Coverage for test/test_config.py: 97%

197 statements  

« prev     ^ index     » next       coverage.py v7.2.3, created at 2023-05-04 13:14 +0700

1import gzip, json 

2import urllib.request 

3from pathlib import Path 

4import pytest 

5from ..agent_model.Model import Model 

6from ..agent_model.util import load_data_file 

7 

8config_mapping = { 

9 '1h': 'simoc-simdata-1-human-preset.json.gz', 

10 '1hrad': 'simoc-simdata-1-human-radish-preset.json.gz', 

11 '4h': 'simoc-simdata-4-human-preset.json.gz', 

12 '4hg': 'simoc-simdata-4-human-garden-preset.json.gz', 

13 '1hg_sam': 'simoc-simdata-sam-1-human-garden-preset.json.gz', 

14 'b2_mission1a': 'simoc-simdata-b2-mission-1a.json.gz', 

15 'b2_mission1b': 'simoc-simdata-b2-mission-1b.json.gz', 

16 'b2_mission2': 'simoc-simdata-b2-mission-2.json.gz', 

17} 

18 

19# --------------------------------- 

20# Download Simdata from simoc.space 

21# --------------------------------- 

22 

23@pytest.fixture(scope="module") 

24def download_simdata(): 

25 directory = Path.cwd() / 'test' / 'v1_simdata' 

26 

27 if not directory.exists(): 

28 directory.mkdir(parents=True) 

29 

30 url = "https://simoc.space/download/simdata/" 

31 try: 

32 for simdata_name in config_mapping.values(): 

33 file_path = directory / simdata_name 

34 if not file_path.exists(): 

35 urllib.request.urlretrieve(url + simdata_name, file_path) 

36 return True 

37 except: 

38 return False 

39 

40def test_download_simdata(download_simdata): 

41 assert download_simdata 

42 

43# ------------------------------------------- 

44# Helper funcs to produce comparision reports 

45# ------------------------------------------- 

46 

47def load_simdata(stem): 

48 """Load gzipped simdata file.""" 

49 fname = config_mapping[stem] 

50 with gzip.open(f'test/v1_simdata/{fname}', 'rb') as f: 

51 data = json.load(f) 

52 return data 

53 

54def lpe(predictions, targets): 

55 """Lifetime percentage error""" 

56 _p = abs(sum(predictions)) 

57 _t = abs(sum(targets)) 

58 return 0 if _t == 0 else (_p-_t)/_t * 100 

59 

60def compare_records(records, stem): 

61 """Generate a report of the differences between data generated by the 

62 current model and current active simdata.""" 

63 

64 # Load current simdata 

65 simdata = load_simdata(stem) 

66 assert records['step_num'][-1] == simdata['steps'], 'Different step numbers' 

67 references = simdata['data'] 

68 

69 # Generate comparison report 

70 reports = {} 

71 for agent_id, record in records['agents'].items(): 

72 substitute_names = {'human': 'human_agent'} 

73 old_name = substitute_names.get(agent_id, agent_id) 

74 reference = references[old_name] 

75 report = {} 

76 

77 # Compare storage 

78 if record['storage']: 

79 report['storage'] = {} 

80 for currency, predictions in record['storage'].items(): 

81 targets = reference['storage'][currency] 

82 report['storage'][currency] = lpe(predictions, targets) 

83 

84 # Compare flows 

85 if record['flows']: 

86 if agent_id == 'atmosphere_equalizer': 

87 continue # Not part of old records 

88 report['flows'] = {} 

89 for direction, flows in record['flows'].items(): 

90 for currency, currency_flows in flows.items(): 

91 if currency == 'par': 

92 continue # Not part of old records 

93 per_connection = [] 

94 for connection, predictions in currency_flows.items(): 

95 targets = reference['flows'][direction][currency][connection] 

96 per_connection.append(lpe(predictions, targets)) 

97 mean_mape = sum(per_connection) / len(per_connection) 

98 report['flows'][f'{direction}_{currency}'] = mean_mape 

99 

100 # Compare attributes 

101 if record['attributes']: 

102 report['attributes'] = {} 

103 for attribute, predictions in record['attributes'].items(): 

104 if 'growth' in reference and attribute in reference['growth']: 

105 targets = reference['growth'][attribute] 

106 elif 'deprive' in attribute: 

107 deprive_attr = attribute[:-8] # e.g. 'co2' from 'co2_deprive' 

108 if deprive_attr in reference['deprive']: 

109 targets = reference['deprive'][deprive_attr] 

110 else: 

111 continue 

112 elif 'criteria' in attribute: 

113 if 'buffer' not in reference: 

114 continue 

115 criteria_attr = '_'.join(attribute.split('_')[:2]) 

116 if criteria_attr in reference['buffer']: 

117 continue # Broken? 

118 targets = reference['buffer'][criteria_attr] 

119 elif (attribute in {'age', 'delay_start', 'par_rate', 'photoperiod'} or 

120 'growth_factor' in attribute): 

121 continue 

122 else: 

123 assert False, f'Unaccounted for attribute: {attribute}' 

124 report['attributes'][attribute] = lpe(predictions, targets) 

125 

126 reports[agent_id] = report 

127 return reports 

128 

129 

130class TestConfigs: 

131 def test_config_1h(self): 

132 config = load_data_file('config_1h.json') 

133 model = Model.from_config(**config, record_initial_state=False) 

134 model.run() 

135 assert model.elapsed_time.days == 10 

136 human = model.agents['human'] 

137 assert human.active == 1 

138 

139 records = model.get_records() 

140 comparison_report = compare_records(records, '1h') 

141 with open('test/v1_simdata/comparison_report_1h.json', 'w') as f: 

142 json.dump(comparison_report, f, indent=2) 

143 for agent, report in comparison_report.items(): 

144 for section, fields in report.items(): 

145 for field, value in fields.items(): 

146 exceptions = { 

147 # ECLSS components are more active because many 

148 # previously had duplicated criteria. e.g.  

149 ('crew_habitat_small', 'storage', 'co2'): 3.3, 

150 ('crew_habitat_small', 'storage', 'h2o'): 1.3, 

151 ('water_storage', 'storage', 'treated'): 5.7, 

152 } 

153 if (agent, section, field) in exceptions: 

154 percent_error = exceptions[(agent, section, field)] 

155 else: 

156 percent_error = 1 

157 # Less than 1% error in lifetime 

158 assert abs(value) < percent_error, f'{agent} {section} {field} error: {value}' 

159 

160 def test_config_1hrad(self): 

161 stem = '1hrad' 

162 config = load_data_file(f'config_{stem}.json') 

163 model = Model.from_config(**config) 

164 model.run() 

165 human = model.agents['human'] 

166 assert human.active == 1 

167 

168 records = model.get_records() 

169 comparison_report = compare_records(records, stem) 

170 with open(f'test/v1_simdata/comparison_report_{stem}.json', 'w') as f: 

171 json.dump(comparison_report, f, indent=2) 

172 for agent, report in comparison_report.items(): 

173 for section, fields in report.items(): 

174 for field, value in fields.items(): 

175 exceptions = { 

176 ('solar_pv_array_mars', 'flows', 'out_kwh'): 3.4, 

177 ('crew_habitat_small', 'storage', 'co2'): 12.2, 

178 ('crew_habitat_small', 'storage', 'h2o'): 3.0, 

179 ('greenhouse_small', 'storage', 'co2'): 8.9, 

180 ('greenhouse_small', 'storage', 'h2o'): 1.9, 

181 # These are mistakenly logged twice in the simdata 

182 ('radish', 'flows', 'out_radish'): 50, 

183 ('radish', 'flows', 'out_inedible_biomass'): 50, 

184 # THis means something different now 

185 ('radish', 'attributes', 'growth_rate'): 102, 

186 # These were formerly multiplied by amount 

187 ('radish', 'attributes', 'in_co2_deprive'): 98, 

188 ('radish', 'attributes', 'in_potable_deprive'): 98, 

189 ('radish', 'attributes', 'in_fertilizer_deprive'): 98, 

190 ('water_storage', 'storage', 'treated'): 7.4, 

191 ('nutrient_storage', 'storage', 'inedible_biomass'): 1.5, 

192 ('power_storage', 'storage', 'kwh'): 11.1, 

193 ('co2_removal_SAWD', 'flows', 'in_co2'): 1.8, 

194 ('co2_removal_SAWD', 'flows', 'in_kwh'): 3.5, 

195 ('co2_removal_SAWD', 'flows', 'out_co2'): 3.5, 

196 ('co2_storage', 'storage', 'co2'): 1.8, 

197 } 

198 if (agent, section, field) in exceptions: 

199 percent_error = exceptions[(agent, section, field)] 

200 else: 

201 percent_error = 1 

202 # Less than 1% error in lifetime 

203 assert abs(value) < percent_error, f'{agent} {section} {field} error: {value}' 

204 

205 

206 def test_config_4h(self): 

207 stem = '4h' 

208 config = load_data_file(f'config_{stem}.json') 

209 model = Model.from_config(**config) 

210 model.run() 

211 human = model.agents['human'] 

212 assert human.active == 4 

213 

214 records = model.get_records() 

215 comparison_report = compare_records(records, stem) 

216 with open(f'test/v1_simdata/comparison_report_{stem}.json', 'w') as f: 

217 json.dump(comparison_report, f, indent=2) 

218 for agent, report in comparison_report.items(): 

219 for section, fields in report.items(): 

220 for field, value in fields.items(): 

221 exceptions = { 

222 ('human', 'attributes', 'in_potable_deprive'): 75, 

223 ('human', 'attributes', 'in_food_deprive'): 75, 

224 ('solar_pv_array_mars', 'flows', 'out_kwh'): 3.3, 

225 ('crew_habitat_small', 'storage', 'co2'): 4.9, 

226 ('crew_habitat_small', 'storage', 'ch4'): 4.2, 

227 ('crew_habitat_small', 'storage', 'h2'): 1.8, 

228 ('crew_habitat_small', 'storage', 'h2o'): 4.3, 

229 ('water_storage', 'storage', 'urine'): 1.2, 

230 ('water_storage', 'storage', 'treated'): 10.3, 

231 ('power_storage', 'storage', 'kwh'): 1.6, 

232 ('co2_reduction_sabatier', 'flows', 'in_h2'): 1.1, 

233 ('co2_reduction_sabatier', 'flows', 'in_co2'): 3.3, 

234 ('co2_reduction_sabatier', 'flows', 'in_kwh'): 3.3, 

235 ('co2_reduction_sabatier', 'flows', 'out_ch4'): 3.3, 

236 ('co2_reduction_sabatier', 'flows', 'out_feces'): 3.3, 

237 } 

238 if (agent, section, field) in exceptions: 

239 percent_error = exceptions[(agent, section, field)] 

240 else: 

241 percent_error = 1 

242 # Less than 1% error in lifetime 

243 assert abs(value) < percent_error, f'{agent} {section} {field} error: {value}' 

244 

245 def test_config_4hg(self): 

246 stem = '4hg' 

247 config = load_data_file(f'config_{stem}.json') 

248 model = Model.from_config(**config) 

249 # For some reason, the simdata only has 2300 steps instead of 2400 

250 for i in range(2300): 

251 model.step() 

252 human = model.agents['human'] 

253 assert human.active == 4 

254 

255 records = model.get_records() 

256 comparison_report = compare_records(records, stem) 

257 with open(f'test/v1_simdata/comparison_report_{stem}.json', 'w') as f: 

258 json.dump(comparison_report, f, indent=2) 

259 

260 

261 def test_config_1hg_sam(self): 

262 stem = '1hg_sam' 

263 config = load_data_file(f'config_{stem}.json') 

264 model = Model.from_config(**config) 

265 model.run() 

266 human = model.agents['human'] 

267 assert human.active == 1 

268 

269 records = model.get_records() 

270 comparison_report = compare_records(records, stem) 

271 with open(f'test/v1_simdata/comparison_report_{stem}.json', 'w') as f: 

272 json.dump(comparison_report, f, indent=2) 

273 

274 

275 def test_config_b2_mission1a(self): 

276 stem = 'b2_mission1a' 

277 config = load_data_file(f'config_{stem}.json') 

278 model = Model.from_config(**config) 

279 model.run() 

280 human = model.agents['human'] 

281 assert human.active == 8 

282 

283 records = model.get_records() 

284 comparison_report = compare_records(records, stem) 

285 with open(f'test/v1_simdata/comparison_report_{stem}.json', 'w') as f: 

286 json.dump(comparison_report, f, indent=2) 

287 

288 

289 def test_config_b2_mission1b(self): 

290 stem = 'b2_mission1b' 

291 config = load_data_file(f'config_{stem}.json') 

292 model = Model.from_config(**config) 

293 model.run() 

294 human = model.agents['human'] 

295 assert human.active == 8 

296 

297 records = model.get_records() 

298 comparison_report = compare_records(records, stem) 

299 with open(f'test/v1_simdata/comparison_report_{stem}.json', 'w') as f: 

300 json.dump(comparison_report, f, indent=2) 

301 

302 

303 

304 def test_config_b2_mission2(self): 

305 stem = 'b2_mission2' 

306 config = load_data_file(f'config_{stem}.json') 

307 model = Model.from_config(**config) 

308 model.run() 

309 human = model.agents['human'] 

310 assert human.active == 8 

311 

312 records = model.get_records() 

313 comparison_report = compare_records(records, stem) 

314 with open(f'test/v1_simdata/comparison_report_{stem}.json', 'w') as f: 

315 json.dump(comparison_report, f, indent=2)