Coverage for tests/stats/test_diebold_mariano.py: 100%

57 statements  

« prev     ^ index     » next       coverage.py v7.3.2, created at 2024-02-28 12:51 +1100

1""" 

2This module contains unit tests for scores.stats.tests.diebold_mariano_impl 

3""" 

4import numpy as np 

5import pytest 

6import xarray as xr 

7 

8from scores.stats.statistical_tests.diebold_mariano_impl import ( 

9 _dm_gamma_hat_k, 

10 _dm_test_statistic, 

11 _dm_v_hat, 

12 _hg_func, 

13 _hg_method_stat, 

14 _hln_method_stat, 

15 diebold_mariano, 

16) 

17 

18 

19@pytest.mark.parametrize( 

20 ( 

21 "da_timeseries", 

22 "ts_dim", 

23 "h_coord", 

24 "method", 

25 "confidence_level", 

26 "statistic_distribution", 

27 "error_msg", 

28 ), 

29 [ 

30 ( 

31 xr.DataArray(data=[1, 2], dims=["x"], coords={"x": [0, 1]}), 

32 "x", 

33 "h", 

34 "KEV", 

35 -0.4, 

36 "t", 

37 "`method` must be one of", 

38 ), 

39 ( 

40 xr.DataArray(data=[1, 2], dims=["x"], coords={"x": [0, 1]}), 

41 "x", 

42 "h", 

43 "HG", 

44 -0.4, 

45 "chi_sq", 

46 "`statistic_distribution` must be one of", 

47 ), 

48 ( 

49 xr.DataArray(data=[1, 2], dims=["x"], coords={"x": [0, 1]}), 

50 "x", 

51 "h", 

52 "HLN", 

53 -0.4, 

54 "t", 

55 "`confidence_level` must be strictly between 0 and 1.", 

56 ), 

57 ( 

58 xr.DataArray(data=[1, 2], dims=["x"], coords={"x": [0, 1]}), 

59 "x", 

60 "h", 

61 "HLN", 

62 1.0, 

63 "t", 

64 "`confidence_level` must be strictly between 0 and 1.", 

65 ), 

66 ( 

67 xr.DataArray(data=[1, 2], dims=["x"], coords={"x": [0, 1]}), 

68 "x", 

69 "h", 

70 "HLN", 

71 0.95, 

72 "t", 

73 "`da_timeseries` must have exactly two dimensions.", 

74 ), 

75 ( 

76 xr.DataArray(data=[[1], [2]], dims=["x", "y"], coords={"x": [0, 1], "y": [1]}), 

77 "z", 

78 "y", 

79 "HLN", 

80 0.95, 

81 "t", 

82 "`ts_dim` 'z' must be a dimension of `da_timeseries`.", 

83 ), 

84 ( 

85 xr.DataArray( 

86 data=[[1, 2]], 

87 dims=["x", "y"], 

88 coords={"x": [0], "y": [0, 1], "h": ("x", [1])}, 

89 ), 

90 "x", 

91 "h1", 

92 "HLN", 

93 0.5, 

94 "t", 

95 "`h_coord` must be among the coordinates of `da_timeseries`.", 

96 ), 

97 ( 

98 xr.DataArray( 

99 data=[[1, 2]], 

100 dims=["x", "y"], 

101 coords={"x": [0], "y": [0, 1], "h": ("x", [1.5])}, 

102 ), 

103 "x", 

104 "h", 

105 "HLN", 

106 0.3, 

107 "t", 

108 " must be an integer.", 

109 ), 

110 ( 

111 xr.DataArray( 

112 data=[[1, 2]], 

113 dims=["x", "y"], 

114 coords={"x": [0], "y": [0, 1], "h": ("x", [np.nan])}, 

115 ), 

116 "x", 

117 "h", 

118 "HLN", 

119 0.9, 

120 "t", 

121 " must be an integer.", 

122 ), 

123 ( 

124 xr.DataArray( 

125 data=[[1, 2]], 

126 dims=["x", "y"], 

127 coords={"x": [0], "y": [0, 1], "h": ("x", [-2])}, 

128 ), 

129 "x", 

130 "h", 

131 "HLN", 

132 0.8, 

133 "t", 

134 " must be positive.", 

135 ), 

136 ( 

137 xr.DataArray( 

138 data=[[1, 2, 3, 4], [1, 2, np.nan, np.nan]], 

139 dims=["x", "y"], 

140 coords={"x": [0, 1], "y": [0, 1, 2, 3], "h": ("x", [3, 3])}, 

141 ), 

142 "x", 

143 "h", 

144 "HLN", 

145 0.7, 

146 "t", 

147 "must be less than the length of the corresponding timeseries", 

148 ), 

149 ( 

150 xr.DataArray( 

151 data=[[1, 2, 3, 4], [1, 2, np.nan, np.nan]], 

152 dims=["x", "y"], 

153 coords={"x": [0, 1], "y": [0, 1, 2, 3], "h": ("x", [4, 1])}, 

154 ), 

155 "x", 

156 "h", 

157 "HLN", 

158 0.6, 

159 "t", 

160 "must be less than the length of the corresponding timeseries", 

161 ), 

162 ], 

163) 

164def test_diebold_mariano_raises( 

165 da_timeseries, 

166 ts_dim, 

167 h_coord, 

168 method, 

169 confidence_level, 

170 statistic_distribution, 

171 error_msg, 

172): 

173 """Tests that diebold_mariano raises a ValueError as expected.""" 

174 with pytest.raises(ValueError, match=error_msg): 

175 diebold_mariano( 

176 da_timeseries, 

177 ts_dim, 

178 h_coord, 

179 method, 

180 confidence_level, 

181 statistic_distribution, 

182 ) 

183 

184 

185def test__hg_func(): 

186 """Tests that _hg_func returns as expected.""" 

187 pars = [2, 0.5] 

188 lag = np.array([0, 1, 2]) 

189 acv = np.array([1, 2, -1]) 

190 expected = 4 * np.exp(np.array([0, -6, -12])) - acv 

191 result = _hg_func(pars, lag, acv) 

192 np.testing.assert_allclose(result, expected) 

193 

194 

195DM_DIFFS = np.array([1.0, 2, 3, 4]) 

196DM_DIFFS2 = np.array([0.0, -1, 0, 2, -1, 0, 1, -4, -1, -1, 2, 1, 0, 2, -1]) 

197DM_DIFFS3 = np.array([0.0, -1, 0, 1, -1, 1, -2, -2, -1, -1]) 

198 

199 

200@pytest.mark.parametrize( 

201 ("diffs", "h", "expected"), 

202 [ 

203 (DM_DIFFS2, 14, -0.169192), # cross-checked with R verification 

204 (DM_DIFFS2, 1, -0.169192), 

205 (DM_DIFFS3, 10, -1.860521), # cross-checked with R verification 

206 ], 

207) 

208def test__hg_method_stat1(diffs, h, expected): 

209 """ 

210 Tests that _hg_method_stat returns result as expected when least_squares 

211 routine is successful. 

212 

213 The results of the first test were cross-checked with the R package `verification`. 

214 The following R code reproduces the results: 

215 library(verification) 

216 obs <- rep(0, 15) 

217 fcst1 <- c(1, 2, -1, 4, -2, 3, 4, 1, 0, 0, -2, 3, 4, -5, -4) 

218 fcst1 <- c(1, 3, 1, 2, -3, 3, 3, 5, 1, -1, 0, 2, 4, -3, -5) 

219 test <- predcomp.test(obs, fcst1, fcst2, test = "HG") 

220 summary(test) 

221 """ 

222 result = _hg_method_stat(diffs, h) 

223 np.testing.assert_allclose(result, expected, atol=1e-5) 

224 

225 

226@pytest.mark.parametrize( 

227 ("k", "expected"), 

228 [ 

229 (1, 1.25), 

230 (2, -1.5), 

231 ], 

232) 

233def test__dm_gamma_hat_k(k, expected): 

234 """Tests that _dm_gamma_hat_k returns values as expected.""" 

235 result = _dm_gamma_hat_k(DM_DIFFS, 2.5, 4, k) 

236 np.testing.assert_allclose(result, expected) 

237 

238 

239@pytest.mark.parametrize( 

240 ("diffs", "h", "expected"), 

241 [ 

242 (DM_DIFFS, 3, (5.0 + 2 * 1.25 + 2 * (-1.5)) / 16), 

243 (np.array([1, -1, 1, -1]), 2, np.nan), # original result = 0, so changed to NaN 

244 ], 

245) 

246def test__dm_v_hat(diffs, h, expected): 

247 """Tests that _dm_v_hat returns values as expected.""" 

248 

249 np.testing.assert_allclose(_dm_v_hat(diffs, np.mean(diffs), len(diffs), h), expected) 

250 

251 

252# DM test stat when timeseries is [1, 2, 3, 4] and h = 2 

253DM_TEST_STAT_EXP1 = ((3 / 8) ** 0.5) * 2.5 * (0.46875 ** (-0.5)) 

254 

255 

256@pytest.mark.parametrize( 

257 ("diffs", "h", "expected"), 

258 [ 

259 (DM_DIFFS, 2, DM_TEST_STAT_EXP1), 

260 (np.array([1, -1, 1, -1]), 2, np.nan), # v_hat = 0, so output NaN 

261 ], 

262) 

263def test__hln_method_stat(diffs, h, expected): 

264 """Tests that _hln_method_stat returns as expected.""" 

265 result = _hln_method_stat(diffs, h) 

266 np.testing.assert_allclose(result, expected) 

267 

268 

269@pytest.mark.parametrize( 

270 ("diffs", "h", "method", "expected"), 

271 [ 

272 (DM_DIFFS, 2, "HLN", DM_TEST_STAT_EXP1), 

273 (np.array([1.0, 1, 1, 1]), 2, "HLN", np.nan), 

274 (np.array([0, 0, 0, 0]), 2, "HLN", np.nan), 

275 (DM_DIFFS2, 14, "HG", -0.1691921), 

276 ], 

277) 

278def test__dm_test_statistic(diffs, h, method, expected): 

279 """Tests that _dm_test_statistic returns values as expected.""" 

280 result = _dm_test_statistic(diffs, h, method) 

281 np.testing.assert_allclose(result, expected, atol=1e-5) 

282 

283 

284def test__dm_test_statistic_with_nan(): 

285 """Tests that _dm_test_statistic returns values as expected with a NaN diffs.""" 

286 diffs = np.array([np.nan, 1, 2, 3, 4.0, np.nan]) 

287 with pytest.warns(RuntimeWarning): 

288 result = _dm_test_statistic(diffs, 2, "HLN") 

289 np.testing.assert_allclose(result, DM_TEST_STAT_EXP1, atol=1e-5) 

290 

291 

292@pytest.mark.filterwarnings("ignore::RuntimeWarning") 

293@pytest.mark.parametrize( 

294 ("diff", "h", "method", "error_msg"), 

295 [ 

296 (DM_DIFFS, 3, "KEV", "`method` must be one of"), 

297 (DM_DIFFS, 100, "HLN", "The condition"), 

298 (np.array([np.nan, 1, np.nan]), 2, "HLN", "The condition"), 

299 (DM_DIFFS, 0, "HG", "The condition"), 

300 ], 

301) 

302def test__dm_test_statistic_raises(diff, h, method, error_msg): 

303 """Tests that _dm_test_statistic raises a ValueError as expected.""" 

304 with pytest.raises(ValueError, match=error_msg): 

305 _dm_test_statistic(diff, h, method) 

306 

307 

308# DM test stat when timeseries is [2.0, 1, -3, -1, 0] and h = 3 

309DM_TEST_STAT_EXP2 = ((6 / 25) ** 0.5) * (-0.2) * (0.0864 ** (-0.5)) 

310 

311# expected outputs for dm_test_stats 

312DM_TEST_STATS_T_EXP = xr.Dataset( 

313 data_vars=dict( 

314 mean=(["lead_day"], [2.5, -0.2, 1.0]), 

315 dm_test_stat=(["lead_day"], [DM_TEST_STAT_EXP1, DM_TEST_STAT_EXP2, np.nan]), 

316 timeseries_len=(["lead_day"], [4, 5, 5]), 

317 confidence_gt_0=( 

318 ["lead_day"], 

319 [0.9443164226429581, 0.3778115634892615, np.nan], 

320 ), 

321 ci_upper=(["lead_day"], [5.131140307989639, 1.079108068801774, np.nan]), 

322 ci_lower=( 

323 ["lead_day"], 

324 [-0.13114030798963894, -1.4791080688017741, np.nan], 

325 ), 

326 ), 

327 coords={"lead_day": [1, 2, 3]}, 

328) 

329 

330DM_TEST_STATS_NORMAL_EXP = xr.Dataset( 

331 data_vars=dict( 

332 mean=(["lead_day"], [2.5, -0.2, 1.0]), 

333 dm_test_stat=(["lead_day"], [DM_TEST_STAT_EXP1, DM_TEST_STAT_EXP2, np.nan]), 

334 timeseries_len=(["lead_day"], [4, 5, 5]), 

335 confidence_gt_0=( 

336 ["lead_day"], 

337 [0.9873263406612659, 0.36944134018176367, np.nan], 

338 ), 

339 ci_upper=(["lead_day"], [4.339002261450286, 0.7869121761708835, np.nan]), 

340 ci_lower=( 

341 ["lead_day"], 

342 [0.6609977385497137, -1.1869121761708834, np.nan], 

343 ), 

344 ), 

345 coords={"lead_day": [1, 2, 3]}, 

346) 

347 

348 

349@pytest.mark.parametrize( 

350 ("distribution", "expected"), 

351 [ 

352 ("t", DM_TEST_STATS_T_EXP), 

353 ("normal", DM_TEST_STATS_NORMAL_EXP), 

354 ], 

355) 

356def test_diebold_mariano(distribution, expected): 

357 """ 

358 Tests that diebold_mariano gives results as expected and raises a warning 

359 due to a NaN in the data. 

360 """ 

361 da_timeseries = xr.DataArray( 

362 data=[[1, 2, 3.0, 4, np.nan], [2.0, 1, -3, -1, 0], [1.0, 1, 1, 1, 1]], 

363 dims=["lead_day", "valid_date"], 

364 coords={ 

365 "lead_day": [1, 2, 3], 

366 "valid_date": ["a", "b", "c", "d", "e"], 

367 "h": ("lead_day", [2, 3, 4]), 

368 }, 

369 ) 

370 with pytest.warns(RuntimeWarning): 

371 result = diebold_mariano( 

372 da_timeseries, 

373 "lead_day", 

374 "h", 

375 method="HLN", 

376 confidence_level=0.9, 

377 statistic_distribution=distribution, 

378 ) 

379 xr.testing.assert_allclose(result, expected, atol=7)