Coverage for tests/stats/test_diebold_mariano.py: 100%
57 statements
« prev ^ index » next coverage.py v7.3.2, created at 2024-02-28 12:51 +1100
« prev ^ index » next coverage.py v7.3.2, created at 2024-02-28 12:51 +1100
1"""
2This module contains unit tests for scores.stats.tests.diebold_mariano_impl
3"""
4import numpy as np
5import pytest
6import xarray as xr
8from scores.stats.statistical_tests.diebold_mariano_impl import (
9 _dm_gamma_hat_k,
10 _dm_test_statistic,
11 _dm_v_hat,
12 _hg_func,
13 _hg_method_stat,
14 _hln_method_stat,
15 diebold_mariano,
16)
19@pytest.mark.parametrize(
20 (
21 "da_timeseries",
22 "ts_dim",
23 "h_coord",
24 "method",
25 "confidence_level",
26 "statistic_distribution",
27 "error_msg",
28 ),
29 [
30 (
31 xr.DataArray(data=[1, 2], dims=["x"], coords={"x": [0, 1]}),
32 "x",
33 "h",
34 "KEV",
35 -0.4,
36 "t",
37 "`method` must be one of",
38 ),
39 (
40 xr.DataArray(data=[1, 2], dims=["x"], coords={"x": [0, 1]}),
41 "x",
42 "h",
43 "HG",
44 -0.4,
45 "chi_sq",
46 "`statistic_distribution` must be one of",
47 ),
48 (
49 xr.DataArray(data=[1, 2], dims=["x"], coords={"x": [0, 1]}),
50 "x",
51 "h",
52 "HLN",
53 -0.4,
54 "t",
55 "`confidence_level` must be strictly between 0 and 1.",
56 ),
57 (
58 xr.DataArray(data=[1, 2], dims=["x"], coords={"x": [0, 1]}),
59 "x",
60 "h",
61 "HLN",
62 1.0,
63 "t",
64 "`confidence_level` must be strictly between 0 and 1.",
65 ),
66 (
67 xr.DataArray(data=[1, 2], dims=["x"], coords={"x": [0, 1]}),
68 "x",
69 "h",
70 "HLN",
71 0.95,
72 "t",
73 "`da_timeseries` must have exactly two dimensions.",
74 ),
75 (
76 xr.DataArray(data=[[1], [2]], dims=["x", "y"], coords={"x": [0, 1], "y": [1]}),
77 "z",
78 "y",
79 "HLN",
80 0.95,
81 "t",
82 "`ts_dim` 'z' must be a dimension of `da_timeseries`.",
83 ),
84 (
85 xr.DataArray(
86 data=[[1, 2]],
87 dims=["x", "y"],
88 coords={"x": [0], "y": [0, 1], "h": ("x", [1])},
89 ),
90 "x",
91 "h1",
92 "HLN",
93 0.5,
94 "t",
95 "`h_coord` must be among the coordinates of `da_timeseries`.",
96 ),
97 (
98 xr.DataArray(
99 data=[[1, 2]],
100 dims=["x", "y"],
101 coords={"x": [0], "y": [0, 1], "h": ("x", [1.5])},
102 ),
103 "x",
104 "h",
105 "HLN",
106 0.3,
107 "t",
108 " must be an integer.",
109 ),
110 (
111 xr.DataArray(
112 data=[[1, 2]],
113 dims=["x", "y"],
114 coords={"x": [0], "y": [0, 1], "h": ("x", [np.nan])},
115 ),
116 "x",
117 "h",
118 "HLN",
119 0.9,
120 "t",
121 " must be an integer.",
122 ),
123 (
124 xr.DataArray(
125 data=[[1, 2]],
126 dims=["x", "y"],
127 coords={"x": [0], "y": [0, 1], "h": ("x", [-2])},
128 ),
129 "x",
130 "h",
131 "HLN",
132 0.8,
133 "t",
134 " must be positive.",
135 ),
136 (
137 xr.DataArray(
138 data=[[1, 2, 3, 4], [1, 2, np.nan, np.nan]],
139 dims=["x", "y"],
140 coords={"x": [0, 1], "y": [0, 1, 2, 3], "h": ("x", [3, 3])},
141 ),
142 "x",
143 "h",
144 "HLN",
145 0.7,
146 "t",
147 "must be less than the length of the corresponding timeseries",
148 ),
149 (
150 xr.DataArray(
151 data=[[1, 2, 3, 4], [1, 2, np.nan, np.nan]],
152 dims=["x", "y"],
153 coords={"x": [0, 1], "y": [0, 1, 2, 3], "h": ("x", [4, 1])},
154 ),
155 "x",
156 "h",
157 "HLN",
158 0.6,
159 "t",
160 "must be less than the length of the corresponding timeseries",
161 ),
162 ],
163)
164def test_diebold_mariano_raises(
165 da_timeseries,
166 ts_dim,
167 h_coord,
168 method,
169 confidence_level,
170 statistic_distribution,
171 error_msg,
172):
173 """Tests that diebold_mariano raises a ValueError as expected."""
174 with pytest.raises(ValueError, match=error_msg):
175 diebold_mariano(
176 da_timeseries,
177 ts_dim,
178 h_coord,
179 method,
180 confidence_level,
181 statistic_distribution,
182 )
185def test__hg_func():
186 """Tests that _hg_func returns as expected."""
187 pars = [2, 0.5]
188 lag = np.array([0, 1, 2])
189 acv = np.array([1, 2, -1])
190 expected = 4 * np.exp(np.array([0, -6, -12])) - acv
191 result = _hg_func(pars, lag, acv)
192 np.testing.assert_allclose(result, expected)
195DM_DIFFS = np.array([1.0, 2, 3, 4])
196DM_DIFFS2 = np.array([0.0, -1, 0, 2, -1, 0, 1, -4, -1, -1, 2, 1, 0, 2, -1])
197DM_DIFFS3 = np.array([0.0, -1, 0, 1, -1, 1, -2, -2, -1, -1])
200@pytest.mark.parametrize(
201 ("diffs", "h", "expected"),
202 [
203 (DM_DIFFS2, 14, -0.169192), # cross-checked with R verification
204 (DM_DIFFS2, 1, -0.169192),
205 (DM_DIFFS3, 10, -1.860521), # cross-checked with R verification
206 ],
207)
208def test__hg_method_stat1(diffs, h, expected):
209 """
210 Tests that _hg_method_stat returns result as expected when least_squares
211 routine is successful.
213 The results of the first test were cross-checked with the R package `verification`.
214 The following R code reproduces the results:
215 library(verification)
216 obs <- rep(0, 15)
217 fcst1 <- c(1, 2, -1, 4, -2, 3, 4, 1, 0, 0, -2, 3, 4, -5, -4)
218 fcst1 <- c(1, 3, 1, 2, -3, 3, 3, 5, 1, -1, 0, 2, 4, -3, -5)
219 test <- predcomp.test(obs, fcst1, fcst2, test = "HG")
220 summary(test)
221 """
222 result = _hg_method_stat(diffs, h)
223 np.testing.assert_allclose(result, expected, atol=1e-5)
226@pytest.mark.parametrize(
227 ("k", "expected"),
228 [
229 (1, 1.25),
230 (2, -1.5),
231 ],
232)
233def test__dm_gamma_hat_k(k, expected):
234 """Tests that _dm_gamma_hat_k returns values as expected."""
235 result = _dm_gamma_hat_k(DM_DIFFS, 2.5, 4, k)
236 np.testing.assert_allclose(result, expected)
239@pytest.mark.parametrize(
240 ("diffs", "h", "expected"),
241 [
242 (DM_DIFFS, 3, (5.0 + 2 * 1.25 + 2 * (-1.5)) / 16),
243 (np.array([1, -1, 1, -1]), 2, np.nan), # original result = 0, so changed to NaN
244 ],
245)
246def test__dm_v_hat(diffs, h, expected):
247 """Tests that _dm_v_hat returns values as expected."""
249 np.testing.assert_allclose(_dm_v_hat(diffs, np.mean(diffs), len(diffs), h), expected)
252# DM test stat when timeseries is [1, 2, 3, 4] and h = 2
253DM_TEST_STAT_EXP1 = ((3 / 8) ** 0.5) * 2.5 * (0.46875 ** (-0.5))
256@pytest.mark.parametrize(
257 ("diffs", "h", "expected"),
258 [
259 (DM_DIFFS, 2, DM_TEST_STAT_EXP1),
260 (np.array([1, -1, 1, -1]), 2, np.nan), # v_hat = 0, so output NaN
261 ],
262)
263def test__hln_method_stat(diffs, h, expected):
264 """Tests that _hln_method_stat returns as expected."""
265 result = _hln_method_stat(diffs, h)
266 np.testing.assert_allclose(result, expected)
269@pytest.mark.parametrize(
270 ("diffs", "h", "method", "expected"),
271 [
272 (DM_DIFFS, 2, "HLN", DM_TEST_STAT_EXP1),
273 (np.array([1.0, 1, 1, 1]), 2, "HLN", np.nan),
274 (np.array([0, 0, 0, 0]), 2, "HLN", np.nan),
275 (DM_DIFFS2, 14, "HG", -0.1691921),
276 ],
277)
278def test__dm_test_statistic(diffs, h, method, expected):
279 """Tests that _dm_test_statistic returns values as expected."""
280 result = _dm_test_statistic(diffs, h, method)
281 np.testing.assert_allclose(result, expected, atol=1e-5)
284def test__dm_test_statistic_with_nan():
285 """Tests that _dm_test_statistic returns values as expected with a NaN diffs."""
286 diffs = np.array([np.nan, 1, 2, 3, 4.0, np.nan])
287 with pytest.warns(RuntimeWarning):
288 result = _dm_test_statistic(diffs, 2, "HLN")
289 np.testing.assert_allclose(result, DM_TEST_STAT_EXP1, atol=1e-5)
292@pytest.mark.filterwarnings("ignore::RuntimeWarning")
293@pytest.mark.parametrize(
294 ("diff", "h", "method", "error_msg"),
295 [
296 (DM_DIFFS, 3, "KEV", "`method` must be one of"),
297 (DM_DIFFS, 100, "HLN", "The condition"),
298 (np.array([np.nan, 1, np.nan]), 2, "HLN", "The condition"),
299 (DM_DIFFS, 0, "HG", "The condition"),
300 ],
301)
302def test__dm_test_statistic_raises(diff, h, method, error_msg):
303 """Tests that _dm_test_statistic raises a ValueError as expected."""
304 with pytest.raises(ValueError, match=error_msg):
305 _dm_test_statistic(diff, h, method)
308# DM test stat when timeseries is [2.0, 1, -3, -1, 0] and h = 3
309DM_TEST_STAT_EXP2 = ((6 / 25) ** 0.5) * (-0.2) * (0.0864 ** (-0.5))
311# expected outputs for dm_test_stats
312DM_TEST_STATS_T_EXP = xr.Dataset(
313 data_vars=dict(
314 mean=(["lead_day"], [2.5, -0.2, 1.0]),
315 dm_test_stat=(["lead_day"], [DM_TEST_STAT_EXP1, DM_TEST_STAT_EXP2, np.nan]),
316 timeseries_len=(["lead_day"], [4, 5, 5]),
317 confidence_gt_0=(
318 ["lead_day"],
319 [0.9443164226429581, 0.3778115634892615, np.nan],
320 ),
321 ci_upper=(["lead_day"], [5.131140307989639, 1.079108068801774, np.nan]),
322 ci_lower=(
323 ["lead_day"],
324 [-0.13114030798963894, -1.4791080688017741, np.nan],
325 ),
326 ),
327 coords={"lead_day": [1, 2, 3]},
328)
330DM_TEST_STATS_NORMAL_EXP = xr.Dataset(
331 data_vars=dict(
332 mean=(["lead_day"], [2.5, -0.2, 1.0]),
333 dm_test_stat=(["lead_day"], [DM_TEST_STAT_EXP1, DM_TEST_STAT_EXP2, np.nan]),
334 timeseries_len=(["lead_day"], [4, 5, 5]),
335 confidence_gt_0=(
336 ["lead_day"],
337 [0.9873263406612659, 0.36944134018176367, np.nan],
338 ),
339 ci_upper=(["lead_day"], [4.339002261450286, 0.7869121761708835, np.nan]),
340 ci_lower=(
341 ["lead_day"],
342 [0.6609977385497137, -1.1869121761708834, np.nan],
343 ),
344 ),
345 coords={"lead_day": [1, 2, 3]},
346)
349@pytest.mark.parametrize(
350 ("distribution", "expected"),
351 [
352 ("t", DM_TEST_STATS_T_EXP),
353 ("normal", DM_TEST_STATS_NORMAL_EXP),
354 ],
355)
356def test_diebold_mariano(distribution, expected):
357 """
358 Tests that diebold_mariano gives results as expected and raises a warning
359 due to a NaN in the data.
360 """
361 da_timeseries = xr.DataArray(
362 data=[[1, 2, 3.0, 4, np.nan], [2.0, 1, -3, -1, 0], [1.0, 1, 1, 1, 1]],
363 dims=["lead_day", "valid_date"],
364 coords={
365 "lead_day": [1, 2, 3],
366 "valid_date": ["a", "b", "c", "d", "e"],
367 "h": ("lead_day", [2, 3, 4]),
368 },
369 )
370 with pytest.warns(RuntimeWarning):
371 result = diebold_mariano(
372 da_timeseries,
373 "lead_day",
374 "h",
375 method="HLN",
376 confidence_level=0.9,
377 statistic_distribution=distribution,
378 )
379 xr.testing.assert_allclose(result, expected, atol=7)