Coverage for /home/martinb/.local/share/virtualenvs/camcops/lib/python3.6/site-packages/statsmodels/tsa/filters/filtertools.py : 9%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# -*- coding: utf-8 -*-
2"""Linear Filters for time series analysis and testing
5TODO:
6* check common sequence in signature of filter functions (ar,ma,x) or (x,ar,ma)
8Created on Sat Oct 23 17:18:03 2010
10Author: Josef-pktd
11"""
12# not original copied from various experimental scripts
13# version control history is there
15import numpy as np
16import scipy.fftpack as fft
17from scipy import signal
18from scipy.signal.signaltools import _centered as trim_centered
20from statsmodels.tools.validation import array_like, PandasWrapper
22def _pad_nans(x, head=None, tail=None):
23 if np.ndim(x) == 1:
24 if head is None and tail is None:
25 return x
26 elif head and tail:
27 return np.r_[[np.nan] * head, x, [np.nan] * tail]
28 elif tail is None:
29 return np.r_[[np.nan] * head, x]
30 elif head is None:
31 return np.r_[x, [np.nan] * tail]
32 elif np.ndim(x) == 2:
33 if head is None and tail is None:
34 return x
35 elif head and tail:
36 return np.r_[[[np.nan] * x.shape[1]] * head, x,
37 [[np.nan] * x.shape[1]] * tail]
38 elif tail is None:
39 return np.r_[[[np.nan] * x.shape[1]] * head, x]
40 elif head is None:
41 return np.r_[x, [[np.nan] * x.shape[1]] * tail]
42 else:
43 raise ValueError("Nan-padding for ndim > 2 not implemented")
45#original changes and examples in sandbox.tsa.try_var_convolve
47# do not do these imports, here just for copied fftconvolve
48#get rid of these imports
49#from scipy.fftpack import fft, ifft, ifftshift, fft2, ifft2, fftn, \
50# ifftn, fftfreq
51#from numpy import product,array
54# previous location in sandbox.tsa.try_var_convolve
55def fftconvolveinv(in1, in2, mode="full"):
56 """
57 Convolve two N-dimensional arrays using FFT. See convolve.
59 copied from scipy.signal.signaltools, but here used to try out inverse
60 filter. does not work or I cannot get it to work
62 2010-10-23:
63 looks ok to me for 1d,
64 from results below with padded data array (fftp)
65 but it does not work for multidimensional inverse filter (fftn)
66 original signal.fftconvolve also uses fftn
67 """
68 s1 = np.array(in1.shape)
69 s2 = np.array(in2.shape)
70 complex_result = (np.issubdtype(in1.dtype, np.complex) or
71 np.issubdtype(in2.dtype, np.complex))
72 size = s1+s2-1
74 # Always use 2**n-sized FFT
75 fsize = 2**np.ceil(np.log2(size))
76 IN1 = fft.fftn(in1,fsize)
77 #IN1 *= fftn(in2,fsize) #JP: this looks like the only change I made
78 IN1 /= fft.fftn(in2,fsize) # use inverse filter
79 # note the inverse is elementwise not matrix inverse
80 # is this correct, NO does not seem to work for VARMA
81 fslice = tuple([slice(0, int(sz)) for sz in size])
82 ret = fft.ifftn(IN1)[fslice].copy()
83 del IN1
84 if not complex_result:
85 ret = ret.real
86 if mode == "full":
87 return ret
88 elif mode == "same":
89 if np.product(s1,axis=0) > np.product(s2,axis=0):
90 osize = s1
91 else:
92 osize = s2
93 return trim_centered(ret,osize)
94 elif mode == "valid":
95 return trim_centered(ret,abs(s2-s1)+1)
98#code duplication with fftconvolveinv
99def fftconvolve3(in1, in2=None, in3=None, mode="full"):
100 """
101 Convolve two N-dimensional arrays using FFT. See convolve.
103 For use with arma (old version: in1=num in2=den in3=data
105 * better for consistency with other functions in1=data in2=num in3=den
106 * note in2 and in3 need to have consistent dimension/shape
107 since I'm using max of in2, in3 shapes and not the sum
109 copied from scipy.signal.signaltools, but here used to try out inverse
110 filter does not work or I cannot get it to work
112 2010-10-23
113 looks ok to me for 1d,
114 from results below with padded data array (fftp)
115 but it does not work for multidimensional inverse filter (fftn)
116 original signal.fftconvolve also uses fftn
117 """
118 if (in2 is None) and (in3 is None):
119 raise ValueError('at least one of in2 and in3 needs to be given')
120 s1 = np.array(in1.shape)
121 if in2 is not None:
122 s2 = np.array(in2.shape)
123 else:
124 s2 = 0
125 if in3 is not None:
126 s3 = np.array(in3.shape)
127 s2 = max(s2, s3) # try this looks reasonable for ARMA
128 #s2 = s3
130 complex_result = (np.issubdtype(in1.dtype, np.complex) or
131 np.issubdtype(in2.dtype, np.complex))
132 size = s1+s2-1
134 # Always use 2**n-sized FFT
135 fsize = 2**np.ceil(np.log2(size))
136 #convolve shorter ones first, not sure if it matters
137 if in2 is not None:
138 IN1 = fft.fftn(in2, fsize)
139 if in3 is not None:
140 IN1 /= fft.fftn(in3, fsize) # use inverse filter
141 # note the inverse is elementwise not matrix inverse
142 # is this correct, NO does not seem to work for VARMA
143 IN1 *= fft.fftn(in1, fsize)
144 fslice = tuple([slice(0, int(sz)) for sz in size])
145 ret = fft.ifftn(IN1)[fslice].copy()
146 del IN1
147 if not complex_result:
148 ret = ret.real
149 if mode == "full":
150 return ret
151 elif mode == "same":
152 if np.product(s1,axis=0) > np.product(s2,axis=0):
153 osize = s1
154 else:
155 osize = s2
156 return trim_centered(ret,osize)
157 elif mode == "valid":
158 return trim_centered(ret,abs(s2-s1)+1)
161#original changes and examples in sandbox.tsa.try_var_convolve
162#examples and tests are there
163def recursive_filter(x, ar_coeff, init=None):
164 """
165 Autoregressive, or recursive, filtering.
167 Parameters
168 ----------
169 x : array_like
170 Time-series data. Should be 1d or n x 1.
171 ar_coeff : array_like
172 AR coefficients in reverse time order. See Notes for details.
173 init : array_like
174 Initial values of the time-series prior to the first value of y.
175 The default is zero.
177 Returns
178 -------
179 array_like
180 Filtered array, number of columns determined by x and ar_coeff. If x
181 is a pandas object than a Series is returned.
183 Notes
184 -----
185 Computes the recursive filter ::
187 y[n] = ar_coeff[0] * y[n-1] + ...
188 + ar_coeff[n_coeff - 1] * y[n - n_coeff] + x[n]
190 where n_coeff = len(n_coeff).
191 """
192 pw = PandasWrapper(x)
193 x = array_like(x, 'x')
194 ar_coeff = array_like(ar_coeff, 'ar_coeff')
196 if init is not None: # integer init are treated differently in lfiltic
197 init = array_like(init, 'init')
198 if len(init) != len(ar_coeff):
199 raise ValueError("ar_coeff must be the same length as init")
201 if init is not None:
202 zi = signal.lfiltic([1], np.r_[1, -ar_coeff], init, x)
203 else:
204 zi = None
206 y = signal.lfilter([1.], np.r_[1, -ar_coeff], x, zi=zi)
208 if init is not None:
209 result = y[0]
210 else:
211 result = y
213 return pw.wrap(result)
217def convolution_filter(x, filt, nsides=2):
218 """
219 Linear filtering via convolution. Centered and backward displaced moving
220 weighted average.
222 Parameters
223 ----------
224 x : array_like
225 data array, 1d or 2d, if 2d then observations in rows
226 filt : array_like
227 Linear filter coefficients in reverse time-order. Should have the
228 same number of dimensions as x though if 1d and ``x`` is 2d will be
229 coerced to 2d.
230 nsides : int, optional
231 If 2, a centered moving average is computed using the filter
232 coefficients. If 1, the filter coefficients are for past values only.
233 Both methods use scipy.signal.convolve.
235 Returns
236 -------
237 y : ndarray, 2d
238 Filtered array, number of columns determined by x and filt. If a
239 pandas object is given, a pandas object is returned. The index of
240 the return is the exact same as the time period in ``x``
242 Notes
243 -----
244 In nsides == 1, x is filtered ::
246 y[n] = filt[0]*x[n-1] + ... + filt[n_filt-1]*x[n-n_filt]
248 where n_filt is len(filt).
250 If nsides == 2, x is filtered around lag 0 ::
252 y[n] = filt[0]*x[n - n_filt/2] + ... + filt[n_filt / 2] * x[n]
253 + ... + x[n + n_filt/2]
255 where n_filt is len(filt). If n_filt is even, then more of the filter
256 is forward in time than backward.
258 If filt is 1d or (nlags,1) one lag polynomial is applied to all
259 variables (columns of x). If filt is 2d, (nlags, nvars) each series is
260 independently filtered with its own lag polynomial, uses loop over nvar.
261 This is different than the usual 2d vs 2d convolution.
263 Filtering is done with scipy.signal.convolve, so it will be reasonably
264 fast for medium sized data. For large data fft convolution would be
265 faster.
266 """
267 # for nsides shift the index instead of using 0 for 0 lag this
268 # allows correct handling of NaNs
269 if nsides == 1:
270 trim_head = len(filt) - 1
271 trim_tail = None
272 elif nsides == 2:
273 trim_head = int(np.ceil(len(filt)/2.) - 1) or None
274 trim_tail = int(np.ceil(len(filt)/2.) - len(filt) % 2) or None
275 else: # pragma : no cover
276 raise ValueError("nsides must be 1 or 2")
278 pw = PandasWrapper(x)
279 x = array_like(x, 'x', maxdim=2)
280 filt = array_like(filt, 'filt', ndim=x.ndim)
282 if filt.ndim == 1 or min(filt.shape) == 1:
283 result = signal.convolve(x, filt, mode='valid')
284 elif filt.ndim == 2:
285 nlags = filt.shape[0]
286 nvar = x.shape[1]
287 result = np.zeros((x.shape[0] - nlags + 1, nvar))
288 if nsides == 2:
289 for i in range(nvar):
290 # could also use np.convolve, but easier for swiching to fft
291 result[:, i] = signal.convolve(x[:, i], filt[:, i],
292 mode='valid')
293 elif nsides == 1:
294 for i in range(nvar):
295 result[:, i] = signal.convolve(x[:, i], np.r_[0, filt[:, i]],
296 mode='valid')
297 result = _pad_nans(result, trim_head, trim_tail)
298 return pw.wrap(result)
301# previously located in sandbox.tsa.garch
302def miso_lfilter(ar, ma, x, useic=False):
303 """
304 Filter multiple time series into a single time series.
306 Uses a convolution to merge inputs, and then lfilter to produce output.
308 Parameters
309 ----------
310 ar : array_like
311 The coefficients of autoregressive lag polynomial including lag zero,
312 ar(L) in the expression ar(L)y_t.
313 ma : array_like, same ndim as x, currently 2d
314 The coefficient of the moving average lag polynomial, ma(L) in
315 ma(L)x_t.
316 x : array_like
317 The 2-d input data series, time in rows, variables in columns.
318 useic : bool
319 Flag indicating whether to use initial conditions.
321 Returns
322 -------
323 y : ndarray
324 The filtered output series.
325 inp : ndarray, 1d
326 The combined input series.
328 Notes
329 -----
330 currently for 2d inputs only, no choice of axis
331 Use of signal.lfilter requires that ar lag polynomial contains
332 floating point numbers
333 does not cut off invalid starting and final values
335 miso_lfilter find array y such that:
337 ar(L)y_t = ma(L)x_t
339 with shapes y (nobs,), x (nobs, nvars), ar (narlags,), and
340 ma (narlags, nvars).
341 """
342 ma = array_like(ma, 'ma')
343 ar = array_like(ar, 'ar')
344 inp = signal.correlate(x, ma[::-1, :])[:, (x.shape[1] + 1) // 2]
345 # for testing 2d equivalence between convolve and correlate
346 # inp2 = signal.convolve(x, ma[:,::-1])[:, (x.shape[1]+1)//2]
347 # np.testing.assert_almost_equal(inp2, inp)
348 nobs = x.shape[0]
349 # cut of extra values at end
351 # TODO: initialize also x for correlate
352 if useic:
353 return signal.lfilter([1], ar, inp,
354 zi=signal.lfiltic(np.array([1., 0.]), ar,
355 useic))[0][:nobs], inp[:nobs]
356 else:
357 return signal.lfilter([1], ar, inp)[:nobs], inp[:nobs]