Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# -*- coding: utf-8 -*- 

2"""Linear Filters for time series analysis and testing 

3 

4 

5TODO: 

6* check common sequence in signature of filter functions (ar,ma,x) or (x,ar,ma) 

7 

8Created on Sat Oct 23 17:18:03 2010 

9 

10Author: Josef-pktd 

11""" 

12# not original copied from various experimental scripts 

13# version control history is there 

14 

15import numpy as np 

16import scipy.fftpack as fft 

17from scipy import signal 

18from scipy.signal.signaltools import _centered as trim_centered 

19 

20from statsmodels.tools.validation import array_like, PandasWrapper 

21 

22def _pad_nans(x, head=None, tail=None): 

23 if np.ndim(x) == 1: 

24 if head is None and tail is None: 

25 return x 

26 elif head and tail: 

27 return np.r_[[np.nan] * head, x, [np.nan] * tail] 

28 elif tail is None: 

29 return np.r_[[np.nan] * head, x] 

30 elif head is None: 

31 return np.r_[x, [np.nan] * tail] 

32 elif np.ndim(x) == 2: 

33 if head is None and tail is None: 

34 return x 

35 elif head and tail: 

36 return np.r_[[[np.nan] * x.shape[1]] * head, x, 

37 [[np.nan] * x.shape[1]] * tail] 

38 elif tail is None: 

39 return np.r_[[[np.nan] * x.shape[1]] * head, x] 

40 elif head is None: 

41 return np.r_[x, [[np.nan] * x.shape[1]] * tail] 

42 else: 

43 raise ValueError("Nan-padding for ndim > 2 not implemented") 

44 

45#original changes and examples in sandbox.tsa.try_var_convolve 

46 

47# do not do these imports, here just for copied fftconvolve 

48#get rid of these imports 

49#from scipy.fftpack import fft, ifft, ifftshift, fft2, ifft2, fftn, \ 

50# ifftn, fftfreq 

51#from numpy import product,array 

52 

53 

54# previous location in sandbox.tsa.try_var_convolve 

55def fftconvolveinv(in1, in2, mode="full"): 

56 """ 

57 Convolve two N-dimensional arrays using FFT. See convolve. 

58 

59 copied from scipy.signal.signaltools, but here used to try out inverse 

60 filter. does not work or I cannot get it to work 

61 

62 2010-10-23: 

63 looks ok to me for 1d, 

64 from results below with padded data array (fftp) 

65 but it does not work for multidimensional inverse filter (fftn) 

66 original signal.fftconvolve also uses fftn 

67 """ 

68 s1 = np.array(in1.shape) 

69 s2 = np.array(in2.shape) 

70 complex_result = (np.issubdtype(in1.dtype, np.complex) or 

71 np.issubdtype(in2.dtype, np.complex)) 

72 size = s1+s2-1 

73 

74 # Always use 2**n-sized FFT 

75 fsize = 2**np.ceil(np.log2(size)) 

76 IN1 = fft.fftn(in1,fsize) 

77 #IN1 *= fftn(in2,fsize) #JP: this looks like the only change I made 

78 IN1 /= fft.fftn(in2,fsize) # use inverse filter 

79 # note the inverse is elementwise not matrix inverse 

80 # is this correct, NO does not seem to work for VARMA 

81 fslice = tuple([slice(0, int(sz)) for sz in size]) 

82 ret = fft.ifftn(IN1)[fslice].copy() 

83 del IN1 

84 if not complex_result: 

85 ret = ret.real 

86 if mode == "full": 

87 return ret 

88 elif mode == "same": 

89 if np.product(s1,axis=0) > np.product(s2,axis=0): 

90 osize = s1 

91 else: 

92 osize = s2 

93 return trim_centered(ret,osize) 

94 elif mode == "valid": 

95 return trim_centered(ret,abs(s2-s1)+1) 

96 

97 

98#code duplication with fftconvolveinv 

99def fftconvolve3(in1, in2=None, in3=None, mode="full"): 

100 """ 

101 Convolve two N-dimensional arrays using FFT. See convolve. 

102 

103 For use with arma (old version: in1=num in2=den in3=data 

104 

105 * better for consistency with other functions in1=data in2=num in3=den 

106 * note in2 and in3 need to have consistent dimension/shape 

107 since I'm using max of in2, in3 shapes and not the sum 

108 

109 copied from scipy.signal.signaltools, but here used to try out inverse 

110 filter does not work or I cannot get it to work 

111 

112 2010-10-23 

113 looks ok to me for 1d, 

114 from results below with padded data array (fftp) 

115 but it does not work for multidimensional inverse filter (fftn) 

116 original signal.fftconvolve also uses fftn 

117 """ 

118 if (in2 is None) and (in3 is None): 

119 raise ValueError('at least one of in2 and in3 needs to be given') 

120 s1 = np.array(in1.shape) 

121 if in2 is not None: 

122 s2 = np.array(in2.shape) 

123 else: 

124 s2 = 0 

125 if in3 is not None: 

126 s3 = np.array(in3.shape) 

127 s2 = max(s2, s3) # try this looks reasonable for ARMA 

128 #s2 = s3 

129 

130 complex_result = (np.issubdtype(in1.dtype, np.complex) or 

131 np.issubdtype(in2.dtype, np.complex)) 

132 size = s1+s2-1 

133 

134 # Always use 2**n-sized FFT 

135 fsize = 2**np.ceil(np.log2(size)) 

136 #convolve shorter ones first, not sure if it matters 

137 if in2 is not None: 

138 IN1 = fft.fftn(in2, fsize) 

139 if in3 is not None: 

140 IN1 /= fft.fftn(in3, fsize) # use inverse filter 

141 # note the inverse is elementwise not matrix inverse 

142 # is this correct, NO does not seem to work for VARMA 

143 IN1 *= fft.fftn(in1, fsize) 

144 fslice = tuple([slice(0, int(sz)) for sz in size]) 

145 ret = fft.ifftn(IN1)[fslice].copy() 

146 del IN1 

147 if not complex_result: 

148 ret = ret.real 

149 if mode == "full": 

150 return ret 

151 elif mode == "same": 

152 if np.product(s1,axis=0) > np.product(s2,axis=0): 

153 osize = s1 

154 else: 

155 osize = s2 

156 return trim_centered(ret,osize) 

157 elif mode == "valid": 

158 return trim_centered(ret,abs(s2-s1)+1) 

159 

160 

161#original changes and examples in sandbox.tsa.try_var_convolve 

162#examples and tests are there 

163def recursive_filter(x, ar_coeff, init=None): 

164 """ 

165 Autoregressive, or recursive, filtering. 

166 

167 Parameters 

168 ---------- 

169 x : array_like 

170 Time-series data. Should be 1d or n x 1. 

171 ar_coeff : array_like 

172 AR coefficients in reverse time order. See Notes for details. 

173 init : array_like 

174 Initial values of the time-series prior to the first value of y. 

175 The default is zero. 

176 

177 Returns 

178 ------- 

179 array_like 

180 Filtered array, number of columns determined by x and ar_coeff. If x 

181 is a pandas object than a Series is returned. 

182 

183 Notes 

184 ----- 

185 Computes the recursive filter :: 

186 

187 y[n] = ar_coeff[0] * y[n-1] + ... 

188 + ar_coeff[n_coeff - 1] * y[n - n_coeff] + x[n] 

189 

190 where n_coeff = len(n_coeff). 

191 """ 

192 pw = PandasWrapper(x) 

193 x = array_like(x, 'x') 

194 ar_coeff = array_like(ar_coeff, 'ar_coeff') 

195 

196 if init is not None: # integer init are treated differently in lfiltic 

197 init = array_like(init, 'init') 

198 if len(init) != len(ar_coeff): 

199 raise ValueError("ar_coeff must be the same length as init") 

200 

201 if init is not None: 

202 zi = signal.lfiltic([1], np.r_[1, -ar_coeff], init, x) 

203 else: 

204 zi = None 

205 

206 y = signal.lfilter([1.], np.r_[1, -ar_coeff], x, zi=zi) 

207 

208 if init is not None: 

209 result = y[0] 

210 else: 

211 result = y 

212 

213 return pw.wrap(result) 

214 

215 

216 

217def convolution_filter(x, filt, nsides=2): 

218 """ 

219 Linear filtering via convolution. Centered and backward displaced moving 

220 weighted average. 

221 

222 Parameters 

223 ---------- 

224 x : array_like 

225 data array, 1d or 2d, if 2d then observations in rows 

226 filt : array_like 

227 Linear filter coefficients in reverse time-order. Should have the 

228 same number of dimensions as x though if 1d and ``x`` is 2d will be 

229 coerced to 2d. 

230 nsides : int, optional 

231 If 2, a centered moving average is computed using the filter 

232 coefficients. If 1, the filter coefficients are for past values only. 

233 Both methods use scipy.signal.convolve. 

234 

235 Returns 

236 ------- 

237 y : ndarray, 2d 

238 Filtered array, number of columns determined by x and filt. If a 

239 pandas object is given, a pandas object is returned. The index of 

240 the return is the exact same as the time period in ``x`` 

241 

242 Notes 

243 ----- 

244 In nsides == 1, x is filtered :: 

245 

246 y[n] = filt[0]*x[n-1] + ... + filt[n_filt-1]*x[n-n_filt] 

247 

248 where n_filt is len(filt). 

249 

250 If nsides == 2, x is filtered around lag 0 :: 

251 

252 y[n] = filt[0]*x[n - n_filt/2] + ... + filt[n_filt / 2] * x[n] 

253 + ... + x[n + n_filt/2] 

254 

255 where n_filt is len(filt). If n_filt is even, then more of the filter 

256 is forward in time than backward. 

257 

258 If filt is 1d or (nlags,1) one lag polynomial is applied to all 

259 variables (columns of x). If filt is 2d, (nlags, nvars) each series is 

260 independently filtered with its own lag polynomial, uses loop over nvar. 

261 This is different than the usual 2d vs 2d convolution. 

262 

263 Filtering is done with scipy.signal.convolve, so it will be reasonably 

264 fast for medium sized data. For large data fft convolution would be 

265 faster. 

266 """ 

267 # for nsides shift the index instead of using 0 for 0 lag this 

268 # allows correct handling of NaNs 

269 if nsides == 1: 

270 trim_head = len(filt) - 1 

271 trim_tail = None 

272 elif nsides == 2: 

273 trim_head = int(np.ceil(len(filt)/2.) - 1) or None 

274 trim_tail = int(np.ceil(len(filt)/2.) - len(filt) % 2) or None 

275 else: # pragma : no cover 

276 raise ValueError("nsides must be 1 or 2") 

277 

278 pw = PandasWrapper(x) 

279 x = array_like(x, 'x', maxdim=2) 

280 filt = array_like(filt, 'filt', ndim=x.ndim) 

281 

282 if filt.ndim == 1 or min(filt.shape) == 1: 

283 result = signal.convolve(x, filt, mode='valid') 

284 elif filt.ndim == 2: 

285 nlags = filt.shape[0] 

286 nvar = x.shape[1] 

287 result = np.zeros((x.shape[0] - nlags + 1, nvar)) 

288 if nsides == 2: 

289 for i in range(nvar): 

290 # could also use np.convolve, but easier for swiching to fft 

291 result[:, i] = signal.convolve(x[:, i], filt[:, i], 

292 mode='valid') 

293 elif nsides == 1: 

294 for i in range(nvar): 

295 result[:, i] = signal.convolve(x[:, i], np.r_[0, filt[:, i]], 

296 mode='valid') 

297 result = _pad_nans(result, trim_head, trim_tail) 

298 return pw.wrap(result) 

299 

300 

301# previously located in sandbox.tsa.garch 

302def miso_lfilter(ar, ma, x, useic=False): 

303 """ 

304 Filter multiple time series into a single time series. 

305 

306 Uses a convolution to merge inputs, and then lfilter to produce output. 

307 

308 Parameters 

309 ---------- 

310 ar : array_like 

311 The coefficients of autoregressive lag polynomial including lag zero, 

312 ar(L) in the expression ar(L)y_t. 

313 ma : array_like, same ndim as x, currently 2d 

314 The coefficient of the moving average lag polynomial, ma(L) in 

315 ma(L)x_t. 

316 x : array_like 

317 The 2-d input data series, time in rows, variables in columns. 

318 useic : bool 

319 Flag indicating whether to use initial conditions. 

320 

321 Returns 

322 ------- 

323 y : ndarray 

324 The filtered output series. 

325 inp : ndarray, 1d 

326 The combined input series. 

327 

328 Notes 

329 ----- 

330 currently for 2d inputs only, no choice of axis 

331 Use of signal.lfilter requires that ar lag polynomial contains 

332 floating point numbers 

333 does not cut off invalid starting and final values 

334 

335 miso_lfilter find array y such that: 

336 

337 ar(L)y_t = ma(L)x_t 

338 

339 with shapes y (nobs,), x (nobs, nvars), ar (narlags,), and 

340 ma (narlags, nvars). 

341 """ 

342 ma = array_like(ma, 'ma') 

343 ar = array_like(ar, 'ar') 

344 inp = signal.correlate(x, ma[::-1, :])[:, (x.shape[1] + 1) // 2] 

345 # for testing 2d equivalence between convolve and correlate 

346 # inp2 = signal.convolve(x, ma[:,::-1])[:, (x.shape[1]+1)//2] 

347 # np.testing.assert_almost_equal(inp2, inp) 

348 nobs = x.shape[0] 

349 # cut of extra values at end 

350 

351 # TODO: initialize also x for correlate 

352 if useic: 

353 return signal.lfilter([1], ar, inp, 

354 zi=signal.lfiltic(np.array([1., 0.]), ar, 

355 useic))[0][:nobs], inp[:nobs] 

356 else: 

357 return signal.lfilter([1], ar, inp)[:nobs], inp[:nobs]