Source code for spike.util.signal_tools

#!/usr/bin/env python 
# encoding: utf-8

'''
Created by Lionel Chiron  18/10/2013 
Copyright (c) 2013 __NMRTEC__. All rights reserved.
Various tools for performing signal processing (SNR calculation,
    synthetic Fids with noise, noise level etc... )
'''

from __future__ import print_function, division
import numpy as np
from numpy.fft import rfft as nprfft, irfft as npirfft
from numpy import pi
import math
import scipy.fftpack as fft
from scipy.signal import firwin2, firwin, convolve, lfilter
from scipy.optimize import minimize
#from matplotlib import pyplot as plt
from ..Display import testplot
plt = testplot.plot()

import unittest
import time
#from .. import NPKData

[docs]def filtering(sig, window = None): ''' Filter for real signal window : list [f1, f2], eg: [0.1, 0.3], 0 < f < 1 ''' def FIR_filter(sig, nbpoles = 80, window = None ): filt = firwin(nbpoles, window, pass_zero = False) filtered_sig = lfilter(filt, 1.0, sig) return filtered_sig filtered_sig = FIR_filter(sig, nbpoles = 100, window = window) return filtered_sig
[docs]def SNR_dB(noisy,target): "computes and return SNR value, in dB" return 10*np.log10(sum(abs(target)**2)/sum(abs(noisy - target)**2))
[docs]def mfft(v): "utility that returns the modulus of the fft of v" s0 = fft.fft(v) s0 = np.sqrt(s0*s0.conj()) # ref spectrum return s0
[docs]def mrfft(v): "utility that returns the modulus of the rfft of v" s0 = nprfft(v) s0 = np.sqrt(s0*s0.conj()) # ref spectrum return s0
[docs]class SIGNAL_NOISE(object): ''' Tool for generating SIGNAL with/without NOISE. input: lenfid : length of the Fid nbpeaks : nb of frequencies amplitude : reference amplitude for signal LB : exponential apodisation factor noise : noise amplitude shape : shape of the spectrum. Available : triangular, list. noisetype : kind of noise, ie : additive, multiplicative etc... shift : shift of the frequencies, the shift is expressed in points. trunc : truncaton ratio for the Fid. seed : for using a precise seed for random processes. baseline : boolean for specifying if a baseline is added to the spectrum output: Noisy signal : self.fid Signal without noise : self.fid0 Noisy spectrum : self.spec Spectrum without noise : self.spec0 eg1: # simplest usage sig = SIGNAL_NOISE( lenfid= 1000, nbpeaks = 20, amplitude = 27, noise = 5) eg2: # with a list of amplitudes lamplit = list(np.random.randn(20)*10) sig = st.SIGNAL_NOISE( lenfid= 10000, nbpeaks = 20, amplitude = lamplit, noise = 4, shape = "list") ''' def __init__(self, lenfid, nbpeaks, amplitude, noise, LB = 1.11,\ shape = "triangular", noisetype = 'additive', shift = 0,\ realfid = False, trunc = None, seed = None, baseline=None): self.LB = LB # linewidth self.nbpeaks = nbpeaks # number of lines in the signal self.lenfid = lenfid # length of the fid self.noise = noise # noise amplitude self.noisetype = noisetype # kind of noise self.shift = shift # shift the spectrum in frequency domain. self.realfid = realfid self.trunc = trunc # factor of truncation of the Fid. if not seed: np.random.seed(seed) # frequency shift self.baseline = baseline self.x = np.arange(self.lenfid*1.0)/self.lenfid # time series self.fid = np.zeros(self.lenfid, dtype = complex) # fid without noise initialization self.fid0 = np.zeros(self.lenfid, dtype = complex) # noisy fid initialization self.shape = shape # shape of spectrum self.slope = 20 # slope for the triangular shape self.mult_Anoise = noise/2 # amplitude for multiplicative noise self.mult_Fnoise = noise/200 # drift in frequency for multiplicative noise self.amplitude = amplitude # amplitude of the signal ####### self.prepare_amplitudes() # make the list of amplitudes self.prepare_frequencies() # make the list of frequencies self.make_pure_signal() # make the signal without noise self.make_noisy_signal() # make the noisy signal if self.realfid: self.mfft = mrfft self.fid = np.real(self.fid) self.fid0 = np.real(self.fid0) else: self.mfft = mfft self.fid_orig = self.fid.copy() # keep initial Fid when performing truncation etc.. self.fid0_orig = self.fid0.copy() # keep initial Fid when performing truncation etc..
[docs] def cut(self): ''' Uses Fid truncated ''' self.fid[self.lenfid//self.trunc:] = 0 # truncate self.fid0[self.lenfid//self.trunc:] = 0 # truncate
[docs] def full(): ''' Uses full Fid. ''' self.fid = self.fid_orig.copy() # retrieve original self.fid0 = self.fid_orig.copy() # retrieve original
[docs] def prepare_amplitudes(self): ''' Makes the list of amplitudes according to the case. triangular: makes a triangular shape beginning at amplitude "self.amplitude" with slope "self.slope" list: retrieves amplitudes from given list of amplitudes. ''' if self.shape == 'triangular': self.Amp = [self.amplitude + (i+1)*self.slope for i in range(self.nbpeaks)] # amplitudes elif self.shape == 'list': self.Amp = self.amplitude # list of amplitudes
[docs] def prepare_frequencies(self): ''' Frequencies for the signal ''' self.Freq = [(i+1+np.sqrt(10))*pi*500.0j for i in range(self.nbpeaks)] # frequencies
[docs] def make_pure_signal(self): ''' Signal without noise ''' for i in range(self.nbpeaks): self.fid0 += self.Amp[i] * np.exp(self.Freq[i]*self.x) self.fid0 *= np.exp(-self.LB*self.x) self.fid0 *= np.exp(1j*self.shift*self.x)
[docs] def make_noisy_signal(self): ''' ''' if self.noisetype == "additive": self._make_additive_noise() elif self.noisetype == "multiplicative": self._make_multiplicative_noise() elif self.noisetype == "sampling": self._make_sampling_noise() elif self.noisetype == "missing points": self._make_missing_point_noise()
def _make_additive_noise(self): ''' Makes complex additive noise. ''' real = np.random.randn(self.x.size) imag = np.random.randn(self.x.size) self.fid = self.fid0 + self.noise*(real + 1j*imag) # additive complex noise def _make_multiplicative_noise(self, ampl = True, freq = False): ''' Adding multiplicative noise on amplitude and frequency. ''' for i in range(self.nbpeaks): nAmp = self.Amp[i] nFreq = self.Freq[i] if ampl: # nAmp *= self.mult_Anoise*np.random.randn(self.x.size) nAmp += self.mult_Anoise*np.random.randn(self.x.size) if freq: nFreq += self.mult_Fnoise*np.random.randn(self.x.size) self.fid += nAmp * np.exp(nFreq*self.x) self.fid *= np.exp(-self.LB*self.x) self.fid *= np.exp(1j*self.shift*self.x) def _make_sampling_noise(self): ''' Takes point with a random step. ''' xn = self.x + 0.5*np.random.randn(self.x.size)/self.lenfid # time series with noisy jitter for i in range(self.nbpeaks): self.fid += self.Amp[i] * np.exp(self.Freq[i]*xn) * np.exp(-self.LB*xn) * np.exp(1j*self.shift*self.x) def _make_missing_point_noise(self): ''' Removes randomly some points after simulation of a regular set of points. ''' miss = np.random.randint(2, size = len(self.x)) self.fid = self.fid0*miss def _spec_baseline(self): maxi = np.abs(self.mfft(self.fid).max()) x = np.linspace(0,10,self.lenfid) return maxi*(0.1*np.sin(1.5*x)+0.2*x/x.max()) def _spec_from_fid(self, fid): ''' Passing from fid to spectrum ''' spec = self.mfft(fid) if self.baseline : spec += self._spec_baseline() return spec @property def spec(self): ''' Noisy spectrum ''' return self._spec_from_fid(self.fid) @property def spec0(self): ''' Spectrum wihtout noise ''' return self._spec_from_fid(self.fid0)
[docs] def zerof(self, fid, fid_ref): ''' Zerofilling ''' zeropadded = np.concatenate((fid, np.zeros(len(fid_ref)-len(fid)))) if self.realfid: fid_zerf = npirfft(nprfft(zeropadded)) else: fid_zerf = npifft(npfft(zeropadded)) return fid_zerf
@property def iSNR(self): ''' Reference SNR ''' return SNR_dB(self.fid, self.fid0)
[docs] def SNR_dB(self, processed_fid): ''' SNR of the denoised signal ''' if processed_fid.size == self.fid0.size: fid_ref = self.fid0 else: fid_ref = self.zerof(self.fid0, processed_fid) return SNR_dB(processed_fid, fid_ref)
[docs]def fid_signoise_type(nbpeaks,lendata, noise, noisetype): "Obsolete" LB = 1.11 # linewidth Freq = [(i+1+np.sqrt(10))*pi*500.0j for i in range(nbpeaks)] # frequencies Amp = [(i+1)*20 for i in range(nbpeaks)] # amplitudes data0 = np.zeros(lendata,dtype=complex) x = np.arange(lendata*1.0)/lendata # time series for i in range(nbpeaks): data0 += Amp[i] * np.exp(Freq[i]*x) * np.exp(-LB*x) if noisetype == "additive": dataadd = data0 + noise*(np.random.randn(x.size) + 1j*np.random.randn(x.size)) # additive complex noise data = dataadd elif noisetype == "multiplicative": Anoise = noise/2 Fnoise = noise/200 for i in range(nbpeaks): nAmp = Amp[i] + Anoise*np.random.randn(x.size) nFreq = Freq[i] + Fnoise*np.random.randn(x.size) data += nAmp * np.exp(nFreq*x) * np.exp(-LB*x) elif noisetype == "sampling": xn = x + 0.5*np.random.randn(x.size)/lendata # time series with noisy jitter data = np.zeros(lendata,dtype = complex) for i in range(nbpeaks): data += Amp[i] * np.exp(Freq[i]*xn) * np.exp(-LB*xn) elif noisetype == "missing points": miss = np.random.randint(2, size=len(x)) dataadd = data0*miss data = dataadd else: raise Exception("unknown noise type") return data
[docs]def fid_signoise(nbpeaks, ampl, lengthfid, noise, shift = 0, shape = "triangular", seed = True): ''' Obsolete Build fid with triangular spectrum from number of peaks : nbpeaks, amplitude : ampl, the length of the fid : lengthfid, and the noise level : noise. if shape is "triangular", uses ampl as minimum amplitude. if shape is "list", uses the given list to make the amplitudes. The seed of random generator can be activated or deactivated with boolean 'seed' ''' if seed: np.random.seed(11232) LB = 1 # linewidth x = np.arange(lengthfid*1.0)/lengthfid fid0 = 1j*np.zeros_like(x) # complex fid omeg = 430.1*1j for i in range(1, nbpeaks + 1): if shape == "triangular": fid0 += i*ampl*np.exp(omeg*(i)*x)*np.exp(-LB*x)*np.exp(1j*shift*x) # elif shape == 'list': fid0 += ampl[i-1]*np.exp(omeg*(i)*x)*np.exp(-LB*x)*np.exp(1j*shift*x) fid = fid0 + noise*(np.random.randn(x.size) + 1j*np.random.randn(x.size)) # additive noise return fid
[docs]def findnoiselevel(fid, nbseg = 20): """ Routine for determining the noise level from a numpy buffer "fid" cut the data in segments then make the standard deviation return makes the average on the 25% lowest segments nbseg=10 nb of segment to cut the spectrum """ less = len(fid)%nbseg # rest of division of length of data by nb of segment restpeaks = fid[less:] # remove the points that avoid to divide correctly the data in segment of same size. newlist = np.array(np.split(restpeaks, nbseg)) #Cutting in segments levels = newlist.std(axis = 1) levels.sort() if nbseg < 4: noiselev = levels[0] else: noiselev = np.mean(levels[0:nbseg//4]) return noiselev
[docs]def findnoiselevel_2D(data_array, nbseg=20, nbrow=200): """ measure noise level on a 2D spectrum """ si1 = data_array.shape[0] if si1 > nbrow: plist = np.random.permutation(si1)[:nbrow] else: plist = range(si1) levels = [findnoiselevel(data_array[p,:], nbseg=nbseg) for p in plist] return np.mean(levels)
def _findnoiselevel_2D(data_array, nbseg = 20): """ obsolete """ dim_array = data_array.size x = np.random.randint(dim_array, size = dim_array[0]) y = np.random.randint(dim_array, size = dim_array[1]) random2D = zip(x, y) noiselev = data_array[random2D].std() return noiselev
[docs]def findnoiselevel_2D_slice(data_array, ratio = 1): """ Routine for finding noise level by dividing the 2D spectrum in vertical slices. """ lstd = [] for i in range(int(data_array.shape[1]*ratio)): randcol = np.random.randint(data_array.shape[1]) cut = data_array[:,randcol] noisecut = findnoiselevel(cut, nbseg=30) lstd.append(noisecut) noiselev = np.array(lstd).mean() return noiselev
[docs]def findnoiselevel2(fid, nbseg=20): """ (findnoiselevel seems more reliable) Routine for determining the noise level from a numpy buffer "fid" cut the data in segments then make the standard deviation and returns the smallest deviation nbseg=10 nb of segment to cut the spectrum """ less = len(fid)%nbseg # rest of division of length of data by nb of segment restpeaks = fid[less:] # remove the points that avoid to divide correctly the data in segment of same size. newlist = np.array(np.hsplit(restpeaks,nbseg)) #Cutting in segments noiselev = newlist.std(axis = 1).min() return noiselev
[docs]def findnoiselevel_offset(fid, nbseg = 10): """ Routine for determining the noise level and mean value (considering a flat signal with gaussian noise and some outliers in the signal) cut the data in segments then make the standard deviation for each segment compares the std dev with the average one and eliminates segments giving a std dev above the mean finally makes the average on the sorted segments. nbseg=10 nb of segment to cut the spectrum """ def fun(x, t, data): return np.sum(np.abs(x[0] + x[1]* t - data)) # L1 norm for the mean. less = len(fid)%nbseg # rest of division of length of data by nb of segment if less !=0: restpeaks = fid[:-less] # remove the points that avoid to divide correctly the data in segment of same size. else: restpeaks = fid newlist = np.array(np.hsplit(restpeaks,nbseg)) #Cutting in segments noiselev = newlist.std(axis=1).min() noisemean = newlist.mean(axis=1) t = np.arange(len(noisemean)) res = minimize(fun, x0 = [0,0], args=(t,noisemean), method="Powell") # Find the mean with L1 norm return noiselev, res.x[0]
[docs]def signal_to_noise(fid, nbseg = 20, peak = "max"): """ determines the signal to noise of a numpy buffer, by dividing le largest peak witht the noiselevel see findnoiselevel """ if peak == "max": sn = max(fid)/findnoiselevel(fid, nbseg) else: sn = fid[peak]/findnoiselevel(fid, nbseg) return sn
[docs]def SNR(dataset, nbseg = 20, peak = "max", unit = None): """ determines the signal to noise of NPKData, by dividing le largest peak witht the noiselevel handles complex spectra, taking the noiselevel and the peak on the real part noise is evaluated by segmenting the data-set in nbseg peices, check in findnoiselevel, peak is either max or an index on which to evaluate the signal intensity if unit is None, value is in ration, if uint is dB, value is in dB """ noise = findnoiselevel(dataset.buffer[::dataset.axis1.itype+1], nbseg = nbseg) if dataset.axis1.itype == 1: buff = dataset.buffer[::2] try: peak = peak/2 except: pass # peak is "max" else: buff = dataset.buffer if peak == "max": pk = max(buff) else: pk = buff[peak] snr = pk/noise if unit == 'dB': snr = dB(snr) return snr
[docs]def dB(val): """ converts a real value to dB 20*math.log(val)/math.log(10) """ return 20*math.log(val)/math.log(10)
[docs]def em_wiener(x,lb): "multiply x(t) by exp(-lb*t)" t = 1.0*np.arange(len(x)) return x*np.exp(-t*lb)
[docs]def apod(lb = 1): ''' apodisation for improving sparsity ''' return em_wiener(x,lb)
[docs]class Test_Units(unittest.TestCase): ''' Unittests test_signoise test_signoise_truncated test_noiselevel test_multitests_urQRdtrick test_multitests_urQRd_npk '''
[docs] def ttest_signoise(self): ''' Testing the class for generating noisy signal. ''' from ..Display import testplot plt = testplot.plot() lendata = 10000 nbpeaks = 10 amplitude = 10 noise = 20 signoise = SIGNAL_NOISE(lendata, nbpeaks, amplitude, noise) plt.plot(signoise.spec) plt.show()
[docs] def ttest_signoise_truncated(self): ''' Testing the class for generating noisy signal. ''' from ..Display import testplot plt = testplot.plot() lendata = 10000 nbpeaks = 10 amplitude = 10 noise = 20 signoise = SIGNAL_NOISE(lendata, nbpeaks, amplitude, noise, trunc = 10) plt.plot(signoise.spec) # spectrum of the full signal signoise.cut() # takes the truncated signal with zerofilling plt.plot(signoise.spec) # spectrum of the truncated signal plt.show()
[docs] def ttest_noiselevel(self): ''' Testing noiselevel on experimental data. ''' from ..Tests import filename from .. import FTICR import math d = FTICR.FTICRData(name = filename("ubiquitine_2D_000002_mr.msh5")) e = d.col(11033) e.display(show = True) for n in (2,5,10,20,50,100,200): print(findnoiselevel(e.buffer, n)) snr = SNR(e) print("total SNR: %.0f %.2f dB"%( snr, 20*math.log(snr)/math.log(10))) snr = SNR(e, peak = 453) print("pk 453 SNR: %.0f %.2f dB"%( snr, 20*math.log(snr)/math.log(10)))
[docs] def ttest_multitests_urQRdtrick(self): ''' Multitests on urQRd_trick ''' from ..Algo.urQRd_trick import urQRd mt = MULTITESTS(algo = urQRd, ampl_sig = 50, ampl_noise = 120.0,\ nbpeaks = 6, nb_tries = 1, plot_fft = True) algo_param = '2*self.nbpeaks' mt.run(algo_param)
[docs] def ttest_multitests_urQRd_npk(self): ''' Multitests on urQRd_trick ''' mt = MULTITESTS(algo = 'urqrd', ampl_sig = 50, ampl_noise = 30.0,\ nbpeaks = 6, nb_tries = 3, npk = True) algo_param = 'k = 2*self.nbpeaks, orda = self.lenfid/2' mt.run(algo_param)
[docs] def ttest_findnoiselevel_2D_slice(self): arr = np.random.randn(1000,2000) arr[25:75,50:150]+=1e9 # Square with high values nl = findnoiselevel_2D_slice(arr, ratio=0.1) # noise level self.assertAlmostEqual(nl, 0.825, 2)
if __name__ == '__main__': unittest.main()