Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1#!/usr/bin/env python 

2# encoding: utf-8 

3""" 

4*Tools for working with 'set-of-files' (sof) files* 

5 

6:Author: 

7 David Young & Marco Landoni 

8 

9:Date Created: 

10 January 22, 2020 

11""" 

12from builtins import object 

13import sys 

14import os 

15os.environ['TERM'] = 'vt100' 

16from fundamentals import tools 

17from astropy.io import fits 

18from ccdproc import ImageFileCollection 

19import codecs 

20from soxspipe.commonutils.keyword_lookup import keyword_lookup 

21 

22 

23class set_of_files(object): 

24 """ 

25 *The worker class for the sof module used to homogenise various frame input formats (sof file, directory of fits fits, list of fits file paths) into a CCDProc ImageFileCollection* 

26 

27 **Key Arguments:** 

28 - ``log`` -- logger 

29 - ``settings`` -- the settings dictionary 

30 - ``inputFrames`` -- can be a directory, a set-of-files (SOF) file or a list of fits frame paths. Default [] 

31 - ``verbose`` -- verbose. True or False. Default *True* 

32 

33 

34 **Usage** 

35 

36 To initiate a sof object, use the following: 

37 

38 ```python 

39 # inputFrames = "/path/to/a/directory" 

40 # inputFrames = ['/path/to/one.fits','/path/to/two.fits','/path/to/three.fits'] 

41 inputFrames = '/path/to/myfiles.sof' 

42 from soxspipe.commonutils import set_of_files 

43 sof = set_of_files( 

44 log=log, 

45 settings=settings, 

46 inputFrames=inputFrames 

47 ) 

48 ``` 

49 

50 `inputFrames` can be a directory, a list of fits filepaths or a set-of-files (SOF) file 

51 """ 

52 # Initialisation 

53 

54 def __init__( 

55 self, 

56 log, 

57 settings=False, 

58 inputFrames=[], 

59 verbose=True 

60 ): 

61 self.log = log 

62 log.debug("instansiating a new 'sof' object") 

63 self.settings = settings 

64 self.inputFrames = inputFrames 

65 self.verbose = verbose 

66 

67 # KEYWORD LOOKUP OBJECT - LOOKUP KEYWORD FROM DICTIONARY IN RESOURCES 

68 # FOLDER 

69 kw = keyword_lookup( 

70 log=self.log, 

71 settings=self.settings 

72 ).get 

73 

74 if self.verbose: 

75 keys = ['MJDOBS', 'CDELT1', 'CDELT2', 'PSZX', 'DPR_TYPE', 'DPR_CATG', 'DPR_TECH', 'SEQ_ARM', 

76 'EXPTIME', 'NAXIS1', 'NAXIS2', 'DET_READ_SPEED', 'CONAD', 'DET_GAIN', 'RON', 'CHIP_RON', 'BUNIT'] 

77 else: 

78 keys = ['MJDOBS', 'DPR_TYPE', 'DPR_CATG', 'DPR_TECH', 'SEQ_ARM', 

79 'EXPTIME'] 

80 

81 keys = kw(keys) 

82 self.keys = [] 

83 self.keys[:] = [k.lower() for k in keys] 

84 

85 # Initial Actions 

86 # FIX RELATIVE HOME PATHS 

87 from os.path import expanduser 

88 home = expanduser("~") 

89 if isinstance(self.inputFrames, str) and self.inputFrames[0] == "~": 

90 self.inputFrames = home + "/" + self.inputFrames[1:] 

91 

92 return None 

93 

94 def _generate_sof_file_from_directory( 

95 self, 

96 directory, 

97 sofPath): 

98 """*generate an sof file from a directory of FITS frames* 

99 

100 **Key Arguments:** 

101 - ``directory`` -- the path to the directory to containing the FITS files. 

102 - ``sofPath`` -- the path to generate the sof file to 

103 

104 **Return:** 

105 - ``sofPath`` -- the path to the sof file 

106 

107 **Usage** 

108 

109 ```python 

110 from soxspipe.commonutils import set_of_files 

111 sof = set_of_files( 

112 log=log, 

113 settings=settings 

114 ) 

115 sofFile = sof._generate_sof_file_from_directory( 

116 directory="path/to/directory", sofPath="/path/to/myFile.sof") 

117 ``` 

118 """ 

119 self.log.debug( 

120 'starting the ``_generate_sof_file_from_directory`` method') 

121 

122 from soxspipe.commonutils import keyword_lookup 

123 kw = keyword_lookup( 

124 log=self.log, 

125 settings=self.settings 

126 ).get 

127 

128 # MAKE RELATIVE HOME PATH ABSOLUTE 

129 from os.path import expanduser 

130 home = expanduser("~") 

131 if directory[0] == "~": 

132 directory = directory.replace("~", home) 

133 if sofPath[0] == "~": 

134 sofPath = sofPath.replace("~", home) 

135 

136 content = "" 

137 for d in sorted(os.listdir(directory)): 

138 if os.path.isfile(os.path.join(directory, d)) and (os.path.splitext(d)[-1].lower() == ".fits"): 

139 fitsPath = os.path.abspath(os.path.join(directory, d)) 

140 # OPEN FITS FILE AT HDULIST - HDU (HEADER DATA UNIT) CONTAINS A HEADER AND A DATA ARRAY (IMAGE) OR 

141 # TABLE. 

142 with fits.open(fitsPath) as hdul: 

143 # READ HEADER INTO MEMORY 

144 hdr = hdul[0].header 

145 # PRINT FULL FITS HEADER TO STDOUT 

146 # print(repr(hdr).strip()) 

147 dpr_type = hdr[kw("DPR_TYPE")].strip() 

148 # CHECK ARM 

149 arm = hdr[kw("SEQ_ARM")] 

150 # CHECK BINNING 

151 if kw('CDELT1') in hdr: 

152 xbin = str(int(hdr[kw('CDELT1')])) 

153 ybin = str(int(hdr[kw('CDELT2')])) 

154 catagory = dpr_type + "_" + arm.strip() 

155 if kw('CDELT1') in hdr: 

156 catagory += "_" + \ 

157 xbin.strip() + "x" + ybin.strip() 

158 

159 content += "%(fitsPath)s %(catagory)s\n" % locals() 

160 

161 # Recursively create missing directories 

162 moduleDirectory = os.path.dirname(sofPath) 

163 if not os.path.exists(moduleDirectory): 

164 os.makedirs(moduleDirectory) 

165 

166 # WRITE TO FILE 

167 with open(sofPath, 'w') as myFile: 

168 myFile.write(content) 

169 

170 self.log.debug( 

171 'completed the ``_generate_sof_file_from_directory`` method') 

172 return sofPath 

173 

174 def get(self): 

175 """*return the set-of-files as a CCDProc ImageFileCollection* 

176 

177 **Return:** 

178 - ``sof`` -- a ccdproc ImageFileCollection of the frames 

179 

180 **Usage** 

181 

182 To generate a ImageFileCollection from a directory, a list of fits filepaths or a set-of-files (SOF) file try the following: 

183 

184 ```python 

185 # inputFrames = "/path/to/a/directory" 

186 # inputFrames = ['/path/to/one.fits','/path/to/two.fits','/path/to/three.fits'] 

187 inputFrames = '/path/to/myfiles.sof' 

188 from soxspipe.commonutils import set_of_files 

189 sof = set_of_files( 

190 log=log, 

191 settings=settings, 

192 inputFrames=inputFrames 

193 ) 

194 sofFile, supplementarySof = sof.get() 

195 print(sofFile.summary) 

196 ``` 

197 

198 `inputFrames` can be a directory, a list of fits filepaths or a set-of-files (SOF) file. 

199 """ 

200 self.log.debug('starting the ``get`` method') 

201 

202 from os.path import expanduser 

203 home = expanduser("~") 

204 

205 if isinstance(self.inputFrames, str) and self.inputFrames[0] == "~": 

206 self.inputFrames = home + "/" + self.inputFrames[1:] 

207 

208 # DIRECTORY OF FRAMES 

209 if isinstance(self.inputFrames, str) and os.path.isdir(self.inputFrames): 

210 sof = ImageFileCollection(self.inputFrames, keywords=self.keys) 

211 supplementaryFilepaths = [] 

212 for d in os.listdir(self.inputFrames): 

213 filepath = os.path.join(self.inputFrames, d) 

214 if os.path.isfile(filepath) and ".fits" not in d.lower() and d[0] != ".": 

215 supplementaryFilepaths.append(filepath) 

216 

217 elif isinstance(self.inputFrames, str) and os.path.isfile(self.inputFrames) and '.sof' in self.inputFrames: 

218 readFile = codecs.open( 

219 self.inputFrames, encoding='utf-8', mode='r') 

220 thisData = readFile.read() 

221 readFile.close() 

222 lines = thisData.split("\n") 

223 

224 # REMOVE COMMENTED LINES 

225 lines = [l for l in lines if len(l) and l[0] != "#"] 

226 

227 fitsFiles = [] 

228 fitsFiles[:] = [l.split(".fits")[0].replace("~/", home + "/") + 

229 ".fits" for l in lines if ".fits" in l] 

230 supplementaryFilepaths = [ 

231 l.replace("~/", home + "/") for l in lines if ".fits" not in l.lower() and len(l) > 3] 

232 # MAKE SURE FILES EXIST 

233 allFiles = fitsFiles.extend(supplementaryFilepaths) 

234 for f in fitsFiles + supplementaryFilepaths: 

235 exists = os.path.exists(f) 

236 if not exists: 

237 raise FileNotFoundError(f"the input file `{f}` does not appear to exist") 

238 

239 locations = [os.path.dirname(f) for f in fitsFiles] 

240 if len(set(locations)) == 1: 

241 location = locations[0] 

242 fitsFiles = [os.path.basename( 

243 f) for f in fitsFiles] 

244 else: 

245 location = None 

246 sof = ImageFileCollection( 

247 filenames=fitsFiles, location=location, keywords=self.keys) 

248 elif isinstance(self.inputFrames, list): 

249 fitsFiles = [f for f in self.inputFrames if ".fits" in f.lower()] 

250 # FIND UNIQUE FILE LOCATIONS 

251 locations = [os.path.dirname(f) for f in fitsFiles] 

252 if len(set(locations)) == 1: 

253 location = locations[0] 

254 fitsFiles = [os.path.basename( 

255 f) for f in fitsFiles] 

256 else: 

257 location = None 

258 sof = ImageFileCollection( 

259 filenames=fitsFiles, location=location, keywords=self.keys) 

260 supplementaryFilepaths = [ 

261 f for f in self.inputFrames if ".fits" not in f.lower() and f[0] != "."] 

262 else: 

263 raise TypeError( 

264 "'inputFrames' should be the path to a directory of files, an SOF file or a list of FITS frame paths") 

265 

266 supplementary_sof = self.create_supplimentary_file_dictionary( 

267 supplementaryFilepaths) 

268 

269 self.log.debug('completed the ``get`` method') 

270 return sof, supplementary_sof 

271 

272 def create_supplimentary_file_dictionary( 

273 self, 

274 supplementaryFilepaths): 

275 """*create supplimentary file dictionary* 

276 

277 **Key Arguments:** 

278 - ``supplementaryFilepaths`` -- the list of filepaths to genereate the dictionary for 

279 

280 **Return:** 

281 - ``supplementary_sof`` -- a dictionary of non-fits files needed for recipe 

282 """ 

283 self.log.debug( 

284 'starting the ``create_supplimentary_file_dictionary`` method') 

285 

286 supplementary_sof = {} 

287 for f in supplementaryFilepaths: 

288 for a in ["NIR", "UVB", "VIS"]: 

289 if a.lower() in f.lower() and a not in supplementary_sof.keys(): 

290 supplementary_sof[a] = {} 

291 

292 for f in supplementaryFilepaths: 

293 if "disp_map" in f.lower(): 

294 for a in ["NIR", "UVB", "VIS"]: 

295 if a.lower() in f.lower(): 

296 supplementary_sof[a]["DISP_MAP"] = f 

297 if "order_locations" in f.lower() or "order_centres" in f.lower(): 

298 for a in ["NIR", "UVB", "VIS"]: 

299 if a.lower() in f.lower(): 

300 supplementary_sof[a]["ORDER_LOCATIONS"] = f 

301 if "2d_map" in f.lower(): 

302 for a in ["NIR", "UVB", "VIS"]: 

303 if a.lower() in f.lower(): 

304 supplementary_sof[a]["2D_MAP"] = f 

305 

306 self.log.debug( 

307 'completed the ``create_supplimentary_file_dictionary`` method') 

308 return supplementary_sof 

309 

310 # use the tab-trigger below for new method 

311 # xt-class-method