Coverage for soxspipe/commonutils/set_of_files.py : 96%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1#!/usr/bin/env python
2# encoding: utf-8
3"""
4*Tools for working with 'set-of-files' (sof) files*
6:Author:
7 David Young & Marco Landoni
9:Date Created:
10 January 22, 2020
11"""
12from builtins import object
13import sys
14import os
15os.environ['TERM'] = 'vt100'
16from fundamentals import tools
17from astropy.io import fits
18from ccdproc import ImageFileCollection
19import codecs
20from soxspipe.commonutils.keyword_lookup import keyword_lookup
23class set_of_files(object):
24 """
25 *The worker class for the sof module used to homogenise various frame input formats (sof file, directory of fits fits, list of fits file paths) into a CCDProc ImageFileCollection*
27 **Key Arguments:**
28 - ``log`` -- logger
29 - ``settings`` -- the settings dictionary
30 - ``inputFrames`` -- can be a directory, a set-of-files (SOF) file or a list of fits frame paths. Default []
31 - ``verbose`` -- verbose. True or False. Default *True*
34 **Usage**
36 To initiate a sof object, use the following:
38 ```python
39 # inputFrames = "/path/to/a/directory"
40 # inputFrames = ['/path/to/one.fits','/path/to/two.fits','/path/to/three.fits']
41 inputFrames = '/path/to/myfiles.sof'
42 from soxspipe.commonutils import set_of_files
43 sof = set_of_files(
44 log=log,
45 settings=settings,
46 inputFrames=inputFrames
47 )
48 ```
50 `inputFrames` can be a directory, a list of fits filepaths or a set-of-files (SOF) file
51 """
52 # Initialisation
54 def __init__(
55 self,
56 log,
57 settings=False,
58 inputFrames=[],
59 verbose=True
60 ):
61 self.log = log
62 log.debug("instansiating a new 'sof' object")
63 self.settings = settings
64 self.inputFrames = inputFrames
65 self.verbose = verbose
67 # KEYWORD LOOKUP OBJECT - LOOKUP KEYWORD FROM DICTIONARY IN RESOURCES
68 # FOLDER
69 kw = keyword_lookup(
70 log=self.log,
71 settings=self.settings
72 ).get
74 if self.verbose:
75 keys = ['MJDOBS', 'CDELT1', 'CDELT2', 'PSZX', 'DPR_TYPE', 'DPR_CATG', 'DPR_TECH', 'SEQ_ARM',
76 'EXPTIME', 'NAXIS1', 'NAXIS2', 'DET_READ_SPEED', 'CONAD', 'DET_GAIN', 'RON', 'CHIP_RON', 'BUNIT']
77 else:
78 keys = ['MJDOBS', 'DPR_TYPE', 'DPR_CATG', 'DPR_TECH', 'SEQ_ARM',
79 'EXPTIME']
81 keys = kw(keys)
82 self.keys = []
83 self.keys[:] = [k.lower() for k in keys]
85 # Initial Actions
86 # FIX RELATIVE HOME PATHS
87 from os.path import expanduser
88 home = expanduser("~")
89 if isinstance(self.inputFrames, str) and self.inputFrames[0] == "~":
90 self.inputFrames = home + "/" + self.inputFrames[1:]
92 return None
94 def _generate_sof_file_from_directory(
95 self,
96 directory,
97 sofPath):
98 """*generate an sof file from a directory of FITS frames*
100 **Key Arguments:**
101 - ``directory`` -- the path to the directory to containing the FITS files.
102 - ``sofPath`` -- the path to generate the sof file to
104 **Return:**
105 - ``sofPath`` -- the path to the sof file
107 **Usage**
109 ```python
110 from soxspipe.commonutils import set_of_files
111 sof = set_of_files(
112 log=log,
113 settings=settings
114 )
115 sofFile = sof._generate_sof_file_from_directory(
116 directory="path/to/directory", sofPath="/path/to/myFile.sof")
117 ```
118 """
119 self.log.debug(
120 'starting the ``_generate_sof_file_from_directory`` method')
122 from soxspipe.commonutils import keyword_lookup
123 kw = keyword_lookup(
124 log=self.log,
125 settings=self.settings
126 ).get
128 # MAKE RELATIVE HOME PATH ABSOLUTE
129 from os.path import expanduser
130 home = expanduser("~")
131 if directory[0] == "~":
132 directory = directory.replace("~", home)
133 if sofPath[0] == "~":
134 sofPath = sofPath.replace("~", home)
136 content = ""
137 for d in sorted(os.listdir(directory)):
138 if os.path.isfile(os.path.join(directory, d)) and (os.path.splitext(d)[-1].lower() == ".fits"):
139 fitsPath = os.path.abspath(os.path.join(directory, d))
140 # OPEN FITS FILE AT HDULIST - HDU (HEADER DATA UNIT) CONTAINS A HEADER AND A DATA ARRAY (IMAGE) OR
141 # TABLE.
142 with fits.open(fitsPath) as hdul:
143 # READ HEADER INTO MEMORY
144 hdr = hdul[0].header
145 # PRINT FULL FITS HEADER TO STDOUT
146 # print(repr(hdr).strip())
147 dpr_type = hdr[kw("DPR_TYPE")].strip()
148 # CHECK ARM
149 arm = hdr[kw("SEQ_ARM")]
150 # CHECK BINNING
151 if kw('CDELT1') in hdr:
152 xbin = str(int(hdr[kw('CDELT1')]))
153 ybin = str(int(hdr[kw('CDELT2')]))
154 catagory = dpr_type + "_" + arm.strip()
155 if kw('CDELT1') in hdr:
156 catagory += "_" + \
157 xbin.strip() + "x" + ybin.strip()
159 content += "%(fitsPath)s %(catagory)s\n" % locals()
161 # Recursively create missing directories
162 moduleDirectory = os.path.dirname(sofPath)
163 if not os.path.exists(moduleDirectory):
164 os.makedirs(moduleDirectory)
166 # WRITE TO FILE
167 with open(sofPath, 'w') as myFile:
168 myFile.write(content)
170 self.log.debug(
171 'completed the ``_generate_sof_file_from_directory`` method')
172 return sofPath
174 def get(self):
175 """*return the set-of-files as a CCDProc ImageFileCollection*
177 **Return:**
178 - ``sof`` -- a ccdproc ImageFileCollection of the frames
180 **Usage**
182 To generate a ImageFileCollection from a directory, a list of fits filepaths or a set-of-files (SOF) file try the following:
184 ```python
185 # inputFrames = "/path/to/a/directory"
186 # inputFrames = ['/path/to/one.fits','/path/to/two.fits','/path/to/three.fits']
187 inputFrames = '/path/to/myfiles.sof'
188 from soxspipe.commonutils import set_of_files
189 sof = set_of_files(
190 log=log,
191 settings=settings,
192 inputFrames=inputFrames
193 )
194 sofFile, supplementarySof = sof.get()
195 print(sofFile.summary)
196 ```
198 `inputFrames` can be a directory, a list of fits filepaths or a set-of-files (SOF) file.
199 """
200 self.log.debug('starting the ``get`` method')
202 from os.path import expanduser
203 home = expanduser("~")
205 if isinstance(self.inputFrames, str) and self.inputFrames[0] == "~":
206 self.inputFrames = home + "/" + self.inputFrames[1:]
208 # DIRECTORY OF FRAMES
209 if isinstance(self.inputFrames, str) and os.path.isdir(self.inputFrames):
210 sof = ImageFileCollection(self.inputFrames, keywords=self.keys)
211 supplementaryFilepaths = []
212 for d in os.listdir(self.inputFrames):
213 filepath = os.path.join(self.inputFrames, d)
214 if os.path.isfile(filepath) and ".fits" not in d.lower() and d[0] != ".":
215 supplementaryFilepaths.append(filepath)
217 elif isinstance(self.inputFrames, str) and os.path.isfile(self.inputFrames) and '.sof' in self.inputFrames:
218 readFile = codecs.open(
219 self.inputFrames, encoding='utf-8', mode='r')
220 thisData = readFile.read()
221 readFile.close()
222 lines = thisData.split("\n")
224 # REMOVE COMMENTED LINES
225 lines = [l for l in lines if len(l) and l[0] != "#"]
227 fitsFiles = []
228 fitsFiles[:] = [l.split(".fits")[0].replace("~/", home + "/") +
229 ".fits" for l in lines if ".fits" in l]
230 supplementaryFilepaths = [
231 l.replace("~/", home + "/") for l in lines if ".fits" not in l.lower() and len(l) > 3]
232 # MAKE SURE FILES EXIST
233 allFiles = fitsFiles.extend(supplementaryFilepaths)
234 for f in fitsFiles + supplementaryFilepaths:
235 exists = os.path.exists(f)
236 if not exists:
237 raise FileNotFoundError(f"the input file `{f}` does not appear to exist")
239 locations = [os.path.dirname(f) for f in fitsFiles]
240 if len(set(locations)) == 1:
241 location = locations[0]
242 fitsFiles = [os.path.basename(
243 f) for f in fitsFiles]
244 else:
245 location = None
246 sof = ImageFileCollection(
247 filenames=fitsFiles, location=location, keywords=self.keys)
248 elif isinstance(self.inputFrames, list):
249 fitsFiles = [f for f in self.inputFrames if ".fits" in f.lower()]
250 # FIND UNIQUE FILE LOCATIONS
251 locations = [os.path.dirname(f) for f in fitsFiles]
252 if len(set(locations)) == 1:
253 location = locations[0]
254 fitsFiles = [os.path.basename(
255 f) for f in fitsFiles]
256 else:
257 location = None
258 sof = ImageFileCollection(
259 filenames=fitsFiles, location=location, keywords=self.keys)
260 supplementaryFilepaths = [
261 f for f in self.inputFrames if ".fits" not in f.lower() and f[0] != "."]
262 else:
263 raise TypeError(
264 "'inputFrames' should be the path to a directory of files, an SOF file or a list of FITS frame paths")
266 supplementary_sof = self.create_supplimentary_file_dictionary(
267 supplementaryFilepaths)
269 self.log.debug('completed the ``get`` method')
270 return sof, supplementary_sof
272 def create_supplimentary_file_dictionary(
273 self,
274 supplementaryFilepaths):
275 """*create supplimentary file dictionary*
277 **Key Arguments:**
278 - ``supplementaryFilepaths`` -- the list of filepaths to genereate the dictionary for
280 **Return:**
281 - ``supplementary_sof`` -- a dictionary of non-fits files needed for recipe
282 """
283 self.log.debug(
284 'starting the ``create_supplimentary_file_dictionary`` method')
286 supplementary_sof = {}
287 for f in supplementaryFilepaths:
288 for a in ["NIR", "UVB", "VIS"]:
289 if a.lower() in f.lower() and a not in supplementary_sof.keys():
290 supplementary_sof[a] = {}
292 for f in supplementaryFilepaths:
293 if "disp_map" in f.lower():
294 for a in ["NIR", "UVB", "VIS"]:
295 if a.lower() in f.lower():
296 supplementary_sof[a]["DISP_MAP"] = f
297 if "order_locations" in f.lower() or "order_centres" in f.lower():
298 for a in ["NIR", "UVB", "VIS"]:
299 if a.lower() in f.lower():
300 supplementary_sof[a]["ORDER_LOCATIONS"] = f
301 if "2d_map" in f.lower():
302 for a in ["NIR", "UVB", "VIS"]:
303 if a.lower() in f.lower():
304 supplementary_sof[a]["2D_MAP"] = f
306 self.log.debug(
307 'completed the ``create_supplimentary_file_dictionary`` method')
308 return supplementary_sof
310 # use the tab-trigger below for new method
311 # xt-class-method