Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1#!/usr/bin/env python 

2# encoding: utf-8 

3""" 

4*The base recipe class which all other recipes inherit* 

5 

6:Author: 

7 David Young & Marco Landoni 

8 

9:Date Created: 

10 January 22, 2020 

11""" 

12################# GLOBAL IMPORTS #################### 

13from builtins import object 

14import sys 

15import os 

16os.environ['TERM'] = 'vt100' 

17from fundamentals import tools 

18import numpy as np 

19from astropy.nddata import CCDData 

20from astropy import units as u 

21from astropy.stats import mad_std 

22import ccdproc 

23import pandas as pd 

24from astropy.nddata.nduncertainty import StdDevUncertainty 

25from ccdproc import Combiner 

26from soxspipe.commonutils import set_of_files 

27from soxspipe.commonutils import keyword_lookup 

28from soxspipe.commonutils import detector_lookup 

29from datetime import datetime 

30from soxspipe.commonutils import filenamer 

31import shutil 

32from tabulate import tabulate 

33 

34 

35class _base_recipe_(object): 

36 """ 

37 The base recipe class which all other recipes inherit 

38 

39 **Key Arguments:** 

40 - ``log`` -- logger 

41 - ``settings`` -- the settings dictionary 

42 - ``verbose`` -- verbose. True or False. Default *False* 

43 

44 **Usage** 

45 

46 To use this base recipe to create a new `soxspipe` recipe, have a look at the code for one of the simpler recipes (e.g. `soxs_mbias`) - copy and modify the code. 

47 """ 

48 

49 def __init__( 

50 self, 

51 log, 

52 settings=False, 

53 verbose=False 

54 ): 

55 self.log = log 

56 log.debug("instansiating a new '__init__' object") 

57 self.settings = settings 

58 self.intermediateRootPath = self._absolute_path( 

59 settings["intermediate-data-root"]) 

60 self.reducedRootPath = self._absolute_path( 

61 settings["reduced-data-root"]) 

62 self.calibrationRootPath = self._absolute_path( 

63 settings["calibration-data-root"]) 

64 

65 self.verbose = verbose 

66 # SET LATER WHEN VERIFYING FRAMES 

67 self.arm = None 

68 self.detectorParams = None 

69 self.dateObs = None 

70 

71 # DATAFRAMES TO COLLECT QCs AND PRODUCTS 

72 self.qc = pd.DataFrame({ 

73 "soxspipe_recipe": [], 

74 "qc_name": [], 

75 "qc_value": [], 

76 "qc_unit": [], 

77 "obs_date_utc": [], 

78 "reduction_date_utc": [] 

79 }) 

80 self.products = pd.DataFrame({ 

81 "soxspipe_recipe": [], 

82 "product_label": [], 

83 "file_name": [], 

84 "file_type": [], 

85 "obs_date_utc": [], 

86 "reduction_date_utc": [], 

87 "file_path": [] 

88 }) 

89 

90 # KEYWORD LOOKUP OBJECT - LOOKUP KEYWORD FROM DICTIONARY IN RESOURCES 

91 # FOLDER 

92 self.kw = keyword_lookup( 

93 log=self.log, 

94 settings=self.settings 

95 ).get 

96 

97 return None 

98 

99 def _prepare_single_frame( 

100 self, 

101 frame, 

102 save=False): 

103 """*prepare a single raw frame by converting pixel data from ADU to electrons and adding mask and uncertainty extensions* 

104 

105 **Key Arguments:** 

106 - ``frame`` -- the path to the frame to prepare, of a CCDData object 

107 

108 **Return:** 

109 - ``frame`` -- the prepared frame with mask and uncertainty extensions (CCDData object) 

110 

111 ```eval_rst 

112 .. todo:: 

113 

114 - write a command-line tool for this method 

115 ``` 

116 """ 

117 self.log.debug('starting the ``_prepare_single_frame`` method') 

118 

119 kw = self.kw 

120 dp = self.detectorParams 

121 

122 # STORE FILEPATH FOR LATER USE 

123 filepath = frame 

124 

125 # CONVERT FILEPATH TO CCDDATA OBJECT 

126 if isinstance(frame, str): 

127 # CONVERT RELATIVE TO ABSOLUTE PATHS 

128 frame = self._absolute_path(frame) 

129 # OPEN THE RAW FRAME - MASK AND UNCERT TO BE POPULATED LATER 

130 frame = CCDData.read(frame, hdu=0, unit=u.adu, hdu_uncertainty='ERRS', 

131 hdu_mask='QUAL', hdu_flags='FLAGS', key_uncertainty_type='UTYPE') 

132 

133 # CHECK THE NUMBER OF EXTENSIONS IS ONLY 1 AND "SXSPRE" DOES NOT 

134 # EXIST. i.e. THIS IS A RAW UNTOUCHED FRAME 

135 if len(frame.to_hdu()) > 1 or "SXSPRE" in frame.header: 

136 return filepath 

137 

138 # MANIPULATE XSH DATA 

139 frame = self.xsh2soxs(frame) 

140 frame = self._trim_frame(frame) 

141 

142 # CORRECT FOR GAIN - CONVERT DATA FROM ADU TO ELECTRONS 

143 frame = ccdproc.gain_correct(frame, dp["gain"]) 

144 

145 # GENERATE UNCERTAINTY MAP AS EXTENSION 

146 if frame.header[kw("DPR_TYPE")] == "BIAS": 

147 # ERROR IS ONLY FROM READNOISE FOR BIAS FRAMES 

148 errorMap = np.ones_like(frame.data) * dp["ron"] 

149 # errorMap = StdDevUncertainty(errorMap) 

150 frame.uncertainty = errorMap 

151 else: 

152 # GENERATE UNCERTAINTY MAP AS EXTENSION 

153 frame = ccdproc.create_deviation( 

154 frame, readnoise=dp["ron"]) 

155 

156 # FIND THE APPROPRIATE BAD-PIXEL BITMAP AND APPEND AS 'FLAG' EXTENSION 

157 # NOTE FLAGS NOTE YET SUPPORTED BY CCDPROC THIS THIS WON'T GET SAVED OUT 

158 # AS AN EXTENSION 

159 arm = self.arm 

160 if kw('WIN_BINX') in frame.header: 

161 binx = int(frame.header[kw('WIN_BINX')]) 

162 biny = int(frame.header[kw('WIN_BINY')]) 

163 else: 

164 binx = 1 

165 biny = 1 

166 

167 bitMapPath = self.calibrationRootPath + "/" + dp["bad-pixel map"][f"{binx}x{biny}"] 

168 

169 if not os.path.exists(bitMapPath): 

170 message = "the path to the bitMapPath %s does not exist on this machine" % ( 

171 bitMapPath,) 

172 self.log.critical(message) 

173 raise IOError(message) 

174 bitMap = CCDData.read(bitMapPath, hdu=0, unit=u.dimensionless_unscaled) 

175 

176 # BIAS FRAMES HAVE NO 'FLUX', JUST READNOISE, SO ADD AN EMPTY BAD-PIXEL 

177 # MAP 

178 if frame.header[kw("DPR_TYPE")] == "BIAS": 

179 bitMap.data = np.zeros_like(bitMap.data) 

180 

181 # print(bitMap.data.shape) 

182 # print(frame.data.shape) 

183 

184 frame.flags = bitMap.data 

185 

186 # FLATTEN BAD-PIXEL BITMAP TO BOOLEAN FALSE (GOOD) OR TRUE (BAD) AND 

187 # APPEND AS 'UNCERT' EXTENSION 

188 boolMask = bitMap.data.astype(bool).data 

189 try: 

190 # FAILS IN PYTHON 2.7 AS BOOLMASK IS A BUFFER - NEED TO CONVERT TO 

191 # 2D ARRAY 

192 boolMask.shape 

193 

194 except: 

195 arr = np.frombuffer(boolMask, dtype=np.uint8) 

196 arr.shape = (frame.data.shape) 

197 boolMask = arr 

198 

199 frame.mask = boolMask 

200 

201 if save: 

202 outDir = self.intermediateRootPath 

203 else: 

204 outDir = self.intermediateRootPath + "/tmp" 

205 

206 # INJECT THE PRE KEYWORD 

207 utcnow = datetime.utcnow() 

208 frame.header["SXSPRE"] = (utcnow.strftime( 

209 "%Y-%m-%dT%H:%M:%S.%f"), "UTC timestamp") 

210 

211 # RECURSIVELY CREATE MISSING DIRECTORIES 

212 if not os.path.exists(outDir): 

213 os.makedirs(outDir) 

214 # CONVERT CCDData TO FITS HDU (INCLUDING HEADER) AND SAVE WITH PRE TAG 

215 # PREPENDED TO FILENAME 

216 basename = os.path.basename(filepath) 

217 filenameNoExtension = os.path.splitext(basename)[0] 

218 extension = os.path.splitext(basename)[1] 

219 filePath = outDir + "/" + \ 

220 filenameNoExtension + "_pre" + extension 

221 

222 # SAVE TO DISK 

223 self._write( 

224 frame=frame, 

225 filedir=outDir, 

226 filename=filenameNoExtension + "_pre" + extension, 

227 overwrite=True 

228 ) 

229 

230 self.log.debug('completed the ``_prepare_single_frame`` method') 

231 return filePath 

232 

233 def _absolute_path( 

234 self, 

235 path): 

236 """*convert paths from home directories to absolute paths* 

237 

238 **Key Arguments:** 

239 - ``path`` -- path possibly relative to home directory 

240 

241 **Return:** 

242 - ``absolutePath`` -- absolute path 

243 

244 **Usage** 

245 

246 ```python 

247 myPath = self._absolute_path(myPath) 

248 ``` 

249 """ 

250 self.log.debug('starting the ``_absolute_path`` method') 

251 

252 from os.path import expanduser 

253 home = expanduser("~") 

254 if path[0] == "~": 

255 path = home + "/" + path[1:] 

256 

257 self.log.debug('completed the ``_absolute_path`` method') 

258 return path.replace("//", "/") 

259 

260 def prepare_frames( 

261 self, 

262 save=False): 

263 """*prepare raw frames by converting pixel data from ADU to electrons and adding mask and uncertainty extensions* 

264 

265 **Key Arguments:** 

266 - ``save`` -- save out the prepared frame to the intermediate products directory. Default False. 

267 

268 **Return:** 

269 - ``preframes`` -- the new image collection containing the prepared frames 

270 

271 **Usage** 

272 

273 Usually called within a recipe class once the input frames have been selected and verified (see `soxs_mbias` code for example): 

274 

275 ```python 

276 self.inputFrames = self.prepare_frames( 

277 save=self.settings["save-intermediate-products"]) 

278 ``` 

279 """ 

280 self.log.debug('starting the ``prepare_frames`` method') 

281 

282 kw = self.kw 

283 

284 filepaths = self.inputFrames.files_filtered(include_path=True) 

285 

286 frameCount = len(filepaths) 

287 print("\n# PREPARING %(frameCount)s RAW FRAMES - TRIMMING OVERSCAN, CONVERTING TO ELECTRON COUNTS, GENERATING UNCERTAINTY MAPS AND APPENDING DEFAULT BAD-PIXEL MASK" % locals()) 

288 preframes = [] 

289 preframes[:] = [self._prepare_single_frame( 

290 frame=frame, save=save) for frame in filepaths] 

291 sof = set_of_files( 

292 log=self.log, 

293 settings=self.settings, 

294 inputFrames=preframes, 

295 verbose=self.verbose 

296 ) 

297 preframes, supplementaryInput = sof.get() 

298 preframes.sort([kw('MJDOBS').lower()]) 

299 

300 print("# PREPARED FRAMES - SUMMARY") 

301 print(preframes.summary) 

302 

303 self.log.debug('completed the ``prepare_frames`` method') 

304 return preframes 

305 

306 def _verify_input_frames_basics( 

307 self): 

308 """*the basic verifications that needs done for all recipes* 

309 

310 **Return:** 

311 - None 

312 

313 If the fits files conform to required input for the recipe everything will pass silently, otherwise an exception shall be raised. 

314 """ 

315 self.log.debug('starting the ``_verify_input_frames_basics`` method') 

316 

317 kw = self.kw 

318 

319 # CHECK WE ACTUALLY HAVE IMAGES 

320 if not len(self.inputFrames.files_filtered(include_path=True)): 

321 raise FileNotFoundError( 

322 "No image frames where passed to the recipe") 

323 

324 arm = self.inputFrames.values( 

325 keyword=kw("SEQ_ARM").lower(), unique=True) 

326 # MIXED INPUT ARMS ARE BAD 

327 if len(arm) > 1: 

328 arms = " and ".join(arms) 

329 print(self.inputFrames.summary) 

330 raise TypeError( 

331 "Input frames are a mix of %(imageTypes)s" % locals()) 

332 else: 

333 self.arm = arm[0] 

334 

335 # CREATE DETECTOR LOOKUP DICTIONARY - SOME VALUES CAN BE OVERWRITTEN 

336 # WITH WHAT IS FOUND HERE IN FITS HEADERS 

337 self.detectorParams = detector_lookup( 

338 log=self.log, 

339 settings=self.settings 

340 ).get(self.arm) 

341 

342 # MIXED BINNING IS BAD 

343 cdelt1 = self.inputFrames.values( 

344 keyword=kw("CDELT1").lower(), unique=True) 

345 cdelt2 = self.inputFrames.values( 

346 keyword=kw("CDELT2").lower(), unique=True) 

347 

348 if len(cdelt1) > 1 or len(cdelt2) > 1: 

349 print(self.inputFrames.summary) 

350 raise TypeError( 

351 "Input frames are a mix of binnings" % locals()) 

352 

353 if cdelt1[0] and cdelt2[0]: 

354 self.detectorParams["binning"] = [int(cdelt2[0]), int(cdelt1[0])] 

355 

356 # MIXED READOUT SPEEDS IS BAD 

357 readSpeed = self.inputFrames.values( 

358 keyword=kw("DET_READ_SPEED").lower(), unique=True) 

359 if len(readSpeed) > 1: 

360 print(self.inputFrames.summary) 

361 raise TypeError( 

362 "Input frames are a mix of readout speeds" % locals()) 

363 

364 # MIXED GAIN SPEEDS IS BAD 

365 # HIERARCH ESO DET OUT1 CONAD - Electrons/ADU 

366 # CONAD IS REALLY GAIN AND HAS UNIT OF Electrons/ADU 

367 gain = self.inputFrames.values( 

368 keyword=kw("CONAD").lower(), unique=True) 

369 if len(gain) > 1: 

370 print(self.inputFrames.summary) 

371 raise TypeError( 

372 "Input frames are a mix of gain" % locals()) 

373 if gain[0]: 

374 # UVB & VIS 

375 self.detectorParams["gain"] = gain[0] * u.electron / u.adu 

376 else: 

377 # NIR 

378 self.detectorParams["gain"] = self.detectorParams[ 

379 "gain"] * u.electron / u.adu 

380 

381 # HIERARCH ESO DET OUT1 RON - Readout noise in electrons 

382 ron = self.inputFrames.values( 

383 keyword=kw("RON").lower(), unique=True) 

384 

385 # MIXED NOISE 

386 if len(ron) > 1: 

387 print(self.inputFrames.summary) 

388 raise TypeError("Input frames are a mix of readnoise" % locals()) 

389 if ron[0]: 

390 # UVB & VIS 

391 self.detectorParams["ron"] = ron[0] * u.electron 

392 else: 

393 # NIR 

394 self.detectorParams["ron"] = self.detectorParams[ 

395 "ron"] * u.electron 

396 

397 self.log.debug('completed the ``_verify_input_frames_basics`` method') 

398 return None 

399 

400 def clean_up( 

401 self): 

402 """*remove intermediate files once recipe is complete* 

403 

404 **Usage** 

405 

406 ```python 

407 recipe.clean_up() 

408 ``` 

409 """ 

410 self.log.debug('starting the ``clean_up`` method') 

411 

412 outDir = self.intermediateRootPath + "/tmp" 

413 

414 try: 

415 shutil.rmtree(outDir) 

416 except: 

417 pass 

418 

419 self.log.debug('completed the ``clean_up`` method') 

420 return None 

421 

422 def xsh2soxs( 

423 self, 

424 frame): 

425 """*perform some massaging of the xshooter data so it more closely resembles soxs data - this function can be removed once code is production ready* 

426 

427 **Key Arguments:** 

428 - ``frame`` -- the CCDDate frame to manipulate 

429 

430 **Return:** 

431 - ``frame`` -- the manipulated soxspipe-ready frame 

432 

433 **Usage:** 

434 

435 ```python 

436 frame = self.xsh2soxs(frame) 

437 ``` 

438 """ 

439 self.log.debug('starting the ``xsh2soxs`` method') 

440 

441 kw = self.kw 

442 dp = self.detectorParams 

443 

444 # NP ROTATION OF ARRAYS IS IN COUNTER-CLOCKWISE DIRECTION 

445 rotationIndex = int(dp["clockwise-rotation"] / 90.) 

446 

447 if self.settings["instrument"] == "xsh" and rotationIndex > 0: 

448 frame.data = np.rot90(frame.data, rotationIndex) 

449 

450 self.log.debug('completed the ``xsh2soxs`` method') 

451 return frame 

452 

453 def _trim_frame( 

454 self, 

455 frame): 

456 """*return frame with pre-scan and overscan regions removed* 

457 

458 **Key Arguments:** 

459 - ``frame`` -- the CCDData frame to be trimmed 

460 """ 

461 self.log.debug('starting the ``_trim_frame`` method') 

462 

463 kw = self.kw 

464 arm = self.arm 

465 dp = self.detectorParams 

466 

467 rs, re, cs, ce = dp["science-pixels"]["rows"]["start"], dp["science-pixels"]["rows"][ 

468 "end"], dp["science-pixels"]["columns"]["start"], dp["science-pixels"]["columns"]["end"] 

469 

470 binning = dp["binning"] 

471 if binning[0] > 1: 

472 rs = int(rs / binning[0]) 

473 re = int(re / binning[0]) 

474 if binning[1] > 1: 

475 cs = int(cs / binning[0]) 

476 ce = int(ce / binning[0]) 

477 

478 trimmed_frame = ccdproc.trim_image(frame[rs: re, cs: ce]) 

479 

480 self.log.debug('completed the ``_trim_frame`` method') 

481 return trimmed_frame 

482 

483 def _write( 

484 self, 

485 frame, 

486 filedir, 

487 filename=False, 

488 overwrite=True): 

489 """*write frame to disk at the specified location* 

490 

491 **Key Arguments:** 

492 - ``frame`` -- the frame to save to disk (CCDData object) 

493 - ``filedir`` -- the location to save the frame 

494 - ``filename`` -- the filename to save the file as. Default: **False** (standardised filename generated in code) 

495 - ``overwrite`` -- if a file exists at the filepath then choose to overwrite the file. Default: True 

496 

497 **Usage:** 

498 

499 Use within a recipe like so: 

500 

501 ```python 

502 self._write(frame, filePath) 

503 ``` 

504 """ 

505 self.log.debug('starting the ``write`` method') 

506 

507 if not filename: 

508 

509 filename = filenamer( 

510 log=self.log, 

511 frame=frame, 

512 settings=self.settings 

513 ) 

514 

515 filepath = filedir + "/" + filename 

516 

517 HDUList = frame.to_hdu( 

518 hdu_mask='QUAL', hdu_uncertainty='ERRS', hdu_flags=None) 

519 HDUList[0].name = "FLUX" 

520 HDUList.writeto(filepath, output_verify='exception', 

521 overwrite=overwrite, checksum=True) 

522 

523 filepath = os.path.abspath(filepath) 

524 

525 self.log.debug('completed the ``write`` method') 

526 return filepath 

527 

528 def clip_and_stack( 

529 self, 

530 frames, 

531 recipe): 

532 """*mean combine input frames after sigma-clipping outlying pixels using a median value with median absolute deviation (mad) as the deviation function* 

533 

534 **Key Arguments:** 

535 - ``frames`` -- an ImageFileCollection of the frames to stack or a list of CCDData objects 

536 - ``recipe`` -- the name of recipe needed to read the correct settings from the yaml files 

537 

538 **Return:** 

539 - ``combined_frame`` -- the combined master frame (with updated bad-pixel and uncertainty maps) 

540 

541 **Usage:** 

542 

543 This snippet can be used within the recipe code to combine individual (using bias frames as an example): 

544 

545 ```python 

546 combined_bias_mean = self.clip_and_stack( 

547 frames=self.inputFrames, recipe="soxs_mbias") 

548 ``` 

549 

550 --- 

551 

552 ```eval_rst 

553 .. todo:: 

554 

555 - revisit error propagation when combining frames: https://github.com/thespacedoctor/soxspipe/issues/42 

556 ``` 

557 """ 

558 self.log.debug('starting the ``clip_and_stack`` method') 

559 

560 arm = self.arm 

561 kw = self.kw 

562 dp = self.detectorParams 

563 imageType = self.imageType 

564 

565 # ALLOW FOR UNDERSCORE AND HYPHENS 

566 recipe = recipe.replace("soxs_", "soxs-") 

567 

568 # UNPACK SETTINGS 

569 clipping_lower_sigma = self.settings[ 

570 recipe]["clipping-lower-simga"] 

571 clipping_upper_sigma = self.settings[ 

572 recipe]["clipping-upper-simga"] 

573 clipping_iteration_count = self.settings[ 

574 recipe]["clipping-iteration-count"] 

575 

576 # LIST OF CCDDATA OBJECTS NEEDED BY COMBINER OBJECT 

577 if not isinstance(frames, list): 

578 ccds = [c for c in frames.ccds(ccd_kwargs={"hdu_uncertainty": 'ERRS', 

579 "hdu_mask": 'QUAL', "hdu_flags": 'FLAGS', "key_uncertainty_type": 'UTYPE'})] 

580 else: 

581 ccds = frames 

582 

583 imageType = ccds[0].header[kw("DPR_TYPE").lower()].replace(",", "-") 

584 imageTech = ccds[0].header[kw("DPR_TECH").lower()].replace(",", "-") 

585 imageCat = ccds[0].header[kw("DPR_CATG").lower()].replace(",", "-") 

586 

587 print(f"\n# MEAN COMBINING {len(ccds)} {arm} {imageCat} {imageTech} {imageType} FRAMES") 

588 

589 # COMBINE MASKS AND THEN RESET 

590 combinedMask = ccds[0].mask 

591 for c in ccds: 

592 combinedMask = c.mask | combinedMask 

593 c.mask[:, :] = False 

594 

595 # COMBINER OBJECT WILL FIRST GENERATE MASKS FOR INDIVIDUAL IMAGES VIA 

596 # CLIPPING AND THEN COMBINE THE IMAGES WITH THE METHOD SELECTED. PIXEL 

597 # MASKED IN ALL INDIVIDUAL IMAGES ARE MASK IN THE FINAL COMBINED IMAGE 

598 combiner = Combiner(ccds) 

599 

600 # print(f"\n# SIGMA-CLIPPING PIXEL WITH OUTLYING VALUES IN INDIVIDUAL {imageType} FRAMES") 

601 # PRINT SOME INFO FOR USER 

602 badCount = combinedMask.sum() 

603 totalPixels = np.size(combinedMask) 

604 percent = (float(badCount) / float(totalPixels)) * 100. 

605 print(f"\tThe basic bad-pixel mask for the {arm} detector {imageType} frames contains {badCount} pixels ({percent:0.2}% of all pixels)") 

606 

607 # GENERATE A MASK FOR EACH OF THE INDIVIDUAL INPUT FRAMES - USING 

608 # MEDIAN WITH MEDIAN ABSOLUTE DEVIATION (MAD) AS THE DEVIATION FUNCTION 

609 old_n_masked = -1 

610 # THIS IS THE SUM OF BAD-PIXELS IN ALL INDIVIDUAL FRAME MASKS 

611 new_n_masked = combiner.data_arr.mask.sum() 

612 iteration = 1 

613 while (new_n_masked > old_n_masked and iteration <= clipping_iteration_count): 

614 combiner.sigma_clipping( 

615 low_thresh=clipping_lower_sigma, high_thresh=clipping_upper_sigma, func=np.ma.median, dev_func=mad_std) 

616 old_n_masked = new_n_masked 

617 # RECOUNT BAD-PIXELS NOW CLIPPING HAS RUN 

618 new_n_masked = combiner.data_arr.mask.sum() 

619 diff = new_n_masked - old_n_masked 

620 extra = "" 

621 if diff == 0: 

622 extra = " - we're done" 

623 if self.verbose: 

624 print("\tClipping iteration %(iteration)s finds %(diff)s more rogue pixels in the set of input frames%(extra)s" % locals()) 

625 iteration += 1 

626 

627 # GENERATE THE COMBINED MEDIAN 

628 # print("\n# MEAN COMBINING FRAMES - WITH UPDATED BAD-PIXEL MASKS") 

629 combined_frame = combiner.average_combine() 

630 

631 # RECOMBINE THE COMBINED MASK FROM ABOVE 

632 combined_frame.mask = combined_frame.mask | combinedMask 

633 

634 # MASSIVE FUDGE - NEED TO CORRECTLY WRITE THE HEADER FOR COMBINED 

635 # IMAGES 

636 combined_frame.header = ccds[0].header 

637 try: 

638 combined_frame.wcs = ccds[0].wcs 

639 except: 

640 pass 

641 combined_frame.header[ 

642 kw("DPR_CATG")] = "MASTER_%(imageType)s_%(arm)s" % locals() 

643 

644 # CALCULATE NEW PIXELS ADDED TO MASK 

645 newBadCount = combined_frame.mask.sum() 

646 diff = newBadCount - badCount 

647 print("\t%(diff)s new pixels made it into the combined bad-pixel map" % locals()) 

648 

649 self.log.debug('completed the ``clip_and_stack`` method') 

650 return combined_frame 

651 

652 def subtract_calibrations( 

653 self, 

654 inputFrame, 

655 master_bias=False, 

656 dark=False): 

657 """*subtract calibration frames from an input frame* 

658 

659 **Key Arguments:** 

660 - ``inputFrame`` -- the input frame to have calibrations subtracted. CCDData object. 

661 - ``master_bias`` -- the master bias frame to be subtracted. CCDData object. Default *False*. 

662 - ``dark`` -- a dark frame to be subtracted. CCDData object. Default *False*. 

663 

664 **Return:** 

665 - ``calibration_subtracted_frame`` -- the input frame with the calibration frame(s) subtracted. CCDData object. 

666 

667 **Usage:** 

668 

669 Within a soxspipe recipe use `subtract_calibrations` like so: 

670 

671 ```python 

672 myCalibratedFrame = self.subtract_calibrations( 

673 inputFrame=inputFrameCCDObject, master_bias=masterBiasCCDObject, dark=darkCCDObject) 

674 ``` 

675 

676 --- 

677 

678 ```eval_rst 

679 .. todo:: 

680 

681 - code needs written to scale dark frame to exposure time of science/calibration frame 

682 ``` 

683 """ 

684 self.log.debug('starting the ``subtract_calibrations`` method') 

685 

686 arm = self.arm 

687 kw = self.kw 

688 dp = self.detectorParams 

689 

690 if master_bias == None: 

691 master_bias = False 

692 if dark == None: 

693 dark = False 

694 

695 # VERIFY DATA IS IN ORDER 

696 if master_bias == False and dark == False: 

697 raise TypeError( 

698 "subtract_calibrations method needs a master-bias frame and/or a dark frame to subtract") 

699 if master_bias == False and dark.header[kw("EXPTIME")] != inputFrame.header[kw("EXPTIME")]: 

700 raise AttributeError( 

701 "Dark and science/calibration frame have differing exposure-times. A master-bias frame needs to be supplied to scale the dark frame to same exposure time as input science/calibration frame") 

702 if master_bias != False and dark != False and dark.header[kw("EXPTIME")] != inputFrame.header[kw("EXPTIME")]: 

703 raise AttributeError( 

704 "CODE NEEDS WRITTEN HERE TO SCALE DARK FRAME TO EXPOSURE TIME OF SCIENCE/CALIBRATION FRAME") 

705 

706 # DARK WITH MATCHING EXPOSURE TIME 

707 tolerence = 0.5 

708 if dark != False and (int(dark.header[kw("EXPTIME")]) < int(inputFrame.header[kw("EXPTIME")]) + tolerence) and (int(dark.header[kw("EXPTIME")]) > int(inputFrame.header[kw("EXPTIME")]) - tolerence): 

709 calibration_subtracted_frame = inputFrame.subtract(dark) 

710 calibration_subtracted_frame.header = inputFrame.header 

711 try: 

712 calibration_subtracted_frame.wcs = inputFrame.wcs 

713 except: 

714 pass 

715 

716 # ONLY A MASTER BIAS FRAME, NO DARK 

717 if dark == False and master_bias != False: 

718 calibration_subtracted_frame = inputFrame.subtract(master_bias) 

719 calibration_subtracted_frame.header = inputFrame.header 

720 try: 

721 calibration_subtracted_frame.wcs = inputFrame.wcs 

722 except: 

723 pass 

724 

725 self.log.debug('completed the ``subtract_calibrations`` method') 

726 return calibration_subtracted_frame 

727 

728 def report_output( 

729 self, 

730 rformat="stdout"): 

731 """*a method to report QC values alongside imtermediate and final products* 

732 

733 **Key Arguments:** 

734 - ``rformat`` -- the format to outout reports as. Default *stdout*. [stdout|....] 

735 

736 **Return:** 

737 - None 

738 

739 **Usage:** 

740 

741 ```python 

742 usage code  

743 ``` 

744 

745 --- 

746 

747 ```eval_rst 

748 .. todo:: 

749 

750 - add usage info 

751 - create a sublime snippet for usage 

752 - write a command-line tool for this method 

753 - update package tutorial with command-line tool info if needed 

754 ``` 

755 """ 

756 self.log.debug('starting the ``report_output`` method') 

757 

758 if not self.verbose: 

759 # REMOVE COLUMN FROM DATA FRAME 

760 self.products.drop(columns=['file_path'], inplace=True) 

761 

762 if rformat == "stdout": 

763 print(tabulate(self.qc, headers='keys', tablefmt='psql')) 

764 print(tabulate(self.products, headers='keys', tablefmt='psql')) 

765 

766 self.log.debug('completed the ``report_output`` method') 

767 return None 

768 

769 # use the tab-trigger below for new method 

770 # xt-class-method