Source code for crappy.blocks.hdf_saver

# coding: utf-8

from os import path, makedirs
import numpy as np

from .._global import OptionalModule
try:
  import tables
except ModuleNotFoundError:
  tables = OptionalModule("tables", "HDFSaver needs the tables module to "
      "write hdf files.")

from .block import Block


[docs]class Hdf_saver(Block): """To save data efficiently in a hdf5 file. This block is is meant to save data coming by arrays at a high rate (`>1kHz`). It uses the module :mod:`tables`. Important: Do not forget to specify the type of data to be saved (see ``atom`` parameter) to avoid casting the data into another type, as this could result in data loss or inefficient saving. """
[docs] def __init__(self, filename, node='table', expected_rows=10**8, atom=None, label='stream', metadata=None): """Sets the args and initializes the parent class. Args: filename (:obj:`str`): Path of the file where the data should be saved. node (:obj:`str`, optional): The name of the node where the data will be saved. expected_rows (:obj:`int`, optional): The number of expected rows in the file. It is used to optimize the dumping. atom (optional): This represent the type of data to be stored in the table. It can be given as a :class:`tables.Atom` instance, as a :class:`numpy.array` or as a :obj:`str`. label (:obj:`str`, optional): The key of the :obj:`dict` that contains the array to save. metadata (:obj:`dict`, optional): A :obj:`dict` containing additional info to save in the `hdf5` file. """ Block.__init__(self) self.filename = filename self.node = node self.expected_rows = expected_rows self.atom = tables.Int16Atom() if atom is None else atom self.label = label self.metadata = {} if metadata is None else metadata if not isinstance(self.atom, tables.Atom): self.atom = tables.Atom.from_dtype(np.dtype(self.atom))
[docs] def prepare(self): assert self.inputs, "No input connected to the hdf_saver!" assert len(self.inputs) == 1,\ "Cannot link more than one block to a hdf_saver!" d = path.dirname(self.filename) if not path.exists(d): # Create the folder if it does not exist try: makedirs(d) except OSError: assert path.exists(d), "Error creating " + d if path.exists(self.filename): # If the file already exists, append a number to the name print("[hdf_saver] WARNING!", self.filename, "already exists !") name, ext = path.splitext(self.filename) i = 1 while path.exists(name + "_%05d" % i + ext): i += 1 self.filename = name+"_%05d" % i + ext print("[hdf_saver] Using", self.filename, "instead!") self.hfile = tables.open_file(self.filename, "w") for name, value in self.metadata.items(): self.hfile.create_array(self.hfile.root, name, value)
[docs] def begin(self): data = self.inputs[0].recv_chunk() w = data[self.label][0].shape[1] self.array = self.hfile.create_earray( self.hfile.root, self.node, self.atom, (0, w), expectedrows=self.expected_rows) for d in data[self.label]: self.array.append(d)
def loop(self): data = self.inputs[0].recv_chunk() for d in data[self.label]: self.array.append(d)
[docs] def finish(self): self.hfile.close()