Source code for crappy.blocks.block

# coding: utf-8

from sys import platform
from multiprocessing import Process, Pipe
from time import sleep, time, localtime, strftime
from weakref import WeakSet
from pickle import UnpicklingError

from .._global import CrappyStop

import subprocess


[docs]def renice(pid, niceness): """Function to renice a process. Warning: Only works in Linux. Note: The user must be allowed to use ``sudo`` to renice with a negative value. May thus ask for a password for negative values. """ if niceness < 0: subprocess.call(['sudo', 'renice', str(niceness), '-p', str(pid)]) else: subprocess.call(['renice', str(niceness), '-p', str(pid)])
[docs]class Block(Process): """This represent a Crappy block, it must be parent of all the blocks.""" instances = WeakSet() def __init__(self): Process.__init__(self) # Block.instances.append(self) self.outputs = [] self.inputs = [] # This pipe allows to send 2 essential signals: # pipe1->pipe2 is to start the main function and set t0 # pipe2->pipe1 to set process status to the parent self.pipe1, self.pipe2 = Pipe() self._status = "idle" self.in_process = False # To know if we are in the process or not self.niceness = 0 def __new__(cls, *args, **kwargs): instance = super().__new__(cls) Block.instances.add(instance) return instance @classmethod def reset(cls): cls.instances = WeakSet()
[docs] def run(self): self.in_process = True # we are in the process self.status = "initializing" try: self.prepare() self.status = "ready" # Wait for parent to tell me to start the main self.t0 = self.pipe2.recv() if self.t0 < 0: try: self.finish() except Exception: pass return self.status = "running" self.begin() self._MB_last_t = time() self._MB_last_FPS = self._MB_last_t self._MB_loops = 0 self.main() self.status = "done" except CrappyStop: print("[%r] Encountered CrappyStop Exception, terminating" % self) self.status = "done" self.stop_all() except KeyboardInterrupt: print("[%r] Keyboard interrupt received" % self) except Exception as e: print("[%r] Exception caught:" % self, e) try: self.finish() except Exception: pass self.status = "error" sleep(1) # To let downstream blocks process the data and avoid loss self.stop_all() raise self.finish() self.status = "done"
@classmethod def get_status(cls): return [x.status for x in cls.instances]
[docs] @classmethod def all_are(cls, s): """Returns :obj:`True` only if all processes status are `s`.""" lst = cls.get_status() return len(set(lst)) == 1 and s in lst
[docs] @classmethod def renice_all(cls, high_prio=True, **_): """Will renice all the blocks processes according to ``block.niceness`` value. Note: If ``high_prio`` is :obj:`False`, blocks with a negative niceness value will be ignored. This is to avoid asking for the ``sudo`` password since only root can lower the niceness of processes. """ if "win" in platform: # Not supported on Windows yet return for b in cls.instances: if b.niceness < 0 and high_prio or b.niceness > 0: print("[renice] Renicing", b.pid, "to", b.niceness) renice(b.pid, b.niceness)
[docs] @classmethod def prepare_all(cls, verbose=True): """Starts all the blocks processes (``block.prepare``), but not the main loop.""" if verbose: def vprint(*args): print("[prepare]", *args) else: def vprint(*_): return vprint("Starting the blocks...") for instance in cls.instances: vprint("Starting", instance) instance.start() vprint("Started, PID:", instance.pid) vprint("All processes are started.")
@classmethod def launch_all(cls, t0=None, verbose=True, bg=False): if verbose: def vprint(*args): print("[launch]", *args) else: def vprint(*_): return if not cls.all_are('ready'): vprint("Waiting for all blocks to be ready...") while not cls.all_are('ready'): sleep(.1) if not all([i in ['ready', 'initializing', 'idle'] for i in cls.get_status()]): print("Crappy failed to start!") for i in cls.instances: if i.status in ['ready', 'initializing']: i.launch(-1) cls.stop_all() return # raise RuntimeError("Crappy failed to start!") vprint("All blocks ready, let's go !") if not t0: t0 = time() vprint("Setting t0 to", strftime("%d %b %Y, %H:%M:%S", localtime(t0))) for instance in cls.instances: instance.launch(t0) t1 = time() vprint("All blocks loop started. It took", (t1 - t0) * 1000, "ms") if bg: return try: # Keep running lst = cls.get_status() while not ("done" in lst or "error" in lst): lst = cls.get_status() sleep(1) except KeyboardInterrupt: print("Main process got keyboard interrupt!") cls.stop_all() # It will automatically propagate to the blocks processes if not cls.all_are('running'): print('Waiting for all processes to finish') t = time() while 'running' in cls.get_status() and time() - t < 3: sleep(.1) if cls.all_are('done'): print("Crappy terminated gracefully") else: print("Crappy terminated, blocks status:") for b in cls.instances: print(b, b.status) @classmethod def start_all(cls, t0=None, verbose=True, bg=False, high_prio=False): cls.prepare_all(verbose) if high_prio and any([b.niceness < 0 for b in cls.instances]): print("[start] High prio: root permission needed to renice") cls.renice_all(high_prio, verbose=verbose) cls.launch_all(t0, verbose, bg)
[docs] @classmethod def stop_all(cls, verbose=True): """Stops all the blocks (``crappy.stop``).""" if verbose: def vprint(*args): print("[stop]", *args) else: def vprint(*_): return vprint("Stopping the blocks...") for instance in cls.instances: if instance.status == 'running': vprint("Stopping", instance) instance.stop() vprint("All blocks are stopped.")
[docs] def begin(self): """If :meth:`main` is not overridden, this method will be called first, before entering the main loop.""" pass
[docs] def finish(self): """If :meth:`main` is not overridden, this method will be called upon exit or after a crash.""" pass
def loop(self): raise NotImplementedError('You must override loop or main in' + str(self))
[docs] def main(self): """This is where you define the main loop of the block. Important: If not overridden, will raise an error. """ while not self.pipe2.poll(): self.loop() self.handle_freq() print("[%r] Got stop signal, interrupting..." % self)
[docs] def handle_freq(self): """For block with a given number of `loops/s` (use ``freq`` attribute to set it).""" self._MB_loops += 1 t = time() if hasattr(self, 'freq') and self.freq: d = self._MB_last_t + 1 / self.freq - t while d > 0: t = time() d = self._MB_last_t + 1 / self.freq - t sleep(max(0, d / 2 - 2e-3)) # Ugly, yet simple and pretty efficient self._MB_last_t = t if hasattr(self, 'verbose') and self.verbose and \ self._MB_last_t - self._MB_last_FPS > 2: print("[%r] loops/s:" % self, self._MB_loops / (self._MB_last_t - self._MB_last_FPS)) self._MB_loops = 0 self._MB_last_FPS = self._MB_last_t
[docs] def launch(self, t0): """To start the :meth:`main` method, will call :meth:`Process.start` if needed. Once the process is started, calling launch will set the starting time and actually start the main method. Args: t0 (:obj:`float`): Time to set as starting time of the block (mandatory, in seconds after epoch). """ if self.status == "idle": print(self, ": Called launch on unprepared process!") self.start() self.pipe1.send(t0) # asking to start main in the process
@property def status(self): """Returns the status of the block, from the process itself or the parent. It can be: - `"idle"`: Block not started yet. - `"initializing"`: :meth:`start` was called and :meth:`prepare` is not over yet. - `"ready"`: :meth:`prepare` is over, waiting to start :meth:`main` by calling :meth:`launch`. - `"running"`: :meth:`main` is running. - `"done"`: :meth:`main` is over. - `"error"`: An error occurred and the block stopped. """ if not self.in_process: while self.pipe1.poll(): try: self._status = self.pipe1.recv() except (EOFError, UnpicklingError): if self._status == 'running': self._status = 'done' # If another process tries to get the status if 'win' not in platform: self.pipe2.send(self._status) # Somehow the previous line makes crappy hang on Windows, no idea why # It is not critical, but it means that process status can now only # be read from the process itself and ONE other process. # Luckily, only the parent (the main process) needs the status for now return self._status @status.setter def status(self, s): assert self.in_process, "Cannot set status from outside of the process!" self.pipe2.send(s) self._status = s
[docs] def prepare(self): """This will be run when creating the process, but before the actual start. The first code to be run in the new process, will only be called once and before the actual start of the main launch of the blocks. It can remain empty and do nothing. """ pass
[docs] def send(self, data): """To send the data to all blocks downstream. Send has 2 ways to operate. You can either build the :obj:`dict` yourself, or you can define ``self.labels`` (usually time first) and call send with a :obj:`list`. It will then map them to the :obj:`dict`. Note: ONLY :obj:`dict` can go through links. """ if isinstance(data, dict): pass elif isinstance(data, list): data = dict(zip(self.labels, data)) elif data == 'stop': pass else: raise IOError("Trying to send a " + str(type(data)) + " in a link!") for o in self.outputs: o.send(data)
[docs] def recv_all(self): """Receives new data from all the inputs (not as chunks). It will simply call :meth:`Pipe.recv` on all non empty links and return a single :obj:`dict`. Important: If the same label comes from multiple links, it may be overridden ! """ r = {} for i in self.inputs: if i.poll(): r.update(i.recv()) return r
[docs] def poll(self): """Tells if any input link has pending data Returns True if l.poll() is True for any input link l. """ return any((l.poll for l in self.inputs))
[docs] def recv_all_last(self): """Like recv_all, but drops older data to return only the latest value This method avoids Pipe congestion that can be induced by recv_all when the receiving block is slower than the block upstream """ r = {} for i in self.inputs: while i.poll(): r.update(i.recv()) return r
[docs] def get_last(self, num=None): """To get the latest value of each labels from all inputs. Warning: Unlike the ``recv`` methods of :ref:`Link`, this method is NOT guaranteed to return all the data going through the links! It is meant to get the latest values, discarding all the previous ones (for a displayer for example). Its mode of operation is completely different since it can operate on multiple inputs at once. Args: num (:obj:`list`, optional): A :obj:`list` containing ll the concerned inputs. If :obj:`None` it will operate on all the input links at once. Note: The first call may be blocking until it receives data, all the others will return instantaneously, giving the latest known reading. """ if not hasattr(self, '_last_values'): self._last_values = [None] * len(self.inputs) if num is None: num = range(len(self.inputs)) elif not isinstance(num, list): num = [num] for i in num: if self._last_values[i] is None: self._last_values[i] = self.inputs[i].recv() while self.inputs[i].poll(): self._last_values[i] = self.inputs[i].recv() ret = {} for i in num: ret.update(self._last_values[i]) return ret
[docs] def get_all_last(self, num=None): """To get the data from all links of the block. It is almost the same as :meth:`get_last`, but will return all the data that goes through the links (as :obj:`list`). Also, if multiple links have the same label, only the last link's value will be kept. """ if not hasattr(self, '_all_last_values'): self._all_last_values = [None] * len(self.inputs) if num is None: num = range(len(self.inputs)) elif not isinstance(num, list): num = [num] for i in num: if self._all_last_values[i] is None or self.inputs[i].poll(): self._all_last_values[i] = self.inputs[i].recv_chunk() else: # Dropping all data (already sent on last call) except the last # to make sure the block has at least one value for key in self._all_last_values[i]: self._all_last_values[i][key][:-1] = [] ret = {} for i in num: ret.update(self._all_last_values[i]) return ret
[docs] def recv_all_delay(self, delay=None, poll_delay=.1): """Method to wait for data, but continuously reading all the links to make sure that it does not block. Return: A :obj:`list` where each entry is what would have been returned by :meth:`Link.recv_chunk` on each link. """ if delay is None: delay = 1 / self.freq t = time() r = [{} for _ in self.inputs] last = t while True: sleep(max(0., poll_delay - time() + last)) for lst, d in zip(self.inputs, r): if not lst.poll(): continue new = lst.recv_chunk() for k, v in new.items(): if k in d: d[k].extend(v) else: d[k] = v if time() - t > delay: break return r
[docs] def drop(self, num=None): """Will clear the inputs of the blocks. This method performs like :meth:`get_last`, but returns :obj:`None` instantly. """ if num is None: num = range(len(self.inputs)) elif not isinstance(num, list): num = [num] for n in num: self.inputs[n].clear()
[docs] def add_output(self, o): """Adds a :ref:`Link` as an output.""" self.outputs.append(o)
[docs] def add_input(self, i): """Adds a :ref:`Link` as an input.""" self.inputs.append(i)
def stop(self): if self.status != 'running': return print('[%r] Stopping' % self) self.pipe1.send(0) for i in self.inputs: i.send('stop') for i in range(10): if self.status == "done": break sleep(.05) # if self.status != "done": if self.status not in ['done', 'idle', 'error']: print('[%r] Could not stop properly, terminating' % self) try: self.terminate() except Exception: pass else: print("[%r] Stopped correctly" % self) def __repr__(self): return str(type(self)) + " (" + str(self.pid or "Not running") + ")"