# coding: utf-8
from .block import Block
from .._global import OptionalModule
from typing import Dict, List, Union, Tuple, Any
from time import time, sleep
from subprocess import Popen, PIPE, STDOUT, TimeoutExpired
from threading import Thread
from queue import Queue, Empty
from ast import literal_eval
from pickle import loads, dumps, UnpicklingError
from socket import timeout, gaierror
try:
import paho.mqtt.client as mqtt
except (ModuleNotFoundError, ImportError):
mqtt = OptionalModule("paho.mqtt.client")
[docs]class Client_server(Block):
"""Block for exchanging data on a local network using the MQTT protocol.
This block can send data to an MQTT broker, receive data from this broker by
subscribing to its topics, and also launch the Mosquitto broker.
"""
[docs] def __init__(self,
broker: bool = False,
address: Any = 'localhost',
port: int = 1148,
init_output: Dict[str, Any] = None,
topics: List[Union[str, Tuple[str, str]]] = None,
cmd_labels: List[Union[str, Tuple[str, str]]] = None,
labels_to_send:
List[Union[str, Tuple[str, str]]] = None) -> None:
"""Checks arguments validity and sets the instance attributes.
Args:
broker (:obj:`bool`, optional): If :obj:`True`, starts the Mosquitto
broker during the prepare loop and stops it during the finish loop. If
Mosquitto is not installed a :exc:`FileNotFoundError` is raised.
address (optional): The network address on which the MQTT broker is
running.
port (:obj:`int`, optional): A network port on which the MQTT broker is
listening.
init_output (:obj:`dict`, optional): A :obj:`dict` containing for each
label in ``topics`` the first value to be sent in the output link. Must
be given if ``topics`` is not :obj:`None`.
topics (:obj:`list`, optional): A :obj:`list` of :obj:`str` and/or
:obj:`tuple` of :obj:`str`. Each string corresponds to the name of a
crappy label to be received from the broker. Each element of the list
is considered to be the name of an MQTT topic, to which the client
subscribes. After a message has been received on that topic, the block
returns for each label in the topic (i.e. each string in the tuple) the
corresponding data from the message.
cmd_labels (:obj:`list`, optional): A :obj:`list` of :obj:`str` and/or
:obj:`tuple` of :obj:`str`. Each string corresponds to the name of a
crappy label to send to the broker. Each element of the list is
considered to be the name of an MQTT topic, in which the client
publishes. Grouping labels in a same topic (i.e. strings in a same
tuple) allows to keep the synchronization between signals coming from a
same block, as they will be published together in a same message. This
is mostly useful for sending a signal along with its timeframe.
labels_to_send (:obj:`list`, optional): A :obj:`list` of :obj:`str`
and/or :obj:`tuple` of :obj:`str`. Allows to rename the labels before
publishing data. The structure of ``labels_to_send`` should be the
exact same as ``cmd_labels``, with each label in ``labels_to_send``
replacing the corresponding one in ``cmd_labels``. This is especially
useful for transferring several signals along with their timestamps, as
the label ``'t(s)'`` should not appear more than once in the topics
list of the receiving block.
Note:
- ``broker``:
In order for the block to run, an MQTT broker must be running at the
specified address on the specified port. If not, an
:exc:`ConnectionRefusedError` is raised. The broker can be started and
stopped manually by the user independently of the execution of crappy.
It also doesn't need to be Mosquitto, any other MQTT broker can be
used.
- ``topics``:
The presence of the same label in multiple topics will most likely lead
to a data loss.
- ``cmd_labels``:
It is not possible to group signals coming from different blocks in a
same topic.
- ``labels_to_send``:
Differences in the structure of ``labels_to_send`` and ``cmd_labels``
will not always raise an error, but may lead to a data loss.
- **Single-value tuples**:
Single-value tuples can be shortened as strings.
::
topics=[('cmd1',), ('cmd2',)]
cmd_labels=[('cmd1',), ('cmd2',)]
labels_to_send=[('cmd1',), ('cmd2',)]
is equivalent to
::
topics=['cmd1', 'cmd2']
cmd_labels=['cmd1', 'cmd2']
labels_to_send=['cmd1', 'cmd2']
Examples:
- ``topics``:
If
::
topics=[('t1', 'cmd1'), 'sign']
the client will subscribe to the topics
::
('t1', 'cmd1')
('sign',)
The block will return data associated with the labels
::
't1', 'cmd1'
'sign'
- ``cmd_labels``:
If
::
cmd_labels=[('t1', 'cmd1'), 'sign']
the client will publish data in the form of
::
[[t1_0, cmd1_0], [t1_1, cmd1_1], ...]
[[sign_0], [sign_1], ...]
in the topics
::
('t1', 'cmd1')
('sign',)
- ``labels_to_send``:
If
::
cmd_labels=[('t(s)', 'cmd'), 'sign']
labels_to_send=[('t1', 'cmd1'), 'sign']
the data from labels
::
't(s)', 'cmd'
will be published in the topic
::
('t1', 'cmd1')
and the data from label
::
'sign'
in the topic
::
('sign',)
"""
Block.__init__(self)
self.niceness = -10
self.stop_mosquitto = False
self.broker = broker
self.address = address
self.port = port
self.init_output = init_output
self.topics = topics
self.cmd_labels = cmd_labels
self.labels_to_send = labels_to_send
self.reader = Thread(target=self._output_reader)
self.client = mqtt.Client(str(time()))
self.client.on_connect = self._on_connect
self.client.on_message = self._on_message
self.client.reconnect_delay_set(max_delay=10)
if self.topics is None and self.cmd_labels is None:
print("[Client_server] WARNING: client-server block is neither an input "
"nor an output !")
if self.topics is not None:
self.last_out_val = init_output
[docs] def prepare(self) -> None:
"""Reorganizes the labels lists, starts the broker and connects to it."""
# Preparing for receiving data
if self.topics is not None:
assert self.outputs, "topics are specified but there's no output link "
self.topics = [(topic,) if not isinstance(topic, tuple) else topic for
topic in self.topics]
# The buffer for received data is a dictionary of queues
self.buffer_output = {topic: Queue() for topic in self.topics}
# Placing the initial values in the queues
assert self.init_output, "init_output values should be provided"
for topic in self.buffer_output:
try:
self.buffer_output[topic].put_nowait([self.init_output[label]
for label in topic])
except KeyError:
print("init_output values should be provided for each label")
raise
# Preparing for publishing data
if self.cmd_labels is not None:
assert self.inputs, "cmd_labels are specified but there's no input link "
self.cmd_labels = [(topic,) if not isinstance(topic, tuple) else topic
for topic in self.cmd_labels]
if self.labels_to_send is not None:
for i, topic in enumerate(self.labels_to_send):
if not isinstance(topic, tuple):
self.labels_to_send[i] = (self.labels_to_send[i],)
# Preparing to rename labels to send using a dictionary
assert len(self.labels_to_send) == len(
self.cmd_labels), "Either a label_to_send should be given for " \
"every cmd_label, or none should be given "
self.labels_to_send = {cmd_label: label_to_send for
cmd_label, label_to_send in
zip(self.cmd_labels, self.labels_to_send)}
# Starting the broker
if self.broker:
self._launch_mosquitto()
self.reader.start()
sleep(5)
print('[Client_server] Waiting for Mosquitto to start')
sleep(5)
# Connecting to the broker
try_count = 15
while True:
try:
self.client.connect(self.address, port=self.port, keepalive=10)
break
except timeout:
print("[Client_server] Impossible to reach the given address, "
"aborting")
raise
except gaierror:
print("[Client_server] Invalid address given, please check the "
"spelling")
raise
except ConnectionRefusedError:
try_count -= 1
if try_count == 0:
print("[Client_server] Connection refused, the broker may not be "
"running or you may not have the rights to connect")
raise
sleep(1)
self.client.loop_start()
[docs] def loop(self) -> None:
"""Receives data from the broker and/or sends data to the broker.
The received data is then sent to the crappy blocks connected to this one.
"""
"""Loop for receiving data
Each queue in the buffer is checked once: if not empty then the first list
of data is popped. This data is then associated to the corresponding
labels in dict_out. dict_out is finally returned if not empty. All the
labels should be returned at each loop iteration, so a buffer stores the
last value for each label and returns it if no other value was received."""
if self.topics is not None:
dict_out = {}
for topic in self.buffer_output:
if not self.buffer_output[topic].empty():
try:
data_list = self.buffer_output[topic].get_nowait()
for label, data in zip(topic, data_list):
dict_out[label] = data
except Empty:
pass
# Updating the last_out_val buffer, and completing dict_out before
# sending data if necessary
if dict_out:
for topic in self.buffer_output:
for label in topic:
if label not in dict_out:
dict_out[label] = self.last_out_val[label]
else:
self.last_out_val[label] = dict_out[label]
self.send(dict_out)
"""Loop for sending data
Data is first received as a list of dictionaries. For each topic, trying to
find a dictionary containing all the corresponding labels. Once this
dictionary has been found, its data is published as a list of list of
values."""
if self.cmd_labels is not None:
received_data = [link.recv_chunk() if link.poll() else {} for link
in self.inputs]
for topic in self.cmd_labels:
for dic in received_data:
if all(label in dic for label in topic):
if self.labels_to_send is not None:
self.client.publish(
str(self.labels_to_send[topic]),
payload=dumps([dic[label] for label in topic]), qos=0)
else:
self.client.publish(str(topic),
payload=dumps(
[dic[label] for label in
topic]),
qos=0)
break
[docs] def finish(self) -> None:
"""Disconnects from the broker and stops it."""
# Disconnecting from the broker
self.client.loop_stop()
self.client.disconnect()
# Stopping the broker
if self.broker:
try:
self.proc.terminate()
self.proc.wait(timeout=15)
print('[Client_server] Mosquitto terminated with return code',
self.proc.returncode)
self.stop_mosquitto = True
self.reader.join()
except TimeoutExpired:
print('[Client_server] Subprocess did not terminate in time')
self.proc.kill()
def _launch_mosquitto(self) -> None:
"""Starts Mosquitto in a subprocess."""
try:
self.proc = Popen(['mosquitto', '-p', str(self.port)],
stdout=PIPE,
stderr=STDOUT)
except FileNotFoundError:
print("[Client_server] Mosquitto is not installed !")
raise
def _output_reader(self) -> None:
"""Reads the output strings from Mosquitto's subprocess."""
while not self.stop_mosquitto:
for line in iter(self.proc.stdout.readline, b''):
print('[Mosquitto] {0}'.format(line.decode('utf-8')), end='')
if 'Error: Address already in use' in line.decode('utf-8'):
print('Mosquitto is already running on this port')
sleep(0.1)
def _on_message(self, _, __, message) -> None:
"""Buffers the received data.
The received message consists in a list of lists of values. Data is placed
in the right buffer according to the topic, in the form of lists of values.
"""
try:
for data_points in zip(*loads(message.payload)):
self.buffer_output[literal_eval(message.topic)].put_nowait(
list(data_points))
except UnpicklingError:
print("[Client_server] Warning ! Message raised UnpicklingError, "
"ignoring it")
def _on_connect(self, _, __, ___, rc: Any) -> None:
"""Automatically subscribes to the topics when connecting to the broker."""
print("[Client_server] Connected with result code " + str(rc))
# Subscribing on connect, so that it automatically resubscribes when
# reconnecting after a connection loss
if self.topics is not None:
for topic in self.topics:
self.client.subscribe(topic=str(topic), qos=0)
print("[Client_server] Subscribed to", topic)
self.client.loop_start()