Source code for cityiq.api

"""

API objects are the primary way to get access to assets and events. :py:class:`CityIq` is the top level access
object. The API offers access to Locations, Events and Events.

Typically, you will construct :py:class:`CityIq` from a :py:class:`Config`. If a configuration is not specific,
the system will look for the file in default locations. You can also override individual configuration parameters
with keyword arguments to the constructor.

Metadata Access
---------------

Metadata, for both locations and assets, can be fetched with property accessors. The bounding box for the queries can
be set in the configuration, or on the  :py:class:`CityIq` constructor.

The asset metadata properties are:

- :py:attr:`CityIq.assets` : All assets
- :py:attr:`CityIq.nodes` : Nodes, the parents for other assets on a pole
- :py:attr:`CityIq.cameras` : All assets
- :py:attr:`CityIq.em_sensors` : ?
- :py:attr:`CityIq.env_sensors` : Environmental sensors

The location metadata properties are:

- :py:attr:`CityIq.locations` : All locations
- :py:attr:`CityIq.walkways` :
- :py:attr:`CityIq.parking_zones` :
- :py:attr:`CityIq.traffic_lane` :

Events can be fetched with :py:func:`cityiq.api.CityIq.events`

Each of these acessor properties or functions returns a generator that generates objects of a specific type,
one base class for each of Locations, Assets or Events:

- :py:class:`Asset`
- :py:class:`Location`
- :py:class:`Event`

.. code-block:: python

    bbox = '32.718987:-117.174244,32.707356:-117.154850'

    c = CityIq(bbox=bbox) # Use default config, override bbox

    # Get Locations
    locations = list(c.locations)

    # Get the assets at this location:
    for location in locations:
        do_something_with(location.assets



"""

import datetime
import logging
import threading
import time

import pytz
import requests

from .config import Config
from .exceptions import CityIqError, ConfigurationError

logger = logging.getLogger(__name__)


[docs]class CityIqObject(object): def __init__(self, client, data): self.client = client self.data = data def __getattr__(self, item): try: return self.data[item] except KeyError: raise AttributeError(item) @property def geometry(self): """Return a Shapely polygon for the coordinates""" from shapely.geometry import Point, Polygon, LineString def numify(e): a, b = e.split(':') return float(b), float(a) if not hasattr(self, 'coordinatesType') or self.coordinatesType == 'GEO': vertices = [numify(e) for e in self.coordinates.split(',')] if len(vertices) == 1: return Point(vertices) elif len(vertices) == 2: return LineString(vertices) else: return Polygon(vertices) def __str__(self): return "<{}: {}>".format(type(self).__name__, self.data)
[docs]class Asset(CityIqObject): detail_url_suffix = '/v2/metadata/assets/{}' locations_url_suffix = '/v2/metadata/assets/{}/locations' children_url_suffix = '/v2/metadata/assets/{}/subAssets' row_header = 'assetUid assetType parentAssetUid mediaType events geometry'.split() # observed values for the assetType field types = ['NODE', 'EM_SENSOR', 'MIC', 'ENV_SENSOR', 'CAMERA'] # Map asset types to subclasses dclass_map = {'NODE': 'NodeAsset', 'CAMERA': 'CameraAsset', 'EM_SENSOR': 'EmSensorAsset', 'XENV_SENSOR': 'EnvSensorAsset', 'MIC': 'MicSensorAsset' } def __new__(cls, *args, **kwargs): dclass_name = Asset.dclass_map.get(args[1]['assetType'], Asset.__name__) # probably fragile dclass = globals()[dclass_name] obj = super(CityIqObject, cls).__new__(dclass) return obj @property def lat(self): return self.coordinates.split(':')[0] @property def lon(self): return self.coordinates.split(':')[1] @property def detail(self): """Asset details""" url = self.client.config.metadata_url + self.detail_url_suffix.format(self.assetUid) r = self.client.http_get(url) return Asset(self.client, r.json()) @property def parent(self): url = self.client.config.metadata_url + self.detail_url_suffix.format(self.parentAssetUid) r = self.client.http_get(url) return Asset(self.client, r.json()) @property def locations(self): """Assets at this location""" url = self.client.config.metadata_url + self.locations_url_suffix.format(self.assetUid) r = self.client.http_get(url) for e in r.json()['locations']: yield Location(self.client, e) @property def children(self): """Sub assets of this asset""" url = self.client.config.metadata_url + self.children_url_suffix.format(self.assetUid) r = self.client.http_get(url) for e in r.json()['assets']: yield Asset(self.client, e) @property def row(self): """Return most important fields in a row format""" from operator import attrgetter def evt_list(events): return ','.join(sorted(set(events or []))) ag = attrgetter(*Asset.row_header[:-2]) return ag(self) + (evt_list(self.eventTypes), self.geometry)
[docs]class NodeAsset(Asset): pass
[docs]class CameraAsset(Asset): pass
[docs]class EnvSensorAsset(Asset): pass
[docs]class EmSensorAsset(Asset): pass
[docs]class MicSensorAsset(Asset): pass
[docs]class Location(CityIqObject): detail_url_suffix = '/v2/metadata/locations/{}' assets_url_suffix = '/v2/metadata/locations/{}/assets' events_url_suffix = '/v2/locations/{locationUid}/events?eventType={eventType}&startTime={startts}&' \ 'endTime={endts}' row_header = 'locationUid locationType parentLocationUid geometry'.split() # observed values for the assetType field types = ['WALKWAY', 'TRAFFIC_LANE', 'PARKING_ZONE'] # Map asset types to subclasses dclass_map = { 'WALKWAY': 'WalkwayLocation', 'TRAFFIC_LANE': 'TrafficLaneLocation', 'PARKING_ZONE': 'ParkingZoneLocation' } def __new__(cls, *args, **kwargs): dclass_name = Location.dclass_map.get(args[1]['locationType'], Location.__name__) # probably fragile dclass = globals()[dclass_name] obj = super(CityIqObject, cls).__new__(dclass) return obj @property def detail(self): url = self.client.config.metadata_url + self.detail_url_suffix.format(self.locationUid) r = self.client.http_get(url) return r.json() @property def assets(self): """Assets at this location""" url = self.client.config.metadata_url + self.assets_url_suffix.format(self.locationUid) r = self.client.http_get(url) for e in r.json()['assets']: yield Asset(self.client, e)
[docs] def events(self, eventType, start_time, end_time=None, span=None, ago=None): start_time, end_time = se_time(self, start_time, end_time, ago, span) url = self.client.config.metadata_url + self.format( locationUid=self.locationUid, eventType=eventType, startts=start_time.timestamp() * 1000, endts=end_time.timestamp() * 100 ) r = self.client.http_get(url) return r.json()
@property def row(self): """Return most important fields in a row format""" from operator import attrgetter ag = attrgetter(*Location.row_header[:-1]) return ag(self) + (self.geometry,)
[docs]class WalkwayLocation(Location): pass
[docs]class TrafficLaneLocation(Location): pass
[docs]class ParkingZoneLocation(Location): pass
[docs]class Event(CityIqObject): types = ['PKIN', 'PKOUT', 'PEDEVT', 'TFEVT', 'TEMPERATURE', 'PRESSURE', 'ORIENTATION', 'METROLOGY', 'HUMIDITY', 'ENERGY_TIMESERIES', 'ENERGY_ALERT']
[docs]class EventWorker(threading.Thread): """Thread worker for websociet events""" def __init__(self, client, events, queue) -> None: super().__init__() self.client = client self.events = events self.queue = queue
[docs] def run(self) -> None: super().run() import websocket import json # websocket.enableTrace(True) # events = ["TFEVT"] if 'TFEVT' in self.events: zone = self.client.config.traffic_zone, else: zone = self.client.config.parking_zone headers = { 'Authorization': 'Bearer ' + self.client.token, 'Predix-Zone-Id': zone, 'Cache-Control': 'no-cache' } def on_message(ws, message): self.queue.put(message) def on_close(ws): self.queue.put(None) def on_open(ws): msg = { 'bbox': self.client.config.bbox, 'eventTypes': self.events } ws.send(json.dumps(msg)) ws = websocket.WebSocketApp(self.client.config.websocket_url + '/events', header=headers, on_message=on_message, on_close=on_close) ws.on_open = on_open try: ws.run_forever() except KeyboardInterrupt: self.queue.put(None)
[docs]def current_time(): '''Return the epoch time in miliseconds''' return int(round(time.time() * 1000, 0))
[docs]def time_ago(days=None, hours=None, minutes=None, seconds=None): t = time.time() if seconds: t -= seconds if minutes: t -= minutes * 60 if hours: t -= hours * 60 * 60 if days: t -= days * 60 * 60 * 26 return int(round(t * 1000, 0))
[docs]def se_time(start_time=None, end_time=None, ago=None, span=None): if ago and span: raise CityIqError("Specify either age or span, but not both") if ago and not end_time: raise CityIqError("If age is specified, end_time must be also") if span and not start_time: raise CityIqError("If span is specified, start_time must be also") if not end_time: if start_time and span: end_time = (start_time + span) * 1000 else: end_time = current_time() else: end_time = int(end_time * 1000) if not start_time: if ago: start_time = end_time - (ago * 1000) else: start_time = time_ago(minutes=15) else: start_time = int(start_time * 1000) return int(start_time), int(end_time)
[docs]class CityIq(object): assets_search_suffix = '/v2/metadata/assets/search' locations_search_suffix = '/v2/metadata/locations/search' events_url = '/v2/locations/events?bbox={bbox}&locationType={location_type}&eventType={event_type}&startTime={' \ 'start_time}&endTime={end_time}&pageSize=100' def __init__(self, config=None, **kwargs): if config: self.config = config else: self.config = Config(**kwargs) self._token = None self.tz = pytz.timezone(self.config.timezone) @property def token(self): from .token import get_cached_token, get_token if not self._token: if self.config.cache_dir: self._token = get_cached_token(self.config.cache_dir, self.config.uaa_url, self.config.client_id, self.config.secret) else: self._token = get_token(self.config.uaa_url, self.config.client_id, self.config.secret) return self._token
[docs] def http_get(self, url, zone=None, params=None, *args, **kwargs): zone = zone if zone else self.config.default_zone if params: # Not using the requests param argument b/c it will urlencode, and these query # parameters can't be url encoded. url = url + '?' + "&".join("{}={}".format(k, v) for k, v in params.items()) headers = { 'Authorization': 'Bearer ' + self.token } if zone: headers['Predix-Zone-Id'] = zone r = requests.get(url, headers=headers, *args, **kwargs) r.raise_for_status() return r
def _get_page(self, url, page, zone, bbox, query): params = { 'page': page, 'size': 5000, 'bbox': bbox, } if query: params['q'] = "{}:{}".format(*query) r = self.http_get(url, zone, params) return r.json()
[docs] def get_pages(self, url, query=None, zone=None, bbox=None): zone = zone if zone else self.config.zone if not zone: ConfigurationError("Must specify a zone, either in the get_assets call, or in the config") bbox = bbox if bbox else self.config.bbox if not zone: ConfigurationError("Must specify a bounding box (bbox) , either in the get_assets call, or in the config") page = 0 index = 0 while True: r = self._get_page(url, page, zone, bbox, query) content = r['content'] for e in content: e['index'] = index e['page'] = page e['total'] = r['totalElements'] yield e index += 1 if r['last']: break page += 1
[docs] def get_assets(self, device_type=None, zone=None, bbox=None): # A space ' ' is interpreted as querying for all records, while a blank '' is # an error. query = ('assetType', device_type if device_type is not None else ' ') for e in self.get_pages(self.config.metadata_url + self.assets_search_suffix, query=query, zone=zone, bbox=bbox): yield Asset(self, e)
@property def assets(self): """Return all system assets""" yield from self.get_assets(' ') @property def asset_dataframe(self): """Return assets in row form in a pandas Dataframe""" from pandas import DataFrame return DataFrame([e.row for e in self.assets], columns=Asset.row_header) @property def nodes(self): """Return all nodes""" yield from self.get_assets('NODE') @property def cameras(self): """Return camera assets""" yield from self.get_assets('CAMERA') @property def env_sensors(self): """Return environmental sensors""" yield from self.get_assets('ENV_SENSOR') @property def em_sensors(self): """Return some other kind of sensor. Electro-magnetic? """ yield from self.get_assets('EM_SENSOR') @property def mics(self): """Return microphone assets""" yield from self.get_assets('MIC')
[docs] def get_locations(self, location_type=None, zone=None, bbox=None): # A space ' ' is interpreted as querying for all records, while a blank '' is # an error. query = ('locationType', location_type if location_type is not None else ' ') for e in self.get_pages(self.config.metadata_url + self.locations_search_suffix, query=query, zone=zone, bbox=bbox): yield Location(self, e)
@property def locations(self): return self.get_locations(' ') @property def locations_dataframe(self): from pandas import DataFrame return DataFrame([e.row for e in self.locations], columns=Location.row_header) @property def walkways(self): return self.get_locations('WALKWAY') @property def traffic_lanes(self): return self.get_locations('TRAFFIC_LANE') @property def parking_zones(self): return self.get_locations('PARKING_ZONE') def _event_params(self, start_time, end_time, event_type, bbox=None): bbox = bbox or self.config.bbox if event_type in ('PKIN', 'PKOUT'): zone = self.config.parking_zone location_type = 'PARKING_ZONE' elif event_type == 'PEDEVT': location_type = 'WALKWAY' zone = self.config.pedestrian_zone elif event_type == 'TFEVT': location_type = 'TRAFFIC_LANE' zone = self.config.traffic_zone else: raise CityIqError("Unknown event type: '{}' ".format(event_type)) params = { 'bbox': bbox, 'locationType': location_type, 'eventType': event_type, 'startTime': start_time, 'endTime': end_time, 'pageSize': 10000 } return zone, params def _events_page(self, start_time, end_time, bbox, event_type): events_url = '/v2/locations/events' zone, params = self._event_params(start_time, end_time, event_type, bbox=bbox) r = self.http_get(self.config.event_url + events_url, params=params, zone=zone) return r.json()
[docs] def events(self, start_time=None, end_time=None, age=None, span=None, bbox=None, event_type=' '): """ :param start_time: :param end_time: :param age: if start_time is not specified, the start time in terms of seconds before the end time :return: """ start_time, end_time = se_time(start_time, end_time, age, span) logger.debug("Starting events start_time={} end_time={}, event_type={}".format( datetime.datetime.utcfromtimestamp(start_time / 1000), datetime.datetime.utcfromtimestamp(end_time / 1000), event_type )) page = 0 while True: data = self._events_page(start_time, end_time, bbox, event_type) meta = data['metaData'] if len(data['content']) == 0: break min_ts = 2 ** 64 max_ts = 0 for i, e in enumerate(data['content']): e['page'] = page e['page_item'] = i min_ts = min(min_ts, int(e['timestamp'])) max_ts = max(max_ts, int(e['timestamp'])) yield e # If the end time is not the end time we asked for then # run another request to get the rest of the events. start_time_r, end_time_r = int(meta['startTs']), int(meta['endTs']) assert (start_time_r == start_time) # 3*60== three minutes, the approximate settling time of events. ( Or so I'm told ... ) if end_time_r == end_time or min_ts == max_ts or (end_time - max_ts) / 1000 < 3 * 60: break assert end_time > end_time_r start_time = max_ts # end_time_r page += 1 logger.debug("Ending events")
[docs] def events_async(self, events=["PKIN", "PKOUT"]): """Use the websocket to get events. The websocket is run in a thread, and this function is a generator that returns results. """ from queue import Queue import json q = Queue() w = EventWorker(self, events, q) w.start() while True: item = q.get() if item is None: break yield json.loads(item) q.task_done()
@property def total_bounds(self): """Return a bounding box for the system from all of the assets. This will be affected by the bbox set in the config, so it should usually be smaller than the one in the config""" assets = list(self.get_assets()) lats = [a.lat for a in assets] lons = [a.lon for a in assets] return "{}:{},{}:{}".format(max(lats), max(lons), min(lats), min(lons))