import os
import copy
import pprint
import importlib
import yaml
import types
import cerberus
import collections
from cis_interface.drivers import import_all_drivers
from cis_interface.communication import import_all_comms
_schema_fname = os.path.abspath(os.path.join(
os.path.dirname(__file__), '.cis_schema.yml'))
_schema = None
_registry = {}
_registry_complete = False
[docs]def register_component(component_class):
r"""Decorator for registering a class as a yaml component."""
global _registry
yaml_typ = component_class._schema_type
if yaml_typ not in _registry:
_registry[yaml_typ] = []
if component_class not in _registry[yaml_typ]:
_registry[yaml_typ].append(component_class)
return component_class
[docs]def inherit_schema(orig, key, value, **kwargs):
r"""Create an inherited schema, adding new value to accepted ones for
dependencies.
Args:
orig (dict): Schema that will be inherited.
key (str): Field that other fields are dependent on.
value (str): New value for key that dependent fields should accept.
**kwargs: Additional keyword arguments will be added to the schema
with dependency on the provided key/value pair.
Returns:
dict: New schema.
"""
if isinstance(value, list):
value_list = value
else:
value_list = [value]
out = copy.deepcopy(orig)
for k in out.keys():
if ('dependencies' in out[k]) and (key in out[k]['dependencies']):
if not isinstance(out[k]['dependencies'][key], list): # pragma: debug
out[k]['dependencies'][key] = [out[k]['dependencies'][key]]
out[k]['dependencies'][key] += value_list
for k, v in kwargs.items():
out[k] = v
out[k].setdefault('dependencies', {})
out[k]['dependencies'].setdefault(key, [])
out[k]['dependencies'][key] += value_list
return out
[docs]def init_registry():
r"""Initialize the registries and schema."""
global _registry_complete
if not _registry_complete:
import_all_drivers()
import_all_comms()
_registry_complete = True
[docs]def clear_schema():
r"""Clear global schema."""
global _schema
_schema = None
[docs]def init_schema(fname=None):
r"""Initialize global schema."""
global _schema
if _schema is None:
_schema = load_schema(fname)
[docs]def create_schema():
r"""Create a new schema from the registry."""
global _registry, _registry_complete
init_registry()
x = SchemaRegistry(_registry)
return x
[docs]def load_schema(fname=None):
r"""Return the cis_interface schema for YAML options.
Args:
fname (str, optional): Full path to the file that the schema should be
loaded from. If the file dosn't exist, it is created. Defaults to
_schema_fname.
Returns:
dict: cis_interface YAML options.
"""
if fname is None:
fname = _schema_fname
if not os.path.isfile(fname):
x = create_schema()
x.save(fname)
return SchemaRegistry.from_file(fname)
[docs]def get_schema(fname=None):
r"""Return the cis_interface schema for YAML options.
Args:
fname (str, optional): Full path to the file that the schema should be
loaded from. If the file dosn't exist, it is created. Defaults to
_schema_fname.
Returns:
dict: cis_interface YAML options.
"""
global _schema
if fname is None:
init_schema()
out = _schema
else:
out = load_schema(fname)
return out
function_type = cerberus.TypeDefinition('function', types.FunctionType, ())
[docs]def str_to_function(value):
r"""Convert a string to a function.
Args:
value (str, list): String or list of strings, specifying function(s).
The format should be "<package.module>:<function>" so that
<function> can be imported from <package>.
Returns:
func: Callable function.
"""
if isinstance(value, list):
single = False
vlist = value
else:
single = True
vlist = [value]
out = []
for s in vlist:
if isinstance(s, str):
pkg_mod = s.split(':')
if len(pkg_mod) == 2:
mod, fun = pkg_mod[:]
else:
raise ValueError("Could not parse function string: %s" % s)
modobj = importlib.import_module(mod)
if not hasattr(modobj, fun):
raise AttributeError("Module %s has no funciton %s" % (
modobj, fun))
out.append(getattr(modobj, fun))
elif hasattr(s, '__call__'):
out.append(s)
else:
raise TypeError("Cannot coerce type %s to function" % s)
if single:
out = out[0]
return out
[docs]class SchemaValidator(cerberus.Validator):
r"""Class for validating the schema."""
types_mapping = cerberus.Validator.types_mapping.copy()
types_mapping['function'] = function_type
cis_type_order = ['list', 'string', 'integer', 'boolean', 'function']
def _resolve_rules_set(self, *args, **kwargs):
rules = super(SchemaValidator, self)._resolve_rules_set(*args, **kwargs)
if isinstance(rules, collections.Mapping):
rules = self._add_coerce(rules)
return rules
def _add_coerce(self, rules):
if 'coerce' in rules:
return rules
t = rules.get('type', None)
if isinstance(t, list):
clist = []
for k in self.cis_type_order:
if (k != 'list') and (k in t):
clist.append(k)
if clist:
rules['coerce'] = clist
elif t in self.cis_type_order:
rules['coerce'] = t
return rules
def _normalize_coerce_string(self, value):
if isinstance(value, list):
return [self._normalize_coerce_string(v) for v in value]
elif isinstance(value, dict):
return {k: self._normalize_coerce_string(v) for k, v in value.items()}
else:
return str(value)
def _normalize_coerce_integer(self, value):
return int(value)
def _normalize_coerce_boolean(self, value):
if isinstance(value, str):
return (value.lower() == 'true')
else:
return bool(value)
def _normalize_coerce_list(self, value):
if isinstance(value, str):
return [v.strip() for v in value.split(',')]
elif isinstance(value, list):
return value
else:
raise TypeError("Cannot coerce type %s to list." % type(value))
def _normalize_coerce_function(self, value):
return str_to_function(value)
[docs]class ComponentSchema(dict):
r"""Schema information for one component.
Args:
schema_type (str): The name of the component.
subtype_attr (str, optional): The attribute that should be used to
log subtypes. Defaults to None.
**kwargs: Additional keyword arguments are entries in the component
schema.
"""
def __init__(self, schema_type, subtype_attr=None, **kwargs):
self._schema_type = schema_type
self._subtype_attr = subtype_attr
self._schema_subtypes = {}
super(ComponentSchema, self).__init__(**kwargs)
@property
def class2subtype(self):
r"""dict: Mapping from class to list of subtypes."""
return self._schema_subtypes
@property
def subtype2class(self):
r"""dict: Mapping from subtype to class."""
out = {}
for k, v in self._schema_subtypes.items():
for iv in v:
out[iv] = k
return out
@property
def subtypes(self):
r"""list: All subtypes for this schema type."""
out = []
for v in self._schema_subtypes.values():
out += v
return list(set(out))
@property
def classes(self):
r"""list: All available classes for this schema."""
return [k for k in self._schema_subtypes.keys()]
[docs] def append(self, comp_cls, subtype=None):
r"""Append component class to the schema.
Args:
comp_cls (class): Component class that should be added.
subtype (str, tuple, optional): Key used to identify the subtype
of the component type. Defaults to subtype_attr if one was
provided, otherwise the subtype will not be logged.
"""
assert(comp_cls._schema_type == self._schema_type)
name = comp_cls.__name__
rule = comp_cls._schema
if (subtype is None) and (self._subtype_attr is not None):
subtype = getattr(comp_cls, self._subtype_attr)
if subtype is not None:
if not isinstance(subtype, list):
self._schema_subtypes[name] = [subtype]
else:
self._schema_subtypes[name] = subtype
self.append_rules(rule)
[docs] def append_rules(self, new):
r"""Add rules from new class's schema to this one.
Args:
new (dict): New schema to add.
"""
for k, v in new.items():
if k not in self:
self[k] = v
else:
diff = []
for ik in v.keys():
if (ik not in self[k]) or (v[ik] != self[k][ik]):
diff.append(ik)
if (len(diff) == 0):
pass
elif (len(diff) == 1) and (diff[0] == 'dependencies'):
alldeps = {}
deps = [self[k]['dependencies'], v['dependencies']]
for idep in deps:
for ik, iv in idep.items():
if ik not in alldeps:
alldeps[ik] = []
if isinstance(iv, list):
alldeps[ik] += iv
else: # pragma: debug
alldeps[ik].append(iv)
for ik in alldeps.keys():
alldeps[ik] = list(set(alldeps[ik]))
vcopy = copy.deepcopy(v)
vcopy['dependencies'] = alldeps
self[k].update(**vcopy)
else: # pragma: debug
print('Existing:')
pprint.pprint(self[k])
print('New:')
pprint.pprint(v)
raise ValueError("Cannot merge schemas.")
[docs]class SchemaRegistry(dict):
r"""Registry of schema's for different integration components.
Args:
registry (dict, optional): Dictionary of registered components.
Defaults to None and the registry will be empty.
required (list, optional): Components that are required. Defaults to
['comm', 'file', 'model', 'connection']. Ignored if registry is None.
Raises:
ValueError: If registry is provided and one of the required components
is missing.
"""
_component_attr = ['_schema_subtypes', '_subtype_attr']
_subtype_attr = {'model': '_language', 'comm': '_commtype',
'file': '_filetype'}
def __init__(self, registry=None, required=None):
comp = {}
if registry is not None:
if required is None:
required = ['comm', 'file', 'model', 'connection']
for k in required:
if k not in registry:
raise ValueError("Component %s required." % k)
for k in registry.keys():
if k not in comp:
isubtype_attr = self._subtype_attr.get(k, None)
comp[k] = ComponentSchema(k, subtype_attr=isubtype_attr)
for x in registry[k]:
subtype = None
if k == 'connection':
subtype = (x._icomm_type, x._ocomm_type, x.direction())
comp[k].append(x, subtype=subtype)
SchemaValidator(comp[k])
# Add lists of required properties
comp['file']['filetype']['allowed'] = comp['file'].subtypes
comp['model']['language']['allowed'] = comp['model'].subtypes
comp['model']['inputs'] = {'type': 'list', 'required': False,
'schema': {'type': 'dict',
'schema': comp['comm']}}
comp['model']['outputs'] = {'type': 'list', 'required': False,
'schema': {'type': 'dict',
'schema': comp['comm']}}
comp['connection']['input_file']['schema'] = comp['file']
comp['connection']['output_file']['schema'] = comp['file']
# Make sure final versions are valid schemas
for x in comp.values():
SchemaValidator(x)
super(SchemaRegistry, self).__init__(**comp)
[docs] @classmethod
def from_file(cls, fname):
r"""Create a SchemaRegistry from a file.
Args:
fname (str): Full path to the file the schema should be loaded from.
"""
out = cls()
out.load(fname)
return out
[docs] def load(self, fname):
r"""Load schema from a file.
Args:
fname (str): Full path to the file the schema should be loaded from.
"""
with open(fname, 'r') as f:
contents = f.read()
schema = yaml.load(contents, Loader=SchemaLoader)
for k, v in schema.items():
is_attr = False
for iattr in self._component_attr:
if k.endswith(iattr):
is_attr = True
break
if is_attr:
continue
self[k] = ComponentSchema(k, **v)
for iattr in self._component_attr:
kattr = k + iattr
if kattr in schema:
setattr(self[k], iattr, schema[kattr])
[docs] def save(self, fname):
r"""Save the schema to a file.
Args:
fname (str): Full path to the file the schema should be saved to.
schema (dict): cis_interface YAML options.
"""
with open(fname, 'w') as f:
yaml.dump(self, f, default_flow_style=False,
Dumper=SchemaDumper)
@property
def class2language(self):
r"""dict: Mapping from ModelDriver class to programming language."""
return self['model'].class2subtype
@property
def language2class(self):
r"""dict: Mapping from programming language to ModelDriver class."""
return self['model'].subtype2class
@property
def class2filetype(self):
r"""dict: Mapping from communication class to filetype."""
return self['file'].class2subtype
@property
def filetype2class(self):
r"""dict: Mapping from filetype to communication class."""
return self['file'].subtype2class
@property
def class2conntype(self):
r"""dict: Mapping from connection class to comm classes & direction."""
return self['connection'].class2subtype
@property
def conntype2class(self):
r"""dict: Mapping from comm classes & direction to connection class."""
return self['connection'].subtype2class
@property
def validator(self):
r"""Compose complete schema for parsing yaml."""
out = {'models': {'type': 'list',
'schema': {'type': 'dict',
'schema': self['model']}},
'connections': {'type': 'list',
'schema': {'type': 'dict',
'schema': self['connection']}}}
return SchemaValidator(out)
[docs]class SchemaLoader(yaml.SafeLoader):
r"""SafeLoader for schema that includes tuples."""
[docs] def construct_python_tuple(self, node):
return tuple(self.construct_sequence(node))
SchemaLoader.add_constructor('tag:yaml.org,2002:python/tuple',
SchemaLoader.construct_python_tuple)
[docs]class SchemaDumper(yaml.Dumper):
r"""SafeDumper for schema that includes tuples and Schema classes."""
[docs] def represent_python_tuple(self, data, **kwargs):
return self.represent_sequence('tag:yaml.org,2002:python/tuple',
list(data), **kwargs)
[docs] def represent_ComponentSchema(self, data):
out = dict(**data)
return self.represent_data(out)
[docs] def represent_SchemaRegistry(self, data):
out = dict(**data)
for k in data.keys():
for iattr in data._component_attr:
if getattr(data[k], iattr, None):
out[k + iattr] = getattr(data[k], iattr)
return self.represent_data(out)
SchemaDumper.add_representer(tuple, SchemaDumper.represent_python_tuple)
SchemaDumper.add_representer(ComponentSchema,
SchemaDumper.represent_ComponentSchema)
SchemaDumper.add_representer(SchemaRegistry,
SchemaDumper.represent_SchemaRegistry)