wallaroo.checks

  1import re
  2from datetime import timedelta
  3from typing import Dict, List, Optional
  4
  5from .object import InvalidNameError
  6
  7
  8class Expression:
  9    """
 10    Root base class for all model-checker expressions. Provides
 11    pythonic magic-method sugar for expression definitions.
 12
 13    """
 14
 15    def model_names(self):
 16        raise NotImplemented()
 17
 18    def as_json(self):
 19        raise NotImplemented()
 20
 21    def one_of(self, *values):
 22        return BinOp("in", self, list(values))
 23
 24    def __gt__(self, other):
 25        return BinOp(">", self, Expression.from_py(other))
 26
 27    def __ge__(self, other):
 28        return BinOp(">=", self, Expression.from_py(other))
 29
 30    def __eq__(self, other):
 31        return BinOp("==", self, Expression.from_py(other))
 32
 33    def __lt__(self, other):
 34        return BinOp("<", self, Expression.from_py(other))
 35
 36    def __le__(self, other):
 37        return BinOp("<=", self, Expression.from_py(other))
 38
 39    def __add__(self, other):
 40        return BinOp("+", self, Expression.from_py(other))
 41
 42    def __sub__(self, other):
 43        return BinOp("-", self, Expression.from_py(other))
 44
 45    @classmethod
 46    def from_py(cls, value):
 47        """Creates an :py:Expression: from a given python `value`."""
 48        if isinstance(value, Expression):
 49            return value
 50        elif any(isinstance(value, T) for T in (float, int, list, str, timedelta)):
 51            return Value(value)
 52        else:
 53            raise RuntimeError("invalid expression for bounds checking", value)
 54
 55    def top_json(self) -> Dict[str, object]:
 56        """Creates a top-level expression that can be passed to the model
 57        checker runtime.
 58        """
 59        return {
 60            "root": self.as_json(),
 61            "required_data": [{"name": mn} for mn in self.model_names()],
 62        }
 63
 64
 65class Function(Expression):
 66    def __init__(self, op, args):
 67        self.op = op
 68        self.args = [Expression.from_py(a) for a in args]
 69
 70    def model_names(self):
 71        return [mn for a in self.args for mn in a.model_names()]
 72
 73    def as_json(self):
 74        return {
 75            "node": "fn",
 76            "fn": self.op,
 77            "arguments": [a.as_json() for a in self.args],
 78        }
 79
 80
 81class BinOp(Expression):
 82    def __init__(self, op, left, right):
 83        self.op = op
 84        self.left = Expression.from_py(left)
 85        self.right = Expression.from_py(right)
 86
 87    def model_names(self):
 88        return [mn for a in (self.left, self.right) for mn in a.model_names()]
 89
 90    def as_json(self):
 91        return {
 92            "node": "binop",
 93            "op": self.op,
 94            "left": self.left.as_json(),
 95            "right": self.right.as_json(),
 96        }
 97
 98
 99class Variable(Expression):
100    """Declares a model variable that can be used as an :py:Expression: in the
101    model checker. Variables are identified by their `model_name`, a `position`
102    of either `"input"` or `"output"`, and the tensor `index`.
103    """
104
105    def __init__(self, model_name, position, index):
106        self.model_name = model_name
107        self.position = position
108        self.index = index
109
110    def model_names(self):
111        return [self.model_name]
112
113    def __getitem__(self, index):
114        assert isinstance(index, int)
115        return Variable(self.model_name, self.position, [*self.index, index])
116
117    def as_json(self):
118        return {
119            "node": "variable",
120            "variant_id": {"name": self.model_name},
121            "position": self.position,
122            "key": self.index,
123        }
124
125
126def value_to_node(value):
127    if isinstance(value, int):
128        return {"node": "literal", "integer": value}
129    if isinstance(value, str):
130        return {"node": "literal", "timedelta": value}
131    if isinstance(value, timedelta):
132        seconds = int(value.total_seconds())
133        assert seconds >= 1
134        return {"node": "literal", "timedelta": f"{seconds}s"}
135    if isinstance(value, float):
136        return {"node": "literal", "float": value}
137    if isinstance(value, list):
138        return {"node": "literal", "list": [value_to_node(i) for i in value]}
139    if isinstance(value, Expression):
140        return value.as_json()
141    raise RuntimeError("invalid type", type(value))
142
143
144class Value(Expression):
145    def __init__(self, value):
146        self.value = value
147
148    def model_names(self):
149        return []
150
151    def as_json(self):
152        return value_to_node(self.value)
153
154
155def is_prom_primitive(v):
156    return isinstance(v, int) or isinstance(v, float)
157
158
159class Aggregate:
160    def __init__(
161        self,
162        name: str,
163        promql_agg: str,
164        inner_expression: Expression,
165        duration: timedelta,
166        bucket_size: Optional[timedelta],
167    ):
168        self.name = name
169        self.promql_agg = promql_agg
170        self.inner_expression = inner_expression
171        self.duration = duration
172        self.bucket_size = bucket_size
173
174    def expression(self):
175        args = [self.inner_expression, self.duration]
176        if self.bucket_size is not None:
177            args.append(self.bucket_size)
178        return Function(self.name, args)
179
180    def promql(self, gauge_name):
181        return f"{self.promql_agg} by (pipeline_id) (pipeline_gauge:{gauge_name})"
182
183    def __gt__(self, other):
184        assert is_prom_primitive(other)
185        return Alert(">", self, other)
186
187    def __ge__(self, other):
188        assert is_prom_primitive(other)
189        return Alert(">=", self, other)
190
191    def __eq__(self, other):
192        assert is_prom_primitive(other)
193        return Alert("==", self, other)
194
195    def __lt__(self, other):
196        assert is_prom_primitive(other)
197        return Alert("<", self, other)
198
199    def __le__(self, other):
200        assert is_prom_primitive(other)
201        return Alert("<=", self, other)
202
203
204class Alert:
205    def __init__(self, op, left, right):
206        self.op = op
207        self.left = left
208        self.right = right
209
210    def promql(self, gauge_name):
211        return f"{self.left.promql(gauge_name)} {self.op} {self.right}"
212
213
214class DefinedFunction:
215    def __init__(self, name):
216        self.name = name
217
218    def __call__(self, *args):
219        return Function(self.name, args)
220
221
222class DefinedAggregate:
223    def __init__(self, name: str, promql_agg):
224        self.name = name
225        self.promql_agg = promql_agg
226
227    def __call__(
228        self,
229        expression: Expression,
230        duration: timedelta,
231        bucket_size: Optional[timedelta] = None,
232    ):
233        return Aggregate(self.name, self.promql_agg, expression, duration, bucket_size)
234
235
236class Variables:
237    def __init__(self, model, position):
238        self.model = model
239        self.position = position
240
241    def __getitem__(self, index):
242        assert isinstance(index, int)
243        return Variable(self.model, self.position, [index])
244
245
246def instrument(
247    values: Dict[str, Expression], gauges: List[str], validations: List[str]
248):
249    return {
250        "values": {name: e.top_json() for name, e in values.items()},
251        "gauges": gauges,
252        "validations": [] if len(gauges) > 0 else validations,
253    }
254
255
256# Cache for the validation regex
257_dns_req = None
258
259
260def dns_compliant(name: str):
261    """Returns true if a string is compliant with DNS label name requirement to
262    ensure it can be a part of a full DNS host name
263    """
264    global _dns_req
265    if not _dns_req:
266        # https://en.wikipedia.org/wiki/Domain_Name_System
267        _dns_req = re.compile("^[a-zA-Z][a-zA-Z0-9-]*[a-zA-Z0-9]*$")
268
269    return len(name) < 64 and _dns_req.match(name) is not None and name[-1] != "-"
270
271
272def require_dns_compliance(name: str):
273    """Validates that 'name' complies with DNS naming requirements or raises an exception"""
274    if not dns_compliant(name):
275        raise InvalidNameError(
276            name, "must be DNS-compatible (ASCII alpha-numeric plus dash (-))"
277        )
class Expression:
 9class Expression:
10    """
11    Root base class for all model-checker expressions. Provides
12    pythonic magic-method sugar for expression definitions.
13
14    """
15
16    def model_names(self):
17        raise NotImplemented()
18
19    def as_json(self):
20        raise NotImplemented()
21
22    def one_of(self, *values):
23        return BinOp("in", self, list(values))
24
25    def __gt__(self, other):
26        return BinOp(">", self, Expression.from_py(other))
27
28    def __ge__(self, other):
29        return BinOp(">=", self, Expression.from_py(other))
30
31    def __eq__(self, other):
32        return BinOp("==", self, Expression.from_py(other))
33
34    def __lt__(self, other):
35        return BinOp("<", self, Expression.from_py(other))
36
37    def __le__(self, other):
38        return BinOp("<=", self, Expression.from_py(other))
39
40    def __add__(self, other):
41        return BinOp("+", self, Expression.from_py(other))
42
43    def __sub__(self, other):
44        return BinOp("-", self, Expression.from_py(other))
45
46    @classmethod
47    def from_py(cls, value):
48        """Creates an :py:Expression: from a given python `value`."""
49        if isinstance(value, Expression):
50            return value
51        elif any(isinstance(value, T) for T in (float, int, list, str, timedelta)):
52            return Value(value)
53        else:
54            raise RuntimeError("invalid expression for bounds checking", value)
55
56    def top_json(self) -> Dict[str, object]:
57        """Creates a top-level expression that can be passed to the model
58        checker runtime.
59        """
60        return {
61            "root": self.as_json(),
62            "required_data": [{"name": mn} for mn in self.model_names()],
63        }

Root base class for all model-checker expressions. Provides pythonic magic-method sugar for expression definitions.

Expression()
def model_names(self):
16    def model_names(self):
17        raise NotImplemented()
def as_json(self):
19    def as_json(self):
20        raise NotImplemented()
def one_of(self, *values):
22    def one_of(self, *values):
23        return BinOp("in", self, list(values))
@classmethod
def from_py(cls, value):
46    @classmethod
47    def from_py(cls, value):
48        """Creates an :py:Expression: from a given python `value`."""
49        if isinstance(value, Expression):
50            return value
51        elif any(isinstance(value, T) for T in (float, int, list, str, timedelta)):
52            return Value(value)
53        else:
54            raise RuntimeError("invalid expression for bounds checking", value)

Creates an :py:Expression: from a given python value.

def top_json(self) -> Dict[str, object]:
56    def top_json(self) -> Dict[str, object]:
57        """Creates a top-level expression that can be passed to the model
58        checker runtime.
59        """
60        return {
61            "root": self.as_json(),
62            "required_data": [{"name": mn} for mn in self.model_names()],
63        }

Creates a top-level expression that can be passed to the model checker runtime.

class Function(Expression):
66class Function(Expression):
67    def __init__(self, op, args):
68        self.op = op
69        self.args = [Expression.from_py(a) for a in args]
70
71    def model_names(self):
72        return [mn for a in self.args for mn in a.model_names()]
73
74    def as_json(self):
75        return {
76            "node": "fn",
77            "fn": self.op,
78            "arguments": [a.as_json() for a in self.args],
79        }

Root base class for all model-checker expressions. Provides pythonic magic-method sugar for expression definitions.

Function(op, args)
67    def __init__(self, op, args):
68        self.op = op
69        self.args = [Expression.from_py(a) for a in args]
def model_names(self):
71    def model_names(self):
72        return [mn for a in self.args for mn in a.model_names()]
def as_json(self):
74    def as_json(self):
75        return {
76            "node": "fn",
77            "fn": self.op,
78            "arguments": [a.as_json() for a in self.args],
79        }
Inherited Members
Expression
one_of
from_py
top_json
class BinOp(Expression):
82class BinOp(Expression):
83    def __init__(self, op, left, right):
84        self.op = op
85        self.left = Expression.from_py(left)
86        self.right = Expression.from_py(right)
87
88    def model_names(self):
89        return [mn for a in (self.left, self.right) for mn in a.model_names()]
90
91    def as_json(self):
92        return {
93            "node": "binop",
94            "op": self.op,
95            "left": self.left.as_json(),
96            "right": self.right.as_json(),
97        }

Root base class for all model-checker expressions. Provides pythonic magic-method sugar for expression definitions.

BinOp(op, left, right)
83    def __init__(self, op, left, right):
84        self.op = op
85        self.left = Expression.from_py(left)
86        self.right = Expression.from_py(right)
def model_names(self):
88    def model_names(self):
89        return [mn for a in (self.left, self.right) for mn in a.model_names()]
def as_json(self):
91    def as_json(self):
92        return {
93            "node": "binop",
94            "op": self.op,
95            "left": self.left.as_json(),
96            "right": self.right.as_json(),
97        }
Inherited Members
Expression
one_of
from_py
top_json
class Variable(Expression):
100class Variable(Expression):
101    """Declares a model variable that can be used as an :py:Expression: in the
102    model checker. Variables are identified by their `model_name`, a `position`
103    of either `"input"` or `"output"`, and the tensor `index`.
104    """
105
106    def __init__(self, model_name, position, index):
107        self.model_name = model_name
108        self.position = position
109        self.index = index
110
111    def model_names(self):
112        return [self.model_name]
113
114    def __getitem__(self, index):
115        assert isinstance(index, int)
116        return Variable(self.model_name, self.position, [*self.index, index])
117
118    def as_json(self):
119        return {
120            "node": "variable",
121            "variant_id": {"name": self.model_name},
122            "position": self.position,
123            "key": self.index,
124        }

Declares a model variable that can be used as an :py:Expression: in the model checker. Variables are identified by their model_name, a position of either "input" or "output", and the tensor index.

Variable(model_name, position, index)
106    def __init__(self, model_name, position, index):
107        self.model_name = model_name
108        self.position = position
109        self.index = index
def model_names(self):
111    def model_names(self):
112        return [self.model_name]
def as_json(self):
118    def as_json(self):
119        return {
120            "node": "variable",
121            "variant_id": {"name": self.model_name},
122            "position": self.position,
123            "key": self.index,
124        }
Inherited Members
Expression
one_of
from_py
top_json
def value_to_node(value):
127def value_to_node(value):
128    if isinstance(value, int):
129        return {"node": "literal", "integer": value}
130    if isinstance(value, str):
131        return {"node": "literal", "timedelta": value}
132    if isinstance(value, timedelta):
133        seconds = int(value.total_seconds())
134        assert seconds >= 1
135        return {"node": "literal", "timedelta": f"{seconds}s"}
136    if isinstance(value, float):
137        return {"node": "literal", "float": value}
138    if isinstance(value, list):
139        return {"node": "literal", "list": [value_to_node(i) for i in value]}
140    if isinstance(value, Expression):
141        return value.as_json()
142    raise RuntimeError("invalid type", type(value))
class Value(Expression):
145class Value(Expression):
146    def __init__(self, value):
147        self.value = value
148
149    def model_names(self):
150        return []
151
152    def as_json(self):
153        return value_to_node(self.value)

Root base class for all model-checker expressions. Provides pythonic magic-method sugar for expression definitions.

Value(value)
146    def __init__(self, value):
147        self.value = value
def model_names(self):
149    def model_names(self):
150        return []
def as_json(self):
152    def as_json(self):
153        return value_to_node(self.value)
Inherited Members
Expression
one_of
from_py
top_json
def is_prom_primitive(v):
156def is_prom_primitive(v):
157    return isinstance(v, int) or isinstance(v, float)
class Aggregate:
160class Aggregate:
161    def __init__(
162        self,
163        name: str,
164        promql_agg: str,
165        inner_expression: Expression,
166        duration: timedelta,
167        bucket_size: Optional[timedelta],
168    ):
169        self.name = name
170        self.promql_agg = promql_agg
171        self.inner_expression = inner_expression
172        self.duration = duration
173        self.bucket_size = bucket_size
174
175    def expression(self):
176        args = [self.inner_expression, self.duration]
177        if self.bucket_size is not None:
178            args.append(self.bucket_size)
179        return Function(self.name, args)
180
181    def promql(self, gauge_name):
182        return f"{self.promql_agg} by (pipeline_id) (pipeline_gauge:{gauge_name})"
183
184    def __gt__(self, other):
185        assert is_prom_primitive(other)
186        return Alert(">", self, other)
187
188    def __ge__(self, other):
189        assert is_prom_primitive(other)
190        return Alert(">=", self, other)
191
192    def __eq__(self, other):
193        assert is_prom_primitive(other)
194        return Alert("==", self, other)
195
196    def __lt__(self, other):
197        assert is_prom_primitive(other)
198        return Alert("<", self, other)
199
200    def __le__(self, other):
201        assert is_prom_primitive(other)
202        return Alert("<=", self, other)
Aggregate( name: str, promql_agg: str, inner_expression: wallaroo.checks.Expression, duration: datetime.timedelta, bucket_size: Optional[datetime.timedelta])
161    def __init__(
162        self,
163        name: str,
164        promql_agg: str,
165        inner_expression: Expression,
166        duration: timedelta,
167        bucket_size: Optional[timedelta],
168    ):
169        self.name = name
170        self.promql_agg = promql_agg
171        self.inner_expression = inner_expression
172        self.duration = duration
173        self.bucket_size = bucket_size
def expression(self):
175    def expression(self):
176        args = [self.inner_expression, self.duration]
177        if self.bucket_size is not None:
178            args.append(self.bucket_size)
179        return Function(self.name, args)
def promql(self, gauge_name):
181    def promql(self, gauge_name):
182        return f"{self.promql_agg} by (pipeline_id) (pipeline_gauge:{gauge_name})"
class Alert:
205class Alert:
206    def __init__(self, op, left, right):
207        self.op = op
208        self.left = left
209        self.right = right
210
211    def promql(self, gauge_name):
212        return f"{self.left.promql(gauge_name)} {self.op} {self.right}"
Alert(op, left, right)
206    def __init__(self, op, left, right):
207        self.op = op
208        self.left = left
209        self.right = right
def promql(self, gauge_name):
211    def promql(self, gauge_name):
212        return f"{self.left.promql(gauge_name)} {self.op} {self.right}"
class DefinedFunction:
215class DefinedFunction:
216    def __init__(self, name):
217        self.name = name
218
219    def __call__(self, *args):
220        return Function(self.name, args)
DefinedFunction(name)
216    def __init__(self, name):
217        self.name = name
class DefinedAggregate:
223class DefinedAggregate:
224    def __init__(self, name: str, promql_agg):
225        self.name = name
226        self.promql_agg = promql_agg
227
228    def __call__(
229        self,
230        expression: Expression,
231        duration: timedelta,
232        bucket_size: Optional[timedelta] = None,
233    ):
234        return Aggregate(self.name, self.promql_agg, expression, duration, bucket_size)
DefinedAggregate(name: str, promql_agg)
224    def __init__(self, name: str, promql_agg):
225        self.name = name
226        self.promql_agg = promql_agg
class Variables:
237class Variables:
238    def __init__(self, model, position):
239        self.model = model
240        self.position = position
241
242    def __getitem__(self, index):
243        assert isinstance(index, int)
244        return Variable(self.model, self.position, [index])
Variables(model, position)
238    def __init__(self, model, position):
239        self.model = model
240        self.position = position
def instrument( values: Dict[str, wallaroo.checks.Expression], gauges: List[str], validations: List[str]):
247def instrument(
248    values: Dict[str, Expression], gauges: List[str], validations: List[str]
249):
250    return {
251        "values": {name: e.top_json() for name, e in values.items()},
252        "gauges": gauges,
253        "validations": [] if len(gauges) > 0 else validations,
254    }
def dns_compliant(name: str):
261def dns_compliant(name: str):
262    """Returns true if a string is compliant with DNS label name requirement to
263    ensure it can be a part of a full DNS host name
264    """
265    global _dns_req
266    if not _dns_req:
267        # https://en.wikipedia.org/wiki/Domain_Name_System
268        _dns_req = re.compile("^[a-zA-Z][a-zA-Z0-9-]*[a-zA-Z0-9]*$")
269
270    return len(name) < 64 and _dns_req.match(name) is not None and name[-1] != "-"

Returns true if a string is compliant with DNS label name requirement to ensure it can be a part of a full DNS host name

def require_dns_compliance(name: str):
273def require_dns_compliance(name: str):
274    """Validates that 'name' complies with DNS naming requirements or raises an exception"""
275    if not dns_compliant(name):
276        raise InvalidNameError(
277            name, "must be DNS-compatible (ASCII alpha-numeric plus dash (-))"
278        )

Validates that 'name' complies with DNS naming requirements or raises an exception