wallaroo.standalone_client

  1import json
  2import os
  3import pathlib
  4import sys
  5import time
  6from typing import Any, Dict, List, Optional, Union
  7
  8import requests
  9
 10from wallaroo.inference_result import InferenceResult
 11from wallaroo.model import Model
 12from wallaroo.object import DeploymentError
 13from wallaroo.pipeline_config import PipelineConfig
 14from wallaroo.version import _user_agent
 15
 16
 17class StandaloneClient:
 18    def __init__(
 19        self,
 20        host: str,
 21        port: int,
 22        model: Optional[Model] = None,
 23        pipeline_config: Optional[PipelineConfig] = None,
 24        interactive: Optional[bool] = None,
 25    ):
 26        if (model and pipeline_config) or not (model or pipeline_config):
 27            raise RuntimeError(
 28                "Specify either a model or a pipeline config for inference"
 29            )
 30        self._host = host
 31        self._port = port
 32        self._model = model
 33        self._pipeline_config = pipeline_config
 34
 35        if interactive is not None:
 36            self._interactive = interactive
 37        elif "JUPYTER_SVC_SERVICE_HOST" in os.environ:
 38            self._interactive = True
 39        else:
 40            self._interactive = False
 41
 42    def _url(self) -> str:
 43        if self._model:
 44            return f"http://{self._host}:{self._port}/models/{self._model.name()}"
 45        elif self._pipeline_config:
 46            return f"http://{self._host}:{self._port}/pipelines/{self._pipeline_config.pipeline_name}"
 47        else:
 48            raise RuntimeError(
 49                "Neither a model or pipeline config was specified for inference"
 50            )
 51
 52    def status(self) -> Dict[str, Any]:
 53        """Returns a dict of standalone engine model status.
 54
 55        Example: {'models': [{'class': 'ccfraud', 'name': 'z5', 'status': 'Running'}]}
 56
 57        Example: {'models': [{'class': 'postprocess',
 58                  'name': 'first-postprocess',
 59                  'status': 'Running'},
 60                  {'class': 'noopfloats', 'name': 'noopv1', 'status': 'Running'},
 61                  {'class': 'preprocess', 'name': 'first-preprocess', 'status': 'Running'}]}
 62
 63        Example: {"models": [{"class":"synerror",
 64                              "name":"v1",
 65                              "status":{"Error":"Python compile or runtime error [\"  File \\\"//tmp/.tmpk7mJpI/syntax_error.py\\\", line 1\", \"    PLEASE CRASH HERE\", \"           ^\", \"SyntaxError: invalid syntax\"]"}}]}
 66
 67        """
 68        url = f"http://{self._host}:{self._port}/models"
 69        headers = {"User-Agent": _user_agent}
 70        try:
 71            res = requests.get(url, timeout=3, headers=headers)
 72        except Exception:
 73            raise DeploymentError(f"Error getting status from {url}")
 74        data = None
 75        if res.status_code == 200:
 76            data = res.json()
 77        else:
 78            raise DeploymentError(
 79                f"Engine at {url} returned code {res.status_code}: {res.text}"
 80            )
 81        return data
 82
 83    def infer(self, tensor: Dict[str, Any]) -> List[InferenceResult]:
 84
 85        if not isinstance(tensor, dict):
 86            raise TypeError(f"tensor is {type(tensor)} but 'dict' is required")
 87
 88        url = self._url()
 89        warning = False
 90        duration = 300
 91        headers = {"User-Agent": _user_agent}
 92        for ix in range(duration + 1):
 93            res = None
 94            try:
 95                res = requests.post(
 96                    url,
 97                    json=tensor,
 98                    timeout=1,
 99                    headers=headers,
100                )
101                data = res.json()
102                break
103            except (requests.exceptions.RequestException, json.JSONDecodeError):
104                if self._interactive:
105                    if not warning:
106                        sys.stdout.write(
107                            "Waiting for deployment to become ready - this may take a few seconds"
108                        )
109                        warning = True
110                    sys.stdout.write(".")
111                time.sleep(1)
112        if ix == duration:
113            raise RuntimeError(f"Deployment did not come up within {duration}s")
114        return [InferenceResult(None, d) for d in data]
115
116    def infer_from_file(
117        self, filename: Union[str, pathlib.Path]
118    ) -> List[InferenceResult]:
119        if not isinstance(filename, pathlib.Path):
120            filename = pathlib.Path(filename)
121        with filename.open("rb") as f:
122            tensor = json.load(f)
123        return self.infer(tensor)
class StandaloneClient:
 18class StandaloneClient:
 19    def __init__(
 20        self,
 21        host: str,
 22        port: int,
 23        model: Optional[Model] = None,
 24        pipeline_config: Optional[PipelineConfig] = None,
 25        interactive: Optional[bool] = None,
 26    ):
 27        if (model and pipeline_config) or not (model or pipeline_config):
 28            raise RuntimeError(
 29                "Specify either a model or a pipeline config for inference"
 30            )
 31        self._host = host
 32        self._port = port
 33        self._model = model
 34        self._pipeline_config = pipeline_config
 35
 36        if interactive is not None:
 37            self._interactive = interactive
 38        elif "JUPYTER_SVC_SERVICE_HOST" in os.environ:
 39            self._interactive = True
 40        else:
 41            self._interactive = False
 42
 43    def _url(self) -> str:
 44        if self._model:
 45            return f"http://{self._host}:{self._port}/models/{self._model.name()}"
 46        elif self._pipeline_config:
 47            return f"http://{self._host}:{self._port}/pipelines/{self._pipeline_config.pipeline_name}"
 48        else:
 49            raise RuntimeError(
 50                "Neither a model or pipeline config was specified for inference"
 51            )
 52
 53    def status(self) -> Dict[str, Any]:
 54        """Returns a dict of standalone engine model status.
 55
 56        Example: {'models': [{'class': 'ccfraud', 'name': 'z5', 'status': 'Running'}]}
 57
 58        Example: {'models': [{'class': 'postprocess',
 59                  'name': 'first-postprocess',
 60                  'status': 'Running'},
 61                  {'class': 'noopfloats', 'name': 'noopv1', 'status': 'Running'},
 62                  {'class': 'preprocess', 'name': 'first-preprocess', 'status': 'Running'}]}
 63
 64        Example: {"models": [{"class":"synerror",
 65                              "name":"v1",
 66                              "status":{"Error":"Python compile or runtime error [\"  File \\\"//tmp/.tmpk7mJpI/syntax_error.py\\\", line 1\", \"    PLEASE CRASH HERE\", \"           ^\", \"SyntaxError: invalid syntax\"]"}}]}
 67
 68        """
 69        url = f"http://{self._host}:{self._port}/models"
 70        headers = {"User-Agent": _user_agent}
 71        try:
 72            res = requests.get(url, timeout=3, headers=headers)
 73        except Exception:
 74            raise DeploymentError(f"Error getting status from {url}")
 75        data = None
 76        if res.status_code == 200:
 77            data = res.json()
 78        else:
 79            raise DeploymentError(
 80                f"Engine at {url} returned code {res.status_code}: {res.text}"
 81            )
 82        return data
 83
 84    def infer(self, tensor: Dict[str, Any]) -> List[InferenceResult]:
 85
 86        if not isinstance(tensor, dict):
 87            raise TypeError(f"tensor is {type(tensor)} but 'dict' is required")
 88
 89        url = self._url()
 90        warning = False
 91        duration = 300
 92        headers = {"User-Agent": _user_agent}
 93        for ix in range(duration + 1):
 94            res = None
 95            try:
 96                res = requests.post(
 97                    url,
 98                    json=tensor,
 99                    timeout=1,
100                    headers=headers,
101                )
102                data = res.json()
103                break
104            except (requests.exceptions.RequestException, json.JSONDecodeError):
105                if self._interactive:
106                    if not warning:
107                        sys.stdout.write(
108                            "Waiting for deployment to become ready - this may take a few seconds"
109                        )
110                        warning = True
111                    sys.stdout.write(".")
112                time.sleep(1)
113        if ix == duration:
114            raise RuntimeError(f"Deployment did not come up within {duration}s")
115        return [InferenceResult(None, d) for d in data]
116
117    def infer_from_file(
118        self, filename: Union[str, pathlib.Path]
119    ) -> List[InferenceResult]:
120        if not isinstance(filename, pathlib.Path):
121            filename = pathlib.Path(filename)
122        with filename.open("rb") as f:
123            tensor = json.load(f)
124        return self.infer(tensor)
StandaloneClient( host: str, port: int, model: Optional[wallaroo.model.Model] = None, pipeline_config: Optional[wallaroo.pipeline_config.PipelineConfig] = None, interactive: Optional[bool] = None)
19    def __init__(
20        self,
21        host: str,
22        port: int,
23        model: Optional[Model] = None,
24        pipeline_config: Optional[PipelineConfig] = None,
25        interactive: Optional[bool] = None,
26    ):
27        if (model and pipeline_config) or not (model or pipeline_config):
28            raise RuntimeError(
29                "Specify either a model or a pipeline config for inference"
30            )
31        self._host = host
32        self._port = port
33        self._model = model
34        self._pipeline_config = pipeline_config
35
36        if interactive is not None:
37            self._interactive = interactive
38        elif "JUPYTER_SVC_SERVICE_HOST" in os.environ:
39            self._interactive = True
40        else:
41            self._interactive = False
def status(self) -> Dict[str, Any]:
53    def status(self) -> Dict[str, Any]:
54        """Returns a dict of standalone engine model status.
55
56        Example: {'models': [{'class': 'ccfraud', 'name': 'z5', 'status': 'Running'}]}
57
58        Example: {'models': [{'class': 'postprocess',
59                  'name': 'first-postprocess',
60                  'status': 'Running'},
61                  {'class': 'noopfloats', 'name': 'noopv1', 'status': 'Running'},
62                  {'class': 'preprocess', 'name': 'first-preprocess', 'status': 'Running'}]}
63
64        Example: {"models": [{"class":"synerror",
65                              "name":"v1",
66                              "status":{"Error":"Python compile or runtime error [\"  File \\\"//tmp/.tmpk7mJpI/syntax_error.py\\\", line 1\", \"    PLEASE CRASH HERE\", \"           ^\", \"SyntaxError: invalid syntax\"]"}}]}
67
68        """
69        url = f"http://{self._host}:{self._port}/models"
70        headers = {"User-Agent": _user_agent}
71        try:
72            res = requests.get(url, timeout=3, headers=headers)
73        except Exception:
74            raise DeploymentError(f"Error getting status from {url}")
75        data = None
76        if res.status_code == 200:
77            data = res.json()
78        else:
79            raise DeploymentError(
80                f"Engine at {url} returned code {res.status_code}: {res.text}"
81            )
82        return data

Returns a dict of standalone engine model status.

Example: {'models': [{'class': 'ccfraud', 'name': 'z5', 'status': 'Running'}]}

Example: {'models': [{'class': 'postprocess', 'name': 'first-postprocess', 'status': 'Running'}, {'class': 'noopfloats', 'name': 'noopv1', 'status': 'Running'}, {'class': 'preprocess', 'name': 'first-preprocess', 'status': 'Running'}]}

Example: {"models": [{"class":"synerror", "name":"v1", "status":{"Error":"Python compile or runtime error [" File \"//tmp/.tmpk7mJpI/syntax_error.py\", line 1", " PLEASE CRASH HERE", " ^", "SyntaxError: invalid syntax"]"}}]}

def infer( self, tensor: Dict[str, Any]) -> List[wallaroo.inference_result.InferenceResult]:
 84    def infer(self, tensor: Dict[str, Any]) -> List[InferenceResult]:
 85
 86        if not isinstance(tensor, dict):
 87            raise TypeError(f"tensor is {type(tensor)} but 'dict' is required")
 88
 89        url = self._url()
 90        warning = False
 91        duration = 300
 92        headers = {"User-Agent": _user_agent}
 93        for ix in range(duration + 1):
 94            res = None
 95            try:
 96                res = requests.post(
 97                    url,
 98                    json=tensor,
 99                    timeout=1,
100                    headers=headers,
101                )
102                data = res.json()
103                break
104            except (requests.exceptions.RequestException, json.JSONDecodeError):
105                if self._interactive:
106                    if not warning:
107                        sys.stdout.write(
108                            "Waiting for deployment to become ready - this may take a few seconds"
109                        )
110                        warning = True
111                    sys.stdout.write(".")
112                time.sleep(1)
113        if ix == duration:
114            raise RuntimeError(f"Deployment did not come up within {duration}s")
115        return [InferenceResult(None, d) for d in data]
def infer_from_file( self, filename: Union[str, pathlib.Path]) -> List[wallaroo.inference_result.InferenceResult]:
117    def infer_from_file(
118        self, filename: Union[str, pathlib.Path]
119    ) -> List[InferenceResult]:
120        if not isinstance(filename, pathlib.Path):
121            filename = pathlib.Path(filename)
122        with filename.open("rb") as f:
123            tensor = json.load(f)
124        return self.infer(tensor)