wallaroo.standalone_client
1import json 2import os 3import pathlib 4import sys 5import time 6from typing import Any, Dict, List, Optional, Union 7 8import requests 9 10from wallaroo.inference_result import InferenceResult 11from wallaroo.model import Model 12from wallaroo.object import DeploymentError 13from wallaroo.pipeline_config import PipelineConfig 14from wallaroo.version import _user_agent 15 16 17class StandaloneClient: 18 def __init__( 19 self, 20 host: str, 21 port: int, 22 model: Optional[Model] = None, 23 pipeline_config: Optional[PipelineConfig] = None, 24 interactive: Optional[bool] = None, 25 ): 26 if (model and pipeline_config) or not (model or pipeline_config): 27 raise RuntimeError( 28 "Specify either a model or a pipeline config for inference" 29 ) 30 self._host = host 31 self._port = port 32 self._model = model 33 self._pipeline_config = pipeline_config 34 35 if interactive is not None: 36 self._interactive = interactive 37 elif "JUPYTER_SVC_SERVICE_HOST" in os.environ: 38 self._interactive = True 39 else: 40 self._interactive = False 41 42 def _url(self) -> str: 43 if self._model: 44 return f"http://{self._host}:{self._port}/models/{self._model.name()}" 45 elif self._pipeline_config: 46 return f"http://{self._host}:{self._port}/pipelines/{self._pipeline_config.pipeline_name}" 47 else: 48 raise RuntimeError( 49 "Neither a model or pipeline config was specified for inference" 50 ) 51 52 def status(self) -> Dict[str, Any]: 53 """Returns a dict of standalone engine model status. 54 55 Example: {'models': [{'class': 'ccfraud', 'name': 'z5', 'status': 'Running'}]} 56 57 Example: {'models': [{'class': 'postprocess', 58 'name': 'first-postprocess', 59 'status': 'Running'}, 60 {'class': 'noopfloats', 'name': 'noopv1', 'status': 'Running'}, 61 {'class': 'preprocess', 'name': 'first-preprocess', 'status': 'Running'}]} 62 63 Example: {"models": [{"class":"synerror", 64 "name":"v1", 65 "status":{"Error":"Python compile or runtime error [\" File \\\"//tmp/.tmpk7mJpI/syntax_error.py\\\", line 1\", \" PLEASE CRASH HERE\", \" ^\", \"SyntaxError: invalid syntax\"]"}}]} 66 67 """ 68 url = f"http://{self._host}:{self._port}/models" 69 headers = {"User-Agent": _user_agent} 70 try: 71 res = requests.get(url, timeout=3, headers=headers) 72 except Exception: 73 raise DeploymentError(f"Error getting status from {url}") 74 data = None 75 if res.status_code == 200: 76 data = res.json() 77 else: 78 raise DeploymentError( 79 f"Engine at {url} returned code {res.status_code}: {res.text}" 80 ) 81 return data 82 83 def infer(self, tensor: Dict[str, Any]) -> List[InferenceResult]: 84 85 if not isinstance(tensor, dict): 86 raise TypeError(f"tensor is {type(tensor)} but 'dict' is required") 87 88 url = self._url() 89 warning = False 90 duration = 300 91 headers = {"User-Agent": _user_agent} 92 for ix in range(duration + 1): 93 res = None 94 try: 95 res = requests.post( 96 url, 97 json=tensor, 98 timeout=1, 99 headers=headers, 100 ) 101 data = res.json() 102 break 103 except (requests.exceptions.RequestException, json.JSONDecodeError): 104 if self._interactive: 105 if not warning: 106 sys.stdout.write( 107 "Waiting for deployment to become ready - this may take a few seconds" 108 ) 109 warning = True 110 sys.stdout.write(".") 111 time.sleep(1) 112 if ix == duration: 113 raise RuntimeError(f"Deployment did not come up within {duration}s") 114 return [InferenceResult(None, d) for d in data] 115 116 def infer_from_file( 117 self, filename: Union[str, pathlib.Path] 118 ) -> List[InferenceResult]: 119 if not isinstance(filename, pathlib.Path): 120 filename = pathlib.Path(filename) 121 with filename.open("rb") as f: 122 tensor = json.load(f) 123 return self.infer(tensor)
class
StandaloneClient:
18class StandaloneClient: 19 def __init__( 20 self, 21 host: str, 22 port: int, 23 model: Optional[Model] = None, 24 pipeline_config: Optional[PipelineConfig] = None, 25 interactive: Optional[bool] = None, 26 ): 27 if (model and pipeline_config) or not (model or pipeline_config): 28 raise RuntimeError( 29 "Specify either a model or a pipeline config for inference" 30 ) 31 self._host = host 32 self._port = port 33 self._model = model 34 self._pipeline_config = pipeline_config 35 36 if interactive is not None: 37 self._interactive = interactive 38 elif "JUPYTER_SVC_SERVICE_HOST" in os.environ: 39 self._interactive = True 40 else: 41 self._interactive = False 42 43 def _url(self) -> str: 44 if self._model: 45 return f"http://{self._host}:{self._port}/models/{self._model.name()}" 46 elif self._pipeline_config: 47 return f"http://{self._host}:{self._port}/pipelines/{self._pipeline_config.pipeline_name}" 48 else: 49 raise RuntimeError( 50 "Neither a model or pipeline config was specified for inference" 51 ) 52 53 def status(self) -> Dict[str, Any]: 54 """Returns a dict of standalone engine model status. 55 56 Example: {'models': [{'class': 'ccfraud', 'name': 'z5', 'status': 'Running'}]} 57 58 Example: {'models': [{'class': 'postprocess', 59 'name': 'first-postprocess', 60 'status': 'Running'}, 61 {'class': 'noopfloats', 'name': 'noopv1', 'status': 'Running'}, 62 {'class': 'preprocess', 'name': 'first-preprocess', 'status': 'Running'}]} 63 64 Example: {"models": [{"class":"synerror", 65 "name":"v1", 66 "status":{"Error":"Python compile or runtime error [\" File \\\"//tmp/.tmpk7mJpI/syntax_error.py\\\", line 1\", \" PLEASE CRASH HERE\", \" ^\", \"SyntaxError: invalid syntax\"]"}}]} 67 68 """ 69 url = f"http://{self._host}:{self._port}/models" 70 headers = {"User-Agent": _user_agent} 71 try: 72 res = requests.get(url, timeout=3, headers=headers) 73 except Exception: 74 raise DeploymentError(f"Error getting status from {url}") 75 data = None 76 if res.status_code == 200: 77 data = res.json() 78 else: 79 raise DeploymentError( 80 f"Engine at {url} returned code {res.status_code}: {res.text}" 81 ) 82 return data 83 84 def infer(self, tensor: Dict[str, Any]) -> List[InferenceResult]: 85 86 if not isinstance(tensor, dict): 87 raise TypeError(f"tensor is {type(tensor)} but 'dict' is required") 88 89 url = self._url() 90 warning = False 91 duration = 300 92 headers = {"User-Agent": _user_agent} 93 for ix in range(duration + 1): 94 res = None 95 try: 96 res = requests.post( 97 url, 98 json=tensor, 99 timeout=1, 100 headers=headers, 101 ) 102 data = res.json() 103 break 104 except (requests.exceptions.RequestException, json.JSONDecodeError): 105 if self._interactive: 106 if not warning: 107 sys.stdout.write( 108 "Waiting for deployment to become ready - this may take a few seconds" 109 ) 110 warning = True 111 sys.stdout.write(".") 112 time.sleep(1) 113 if ix == duration: 114 raise RuntimeError(f"Deployment did not come up within {duration}s") 115 return [InferenceResult(None, d) for d in data] 116 117 def infer_from_file( 118 self, filename: Union[str, pathlib.Path] 119 ) -> List[InferenceResult]: 120 if not isinstance(filename, pathlib.Path): 121 filename = pathlib.Path(filename) 122 with filename.open("rb") as f: 123 tensor = json.load(f) 124 return self.infer(tensor)
StandaloneClient( host: str, port: int, model: Optional[wallaroo.model.Model] = None, pipeline_config: Optional[wallaroo.pipeline_config.PipelineConfig] = None, interactive: Optional[bool] = None)
19 def __init__( 20 self, 21 host: str, 22 port: int, 23 model: Optional[Model] = None, 24 pipeline_config: Optional[PipelineConfig] = None, 25 interactive: Optional[bool] = None, 26 ): 27 if (model and pipeline_config) or not (model or pipeline_config): 28 raise RuntimeError( 29 "Specify either a model or a pipeline config for inference" 30 ) 31 self._host = host 32 self._port = port 33 self._model = model 34 self._pipeline_config = pipeline_config 35 36 if interactive is not None: 37 self._interactive = interactive 38 elif "JUPYTER_SVC_SERVICE_HOST" in os.environ: 39 self._interactive = True 40 else: 41 self._interactive = False
def
status(self) -> Dict[str, Any]:
53 def status(self) -> Dict[str, Any]: 54 """Returns a dict of standalone engine model status. 55 56 Example: {'models': [{'class': 'ccfraud', 'name': 'z5', 'status': 'Running'}]} 57 58 Example: {'models': [{'class': 'postprocess', 59 'name': 'first-postprocess', 60 'status': 'Running'}, 61 {'class': 'noopfloats', 'name': 'noopv1', 'status': 'Running'}, 62 {'class': 'preprocess', 'name': 'first-preprocess', 'status': 'Running'}]} 63 64 Example: {"models": [{"class":"synerror", 65 "name":"v1", 66 "status":{"Error":"Python compile or runtime error [\" File \\\"//tmp/.tmpk7mJpI/syntax_error.py\\\", line 1\", \" PLEASE CRASH HERE\", \" ^\", \"SyntaxError: invalid syntax\"]"}}]} 67 68 """ 69 url = f"http://{self._host}:{self._port}/models" 70 headers = {"User-Agent": _user_agent} 71 try: 72 res = requests.get(url, timeout=3, headers=headers) 73 except Exception: 74 raise DeploymentError(f"Error getting status from {url}") 75 data = None 76 if res.status_code == 200: 77 data = res.json() 78 else: 79 raise DeploymentError( 80 f"Engine at {url} returned code {res.status_code}: {res.text}" 81 ) 82 return data
Returns a dict of standalone engine model status.
Example: {'models': [{'class': 'ccfraud', 'name': 'z5', 'status': 'Running'}]}
Example: {'models': [{'class': 'postprocess', 'name': 'first-postprocess', 'status': 'Running'}, {'class': 'noopfloats', 'name': 'noopv1', 'status': 'Running'}, {'class': 'preprocess', 'name': 'first-preprocess', 'status': 'Running'}]}
Example: {"models": [{"class":"synerror", "name":"v1", "status":{"Error":"Python compile or runtime error [" File \"//tmp/.tmpk7mJpI/syntax_error.py\", line 1", " PLEASE CRASH HERE", " ^", "SyntaxError: invalid syntax"]"}}]}
84 def infer(self, tensor: Dict[str, Any]) -> List[InferenceResult]: 85 86 if not isinstance(tensor, dict): 87 raise TypeError(f"tensor is {type(tensor)} but 'dict' is required") 88 89 url = self._url() 90 warning = False 91 duration = 300 92 headers = {"User-Agent": _user_agent} 93 for ix in range(duration + 1): 94 res = None 95 try: 96 res = requests.post( 97 url, 98 json=tensor, 99 timeout=1, 100 headers=headers, 101 ) 102 data = res.json() 103 break 104 except (requests.exceptions.RequestException, json.JSONDecodeError): 105 if self._interactive: 106 if not warning: 107 sys.stdout.write( 108 "Waiting for deployment to become ready - this may take a few seconds" 109 ) 110 warning = True 111 sys.stdout.write(".") 112 time.sleep(1) 113 if ix == duration: 114 raise RuntimeError(f"Deployment did not come up within {duration}s") 115 return [InferenceResult(None, d) for d in data]
def
infer_from_file( self, filename: Union[str, pathlib.Path]) -> List[wallaroo.inference_result.InferenceResult]: