Coverage for /Users/buh/.pyenv/versions/3.12.2/envs/es-testbed/lib/python3.12/site-packages/es_testbed/helpers/es_api.py: 74%
213 statements
« prev ^ index » next coverage.py v7.4.4, created at 2024-04-25 19:21 -0600
« prev ^ index » next coverage.py v7.4.4, created at 2024-04-25 19:21 -0600
1"""Functions that make Elasticsearch API Calls"""
2import typing as t
3from os import getenv
4from elasticsearch8 import Elasticsearch, exceptions as esx
5from es_wait import Exists, Snapshot
6from es_testbed.defaults import MAPPING, PAUSE_DEFAULT, PAUSE_ENVVAR
7from es_testbed import exceptions as exc
8from es_testbed.helpers.utils import doc_gen, get_routing, getlogger, mounted_name, storage_type
9LOGGER = getlogger(__name__)
10PAUSE_VALUE = float(getenv(PAUSE_ENVVAR, default=PAUSE_DEFAULT))
11# pylint: disable=broad-except
13def emap(kind: str, es: Elasticsearch, value=None):
14 """Return a value from a dictionary"""
15 _ = {
16 'alias': {
17 'delete': es.indices.delete_alias,
18 'exists': es.indices.exists_alias,
19 'get': es.indices.get_alias,
20 'kwargs': {'index': value, 'expand_wildcards': ['open', 'closed']},
21 'plural': 'alias(es)'
22 },
23 'data_stream': {
24 'delete': es.indices.delete_data_stream,
25 'exists': es.indices.exists,
26 'get': es.indices.get_data_stream,
27 'kwargs': {'name': value, 'expand_wildcards': ['open', 'closed']},
28 'plural': 'data_stream(s)',
29 'key': 'data_streams',
30 },
31 'index': {
32 'delete': es.indices.delete,
33 'exists': es.indices.exists,
34 'get': es.indices.get,
35 'kwargs': {'index': value, 'expand_wildcards': ['open', 'closed']},
36 'plural': 'index(es)'
37 },
38 'template': {
39 'delete': es.indices.delete_index_template,
40 'exists': es.indices.exists_index_template,
41 'get': es.indices.get_index_template,
42 'kwargs': {'name': value},
43 'plural': 'index template(s)',
44 'key': 'index_templates',
45 },
46 'ilm': {
47 'delete': es.ilm.delete_lifecycle,
48 'exists': es.ilm.get_lifecycle,
49 'get': es.ilm.get_lifecycle,
50 'kwargs': {'name': value},
51 'plural': 'ilm policy(ies)'
52 },
53 'component': {
54 'delete': es.cluster.delete_component_template,
55 'exists': es.cluster.exists_component_template,
56 'get': es.cluster.get_component_template,
57 'kwargs': {'name': value},
58 'plural': 'component template(s)',
59 'key': 'component_templates',
60 },
61 'snapshot': {
62 'delete': es.snapshot.delete,
63 'exists': es.snapshot.get,
64 'get': es.snapshot.get,
65 'kwargs': {},
66 'plural': 'snapshot(s)'
67 }
68 }
69 return _[kind]
71def change_ds(client: Elasticsearch, actions: dict=None) -> None:
72 """Change/Modify/Update a datastream"""
73 try:
74 client.indices.modify_data_stream(actions=actions, body=None)
75 except Exception as err:
76 raise exc.ResultNotExpected(f'Unable to modify datastreams. {err}') from err
78def create_data_stream(client: Elasticsearch, name: str) -> None:
79 """Create a datastream"""
80 try:
81 client.indices.create_data_stream(name=name)
82 test = Exists(client, name=name, kind='datastream', pause=PAUSE_VALUE)
83 test.wait_for_it()
84 except Exception as err:
85 raise exc.TestbedFailure(
86 f'Unable to create datastream {name}. Error: {err}') from err
88def create_index(
89 client: Elasticsearch,
90 name: str,
91 aliases: dict=None,
92 settings: dict=None,
93 tier: str='hot'
94 ) -> None:
95 """Create named index"""
96 if not settings:
97 settings = get_routing(tier=tier)
98 else:
99 settings.update(get_routing(tier=tier))
100 client.indices.create(
101 index=name,
102 aliases=aliases,
103 mappings=MAPPING,
104 settings=settings
105 )
106 try:
107 test = Exists(client, name=name, kind='index', pause=PAUSE_VALUE)
108 test.wait_for_it()
109 except exc.TimeoutException as err:
110 raise exc.ResultNotExpected(f'Failed to create index {name}') from err
111 return exists(client, 'index', name)
113def delete(client: Elasticsearch, kind: str, name: str, repository: str=None):
114 """Delete the named object of type kind"""
115 which = emap(kind, client)
116 func = which['delete']
117 if name is None: # Typically only with ilm 117 ↛ 118line 117 didn't jump to line 118, because the condition on line 117 was never true
118 LOGGER.debug('"%s" has a None value for name', kind)
119 return
120 try:
121 if kind == 'snapshot':
122 func(snapshot=name, repository=repository)
123 elif kind == 'index':
124 func(index=name)
125 else:
126 func(name=name)
127 except esx.NotFoundError:
128 LOGGER.warning('%s named %s not found.', kind, name)
129 except Exception as err:
130 raise exc.ResultNotExpected(f'Unexpected result: {err}') from err
131 if exists(client, kind, name, repository=repository): 131 ↛ 132line 131 didn't jump to line 132, because the condition on line 131 was never true
132 LOGGER.critical('Unable to delete "%s" %s', kind, name)
133 raise exc.ResultNotExpected(f'{kind} "{name}" still exists.')
134 LOGGER.info('Successfully deleted %s: "%s"', which['plural'], name)
136def do_snap(client: Elasticsearch, repo: str, snap: str, idx: str, tier: str='cold') -> None:
137 """Perform a snapshot"""
138 client.snapshot.create(repository=repo, snapshot=snap, indices=idx)
139 test = Snapshot(client, action='snapshot', snapshot=snap, repository=repo, pause=1, timeout=60)
140 test.wait_for_it()
142 # Mount the index accordingly
143 client.searchable_snapshots.mount(
144 repository=repo, snapshot=snap, index=idx, index_settings=get_routing(tier=tier),
145 renamed_index=mounted_name(idx, tier), storage=storage_type(tier),
146 wait_for_completion=True)
147 # Fix aliases
148 fix_aliases(client, idx, mounted_name(idx, tier))
150def exists(client: Elasticsearch, kind: str, name: str, repository: str=None) -> bool:
151 """Return boolean existence of the named kind of object"""
152 if name is None: 152 ↛ 153line 152 didn't jump to line 153, because the condition on line 152 was never true
153 return False
154 retval = True
155 func = emap(kind, client)['exists']
156 try:
157 if kind == 'snapshot':
158 retval = func(snapshot=name, repository=repository)
159 elif kind == 'ilm':
160 retval = func(name=name)
161 elif kind in ['index', 'data_stream']:
162 retval = func(index=name)
163 else:
164 retval = func(name=name)
165 except esx.NotFoundError: 165 ↛ 167line 165 didn't jump to line 167
166 retval = False
167 except Exception as err:
168 raise exc.ResultNotExpected(f'Unexpected result: {err}') from err
169 return retval
171def fill_index(
172 client: Elasticsearch,
173 name: str=None,
174 count: int=None,
175 start_num: int=None,
176 match: bool=True
177 ) -> None:
178 """
179 Create and fill the named index with mappings and settings as directed
181 :param client: ES client
182 :param name: Index name
183 :param count: The number of docs to create
184 :param start_number: Where to start the incrementing number
185 :param match: Whether to use the default values for key (True) or random strings (False)
187 :type client: es
188 :type name: str
189 :type count: int
190 :type start_number: int
191 :type match: bool
193 :rtype: None
194 :returns: No return value
195 """
196 for doc in doc_gen(count=count, start_at=start_num, match=match):
197 client.index(index=name, document=doc)
198 client.indices.flush(index=name)
199 client.indices.refresh(index=name)
201def find_write_index(client: Elasticsearch, name: str) -> t.AnyStr:
202 """Find the write_index for an alias by searching any index the alias points to"""
203 retval = None
204 for alias in get_aliases(client, name):
205 retval = get_write_index(client, alias)
206 if retval: 206 ↛ 204line 206 didn't jump to line 204, because the condition on line 206 was never false
207 break
208 return retval
210def fix_aliases(client: Elasticsearch, oldidx: str, newidx: str) -> None:
211 """Fix aliases using the new and old index names as data"""
212 # Delete the original index
213 client.indices.delete(index=oldidx)
214 # Add the original index name as an alias to the mounted index
215 client.indices.put_alias(index=f'{newidx}', name=oldidx)
217def get(client: Elasticsearch, kind: str, pattern: str) -> t.Sequence[str]:
218 """get any/all objects of type kind matching pattern"""
219 if pattern is None: 219 ↛ 220line 219 didn't jump to line 220, because the condition on line 219 was never true
220 msg = f'"{kind}" has a None value for pattern'
221 LOGGER.error(msg)
222 raise exc.TestbedMisconfig(msg)
223 which = emap(kind, client, value=pattern)
224 func = which['get']
225 kwargs = which['kwargs']
226 try:
227 result = func(**kwargs)
228 except Exception as err:
229 raise exc.ResultNotExpected(f'Unexpected result: {err}') from err
230 if kind in ['data_stream', 'template', 'component']:
231 retval = [x['name'] for x in result[which['key']]]
232 else:
233 # ['alias', 'ilm', 'index']
234 retval = list(result.keys())
235 return retval
237def get_aliases(client: Elasticsearch, name: str) -> t.Sequence[str]:
238 """Get aliases from index 'name'"""
239 res = client.indices.get(index=name)
240 try:
241 retval = list(res[name]['aliases'].keys())
242 except KeyError:
243 retval = None
244 return retval
246def get_backing_indices(client: Elasticsearch, name: str) -> t.Sequence[str]:
247 """Get the backing indices from the named data_stream"""
248 resp = resolver(client, name)
249 data_streams = resp['data_streams']
250 retval = []
251 if data_streams: 251 ↛ 255line 251 didn't jump to line 255, because the condition on line 251 was never false
252 if len(data_streams) > 1: 252 ↛ 253line 252 didn't jump to line 253, because the condition on line 252 was never true
253 raise exc.ResultNotExpected(f'Expected only a single data_stream matching {name}')
254 retval = data_streams[0]['backing_indices']
255 return retval
257def get_ds_current(client: Elasticsearch, name: str) -> str:
258 """
259 Find which index is the current 'write' index of the datastream
260 This is best accomplished by grabbing the last backing_index
261 """
262 backers = get_backing_indices(client, name)
263 retval = None
264 if backers: 264 ↛ 266line 264 didn't jump to line 266, because the condition on line 264 was never false
265 retval = backers[-1]
266 return retval
268def get_ilm(client: Elasticsearch, pattern: str) -> t.Union[t.Dict[str,str], None]:
269 """Get any ILM entity in ES that matches pattern"""
270 try:
271 return client.ilm.get_lifecycle(name=pattern)
272 except Exception as err:
273 msg = f'Unable to get ILM lifecycle matching {pattern}. Error: {err}'
274 LOGGER.critical(msg)
275 raise exc.ResultNotExpected(msg) from err
277def get_ilm_phases(client: Elasticsearch, name: str) -> dict:
278 """Return the policy/phases part of the ILM policy identified by 'name'"""
279 ilm = get_ilm(client, name)
280 try:
281 return ilm[name]['policy']['phases']
282 except KeyError as err:
283 msg = f'Unable to get ILM lifecycle named {name}. Error: {err}'
284 LOGGER.critical(msg)
285 raise exc.ResultNotExpected(msg) from err
287def get_write_index(client: Elasticsearch, name: str) -> str:
288 """
289 Calls :py:meth:`~.elasticsearch.client.IndicesClient.get_alias`
291 :param client: A client connection object
292 :param name: An alias name
294 :type client: :py:class:`~.elasticsearch.Elasticsearch`
295 :type name: str
297 :returns: The the index name associated with the alias that is designated ``is_write_index``
298 :rtype: str
299 """
300 response = client.indices.get_alias(index=name)
301 retval = None
302 for index in list(response.keys()): 302 ↛ 309line 302 didn't jump to line 309, because the loop on line 302 didn't complete
303 try:
304 if response[index]['aliases'][name]['is_write_index']:
305 retval = index
306 break
307 except KeyError:
308 continue
309 return retval
311def snapshot_name(client: Elasticsearch, name: str) -> t.Union[t.AnyStr, None]:
312 """Get the name of the snapshot behind the mounted index data"""
313 res = {}
314 if exists(client, 'index', name): # Can jump straight to nested keys if it exists 314 ↛ 316line 314 didn't jump to line 316, because the condition on line 314 was never false
315 res = client.indices.get(index=name)[name]['settings']['index']
316 try:
317 retval = res['store']['snapshot']['snapshot_name']
318 except KeyError:
319 LOGGER.error('%s is not a searchable snapshot')
320 retval = None
321 return retval
323def ilm_explain(client: Elasticsearch, name: str) -> t.Union[t.Dict, None]:
324 """Return the results from the ILM Explain API call for the named index"""
325 try:
326 retval = client.ilm.explain_lifecycle(index=name)['indices'][name]
327 except KeyError:
328 LOGGER.debug('Index name changed')
329 new = list(client.ilm.explain_lifecycle(index=name)['indices'].keys())[0]
330 retval = client.ilm.explain_lifecycle(index=new)['indices'][new]
331 except esx.NotFoundError as err: 331 ↛ 334line 331 didn't jump to line 334
332 LOGGER.warning('Datastream/Index Name changed. %s was not found', name)
333 raise exc.NameChanged(f'{name} was not found, likely due to a name change') from err
334 except Exception as err:
335 msg = f'Unable to get ILM information for index {name}'
336 LOGGER.critical(msg)
337 raise exc.ResultNotExpected(f'{msg}. Exception: {err}') from err
338 return retval
340def ilm_move(client: Elasticsearch, name: str, current_step: dict, next_step: dict) -> None:
341 """Move index 'name' from the current step to the next step"""
342 try:
343 client.ilm.move_to_step(index=name, current_step=current_step, next_step=next_step)
344 except Exception as err:
345 msg = f'Unable to move index {name} to ILM next step: {next}. Error: {err}'
346 LOGGER.critical(msg)
347 raise exc.ResultNotExpected(msg)
349def put_comp_tmpl(client: Elasticsearch, name: str, component: dict) -> None:
350 """Publish a component template"""
351 try:
352 client.cluster.put_component_template(name=name, template=component, create=True)
353 test = Exists(client, name=name, kind='component', pause=PAUSE_VALUE)
354 test.wait_for_it()
355 except Exception as err:
356 raise exc.TestbedFailure(
357 f'Unable to create component template {name}. Error: {err}') from err
359def put_idx_tmpl(
360 client, name: str, index_patterns: list, components: list,
361 data_stream: dict=None) -> None:
362 """Publish an index template"""
363 try:
364 client.indices.put_index_template(
365 name=name,
366 composed_of=components,
367 data_stream=data_stream,
368 index_patterns=index_patterns,
369 create=True,
370 )
371 test = Exists(client, name=name, kind='template', pause=PAUSE_VALUE)
372 test.wait_for_it()
373 except Exception as err:
374 raise exc.TestbedFailure(
375 f'Unable to create index template {name}. Error: {err}') from err
377def put_ilm(client: Elasticsearch, name: str, policy: dict=None) -> None:
378 """Publish an ILM Policy"""
379 try:
380 client.ilm.put_lifecycle(name=name, policy=policy)
381 except Exception as err:
382 raise exc.TestbedFailure(
383 f'Unable to put index lifecycle policy {name}. Error: {err}') from err
385def resolver(client: Elasticsearch, name: str) -> dict:
386 """
387 Resolve details about the entity, be it an index, alias, or data_stream
389 Because you can pass search patterns and aliases as name, each element comes back as an array:
391 {'indices': [], 'aliases': [], 'data_streams': []}
393 If you only resolve a single index or data stream, you will still have a 1-element list
394 """
395 return client.indices.resolve_index(name=name, expand_wildcards=['open', 'closed'])
397def rollover(client: Elasticsearch, name: str) -> None:
398 """Rollover alias or datastream identified by name"""
399 client.indices.rollover(alias=name, wait_for_active_shards='all')