Coverage for src/es_testbed/helpers/es_api.py: 61%
209 statements
« prev ^ index » next coverage.py v7.5.0, created at 2024-04-23 15:01 -0600
« prev ^ index » next coverage.py v7.5.0, created at 2024-04-23 15:01 -0600
1"""Functions that make Elasticsearch API Calls"""
2import typing as t
3from os import getenv
4from elasticsearch8 import Elasticsearch, exceptions as esx
5from es_wait import Exists, Snapshot
6from es_testbed.defaults import MAPPING, PAUSE_DEFAULT, PAUSE_ENVVAR
7from es_testbed import exceptions as exc
8from es_testbed.helpers.utils import doc_gen, get_routing, getlogger, mounted_name, storage_type
9LOGGER = getlogger(__name__)
10PAUSE_VALUE = float(getenv(PAUSE_ENVVAR, default=PAUSE_DEFAULT))
11# pylint: disable=broad-except
13def emap(kind: str, es: Elasticsearch, value=None):
14 """Return a value from a dictionary"""
15 _ = {
16 'alias': {
17 'delete': es.indices.delete_alias,
18 'exists': es.indices.exists_alias,
19 'get': es.indices.get_alias,
20 'kwargs': {'index': value, 'expand_wildcards': ['open', 'closed']},
21 'plural': 'alias(es)'
22 },
23 'data_stream': {
24 'delete': es.indices.delete_data_stream,
25 'exists': es.indices.exists,
26 'get': es.indices.get_data_stream,
27 'kwargs': {'name': value, 'expand_wildcards': ['open', 'closed']},
28 'plural': 'data_stream(s)',
29 'key': 'data_streams',
30 },
31 'index': {
32 'delete': es.indices.delete,
33 'exists': es.indices.exists,
34 'get': es.indices.get,
35 'kwargs': {'index': value, 'expand_wildcards': ['open', 'closed']},
36 'plural': 'index(es)'
37 },
38 'template': {
39 'delete': es.indices.delete_index_template,
40 'exists': es.indices.exists_index_template,
41 'get': es.indices.get_index_template,
42 'kwargs': {'name': value},
43 'plural': 'index template(s)',
44 'key': 'index_templates',
45 },
46 'ilm': {
47 'delete': es.ilm.delete_lifecycle,
48 'exists': es.ilm.get_lifecycle,
49 'get': es.ilm.get_lifecycle,
50 'kwargs': {'name': value},
51 'plural': 'ilm policy(ies)'
52 },
53 'component': {
54 'delete': es.cluster.delete_component_template,
55 'exists': es.cluster.exists_component_template,
56 'get': es.cluster.get_component_template,
57 'kwargs': {'name': value},
58 'plural': 'component template(s)',
59 'key': 'component_templates',
60 },
61 'snapshot': {
62 'delete': es.snapshot.delete,
63 'exists': es.snapshot.get,
64 'get': es.snapshot.get,
65 'kwargs': {},
66 'plural': 'snapshot(s)'
67 }
68 }
69 return _[kind]
71def change_ds(client: Elasticsearch, actions: dict=None) -> None:
72 """Change/Modify/Update a datastream"""
73 try:
74 client.indices.modify_data_stream(actions=actions, body=None)
75 except Exception as err:
76 raise exc.ResultNotExpected(f'Unable to modify datastreams. {err}') from err
78def create_data_stream(client: Elasticsearch, name: str) -> None:
79 """Create a datastream"""
80 try:
81 client.indices.create_data_stream(name=name)
82 test = Exists(client, name=name, kind='datastream', pause=PAUSE_VALUE)
83 test.wait_for_it()
84 except Exception as err:
85 raise exc.TestbedFailure(
86 f'Unable to create datastream {name}. Error: {err}') from err
88def create_index(
89 client: Elasticsearch,
90 name: str,
91 aliases: dict=None,
92 settings: dict=None,
93 tier: str='hot'
94 ) -> None:
95 """Create named index"""
96 if not settings: 96 ↛ 97line 96 didn't jump to line 97, because the condition on line 96 was never true
97 settings = get_routing(tier=tier)
98 else:
99 settings.update(get_routing(tier=tier))
100 client.indices.create(
101 index=name,
102 aliases=aliases,
103 mappings=MAPPING,
104 settings=settings
105 )
106 try:
107 test = Exists(client, name=name, kind='index', pause=PAUSE_VALUE)
108 test.wait_for_it()
109 except exc.TimeoutException as err:
110 raise exc.ResultNotExpected(f'Failed to create index {name}') from err
111 return exists(client, 'index', name)
113def delete(client: Elasticsearch, kind: str, name: str, repository: str=None):
114 """Delete the named object of type kind"""
115 which = emap(kind, client)
116 func = which['delete']
117 if name is None: # Typically only with ilm 117 ↛ 118line 117 didn't jump to line 118, because the condition on line 117 was never true
118 LOGGER.debug('"%s" has a None value for name', kind)
119 return
120 try:
121 if kind == 'snapshot': 121 ↛ 122line 121 didn't jump to line 122, because the condition on line 121 was never true
122 func(snapshot=name, repository=repository)
123 elif kind == 'index':
124 func(index=name)
125 else:
126 func(name=name)
127 except esx.NotFoundError:
128 LOGGER.warning('%s named %s not found.', kind, name)
129 except Exception as err:
130 raise exc.ResultNotExpected(f'Unexpected result: {err}') from err
131 if exists(client, kind, name, repository=repository): 131 ↛ 132line 131 didn't jump to line 132, because the condition on line 131 was never true
132 LOGGER.critical('Unable to delete "%s" %s', kind, name)
133 raise exc.ResultNotExpected(f'{kind} "{name}" still exists.')
134 LOGGER.info('Successfully deleted %s: "%s"', which['plural'], name)
136def do_snap(client: Elasticsearch, repo: str, snap: str, idx: str, tier: str='cold') -> None:
137 """Perform a snapshot"""
138 client.snapshot.create(repository=repo, snapshot=snap, indices=idx)
139 test = Snapshot(client, action='snapshot', snapshot=snap, repository=repo, pause=1, timeout=60)
140 test.wait_for_it()
142 # Mount the index accordingly
143 client.searchable_snapshots.mount(
144 repository=repo, snapshot=snap, index=idx, index_settings=get_routing(tier=tier),
145 renamed_index=mounted_name(idx, tier), storage=storage_type(tier),
146 wait_for_completion=True)
148def exists(client: Elasticsearch, kind: str, name: str, repository: str=None) -> bool:
149 """Return boolean existence of the named kind of object"""
150 if name is None: 150 ↛ 151line 150 didn't jump to line 151, because the condition on line 150 was never true
151 return False
152 retval = True
153 func = emap(kind, client)['exists']
154 try:
155 if kind == 'snapshot': 155 ↛ 156line 155 didn't jump to line 156, because the condition on line 155 was never true
156 retval = func(snapshot=name, repository=repository)
157 elif kind == 'ilm':
158 retval = func(name=name)
159 elif kind in ['index', 'data_stream']:
160 retval = func(index=name)
161 else:
162 retval = func(name=name)
163 except esx.NotFoundError: 163 ↛ 165line 163 didn't jump to line 165
164 retval = False
165 except Exception as err:
166 raise exc.ResultNotExpected(f'Unexpected result: {err}') from err
167 return retval
169def fill_index(
170 client: Elasticsearch,
171 name: str=None,
172 count: int=None,
173 start_num: int=None,
174 match: bool=True
175 ) -> None:
176 """
177 Create and fill the named index with mappings and settings as directed
179 :param client: ES client
180 :param name: Index name
181 :param count: The number of docs to create
182 :param start_number: Where to start the incrementing number
183 :param match: Whether to use the default values for key (True) or random strings (False)
185 :type client: es
186 :type name: str
187 :type count: int
188 :type start_number: int
189 :type match: bool
191 :rtype: None
192 :returns: No return value
193 """
194 for doc in doc_gen(count=count, start_at=start_num, match=match):
195 client.index(index=name, document=doc)
196 client.indices.flush(index=name)
197 client.indices.refresh(index=name)
199def find_write_index(client: Elasticsearch, name: str) -> t.AnyStr:
200 """Find the write_index for an alias by searching any index the alias points to"""
201 retval = None
202 for alias in get_aliases(client, name): 202 ↛ 206line 202 didn't jump to line 206, because the loop on line 202 didn't complete
203 retval = get_write_index(client, alias)
204 if retval: 204 ↛ 202line 204 didn't jump to line 202, because the condition on line 204 was never false
205 break
206 return retval
208def fix_aliases(client: Elasticsearch, oldidx: str, newidx: str) -> None:
209 """Fix aliases using the new and old index names as data"""
210 # Delete the original index
211 client.indices.delete(index=oldidx)
212 # Add the original index name as an alias to the mounted index
213 client.indices.put_alias(index=f'{newidx}', name=oldidx)
215def get(client: Elasticsearch, kind: str, pattern: str) -> t.Sequence[str]:
216 """get any/all objects of type kind matching pattern"""
217 if pattern is None: 217 ↛ 218line 217 didn't jump to line 218, because the condition on line 217 was never true
218 msg = f'"{kind}" has a None value for pattern'
219 LOGGER.error(msg)
220 raise exc.TestbedMisconfig(msg)
221 which = emap(kind, client, value=pattern)
222 func = which['get']
223 kwargs = which['kwargs']
224 try:
225 result = func(**kwargs)
226 except Exception as err:
227 raise exc.ResultNotExpected(f'Unexpected result: {err}') from err
228 if kind in ['data_stream', 'template', 'component']:
229 retval = [x['name'] for x in result[which['key']]]
230 else:
231 # ['alias', 'ilm', 'index']
232 retval = list(result.keys())
233 return retval
235def get_aliases(client: Elasticsearch, name: str) -> t.Sequence[str]:
236 """Get aliases from index 'name'"""
237 res = client.indices.get(index=name)
238 try:
239 retval = list(res[name]['aliases'].keys())
240 except KeyError:
241 retval = None
242 return retval
244def get_backing_indices(client: Elasticsearch, name: str) -> t.Sequence[str]:
245 """Get the backing indices from the named data_stream"""
246 resp = resolver(client, name)
247 data_streams = resp['data_streams']
248 retval = []
249 if data_streams: 249 ↛ 253line 249 didn't jump to line 253, because the condition on line 249 was never false
250 if len(data_streams) > 1: 250 ↛ 251line 250 didn't jump to line 251, because the condition on line 250 was never true
251 raise exc.ResultNotExpected(f'Expected only a single data_stream matching {name}')
252 retval = data_streams[0]['backing_indices']
253 return retval
255def get_ds_current(client: Elasticsearch, name: str) -> str:
256 """
257 Find which index is the current 'write' index of the datastream
258 This is best accomplished by grabbing the last backing_index
259 """
260 backers = get_backing_indices(client, name)
261 retval = None
262 if backers: 262 ↛ 264line 262 didn't jump to line 264, because the condition on line 262 was never false
263 retval = backers[-1]
264 return retval
266def get_ilm(client: Elasticsearch, pattern: str) -> t.Union[t.Dict[str,str], None]:
267 """Get any ILM entity in ES that matches pattern"""
268 try:
269 return client.ilm.get_lifecycle(name=pattern)
270 except Exception as err:
271 msg = f'Unable to get ILM lifecycle matching {pattern}. Error: {err}'
272 LOGGER.critical(msg)
273 raise exc.ResultNotExpected(msg) from err
275def get_ilm_phases(client: Elasticsearch, name: str) -> dict:
276 """Return the policy/phases part of the ILM policy identified by 'name'"""
277 ilm = get_ilm(client, name)
278 try:
279 return ilm[name]['policy']['phases']
280 except KeyError as err:
281 msg = f'Unable to get ILM lifecycle named {name}. Error: {err}'
282 LOGGER.critical(msg)
283 raise exc.ResultNotExpected(msg) from err
285def get_write_index(client: Elasticsearch, name: str) -> str:
286 """
287 Calls :py:meth:`~.elasticsearch.client.IndicesClient.get_alias`
289 :param client: A client connection object
290 :param name: An alias name
292 :type client: :py:class:`~.elasticsearch.Elasticsearch`
293 :type name: str
295 :returns: The the index name associated with the alias that is designated ``is_write_index``
296 :rtype: str
297 """
298 response = client.indices.get_alias(index=name)
299 retval = None
300 for index in list(response.keys()): 300 ↛ 304line 300 didn't jump to line 304, because the loop on line 300 didn't complete
301 if response[index]['aliases'][name]['is_write_index']: 301 ↛ 300line 301 didn't jump to line 300, because the condition on line 301 was never false
302 retval = index
303 break
304 return retval
306def snapshot_name(client: Elasticsearch, name: str) -> t.Union[t.AnyStr, None]:
307 """Get the name of the snapshot behind the mounted index data"""
308 res = {}
309 if exists(client, 'index', name): # Can jump straight to nested keys if it exists
310 res = client.indices.get(index=name)[name]['settings']['index']
311 try:
312 retval = res['store']['snapshot']['snapshot_name']
313 except KeyError:
314 LOGGER.error('%s is not a searchable snapshot')
315 retval = None
316 return retval
318def ilm_explain(client: Elasticsearch, name: str) -> t.Union[t.Dict, None]:
319 """Return the results from the ILM Explain API call for the named index"""
320 try:
321 retval = client.ilm.explain_lifecycle(index=name)['indices'][name]
322 except KeyError:
323 LOGGER.debug('Index name changed')
324 new = list(client.ilm.explain_lifecycle(index=name)['indices'].keys())[0]
325 retval = client.ilm.explain_lifecycle(index=new)['indices'][new]
326 except esx.NotFoundError as err:
327 LOGGER.warning('Datastream/Index Name changed. %s was not found', name)
328 raise exc.NameChanged(f'{name} was not found, likely due to a name change') from err
329 except Exception as err:
330 msg = f'Unable to get ILM information for index {name}'
331 LOGGER.critical(msg)
332 raise exc.ResultNotExpected(f'{msg}. Exception: {err}') from err
333 return retval
335def ilm_move(client: Elasticsearch, name: str, current_step: dict, next_step: dict) -> None:
336 """Move index 'name' from the current step to the next step"""
337 try:
338 client.ilm.move_to_step(index=name, current_step=current_step, next_step=next_step)
339 except Exception as err:
340 msg = f'Unable to move index {name} to ILM next step: {next}. Error: {err}'
341 LOGGER.critical(msg)
342 raise exc.ResultNotExpected(msg)
344def put_comp_tmpl(client: Elasticsearch, name: str, component: dict) -> None:
345 """Publish a component template"""
346 try:
347 client.cluster.put_component_template(name=name, template=component, create=True)
348 test = Exists(client, name=name, kind='component', pause=PAUSE_VALUE)
349 test.wait_for_it()
350 except Exception as err:
351 raise exc.TestbedFailure(
352 f'Unable to create component template {name}. Error: {err}') from err
354def put_idx_tmpl(
355 client, name: str, index_patterns: list, components: list,
356 data_stream: dict=None) -> None:
357 """Publish an index template"""
358 try:
359 client.indices.put_index_template(
360 name=name,
361 composed_of=components,
362 data_stream=data_stream,
363 index_patterns=index_patterns,
364 create=True,
365 )
366 test = Exists(client, name=name, kind='template', pause=PAUSE_VALUE)
367 test.wait_for_it()
368 except Exception as err:
369 raise exc.TestbedFailure(
370 f'Unable to create index template {name}. Error: {err}') from err
372def put_ilm(client: Elasticsearch, name: str, policy: dict=None) -> None:
373 """Publish an ILM Policy"""
374 try:
375 client.ilm.put_lifecycle(name=name, policy=policy)
376 except Exception as err:
377 raise exc.TestbedFailure(
378 f'Unable to put index lifecycle policy {name}. Error: {err}') from err
380def resolver(client: Elasticsearch, name: str) -> dict:
381 """
382 Resolve details about the entity, be it an index, alias, or data_stream
384 Because you can pass search patterns and aliases as name, each element comes back as an array:
386 {'indices': [], 'aliases': [], 'data_streams': []}
388 If you only resolve a single index or data stream, you will still have a 1-element list
389 """
390 return client.indices.resolve_index(name=name, expand_wildcards=['open', 'closed'])
392def rollover(client: Elasticsearch, name: str) -> None:
393 """Rollover alias or datastream identified by name"""
394 client.indices.rollover(alias=name, wait_for_active_shards='all')