Coverage for /Users/buh/.pyenv/versions/3.12.2/envs/pii/lib/python3.12/site-packages/es_pii_tool/helpers/steps.py: 71%
468 statements
« prev ^ index » next coverage.py v7.5.0, created at 2025-03-17 23:33 -0600
« prev ^ index » next coverage.py v7.5.0, created at 2025-03-17 23:33 -0600
1"""Each function is a single step in PII redaction"""
3import typing as t
4import time
5import logging
6from dotmap import DotMap # type: ignore
7from es_wait import IlmPhase, IlmStep
8from es_pii_tool.defaults import PAUSE_DEFAULT
9from es_pii_tool.exceptions import (
10 BadClientResult,
11 FatalError,
12 MissingArgument,
13 MissingError,
14 MissingIndex,
15 ValueMismatch,
16)
17from es_pii_tool.trackables import Step
18from es_pii_tool.helpers import elastic_api as api
19from es_pii_tool.helpers.utils import (
20 configure_ilm_policy,
21 get_alias_actions,
22 strip_ilm_name,
23 es_waiter,
24 timing,
25)
27if t.TYPE_CHECKING:
28 from es_pii_tool.trackables import Task
30logger = logging.getLogger(__name__)
33def failed_step(task: 'Task', step: 'Step', exc):
34 """Function to avoid repetition of code if a step fails"""
35 # MissingIndex, BadClientResult are the only ones inbound
36 if isinstance(exc, MissingIndex):
37 msg = (
38 f'Step failed because index {exc.missing} was not found. The upstream '
39 f'exception type was MissingIndex, with error message: '
40 f'{exc.upstream.args[0]}'
41 )
42 elif isinstance(exc, BadClientResult):
43 msg = (
44 f'Step failed because of a bad or unexpected response or result from '
45 f'the Elasticsearch cluster. The upstream exception type was '
46 f'BadClientResult, with error message: {exc.upstream.args[0]}'
47 )
48 else:
49 msg = f'Step failed for an unexpected reason: {exc}'
50 logger.critical(msg)
51 step.end(False, errors=True, logmsg=f'{msg}')
52 task.end(False, errors=True, logmsg=f'Failed {step.stepname}')
53 raise FatalError(msg, exc)
56def metastep(task: 'Task', stepname: str, func, *args, **kwargs) -> None:
57 """The reusable step"""
58 step = Step(task=task, stepname=stepname)
59 if step.finished():
60 logger.info('%s: already completed', step.stub)
61 return
62 step.begin()
63 dry_run_safe = kwargs.pop('dry_run_safe', False)
64 dry_run_msg = kwargs.pop('dry_run_msg', None)
65 include_step = kwargs.pop('include_step', False)
66 if include_step:
67 kwargs['step'] = step
68 if (dry_run_safe and task.job.dry_run) or not task.job.dry_run:
69 try:
70 response = func(*args, **kwargs)
71 except (MissingIndex, BadClientResult, ValueMismatch) as exc:
72 failed_step(task, step, exc)
73 if response:
74 step.add_log(f'{response}')
75 else:
76 if dry_run_msg is None:
77 dry_run_msg = 'No action logged'
78 msg = f'Dry-Run: No changes, but expected behavior: {dry_run_msg}'
79 step.add_log(msg)
80 logger.debug(msg)
81 step.end(completed=True, errors=False, logmsg=f'{stepname} completed')
84def missing_data(stepname, kwargs) -> None:
85 """Avoid duplicated code for data check"""
86 if 'data' not in kwargs:
87 msg = f'"{stepname}" is missing keyword argument(s)'
88 what = 'type: DotMap'
89 names = ['data']
90 raise MissingArgument(msg, what, names)
93def _meta_resolve_index(var: DotMap, data: DotMap) -> str:
94 """Make a metastep for resolve_index"""
95 result = api.resolve_index(var.client, var.index)
96 logger.debug('resolve data: %s', result)
97 response = ''
98 try:
99 data.data_stream = result['indices'][0]['data_stream']
100 except KeyError:
101 response = f'Index {var.index} is not part of a data_stream'
102 logger.debug(response)
103 return response
106def resolve_index(task: 'Task', stepname: str, var: DotMap, **kwargs) -> None:
107 """
108 Resolve the index to see if it's part of a data stream
109 """
110 missing_data(stepname, kwargs)
111 data = kwargs['data']
112 metastep(task, stepname, _meta_resolve_index, var, data, dry_run_safe=True)
115def _meta_pre_delete(var: DotMap) -> str:
116 """Make a metastep for pre_delete"""
117 response = ''
118 # The metastep will handle the "don't do this if dry_run" logic
119 try:
120 api.delete_index(var.client, var.redaction_target)
121 except MissingIndex:
122 # Not a problem. This is normal and expected.
123 response = f'Pre-delete did not find index "{var.redaction_target}"'
124 logger.debug(response)
125 return response
128def pre_delete(task: 'Task', stepname: str, var: DotMap, **kwargs) -> None:
129 """
130 Pre-delete the redacted index to ensure no collisions. Ignore if not present
131 """
132 missing_data(stepname, kwargs)
133 drm = 'Delete index {var.redaction_target} (if it exists)'
134 metastep(task, stepname, _meta_pre_delete, var, dry_run_msg=drm)
137def _meta_restore_index(var: DotMap) -> str:
138 """Make a metastep for restore_index"""
139 response = f'Restored {var.ss_idx} to {var.redaction_target}'
140 try:
141 api.restore_index(
142 var.client,
143 var.repository,
144 var.ss_snap,
145 var.ss_idx,
146 var.redaction_target,
147 index_settings=var.restore_settings.toDict(),
148 )
149 except BadClientResult as bad:
150 response = f'Unable to restore {var.ss_idx} to {var.redaction_target}: {bad}'
151 logger.error(response)
152 return response
155def restore_index(task: 'Task', stepname, var: DotMap, **kwargs) -> None:
156 """Restore index from snapshot"""
157 missing_data(stepname, kwargs)
158 drm = f'Restore {var.ss_idx} to {var.redaction_target}'
159 metastep(task, stepname, _meta_restore_index, var, dry_run_msg=drm)
162def _meta_get_ilm_data(var: DotMap, data: DotMap) -> str:
163 """Make a metastep for get_index_lifecycle_data"""
164 res = api.get_settings(var.client, var.index)
165 response = ''
166 data.index = DotMap()
167 data.index.lifecycle = DotMap(
168 {'name': None, 'rollover_alias': None, 'indexing_complete': True}
169 )
170 try:
171 data.index.lifecycle = DotMap(res[var.index]['settings']['index']['lifecycle'])
172 except KeyError as err:
173 response = f'Index {var.index} missing one or more lifecycle keys: {err}'
174 if data.index.lifecycle.name:
175 response = f'Index lifecycle settings: {data.index.lifecycle}'
176 else:
177 response = f'Index {var.index} has no ILM lifecycle'
178 logger.debug(response)
179 return response
182def get_index_lifecycle_data(task: 'Task', stepname, var: DotMap, **kwargs) -> None:
183 """
184 Populate data.index with index settings results referenced at
185 INDEXNAME.settings.index.lifecycle
186 """
187 missing_data(stepname, kwargs)
188 data = kwargs['data']
189 metastep(task, stepname, _meta_get_ilm_data, var, data, dry_run_safe=True)
192def _meta_get_ilm_explain_data(var: DotMap, data: DotMap) -> str:
193 """Make a metastep for get_ilm_explain_data"""
194 response = ''
195 if data.index.lifecycle.name:
196 data.ilm = DotMap()
197 try:
198 res = api.get_ilm(var.client, var.index)
199 data.ilm.explain = DotMap(res['indices'][var.index])
200 response = f'ILM explain settings: {data.ilm.explain}'
201 except MissingIndex as exc:
202 logger.error('Index %s not found in ILM explain data', var.index)
203 raise exc
204 else:
205 response = f'Index {var.index} has no ILM explain data'
206 logger.debug(response)
207 return response
210def get_ilm_explain_data(task: 'Task', stepname, var: DotMap, **kwargs) -> None:
211 """
212 Populate data.ilm.explain with ilm_explain data
213 """
214 missing_data(stepname, kwargs)
215 data = kwargs['data']
216 metastep(task, stepname, _meta_get_ilm_explain_data, var, data, dry_run_safe=True)
219def _meta_get_ilm_lifecycle_data(var: DotMap, data: DotMap) -> str:
220 """Make a metastep for get_ilm_lifecycle_data"""
221 response = ''
222 if data.index.lifecycle.name:
223 res = api.get_ilm_lifecycle(var.client, data.index.lifecycle.name)
224 if not res:
225 msg = f'No such ILM policy: {data.index.lifecycle.name}'
226 raise BadClientResult(msg, Exception())
227 data.ilm.lifecycle = DotMap(res[data.index.lifecycle.name])
228 response = f'ILM lifecycle settings: {data.ilm.lifecycle}'
229 else:
230 response = f'Index {var.index} has no ILM lifecycle data'
231 logger.debug(response)
232 return response
235def get_ilm_lifecycle_data(task: 'Task', stepname, var: DotMap, **kwargs) -> None:
236 """
237 Populate data.ilm.explain with ilm_explain data
238 """
239 missing_data(stepname, kwargs)
240 data = kwargs['data']
241 metastep(task, stepname, _meta_get_ilm_lifecycle_data, var, data, dry_run_safe=True)
244def clone_ilm_policy(task: 'Task', stepname, var: DotMap, **kwargs) -> None:
245 """
246 If this index has an ILM policy, we need to clone it so we can attach
247 the new index to it.
248 """
249 missing_data(stepname, kwargs)
250 data = kwargs['data']
251 step = Step(task=task, stepname=stepname)
252 if step.finished():
253 logger.info('%s: already completed', step.stub)
254 return
255 step.begin()
256 if data.index.lifecycle.name is None or not data.ilm.lifecycle.policy:
257 _ = f'{stepname}: Index {var.index} has no ILM lifecycle or policy data'
258 logger.debug(_)
259 step.add_log(_)
260 return
261 data.new = DotMap()
263 # From here, we check for matching named cloned policy
265 configure_ilm_policy(task, data)
267 # New ILM policy naming: pii-tool-POLICYNAME---v###
268 stub = f'pii-tool-{strip_ilm_name(data.index.lifecycle.name)}'
269 policy = data.new.ilmpolicy.toDict() # For comparison
270 resp = {'dummy': 'startval'} # So the while loop can start with something
271 policyver = 0 # Our version number starting point.
272 policymatch = False
273 while resp:
274 data.new.ilmname = f'{stub}---v{policyver + 1:03}'
275 resp = api.get_ilm_lifecycle(var.client, data.new.ilmname) # type: ignore
276 if resp: # We have data, so the name matches
277 # Compare the new policy to the one just returned
278 if policy == resp[data.new.ilmname]['policy']: # type: ignore
279 msg = f'New policy data matches: {data.new.ilmname}'
280 logger.debug(msg)
281 step.add_log(msg)
282 policymatch = True
283 break # We can drop out of the loop here.
284 # Implied else: resp has no value, so the while loop will end.
285 policyver += 1
286 msg = f'New ILM policy name (may already exist): {data.new.ilmname}'
287 logger.debug(msg)
288 step.add_log(msg)
289 if not task.job.dry_run: # Don't create if dry_run
290 if not policymatch:
291 # Create the cloned ILM policy
292 try:
293 gkw = {'name': data.new.ilmname, 'policy': policy}
294 api.generic_get(var.client.ilm.put_lifecycle, **gkw)
295 except (MissingError, BadClientResult) as exc:
296 _ = f'Unable to put new ILM policy: {exc}'
297 logger.error(_)
298 step.add_log(_)
299 failed_step(task, step, exc)
300 # Implied else: We've arrived at the expected new ILM name
301 # and it does match an existing policy in name and content
302 # so we don't need to create a new one.
303 else:
304 _ = (
305 f'Dry-Run: No changes, but expected behavior: '
306 f'ILM policy {data.new.ilmname} created'
307 )
308 logger.debug(_)
309 step.add_log(_)
310 step.end(completed=True, errors=False, logmsg=f'{stepname} completed')
313def un_ilm_the_restored_index(task: 'Task', stepname, var: DotMap, **kwargs) -> None:
314 """Remove the lifecycle data from the settings of the restored index"""
315 missing_data(stepname, kwargs)
316 drm = f'Any existing ILM policy removed from {var.redaction_target}'
317 metastep(
318 task,
319 stepname,
320 api.remove_ilm_policy,
321 var.client,
322 var.redaction_target,
323 dry_run_msg=drm,
324 )
327def redact_from_index(task: 'Task', stepname, var: DotMap, **kwargs) -> None:
328 """Run update by query on new restored index"""
329 missing_data(stepname, kwargs)
330 drm = (
331 f'Redact index {var.redaction_target} replacing content of fields: '
332 f'{task.job.config["fields"]} with message: {task.job.config["message"]}'
333 )
334 metastep(
335 task,
336 stepname,
337 api.redact_from_index,
338 var.client,
339 var.redaction_target,
340 task.job.config,
341 dry_run_msg=drm,
342 )
345def _meta_forcemerge_index(task: 'Task', var: DotMap, **kwargs) -> str:
346 """Do some task logging around the forcemerge api call"""
347 step = kwargs.pop('step', None)
348 if step is None:
349 raise MissingArgument('_meta_forcemerge_index', 'keyword argument', 'step')
350 index = var.redaction_target
351 msg = f'Before forcemerge, {api.report_segment_count(var.client, index)}'
352 logger.info(msg)
353 step.add_log(msg)
354 fmkwargs = {}
355 if 'forcemerge' in task.job.config:
356 fmkwargs = task.job.config['forcemerge']
357 fmkwargs['index'] = index
358 if 'only_expunge_deletes' in fmkwargs and fmkwargs['only_expunge_deletes']:
359 msg = 'Forcemerge will only expunge deleted docs!'
360 logger.info(msg)
361 step.add_log(msg)
362 else:
363 mns = 1 # default value
364 if 'max_num_segments' in fmkwargs and isinstance(
365 fmkwargs['max_num_segments'], int
366 ):
367 mns = fmkwargs['max_num_segments']
368 msg = f'Proceeding to forcemerge to {mns} segments per shard'
369 logger.info(msg)
370 step.add_log(msg)
371 logger.debug('forcemerge kwargs = %s', fmkwargs)
372 # Do the actual forcemerging
373 api.forcemerge_index(var.client, **fmkwargs)
374 msg = f'After forcemerge, {api.report_segment_count(var.client, index)}'
375 logger.info(msg)
376 step.add_log(msg)
377 return msg
380def forcemerge_index(task: 'Task', stepname, var: DotMap, **kwargs) -> None:
381 """Force merge redacted index"""
382 missing_data(stepname, kwargs)
383 msg = ''
384 fmkwargs = {}
385 if 'forcemerge' in task.job.config:
386 fmkwargs = task.job.config['forcemerge']
387 if 'only_expunge_deletes' in fmkwargs and fmkwargs['only_expunge_deletes']:
388 msg = 'only expunging deleted docs'
389 else:
390 mns = 1 # default value
391 if 'max_num_segments' in fmkwargs and isinstance(
392 fmkwargs['max_num_segments'], int
393 ):
394 mns = fmkwargs['max_num_segments']
395 msg = f'to {mns} segments per shard'
396 drm = f'Forcemerge index {var.redaction_target} {msg}'
397 metastep(
398 task,
399 stepname,
400 _meta_forcemerge_index,
401 task,
402 var,
403 include_step=True,
404 dry_run_msg=drm,
405 )
408def clear_cache(task: 'Task', stepname, var: DotMap, **kwargs) -> None:
409 """Clear cache of redacted index"""
410 missing_data(stepname, kwargs)
411 drm = f'Clear cache of index {var.redaction_target}'
412 metastep(
413 task,
414 stepname,
415 api.clear_cache,
416 var.client,
417 var.redaction_target,
418 dry_run_msg=drm,
419 )
422def confirm_redaction(task: 'Task', stepname, var: DotMap, **kwargs) -> None:
423 """Check update by query did its job"""
424 missing_data(stepname, kwargs)
425 drm = f'Confirm redaction of index {var.redaction_target}'
426 metastep(
427 task,
428 stepname,
429 api.check_index,
430 var.client,
431 var.redaction_target,
432 task.job.config,
433 dry_run_msg=drm,
434 )
437def snapshot_index(task: 'Task', stepname, var: DotMap, **kwargs) -> None:
438 """Create a new snapshot for mounting our redacted index"""
439 missing_data(stepname, kwargs)
440 drm = f'Snapshot index {var.redaction_target} to {var.new_snap_name}'
441 metastep(
442 task,
443 stepname,
444 api.take_snapshot,
445 var.client,
446 var.repository,
447 var.new_snap_name,
448 var.redaction_target,
449 dry_run_msg=drm,
450 )
453def mount_snapshot(task: 'Task', stepname, var: DotMap, **kwargs) -> None:
454 """
455 Mount the index as a searchable snapshot to make the redacted index available
456 """
457 missing_data(stepname, kwargs)
458 drm = (
459 f'Mount index {var.redaction_target} in snapshot '
460 f'{var.new_snap_name} as {var.mount_name}'
461 )
462 metastep(task, stepname, api.mount_index, var, dry_run_msg=drm)
465def apply_ilm_policy(task: 'Task', stepname, var: DotMap, **kwargs) -> None:
466 """
467 If the index was associated with an ILM policy, associate it with the
468 new, cloned ILM policy.
469 """
470 missing_data(stepname, kwargs)
471 data = kwargs['data']
472 if data.new.ilmname:
473 settings = {'index': {}} # type: ignore
474 # Add all of the original lifecycle settings
475 settings['index']['lifecycle'] = data.index.lifecycle.toDict()
476 # Replace the name with the new ILM policy name
477 settings['index']['lifecycle']['name'] = data.new.ilmname
478 drm = f'Apply new ILM policy {data.new.ilmname} to {var.mount_name}'
479 metastep(
480 task,
481 stepname,
482 api.put_settings,
483 var.client,
484 var.mount_name,
485 settings,
486 dry_run_msg=drm,
487 )
490def confirm_ilm_phase(task: 'Task', stepname, var: DotMap, **kwargs) -> None:
491 """
492 Confirm the mounted index is in the expected ILM phase
493 This is done by using move_to_step. If it's already in the step, no problem.
494 If it's in step ``new``, this will advance the index to the expected step.
495 """
496 missing_data(stepname, kwargs)
497 step = Step(task=task, stepname=stepname)
498 if step.finished():
499 logger.info('%s: already completed', step.stub)
500 return
501 step.begin()
502 if task.job.dry_run:
503 msg = f'Dry-Run: {var.mount_name} moved to ILM phase {var.phase}'
504 logger.debug(msg)
505 step.add_log(msg)
506 step.end(completed=True, errors=False, logmsg=f'{stepname} completed')
507 return
508 # Wait for phase to be "new"
509 pause, timeout = timing('ilm')
510 logger.debug(f'ENV pause = {pause}, timeout = {timeout}')
511 try:
512 # Update in es_wait 0.9.2:
513 # - If you send phase='new', it will wait for the phase to be 'new' or higher
514 # - This is where a user was getting stuck. They were waiting for 'new' but
515 # - the phase was already 'frozen', so it was endlessly checking for 'new'.
516 es_waiter(
517 var.client,
518 IlmPhase,
519 name=var.mount_name,
520 phase='new',
521 pause=pause,
522 timeout=timeout,
523 )
524 # Wait for step to be "complete"
525 es_waiter(
526 var.client, IlmStep, name=var.mount_name, pause=pause, timeout=timeout
527 )
528 except BadClientResult as bad:
529 _ = f'ILM step confirmation problem -- ERROR: {bad}'
530 logger.error(_)
531 step.add_log(_)
532 failed_step(task, step, bad)
534 def get_currstep():
535 try:
536 _ = api.generic_get(var.client.ilm.explain_lifecycle, index=var.mount_name)
537 except MissingError as exc:
538 _ = f'Unable to get ILM phase of {var.mount_name}'
539 logger.error(_)
540 step.add_log(_)
541 failed_step(task, step, exc)
542 try:
543 expl = _['indices'][var.mount_name]
544 except KeyError as err:
545 msg = f'{var.mount_name} not found in ILM explain data: {err}'
546 logger.error(msg)
547 step.add_log(msg)
548 failed_step(task, step, err)
549 if 'managed' not in expl:
550 msg = f'Index {var.mount_name} is not managed by ILM'
551 step.add_log(msg)
552 failed_step(
553 task, step, ValueMismatch(msg, expl['managed'], '{"managed": True}')
554 )
555 return {'phase': expl['phase'], 'action': expl['action'], 'name': expl['step']}
557 nextstep = {'phase': var.phase, 'action': 'complete', 'name': 'complete'}
558 if task.job.dry_run: # Don't actually move_to_step if dry_run
559 msg = (
560 f'{stepname}: Dry-Run: {var.mount_name} not moved/confirmed to ILM '
561 f'phase {var.phase}'
562 )
563 logger.debug(msg)
564 step.end(completed=True, errors=False, logmsg=f'{stepname} completed')
565 return
567 # We will try to move the index to the expected phase up to 3 times
568 # before failing the step.
569 attempts = 0
570 success = False
571 while attempts < 3 and not success:
572 # Since we are now testing for 'new' or higher, we may not need to advance
573 # ILM phases. If the current step is already where we expect to be, log
574 # confirmation and move on.
575 logger.debug('Attempt number: %s', attempts)
576 currstep = get_currstep()
577 if currstep == nextstep:
578 msg = (
579 f'{stepname}: {var.mount_name} is confirmed to be in ILM phase '
580 f'{var.phase}'
581 )
582 logger.debug(msg)
583 step.add_log(msg)
584 # Set both while loop critera to values that will end the loop
585 success = True
586 attempts = 3
587 else:
588 # If we are not yet in the expected target phase, then proceed with the
589 # ILM phase change.
590 logger.debug('Current ILM Phase: %s', currstep)
591 logger.debug('Target ILM Phase: %s', nextstep)
592 logger.debug('PHASE: %s', var.phase)
593 try:
594 api.ilm_move(var.client, var.mount_name, currstep, nextstep)
595 success = True
596 except BadClientResult as bad:
597 logger.debug('Attempt failed. Incrementing attempts.')
598 attempts += 1
599 if attempts == 3:
600 _ = 'Attempt limit reached. Failing step.'
601 logger.error(_)
602 step.add_log(_)
603 failed_step(task, step, bad)
604 logger.debug('Waiting %s seconds before retrying...', PAUSE_DEFAULT)
605 time.sleep(float(PAUSE_DEFAULT))
606 logger.warning('ILM move failed: %s -- Retrying...', bad.message)
607 continue
608 pause, timeout = timing('ilm')
609 logger.debug(f'ENV pause = {pause}, timeout = {timeout}')
610 try:
611 es_waiter(
612 var.client,
613 IlmPhase,
614 name=var.mount_name,
615 phase=var.phase,
616 pause=pause,
617 timeout=timeout,
618 )
619 es_waiter(
620 var.client,
621 IlmStep,
622 name=var.mount_name,
623 pause=pause,
624 timeout=timeout,
625 )
626 except BadClientResult as phase_err:
627 msg = f'Unable to wait for ILM step to complete -- ERROR: {phase_err}'
628 logger.error(msg)
629 step.add_log(msg)
630 failed_step(task, step, phase_err)
631 # If we make it here, we have successfully moved the index to the expected phase
632 step.end(completed=True, errors=False, logmsg=f'{stepname} completed')
635def delete_redaction_target(task: 'Task', stepname, var: DotMap, **kwargs) -> None:
636 """
637 Now that it's mounted (with a new name), we should delete the redaction_target
638 index
639 """
640 missing_data(stepname, kwargs)
641 drm = f'Delete redaction target index {var.redaction_target}'
642 metastep(
643 task,
644 stepname,
645 api.delete_index,
646 var.client,
647 var.redaction_target,
648 dry_run_msg=drm,
649 )
652def fix_aliases(task: 'Task', stepname, var: DotMap, **kwargs) -> None:
653 """Using the aliases collected from var.index, update mount_name and verify"""
654 missing_data(stepname, kwargs)
655 data = kwargs['data']
656 step = Step(task=task, stepname=stepname)
657 if step.finished():
658 logger.info('%s: already completed', step.stub)
659 return
660 step.begin()
661 if data.data_stream:
662 msg = 'Cannot apply aliases to indices in data_stream'
663 logger.debug(msg)
664 step.add_log(msg)
665 step.end(completed=True, errors=False, logmsg=f'{stepname} completed')
666 return
667 alias_names = var.aliases.toDict().keys()
668 if not alias_names:
669 msg = f'No aliases associated with index {var.index}'
670 step.add_log(msg)
671 logger.info(msg)
672 elif not task.job.dry_run:
673 msg = f'Transferring aliases to new index ' f'{var.mount_name}'
674 logger.debug(msg)
675 step.add_log(msg)
676 var.client.indices.update_aliases(
677 actions=get_alias_actions(var.index, var.mount_name, var.aliases.toDict())
678 )
679 verify = var.client.indices.get(index=var.mount_name)[var.mount_name][
680 'aliases'
681 ].keys()
682 if alias_names != verify:
683 msg = f'Alias names do not match! {alias_names} does not match: {verify}'
684 logger.critical(msg)
685 step.add_log(msg)
686 failed_step(
687 task, step, ValueMismatch(msg, 'alias names mismatch', alias_names)
688 )
689 else:
690 msg = 'Dry-Run: alias transfer not executed'
691 logger.debug(msg)
692 step.add_log(msg)
693 step.end(completed=True, errors=False, logmsg=f'{stepname} completed')
696def un_ilm_the_original_index(task: 'Task', stepname, var: DotMap, **kwargs) -> None:
697 """
698 Remove the lifecycle data from the settings of the original index
700 This is chiefly done as a safety measure.
701 """
702 missing_data(stepname, kwargs)
703 metastep(task, stepname, api.remove_ilm_policy, var.client, var.index)
706def close_old_index(task: 'Task', stepname, var: DotMap, **kwargs) -> None:
707 """Close old mounted snapshot"""
708 missing_data(stepname, kwargs)
709 metastep(task, stepname, api.close_index, var.client, var.index)
712def delete_old_index(task: 'Task', stepname, var: DotMap, **kwargs) -> None:
713 """Delete old mounted snapshot, if configured to do so"""
714 missing_data(stepname, kwargs)
715 step = Step(task=task, stepname=stepname)
716 if step.finished():
717 logger.info('%s: already completed', step.stub)
718 return
719 step.begin()
720 if task.job.config['delete']:
721 msg = f'Deleting original mounted index: {var.index}'
722 task.add_log(msg)
723 logger.info(msg)
724 try:
725 api.delete_index(var.client, var.index)
726 except MissingIndex as miss:
727 msg = f'Index {var.index} not found for deletion: {miss}'
728 logger.error(msg)
729 step.add_log(msg)
730 except BadClientResult as bad:
731 msg = f'Bad client result: {bad}'
732 logger.error(msg)
733 step.add_log(msg)
734 failed_step(task, step, bad)
735 else:
736 msg = (
737 f'delete set to False — not deleting original mounted index: '
738 f'{var.index}'
739 )
740 task.add_log(msg)
741 logger.warning(msg)
742 step.end(completed=True, errors=False, logmsg=f'{stepname} completed')
745def assign_aliases(task: 'Task', stepname, var: DotMap, **kwargs) -> None:
746 """Put the starting index name on new mounted index as alias"""
747 missing_data(stepname, kwargs)
748 data = kwargs['data']
749 step = Step(task=task, stepname=stepname)
750 if step.finished():
751 logger.info('%s: already completed', step.stub)
752 return
753 step.begin()
754 if data.data_stream:
755 msg = 'Cannot apply aliases to indices in data_stream'
756 logger.debug(msg)
757 step.add_log(msg)
758 step.end(completed=True, errors=False, logmsg=f'{stepname} completed')
759 return
760 if not task.job.dry_run:
761 msg = f'Assigning aliases {var.index} to index {var.mount_name}'
762 logger.debug(msg)
763 step.add_log(msg)
764 try:
765 api.assign_alias(var.client, var.mount_name, var.index)
766 except BadClientResult as bad:
767 failed_step(task, step, bad)
768 else:
769 msg = f'Assigning aliases {var.index} to index {var.mount_name}'
770 _ = f'Dry-Run: No changes, but expected behavior: {msg}'
771 logger.debug(_)
772 step.add_log(_)
773 step.end(completed=True, errors=False, logmsg=f'{stepname} completed')
776def reassociate_index_with_ds(task: 'Task', stepname, var: DotMap, **kwargs) -> None:
777 """
778 If the index was associated with a data_stream, reassociate it with the
779 data_stream again.
780 """
781 missing_data(stepname, kwargs)
782 data = kwargs['data']
783 acts = [{'add_backing_index': {'index': var.mount_name}}]
784 if data.data_stream:
785 acts[0]['add_backing_index']['data_stream'] = data.data_stream
786 logger.debug('%s: Modify data_stream actions: %s', stepname, acts)
787 drm = f'Reassociate index {var.mount_name} with data_stream {data.data_stream}'
788 metastep(
789 task, stepname, api.modify_data_stream, var.client, acts, dry_run_msg=drm
790 )
793def _meta_record_it(task: 'Task', snapname: str) -> str:
794 """Make a metastep for record_it"""
795 task.job.cleanup.append(snapname)
796 return f'Snapshot {snapname} added to cleanup list'
799def record_it(task: 'Task', stepname, var: DotMap, **kwargs) -> None:
800 """Record the now-deletable snapshot in the job's tracking index."""
801 missing_data(stepname, kwargs)
802 drm = f'Snapshot {var.ss_snap} added to cleanup list'
803 metastep(task, stepname, _meta_record_it, task, var.ss_snap, dry_run_msg=drm)