Coverage for /Users/buh/.pyenv/versions/3.12.2/envs/pii/lib/python3.12/site-packages/es_pii_tool/redacters/steps.py: 94%

54 statements  

« prev     ^ index     » next       coverage.py v7.5.0, created at 2025-03-17 23:33 -0600

1"""Steps to redact a snapshot mounted index""" 

2 

3import typing as t 

4import logging 

5from dotmap import DotMap # type: ignore 

6from es_pii_tool.trackables import Task 

7from es_pii_tool.helpers import steps as s 

8 

9logger = logging.getLogger(__name__) 

10 

11# pylint: disable=W0718 

12 

13 

14class RedactionSteps: 

15 """All of the redaction steps for the final flow""" 

16 

17 def __init__(self, task: Task, var: DotMap): 

18 self.task = task 

19 self.var = var # These are the variables from RedactSnapshot 

20 self.counter = 1 # Counter will track the step number for us 

21 self.steps: t.Sequence = [] # Steps to execute will be ordered here 

22 self.data = DotMap() 

23 

24 def prep_steps(self): 

25 """Execute the preparatory steps for all indices""" 

26 # PREPARATORY STEPS 

27 # All indices will do these steps 

28 prepsteps = [ 

29 s.resolve_index, # Resolve whether an index or data_stream 

30 s.get_index_lifecycle_data, # Get INDEX lifecycle from settings, if any 

31 s.get_ilm_explain_data, # Get ILM explain data, if any 

32 s.get_ilm_lifecycle_data, # Get ILM lifecycle data, if any 

33 s.clone_ilm_policy, # If an ILM policy exists for index, clone it. 

34 ] 

35 

36 for func in prepsteps: 

37 stepname = f'step{str(self.counter).zfill(2)}_{func.__name__}' 

38 logger.debug('Attempting %s', stepname) 

39 try: 

40 func(self.task, stepname, self.var, data=self.data) 

41 except Exception as exc: 

42 logger.error('Failed to execute %s: %s', stepname, exc) 

43 raise exc 

44 self.counter += 1 

45 

46 def first_steps(self): 

47 """ 

48 Append the first steps to :py:attr:`steps` 

49 """ 

50 self.steps = [ 

51 s.pre_delete, # Force delete var.redaction_target, just in case 

52 s.restore_index, # Restore to var.redaction_target 

53 s.un_ilm_the_restored_index, # Remove ILM from var.redaction_target 

54 s.redact_from_index, # Redact specified docs from var.redaction_target 

55 s.forcemerge_index, # Force merge, if configured to do so 

56 s.clear_cache, # Clear the index cache for var.redaction_target 

57 s.confirm_redaction, # Confirm the docs were redacted 

58 s.snapshot_index, # Snapshot var.redaction_target 

59 s.mount_snapshot, # Mount the snapshotted index as var.mount_name 

60 ] 

61 

62 def ilm_steps(self): 

63 """ 

64 Append ILM specific steps only if there is a new ILM lifecycle name 

65 """ 

66 # After the prep steps, this value should be known 

67 if bool(self.data.new.ilmname): 

68 self.steps.append(s.apply_ilm_policy) # Apply the cloned ILM policy 

69 self.steps.append(s.confirm_ilm_phase) 

70 # Confirm we're in the expected phase and steps are "completed" 

71 

72 def delete_original(self): 

73 """ 

74 Append steps to delete the original index 

75 """ 

76 self.steps.append(s.un_ilm_the_original_index) # Remove ILM as a precaution 

77 self.steps.append(s.close_old_index) # Close it - Also precautionary 

78 self.steps.append(s.delete_old_index) # Delete it 

79 

80 def get_steps(self): 

81 """ 

82 Meta-step to populate :py:attr:`steps` 

83 """ 

84 # Do preparatory steps on all indices 

85 self.prep_steps() 

86 

87 # Set the first steps in self.steps 

88 self.first_steps() 

89 

90 # Add configuration dependent steps 

91 self.ilm_steps() 

92 

93 self.steps.append(s.delete_redaction_target) # Delete var.redaction_target 

94 

95 # After the prep steps, these values should be known 

96 is_data_stream = bool(self.data.data_stream) 

97 

98 # Only if original index was not a data_stream 

99 if not is_data_stream: 

100 self.steps.append(s.fix_aliases) # Collect and fix aliases to apply 

101 

102 # Remove original index 

103 self.delete_original() 

104 

105 # Reassociate as needed 

106 if is_data_stream: 

107 self.steps.append(s.reassociate_index_with_ds) # Reassociate with ds 

108 else: 

109 self.steps.append(s.assign_aliases) # Reassociate with aliases 

110 

111 # Final step 

112 self.steps.append(s.record_it) 

113 

114 def run(self) -> None: 

115 """ 

116 Run the steps in sequence 

117 

118 Step numbers are calculated by :py:attr:`counter`, which makes it easier 

119 to number steps if they are changed or reordered. 

120 """ 

121 self.get_steps() 

122 

123 # Now we finish the steps 

124 for func in self.steps: 

125 stepname = f'step{str(self.counter).zfill(2)}_{func.__name__}' 

126 logger.debug('Attempting %s', stepname) 

127 func(self.task, stepname, self.var, data=self.data) 

128 self.counter += 1