Coverage for /Users/buh/.pyenv/versions/3.12.2/envs/es-testbed/lib/python3.12/site-packages/es_testbed/classes/ilm.py: 68%

129 statements  

« prev     ^ index     » next       coverage.py v7.4.4, created at 2024-04-25 19:21 -0600

1"""ILM Defining Class""" 

2import typing as t 

3from os import getenv 

4from time import sleep 

5from dotmap import DotMap 

6from elasticsearch8 import Elasticsearch 

7from es_testbed.defaults import PAUSE_ENVVAR, PAUSE_DEFAULT 

8from es_testbed.exceptions import NameChanged, ResultNotExpected, TestbedMisconfig 

9from es_testbed.helpers import es_api 

10from es_testbed.helpers.utils import getlogger 

11PAUSE_VALUE = float(getenv(PAUSE_ENVVAR, default=PAUSE_DEFAULT)) 

12 

13# pylint: disable=missing-docstring 

14 

15### Example ILM explain output 

16# { 

17# 'action': 'complete', 

18# 'action_time_millis': 0, 

19# 'age': '5.65m', 

20# 'index': 'INDEX_NAME', 

21# 'index_creation_date_millis': 0, 

22# 'lifecycle_date_millis': 0, 

23# 'managed': True, 

24# 'phase': 'hot', 

25# 'phase_execution': { 

26# 'modified_date_in_millis': 0, 

27# 'phase_definition': { 

28# 'actions': { 

29# 'rollover': { 

30# 'max_age': 'MAX_AGE', 

31# 'max_primary_shard_docs': 1000, 

32# 'max_primary_shard_size': 'MAX_SIZE', 

33# 'min_docs': 1 

34# } 

35# }, 

36# 'min_age': '0ms' 

37# }, 

38# 'policy': 'POLICY_NAME', 

39# 'version': 1 

40# }, 

41# 'phase_time_millis': 0, 

42# 'policy': 'POLICY_NAME', 

43# 'step': 'complete', 

44# 'step_time_millis': 0, 

45# 'time_since_index_creation': '5.65m' 

46# } 

47 

48class IlmTracker: 

49 def __init__(self, client: Elasticsearch, name: str): 

50 self.logger = getlogger('es_testbed.IlmTracker') 

51 self.client = client 

52 self.name = self.resolve(name) # A single index name 

53 self._explain = DotMap(self.get_explain_data()) 

54 self._phases = es_api.get_ilm_phases(self.client, self._explain.policy) 

55 

56 @property 

57 def current_step(self) -> dict: 

58 return { 

59 'phase': self._explain.phase, 

60 'action': self._explain.action, 

61 'name': self._explain.step, 

62 } 

63 

64 @property 

65 def explain(self) -> DotMap: 

66 return self._explain 

67 

68 @property 

69 def next_phase(self) -> str: 

70 retval = None 

71 if self._explain.phase == 'delete': 

72 self.logger.warning('Already on "delete" phase. No more phases to advance') 

73 else: 

74 curr = self.pnum(self._explain.phase) # A numeric representation of the current phase 

75 # A list of any remaining phases in the policy with a higher number than the current 

76 remaining = [self.pnum(x) for x in self.policy_phases if self.pnum(x) > curr] 

77 if remaining: # If any: 

78 retval = self.pname(remaining[0]) 

79 # Get the phase name from the number stored in the first element 

80 return retval 

81 

82 @property 

83 def policy_phases(self) -> t.Sequence[str]: 

84 return list(self._phases.keys()) 

85 

86 def advance(self, phase: str=None, action: str=None, name: str=None) -> None: 

87 def wait(phase: str) -> None: 

88 counter = 0 

89 sleep(1.5) # Initial wait since we set ILM to poll every second 

90 while self._explain.phase != phase: 

91 sleep(PAUSE_VALUE) 

92 self.update() 

93 counter += 1 

94 self.count_logging(counter) 

95 

96 if self._explain.phase == 'delete': 96 ↛ 97line 96 didn't jump to line 97, because the condition on line 96 was never true

97 self.logger.warning('Already on "delete" phase. No more phases to advance') 

98 else: 

99 self.logger.debug('current_step: %s', self.current_step) 

100 next_step = self.next_step(phase, action=action, name=name) 

101 self.logger.debug('next_step: %s', next_step) 

102 if next_step: 102 ↛ 107line 102 didn't jump to line 107, because the condition on line 102 was never false

103 es_api.ilm_move(self.client, self.name, self.current_step, next_step) 

104 wait(phase) 

105 self.logger.info('Index %s now on phase %s', self.name, phase) 

106 else: 

107 self.logger.error('next_step is a None value') 

108 self.logger.error('current_step: %s', self.current_step) 

109 

110 def count_logging(self, counter: int) -> None: 

111 # Send a message every 10 loops 

112 if counter % 40 == 0: 112 ↛ 113line 112 didn't jump to line 113, because the condition on line 112 was never true

113 self.logger.info('Still working... Explain: %s', self._explain.asdict) 

114 if counter == 480: 114 ↛ 115line 114 didn't jump to line 115, because the condition on line 114 was never true

115 msg = 'Taking too long! Giving up on waiting' 

116 self.logger.critical(msg) 

117 raise ResultNotExpected(msg) 

118 

119 def get_explain_data(self) -> t.Dict: 

120 try: 

121 return es_api.ilm_explain(self.client, self.name) 

122 except NameChanged as err: 122 ↛ 125line 122 didn't jump to line 125

123 self.logger.debug('Passing along upstream exception...') 

124 raise NameChanged from err 

125 except ResultNotExpected as err: 

126 msg = f'Unable to get ilm_explain API call results. Error: {err}' 

127 self.logger.critical(msg) 

128 raise ResultNotExpected(msg) from err 

129 

130 def next_step(self, phase: str=None, action: str=None, name: str=None) -> t.Dict: 

131 err1 = bool((action is not None) and (name is None)) 

132 err2 = bool((action is None) and (name is not None)) 

133 if err1 or err2: 133 ↛ 134line 133 didn't jump to line 134, because the condition on line 133 was never true

134 msg = 'If either action or name is specified, both must be' 

135 self.logger.critical(msg) 

136 raise TestbedMisconfig(msg) 

137 if not phase: 137 ↛ 138line 137 didn't jump to line 138, because the condition on line 137 was never true

138 phase = self.next_phase 

139 retval = {'phase': phase} 

140 if action: 140 ↛ 141line 140 didn't jump to line 141, because the condition on line 140 was never true

141 retval['action'] = action 

142 retval['name'] = name 

143 return retval 

144 

145 def pnum(self, phase: str) -> int: 

146 _ = {'new': 0, 'hot': 1, 'warm': 2, 'cold': 3, 'frozen': 4, 'delete': 5} 

147 return _[phase] 

148 

149 def pname(self, num: int) -> str: 

150 _ = {0: 'new', 1: 'hot', 2: 'warm', 3: 'cold', 4: 'frozen', 5: 'delete'} 

151 return _[num] 

152 

153 def resolve(self, name: str) -> str: 

154 """Resolve that we have an index and NOT an alias or a datastream""" 

155 res = es_api.resolver(self.client, name) 

156 if len(res['aliases']) > 0 or len(res['data_streams']) > 0: 156 ↛ 157line 156 didn't jump to line 157, because the condition on line 156 was never true

157 msg = f'{name} is not an index: {res}' 

158 self.logger.critical(msg) 

159 raise ResultNotExpected(msg) 

160 if len(res['indices']) > 1: 160 ↛ 161line 160 didn't jump to line 161, because the condition on line 160 was never true

161 msg = f'{name} resolved to multiple indices: {res['indices']}' 

162 self.logger.critical(msg) 

163 raise ResultNotExpected(msg) 

164 return res['indices'][0]['name'] 

165 

166 def update(self) -> None: 

167 try: 

168 self._explain = DotMap(self.get_explain_data()) 

169 except NameChanged as err: 

170 self.logger.debug('Passing along upstream exception...') 

171 raise NameChanged from err 

172 

173 def wait4complete(self) -> None: 

174 counter = 0 

175 self.logger.debug('Waiting for current action and step to complete') 

176 self.logger.debug('Action: %s --- Step: %s', self._explain.action, self._explain.step) 

177 while not bool( 

178 self._explain.action == 'complete' and 

179 self._explain.step == 'complete'): 

180 counter += 1 

181 sleep(PAUSE_VALUE) 

182 if counter % 10 == 0: 182 ↛ 183line 182 didn't jump to line 183, because the condition on line 182 was never true

183 self.logger.debug( 

184 'Action: %s --- Step: %s', self._explain.action, self._explain.step) 

185 try: 

186 self.count_logging(counter) 

187 except ResultNotExpected as err: 

188 self.logger.critical('Breaking the loop. Explain: %s', self._explain.toDict()) 

189 raise ResultNotExpected from err 

190 try: 

191 self.update() 

192 except NameChanged as err: 

193 self.logger.debug('Passing along upstream exception...') 

194 raise NameChanged from err