Coverage for /Users/buh/.pyenv/versions/3.12.2/envs/es-testbed/lib/python3.12/site-packages/es_testbed/classes/ilm.py: 67%

130 statements  

« prev     ^ index     » next       coverage.py v7.4.4, created at 2024-04-27 20:59 -0600

1"""ILM Defining Class""" 

2 

3import typing as t 

4from os import getenv 

5from time import sleep 

6from dotmap import DotMap 

7from es_testbed.defaults import PAUSE_ENVVAR, PAUSE_DEFAULT 

8from es_testbed.exceptions import NameChanged, ResultNotExpected, TestbedMisconfig 

9from es_testbed.helpers import es_api 

10from es_testbed.helpers.utils import getlogger 

11 

12if t.TYPE_CHECKING: 12 ↛ 13line 12 didn't jump to line 13, because the condition on line 12 was never true

13 from elasticsearch8 import Elasticsearch 

14 

15PAUSE_VALUE = float(getenv(PAUSE_ENVVAR, default=PAUSE_DEFAULT)) 

16 

17# pylint: disable=missing-docstring 

18 

19# ## Example ILM explain output 

20# { 

21# 'action': 'complete', 

22# 'action_time_millis': 0, 

23# 'age': '5.65m', 

24# 'index': 'INDEX_NAME', 

25# 'index_creation_date_millis': 0, 

26# 'lifecycle_date_millis': 0, 

27# 'managed': True, 

28# 'phase': 'hot', 

29# 'phase_execution': { 

30# 'modified_date_in_millis': 0, 

31# 'phase_definition': { 

32# 'actions': { 

33# 'rollover': { 

34# 'max_age': 'MAX_AGE', 

35# 'max_primary_shard_docs': 1000, 

36# 'max_primary_shard_size': 'MAX_SIZE', 

37# 'min_docs': 1 

38# } 

39# }, 

40# 'min_age': '0ms' 

41# }, 

42# 'policy': 'POLICY_NAME', 

43# 'version': 1 

44# }, 

45# 'phase_time_millis': 0, 

46# 'policy': 'POLICY_NAME', 

47# 'step': 'complete', 

48# 'step_time_millis': 0, 

49# 'time_since_index_creation': '5.65m' 

50# } 

51 

52 

53class IlmTracker: 

54 """ILM Phase Tracking Class""" 

55 

56 def __init__(self, client: 'Elasticsearch', name: str): 

57 self.logger = getlogger('es_testbed.IlmTracker') 

58 self.client = client 

59 self.name = self.resolve(name) # A single index name 

60 self._explain = DotMap(self.get_explain_data()) 

61 self._phases = es_api.get_ilm_phases(self.client, self._explain.policy) 

62 

63 @property 

64 def current_step(self) -> t.Dict: 

65 """Return the current ILM step information""" 

66 return { 

67 'phase': self._explain.phase, 

68 'action': self._explain.action, 

69 'name': self._explain.step, 

70 } 

71 

72 @property 

73 def explain(self) -> DotMap: 

74 """Return the current stored value of ILM Explain""" 

75 return self._explain 

76 

77 @property 

78 def next_phase(self) -> str: 

79 """Return the next phase in the index's ILM journey""" 

80 retval = None 

81 if self._explain.phase == 'delete': 

82 self.logger.warning('Already on "delete" phase. No more phases to advance') 

83 else: 

84 curr = self.pnum(self._explain.phase) # A numeric representation 

85 # A list of any remaining phases in the policy with a higher number than 

86 # the current 

87 remaining = [ 

88 self.pnum(x) for x in self.policy_phases if self.pnum(x) > curr 

89 ] 

90 if remaining: # If any: 

91 retval = self.pname(remaining[0]) 

92 # Get the phase name from the number stored in the first element 

93 return retval 

94 

95 @property 

96 def policy_phases(self) -> t.Sequence[str]: 

97 """Return a list of phases in the ILM policy""" 

98 return list(self._phases.keys()) 

99 

100 def advance( 

101 self, 

102 phase: t.Union[str, None] = None, 

103 action: t.Union[str, None] = None, 

104 name: t.Union[str, None] = None, 

105 ) -> None: 

106 """Advance index to next ILM phase""" 

107 

108 def wait(phase: str) -> None: 

109 """Wait for the phase change""" 

110 counter = 0 

111 sleep(1.5) # Initial wait since we set ILM to poll every second 

112 while self._explain.phase != phase: 

113 sleep(PAUSE_VALUE) 

114 self.update() 

115 counter += 1 

116 self.count_logging(counter) 

117 

118 if self._explain.phase == 'delete': 118 ↛ 119line 118 didn't jump to line 119, because the condition on line 118 was never true

119 self.logger.warning('Already on "delete" phase. No more phases to advance') 

120 else: 

121 self.logger.debug('current_step: %s', self.current_step) 

122 next_step = self.next_step(phase, action=action, name=name) 

123 self.logger.debug('next_step: %s', next_step) 

124 if next_step: 124 ↛ 129line 124 didn't jump to line 129, because the condition on line 124 was never false

125 es_api.ilm_move(self.client, self.name, self.current_step, next_step) 

126 wait(phase) 

127 self.logger.info('Index %s now on phase %s', self.name, phase) 

128 else: 

129 self.logger.error('next_step is a None value') 

130 self.logger.error('current_step: %s', self.current_step) 

131 

132 def count_logging(self, counter: int) -> None: 

133 """Log messages based on how big counter is""" 

134 # Send a message every 10 loops 

135 if counter % 40 == 0: 135 ↛ 136line 135 didn't jump to line 136, because the condition on line 135 was never true

136 self.logger.info('Still working... Explain: %s', self._explain.asdict) 

137 if counter == 480: 137 ↛ 138line 137 didn't jump to line 138, because the condition on line 137 was never true

138 msg = 'Taking too long! Giving up on waiting' 

139 self.logger.critical(msg) 

140 raise ResultNotExpected(msg) 

141 

142 def get_explain_data(self) -> t.Dict: 

143 """Get the ILM explain data and return it""" 

144 try: 

145 return es_api.ilm_explain(self.client, self.name) 

146 except NameChanged as err: 146 ↛ 149line 146 didn't jump to line 149

147 self.logger.debug('Passing along upstream exception...') 

148 raise NameChanged from err 

149 except ResultNotExpected as err: 

150 msg = f'Unable to get ilm_explain API call results. Error: {err}' 

151 self.logger.critical(msg) 

152 raise ResultNotExpected(msg) from err 

153 

154 def next_step( 

155 self, 

156 phase: t.Union[str, None] = None, 

157 action: t.Union[str, None] = None, 

158 name: t.Union[str, None] = None, 

159 ) -> t.Dict: 

160 """Determine the next ILM step based on the current phase, action, and name""" 

161 err1 = bool((action is not None) and (name is None)) 

162 err2 = bool((action is None) and (name is not None)) 

163 if err1 or err2: 163 ↛ 164line 163 didn't jump to line 164, because the condition on line 163 was never true

164 msg = 'If either action or name is specified, both must be' 

165 self.logger.critical(msg) 

166 raise TestbedMisconfig(msg) 

167 if not phase: 167 ↛ 168line 167 didn't jump to line 168, because the condition on line 167 was never true

168 phase = self.next_phase 

169 retval = {'phase': phase} 

170 if action: 170 ↛ 171line 170 didn't jump to line 171, because the condition on line 170 was never true

171 retval['action'] = action 

172 retval['name'] = name 

173 return retval 

174 

175 def pnum(self, phase: str) -> int: 

176 """Map a phase name to a phase number""" 

177 _ = {'new': 0, 'hot': 1, 'warm': 2, 'cold': 3, 'frozen': 4, 'delete': 5} 

178 return _[phase] 

179 

180 def pname(self, num: int) -> str: 

181 """Map a phase number to a phase name""" 

182 _ = {0: 'new', 1: 'hot', 2: 'warm', 3: 'cold', 4: 'frozen', 5: 'delete'} 

183 return _[num] 

184 

185 def resolve(self, name: str) -> str: 

186 """Resolve that we have an index and NOT an alias or a datastream""" 

187 res = es_api.resolver(self.client, name) 

188 if len(res['aliases']) > 0 or len(res['data_streams']) > 0: 188 ↛ 189line 188 didn't jump to line 189, because the condition on line 188 was never true

189 msg = f'{name} is not an index: {res}' 

190 self.logger.critical(msg) 

191 raise ResultNotExpected(msg) 

192 if len(res['indices']) > 1: 192 ↛ 193line 192 didn't jump to line 193, because the condition on line 192 was never true

193 msg = f'{name} resolved to multiple indices: {res["indices"]}' 

194 self.logger.critical(msg) 

195 raise ResultNotExpected(msg) 

196 return res['indices'][0]['name'] 

197 

198 def update(self) -> None: 

199 """Update self._explain with the latest from :py:meth:`get_explain_data`""" 

200 try: 

201 self._explain = DotMap(self.get_explain_data()) 

202 except NameChanged as err: 

203 self.logger.debug('Passing along upstream exception...') 

204 raise NameChanged from err 

205 

206 def wait4complete(self) -> None: 

207 """Wait for the ILM phase change to complete both the action and step""" 

208 counter = 0 

209 self.logger.debug('Waiting for current action and step to complete') 

210 self.logger.debug( 

211 'Action: %s --- Step: %s', self._explain.action, self._explain.step 

212 ) 

213 while not bool( 

214 self._explain.action == 'complete' and self._explain.step == 'complete' 

215 ): 

216 counter += 1 

217 sleep(PAUSE_VALUE) 

218 if counter % 10 == 0: 218 ↛ 219line 218 didn't jump to line 219, because the condition on line 218 was never true

219 self.logger.debug( 

220 'Action: %s --- Step: %s', self._explain.action, self._explain.step 

221 ) 

222 try: 

223 self.count_logging(counter) 

224 except ResultNotExpected as err: 

225 self.logger.critical( 

226 'Breaking the loop. Explain: %s', self._explain.toDict() 

227 ) 

228 raise ResultNotExpected from err 

229 try: 

230 self.update() 

231 except NameChanged as err: 

232 self.logger.debug('Passing along upstream exception...') 

233 raise NameChanged from err