Coverage for src/es_testbed/classes/ilm.py: 45%

164 statements  

« prev     ^ index     » next       coverage.py v7.5.0, created at 2024-04-23 13:32 -0600

1"""ILM Defining Class""" 

2import typing as t 

3from os import getenv 

4from time import sleep 

5from elasticsearch8 import Elasticsearch 

6from es_testbed.defaults import TESTPLAN, PAUSE_ENVVAR, PAUSE_DEFAULT 

7from es_testbed.exceptions import NameChanged, ResultNotExpected, TestbedMisconfig 

8from es_testbed.helpers import es_api 

9from es_testbed.helpers.utils import build_ilm_policy, getlogger 

10from .args import Args 

11PAUSE_VALUE = float(getenv(PAUSE_ENVVAR, default=PAUSE_DEFAULT)) 

12 

13# pylint: disable=missing-docstring 

14class IlmBuilder(Args): 

15 """Define elements of an ILM policy""" 

16 def __init__( 

17 self, 

18 settings: t.Dict[str, t.Any] = None, 

19 defaults: t.Dict[str, t.Any] = None, 

20 enabled: t.Optional[bool] = True, 

21 ): 

22 super().__init__(settings=settings, defaults=defaults) 

23 self.logger = getlogger('es_testbed.IlmBuilder') 

24 self.enabled = enabled 

25 self.tiers = [] 

26 if enabled: 26 ↛ exitline 26 didn't return from function '__init__', because the condition on line 26 was never false

27 self.logger.debug('ILM Enabled? %s', enabled) 

28 self.logger.debug('Apply defaults: %s', TESTPLAN['ilm']) 

29 self.set_defaults(TESTPLAN['ilm']) # Set defaults 

30 self.logger.debug('Defaults set: %s', self.asdict) 

31 if settings: 31 ↛ 32line 31 didn't jump to line 32, because the condition on line 31 was never true

32 self.update_settings(settings) # Update beyond defaults 

33 self.logger.debug('Settings passed: %s', self.asdict) 

34 

35 @property 

36 def enabled(self) -> bool: 

37 return self._enabled 

38 @enabled.setter 

39 def enabled(self, value: bool): 

40 self._enabled = value 

41 

42 @property 

43 def policy(self) -> dict: 

44 if self.enabled: 44 ↛ 46line 44 didn't jump to line 46, because the condition on line 44 was never false

45 return build_ilm_policy(**self.asdict) 

46 return None 

47 @policy.setter 

48 def policy(self, value: dict): 

49 self._policy = value 

50 

51class IlmExplain(Args): 

52 """Track Index ilm.explain_lifecycle API results""" 

53 def __init__( 

54 self, 

55 settings: t.Dict[str, t.Any] = None, 

56 defaults: t.Dict[str, t.Any] = None, 

57 ): 

58 self.action = None 

59 self.phase = None 

60 self.policy = None 

61 self.step = None 

62 super().__init__(settings=settings, defaults=defaults) 

63 self.logger = getlogger('es_testbed.IlmExplain') 

64 ### The important keys are: 

65 ### [action, index, managed, phase, policy, step] 

66 # 

67 # kwarg settings should be passed the output of  

68 # client.ilm.explain_lifecycle(index=name)['indices'][name] 

69 # { 

70 # 'action': 'complete', 

71 # 'action_time_millis': 0, 

72 # 'age': '5.65m', 

73 # 'index': 'INDEX_NAME', 

74 # 'index_creation_date_millis': 0, 

75 # 'lifecycle_date_millis': 0, 

76 # 'managed': True, 

77 # 'phase': 'hot', 

78 # 'phase_execution': { 

79 # 'modified_date_in_millis': 0, 

80 # 'phase_definition': { 

81 # 'actions': { 

82 # 'rollover': { 

83 # 'max_age': 'MAX_AGE', 

84 # 'max_primary_shard_docs': 1000, 

85 # 'max_primary_shard_size': 'MAX_SIZE', 

86 # 'min_docs': 1 

87 # } 

88 # }, 

89 # 'min_age': '0ms' 

90 # }, 

91 # 'policy': 'POLICY_NAME', 

92 # 'version': 1 

93 # }, 

94 # 'phase_time_millis': 0, 

95 # 'policy': 'POLICY_NAME', 

96 # 'step': 'complete', 

97 # 'step_time_millis': 0, 

98 # 'time_since_index_creation': '5.65m' 

99 # } 

100 

101class IlmTracker: 

102 def __init__(self, client: Elasticsearch, name: str): 

103 self.logger = getlogger('es_testbed.IlmTracker') 

104 self.client = client 

105 self.name = self.resolve(name) # A single index name 

106 self._explain = IlmExplain(settings=self.get_explain_data()) 

107 self._phases = es_api.get_ilm_phases(self.client, self._explain.policy) 

108 # IlmExplain is a subclass of Args, which allows us to treat each dictionary key 

109 # as an attribute. This means we can access as attributes: 

110 # self._explain. 

111 # 'action': 'complete', 

112 # 'index': 'INDEX_NAME', 

113 # 'managed': True, 

114 # 'phase': 'hot', 

115 # 'policy': 'POLICY_NAME', 

116 # 'step': 'complete', 

117 # (among others, as needed) 

118 

119 @property 

120 def current_step(self) -> dict: 

121 return { 

122 'phase': self._explain.phase, 

123 'action': self._explain.action, 

124 'name': self._explain.step, 

125 } 

126 

127 @property 

128 def explain(self): 

129 return self._explain 

130 

131 @property 

132 def next_phase(self): 

133 retval = None 

134 if self._explain.phase == 'delete': 

135 self.logger.warning('Already on "delete" phase. No more phases to advance') 

136 else: 

137 curr = self.pnum(self._explain.phase) # A numeric representation of the current phase 

138 # A list of any remaining phases in the policy with a higher number than the current 

139 remaining = [self.pnum(x) for x in self.policy_phases if self.pnum(x) > curr] 

140 if remaining: # If any: 

141 retval = self.pname(remaining[0]) 

142 # Get the phase name from the number stored in the first element 

143 return retval 

144 

145 @property 

146 def policy_phases(self): 

147 return list(self._phases.keys()) 

148 

149 def advance(self, phase: str=None, action: str=None, name: str=None) -> None: 

150 def wait(phase: str) -> None: 

151 counter = 0 

152 while self._explain.phase != phase: 

153 sleep(1.5) 

154 self.update() 

155 counter += 1 

156 self.count_logging(counter) 

157 

158 if self._explain.phase == 'delete': 

159 self.logger.warning('Already on "delete" phase. No more phases to advance') 

160 else: 

161 self.logger.debug('current_step: %s', self.current_step) 

162 next_step = self.next_step(phase, action=action, name=name) 

163 self.logger.debug('next_step: %s', next_step) 

164 if next_step: 

165 es_api.ilm_move( 

166 self.client, 

167 self.name, 

168 self.current_step, 

169 next_step 

170 ) 

171 wait(phase) 

172 self.logger.info('Index %s now on phase %s', self.name, phase) 

173 else: 

174 self.logger.error('next_step is a None value') 

175 self.logger.error('current_step: %s', self.current_step) 

176 

177 def count_logging(self, counter: int) -> None: 

178 # Send a message every 10 loops 

179 if counter % 40 == 0: 

180 self.logger.info('Still working... Explain: %s', self._explain.asdict) 

181 if counter == 480: 

182 msg = 'Taking too long! Giving up on waiting' 

183 self.logger.critical(msg) 

184 raise ResultNotExpected(msg) 

185 

186 def get_explain_data(self): 

187 try: 

188 return es_api.ilm_explain(self.client, self.name) 

189 except NameChanged as err: 

190 self.logger.debug('Passing along upstream exception...') 

191 raise NameChanged from err 

192 except ResultNotExpected as err: 

193 msg = f'Unable to get ilm_explain API call results. Error: {err}' 

194 self.logger.critical(msg) 

195 raise ResultNotExpected(msg) from err 

196 

197 def next_step(self, phase: str=None, action: str=None, name: str=None) -> dict: 

198 err1 = bool((action is not None) and (name is None)) 

199 err2 = bool((action is None) and (name is not None)) 

200 if err1 or err2: 

201 msg = 'If either action or name is specified, both must be' 

202 self.logger.critical(msg) 

203 raise TestbedMisconfig(msg) 

204 if not phase: 

205 phase = self.next_phase 

206 retval = {'phase': phase} 

207 if action: 

208 retval['action'] = action 

209 retval['name'] = name 

210 return retval 

211 

212 def pnum(self, phase: str): 

213 _ = {'new': 0, 'hot': 1, 'warm': 2, 'cold': 3, 'frozen': 4, 'delete': 5} 

214 return _[phase] 

215 

216 def pname(self, num: int): 

217 _ = {0: 'new', 1: 'hot', 2: 'warm', 3: 'cold', 4: 'frozen', 5: 'delete'} 

218 return _[num] 

219 

220 def resolve(self, name: str) -> str: 

221 """Resolve that we have an index and NOT an alias or a datastream""" 

222 res = es_api.resolver(self.client, name) 

223 if len(res['aliases']) > 0 or len(res['data_streams']) > 0: 223 ↛ 224line 223 didn't jump to line 224, because the condition on line 223 was never true

224 msg = f'{name} is not an index: {res}' 

225 self.logger.critical(msg) 

226 raise ResultNotExpected(msg) 

227 if len(res['indices']) > 1: 227 ↛ 228line 227 didn't jump to line 228, because the condition on line 227 was never true

228 msg = f'{name} resolved to multiple indices: {res['indices']}' 

229 self.logger.critical(msg) 

230 raise ResultNotExpected(msg) 

231 return res['indices'][0]['name'] 

232 

233 def update(self): 

234 try: 

235 self._explain.update_settings(self.get_explain_data()) 

236 except NameChanged as err: 

237 self.logger.debug('Passing along upstream exception...') 

238 raise NameChanged from err 

239 

240 def wait4complete(self) -> None: 

241 counter = 0 

242 self.logger.debug('Waiting for current action and step to complete') 

243 self.logger.debug('Action: %s --- Step: %s', self._explain.action, self._explain.step) 

244 while not bool( 

245 self._explain.action == 'complete' and 

246 self._explain.step == 'complete'): 

247 counter += 1 

248 sleep(PAUSE_VALUE) 

249 if counter % 10 == 0: 

250 self.logger.debug( 

251 'Action: %s --- Step: %s', self._explain.action, self._explain.step) 

252 try: 

253 self.count_logging(counter) 

254 except ResultNotExpected as err: 

255 self.logger.critical('Breaking the loop. Explain: %s', self._explain.asdict) 

256 raise ResultNotExpected from err 

257 try: 

258 self.update() 

259 except NameChanged as err: 

260 self.logger.debug('Passing along upstream exception...') 

261 raise NameChanged from err