Coverage for src/es_testbed/classes/ilm.py: 45%
164 statements
« prev ^ index » next coverage.py v7.5.0, created at 2024-04-23 13:32 -0600
« prev ^ index » next coverage.py v7.5.0, created at 2024-04-23 13:32 -0600
1"""ILM Defining Class"""
2import typing as t
3from os import getenv
4from time import sleep
5from elasticsearch8 import Elasticsearch
6from es_testbed.defaults import TESTPLAN, PAUSE_ENVVAR, PAUSE_DEFAULT
7from es_testbed.exceptions import NameChanged, ResultNotExpected, TestbedMisconfig
8from es_testbed.helpers import es_api
9from es_testbed.helpers.utils import build_ilm_policy, getlogger
10from .args import Args
11PAUSE_VALUE = float(getenv(PAUSE_ENVVAR, default=PAUSE_DEFAULT))
13# pylint: disable=missing-docstring
14class IlmBuilder(Args):
15 """Define elements of an ILM policy"""
16 def __init__(
17 self,
18 settings: t.Dict[str, t.Any] = None,
19 defaults: t.Dict[str, t.Any] = None,
20 enabled: t.Optional[bool] = True,
21 ):
22 super().__init__(settings=settings, defaults=defaults)
23 self.logger = getlogger('es_testbed.IlmBuilder')
24 self.enabled = enabled
25 self.tiers = []
26 if enabled: 26 ↛ exitline 26 didn't return from function '__init__', because the condition on line 26 was never false
27 self.logger.debug('ILM Enabled? %s', enabled)
28 self.logger.debug('Apply defaults: %s', TESTPLAN['ilm'])
29 self.set_defaults(TESTPLAN['ilm']) # Set defaults
30 self.logger.debug('Defaults set: %s', self.asdict)
31 if settings: 31 ↛ 32line 31 didn't jump to line 32, because the condition on line 31 was never true
32 self.update_settings(settings) # Update beyond defaults
33 self.logger.debug('Settings passed: %s', self.asdict)
35 @property
36 def enabled(self) -> bool:
37 return self._enabled
38 @enabled.setter
39 def enabled(self, value: bool):
40 self._enabled = value
42 @property
43 def policy(self) -> dict:
44 if self.enabled: 44 ↛ 46line 44 didn't jump to line 46, because the condition on line 44 was never false
45 return build_ilm_policy(**self.asdict)
46 return None
47 @policy.setter
48 def policy(self, value: dict):
49 self._policy = value
51class IlmExplain(Args):
52 """Track Index ilm.explain_lifecycle API results"""
53 def __init__(
54 self,
55 settings: t.Dict[str, t.Any] = None,
56 defaults: t.Dict[str, t.Any] = None,
57 ):
58 self.action = None
59 self.phase = None
60 self.policy = None
61 self.step = None
62 super().__init__(settings=settings, defaults=defaults)
63 self.logger = getlogger('es_testbed.IlmExplain')
64 ### The important keys are:
65 ### [action, index, managed, phase, policy, step]
66 #
67 # kwarg settings should be passed the output of
68 # client.ilm.explain_lifecycle(index=name)['indices'][name]
69 # {
70 # 'action': 'complete',
71 # 'action_time_millis': 0,
72 # 'age': '5.65m',
73 # 'index': 'INDEX_NAME',
74 # 'index_creation_date_millis': 0,
75 # 'lifecycle_date_millis': 0,
76 # 'managed': True,
77 # 'phase': 'hot',
78 # 'phase_execution': {
79 # 'modified_date_in_millis': 0,
80 # 'phase_definition': {
81 # 'actions': {
82 # 'rollover': {
83 # 'max_age': 'MAX_AGE',
84 # 'max_primary_shard_docs': 1000,
85 # 'max_primary_shard_size': 'MAX_SIZE',
86 # 'min_docs': 1
87 # }
88 # },
89 # 'min_age': '0ms'
90 # },
91 # 'policy': 'POLICY_NAME',
92 # 'version': 1
93 # },
94 # 'phase_time_millis': 0,
95 # 'policy': 'POLICY_NAME',
96 # 'step': 'complete',
97 # 'step_time_millis': 0,
98 # 'time_since_index_creation': '5.65m'
99 # }
101class IlmTracker:
102 def __init__(self, client: Elasticsearch, name: str):
103 self.logger = getlogger('es_testbed.IlmTracker')
104 self.client = client
105 self.name = self.resolve(name) # A single index name
106 self._explain = IlmExplain(settings=self.get_explain_data())
107 self._phases = es_api.get_ilm_phases(self.client, self._explain.policy)
108 # IlmExplain is a subclass of Args, which allows us to treat each dictionary key
109 # as an attribute. This means we can access as attributes:
110 # self._explain.
111 # 'action': 'complete',
112 # 'index': 'INDEX_NAME',
113 # 'managed': True,
114 # 'phase': 'hot',
115 # 'policy': 'POLICY_NAME',
116 # 'step': 'complete',
117 # (among others, as needed)
119 @property
120 def current_step(self) -> dict:
121 return {
122 'phase': self._explain.phase,
123 'action': self._explain.action,
124 'name': self._explain.step,
125 }
127 @property
128 def explain(self):
129 return self._explain
131 @property
132 def next_phase(self):
133 retval = None
134 if self._explain.phase == 'delete':
135 self.logger.warning('Already on "delete" phase. No more phases to advance')
136 else:
137 curr = self.pnum(self._explain.phase) # A numeric representation of the current phase
138 # A list of any remaining phases in the policy with a higher number than the current
139 remaining = [self.pnum(x) for x in self.policy_phases if self.pnum(x) > curr]
140 if remaining: # If any:
141 retval = self.pname(remaining[0])
142 # Get the phase name from the number stored in the first element
143 return retval
145 @property
146 def policy_phases(self):
147 return list(self._phases.keys())
149 def advance(self, phase: str=None, action: str=None, name: str=None) -> None:
150 def wait(phase: str) -> None:
151 counter = 0
152 while self._explain.phase != phase:
153 sleep(1.5)
154 self.update()
155 counter += 1
156 self.count_logging(counter)
158 if self._explain.phase == 'delete':
159 self.logger.warning('Already on "delete" phase. No more phases to advance')
160 else:
161 self.logger.debug('current_step: %s', self.current_step)
162 next_step = self.next_step(phase, action=action, name=name)
163 self.logger.debug('next_step: %s', next_step)
164 if next_step:
165 es_api.ilm_move(
166 self.client,
167 self.name,
168 self.current_step,
169 next_step
170 )
171 wait(phase)
172 self.logger.info('Index %s now on phase %s', self.name, phase)
173 else:
174 self.logger.error('next_step is a None value')
175 self.logger.error('current_step: %s', self.current_step)
177 def count_logging(self, counter: int) -> None:
178 # Send a message every 10 loops
179 if counter % 40 == 0:
180 self.logger.info('Still working... Explain: %s', self._explain.asdict)
181 if counter == 480:
182 msg = 'Taking too long! Giving up on waiting'
183 self.logger.critical(msg)
184 raise ResultNotExpected(msg)
186 def get_explain_data(self):
187 try:
188 return es_api.ilm_explain(self.client, self.name)
189 except NameChanged as err:
190 self.logger.debug('Passing along upstream exception...')
191 raise NameChanged from err
192 except ResultNotExpected as err:
193 msg = f'Unable to get ilm_explain API call results. Error: {err}'
194 self.logger.critical(msg)
195 raise ResultNotExpected(msg) from err
197 def next_step(self, phase: str=None, action: str=None, name: str=None) -> dict:
198 err1 = bool((action is not None) and (name is None))
199 err2 = bool((action is None) and (name is not None))
200 if err1 or err2:
201 msg = 'If either action or name is specified, both must be'
202 self.logger.critical(msg)
203 raise TestbedMisconfig(msg)
204 if not phase:
205 phase = self.next_phase
206 retval = {'phase': phase}
207 if action:
208 retval['action'] = action
209 retval['name'] = name
210 return retval
212 def pnum(self, phase: str):
213 _ = {'new': 0, 'hot': 1, 'warm': 2, 'cold': 3, 'frozen': 4, 'delete': 5}
214 return _[phase]
216 def pname(self, num: int):
217 _ = {0: 'new', 1: 'hot', 2: 'warm', 3: 'cold', 4: 'frozen', 5: 'delete'}
218 return _[num]
220 def resolve(self, name: str) -> str:
221 """Resolve that we have an index and NOT an alias or a datastream"""
222 res = es_api.resolver(self.client, name)
223 if len(res['aliases']) > 0 or len(res['data_streams']) > 0: 223 ↛ 224line 223 didn't jump to line 224, because the condition on line 223 was never true
224 msg = f'{name} is not an index: {res}'
225 self.logger.critical(msg)
226 raise ResultNotExpected(msg)
227 if len(res['indices']) > 1: 227 ↛ 228line 227 didn't jump to line 228, because the condition on line 227 was never true
228 msg = f'{name} resolved to multiple indices: {res['indices']}'
229 self.logger.critical(msg)
230 raise ResultNotExpected(msg)
231 return res['indices'][0]['name']
233 def update(self):
234 try:
235 self._explain.update_settings(self.get_explain_data())
236 except NameChanged as err:
237 self.logger.debug('Passing along upstream exception...')
238 raise NameChanged from err
240 def wait4complete(self) -> None:
241 counter = 0
242 self.logger.debug('Waiting for current action and step to complete')
243 self.logger.debug('Action: %s --- Step: %s', self._explain.action, self._explain.step)
244 while not bool(
245 self._explain.action == 'complete' and
246 self._explain.step == 'complete'):
247 counter += 1
248 sleep(PAUSE_VALUE)
249 if counter % 10 == 0:
250 self.logger.debug(
251 'Action: %s --- Step: %s', self._explain.action, self._explain.step)
252 try:
253 self.count_logging(counter)
254 except ResultNotExpected as err:
255 self.logger.critical('Breaking the loop. Explain: %s', self._explain.asdict)
256 raise ResultNotExpected from err
257 try:
258 self.update()
259 except NameChanged as err:
260 self.logger.debug('Passing along upstream exception...')
261 raise NameChanged from err