Coverage for /Users/buh/.pyenv/versions/3.12.2/envs/es-testbed/lib/python3.12/site-packages/es_testbed/classes/ilm.py: 67%
130 statements
« prev ^ index » next coverage.py v7.4.4, created at 2024-04-27 20:59 -0600
« prev ^ index » next coverage.py v7.4.4, created at 2024-04-27 20:59 -0600
1"""ILM Defining Class"""
3import typing as t
4from os import getenv
5from time import sleep
6from dotmap import DotMap
7from es_testbed.defaults import PAUSE_ENVVAR, PAUSE_DEFAULT
8from es_testbed.exceptions import NameChanged, ResultNotExpected, TestbedMisconfig
9from es_testbed.helpers import es_api
10from es_testbed.helpers.utils import getlogger
12if t.TYPE_CHECKING: 12 ↛ 13line 12 didn't jump to line 13, because the condition on line 12 was never true
13 from elasticsearch8 import Elasticsearch
15PAUSE_VALUE = float(getenv(PAUSE_ENVVAR, default=PAUSE_DEFAULT))
17# pylint: disable=missing-docstring
19# ## Example ILM explain output
20# {
21# 'action': 'complete',
22# 'action_time_millis': 0,
23# 'age': '5.65m',
24# 'index': 'INDEX_NAME',
25# 'index_creation_date_millis': 0,
26# 'lifecycle_date_millis': 0,
27# 'managed': True,
28# 'phase': 'hot',
29# 'phase_execution': {
30# 'modified_date_in_millis': 0,
31# 'phase_definition': {
32# 'actions': {
33# 'rollover': {
34# 'max_age': 'MAX_AGE',
35# 'max_primary_shard_docs': 1000,
36# 'max_primary_shard_size': 'MAX_SIZE',
37# 'min_docs': 1
38# }
39# },
40# 'min_age': '0ms'
41# },
42# 'policy': 'POLICY_NAME',
43# 'version': 1
44# },
45# 'phase_time_millis': 0,
46# 'policy': 'POLICY_NAME',
47# 'step': 'complete',
48# 'step_time_millis': 0,
49# 'time_since_index_creation': '5.65m'
50# }
53class IlmTracker:
54 """ILM Phase Tracking Class"""
56 def __init__(self, client: 'Elasticsearch', name: str):
57 self.logger = getlogger('es_testbed.IlmTracker')
58 self.client = client
59 self.name = self.resolve(name) # A single index name
60 self._explain = DotMap(self.get_explain_data())
61 self._phases = es_api.get_ilm_phases(self.client, self._explain.policy)
63 @property
64 def current_step(self) -> t.Dict:
65 """Return the current ILM step information"""
66 return {
67 'phase': self._explain.phase,
68 'action': self._explain.action,
69 'name': self._explain.step,
70 }
72 @property
73 def explain(self) -> DotMap:
74 """Return the current stored value of ILM Explain"""
75 return self._explain
77 @property
78 def next_phase(self) -> str:
79 """Return the next phase in the index's ILM journey"""
80 retval = None
81 if self._explain.phase == 'delete':
82 self.logger.warning('Already on "delete" phase. No more phases to advance')
83 else:
84 curr = self.pnum(self._explain.phase) # A numeric representation
85 # A list of any remaining phases in the policy with a higher number than
86 # the current
87 remaining = [
88 self.pnum(x) for x in self.policy_phases if self.pnum(x) > curr
89 ]
90 if remaining: # If any:
91 retval = self.pname(remaining[0])
92 # Get the phase name from the number stored in the first element
93 return retval
95 @property
96 def policy_phases(self) -> t.Sequence[str]:
97 """Return a list of phases in the ILM policy"""
98 return list(self._phases.keys())
100 def advance(
101 self,
102 phase: t.Union[str, None] = None,
103 action: t.Union[str, None] = None,
104 name: t.Union[str, None] = None,
105 ) -> None:
106 """Advance index to next ILM phase"""
108 def wait(phase: str) -> None:
109 """Wait for the phase change"""
110 counter = 0
111 sleep(1.5) # Initial wait since we set ILM to poll every second
112 while self._explain.phase != phase:
113 sleep(PAUSE_VALUE)
114 self.update()
115 counter += 1
116 self.count_logging(counter)
118 if self._explain.phase == 'delete': 118 ↛ 119line 118 didn't jump to line 119, because the condition on line 118 was never true
119 self.logger.warning('Already on "delete" phase. No more phases to advance')
120 else:
121 self.logger.debug('current_step: %s', self.current_step)
122 next_step = self.next_step(phase, action=action, name=name)
123 self.logger.debug('next_step: %s', next_step)
124 if next_step: 124 ↛ 129line 124 didn't jump to line 129, because the condition on line 124 was never false
125 es_api.ilm_move(self.client, self.name, self.current_step, next_step)
126 wait(phase)
127 self.logger.info('Index %s now on phase %s', self.name, phase)
128 else:
129 self.logger.error('next_step is a None value')
130 self.logger.error('current_step: %s', self.current_step)
132 def count_logging(self, counter: int) -> None:
133 """Log messages based on how big counter is"""
134 # Send a message every 10 loops
135 if counter % 40 == 0: 135 ↛ 136line 135 didn't jump to line 136, because the condition on line 135 was never true
136 self.logger.info('Still working... Explain: %s', self._explain.asdict)
137 if counter == 480: 137 ↛ 138line 137 didn't jump to line 138, because the condition on line 137 was never true
138 msg = 'Taking too long! Giving up on waiting'
139 self.logger.critical(msg)
140 raise ResultNotExpected(msg)
142 def get_explain_data(self) -> t.Dict:
143 """Get the ILM explain data and return it"""
144 try:
145 return es_api.ilm_explain(self.client, self.name)
146 except NameChanged as err: 146 ↛ 149line 146 didn't jump to line 149
147 self.logger.debug('Passing along upstream exception...')
148 raise NameChanged from err
149 except ResultNotExpected as err:
150 msg = f'Unable to get ilm_explain API call results. Error: {err}'
151 self.logger.critical(msg)
152 raise ResultNotExpected(msg) from err
154 def next_step(
155 self,
156 phase: t.Union[str, None] = None,
157 action: t.Union[str, None] = None,
158 name: t.Union[str, None] = None,
159 ) -> t.Dict:
160 """Determine the next ILM step based on the current phase, action, and name"""
161 err1 = bool((action is not None) and (name is None))
162 err2 = bool((action is None) and (name is not None))
163 if err1 or err2: 163 ↛ 164line 163 didn't jump to line 164, because the condition on line 163 was never true
164 msg = 'If either action or name is specified, both must be'
165 self.logger.critical(msg)
166 raise TestbedMisconfig(msg)
167 if not phase: 167 ↛ 168line 167 didn't jump to line 168, because the condition on line 167 was never true
168 phase = self.next_phase
169 retval = {'phase': phase}
170 if action: 170 ↛ 171line 170 didn't jump to line 171, because the condition on line 170 was never true
171 retval['action'] = action
172 retval['name'] = name
173 return retval
175 def pnum(self, phase: str) -> int:
176 """Map a phase name to a phase number"""
177 _ = {'new': 0, 'hot': 1, 'warm': 2, 'cold': 3, 'frozen': 4, 'delete': 5}
178 return _[phase]
180 def pname(self, num: int) -> str:
181 """Map a phase number to a phase name"""
182 _ = {0: 'new', 1: 'hot', 2: 'warm', 3: 'cold', 4: 'frozen', 5: 'delete'}
183 return _[num]
185 def resolve(self, name: str) -> str:
186 """Resolve that we have an index and NOT an alias or a datastream"""
187 res = es_api.resolver(self.client, name)
188 if len(res['aliases']) > 0 or len(res['data_streams']) > 0: 188 ↛ 189line 188 didn't jump to line 189, because the condition on line 188 was never true
189 msg = f'{name} is not an index: {res}'
190 self.logger.critical(msg)
191 raise ResultNotExpected(msg)
192 if len(res['indices']) > 1: 192 ↛ 193line 192 didn't jump to line 193, because the condition on line 192 was never true
193 msg = f'{name} resolved to multiple indices: {res["indices"]}'
194 self.logger.critical(msg)
195 raise ResultNotExpected(msg)
196 return res['indices'][0]['name']
198 def update(self) -> None:
199 """Update self._explain with the latest from :py:meth:`get_explain_data`"""
200 try:
201 self._explain = DotMap(self.get_explain_data())
202 except NameChanged as err:
203 self.logger.debug('Passing along upstream exception...')
204 raise NameChanged from err
206 def wait4complete(self) -> None:
207 """Wait for the ILM phase change to complete both the action and step"""
208 counter = 0
209 self.logger.debug('Waiting for current action and step to complete')
210 self.logger.debug(
211 'Action: %s --- Step: %s', self._explain.action, self._explain.step
212 )
213 while not bool(
214 self._explain.action == 'complete' and self._explain.step == 'complete'
215 ):
216 counter += 1
217 sleep(PAUSE_VALUE)
218 if counter % 10 == 0: 218 ↛ 219line 218 didn't jump to line 219, because the condition on line 218 was never true
219 self.logger.debug(
220 'Action: %s --- Step: %s', self._explain.action, self._explain.step
221 )
222 try:
223 self.count_logging(counter)
224 except ResultNotExpected as err:
225 self.logger.critical(
226 'Breaking the loop. Explain: %s', self._explain.toDict()
227 )
228 raise ResultNotExpected from err
229 try:
230 self.update()
231 except NameChanged as err:
232 self.logger.debug('Passing along upstream exception...')
233 raise NameChanged from err