Coverage for /Users/buh/.pyenv/versions/3.12.2/envs/es-testbed/lib/python3.12/site-packages/es_testbed/ilm.py: 73%
143 statements
« prev ^ index » next coverage.py v7.4.4, created at 2024-05-02 18:22 -0600
« prev ^ index » next coverage.py v7.4.4, created at 2024-05-02 18:22 -0600
1"""ILM Defining Class"""
3import typing as t
4import logging
5from os import getenv
6from dotmap import DotMap
7from elasticsearch8.exceptions import BadRequestError
8from es_wait import IlmPhase, IlmStep
9from es_wait.exceptions import IlmWaitError
10from es_testbed.defaults import (
11 PAUSE_ENVVAR,
12 PAUSE_DEFAULT,
13 TIMEOUT_DEFAULT,
14 TIMEOUT_ENVVAR,
15)
16from es_testbed.exceptions import NameChanged, ResultNotExpected, TestbedMisconfig
17from es_testbed.helpers.es_api import get_ilm_phases, ilm_explain, ilm_move, resolver
18from es_testbed.helpers.utils import prettystr
20if t.TYPE_CHECKING:
21 from elasticsearch8 import Elasticsearch
23PAUSE_VALUE = float(getenv(PAUSE_ENVVAR, default=PAUSE_DEFAULT))
24TIMEOUT_VALUE = float(getenv(TIMEOUT_ENVVAR, default=TIMEOUT_DEFAULT))
26logger = logging.getLogger('es_testbed.IlmTracker')
28# ## Example ILM explain output
29# {
30# 'action': 'complete',
31# 'action_time_millis': 0,
32# 'age': '5.65m',
33# 'index': 'INDEX_NAME',
34# 'index_creation_date_millis': 0,
35# 'lifecycle_date_millis': 0,
36# 'managed': True,
37# 'phase': 'hot',
38# 'phase_execution': {
39# 'modified_date_in_millis': 0,
40# 'phase_definition': {
41# 'actions': {
42# 'rollover': {
43# 'max_age': 'MAX_AGE',
44# 'max_primary_shard_docs': 1000,
45# 'max_primary_shard_size': 'MAX_SIZE',
46# 'min_docs': 1
47# }
48# },
49# 'min_age': '0ms'
50# },
51# 'policy': 'POLICY_NAME',
52# 'version': 1
53# },
54# 'phase_time_millis': 0,
55# 'policy': 'POLICY_NAME',
56# 'step': 'complete',
57# 'step_time_millis': 0,
58# 'time_since_index_creation': '5.65m'
59# }
62class IlmTracker:
63 """ILM Phase Tracking Class"""
65 def __init__(self, client: 'Elasticsearch', name: str):
66 self.client = client
67 self.name = self.resolve(name) # A single index name
68 self._explain = DotMap(self.get_explain_data())
69 self._phases = get_ilm_phases(self.client, self._explain.policy)
71 @property
72 def current_step(self) -> t.Dict:
73 """Return the current ILM step information"""
74 self.update()
75 return {
76 'phase': self._explain.phase,
77 'action': self._explain.action,
78 'name': self._explain.step,
79 }
81 @property
82 def explain(self) -> DotMap:
83 """Return the current stored value of ILM Explain"""
84 return self._explain
86 @property
87 def next_phase(self) -> str:
88 """Return the next phase in the index's ILM journey"""
89 retval = None
90 if self._explain.phase == 'delete':
91 logger.warning('Already on "delete" phase. No more phases to advance')
92 else:
93 curr = self.pnum(self._explain.phase) # A numeric representation
94 # A list of any remaining phases in the policy with a higher number than
95 # the current
96 remaining = [
97 self.pnum(x) for x in self.policy_phases if self.pnum(x) > curr
98 ]
99 if remaining: # If any:
100 retval = self.pname(remaining[0])
101 # Get the phase name from the number stored in the first element
102 return retval
104 @property
105 def policy_phases(self) -> t.Sequence[str]:
106 """Return a list of phases in the ILM policy"""
107 return list(self._phases.keys())
109 def _log_phase(self, phase: str) -> None:
110 logger.debug('ILM Explain Index: %s', self._explain.index)
111 logger.info('Index "%s" now on phase "%s"', self.name, phase)
113 def _phase_wait(
114 self, phase: str, pause: float = PAUSE_VALUE, timeout: float = TIMEOUT_VALUE
115 ) -> None:
116 """Wait until the new phase shows up in ILM Explain"""
117 kw = {'name': self.name, 'phase': phase, 'pause': pause, 'timeout': timeout}
118 phasechk = IlmPhase(self.client, **kw)
119 phasechk.wait_for_it()
121 def _ssphz(self, phase: str) -> bool:
122 return bool(self.pnum(phase) > self.pnum('warm'))
124 def advance(
125 self,
126 phase: t.Union[str, None] = None,
127 action: t.Union[str, None] = None,
128 name: t.Union[str, None] = None,
129 ) -> None:
130 """Advance index to next ILM phase"""
131 if self._explain.phase == 'delete':
132 logger.warning('Already on "delete" phase. No more phases to advance')
133 else:
134 logger.debug('current_step: %s', prettystr(self.current_step))
135 next_step = self.next_step(phase, action=action, name=name)
136 logger.debug('next_step: %s', prettystr(next_step))
137 if self._explain.phase == 'new' and phase == 'hot':
138 # It won't be for very long.
139 self._phase_wait('hot')
141 # Regardless of the remaining phases, the current phase steps must be
142 # complete before proceeding with ilm_move
143 self.update()
144 self.wait4complete()
145 self.update()
147 # We could have arrived with it hot, but incomplete
148 if phase == 'hot':
149 self._log_phase(phase)
150 # we've advanced to our target phase, and all steps are completed
152 # Remaining phases could be warm through frozen
153 elif self._explain.phase != phase:
155 # We will only wait for steps to complete for the hot and warm tiers
156 wait4steps = False if self._ssphz(phase) else False
158 ilm_move(self.client, self.name, self.current_step, next_step)
159 self._phase_wait(phase)
160 # If cold or frozen, we can return now. We let the calling function
161 # worry about the weird name changing behavior of searchable mounts
163 if wait4steps:
164 self.update()
165 logger.debug(
166 'Waiting for "%s" phase steps to complete...',
167 phase,
168 )
169 self.wait4complete()
170 self.update()
171 self._log_phase(phase)
172 else:
173 logger.error('next_step is a None value')
174 logger.error('current_step: %s', prettystr(self.current_step))
176 def get_explain_data(self) -> t.Dict:
177 """Get the ILM explain data and return it"""
178 try:
179 return ilm_explain(self.client, self.name)
180 except NameChanged as err:
181 logger.debug('Passing along upstream exception...')
182 raise NameChanged from err
183 except ResultNotExpected as err:
184 msg = f'Unable to get ilm_explain results. Error: {prettystr(err)}'
185 logger.critical(msg)
186 raise ResultNotExpected(msg) from err
188 def next_step(
189 self,
190 phase: t.Union[str, None] = None,
191 action: t.Union[str, None] = None,
192 name: t.Union[str, None] = None,
193 ) -> t.Dict:
194 """Determine the next ILM step based on the current phase, action, and name"""
195 err1 = bool((action is not None) and (name is None))
196 err2 = bool((action is None) and (name is not None))
197 if err1 or err2:
198 msg = 'If either action or name is specified, both must be'
199 logger.critical(msg)
200 raise TestbedMisconfig(msg)
201 if not phase:
202 phase = self.next_phase
203 retval = {'phase': phase}
204 if action:
205 retval['action'] = action
206 retval['name'] = name
207 return retval
209 def pnum(self, phase: str) -> int:
210 """Map a phase name to a phase number"""
211 _ = {'new': 0, 'hot': 1, 'warm': 2, 'cold': 3, 'frozen': 4, 'delete': 5}
212 return _[phase]
214 def pname(self, num: int) -> str:
215 """Map a phase number to a phase name"""
216 _ = {0: 'new', 1: 'hot', 2: 'warm', 3: 'cold', 4: 'frozen', 5: 'delete'}
217 return _[num]
219 def resolve(self, name: str) -> str:
220 """Resolve that we have an index and NOT an alias or a datastream"""
221 res = resolver(self.client, name)
222 if len(res['aliases']) > 0 or len(res['data_streams']) > 0:
223 msg = f'{name} is not an index: {res}'
224 logger.critical(msg)
225 raise ResultNotExpected(msg)
226 if len(res['indices']) > 1:
227 msg = f'{name} resolved to multiple indices: {prettystr(res["indices"])}'
228 logger.critical(msg)
229 raise ResultNotExpected(msg)
230 return res['indices'][0]['name']
232 def update(self) -> None:
233 """Update self._explain with the latest from :py:meth:`get_explain_data`"""
234 try:
235 self._explain = DotMap(self.get_explain_data())
236 except NameChanged as err:
237 logger.debug('Passing along upstream exception...')
238 raise NameChanged from err
240 def wait4complete(self) -> None:
241 """Subroutine for waiting for an ILM step to complete"""
242 step_action = bool(self._explain.action == 'complete')
243 step_name = bool(self._explain.name == 'complete')
244 if bool(step_action and step_name):
245 logger.debug(
246 '%s: Current step complete: %s', self.name, prettystr(self.current_step)
247 )
248 return
249 logger.debug(
250 '%s: Current step not complete. %s', self.name, prettystr(self.current_step)
251 )
252 kw = {'name': self.name, 'pause': PAUSE_VALUE, 'timeout': TIMEOUT_VALUE}
253 step = IlmStep(self.client, **kw)
254 try:
255 step.wait_for_it()
256 logger.debug('ILM Step successful. The wait is over')
257 except KeyError as exc:
258 logger.error('KeyError: The index name has changed: %s', prettystr(exc))
259 raise exc
260 except BadRequestError as exc:
261 logger.error('Index not found')
262 raise exc
263 except IlmWaitError as exc:
264 logger.error('Other IlmWait error encountered: %s', prettystr(exc))
265 raise exc