Coverage for /Users/buh/.pyenv/versions/3.12.9/envs/es-testbed/lib/python3.12/site-packages/es_testbed/entities/index.py: 100%
178 statements
« prev ^ index » next coverage.py v7.6.12, created at 2025-03-31 14:55 -0600
« prev ^ index » next coverage.py v7.6.12, created at 2025-03-31 14:55 -0600
1"""Index Entity Class"""
3import typing as t
4import logging
5from os import getenv
6import tiered_debug as debug
7from elasticsearch8.exceptions import BadRequestError
8from es_wait import Exists, IlmPhase, IlmStep
9from es_wait.exceptions import EsWaitFatal, EsWaitTimeout
10from es_testbed.defaults import (
11 PAUSE_DEFAULT,
12 PAUSE_ENVVAR,
13 TIMEOUT_DEFAULT,
14 TIMEOUT_ENVVAR,
15)
16from es_testbed.exceptions import TestbedFailure
17from es_testbed.entities.entity import Entity
18from es_testbed.helpers.es_api import snapshot_name
19from es_testbed.helpers.utils import mounted_name, prettystr
20from es_testbed.ilm import IlmTracker
22if t.TYPE_CHECKING:
23 from elasticsearch8 import Elasticsearch
25PAUSE_VALUE = float(getenv(PAUSE_ENVVAR, default=PAUSE_DEFAULT))
26TIMEOUT_VALUE = float(getenv(TIMEOUT_ENVVAR, default=TIMEOUT_DEFAULT))
28logger = logging.getLogger(__name__)
31class Index(Entity):
32 """Index Entity Class"""
34 def __init__(
35 self,
36 client: 'Elasticsearch',
37 name: t.Union[str, None] = None,
38 snapmgr=None,
39 policy_name: str = None,
40 ):
41 debug.lv2('Initializing Index entity object...')
42 super().__init__(client=client, name=name)
43 self.policy_name = policy_name
44 self.ilm_tracker = None
45 self.snapmgr = snapmgr
46 debug.lv3('Index entity object initialized')
48 @property
49 def _get_target(self) -> str:
50 target = None
51 phases = self.ilm_tracker.policy_phases
52 curr = self.ilm_tracker.explain.phase
53 if not bool(('cold' in phases) or ('frozen' in phases)):
54 debug.lv3(f'ILM Policy for "{self.name}" has no cold/frozen phases')
55 target = curr # Keep the same
56 if bool(('cold' in phases) and ('frozen' in phases)):
57 if self.ilm_tracker.pname(curr) < self.ilm_tracker.pname('cold'):
58 target = 'cold'
59 elif curr == 'cold':
60 target = 'frozen'
61 elif self.ilm_tracker.pname(curr) >= self.ilm_tracker.pname('frozen'):
62 target = curr
63 elif bool(('cold' in phases) and ('frozen' not in phases)):
64 target = 'cold'
65 elif bool(('cold' not in phases) and ('frozen' in phases)):
66 target = 'frozen'
67 return target
69 @property
70 def phase_tuple(self) -> t.Tuple[str, str]:
71 """Return the current phase and the target phase as a Tuple"""
72 return self.ilm_tracker.explain.phase, self._get_target
74 def _add_snap_step(self) -> None:
75 debug.lv2('Starting method...')
76 debug.lv4('Getting snapshot name for tracking...')
77 snapname = snapshot_name(self.client, self.name)
78 debug.lv5(f'Snapshot {snapname} backs {self.name}')
79 self.snapmgr.add_existing(snapname)
80 debug.lv3('Exiting method')
82 def _ilm_step(self) -> None:
83 """Subroutine for waiting for an ILM step to complete"""
84 debug.lv2('Starting method...')
85 step = {
86 'phase': self.ilm_tracker.explain.phase,
87 'action': self.ilm_tracker.explain.action,
88 'name': self.ilm_tracker.explain.step,
89 }
90 debug.lv5(f'{self.name}: Current Step: {step}')
91 step = IlmStep(
92 self.client, pause=PAUSE_VALUE, timeout=TIMEOUT_VALUE, name=self.name
93 )
94 try:
95 debug.lv4('TRY: Waiting for ILM step to complete...')
96 self._wait_try(step.wait)
97 except TestbedFailure as err:
98 logger.error(err.message)
99 debug.lv3('Exiting method, raising exception')
100 debug.lv5(f'Exception: {prettystr(err)}')
101 raise err
102 debug.lv3('Exiting method')
104 def _wait_try(self, func: t.Callable) -> None:
105 """Wait for an es-wait function to complete"""
106 debug.lv2('Starting method...')
107 try:
108 debug.lv4('TRY: To run func()...')
109 func()
110 except EsWaitFatal as wait:
111 # EsWaitFatal indicates we had more than the allowed number of exceptions
112 msg = f'{wait.message}. Elapsed time: {wait.elapsed}. Errors: {wait.errors}'
113 debug.lv3('Exiting method, raising exception')
114 debug.lv5(f'Exception: {prettystr(wait)}')
115 raise TestbedFailure(msg) from wait
116 except EsWaitTimeout as wait:
117 # EsWaitTimeout indicates we hit the timeout
118 msg = f'{wait.message}. Total elapsed time: {wait.elapsed}.'
119 debug.lv3('Exiting method, raising exception')
120 debug.lv5(f'Exception: {prettystr(wait)}')
121 raise TestbedFailure(msg) from wait
122 except Exception as err:
123 debug.lv3('Exiting method, raising exception')
124 debug.lv5(f'Exception: {prettystr(err)}')
125 raise TestbedFailure(f'General Exception caught: {prettystr(err)}') from err
126 debug.lv3('Exiting method')
128 def _mounted_step(self, target: str) -> str:
129 debug.lv2('Starting method...')
130 try:
131 debug.lv4(f'TRY: Moving "{self.name}" to ILM phase "{target}"')
132 self.ilm_tracker.advance(phase=target)
133 except BadRequestError as err:
134 logger.critical(f'err: {prettystr(err)}')
135 debug.lv3('Exiting method, raising exception')
136 debug.lv5(f'Exception: {prettystr(err)}')
137 raise err # Re-raise after logging
138 # At this point, it's "in" a searchable tier, but the index name hasn't
139 # changed yet
140 newidx = mounted_name(self.name, target)
141 debug.lv3(f'Waiting for ILM phase change to complete. New index: {newidx}')
142 wait_kwargs = {
143 'name': newidx,
144 'kind': 'index',
145 'pause': PAUSE_VALUE,
146 'timeout': TIMEOUT_VALUE,
147 }
149 test = Exists(self.client, **wait_kwargs)
150 debug.lv5(f'Exists response: {prettystr(test)}')
151 try:
152 debug.lv4(f'TRY: Waiting for "{newidx}" to exist...')
153 self._wait_try(test.wait)
154 except TestbedFailure as err:
155 logger.error(err.message)
156 debug.lv3('Exiting method, raising exception')
157 debug.lv5(f'Exception: {prettystr(err)}')
158 raise err
160 # Update the name and run
161 debug.lv3(f'Updating self.name from "{self.name}" to "{newidx}"...')
162 self.name = newidx
164 # Wait for the ILM steps to complete
165 debug.lv3('Waiting for the ILM steps to complete...')
166 self._ilm_step()
168 # Track the new index
169 debug.lv3(f'Switching to track "{newidx}" as self.name...')
170 self.track_ilm(newidx)
171 debug.lv3('Exiting method')
173 def manual_ss(self, scheme: t.Dict[str, t.Any]) -> None:
174 """
175 If we are NOT using ILM but have specified searchable snapshots in the plan
176 entities
177 """
178 debug.lv2('Starting method...')
179 if 'target_tier' in scheme and scheme['target_tier'] in ['cold', 'frozen']:
180 debug.lv3('Manually mounting as searchable snapshot...')
181 debug.lv5(
182 f'Mounting "{self.name}" as searchable snapshot '
183 f'({scheme["target_tier"]})'
184 )
185 self.snapmgr.add(self.name, scheme['target_tier'])
186 # Replace self.name with the renamed name
187 self.name = mounted_name(self.name, scheme['target_tier'])
188 debug.lv3('Exiting method')
190 def mount_ss(self, scheme: dict) -> None:
191 """If the index is planned to become a searchable snapshot, we do that now"""
192 debug.lv2('Starting method...')
193 debug.lv3(f'Checking if "{self.name}" should be a searchable snapshot')
194 if self.am_i_write_idx:
195 debug.lv5(
196 f'"{self.name}" is the write_index. Cannot mount as searchable '
197 f'snapshot'
198 )
199 debug.lv3('Exiting method')
200 return
201 if not self.policy_name: # If we have this, chances are we have a policy
202 debug.lv3(f'No ILM policy for "{self.name}". Trying manual...')
203 self.manual_ss(scheme)
204 debug.lv3('Exiting method')
205 return
206 phase = self.ilm_tracker.next_phase
207 debug.lv5(f'Next phase for "{self.name}" = {phase}')
208 current = self.ilm_tracker.explain.phase
209 debug.lv5(f'Current phase for "{self.name}" = {current}')
210 if current == 'new':
211 # This is a problem. We need to be in 'hot', with rollover completed.
212 debug.lv3(
213 f'Our index is still in phase "{current}"!. '
214 f'We need it to be in "{phase}"'
215 )
217 phasenext = IlmPhase(
218 self.client,
219 pause=PAUSE_VALUE,
220 timeout=TIMEOUT_VALUE,
221 name=self.name,
222 phase=self.ilm_tracker.next_phase,
223 )
224 try:
225 debug.lv4('TRY: Waiting for ILM phase to complete...')
226 self._wait_try(phasenext.wait)
227 except TestbedFailure as err:
228 logger.error(err.message)
229 debug.lv3('Exiting method, raising exception')
230 debug.lv5(f'Exception: {prettystr(err)}')
231 raise err
232 target = self._get_target
233 debug.lv5(f'Target phase for "{self.name}" = {target}')
234 if current != target:
235 debug.lv5(f'Current ({current}) and target ({target}) mismatch')
236 debug.lv5('Waiting for ILM step to complete...')
237 self.ilm_tracker.wait4complete()
238 # Because the step is completed, we must now update OUR tracker to
239 # reflect the updated ILM Explain information
240 debug.lv5('Updating ILM tracker...')
241 self.ilm_tracker.update()
243 # ILM snapshot mount phase. The biggest pain of them all...
244 debug.lv3(f'Moving "{self.name}" to ILM phase "{target}"')
245 self._mounted_step(target)
246 debug.lv3(f'ILM advance to phase "{target}" completed')
248 # Record the snapshot in our tracker
249 debug.lv5('Adding snapshot step to snapmgr...')
250 self._add_snap_step()
251 debug.lv3('Exiting method')
253 def track_ilm(self, name: str) -> None:
254 """
255 Get ILM phase information and put it in self.ilm_tracker
256 Name as an arg makes it configurable
257 """
258 debug.lv2('Starting method...')
259 if self.policy_name:
260 debug.lv5(f'ILM policy name: {self.policy_name}')
261 debug.lv5(f'Creating ILM tracker for "{name}"')
262 self.ilm_tracker = IlmTracker(self.client, name)
263 debug.lv5(f'Updating ILM tracker for "{name}"')
264 self.ilm_tracker.update()
265 else:
266 debug.lv5('No ILM policy name. Skipping ILM tracking')
267 debug.lv3('Exiting method')