Coverage for /Users/buh/.pyenv/versions/3.12.9/envs/es-testbed/lib/python3.12/site-packages/es_testbed/entities/index.py: 100%
170 statements
« prev ^ index » next coverage.py v7.6.12, created at 2025-04-16 12:23 -0600
« prev ^ index » next coverage.py v7.6.12, created at 2025-04-16 12:23 -0600
1"""Index Entity Class"""
3import typing as t
4import logging
5from os import getenv
6from elasticsearch8.exceptions import BadRequestError
7from es_wait import Exists, IlmPhase, IlmStep
8from es_wait.exceptions import EsWaitFatal, EsWaitTimeout
9from ..debug import debug, begin_end
10from ..defaults import PAUSE_DEFAULT, PAUSE_ENVVAR, TIMEOUT_DEFAULT, TIMEOUT_ENVVAR
11from ..es_api import snapshot_name
12from ..exceptions import TestbedFailure
13from ..ilm import IlmTracker
14from ..utils import mounted_name, prettystr
15from .entity import Entity
17if t.TYPE_CHECKING:
18 from elasticsearch8 import Elasticsearch
20PAUSE_VALUE = float(getenv(PAUSE_ENVVAR, default=PAUSE_DEFAULT))
21TIMEOUT_VALUE = float(getenv(TIMEOUT_ENVVAR, default=TIMEOUT_DEFAULT))
23logger = logging.getLogger(__name__)
26class Index(Entity):
27 """Index Entity Class"""
29 def __init__(
30 self,
31 client: 'Elasticsearch',
32 name: t.Union[str, None] = None,
33 snapmgr=None,
34 policy_name: str = None,
35 ):
36 debug.lv2('Initializing Index entity object...')
37 super().__init__(client=client, name=name)
38 self.policy_name = policy_name
39 self.ilm_tracker = None
40 self.snapmgr = snapmgr
41 debug.lv3('Index entity object initialized')
43 @property
44 @begin_end()
45 def _get_target(self) -> str:
46 target = None
47 phases = self.ilm_tracker.policy_phases
48 curr = self.ilm_tracker.explain.phase
49 if not bool(('cold' in phases) or ('frozen' in phases)):
50 debug.lv3(f'ILM Policy for "{self.name}" has no cold/frozen phases')
51 target = curr # Keep the same
52 if bool(('cold' in phases) and ('frozen' in phases)):
53 if self.ilm_tracker.pname(curr) < self.ilm_tracker.pname('cold'):
54 target = 'cold'
55 elif curr == 'cold':
56 target = 'frozen'
57 elif self.ilm_tracker.pname(curr) >= self.ilm_tracker.pname('frozen'):
58 target = curr
59 elif bool(('cold' in phases) and ('frozen' not in phases)):
60 target = 'cold'
61 elif bool(('cold' not in phases) and ('frozen' in phases)):
62 target = 'frozen'
63 return target
65 @property
66 def phase_tuple(self) -> t.Tuple[str, str]:
67 """Return the current phase and the target phase as a Tuple"""
68 return self.ilm_tracker.explain.phase, self._get_target
70 @begin_end()
71 def _add_snap_step(self) -> None:
72 debug.lv4('Getting snapshot name for tracking...')
73 snapname = snapshot_name(self.client, self.name)
74 debug.lv5(f'Snapshot {snapname} backs {self.name}')
75 self.snapmgr.add_existing(snapname)
77 @begin_end()
78 def _ilm_step(self) -> None:
79 """Subroutine for waiting for an ILM step to complete"""
80 step = {
81 'phase': self.ilm_tracker.explain.phase,
82 'action': self.ilm_tracker.explain.action,
83 'name': self.ilm_tracker.explain.step,
84 }
85 debug.lv5(f'{self.name}: Current Step: {step}')
86 step = IlmStep(
87 self.client, pause=PAUSE_VALUE, timeout=TIMEOUT_VALUE, name=self.name
88 )
89 try:
90 debug.lv4('TRY: Waiting for ILM step to complete...')
91 self._wait_try(step.wait)
92 except TestbedFailure as err:
93 logger.error(err.message)
94 debug.lv3('Exiting method, raising exception')
95 debug.lv5(f'Exception: {prettystr(err)}')
96 raise err
98 @begin_end()
99 def _wait_try(self, func: t.Callable) -> None:
100 """Wait for an es-wait function to complete"""
101 try:
102 debug.lv4('TRY: To run func()...')
103 func()
104 except EsWaitFatal as wait:
105 # EsWaitFatal indicates we had more than the allowed number of exceptions
106 msg = f'{wait.message}. Elapsed time: {wait.elapsed}. Errors: {wait.errors}'
107 debug.lv3('Exiting method, raising exception')
108 debug.lv5(f'Exception: {prettystr(wait)}')
109 raise TestbedFailure(msg) from wait
110 except EsWaitTimeout as wait:
111 # EsWaitTimeout indicates we hit the timeout
112 msg = f'{wait.message}. Total elapsed time: {wait.elapsed}.'
113 debug.lv3('Exiting method, raising exception')
114 debug.lv5(f'Exception: {prettystr(wait)}')
115 raise TestbedFailure(msg) from wait
116 except Exception as err:
117 debug.lv3('Exiting method, raising exception')
118 debug.lv5(f'Exception: {prettystr(err)}')
119 raise TestbedFailure(f'General Exception caught: {prettystr(err)}') from err
121 @begin_end()
122 def _mounted_step(self, target: str) -> str:
123 try:
124 debug.lv4(f'TRY: Moving "{self.name}" to ILM phase "{target}"')
125 self.ilm_tracker.advance(phase=target)
126 except BadRequestError as err:
127 logger.critical(f'err: {prettystr(err)}')
128 debug.lv3('Exiting method, raising exception')
129 debug.lv5(f'Exception: {prettystr(err)}')
130 raise err # Re-raise after logging
131 # At this point, it's "in" a searchable tier, but the index name hasn't
132 # changed yet
133 newidx = mounted_name(self.name, target)
134 debug.lv3(f'Waiting for ILM phase change to complete. New index: {newidx}')
135 wait_kwargs = {
136 'name': newidx,
137 'kind': 'index',
138 'pause': PAUSE_VALUE,
139 'timeout': TIMEOUT_VALUE,
140 }
142 test = Exists(self.client, **wait_kwargs)
143 debug.lv5(f'Exists response: {prettystr(test)}')
144 try:
145 debug.lv4(f'TRY: Waiting for "{newidx}" to exist...')
146 self._wait_try(test.wait)
147 except TestbedFailure as err:
148 logger.error(err.message)
149 debug.lv3('Exiting method, raising exception')
150 debug.lv5(f'Exception: {prettystr(err)}')
151 raise err
153 # Update the name and run
154 debug.lv3(f'Updating self.name from "{self.name}" to "{newidx}"...')
155 self.name = newidx
157 # Wait for the ILM steps to complete
158 debug.lv3('Waiting for the ILM steps to complete...')
159 self._ilm_step()
161 # Track the new index
162 debug.lv3(f'Switching to track "{newidx}" as self.name...')
163 self.track_ilm(newidx)
165 @begin_end()
166 def manual_ss(self, scheme: t.Dict[str, t.Any]) -> None:
167 """
168 If we are NOT using ILM but have specified searchable snapshots in the plan
169 entities
170 """
171 if 'target_tier' in scheme and scheme['target_tier'] in ['cold', 'frozen']:
172 debug.lv3('Manually mounting as searchable snapshot...')
173 debug.lv5(
174 f'Mounting "{self.name}" as searchable snapshot '
175 f'({scheme["target_tier"]})'
176 )
177 self.snapmgr.add(self.name, scheme['target_tier'])
178 # Replace self.name with the renamed name
179 self.name = mounted_name(self.name, scheme['target_tier'])
181 @begin_end()
182 def mount_ss(self, scheme: dict) -> None:
183 """If the index is planned to become a searchable snapshot, we do that now"""
184 debug.lv3(f'Checking if "{self.name}" should be a searchable snapshot')
185 if self.am_i_write_idx:
186 debug.lv5(
187 f'"{self.name}" is the write_index. Cannot mount as searchable '
188 f'snapshot'
189 )
191 return
192 if not self.policy_name: # If we have this, chances are we have a policy
193 debug.lv3(f'No ILM policy for "{self.name}". Trying manual...')
194 self.manual_ss(scheme)
196 return
197 phase = self.ilm_tracker.next_phase
198 debug.lv5(f'Next phase for "{self.name}" = {phase}')
199 current = self.ilm_tracker.explain.phase
200 debug.lv5(f'Current phase for "{self.name}" = {current}')
201 if current == 'new':
202 # This is a problem. We need to be in 'hot', with rollover completed.
203 debug.lv3(
204 f'Our index is still in phase "{current}"!. '
205 f'We need it to be in "{phase}"'
206 )
208 phasenext = IlmPhase(
209 self.client,
210 pause=PAUSE_VALUE,
211 timeout=TIMEOUT_VALUE,
212 name=self.name,
213 phase=self.ilm_tracker.next_phase,
214 )
215 try:
216 debug.lv4('TRY: Waiting for ILM phase to complete...')
217 self._wait_try(phasenext.wait)
218 except TestbedFailure as err:
219 logger.error(err.message)
220 debug.lv3('Exiting method, raising exception')
221 debug.lv5(f'Exception: {prettystr(err)}')
222 raise err
223 target = self._get_target
224 debug.lv5(f'Target phase for "{self.name}" = {target}')
225 if current != target:
226 debug.lv5(f'Current ({current}) and target ({target}) mismatch')
227 debug.lv5('Waiting for ILM step to complete...')
228 self.ilm_tracker.wait4complete()
229 # Because the step is completed, we must now update OUR tracker to
230 # reflect the updated ILM Explain information
231 debug.lv5('Updating ILM tracker...')
232 self.ilm_tracker.update()
234 # ILM snapshot mount phase. The biggest pain of them all...
235 debug.lv3(f'Moving "{self.name}" to ILM phase "{target}"')
236 self._mounted_step(target)
237 debug.lv3(f'ILM advance to phase "{target}" completed')
239 # Record the snapshot in our tracker
240 debug.lv5('Adding snapshot step to snapmgr...')
241 self._add_snap_step()
243 @begin_end()
244 def track_ilm(self, name: str) -> None:
245 """
246 Get ILM phase information and put it in self.ilm_tracker
247 Name as an arg makes it configurable
248 """
249 if self.policy_name:
250 debug.lv5(f'ILM policy name: {self.policy_name}')
251 debug.lv5(f'Creating ILM tracker for "{name}"')
252 self.ilm_tracker = IlmTracker(self.client, name)
253 debug.lv5(f'Updating ILM tracker for "{name}"')
254 self.ilm_tracker.update()
255 else:
256 debug.lv5('No ILM policy name. Skipping ILM tracking')