Coverage for /Users/buh/.pyenv/versions/3.12.2/envs/es-testbed/lib/python3.12/site-packages/es_testbed/classes/ilm.py: 68%
129 statements
« prev ^ index » next coverage.py v7.4.4, created at 2024-04-25 19:21 -0600
« prev ^ index » next coverage.py v7.4.4, created at 2024-04-25 19:21 -0600
1"""ILM Defining Class"""
2import typing as t
3from os import getenv
4from time import sleep
5from dotmap import DotMap
6from elasticsearch8 import Elasticsearch
7from es_testbed.defaults import PAUSE_ENVVAR, PAUSE_DEFAULT
8from es_testbed.exceptions import NameChanged, ResultNotExpected, TestbedMisconfig
9from es_testbed.helpers import es_api
10from es_testbed.helpers.utils import getlogger
11PAUSE_VALUE = float(getenv(PAUSE_ENVVAR, default=PAUSE_DEFAULT))
13# pylint: disable=missing-docstring
15### Example ILM explain output
16# {
17# 'action': 'complete',
18# 'action_time_millis': 0,
19# 'age': '5.65m',
20# 'index': 'INDEX_NAME',
21# 'index_creation_date_millis': 0,
22# 'lifecycle_date_millis': 0,
23# 'managed': True,
24# 'phase': 'hot',
25# 'phase_execution': {
26# 'modified_date_in_millis': 0,
27# 'phase_definition': {
28# 'actions': {
29# 'rollover': {
30# 'max_age': 'MAX_AGE',
31# 'max_primary_shard_docs': 1000,
32# 'max_primary_shard_size': 'MAX_SIZE',
33# 'min_docs': 1
34# }
35# },
36# 'min_age': '0ms'
37# },
38# 'policy': 'POLICY_NAME',
39# 'version': 1
40# },
41# 'phase_time_millis': 0,
42# 'policy': 'POLICY_NAME',
43# 'step': 'complete',
44# 'step_time_millis': 0,
45# 'time_since_index_creation': '5.65m'
46# }
48class IlmTracker:
49 def __init__(self, client: Elasticsearch, name: str):
50 self.logger = getlogger('es_testbed.IlmTracker')
51 self.client = client
52 self.name = self.resolve(name) # A single index name
53 self._explain = DotMap(self.get_explain_data())
54 self._phases = es_api.get_ilm_phases(self.client, self._explain.policy)
56 @property
57 def current_step(self) -> dict:
58 return {
59 'phase': self._explain.phase,
60 'action': self._explain.action,
61 'name': self._explain.step,
62 }
64 @property
65 def explain(self) -> DotMap:
66 return self._explain
68 @property
69 def next_phase(self) -> str:
70 retval = None
71 if self._explain.phase == 'delete':
72 self.logger.warning('Already on "delete" phase. No more phases to advance')
73 else:
74 curr = self.pnum(self._explain.phase) # A numeric representation of the current phase
75 # A list of any remaining phases in the policy with a higher number than the current
76 remaining = [self.pnum(x) for x in self.policy_phases if self.pnum(x) > curr]
77 if remaining: # If any:
78 retval = self.pname(remaining[0])
79 # Get the phase name from the number stored in the first element
80 return retval
82 @property
83 def policy_phases(self) -> t.Sequence[str]:
84 return list(self._phases.keys())
86 def advance(self, phase: str=None, action: str=None, name: str=None) -> None:
87 def wait(phase: str) -> None:
88 counter = 0
89 sleep(1.5) # Initial wait since we set ILM to poll every second
90 while self._explain.phase != phase:
91 sleep(PAUSE_VALUE)
92 self.update()
93 counter += 1
94 self.count_logging(counter)
96 if self._explain.phase == 'delete': 96 ↛ 97line 96 didn't jump to line 97, because the condition on line 96 was never true
97 self.logger.warning('Already on "delete" phase. No more phases to advance')
98 else:
99 self.logger.debug('current_step: %s', self.current_step)
100 next_step = self.next_step(phase, action=action, name=name)
101 self.logger.debug('next_step: %s', next_step)
102 if next_step: 102 ↛ 107line 102 didn't jump to line 107, because the condition on line 102 was never false
103 es_api.ilm_move(self.client, self.name, self.current_step, next_step)
104 wait(phase)
105 self.logger.info('Index %s now on phase %s', self.name, phase)
106 else:
107 self.logger.error('next_step is a None value')
108 self.logger.error('current_step: %s', self.current_step)
110 def count_logging(self, counter: int) -> None:
111 # Send a message every 10 loops
112 if counter % 40 == 0: 112 ↛ 113line 112 didn't jump to line 113, because the condition on line 112 was never true
113 self.logger.info('Still working... Explain: %s', self._explain.asdict)
114 if counter == 480: 114 ↛ 115line 114 didn't jump to line 115, because the condition on line 114 was never true
115 msg = 'Taking too long! Giving up on waiting'
116 self.logger.critical(msg)
117 raise ResultNotExpected(msg)
119 def get_explain_data(self) -> t.Dict:
120 try:
121 return es_api.ilm_explain(self.client, self.name)
122 except NameChanged as err: 122 ↛ 125line 122 didn't jump to line 125
123 self.logger.debug('Passing along upstream exception...')
124 raise NameChanged from err
125 except ResultNotExpected as err:
126 msg = f'Unable to get ilm_explain API call results. Error: {err}'
127 self.logger.critical(msg)
128 raise ResultNotExpected(msg) from err
130 def next_step(self, phase: str=None, action: str=None, name: str=None) -> t.Dict:
131 err1 = bool((action is not None) and (name is None))
132 err2 = bool((action is None) and (name is not None))
133 if err1 or err2: 133 ↛ 134line 133 didn't jump to line 134, because the condition on line 133 was never true
134 msg = 'If either action or name is specified, both must be'
135 self.logger.critical(msg)
136 raise TestbedMisconfig(msg)
137 if not phase: 137 ↛ 138line 137 didn't jump to line 138, because the condition on line 137 was never true
138 phase = self.next_phase
139 retval = {'phase': phase}
140 if action: 140 ↛ 141line 140 didn't jump to line 141, because the condition on line 140 was never true
141 retval['action'] = action
142 retval['name'] = name
143 return retval
145 def pnum(self, phase: str) -> int:
146 _ = {'new': 0, 'hot': 1, 'warm': 2, 'cold': 3, 'frozen': 4, 'delete': 5}
147 return _[phase]
149 def pname(self, num: int) -> str:
150 _ = {0: 'new', 1: 'hot', 2: 'warm', 3: 'cold', 4: 'frozen', 5: 'delete'}
151 return _[num]
153 def resolve(self, name: str) -> str:
154 """Resolve that we have an index and NOT an alias or a datastream"""
155 res = es_api.resolver(self.client, name)
156 if len(res['aliases']) > 0 or len(res['data_streams']) > 0: 156 ↛ 157line 156 didn't jump to line 157, because the condition on line 156 was never true
157 msg = f'{name} is not an index: {res}'
158 self.logger.critical(msg)
159 raise ResultNotExpected(msg)
160 if len(res['indices']) > 1: 160 ↛ 161line 160 didn't jump to line 161, because the condition on line 160 was never true
161 msg = f'{name} resolved to multiple indices: {res['indices']}'
162 self.logger.critical(msg)
163 raise ResultNotExpected(msg)
164 return res['indices'][0]['name']
166 def update(self) -> None:
167 try:
168 self._explain = DotMap(self.get_explain_data())
169 except NameChanged as err:
170 self.logger.debug('Passing along upstream exception...')
171 raise NameChanged from err
173 def wait4complete(self) -> None:
174 counter = 0
175 self.logger.debug('Waiting for current action and step to complete')
176 self.logger.debug('Action: %s --- Step: %s', self._explain.action, self._explain.step)
177 while not bool(
178 self._explain.action == 'complete' and
179 self._explain.step == 'complete'):
180 counter += 1
181 sleep(PAUSE_VALUE)
182 if counter % 10 == 0: 182 ↛ 183line 182 didn't jump to line 183, because the condition on line 182 was never true
183 self.logger.debug(
184 'Action: %s --- Step: %s', self._explain.action, self._explain.step)
185 try:
186 self.count_logging(counter)
187 except ResultNotExpected as err:
188 self.logger.critical('Breaking the loop. Explain: %s', self._explain.toDict())
189 raise ResultNotExpected from err
190 try:
191 self.update()
192 except NameChanged as err:
193 self.logger.debug('Passing along upstream exception...')
194 raise NameChanged from err