Coverage for /Users/buh/.pyenv/versions/3.12.9/envs/es-testbed/lib/python3.12/site-packages/es_testbed/entities/index.py: 100%

170 statements  

« prev     ^ index     » next       coverage.py v7.6.12, created at 2025-04-16 12:23 -0600

1"""Index Entity Class""" 

2 

3import typing as t 

4import logging 

5from os import getenv 

6from elasticsearch8.exceptions import BadRequestError 

7from es_wait import Exists, IlmPhase, IlmStep 

8from es_wait.exceptions import EsWaitFatal, EsWaitTimeout 

9from ..debug import debug, begin_end 

10from ..defaults import PAUSE_DEFAULT, PAUSE_ENVVAR, TIMEOUT_DEFAULT, TIMEOUT_ENVVAR 

11from ..es_api import snapshot_name 

12from ..exceptions import TestbedFailure 

13from ..ilm import IlmTracker 

14from ..utils import mounted_name, prettystr 

15from .entity import Entity 

16 

17if t.TYPE_CHECKING: 

18 from elasticsearch8 import Elasticsearch 

19 

20PAUSE_VALUE = float(getenv(PAUSE_ENVVAR, default=PAUSE_DEFAULT)) 

21TIMEOUT_VALUE = float(getenv(TIMEOUT_ENVVAR, default=TIMEOUT_DEFAULT)) 

22 

23logger = logging.getLogger(__name__) 

24 

25 

26class Index(Entity): 

27 """Index Entity Class""" 

28 

29 def __init__( 

30 self, 

31 client: 'Elasticsearch', 

32 name: t.Union[str, None] = None, 

33 snapmgr=None, 

34 policy_name: str = None, 

35 ): 

36 debug.lv2('Initializing Index entity object...') 

37 super().__init__(client=client, name=name) 

38 self.policy_name = policy_name 

39 self.ilm_tracker = None 

40 self.snapmgr = snapmgr 

41 debug.lv3('Index entity object initialized') 

42 

43 @property 

44 @begin_end() 

45 def _get_target(self) -> str: 

46 target = None 

47 phases = self.ilm_tracker.policy_phases 

48 curr = self.ilm_tracker.explain.phase 

49 if not bool(('cold' in phases) or ('frozen' in phases)): 

50 debug.lv3(f'ILM Policy for "{self.name}" has no cold/frozen phases') 

51 target = curr # Keep the same 

52 if bool(('cold' in phases) and ('frozen' in phases)): 

53 if self.ilm_tracker.pname(curr) < self.ilm_tracker.pname('cold'): 

54 target = 'cold' 

55 elif curr == 'cold': 

56 target = 'frozen' 

57 elif self.ilm_tracker.pname(curr) >= self.ilm_tracker.pname('frozen'): 

58 target = curr 

59 elif bool(('cold' in phases) and ('frozen' not in phases)): 

60 target = 'cold' 

61 elif bool(('cold' not in phases) and ('frozen' in phases)): 

62 target = 'frozen' 

63 return target 

64 

65 @property 

66 def phase_tuple(self) -> t.Tuple[str, str]: 

67 """Return the current phase and the target phase as a Tuple""" 

68 return self.ilm_tracker.explain.phase, self._get_target 

69 

70 @begin_end() 

71 def _add_snap_step(self) -> None: 

72 debug.lv4('Getting snapshot name for tracking...') 

73 snapname = snapshot_name(self.client, self.name) 

74 debug.lv5(f'Snapshot {snapname} backs {self.name}') 

75 self.snapmgr.add_existing(snapname) 

76 

77 @begin_end() 

78 def _ilm_step(self) -> None: 

79 """Subroutine for waiting for an ILM step to complete""" 

80 step = { 

81 'phase': self.ilm_tracker.explain.phase, 

82 'action': self.ilm_tracker.explain.action, 

83 'name': self.ilm_tracker.explain.step, 

84 } 

85 debug.lv5(f'{self.name}: Current Step: {step}') 

86 step = IlmStep( 

87 self.client, pause=PAUSE_VALUE, timeout=TIMEOUT_VALUE, name=self.name 

88 ) 

89 try: 

90 debug.lv4('TRY: Waiting for ILM step to complete...') 

91 self._wait_try(step.wait) 

92 except TestbedFailure as err: 

93 logger.error(err.message) 

94 debug.lv3('Exiting method, raising exception') 

95 debug.lv5(f'Exception: {prettystr(err)}') 

96 raise err 

97 

98 @begin_end() 

99 def _wait_try(self, func: t.Callable) -> None: 

100 """Wait for an es-wait function to complete""" 

101 try: 

102 debug.lv4('TRY: To run func()...') 

103 func() 

104 except EsWaitFatal as wait: 

105 # EsWaitFatal indicates we had more than the allowed number of exceptions 

106 msg = f'{wait.message}. Elapsed time: {wait.elapsed}. Errors: {wait.errors}' 

107 debug.lv3('Exiting method, raising exception') 

108 debug.lv5(f'Exception: {prettystr(wait)}') 

109 raise TestbedFailure(msg) from wait 

110 except EsWaitTimeout as wait: 

111 # EsWaitTimeout indicates we hit the timeout 

112 msg = f'{wait.message}. Total elapsed time: {wait.elapsed}.' 

113 debug.lv3('Exiting method, raising exception') 

114 debug.lv5(f'Exception: {prettystr(wait)}') 

115 raise TestbedFailure(msg) from wait 

116 except Exception as err: 

117 debug.lv3('Exiting method, raising exception') 

118 debug.lv5(f'Exception: {prettystr(err)}') 

119 raise TestbedFailure(f'General Exception caught: {prettystr(err)}') from err 

120 

121 @begin_end() 

122 def _mounted_step(self, target: str) -> str: 

123 try: 

124 debug.lv4(f'TRY: Moving "{self.name}" to ILM phase "{target}"') 

125 self.ilm_tracker.advance(phase=target) 

126 except BadRequestError as err: 

127 logger.critical(f'err: {prettystr(err)}') 

128 debug.lv3('Exiting method, raising exception') 

129 debug.lv5(f'Exception: {prettystr(err)}') 

130 raise err # Re-raise after logging 

131 # At this point, it's "in" a searchable tier, but the index name hasn't 

132 # changed yet 

133 newidx = mounted_name(self.name, target) 

134 debug.lv3(f'Waiting for ILM phase change to complete. New index: {newidx}') 

135 wait_kwargs = { 

136 'name': newidx, 

137 'kind': 'index', 

138 'pause': PAUSE_VALUE, 

139 'timeout': TIMEOUT_VALUE, 

140 } 

141 

142 test = Exists(self.client, **wait_kwargs) 

143 debug.lv5(f'Exists response: {prettystr(test)}') 

144 try: 

145 debug.lv4(f'TRY: Waiting for "{newidx}" to exist...') 

146 self._wait_try(test.wait) 

147 except TestbedFailure as err: 

148 logger.error(err.message) 

149 debug.lv3('Exiting method, raising exception') 

150 debug.lv5(f'Exception: {prettystr(err)}') 

151 raise err 

152 

153 # Update the name and run 

154 debug.lv3(f'Updating self.name from "{self.name}" to "{newidx}"...') 

155 self.name = newidx 

156 

157 # Wait for the ILM steps to complete 

158 debug.lv3('Waiting for the ILM steps to complete...') 

159 self._ilm_step() 

160 

161 # Track the new index 

162 debug.lv3(f'Switching to track "{newidx}" as self.name...') 

163 self.track_ilm(newidx) 

164 

165 @begin_end() 

166 def manual_ss(self, scheme: t.Dict[str, t.Any]) -> None: 

167 """ 

168 If we are NOT using ILM but have specified searchable snapshots in the plan 

169 entities 

170 """ 

171 if 'target_tier' in scheme and scheme['target_tier'] in ['cold', 'frozen']: 

172 debug.lv3('Manually mounting as searchable snapshot...') 

173 debug.lv5( 

174 f'Mounting "{self.name}" as searchable snapshot ' 

175 f'({scheme["target_tier"]})' 

176 ) 

177 self.snapmgr.add(self.name, scheme['target_tier']) 

178 # Replace self.name with the renamed name 

179 self.name = mounted_name(self.name, scheme['target_tier']) 

180 

181 @begin_end() 

182 def mount_ss(self, scheme: dict) -> None: 

183 """If the index is planned to become a searchable snapshot, we do that now""" 

184 debug.lv3(f'Checking if "{self.name}" should be a searchable snapshot') 

185 if self.am_i_write_idx: 

186 debug.lv5( 

187 f'"{self.name}" is the write_index. Cannot mount as searchable ' 

188 f'snapshot' 

189 ) 

190 

191 return 

192 if not self.policy_name: # If we have this, chances are we have a policy 

193 debug.lv3(f'No ILM policy for "{self.name}". Trying manual...') 

194 self.manual_ss(scheme) 

195 

196 return 

197 phase = self.ilm_tracker.next_phase 

198 debug.lv5(f'Next phase for "{self.name}" = {phase}') 

199 current = self.ilm_tracker.explain.phase 

200 debug.lv5(f'Current phase for "{self.name}" = {current}') 

201 if current == 'new': 

202 # This is a problem. We need to be in 'hot', with rollover completed. 

203 debug.lv3( 

204 f'Our index is still in phase "{current}"!. ' 

205 f'We need it to be in "{phase}"' 

206 ) 

207 

208 phasenext = IlmPhase( 

209 self.client, 

210 pause=PAUSE_VALUE, 

211 timeout=TIMEOUT_VALUE, 

212 name=self.name, 

213 phase=self.ilm_tracker.next_phase, 

214 ) 

215 try: 

216 debug.lv4('TRY: Waiting for ILM phase to complete...') 

217 self._wait_try(phasenext.wait) 

218 except TestbedFailure as err: 

219 logger.error(err.message) 

220 debug.lv3('Exiting method, raising exception') 

221 debug.lv5(f'Exception: {prettystr(err)}') 

222 raise err 

223 target = self._get_target 

224 debug.lv5(f'Target phase for "{self.name}" = {target}') 

225 if current != target: 

226 debug.lv5(f'Current ({current}) and target ({target}) mismatch') 

227 debug.lv5('Waiting for ILM step to complete...') 

228 self.ilm_tracker.wait4complete() 

229 # Because the step is completed, we must now update OUR tracker to 

230 # reflect the updated ILM Explain information 

231 debug.lv5('Updating ILM tracker...') 

232 self.ilm_tracker.update() 

233 

234 # ILM snapshot mount phase. The biggest pain of them all... 

235 debug.lv3(f'Moving "{self.name}" to ILM phase "{target}"') 

236 self._mounted_step(target) 

237 debug.lv3(f'ILM advance to phase "{target}" completed') 

238 

239 # Record the snapshot in our tracker 

240 debug.lv5('Adding snapshot step to snapmgr...') 

241 self._add_snap_step() 

242 

243 @begin_end() 

244 def track_ilm(self, name: str) -> None: 

245 """ 

246 Get ILM phase information and put it in self.ilm_tracker 

247 Name as an arg makes it configurable 

248 """ 

249 if self.policy_name: 

250 debug.lv5(f'ILM policy name: {self.policy_name}') 

251 debug.lv5(f'Creating ILM tracker for "{name}"') 

252 self.ilm_tracker = IlmTracker(self.client, name) 

253 debug.lv5(f'Updating ILM tracker for "{name}"') 

254 self.ilm_tracker.update() 

255 else: 

256 debug.lv5('No ILM policy name. Skipping ILM tracking')