Coverage for /Users/buh/.pyenv/versions/3.12.9/envs/es-testbed/lib/python3.12/site-packages/es_testbed/entities/index.py: 100%

178 statements  

« prev     ^ index     » next       coverage.py v7.6.12, created at 2025-03-31 14:55 -0600

1"""Index Entity Class""" 

2 

3import typing as t 

4import logging 

5from os import getenv 

6import tiered_debug as debug 

7from elasticsearch8.exceptions import BadRequestError 

8from es_wait import Exists, IlmPhase, IlmStep 

9from es_wait.exceptions import EsWaitFatal, EsWaitTimeout 

10from es_testbed.defaults import ( 

11 PAUSE_DEFAULT, 

12 PAUSE_ENVVAR, 

13 TIMEOUT_DEFAULT, 

14 TIMEOUT_ENVVAR, 

15) 

16from es_testbed.exceptions import TestbedFailure 

17from es_testbed.entities.entity import Entity 

18from es_testbed.helpers.es_api import snapshot_name 

19from es_testbed.helpers.utils import mounted_name, prettystr 

20from es_testbed.ilm import IlmTracker 

21 

22if t.TYPE_CHECKING: 

23 from elasticsearch8 import Elasticsearch 

24 

25PAUSE_VALUE = float(getenv(PAUSE_ENVVAR, default=PAUSE_DEFAULT)) 

26TIMEOUT_VALUE = float(getenv(TIMEOUT_ENVVAR, default=TIMEOUT_DEFAULT)) 

27 

28logger = logging.getLogger(__name__) 

29 

30 

31class Index(Entity): 

32 """Index Entity Class""" 

33 

34 def __init__( 

35 self, 

36 client: 'Elasticsearch', 

37 name: t.Union[str, None] = None, 

38 snapmgr=None, 

39 policy_name: str = None, 

40 ): 

41 debug.lv2('Initializing Index entity object...') 

42 super().__init__(client=client, name=name) 

43 self.policy_name = policy_name 

44 self.ilm_tracker = None 

45 self.snapmgr = snapmgr 

46 debug.lv3('Index entity object initialized') 

47 

48 @property 

49 def _get_target(self) -> str: 

50 target = None 

51 phases = self.ilm_tracker.policy_phases 

52 curr = self.ilm_tracker.explain.phase 

53 if not bool(('cold' in phases) or ('frozen' in phases)): 

54 debug.lv3(f'ILM Policy for "{self.name}" has no cold/frozen phases') 

55 target = curr # Keep the same 

56 if bool(('cold' in phases) and ('frozen' in phases)): 

57 if self.ilm_tracker.pname(curr) < self.ilm_tracker.pname('cold'): 

58 target = 'cold' 

59 elif curr == 'cold': 

60 target = 'frozen' 

61 elif self.ilm_tracker.pname(curr) >= self.ilm_tracker.pname('frozen'): 

62 target = curr 

63 elif bool(('cold' in phases) and ('frozen' not in phases)): 

64 target = 'cold' 

65 elif bool(('cold' not in phases) and ('frozen' in phases)): 

66 target = 'frozen' 

67 return target 

68 

69 @property 

70 def phase_tuple(self) -> t.Tuple[str, str]: 

71 """Return the current phase and the target phase as a Tuple""" 

72 return self.ilm_tracker.explain.phase, self._get_target 

73 

74 def _add_snap_step(self) -> None: 

75 debug.lv2('Starting method...') 

76 debug.lv4('Getting snapshot name for tracking...') 

77 snapname = snapshot_name(self.client, self.name) 

78 debug.lv5(f'Snapshot {snapname} backs {self.name}') 

79 self.snapmgr.add_existing(snapname) 

80 debug.lv3('Exiting method') 

81 

82 def _ilm_step(self) -> None: 

83 """Subroutine for waiting for an ILM step to complete""" 

84 debug.lv2('Starting method...') 

85 step = { 

86 'phase': self.ilm_tracker.explain.phase, 

87 'action': self.ilm_tracker.explain.action, 

88 'name': self.ilm_tracker.explain.step, 

89 } 

90 debug.lv5(f'{self.name}: Current Step: {step}') 

91 step = IlmStep( 

92 self.client, pause=PAUSE_VALUE, timeout=TIMEOUT_VALUE, name=self.name 

93 ) 

94 try: 

95 debug.lv4('TRY: Waiting for ILM step to complete...') 

96 self._wait_try(step.wait) 

97 except TestbedFailure as err: 

98 logger.error(err.message) 

99 debug.lv3('Exiting method, raising exception') 

100 debug.lv5(f'Exception: {prettystr(err)}') 

101 raise err 

102 debug.lv3('Exiting method') 

103 

104 def _wait_try(self, func: t.Callable) -> None: 

105 """Wait for an es-wait function to complete""" 

106 debug.lv2('Starting method...') 

107 try: 

108 debug.lv4('TRY: To run func()...') 

109 func() 

110 except EsWaitFatal as wait: 

111 # EsWaitFatal indicates we had more than the allowed number of exceptions 

112 msg = f'{wait.message}. Elapsed time: {wait.elapsed}. Errors: {wait.errors}' 

113 debug.lv3('Exiting method, raising exception') 

114 debug.lv5(f'Exception: {prettystr(wait)}') 

115 raise TestbedFailure(msg) from wait 

116 except EsWaitTimeout as wait: 

117 # EsWaitTimeout indicates we hit the timeout 

118 msg = f'{wait.message}. Total elapsed time: {wait.elapsed}.' 

119 debug.lv3('Exiting method, raising exception') 

120 debug.lv5(f'Exception: {prettystr(wait)}') 

121 raise TestbedFailure(msg) from wait 

122 except Exception as err: 

123 debug.lv3('Exiting method, raising exception') 

124 debug.lv5(f'Exception: {prettystr(err)}') 

125 raise TestbedFailure(f'General Exception caught: {prettystr(err)}') from err 

126 debug.lv3('Exiting method') 

127 

128 def _mounted_step(self, target: str) -> str: 

129 debug.lv2('Starting method...') 

130 try: 

131 debug.lv4(f'TRY: Moving "{self.name}" to ILM phase "{target}"') 

132 self.ilm_tracker.advance(phase=target) 

133 except BadRequestError as err: 

134 logger.critical(f'err: {prettystr(err)}') 

135 debug.lv3('Exiting method, raising exception') 

136 debug.lv5(f'Exception: {prettystr(err)}') 

137 raise err # Re-raise after logging 

138 # At this point, it's "in" a searchable tier, but the index name hasn't 

139 # changed yet 

140 newidx = mounted_name(self.name, target) 

141 debug.lv3(f'Waiting for ILM phase change to complete. New index: {newidx}') 

142 wait_kwargs = { 

143 'name': newidx, 

144 'kind': 'index', 

145 'pause': PAUSE_VALUE, 

146 'timeout': TIMEOUT_VALUE, 

147 } 

148 

149 test = Exists(self.client, **wait_kwargs) 

150 debug.lv5(f'Exists response: {prettystr(test)}') 

151 try: 

152 debug.lv4(f'TRY: Waiting for "{newidx}" to exist...') 

153 self._wait_try(test.wait) 

154 except TestbedFailure as err: 

155 logger.error(err.message) 

156 debug.lv3('Exiting method, raising exception') 

157 debug.lv5(f'Exception: {prettystr(err)}') 

158 raise err 

159 

160 # Update the name and run 

161 debug.lv3(f'Updating self.name from "{self.name}" to "{newidx}"...') 

162 self.name = newidx 

163 

164 # Wait for the ILM steps to complete 

165 debug.lv3('Waiting for the ILM steps to complete...') 

166 self._ilm_step() 

167 

168 # Track the new index 

169 debug.lv3(f'Switching to track "{newidx}" as self.name...') 

170 self.track_ilm(newidx) 

171 debug.lv3('Exiting method') 

172 

173 def manual_ss(self, scheme: t.Dict[str, t.Any]) -> None: 

174 """ 

175 If we are NOT using ILM but have specified searchable snapshots in the plan 

176 entities 

177 """ 

178 debug.lv2('Starting method...') 

179 if 'target_tier' in scheme and scheme['target_tier'] in ['cold', 'frozen']: 

180 debug.lv3('Manually mounting as searchable snapshot...') 

181 debug.lv5( 

182 f'Mounting "{self.name}" as searchable snapshot ' 

183 f'({scheme["target_tier"]})' 

184 ) 

185 self.snapmgr.add(self.name, scheme['target_tier']) 

186 # Replace self.name with the renamed name 

187 self.name = mounted_name(self.name, scheme['target_tier']) 

188 debug.lv3('Exiting method') 

189 

190 def mount_ss(self, scheme: dict) -> None: 

191 """If the index is planned to become a searchable snapshot, we do that now""" 

192 debug.lv2('Starting method...') 

193 debug.lv3(f'Checking if "{self.name}" should be a searchable snapshot') 

194 if self.am_i_write_idx: 

195 debug.lv5( 

196 f'"{self.name}" is the write_index. Cannot mount as searchable ' 

197 f'snapshot' 

198 ) 

199 debug.lv3('Exiting method') 

200 return 

201 if not self.policy_name: # If we have this, chances are we have a policy 

202 debug.lv3(f'No ILM policy for "{self.name}". Trying manual...') 

203 self.manual_ss(scheme) 

204 debug.lv3('Exiting method') 

205 return 

206 phase = self.ilm_tracker.next_phase 

207 debug.lv5(f'Next phase for "{self.name}" = {phase}') 

208 current = self.ilm_tracker.explain.phase 

209 debug.lv5(f'Current phase for "{self.name}" = {current}') 

210 if current == 'new': 

211 # This is a problem. We need to be in 'hot', with rollover completed. 

212 debug.lv3( 

213 f'Our index is still in phase "{current}"!. ' 

214 f'We need it to be in "{phase}"' 

215 ) 

216 

217 phasenext = IlmPhase( 

218 self.client, 

219 pause=PAUSE_VALUE, 

220 timeout=TIMEOUT_VALUE, 

221 name=self.name, 

222 phase=self.ilm_tracker.next_phase, 

223 ) 

224 try: 

225 debug.lv4('TRY: Waiting for ILM phase to complete...') 

226 self._wait_try(phasenext.wait) 

227 except TestbedFailure as err: 

228 logger.error(err.message) 

229 debug.lv3('Exiting method, raising exception') 

230 debug.lv5(f'Exception: {prettystr(err)}') 

231 raise err 

232 target = self._get_target 

233 debug.lv5(f'Target phase for "{self.name}" = {target}') 

234 if current != target: 

235 debug.lv5(f'Current ({current}) and target ({target}) mismatch') 

236 debug.lv5('Waiting for ILM step to complete...') 

237 self.ilm_tracker.wait4complete() 

238 # Because the step is completed, we must now update OUR tracker to 

239 # reflect the updated ILM Explain information 

240 debug.lv5('Updating ILM tracker...') 

241 self.ilm_tracker.update() 

242 

243 # ILM snapshot mount phase. The biggest pain of them all... 

244 debug.lv3(f'Moving "{self.name}" to ILM phase "{target}"') 

245 self._mounted_step(target) 

246 debug.lv3(f'ILM advance to phase "{target}" completed') 

247 

248 # Record the snapshot in our tracker 

249 debug.lv5('Adding snapshot step to snapmgr...') 

250 self._add_snap_step() 

251 debug.lv3('Exiting method') 

252 

253 def track_ilm(self, name: str) -> None: 

254 """ 

255 Get ILM phase information and put it in self.ilm_tracker 

256 Name as an arg makes it configurable 

257 """ 

258 debug.lv2('Starting method...') 

259 if self.policy_name: 

260 debug.lv5(f'ILM policy name: {self.policy_name}') 

261 debug.lv5(f'Creating ILM tracker for "{name}"') 

262 self.ilm_tracker = IlmTracker(self.client, name) 

263 debug.lv5(f'Updating ILM tracker for "{name}"') 

264 self.ilm_tracker.update() 

265 else: 

266 debug.lv5('No ILM policy name. Skipping ILM tracking') 

267 debug.lv3('Exiting method')