Coverage for /Users/buh/.pyenv/versions/3.12.9/envs/es-testbed/lib/python3.12/site-packages/es_testbed/entities/index.py: 100%

127 statements  

« prev     ^ index     » next       coverage.py v7.6.12, created at 2025-03-17 19:30 -0600

1"""Index Entity Class""" 

2 

3import typing as t 

4import logging 

5from os import getenv 

6from elasticsearch8.exceptions import BadRequestError 

7from es_wait import Exists, IlmPhase, IlmStep 

8from es_wait.exceptions import EsWaitFatal, EsWaitTimeout 

9from es_testbed.defaults import ( 

10 PAUSE_DEFAULT, 

11 PAUSE_ENVVAR, 

12 TIMEOUT_DEFAULT, 

13 TIMEOUT_ENVVAR, 

14) 

15from es_testbed.exceptions import TestbedFailure 

16from es_testbed.entities.entity import Entity 

17from es_testbed.helpers.es_api import snapshot_name 

18from es_testbed.helpers.utils import mounted_name, prettystr 

19from es_testbed.ilm import IlmTracker 

20 

21if t.TYPE_CHECKING: 

22 from elasticsearch8 import Elasticsearch 

23 

24PAUSE_VALUE = float(getenv(PAUSE_ENVVAR, default=PAUSE_DEFAULT)) 

25TIMEOUT_VALUE = float(getenv(TIMEOUT_ENVVAR, default=TIMEOUT_DEFAULT)) 

26 

27logger = logging.getLogger(__name__) 

28 

29 

30class Index(Entity): 

31 """Index Entity Class""" 

32 

33 def __init__( 

34 self, 

35 client: 'Elasticsearch', 

36 name: t.Union[str, None] = None, 

37 snapmgr=None, 

38 policy_name: str = None, 

39 ): 

40 super().__init__(client=client, name=name) 

41 self.policy_name = policy_name 

42 self.ilm_tracker = None 

43 self.snapmgr = snapmgr 

44 

45 @property 

46 def _get_target(self) -> str: 

47 target = None 

48 phases = self.ilm_tracker.policy_phases 

49 curr = self.ilm_tracker.explain.phase 

50 if not bool(('cold' in phases) or ('frozen' in phases)): 

51 logger.info(f'ILM Policy for "{self.name}" has no cold/frozen phases') 

52 target = curr # Keep the same 

53 if bool(('cold' in phases) and ('frozen' in phases)): 

54 if self.ilm_tracker.pname(curr) < self.ilm_tracker.pname('cold'): 

55 target = 'cold' 

56 elif curr == 'cold': 

57 target = 'frozen' 

58 elif self.ilm_tracker.pname(curr) >= self.ilm_tracker.pname('frozen'): 

59 target = curr 

60 elif bool(('cold' in phases) and ('frozen' not in phases)): 

61 target = 'cold' 

62 elif bool(('cold' not in phases) and ('frozen' in phases)): 

63 target = 'frozen' 

64 return target 

65 

66 @property 

67 def phase_tuple(self) -> t.Tuple[str, str]: 

68 """Return the current phase and the target phase as a Tuple""" 

69 return self.ilm_tracker.explain.phase, self._get_target 

70 

71 def _add_snap_step(self) -> None: 

72 logger.debug('Getting snapshot name for tracking...') 

73 snapname = snapshot_name(self.client, self.name) 

74 logger.debug(f'Snapshot {snapname} backs {self.name}') 

75 self.snapmgr.add_existing(snapname) 

76 

77 def _ilm_step(self) -> None: 

78 """Subroutine for waiting for an ILM step to complete""" 

79 step = { 

80 'phase': self.ilm_tracker.explain.phase, 

81 'action': self.ilm_tracker.explain.action, 

82 'name': self.ilm_tracker.explain.step, 

83 } 

84 logger.debug(f'{self.name}: Current Step: {step}') 

85 step = IlmStep( 

86 self.client, pause=PAUSE_VALUE, timeout=TIMEOUT_VALUE, name=self.name 

87 ) 

88 try: 

89 self._wait_try(step.wait) 

90 except TestbedFailure as err: 

91 logger.error(err.message) 

92 raise err 

93 

94 def _wait_try(self, func: t.Callable) -> None: 

95 """Wait for an es-wait function to complete""" 

96 try: 

97 func() 

98 except EsWaitFatal as wait: 

99 # EsWaitFatal indicates we had more than the allowed number of exceptions 

100 msg = f'{wait.message}. Elapsed time: {wait.elapsed}. Errors: {wait.errors}' 

101 raise TestbedFailure(msg) from wait 

102 except EsWaitTimeout as wait: 

103 # EsWaitTimeout indicates we hit the timeout 

104 msg = f'{wait.message}. Total elapsed time: {wait.elapsed}.' 

105 raise TestbedFailure(msg) from wait 

106 except Exception as err: 

107 raise TestbedFailure(f'General Exception caught: {prettystr(err)}') from err 

108 

109 def _mounted_step(self, target: str) -> str: 

110 try: 

111 self.ilm_tracker.advance(phase=target) 

112 except BadRequestError as err: 

113 logger.critical(f'err: {prettystr(err)}') 

114 raise err # Re-raise after logging 

115 # At this point, it's "in" a searchable tier, but the index name hasn't 

116 # changed yet 

117 newidx = mounted_name(self.name, target) 

118 logger.debug(f'Waiting for ILM phase change to complete. New index: {newidx}') 

119 wait_kwargs = { 

120 'name': newidx, 

121 'kind': 'index', 

122 'pause': PAUSE_VALUE, 

123 'timeout': TIMEOUT_VALUE, 

124 } 

125 

126 test = Exists(self.client, **wait_kwargs) 

127 try: 

128 self._wait_try(test.wait) 

129 except TestbedFailure as err: 

130 logger.error(err.message) 

131 raise err 

132 

133 # Update the name and run 

134 logger.debug(f'Updating self.name from "{self.name}" to "{newidx}"...') 

135 self.name = newidx 

136 

137 # Wait for the ILM steps to complete 

138 logger.debug('Waiting for the ILM steps to complete...') 

139 self._ilm_step() 

140 

141 # Track the new index 

142 logger.debug(f'Switching to track "{newidx}" as self.name...') 

143 self.track_ilm(newidx) 

144 

145 def manual_ss(self, scheme: t.Dict[str, t.Any]) -> None: 

146 """ 

147 If we are NOT using ILM but have specified searchable snapshots in the plan 

148 entities 

149 """ 

150 if 'target_tier' in scheme and scheme['target_tier'] in ['cold', 'frozen']: 

151 self.snapmgr.add(self.name, scheme['target_tier']) 

152 # Replace self.name with the renamed name 

153 self.name = mounted_name(self.name, scheme['target_tier']) 

154 

155 def mount_ss(self, scheme: dict) -> None: 

156 """If the index is planned to become a searchable snapshot, we do that now""" 

157 logger.debug(f'Checking if "{self.name}" should be a searchable snapshot') 

158 if self.am_i_write_idx: 

159 logger.debug( 

160 f'"{self.name}" is the write_index. Cannot mount as searchable ' 

161 f'snapshot' 

162 ) 

163 return 

164 if not self.policy_name: # If we have this, chances are we have a policy 

165 logger.debug(f'No ILM policy for "{self.name}". Trying manual...') 

166 self.manual_ss(scheme) 

167 return 

168 phase = self.ilm_tracker.next_phase 

169 current = self.ilm_tracker.explain.phase 

170 if current == 'new': 

171 # This is a problem. We need to be in 'hot', with rollover completed. 

172 logger.debug( 

173 f'Our index is still in phase "{current}"!. ' 

174 f'We need it to be in "{phase}"' 

175 ) 

176 

177 phasenext = IlmPhase( 

178 self.client, 

179 pause=PAUSE_VALUE, 

180 timeout=TIMEOUT_VALUE, 

181 name=self.name, 

182 phase=self.ilm_tracker.next_phase, 

183 ) 

184 try: 

185 self._wait_try(phasenext.wait) 

186 except TestbedFailure as err: 

187 logger.error(err.message) 

188 raise err 

189 target = self._get_target 

190 if current != target: 

191 logger.debug(f'Current ({current}) and target ({target}) mismatch') 

192 self.ilm_tracker.wait4complete() 

193 # Because the step is completed, we must now update OUR tracker to 

194 # reflect the updated ILM Explain information 

195 self.ilm_tracker.update() 

196 

197 # ILM snapshot mount phase. The biggest pain of them all... 

198 logger.debug(f'Moving "{self.name}" to ILM phase "{target}"') 

199 self._mounted_step(target) 

200 logger.info(f'ILM advance to phase "{target}" completed') 

201 

202 # Record the snapshot in our tracker 

203 self._add_snap_step() 

204 

205 def track_ilm(self, name: str) -> None: 

206 """ 

207 Get ILM phase information and put it in self.ilm_tracker 

208 Name as an arg makes it configurable 

209 """ 

210 if self.policy_name: 

211 self.ilm_tracker = IlmTracker(self.client, name) 

212 self.ilm_tracker.update()