Coverage for /Users/buh/.pyenv/versions/3.12.2/envs/es-testbed/lib/python3.12/site-packages/es_testbed/entities/index.py: 38%

114 statements  

« prev     ^ index     » next       coverage.py v7.4.4, created at 2024-08-21 12:05 -0600

1"""Index Entity Class""" 

2 

3import typing as t 

4import logging 

5from os import getenv 

6from elasticsearch8.exceptions import BadRequestError 

7from es_wait import Exists, IlmPhase, IlmStep 

8from es_wait.exceptions import IlmWaitError 

9from es_testbed.defaults import ( 

10 PAUSE_DEFAULT, 

11 PAUSE_ENVVAR, 

12 TIMEOUT_DEFAULT, 

13 TIMEOUT_ENVVAR, 

14) 

15from es_testbed.entities.entity import Entity 

16from es_testbed.helpers.es_api import snapshot_name 

17from es_testbed.helpers.utils import mounted_name, prettystr 

18from es_testbed.ilm import IlmTracker 

19 

20if t.TYPE_CHECKING: 

21 from elasticsearch8 import Elasticsearch 

22 

23PAUSE_VALUE = float(getenv(PAUSE_ENVVAR, default=PAUSE_DEFAULT)) 

24TIMEOUT_VALUE = float(getenv(TIMEOUT_ENVVAR, default=TIMEOUT_DEFAULT)) 

25 

26logger = logging.getLogger(__name__) 

27 

28 

29class Index(Entity): 

30 """Index Entity Class""" 

31 

32 def __init__( 

33 self, 

34 client: 'Elasticsearch', 

35 name: t.Union[str, None] = None, 

36 snapmgr=None, 

37 policy_name: str = None, 

38 ): 

39 super().__init__(client=client, name=name) 

40 self.policy_name = policy_name 

41 self.ilm_tracker = None 

42 self.snapmgr = snapmgr 

43 

44 @property 

45 def _get_target(self) -> str: 

46 target = None 

47 phases = self.ilm_tracker.policy_phases 

48 curr = self.ilm_tracker.explain.phase 

49 if not bool(('cold' in phases) or ('frozen' in phases)): 

50 logger.info('ILM Policy for "%s" has no cold/frozen phases', self.name) 

51 target = curr # Keep the same 

52 if bool(('cold' in phases) and ('frozen' in phases)): 

53 if self.ilm_tracker.pname(curr) < self.ilm_tracker.pname('cold'): 

54 target = 'cold' 

55 elif curr == 'cold': 

56 target = 'frozen' 

57 elif self.ilm_tracker.pname(curr) >= self.ilm_tracker.pname('frozen'): 

58 target = curr 

59 elif bool(('cold' in phases) and ('frozen' not in phases)): 

60 target = 'cold' 

61 elif bool(('cold' not in phases) and ('frozen' in phases)): 

62 target = 'frozen' 

63 return target 

64 

65 @property 

66 def phase_tuple(self) -> t.Tuple[str, str]: 

67 """Return the current phase and the target phase as a Tuple""" 

68 return self.ilm_tracker.explain.phase, self._get_target 

69 

70 def _add_snap_step(self) -> None: 

71 logger.debug('Getting snapshot name for tracking...') 

72 snapname = snapshot_name(self.client, self.name) 

73 logger.debug('Snapshot %s backs %s', snapname, self.name) 

74 self.snapmgr.add_existing(snapname) 

75 

76 def _ilm_step(self) -> None: 

77 """Subroutine for waiting for an ILM step to complete""" 

78 step = { 

79 'phase': self.ilm_tracker.explain.phase, 

80 'action': self.ilm_tracker.explain.action, 

81 'name': self.ilm_tracker.explain.step, 

82 } 

83 logger.debug('%s: Current Step: %s', self.name, step) 

84 step = IlmStep( 

85 self.client, pause=PAUSE_VALUE, timeout=TIMEOUT_VALUE, name=self.name 

86 ) 

87 try: 

88 step.wait() 

89 logger.debug('ILM Step successful. The wait is over') 

90 except KeyError as exc: 

91 logger.error('KeyError: The index name has changed: "%s"', exc) 

92 raise exc 

93 except BadRequestError as exc: 

94 logger.error('Index not found') 

95 raise exc 

96 except IlmWaitError as exc: 

97 logger.error('Other IlmWait error encountered: "%s"', exc) 

98 raise exc 

99 

100 def _mounted_step(self, target: str) -> str: 

101 try: 

102 self.ilm_tracker.advance(phase=target) 

103 except BadRequestError as err: 

104 logger.critical('err: %s', prettystr(err)) 

105 raise BadRequestError from err 

106 # At this point, it's "in" a searchable tier, but the index name hasn't 

107 # changed yet 

108 newidx = mounted_name(self.name, target) 

109 logger.debug('Waiting for ILM phase change to complete. New index: %s', newidx) 

110 kwargs = { 

111 'name': newidx, 

112 'kind': 'index', 

113 'pause': PAUSE_VALUE, 

114 'timeout': TIMEOUT_VALUE, 

115 } 

116 test = Exists(self.client, **kwargs) 

117 test.wait() 

118 

119 # Update the name and run 

120 logger.debug('Updating self.name from "%s" to "%s"...', self.name, newidx) 

121 self.name = newidx 

122 

123 # Wait for the ILM steps to complete 

124 logger.debug('Waiting for the ILM steps to complete...') 

125 self._ilm_step() 

126 

127 # Track the new index 

128 logger.debug('Switching to track "%s" as self.name...', newidx) 

129 self.track_ilm(newidx) 

130 

131 # This is maybe unnecessary. This is for progressing ILM, e.g. from 

132 # hot -> warm -> cold -> frozen (and even through delete). 

133 # 

134 # def _loop_until_target(self) -> None: 

135 # current, target = self.phase_tuple 

136 # while current != target: 

137 # logger.debug( 

138 # 'Attempting to move %s to ILM phase %s', self.name, target 

139 # ) 

140 # self.ilm_tracker.advance(phase=target) 

141 # # At this point, it's "in" a searchable tier, but the index name hasn't 

142 # # changed yet 

143 # newidx = mounted_name(self.name, target) 

144 # logger.debug( 

145 # 'Waiting for ILM phase change to complete. New index: %s', newidx 

146 # ) 

147 # kwargs = { 

148 # 'name': newidx, 

149 # 'kind': 'index', 

150 # 'pause': PAUSE_VALUE, 

151 # 'timeout': TIMEOUT_VALUE, 

152 # } 

153 # test = Exists(self.client, **kwargs) 

154 # test.wait() 

155 # logger.info('ILM advance to phase %s completed', target) 

156 # self.name = newidx 

157 # self.track_ilm(self.name) # Refresh the ilm_tracker with the new name 

158 # current, target = self.phase_tuple 

159 

160 def manual_ss(self, scheme) -> None: 

161 """ 

162 If we are NOT using ILM but have specified searchable snapshots in the plan 

163 entities 

164 """ 

165 if 'target_tier' in scheme and scheme['target_tier'] in ['cold', 'frozen']: 

166 self.snapmgr.add(self.name, scheme['target_tier']) 

167 # Replace self.name with the renamed name 

168 self.name = mounted_name(self.name, scheme['target_tier']) 

169 

170 def mount_ss(self, scheme: dict) -> None: 

171 """If the index is planned to become a searchable snapshot, we do that now""" 

172 logger.debug('Checking if "%s" should be a searchable snapshot', self.name) 

173 if self.am_i_write_idx: 

174 logger.debug( 

175 '"%s" is the write_index. Cannot mount as searchable snapshot', 

176 self.name, 

177 ) 

178 return 

179 if not self.policy_name: # If we have this, chances are we have a policy 

180 logger.debug('No ILM policy for "%s". Trying manual...', self.name) 

181 self.manual_ss(scheme) 

182 return 

183 phase = self.ilm_tracker.next_phase 

184 current = self.ilm_tracker.explain.phase 

185 if current == 'new': 

186 # This is a problem. We need to be in 'hot', with rollover completed. 

187 logger.debug( 

188 'Our index is still in phase "%s"!. We need it to be in "%s"', 

189 current, 

190 phase, 

191 ) 

192 

193 phasenext = IlmPhase( 

194 self.client, 

195 pause=PAUSE_VALUE, 

196 timeout=TIMEOUT_VALUE, 

197 name=self.name, 

198 phase=self.ilm_tracker.next_phase, 

199 ) 

200 phasenext.wait() 

201 target = self._get_target 

202 if current != target: 

203 logger.debug('Current (%s) and target (%s) mismatch', current, target) 

204 self.ilm_tracker.wait4complete() 

205 # Because the step is completed, we must now update OUR tracker to 

206 # reflect the updated ILM Explain information 

207 self.ilm_tracker.update() 

208 

209 # ILM snapshot mount phase. The biggest pain of them all... 

210 logger.debug('Moving "%s" to ILM phase "%s"', self.name, target) 

211 self._mounted_step(target) 

212 logger.info('ILM advance to phase "%s" completed', target) 

213 

214 # Record the snapshot in our tracker 

215 self._add_snap_step() 

216 

217 def track_ilm(self, name: str) -> None: 

218 """ 

219 Get ILM phase information and put it in self.ilm_tracker 

220 Name as an arg makes it configurable 

221 """ 

222 if self.policy_name: 

223 self.ilm_tracker = IlmTracker(self.client, name) 

224 self.ilm_tracker.update()