Coverage for /Users/buh/.pyenv/versions/3.12.2/envs/es-testbed/lib/python3.12/site-packages/es_testbed/helpers/es_api.py: 74%

213 statements  

« prev     ^ index     » next       coverage.py v7.4.4, created at 2024-04-25 19:21 -0600

1"""Functions that make Elasticsearch API Calls""" 

2import typing as t 

3from os import getenv 

4from elasticsearch8 import Elasticsearch, exceptions as esx 

5from es_wait import Exists, Snapshot 

6from es_testbed.defaults import MAPPING, PAUSE_DEFAULT, PAUSE_ENVVAR 

7from es_testbed import exceptions as exc 

8from es_testbed.helpers.utils import doc_gen, get_routing, getlogger, mounted_name, storage_type 

9LOGGER = getlogger(__name__) 

10PAUSE_VALUE = float(getenv(PAUSE_ENVVAR, default=PAUSE_DEFAULT)) 

11# pylint: disable=broad-except 

12 

13def emap(kind: str, es: Elasticsearch, value=None): 

14 """Return a value from a dictionary""" 

15 _ = { 

16 'alias': { 

17 'delete': es.indices.delete_alias, 

18 'exists': es.indices.exists_alias, 

19 'get': es.indices.get_alias, 

20 'kwargs': {'index': value, 'expand_wildcards': ['open', 'closed']}, 

21 'plural': 'alias(es)' 

22 }, 

23 'data_stream': { 

24 'delete': es.indices.delete_data_stream, 

25 'exists': es.indices.exists, 

26 'get': es.indices.get_data_stream, 

27 'kwargs': {'name': value, 'expand_wildcards': ['open', 'closed']}, 

28 'plural': 'data_stream(s)', 

29 'key': 'data_streams', 

30 }, 

31 'index': { 

32 'delete': es.indices.delete, 

33 'exists': es.indices.exists, 

34 'get': es.indices.get, 

35 'kwargs': {'index': value, 'expand_wildcards': ['open', 'closed']}, 

36 'plural': 'index(es)' 

37 }, 

38 'template': { 

39 'delete': es.indices.delete_index_template, 

40 'exists': es.indices.exists_index_template, 

41 'get': es.indices.get_index_template, 

42 'kwargs': {'name': value}, 

43 'plural': 'index template(s)', 

44 'key': 'index_templates', 

45 }, 

46 'ilm': { 

47 'delete': es.ilm.delete_lifecycle, 

48 'exists': es.ilm.get_lifecycle, 

49 'get': es.ilm.get_lifecycle, 

50 'kwargs': {'name': value}, 

51 'plural': 'ilm policy(ies)' 

52 }, 

53 'component': { 

54 'delete': es.cluster.delete_component_template, 

55 'exists': es.cluster.exists_component_template, 

56 'get': es.cluster.get_component_template, 

57 'kwargs': {'name': value}, 

58 'plural': 'component template(s)', 

59 'key': 'component_templates', 

60 }, 

61 'snapshot': { 

62 'delete': es.snapshot.delete, 

63 'exists': es.snapshot.get, 

64 'get': es.snapshot.get, 

65 'kwargs': {}, 

66 'plural': 'snapshot(s)' 

67 } 

68 } 

69 return _[kind] 

70 

71def change_ds(client: Elasticsearch, actions: dict=None) -> None: 

72 """Change/Modify/Update a datastream""" 

73 try: 

74 client.indices.modify_data_stream(actions=actions, body=None) 

75 except Exception as err: 

76 raise exc.ResultNotExpected(f'Unable to modify datastreams. {err}') from err 

77 

78def create_data_stream(client: Elasticsearch, name: str) -> None: 

79 """Create a datastream""" 

80 try: 

81 client.indices.create_data_stream(name=name) 

82 test = Exists(client, name=name, kind='datastream', pause=PAUSE_VALUE) 

83 test.wait_for_it() 

84 except Exception as err: 

85 raise exc.TestbedFailure( 

86 f'Unable to create datastream {name}. Error: {err}') from err 

87 

88def create_index( 

89 client: Elasticsearch, 

90 name: str, 

91 aliases: dict=None, 

92 settings: dict=None, 

93 tier: str='hot' 

94 ) -> None: 

95 """Create named index""" 

96 if not settings: 

97 settings = get_routing(tier=tier) 

98 else: 

99 settings.update(get_routing(tier=tier)) 

100 client.indices.create( 

101 index=name, 

102 aliases=aliases, 

103 mappings=MAPPING, 

104 settings=settings 

105 ) 

106 try: 

107 test = Exists(client, name=name, kind='index', pause=PAUSE_VALUE) 

108 test.wait_for_it() 

109 except exc.TimeoutException as err: 

110 raise exc.ResultNotExpected(f'Failed to create index {name}') from err 

111 return exists(client, 'index', name) 

112 

113def delete(client: Elasticsearch, kind: str, name: str, repository: str=None): 

114 """Delete the named object of type kind""" 

115 which = emap(kind, client) 

116 func = which['delete'] 

117 if name is None: # Typically only with ilm 117 ↛ 118line 117 didn't jump to line 118, because the condition on line 117 was never true

118 LOGGER.debug('"%s" has a None value for name', kind) 

119 return 

120 try: 

121 if kind == 'snapshot': 

122 func(snapshot=name, repository=repository) 

123 elif kind == 'index': 

124 func(index=name) 

125 else: 

126 func(name=name) 

127 except esx.NotFoundError: 

128 LOGGER.warning('%s named %s not found.', kind, name) 

129 except Exception as err: 

130 raise exc.ResultNotExpected(f'Unexpected result: {err}') from err 

131 if exists(client, kind, name, repository=repository): 131 ↛ 132line 131 didn't jump to line 132, because the condition on line 131 was never true

132 LOGGER.critical('Unable to delete "%s" %s', kind, name) 

133 raise exc.ResultNotExpected(f'{kind} "{name}" still exists.') 

134 LOGGER.info('Successfully deleted %s: "%s"', which['plural'], name) 

135 

136def do_snap(client: Elasticsearch, repo: str, snap: str, idx: str, tier: str='cold') -> None: 

137 """Perform a snapshot""" 

138 client.snapshot.create(repository=repo, snapshot=snap, indices=idx) 

139 test = Snapshot(client, action='snapshot', snapshot=snap, repository=repo, pause=1, timeout=60) 

140 test.wait_for_it() 

141 

142 # Mount the index accordingly 

143 client.searchable_snapshots.mount( 

144 repository=repo, snapshot=snap, index=idx, index_settings=get_routing(tier=tier), 

145 renamed_index=mounted_name(idx, tier), storage=storage_type(tier), 

146 wait_for_completion=True) 

147 # Fix aliases 

148 fix_aliases(client, idx, mounted_name(idx, tier)) 

149 

150def exists(client: Elasticsearch, kind: str, name: str, repository: str=None) -> bool: 

151 """Return boolean existence of the named kind of object""" 

152 if name is None: 152 ↛ 153line 152 didn't jump to line 153, because the condition on line 152 was never true

153 return False 

154 retval = True 

155 func = emap(kind, client)['exists'] 

156 try: 

157 if kind == 'snapshot': 

158 retval = func(snapshot=name, repository=repository) 

159 elif kind == 'ilm': 

160 retval = func(name=name) 

161 elif kind in ['index', 'data_stream']: 

162 retval = func(index=name) 

163 else: 

164 retval = func(name=name) 

165 except esx.NotFoundError: 165 ↛ 167line 165 didn't jump to line 167

166 retval = False 

167 except Exception as err: 

168 raise exc.ResultNotExpected(f'Unexpected result: {err}') from err 

169 return retval 

170 

171def fill_index( 

172 client: Elasticsearch, 

173 name: str=None, 

174 count: int=None, 

175 start_num: int=None, 

176 match: bool=True 

177 ) -> None: 

178 """ 

179 Create and fill the named index with mappings and settings as directed 

180 

181 :param client: ES client 

182 :param name: Index name 

183 :param count: The number of docs to create 

184 :param start_number: Where to start the incrementing number 

185 :param match: Whether to use the default values for key (True) or random strings (False) 

186 

187 :type client: es 

188 :type name: str 

189 :type count: int 

190 :type start_number: int 

191 :type match: bool 

192 

193 :rtype: None 

194 :returns: No return value 

195 """ 

196 for doc in doc_gen(count=count, start_at=start_num, match=match): 

197 client.index(index=name, document=doc) 

198 client.indices.flush(index=name) 

199 client.indices.refresh(index=name) 

200 

201def find_write_index(client: Elasticsearch, name: str) -> t.AnyStr: 

202 """Find the write_index for an alias by searching any index the alias points to""" 

203 retval = None 

204 for alias in get_aliases(client, name): 

205 retval = get_write_index(client, alias) 

206 if retval: 206 ↛ 204line 206 didn't jump to line 204, because the condition on line 206 was never false

207 break 

208 return retval 

209 

210def fix_aliases(client: Elasticsearch, oldidx: str, newidx: str) -> None: 

211 """Fix aliases using the new and old index names as data""" 

212 # Delete the original index 

213 client.indices.delete(index=oldidx) 

214 # Add the original index name as an alias to the mounted index 

215 client.indices.put_alias(index=f'{newidx}', name=oldidx) 

216 

217def get(client: Elasticsearch, kind: str, pattern: str) -> t.Sequence[str]: 

218 """get any/all objects of type kind matching pattern""" 

219 if pattern is None: 219 ↛ 220line 219 didn't jump to line 220, because the condition on line 219 was never true

220 msg = f'"{kind}" has a None value for pattern' 

221 LOGGER.error(msg) 

222 raise exc.TestbedMisconfig(msg) 

223 which = emap(kind, client, value=pattern) 

224 func = which['get'] 

225 kwargs = which['kwargs'] 

226 try: 

227 result = func(**kwargs) 

228 except Exception as err: 

229 raise exc.ResultNotExpected(f'Unexpected result: {err}') from err 

230 if kind in ['data_stream', 'template', 'component']: 

231 retval = [x['name'] for x in result[which['key']]] 

232 else: 

233 # ['alias', 'ilm', 'index'] 

234 retval = list(result.keys()) 

235 return retval 

236 

237def get_aliases(client: Elasticsearch, name: str) -> t.Sequence[str]: 

238 """Get aliases from index 'name'""" 

239 res = client.indices.get(index=name) 

240 try: 

241 retval = list(res[name]['aliases'].keys()) 

242 except KeyError: 

243 retval = None 

244 return retval 

245 

246def get_backing_indices(client: Elasticsearch, name: str) -> t.Sequence[str]: 

247 """Get the backing indices from the named data_stream""" 

248 resp = resolver(client, name) 

249 data_streams = resp['data_streams'] 

250 retval = [] 

251 if data_streams: 251 ↛ 255line 251 didn't jump to line 255, because the condition on line 251 was never false

252 if len(data_streams) > 1: 252 ↛ 253line 252 didn't jump to line 253, because the condition on line 252 was never true

253 raise exc.ResultNotExpected(f'Expected only a single data_stream matching {name}') 

254 retval = data_streams[0]['backing_indices'] 

255 return retval 

256 

257def get_ds_current(client: Elasticsearch, name: str) -> str: 

258 """ 

259 Find which index is the current 'write' index of the datastream 

260 This is best accomplished by grabbing the last backing_index 

261 """ 

262 backers = get_backing_indices(client, name) 

263 retval = None 

264 if backers: 264 ↛ 266line 264 didn't jump to line 266, because the condition on line 264 was never false

265 retval = backers[-1] 

266 return retval 

267 

268def get_ilm(client: Elasticsearch, pattern: str) -> t.Union[t.Dict[str,str], None]: 

269 """Get any ILM entity in ES that matches pattern""" 

270 try: 

271 return client.ilm.get_lifecycle(name=pattern) 

272 except Exception as err: 

273 msg = f'Unable to get ILM lifecycle matching {pattern}. Error: {err}' 

274 LOGGER.critical(msg) 

275 raise exc.ResultNotExpected(msg) from err 

276 

277def get_ilm_phases(client: Elasticsearch, name: str) -> dict: 

278 """Return the policy/phases part of the ILM policy identified by 'name'""" 

279 ilm = get_ilm(client, name) 

280 try: 

281 return ilm[name]['policy']['phases'] 

282 except KeyError as err: 

283 msg = f'Unable to get ILM lifecycle named {name}. Error: {err}' 

284 LOGGER.critical(msg) 

285 raise exc.ResultNotExpected(msg) from err 

286 

287def get_write_index(client: Elasticsearch, name: str) -> str: 

288 """ 

289 Calls :py:meth:`~.elasticsearch.client.IndicesClient.get_alias` 

290 

291 :param client: A client connection object 

292 :param name: An alias name 

293 

294 :type client: :py:class:`~.elasticsearch.Elasticsearch` 

295 :type name: str 

296 

297 :returns: The the index name associated with the alias that is designated ``is_write_index`` 

298 :rtype: str 

299 """ 

300 response = client.indices.get_alias(index=name) 

301 retval = None 

302 for index in list(response.keys()): 302 ↛ 309line 302 didn't jump to line 309, because the loop on line 302 didn't complete

303 try: 

304 if response[index]['aliases'][name]['is_write_index']: 

305 retval = index 

306 break 

307 except KeyError: 

308 continue 

309 return retval 

310 

311def snapshot_name(client: Elasticsearch, name: str) -> t.Union[t.AnyStr, None]: 

312 """Get the name of the snapshot behind the mounted index data""" 

313 res = {} 

314 if exists(client, 'index', name): # Can jump straight to nested keys if it exists 314 ↛ 316line 314 didn't jump to line 316, because the condition on line 314 was never false

315 res = client.indices.get(index=name)[name]['settings']['index'] 

316 try: 

317 retval = res['store']['snapshot']['snapshot_name'] 

318 except KeyError: 

319 LOGGER.error('%s is not a searchable snapshot') 

320 retval = None 

321 return retval 

322 

323def ilm_explain(client: Elasticsearch, name: str) -> t.Union[t.Dict, None]: 

324 """Return the results from the ILM Explain API call for the named index""" 

325 try: 

326 retval = client.ilm.explain_lifecycle(index=name)['indices'][name] 

327 except KeyError: 

328 LOGGER.debug('Index name changed') 

329 new = list(client.ilm.explain_lifecycle(index=name)['indices'].keys())[0] 

330 retval = client.ilm.explain_lifecycle(index=new)['indices'][new] 

331 except esx.NotFoundError as err: 331 ↛ 334line 331 didn't jump to line 334

332 LOGGER.warning('Datastream/Index Name changed. %s was not found', name) 

333 raise exc.NameChanged(f'{name} was not found, likely due to a name change') from err 

334 except Exception as err: 

335 msg = f'Unable to get ILM information for index {name}' 

336 LOGGER.critical(msg) 

337 raise exc.ResultNotExpected(f'{msg}. Exception: {err}') from err 

338 return retval 

339 

340def ilm_move(client: Elasticsearch, name: str, current_step: dict, next_step: dict) -> None: 

341 """Move index 'name' from the current step to the next step""" 

342 try: 

343 client.ilm.move_to_step(index=name, current_step=current_step, next_step=next_step) 

344 except Exception as err: 

345 msg = f'Unable to move index {name} to ILM next step: {next}. Error: {err}' 

346 LOGGER.critical(msg) 

347 raise exc.ResultNotExpected(msg) 

348 

349def put_comp_tmpl(client: Elasticsearch, name: str, component: dict) -> None: 

350 """Publish a component template""" 

351 try: 

352 client.cluster.put_component_template(name=name, template=component, create=True) 

353 test = Exists(client, name=name, kind='component', pause=PAUSE_VALUE) 

354 test.wait_for_it() 

355 except Exception as err: 

356 raise exc.TestbedFailure( 

357 f'Unable to create component template {name}. Error: {err}') from err 

358 

359def put_idx_tmpl( 

360 client, name: str, index_patterns: list, components: list, 

361 data_stream: dict=None) -> None: 

362 """Publish an index template""" 

363 try: 

364 client.indices.put_index_template( 

365 name=name, 

366 composed_of=components, 

367 data_stream=data_stream, 

368 index_patterns=index_patterns, 

369 create=True, 

370 ) 

371 test = Exists(client, name=name, kind='template', pause=PAUSE_VALUE) 

372 test.wait_for_it() 

373 except Exception as err: 

374 raise exc.TestbedFailure( 

375 f'Unable to create index template {name}. Error: {err}') from err 

376 

377def put_ilm(client: Elasticsearch, name: str, policy: dict=None) -> None: 

378 """Publish an ILM Policy""" 

379 try: 

380 client.ilm.put_lifecycle(name=name, policy=policy) 

381 except Exception as err: 

382 raise exc.TestbedFailure( 

383 f'Unable to put index lifecycle policy {name}. Error: {err}') from err 

384 

385def resolver(client: Elasticsearch, name: str) -> dict: 

386 """ 

387 Resolve details about the entity, be it an index, alias, or data_stream 

388  

389 Because you can pass search patterns and aliases as name, each element comes back as an array: 

390  

391 {'indices': [], 'aliases': [], 'data_streams': []} 

392  

393 If you only resolve a single index or data stream, you will still have a 1-element list 

394 """ 

395 return client.indices.resolve_index(name=name, expand_wildcards=['open', 'closed']) 

396 

397def rollover(client: Elasticsearch, name: str) -> None: 

398 """Rollover alias or datastream identified by name""" 

399 client.indices.rollover(alias=name, wait_for_active_shards='all')