Coverage for src/es_testbed/helpers/es_api.py: 61%

209 statements  

« prev     ^ index     » next       coverage.py v7.5.0, created at 2024-04-23 15:01 -0600

1"""Functions that make Elasticsearch API Calls""" 

2import typing as t 

3from os import getenv 

4from elasticsearch8 import Elasticsearch, exceptions as esx 

5from es_wait import Exists, Snapshot 

6from es_testbed.defaults import MAPPING, PAUSE_DEFAULT, PAUSE_ENVVAR 

7from es_testbed import exceptions as exc 

8from es_testbed.helpers.utils import doc_gen, get_routing, getlogger, mounted_name, storage_type 

9LOGGER = getlogger(__name__) 

10PAUSE_VALUE = float(getenv(PAUSE_ENVVAR, default=PAUSE_DEFAULT)) 

11# pylint: disable=broad-except 

12 

13def emap(kind: str, es: Elasticsearch, value=None): 

14 """Return a value from a dictionary""" 

15 _ = { 

16 'alias': { 

17 'delete': es.indices.delete_alias, 

18 'exists': es.indices.exists_alias, 

19 'get': es.indices.get_alias, 

20 'kwargs': {'index': value, 'expand_wildcards': ['open', 'closed']}, 

21 'plural': 'alias(es)' 

22 }, 

23 'data_stream': { 

24 'delete': es.indices.delete_data_stream, 

25 'exists': es.indices.exists, 

26 'get': es.indices.get_data_stream, 

27 'kwargs': {'name': value, 'expand_wildcards': ['open', 'closed']}, 

28 'plural': 'data_stream(s)', 

29 'key': 'data_streams', 

30 }, 

31 'index': { 

32 'delete': es.indices.delete, 

33 'exists': es.indices.exists, 

34 'get': es.indices.get, 

35 'kwargs': {'index': value, 'expand_wildcards': ['open', 'closed']}, 

36 'plural': 'index(es)' 

37 }, 

38 'template': { 

39 'delete': es.indices.delete_index_template, 

40 'exists': es.indices.exists_index_template, 

41 'get': es.indices.get_index_template, 

42 'kwargs': {'name': value}, 

43 'plural': 'index template(s)', 

44 'key': 'index_templates', 

45 }, 

46 'ilm': { 

47 'delete': es.ilm.delete_lifecycle, 

48 'exists': es.ilm.get_lifecycle, 

49 'get': es.ilm.get_lifecycle, 

50 'kwargs': {'name': value}, 

51 'plural': 'ilm policy(ies)' 

52 }, 

53 'component': { 

54 'delete': es.cluster.delete_component_template, 

55 'exists': es.cluster.exists_component_template, 

56 'get': es.cluster.get_component_template, 

57 'kwargs': {'name': value}, 

58 'plural': 'component template(s)', 

59 'key': 'component_templates', 

60 }, 

61 'snapshot': { 

62 'delete': es.snapshot.delete, 

63 'exists': es.snapshot.get, 

64 'get': es.snapshot.get, 

65 'kwargs': {}, 

66 'plural': 'snapshot(s)' 

67 } 

68 } 

69 return _[kind] 

70 

71def change_ds(client: Elasticsearch, actions: dict=None) -> None: 

72 """Change/Modify/Update a datastream""" 

73 try: 

74 client.indices.modify_data_stream(actions=actions, body=None) 

75 except Exception as err: 

76 raise exc.ResultNotExpected(f'Unable to modify datastreams. {err}') from err 

77 

78def create_data_stream(client: Elasticsearch, name: str) -> None: 

79 """Create a datastream""" 

80 try: 

81 client.indices.create_data_stream(name=name) 

82 test = Exists(client, name=name, kind='datastream', pause=PAUSE_VALUE) 

83 test.wait_for_it() 

84 except Exception as err: 

85 raise exc.TestbedFailure( 

86 f'Unable to create datastream {name}. Error: {err}') from err 

87 

88def create_index( 

89 client: Elasticsearch, 

90 name: str, 

91 aliases: dict=None, 

92 settings: dict=None, 

93 tier: str='hot' 

94 ) -> None: 

95 """Create named index""" 

96 if not settings: 96 ↛ 97line 96 didn't jump to line 97, because the condition on line 96 was never true

97 settings = get_routing(tier=tier) 

98 else: 

99 settings.update(get_routing(tier=tier)) 

100 client.indices.create( 

101 index=name, 

102 aliases=aliases, 

103 mappings=MAPPING, 

104 settings=settings 

105 ) 

106 try: 

107 test = Exists(client, name=name, kind='index', pause=PAUSE_VALUE) 

108 test.wait_for_it() 

109 except exc.TimeoutException as err: 

110 raise exc.ResultNotExpected(f'Failed to create index {name}') from err 

111 return exists(client, 'index', name) 

112 

113def delete(client: Elasticsearch, kind: str, name: str, repository: str=None): 

114 """Delete the named object of type kind""" 

115 which = emap(kind, client) 

116 func = which['delete'] 

117 if name is None: # Typically only with ilm 117 ↛ 118line 117 didn't jump to line 118, because the condition on line 117 was never true

118 LOGGER.debug('"%s" has a None value for name', kind) 

119 return 

120 try: 

121 if kind == 'snapshot': 121 ↛ 122line 121 didn't jump to line 122, because the condition on line 121 was never true

122 func(snapshot=name, repository=repository) 

123 elif kind == 'index': 

124 func(index=name) 

125 else: 

126 func(name=name) 

127 except esx.NotFoundError: 

128 LOGGER.warning('%s named %s not found.', kind, name) 

129 except Exception as err: 

130 raise exc.ResultNotExpected(f'Unexpected result: {err}') from err 

131 if exists(client, kind, name, repository=repository): 131 ↛ 132line 131 didn't jump to line 132, because the condition on line 131 was never true

132 LOGGER.critical('Unable to delete "%s" %s', kind, name) 

133 raise exc.ResultNotExpected(f'{kind} "{name}" still exists.') 

134 LOGGER.info('Successfully deleted %s: "%s"', which['plural'], name) 

135 

136def do_snap(client: Elasticsearch, repo: str, snap: str, idx: str, tier: str='cold') -> None: 

137 """Perform a snapshot""" 

138 client.snapshot.create(repository=repo, snapshot=snap, indices=idx) 

139 test = Snapshot(client, action='snapshot', snapshot=snap, repository=repo, pause=1, timeout=60) 

140 test.wait_for_it() 

141 

142 # Mount the index accordingly 

143 client.searchable_snapshots.mount( 

144 repository=repo, snapshot=snap, index=idx, index_settings=get_routing(tier=tier), 

145 renamed_index=mounted_name(idx, tier), storage=storage_type(tier), 

146 wait_for_completion=True) 

147 

148def exists(client: Elasticsearch, kind: str, name: str, repository: str=None) -> bool: 

149 """Return boolean existence of the named kind of object""" 

150 if name is None: 150 ↛ 151line 150 didn't jump to line 151, because the condition on line 150 was never true

151 return False 

152 retval = True 

153 func = emap(kind, client)['exists'] 

154 try: 

155 if kind == 'snapshot': 155 ↛ 156line 155 didn't jump to line 156, because the condition on line 155 was never true

156 retval = func(snapshot=name, repository=repository) 

157 elif kind == 'ilm': 

158 retval = func(name=name) 

159 elif kind in ['index', 'data_stream']: 

160 retval = func(index=name) 

161 else: 

162 retval = func(name=name) 

163 except esx.NotFoundError: 163 ↛ 165line 163 didn't jump to line 165

164 retval = False 

165 except Exception as err: 

166 raise exc.ResultNotExpected(f'Unexpected result: {err}') from err 

167 return retval 

168 

169def fill_index( 

170 client: Elasticsearch, 

171 name: str=None, 

172 count: int=None, 

173 start_num: int=None, 

174 match: bool=True 

175 ) -> None: 

176 """ 

177 Create and fill the named index with mappings and settings as directed 

178 

179 :param client: ES client 

180 :param name: Index name 

181 :param count: The number of docs to create 

182 :param start_number: Where to start the incrementing number 

183 :param match: Whether to use the default values for key (True) or random strings (False) 

184 

185 :type client: es 

186 :type name: str 

187 :type count: int 

188 :type start_number: int 

189 :type match: bool 

190 

191 :rtype: None 

192 :returns: No return value 

193 """ 

194 for doc in doc_gen(count=count, start_at=start_num, match=match): 

195 client.index(index=name, document=doc) 

196 client.indices.flush(index=name) 

197 client.indices.refresh(index=name) 

198 

199def find_write_index(client: Elasticsearch, name: str) -> t.AnyStr: 

200 """Find the write_index for an alias by searching any index the alias points to""" 

201 retval = None 

202 for alias in get_aliases(client, name): 202 ↛ 206line 202 didn't jump to line 206, because the loop on line 202 didn't complete

203 retval = get_write_index(client, alias) 

204 if retval: 204 ↛ 202line 204 didn't jump to line 202, because the condition on line 204 was never false

205 break 

206 return retval 

207 

208def fix_aliases(client: Elasticsearch, oldidx: str, newidx: str) -> None: 

209 """Fix aliases using the new and old index names as data""" 

210 # Delete the original index 

211 client.indices.delete(index=oldidx) 

212 # Add the original index name as an alias to the mounted index 

213 client.indices.put_alias(index=f'{newidx}', name=oldidx) 

214 

215def get(client: Elasticsearch, kind: str, pattern: str) -> t.Sequence[str]: 

216 """get any/all objects of type kind matching pattern""" 

217 if pattern is None: 217 ↛ 218line 217 didn't jump to line 218, because the condition on line 217 was never true

218 msg = f'"{kind}" has a None value for pattern' 

219 LOGGER.error(msg) 

220 raise exc.TestbedMisconfig(msg) 

221 which = emap(kind, client, value=pattern) 

222 func = which['get'] 

223 kwargs = which['kwargs'] 

224 try: 

225 result = func(**kwargs) 

226 except Exception as err: 

227 raise exc.ResultNotExpected(f'Unexpected result: {err}') from err 

228 if kind in ['data_stream', 'template', 'component']: 

229 retval = [x['name'] for x in result[which['key']]] 

230 else: 

231 # ['alias', 'ilm', 'index'] 

232 retval = list(result.keys()) 

233 return retval 

234 

235def get_aliases(client: Elasticsearch, name: str) -> t.Sequence[str]: 

236 """Get aliases from index 'name'""" 

237 res = client.indices.get(index=name) 

238 try: 

239 retval = list(res[name]['aliases'].keys()) 

240 except KeyError: 

241 retval = None 

242 return retval 

243 

244def get_backing_indices(client: Elasticsearch, name: str) -> t.Sequence[str]: 

245 """Get the backing indices from the named data_stream""" 

246 resp = resolver(client, name) 

247 data_streams = resp['data_streams'] 

248 retval = [] 

249 if data_streams: 249 ↛ 253line 249 didn't jump to line 253, because the condition on line 249 was never false

250 if len(data_streams) > 1: 250 ↛ 251line 250 didn't jump to line 251, because the condition on line 250 was never true

251 raise exc.ResultNotExpected(f'Expected only a single data_stream matching {name}') 

252 retval = data_streams[0]['backing_indices'] 

253 return retval 

254 

255def get_ds_current(client: Elasticsearch, name: str) -> str: 

256 """ 

257 Find which index is the current 'write' index of the datastream 

258 This is best accomplished by grabbing the last backing_index 

259 """ 

260 backers = get_backing_indices(client, name) 

261 retval = None 

262 if backers: 262 ↛ 264line 262 didn't jump to line 264, because the condition on line 262 was never false

263 retval = backers[-1] 

264 return retval 

265 

266def get_ilm(client: Elasticsearch, pattern: str) -> t.Union[t.Dict[str,str], None]: 

267 """Get any ILM entity in ES that matches pattern""" 

268 try: 

269 return client.ilm.get_lifecycle(name=pattern) 

270 except Exception as err: 

271 msg = f'Unable to get ILM lifecycle matching {pattern}. Error: {err}' 

272 LOGGER.critical(msg) 

273 raise exc.ResultNotExpected(msg) from err 

274 

275def get_ilm_phases(client: Elasticsearch, name: str) -> dict: 

276 """Return the policy/phases part of the ILM policy identified by 'name'""" 

277 ilm = get_ilm(client, name) 

278 try: 

279 return ilm[name]['policy']['phases'] 

280 except KeyError as err: 

281 msg = f'Unable to get ILM lifecycle named {name}. Error: {err}' 

282 LOGGER.critical(msg) 

283 raise exc.ResultNotExpected(msg) from err 

284 

285def get_write_index(client: Elasticsearch, name: str) -> str: 

286 """ 

287 Calls :py:meth:`~.elasticsearch.client.IndicesClient.get_alias` 

288 

289 :param client: A client connection object 

290 :param name: An alias name 

291 

292 :type client: :py:class:`~.elasticsearch.Elasticsearch` 

293 :type name: str 

294 

295 :returns: The the index name associated with the alias that is designated ``is_write_index`` 

296 :rtype: str 

297 """ 

298 response = client.indices.get_alias(index=name) 

299 retval = None 

300 for index in list(response.keys()): 300 ↛ 304line 300 didn't jump to line 304, because the loop on line 300 didn't complete

301 if response[index]['aliases'][name]['is_write_index']: 301 ↛ 300line 301 didn't jump to line 300, because the condition on line 301 was never false

302 retval = index 

303 break 

304 return retval 

305 

306def snapshot_name(client: Elasticsearch, name: str) -> t.Union[t.AnyStr, None]: 

307 """Get the name of the snapshot behind the mounted index data""" 

308 res = {} 

309 if exists(client, 'index', name): # Can jump straight to nested keys if it exists 

310 res = client.indices.get(index=name)[name]['settings']['index'] 

311 try: 

312 retval = res['store']['snapshot']['snapshot_name'] 

313 except KeyError: 

314 LOGGER.error('%s is not a searchable snapshot') 

315 retval = None 

316 return retval 

317 

318def ilm_explain(client: Elasticsearch, name: str) -> t.Union[t.Dict, None]: 

319 """Return the results from the ILM Explain API call for the named index""" 

320 try: 

321 retval = client.ilm.explain_lifecycle(index=name)['indices'][name] 

322 except KeyError: 

323 LOGGER.debug('Index name changed') 

324 new = list(client.ilm.explain_lifecycle(index=name)['indices'].keys())[0] 

325 retval = client.ilm.explain_lifecycle(index=new)['indices'][new] 

326 except esx.NotFoundError as err: 

327 LOGGER.warning('Datastream/Index Name changed. %s was not found', name) 

328 raise exc.NameChanged(f'{name} was not found, likely due to a name change') from err 

329 except Exception as err: 

330 msg = f'Unable to get ILM information for index {name}' 

331 LOGGER.critical(msg) 

332 raise exc.ResultNotExpected(f'{msg}. Exception: {err}') from err 

333 return retval 

334 

335def ilm_move(client: Elasticsearch, name: str, current_step: dict, next_step: dict) -> None: 

336 """Move index 'name' from the current step to the next step""" 

337 try: 

338 client.ilm.move_to_step(index=name, current_step=current_step, next_step=next_step) 

339 except Exception as err: 

340 msg = f'Unable to move index {name} to ILM next step: {next}. Error: {err}' 

341 LOGGER.critical(msg) 

342 raise exc.ResultNotExpected(msg) 

343 

344def put_comp_tmpl(client: Elasticsearch, name: str, component: dict) -> None: 

345 """Publish a component template""" 

346 try: 

347 client.cluster.put_component_template(name=name, template=component, create=True) 

348 test = Exists(client, name=name, kind='component', pause=PAUSE_VALUE) 

349 test.wait_for_it() 

350 except Exception as err: 

351 raise exc.TestbedFailure( 

352 f'Unable to create component template {name}. Error: {err}') from err 

353 

354def put_idx_tmpl( 

355 client, name: str, index_patterns: list, components: list, 

356 data_stream: dict=None) -> None: 

357 """Publish an index template""" 

358 try: 

359 client.indices.put_index_template( 

360 name=name, 

361 composed_of=components, 

362 data_stream=data_stream, 

363 index_patterns=index_patterns, 

364 create=True, 

365 ) 

366 test = Exists(client, name=name, kind='template', pause=PAUSE_VALUE) 

367 test.wait_for_it() 

368 except Exception as err: 

369 raise exc.TestbedFailure( 

370 f'Unable to create index template {name}. Error: {err}') from err 

371 

372def put_ilm(client: Elasticsearch, name: str, policy: dict=None) -> None: 

373 """Publish an ILM Policy""" 

374 try: 

375 client.ilm.put_lifecycle(name=name, policy=policy) 

376 except Exception as err: 

377 raise exc.TestbedFailure( 

378 f'Unable to put index lifecycle policy {name}. Error: {err}') from err 

379 

380def resolver(client: Elasticsearch, name: str) -> dict: 

381 """ 

382 Resolve details about the entity, be it an index, alias, or data_stream 

383  

384 Because you can pass search patterns and aliases as name, each element comes back as an array: 

385  

386 {'indices': [], 'aliases': [], 'data_streams': []} 

387  

388 If you only resolve a single index or data stream, you will still have a 1-element list 

389 """ 

390 return client.indices.resolve_index(name=name, expand_wildcards=['open', 'closed']) 

391 

392def rollover(client: Elasticsearch, name: str) -> None: 

393 """Rollover alias or datastream identified by name""" 

394 client.indices.rollover(alias=name, wait_for_active_shards='all')