Coverage for /Users/buh/.pyenv/versions/3.12.2/envs/es-testbed/lib/python3.12/site-packages/es_testbed/mgrs/data_stream.py: 95%

61 statements  

« prev     ^ index     » next       coverage.py v7.4.4, created at 2024-08-21 12:05 -0600

1"""data_stream Entity Manager Class""" 

2 

3import typing as t 

4import logging 

5from importlib import import_module 

6from es_testbed.entities import DataStream, Index 

7from es_testbed.helpers.es_api import create_data_stream, fill_index 

8from es_testbed.helpers.utils import prettystr 

9from es_testbed.mgrs.index import IndexMgr 

10from es_testbed.mgrs.snapshot import SnapshotMgr 

11 

12if t.TYPE_CHECKING: 

13 from elasticsearch8 import Elasticsearch 

14 from dotmap import DotMap 

15 

16logger = logging.getLogger(__name__) 

17 

18 

19class DataStreamMgr(IndexMgr): 

20 """data_stream Entity Manager Class""" 

21 

22 kind = 'data_stream' 

23 listname = 'data_stream' 

24 

25 def __init__( 

26 self, 

27 client: t.Union['Elasticsearch', None] = None, 

28 plan: t.Union['DotMap', None] = None, 

29 snapmgr: t.Union[SnapshotMgr, None] = None, 

30 ): 

31 self.ds = None 

32 self.index_trackers = [] 

33 super().__init__(client=client, plan=plan, snapmgr=snapmgr) 

34 

35 @property 

36 def suffix(self): 

37 """Return nothing, as there is no index count suffix to a data_stream name""" 

38 return '' 

39 

40 @property 

41 def indexlist(self) -> t.Sequence[str]: 

42 """Get the current list of indices in the data_stream""" 

43 return [x.name for x in self.index_trackers] 

44 

45 def add(self, value): 

46 """Create a data stream and track it""" 

47 try: 

48 create_data_stream(self.client, value) 

49 except Exception as err: 

50 logger.critical('Error creating data_stream: %s', err) 

51 raise err 

52 self.track_data_stream() 

53 

54 def searchable(self): 

55 """If the indices were marked as searchable snapshots, we do that now""" 

56 for idx, scheme in enumerate(self.plan.index_buildlist): 

57 self.index_trackers[idx].mount_ss(scheme) 

58 logger.info('Completed backing index promotion to searchable snapshots.') 

59 logger.info( 

60 'data_stream backing indices: %s', prettystr(self.ds.backing_indices) 

61 ) 

62 

63 def setup(self) -> None: 

64 """Setup the entity manager""" 

65 self.index_trackers = [] # Inheritance oddity requires redeclaration here 

66 mod = import_module(f'{self.plan.modpath}.functions') 

67 func = getattr(mod, 'doc_generator') 

68 for scheme in self.plan.index_buildlist: 

69 if not self.entity_list: 

70 self.add(self.name) 

71 else: 

72 self.ds.rollover() 

73 fill_index( 

74 self.client, 

75 name=self.name, 

76 doc_generator=func, 

77 options=scheme['options'], 

78 ) 

79 logger.debug('Created data_stream: %s', self.ds.name) 

80 logger.debug( 

81 'Created data_stream backing indices: %s', 

82 prettystr(self.ds.backing_indices), 

83 ) 

84 for index in self.ds.backing_indices: 

85 self.track_index(index) 

86 self.ds.verify(self.indexlist) 

87 self.searchable() 

88 self.ds.verify(self.indexlist) 

89 logger.info('Successfully completed data_stream buildout.') 

90 

91 def track_data_stream(self) -> None: 

92 """Add a DataStream entity and append it to entity_list""" 

93 logger.debug('Tracking data_stream: %s', self.name) 

94 self.ds = DataStream(client=self.client, name=self.name) 

95 self.appender(self.name) 

96 

97 def track_index(self, name: str) -> None: 

98 """Add an Index entity and append it to index_trackers""" 

99 logger.debug('Tracking index: "%s"', name) 

100 entity = Index( 

101 client=self.client, 

102 name=name, 

103 snapmgr=self.snapmgr, 

104 policy_name=self.policy_name, 

105 ) 

106 entity.track_ilm(name) 

107 self.index_trackers.append(entity)