Coverage for /Users/buh/.pyenv/versions/3.12.9/envs/es-testbed/lib/python3.12/site-packages/es_testbed/mgrs/data_stream.py: 96%

75 statements  

« prev     ^ index     » next       coverage.py v7.6.12, created at 2025-03-31 14:55 -0600

1"""data_stream Entity Manager Class""" 

2 

3import typing as t 

4import logging 

5from importlib import import_module 

6import tiered_debug as debug 

7from es_testbed.entities import DataStream, Index 

8from es_testbed.helpers.es_api import create_data_stream, fill_index 

9from es_testbed.helpers.utils import prettystr 

10from es_testbed.mgrs.index import IndexMgr 

11from es_testbed.mgrs.snapshot import SnapshotMgr 

12 

13if t.TYPE_CHECKING: 

14 from elasticsearch8 import Elasticsearch 

15 from dotmap import DotMap 

16 

17logger = logging.getLogger(__name__) 

18 

19 

20class DataStreamMgr(IndexMgr): 

21 """data_stream Entity Manager Class""" 

22 

23 kind = 'data_stream' 

24 listname = 'data_stream' 

25 

26 def __init__( 

27 self, 

28 client: t.Union['Elasticsearch', None] = None, 

29 plan: t.Union['DotMap', None] = None, 

30 snapmgr: t.Union[SnapshotMgr, None] = None, 

31 ): 

32 self.ds = None 

33 self.index_trackers = [] 

34 debug.lv3('Initializing DataStreamMgr object...') 

35 super().__init__(client=client, plan=plan, snapmgr=snapmgr) 

36 debug.lv3('DataStreamMgr object initialized') 

37 

38 @property 

39 def suffix(self): 

40 """Return nothing, as there is no index count suffix to a data_stream name""" 

41 return '' 

42 

43 @property 

44 def indexlist(self) -> t.Sequence[str]: 

45 """Get the current list of indices in the data_stream""" 

46 return [x.name for x in self.index_trackers] 

47 

48 def add(self, value): 

49 """Create a data stream and track it""" 

50 debug.lv2('Starting method...') 

51 try: 

52 debug.lv4(f'TRY: Creating data_stream {value}') 

53 create_data_stream(self.client, value) 

54 except Exception as err: 

55 logger.critical(f'Error creating data_stream: {prettystr(err)}') 

56 raise err 

57 self.track_data_stream() 

58 debug.lv3('Exiting method') 

59 

60 def searchable(self): 

61 """If the indices were marked as searchable snapshots, we do that now""" 

62 debug.lv2('Starting method...') 

63 for idx, scheme in enumerate(self.plan.index_buildlist): 

64 self.index_trackers[idx].mount_ss(scheme) 

65 logger.info('Completed backing index promotion to searchable snapshots.') 

66 debug.lv5(f'data_stream backing indices: {prettystr(self.ds.backing_indices)}') 

67 debug.lv3('Exiting method') 

68 

69 def setup(self) -> None: 

70 """Setup the entity manager""" 

71 debug.lv2('Starting method...') 

72 self.index_trackers = [] # Inheritance oddity requires redeclaration here 

73 mod = import_module(f'{self.plan.modpath}.functions') 

74 func = getattr(mod, 'doc_generator') 

75 for scheme in self.plan.index_buildlist: 

76 if not self.entity_list: 

77 self.add(self.name) 

78 else: 

79 self.ds.rollover() 

80 fill_index( 

81 self.client, 

82 name=self.name, 

83 doc_generator=func, 

84 options=scheme['options'], 

85 ) 

86 debug.lv2(f'Created data_stream: {self.ds.name}') 

87 debug.lv3( 

88 f'Created data_stream backing indices: {prettystr(self.ds.backing_indices)}' 

89 ) 

90 for index in self.ds.backing_indices: 

91 self.track_index(index) 

92 self.ds.verify(self.indexlist) 

93 self.searchable() 

94 self.ds.verify(self.indexlist) 

95 logger.info('Successfully completed data_stream buildout.') 

96 debug.lv3('Exiting method') 

97 

98 def track_data_stream(self) -> None: 

99 """Add a DataStream entity and append it to entity_list""" 

100 debug.lv2('Starting method...') 

101 debug.lv3(f'Tracking data_stream: {self.name}') 

102 self.ds = DataStream(client=self.client, name=self.name) 

103 self.appender(self.name) 

104 debug.lv3('Exiting method') 

105 

106 def track_index(self, name: str) -> None: 

107 """Add an Index entity and append it to index_trackers""" 

108 debug.lv2('Starting method...') 

109 debug.lv3(f'Tracking index: "{name}"') 

110 entity = Index( 

111 client=self.client, 

112 name=name, 

113 snapmgr=self.snapmgr, 

114 policy_name=self.policy_name, 

115 ) 

116 entity.track_ilm(name) 

117 self.index_trackers.append(entity) 

118 debug.lv3('Exiting method')