Coverage for /Users/buh/.pyenv/versions/3.12.9/envs/es-testbed/lib/python3.12/site-packages/es_testbed/mgrs/data_stream.py: 96%

70 statements  

« prev     ^ index     » next       coverage.py v7.6.12, created at 2025-04-21 21:08 -0600

1"""data_stream Entity Manager Class""" 

2 

3# pylint: disable=W0221 

4import typing as t 

5import logging 

6from importlib import import_module 

7from ..debug import debug, begin_end 

8from ..entities import DataStream, Index 

9from ..es_api import create_data_stream, fill_index 

10from ..utils import prettystr 

11from .index import IndexMgr 

12from .snapshot import SnapshotMgr 

13 

14if t.TYPE_CHECKING: 

15 from elasticsearch8 import Elasticsearch 

16 from dotmap import DotMap 

17 

18logger = logging.getLogger(__name__) 

19 

20 

21class DataStreamMgr(IndexMgr): 

22 """data_stream Entity Manager Class""" 

23 

24 kind = "data_stream" 

25 listname = "data_stream" 

26 

27 def __init__( 

28 self, 

29 client: t.Union["Elasticsearch", None] = None, 

30 plan: t.Union["DotMap", None] = None, 

31 snapmgr: t.Union[SnapshotMgr, None] = None, 

32 ): 

33 self.ds = None 

34 self.index_trackers = [] 

35 debug.lv3("Initializing DataStreamMgr object...") 

36 super().__init__(client=client, plan=plan, snapmgr=snapmgr) 

37 debug.lv3("DataStreamMgr object initialized") 

38 

39 @property 

40 def suffix(self): 

41 """Return nothing, as there is no index count suffix to a data_stream name""" 

42 return "" 

43 

44 @property 

45 def indexlist(self) -> t.Sequence[str]: 

46 """Get the current list of indices in the data_stream""" 

47 return [x.name for x in self.index_trackers] 

48 

49 @begin_end() 

50 def add(self, value): 

51 """Create a data stream and track it""" 

52 try: 

53 debug.lv4(f"TRY: Creating data_stream {value}") 

54 create_data_stream(self.client, value) 

55 except Exception as err: 

56 logger.critical(f"Error creating data_stream: {prettystr(err)}") 

57 raise err 

58 self.track_data_stream() 

59 

60 @begin_end() 

61 def searchable(self): 

62 """If the indices were marked as searchable snapshots, we do that now""" 

63 for idx, scheme in enumerate(self.plan.index_buildlist): 

64 self.index_trackers[idx].mount_ss(scheme) 

65 logger.info("Completed backing index promotion to searchable snapshots.") 

66 debug.lv5(f"data_stream backing indices: {prettystr(self.ds.backing_indices)}") 

67 

68 @begin_end() 

69 def setup(self) -> None: 

70 """Setup the entity manager""" 

71 self.index_trackers = [] # Inheritance oddity requires redeclaration here 

72 mod = import_module(f"{self.plan.modpath}.functions") 

73 func = getattr(mod, "doc_generator") 

74 for scheme in self.plan.index_buildlist: 

75 if not self.entity_list: 

76 self.add(self.name) 

77 else: 

78 self.ds.rollover() 

79 fill_index( 

80 self.client, 

81 name=self.name, 

82 doc_generator=func, 

83 options=scheme["options"], 

84 ) 

85 debug.lv2(f"Created data_stream: {self.ds.name}") 

86 debug.lv3( 

87 f"Created data_stream backing indices: {prettystr(self.ds.backing_indices)}" 

88 ) 

89 for index in self.ds.backing_indices: 

90 self.track_index(index) 

91 self.ds.verify(self.indexlist) 

92 self.searchable() 

93 self.ds.verify(self.indexlist) 

94 logger.info("Successfully completed data_stream buildout.") 

95 

96 @begin_end() 

97 def track_data_stream(self) -> None: 

98 """Add a DataStream entity and append it to entity_list""" 

99 debug.lv3(f"Tracking data_stream: {self.name}") 

100 self.ds = DataStream(client=self.client, name=self.name) 

101 self.appender(self.name) 

102 

103 @begin_end() 

104 def track_index(self, name: str) -> None: 

105 """Add an Index entity and append it to index_trackers""" 

106 debug.lv3(f'Tracking index: "{name}"') 

107 entity = Index( 

108 client=self.client, 

109 name=name, 

110 snapmgr=self.snapmgr, 

111 policy_name=self.policy_name, 

112 ) 

113 entity.track_ilm(name) 

114 self.index_trackers.append(entity)