Coverage for /Users/buh/.pyenv/versions/3.12.9/envs/es-testbed/lib/python3.12/site-packages/es_testbed/mgrs/data_stream.py: 96%
75 statements
« prev ^ index » next coverage.py v7.6.12, created at 2025-03-31 14:55 -0600
« prev ^ index » next coverage.py v7.6.12, created at 2025-03-31 14:55 -0600
1"""data_stream Entity Manager Class"""
3import typing as t
4import logging
5from importlib import import_module
6import tiered_debug as debug
7from es_testbed.entities import DataStream, Index
8from es_testbed.helpers.es_api import create_data_stream, fill_index
9from es_testbed.helpers.utils import prettystr
10from es_testbed.mgrs.index import IndexMgr
11from es_testbed.mgrs.snapshot import SnapshotMgr
13if t.TYPE_CHECKING:
14 from elasticsearch8 import Elasticsearch
15 from dotmap import DotMap
17logger = logging.getLogger(__name__)
20class DataStreamMgr(IndexMgr):
21 """data_stream Entity Manager Class"""
23 kind = 'data_stream'
24 listname = 'data_stream'
26 def __init__(
27 self,
28 client: t.Union['Elasticsearch', None] = None,
29 plan: t.Union['DotMap', None] = None,
30 snapmgr: t.Union[SnapshotMgr, None] = None,
31 ):
32 self.ds = None
33 self.index_trackers = []
34 debug.lv3('Initializing DataStreamMgr object...')
35 super().__init__(client=client, plan=plan, snapmgr=snapmgr)
36 debug.lv3('DataStreamMgr object initialized')
38 @property
39 def suffix(self):
40 """Return nothing, as there is no index count suffix to a data_stream name"""
41 return ''
43 @property
44 def indexlist(self) -> t.Sequence[str]:
45 """Get the current list of indices in the data_stream"""
46 return [x.name for x in self.index_trackers]
48 def add(self, value):
49 """Create a data stream and track it"""
50 debug.lv2('Starting method...')
51 try:
52 debug.lv4(f'TRY: Creating data_stream {value}')
53 create_data_stream(self.client, value)
54 except Exception as err:
55 logger.critical(f'Error creating data_stream: {prettystr(err)}')
56 raise err
57 self.track_data_stream()
58 debug.lv3('Exiting method')
60 def searchable(self):
61 """If the indices were marked as searchable snapshots, we do that now"""
62 debug.lv2('Starting method...')
63 for idx, scheme in enumerate(self.plan.index_buildlist):
64 self.index_trackers[idx].mount_ss(scheme)
65 logger.info('Completed backing index promotion to searchable snapshots.')
66 debug.lv5(f'data_stream backing indices: {prettystr(self.ds.backing_indices)}')
67 debug.lv3('Exiting method')
69 def setup(self) -> None:
70 """Setup the entity manager"""
71 debug.lv2('Starting method...')
72 self.index_trackers = [] # Inheritance oddity requires redeclaration here
73 mod = import_module(f'{self.plan.modpath}.functions')
74 func = getattr(mod, 'doc_generator')
75 for scheme in self.plan.index_buildlist:
76 if not self.entity_list:
77 self.add(self.name)
78 else:
79 self.ds.rollover()
80 fill_index(
81 self.client,
82 name=self.name,
83 doc_generator=func,
84 options=scheme['options'],
85 )
86 debug.lv2(f'Created data_stream: {self.ds.name}')
87 debug.lv3(
88 f'Created data_stream backing indices: {prettystr(self.ds.backing_indices)}'
89 )
90 for index in self.ds.backing_indices:
91 self.track_index(index)
92 self.ds.verify(self.indexlist)
93 self.searchable()
94 self.ds.verify(self.indexlist)
95 logger.info('Successfully completed data_stream buildout.')
96 debug.lv3('Exiting method')
98 def track_data_stream(self) -> None:
99 """Add a DataStream entity and append it to entity_list"""
100 debug.lv2('Starting method...')
101 debug.lv3(f'Tracking data_stream: {self.name}')
102 self.ds = DataStream(client=self.client, name=self.name)
103 self.appender(self.name)
104 debug.lv3('Exiting method')
106 def track_index(self, name: str) -> None:
107 """Add an Index entity and append it to index_trackers"""
108 debug.lv2('Starting method...')
109 debug.lv3(f'Tracking index: "{name}"')
110 entity = Index(
111 client=self.client,
112 name=name,
113 snapmgr=self.snapmgr,
114 policy_name=self.policy_name,
115 )
116 entity.track_ilm(name)
117 self.index_trackers.append(entity)
118 debug.lv3('Exiting method')