Coverage for /Users/buh/.pyenv/versions/3.12.9/envs/es-testbed/lib/python3.12/site-packages/es_testbed/mgrs/data_stream.py: 96%
70 statements
« prev ^ index » next coverage.py v7.6.12, created at 2025-04-21 21:08 -0600
« prev ^ index » next coverage.py v7.6.12, created at 2025-04-21 21:08 -0600
1"""data_stream Entity Manager Class"""
3# pylint: disable=W0221
4import typing as t
5import logging
6from importlib import import_module
7from ..debug import debug, begin_end
8from ..entities import DataStream, Index
9from ..es_api import create_data_stream, fill_index
10from ..utils import prettystr
11from .index import IndexMgr
12from .snapshot import SnapshotMgr
14if t.TYPE_CHECKING:
15 from elasticsearch8 import Elasticsearch
16 from dotmap import DotMap
18logger = logging.getLogger(__name__)
21class DataStreamMgr(IndexMgr):
22 """data_stream Entity Manager Class"""
24 kind = "data_stream"
25 listname = "data_stream"
27 def __init__(
28 self,
29 client: t.Union["Elasticsearch", None] = None,
30 plan: t.Union["DotMap", None] = None,
31 snapmgr: t.Union[SnapshotMgr, None] = None,
32 ):
33 self.ds = None
34 self.index_trackers = []
35 debug.lv3("Initializing DataStreamMgr object...")
36 super().__init__(client=client, plan=plan, snapmgr=snapmgr)
37 debug.lv3("DataStreamMgr object initialized")
39 @property
40 def suffix(self):
41 """Return nothing, as there is no index count suffix to a data_stream name"""
42 return ""
44 @property
45 def indexlist(self) -> t.Sequence[str]:
46 """Get the current list of indices in the data_stream"""
47 return [x.name for x in self.index_trackers]
49 @begin_end()
50 def add(self, value):
51 """Create a data stream and track it"""
52 try:
53 debug.lv4(f"TRY: Creating data_stream {value}")
54 create_data_stream(self.client, value)
55 except Exception as err:
56 logger.critical(f"Error creating data_stream: {prettystr(err)}")
57 raise err
58 self.track_data_stream()
60 @begin_end()
61 def searchable(self):
62 """If the indices were marked as searchable snapshots, we do that now"""
63 for idx, scheme in enumerate(self.plan.index_buildlist):
64 self.index_trackers[idx].mount_ss(scheme)
65 logger.info("Completed backing index promotion to searchable snapshots.")
66 debug.lv5(f"data_stream backing indices: {prettystr(self.ds.backing_indices)}")
68 @begin_end()
69 def setup(self) -> None:
70 """Setup the entity manager"""
71 self.index_trackers = [] # Inheritance oddity requires redeclaration here
72 mod = import_module(f"{self.plan.modpath}.functions")
73 func = getattr(mod, "doc_generator")
74 for scheme in self.plan.index_buildlist:
75 if not self.entity_list:
76 self.add(self.name)
77 else:
78 self.ds.rollover()
79 fill_index(
80 self.client,
81 name=self.name,
82 doc_generator=func,
83 options=scheme["options"],
84 )
85 debug.lv2(f"Created data_stream: {self.ds.name}")
86 debug.lv3(
87 f"Created data_stream backing indices: {prettystr(self.ds.backing_indices)}"
88 )
89 for index in self.ds.backing_indices:
90 self.track_index(index)
91 self.ds.verify(self.indexlist)
92 self.searchable()
93 self.ds.verify(self.indexlist)
94 logger.info("Successfully completed data_stream buildout.")
96 @begin_end()
97 def track_data_stream(self) -> None:
98 """Add a DataStream entity and append it to entity_list"""
99 debug.lv3(f"Tracking data_stream: {self.name}")
100 self.ds = DataStream(client=self.client, name=self.name)
101 self.appender(self.name)
103 @begin_end()
104 def track_index(self, name: str) -> None:
105 """Add an Index entity and append it to index_trackers"""
106 debug.lv3(f'Tracking index: "{name}"')
107 entity = Index(
108 client=self.client,
109 name=name,
110 snapmgr=self.snapmgr,
111 policy_name=self.policy_name,
112 )
113 entity.track_ilm(name)
114 self.index_trackers.append(entity)