Coverage for /Users/buh/.pyenv/versions/3.12.9/envs/es-testbed/lib/python3.12/site-packages/es_testbed/_plan.py: 100%

72 statements  

« prev     ^ index     » next       coverage.py v7.6.12, created at 2025-03-17 19:30 -0600

1"""TestPlan Class Definition""" 

2 

3import typing as t 

4import logging 

5from dotmap import DotMap 

6from es_testbed.defaults import TESTPLAN 

7from es_testbed.helpers.utils import build_ilm_policy, prettystr, randomstr 

8 

9logger = logging.getLogger(__name__) 

10 

11 

12class PlanBuilder: 

13 """ 

14 Plan builder class 

15 

16 """ 

17 

18 def __init__( 

19 self, 

20 settings: t.Dict, 

21 autobuild: t.Optional[bool] = True, 

22 ): 

23 """Class initializer 

24 

25 Receives a settings dictionary and builds the object based on it. This 

26 dictionary is converted into a DotMap object to make it easier to access 

27 and modify the settings. 

28 

29 The settings dictionary should have the following structure: 

30 

31 .. code-block:: python 

32 

33 { 

34 'type': 'indices', # indices or data_streams 

35 'prefix': 'es-testbed', # The default prefix for everything we create 

36 'rollover_alias': False, # Only respected if 'type' == 'indices'. 

37 # Will rollover after creation and filling 1st 

38 # If True, will be overridden to value of alias 

39 # If False, will be overridden with None 

40 'uniq': 'my-unique-str', # If not provided, randomstr() 

41 'repository': # Only used for cold/frozen tier for snapshots 

42 'ilm': { # All of these ILM values are defaults 

43 'enabled': False, 

44 'phases': ['hot', 'delete'], 

45 'readonly': PHASE # Define readonly action during named PHASE 

46 'forcemerge': False, 

47 'max_num_segments': 1, 

48 'policy': {} # Define full ILM policy in advance. 

49 }, 

50 'index_buildlist': [], # List of indices to create 

51 } 

52 

53 The index_buildlist is a list of dictionaries, each with a similar structure: 

54 

55 .. code-block:: python 

56 

57 'index_buildlist': [ 

58 { 

59 'preset': 'NAME', # docgen_preset name, included or otherwise 

60 'options': { # kwargs for the generator function 

61 'docs': 10, 

62 'start_at': 0, 

63 'match': True, 

64 } 

65 'target_tier': 'frozen' # Target tier for 1st (oldest) index created 

66 }, 

67 { 

68 'preset': 'NAME', # docgen_preset name, included or otherwise 

69 'options': { # kwargs for the generator function 

70 'docs': 10, 

71 'start_at': 10, 

72 'match': True, 

73 } 

74 'target_tier': 'cold' # Target tier for 2nd index created 

75 }, 

76 { 

77 'preset': 'NAME', # docgen_preset name, included or otherwise 

78 'options': { # kwargs for the generator function 

79 'docs': 10, 

80 'start_at': 20, 

81 'match': True, 

82 } 

83 'target_tier': 'hot' # Target tier for last (newest) index created 

84 }, 

85 ] 

86 

87 The index_buildlist can be provided after the object is created, and is, 

88 in fact, if using a preset, which will provide the entire plan, including 

89 the index_buildlist, and the values that need to go in the indices. 

90 

91 Args: 

92 settings (t.Dict): The settings dictionary 

93 autobuild (t.Optional[bool], optional): Whether to autobuild the plan. 

94 Defaults to True. 

95 

96 Raises: 

97 ValueError: Must provide a settings dictionary 

98 """ 

99 if settings is None: 

100 raise ValueError('Must provide a settings dictionary') 

101 self.settings = settings 

102 logger.debug(f'SETTINGS: {settings}') 

103 self._plan = DotMap(TESTPLAN) 

104 logger.debug(f'INITIAL PLAN: {prettystr(self._plan)}') 

105 self._plan.cleanup = 'UNSET' # Future use? 

106 if autobuild: 

107 self.setup() 

108 

109 @property 

110 def plan(self) -> DotMap: 

111 """Return the Plan""" 

112 return self._plan 

113 

114 def _create_lists(self) -> None: 

115 names = [ 

116 'indices', 

117 'data_stream', 

118 'snapshots', 

119 'ilm_policies', 

120 'index_templates', 

121 'component_templates', 

122 ] 

123 for name in names: 

124 self._plan[name] = [] 

125 

126 def setup(self) -> None: 

127 """Do initial setup of the Plan DotMap""" 

128 self._plan.uniq = randomstr(length=8, lowercase=True) 

129 self._create_lists() 

130 self.update(self.settings) # Override with settings. 

131 self.update_rollover_alias() 

132 logger.debug('Rollover alias updated') 

133 self.update_ilm() 

134 logger.debug(f'FINAL PLAN: {prettystr(self._plan.toDict())}') 

135 

136 def update(self, settings: t.Dict) -> None: 

137 """Update the Plan DotMap""" 

138 self._plan.update(DotMap(settings)) 

139 

140 def update_ilm(self) -> None: 

141 """Update the ILM portion of the Plan DotMap""" 

142 setdefault = False 

143 if 'ilm' not in self._plan: 

144 logger.debug('key "ilm" is not in plan') 

145 setdefault = True 

146 if isinstance(self._plan.ilm, dict): 

147 _ = DotMap(self._plan.ilm) 

148 self._plan.ilm = _ 

149 if isinstance(self._plan.ilm, DotMap): 

150 if 'enabled' not in self._plan.ilm: 

151 # Override with defaults 

152 logger.debug( 

153 'plan.ilm does not have key "enabled". Overriding with defaults' 

154 ) 

155 setdefault = True 

156 elif isinstance(self._plan.ilm, bool): 

157 if self._plan.ilm: 

158 logger.warning( 

159 '"plan.ilm: True" is incorrect. Use plan.ilm.enabled: True' 

160 ) 

161 logger.debug('plan.ilm is boolean. Overriding with defaults') 

162 setdefault = True 

163 if setdefault: 

164 logger.debug('Setting defaults for ILM') 

165 self._plan.ilm = DotMap(TESTPLAN['ilm']) 

166 if self._plan.ilm.enabled: 

167 ilm = self._plan.ilm 

168 if not isinstance(self._plan.ilm.phases, list): 

169 logger.error('Phases is not a list!') 

170 self._plan.ilm.phases = TESTPLAN['ilm']['phases'] 

171 for entity in self._plan.index_buildlist: 

172 if 'searchable' in entity and entity['searchable'] is not None: 

173 if not entity['searchable'] in ilm.phases: 

174 ilm.phases.append(entity['searchable']) 

175 logger.debug(f'ILM = {ilm}') 

176 logger.debug(f'self._plan.ilm = {self._plan.ilm}') 

177 kwargs = { 

178 'phases': ilm.phases, 

179 'forcemerge': ilm.forcemerge, 

180 'max_num_segments': ilm.max_num_segments, 

181 'readonly': ilm.readonly, 

182 'repository': self._plan.repository, 

183 } 

184 logger.debug(f'KWARGS = {kwargs}') 

185 self._plan.ilm.policy = build_ilm_policy(**kwargs) 

186 

187 def update_rollover_alias(self) -> None: 

188 """Update the Rollover Alias value""" 

189 if self._plan.rollover_alias: 

190 self._plan.rollover_alias = f'{self._plan.prefix}-idx-{self._plan.uniq}' 

191 else: 

192 self._plan.rollover_alias = None