Coverage for /Users/buh/.pyenv/versions/3.12.2/envs/pii/lib/python3.12/site-packages/es_pii_tool/defaults.py: 100%
23 statements
« prev ^ index » next coverage.py v7.5.0, created at 2025-03-17 23:49 -0600
« prev ^ index » next coverage.py v7.5.0, created at 2025-03-17 23:49 -0600
1"""App Defaults"""
3import typing as t
4from voluptuous import All, Any, Boolean, Coerce, Optional, Range, Required, Schema
5from es_wait.defaults import EXISTS, HEALTH, ILM, RELOCATE, RESTORE, SNAPSHOT, TASK
7TRACKING_INDEX = 'redactions-tracker'
9CLICK_DRYRUN = {
10 'dry-run': {
11 'help': 'Do not perform any changes.',
12 'is_flag': True,
13 'show_envvar': True,
14 'envvar': 'PII_TOOL_DRY_RUN',
15 }
16}
18CLICK_TRACKING = {
19 'tracking-index': {
20 'help': 'Name for the tracking index.',
21 'default': TRACKING_INDEX,
22 'show_default': True,
23 'show_envvar': True,
24 'envvar': 'PII_TOOL_TRACKING_INDEX',
25 }
26}
28PHASES: t.Sequence = ['hot', 'warm', 'cold', 'frozen', 'delete']
30PAUSE_DEFAULT: str = '9.0'
31PAUSE_ENVVAR: str = 'PII_TOOL_PAUSE'
32TIMEOUT_DEFAULT: str = '7200.0'
33TIMEOUT_ENVVAR: str = 'PII_TOOL_TIMEOUT'
35TIMINGS: dict = {
36 'exists': {
37 'pause': {
38 'testing': 0.3,
39 'default': EXISTS.get('pause'),
40 },
41 'timeout': {
42 'testing': 10.0,
43 'default': EXISTS.get('timeout'),
44 },
45 },
46 'health': {
47 'pause': {
48 'testing': 0.3,
49 'default': HEALTH.get('pause'),
50 },
51 'timeout': {
52 'testing': 10.0,
53 'default': HEALTH.get('timeout'),
54 },
55 },
56 'ilm': {
57 'pause': {
58 'testing': 1.0,
59 'default': ILM.get('pause'),
60 },
61 'timeout': {
62 'testing': 30.0,
63 'default': ILM.get('timeout'),
64 },
65 },
66 'relocate': {
67 'pause': {
68 'testing': 0.5,
69 'default': RELOCATE.get('pause'),
70 },
71 'timeout': {
72 'testing': 30.0,
73 'default': RELOCATE.get('timeout'),
74 },
75 },
76 'restore': {
77 'pause': {
78 'testing': 0.5,
79 'default': RESTORE.get('pause'),
80 },
81 'timeout': {
82 'testing': 30.0,
83 'default': RESTORE.get('timeout'),
84 },
85 },
86 'snapshot': {
87 'pause': {
88 'testing': 0.5,
89 'default': SNAPSHOT.get('pause'),
90 },
91 'timeout': {
92 'testing': 30.0,
93 'default': SNAPSHOT.get('timeout'),
94 },
95 },
96 'task': {
97 'pause': {
98 'testing': 0.3,
99 'default': TASK.get('pause'),
100 },
101 'timeout': {
102 'testing': 30.0,
103 'default': TASK.get('timeout'),
104 },
105 },
106}
109def forcemerge_schema() -> t.Dict[Optional, t.Union[All, Any, Coerce, Range, Required]]:
110 """Define the forcemerge schema"""
111 return {
112 Optional('max_num_segments', default=1): All(
113 Coerce(int), Range(min=1, max=32768)
114 ),
115 # The Boolean() here is a capitalized function, not a class. This code passes
116 # without the need for the passed value because of how voluptuous Schema
117 # validation works.
118 # pylint: disable=no-value-for-parameter
119 Optional('only_expunge_deletes', default=False): Any(
120 bool, All(Any(str), Boolean())
121 ),
122 }
125def redactions_schema() -> t.Dict[
126 Optional,
127 t.Dict[
128 t.Union[Required, Optional],
129 t.Union[All, Any, t.Dict, t.Sequence[Any], Optional],
130 ],
131]:
132 """An index pattern to search and redact data from"""
133 merge = forcemerge_schema()
134 return {
135 Optional(Any(str)): {
136 Required('pattern'): Any(str),
137 Required('query'): {Any(str): dict},
138 Required('fields'): [Any(str)],
139 Required('message', default='REDACTED'): Any(str),
140 # The Boolean() here is a capitalized function, not a class. This code
141 # passes without the need for the passed value because of how voluptuous
142 # Schema validation works.
143 # pylint: disable=no-value-for-parameter
144 Optional('delete', default=True): Any(bool, All(Any(str), Boolean())),
145 Required('expected_docs'): All(Coerce(int), Range(min=1, max=32768)),
146 Optional('restore_settings', default=None): Any(dict, None),
147 Optional('forcemerge'): merge,
148 }
149 }
152def index_settings() -> t.Dict:
153 """The Elasticsearch index settings for the progress/status tracking index"""
154 return {
155 'index': {
156 'number_of_shards': '1',
157 'auto_expand_replicas': '0-1',
158 }
159 }
162def status_mappings() -> t.Dict:
163 """The Elasticsearch index mappings for the progress/status tracking index"""
164 return {
165 'properties': {
166 'job': {'type': 'keyword'},
167 'task': {'type': 'keyword'},
168 'step': {'type': 'keyword'},
169 'join_field': {'type': 'join', 'relations': {'job': 'task'}},
170 'cleanup': {'type': 'keyword'},
171 'completed': {'type': 'boolean'},
172 'end_time': {'type': 'date'},
173 'errors': {'type': 'boolean'},
174 'dry_run': {'type': 'boolean'},
175 'index': {'type': 'keyword'},
176 'logs': {'type': 'text'},
177 'start_time': {'type': 'date'},
178 },
179 'dynamic_templates': [
180 {
181 'configuration': {
182 'path_match': 'config.*',
183 'mapping': {'type': 'keyword', 'index': False},
184 }
185 }
186 ],
187 }
190def redaction_schema() -> Schema:
191 """The full voluptuous Schema for a redaction file"""
192 return Schema({Required('redactions'): [redactions_schema()]})