Coverage for src/seqrule/rulesets/pipeline.py: 58%
113 statements
« prev ^ index » next coverage.py v7.6.12, created at 2025-02-27 10:39 -0600
« prev ^ index » next coverage.py v7.6.12, created at 2025-02-27 10:39 -0600
1"""
2Software Release Pipeline Rules.
4This module implements sequence rules for CI/CD pipelines, with support for:
5- Stage sequencing and dependencies
6- Approval requirements
7- Duration constraints
8- Retry limits
9- Required security checks
10- Environment promotion rules
11- Resource constraints
12- Parallel execution
13- Stage dependencies
15Example pipeline:
16 lint -> unit_tests -> security_scan -> build -> staging -> integration_tests -> prod
18Each stage has properties like duration, required approvals, status, and resource requirements.
19"""
21from datetime import datetime
22from enum import Enum
23from typing import Dict, List, Optional, Set
25from ..core import AbstractObject, Sequence
26from ..dsl import DSLRule
29class StageStatus(Enum):
30 """Status of a pipeline stage."""
32 PENDING = "pending"
33 RUNNING = "running"
34 PASSED = "passed"
35 FAILED = "failed"
36 SKIPPED = "skipped"
37 BLOCKED = "blocked" # Blocked by approval or dependency
38 WAITING = "waiting" # Waiting for resources or parallel stages
41class Environment(Enum):
42 """Deployment environment."""
44 DEV = "dev"
45 STAGING = "staging"
46 PROD = "prod"
49class ResourceType(Enum):
50 """Types of resources required by pipeline stages."""
52 CPU = "cpu" # CPU cores
53 MEMORY = "memory" # Memory in GB
54 GPU = "gpu" # GPU units
55 WORKER = "worker" # CI/CD workers
56 DEPLOY_SLOT = "deploy_slot" # Deployment slots
59class PipelineStage(AbstractObject):
60 """A stage in the software release pipeline."""
62 def __init__(
63 self,
64 name: str,
65 duration_mins: int,
66 required_approvals: int = 0,
67 environment: Optional[Environment] = None,
68 retry_count: int = 0,
69 status: StageStatus = StageStatus.PENDING,
70 started_at: Optional[datetime] = None,
71 completed_at: Optional[datetime] = None,
72 dependencies: Optional[Set[str]] = None,
73 parallel_group: Optional[str] = None,
74 resources: Optional[Dict[ResourceType, float]] = None,
75 ):
76 """
77 Initialize a pipeline stage.
79 Args:
80 name: Stage name (e.g., "unit_tests", "deploy")
81 duration_mins: Expected duration in minutes (must be positive)
82 required_approvals: Number of approvals needed
83 environment: Target environment for deployment stages
84 retry_count: Number of times this stage has been retried
85 status: Current stage status
86 started_at: When the stage started
87 completed_at: When the stage completed
88 dependencies: Set of stage names that must complete before this stage
89 parallel_group: Group name for stages that can run in parallel
90 resources: Resource requirements {ResourceType: amount}
91 """
92 if duration_mins <= 0:
93 raise ValueError("Duration must be positive")
95 if retry_count < 0:
96 raise ValueError("Retry count cannot be negative")
98 if required_approvals < 0:
99 raise ValueError("Required approvals cannot be negative")
101 super().__init__(
102 name=name,
103 duration_mins=duration_mins,
104 required_approvals=required_approvals,
105 environment=environment.value if environment else None,
106 retry_count=retry_count,
107 status=status.value,
108 started_at=started_at,
109 completed_at=completed_at,
110 dependencies=frozenset(dependencies) if dependencies else frozenset(),
111 parallel_group=parallel_group,
112 resources=resources or {},
113 )
115 def __repr__(self) -> str:
116 return (
117 f"PipelineStage(name={self['name']}, "
118 f"status={self['status'].upper()}, "
119 f"env={self['environment']})"
120 )
123def create_stage_order_rule(before: str, after: str) -> DSLRule:
124 """
125 Creates a rule requiring one stage to complete before another starts.
127 Example:
128 tests_before_deploy = create_stage_order_rule("unit_tests", "deploy")
129 """
131 def check_order(seq: Sequence) -> bool:
132 before_passed = False
133 for stage in seq:
134 if stage["name"] == before:
135 before_passed = stage["status"] == StageStatus.PASSED.value
136 elif stage["name"] == after and not before_passed:
137 return False
138 return True
140 return DSLRule(check_order, f"'{before}' must pass before '{after}' starts")
143def create_approval_rule(stage_name: str, min_approvals: int) -> DSLRule:
144 """
145 Creates a rule requiring a minimum number of approvals for a stage.
147 Example:
148 prod_approval = create_approval_rule("prod_deploy", min_approvals=2)
149 """
151 def check_approvals(seq: Sequence) -> bool:
152 for stage in seq:
153 if (
154 stage["name"] == stage_name
155 and stage["required_approvals"] >= min_approvals
156 and stage["status"] != StageStatus.BLOCKED.value
157 ):
158 return False
159 return True
161 return DSLRule(
162 check_approvals, f"'{stage_name}' requires {min_approvals} approvals"
163 )
166def create_duration_rule(max_minutes: int) -> DSLRule:
167 """
168 Creates a rule limiting the total pipeline duration.
169 Skipped and pending stages are excluded from the total duration.
171 Example:
172 time_limit = create_duration_rule(max_minutes=60)
173 """
175 def check_duration(seq: Sequence) -> bool:
176 excluded_statuses = {StageStatus.SKIPPED.value, StageStatus.PENDING.value}
177 total = sum(
178 stage["duration_mins"]
179 for stage in seq
180 if stage["status"] not in excluded_statuses
181 )
182 return total <= max_minutes
184 return DSLRule(
185 check_duration, f"pipeline must complete within {max_minutes} minutes"
186 )
189def create_retry_rule(max_retries: int) -> DSLRule:
190 """
191 Creates a rule limiting the number of retries for failed stages.
193 Example:
194 retry_limit = create_retry_rule(max_retries=3)
195 """
197 def check_retries(seq: Sequence) -> bool:
198 return all(stage["retry_count"] <= max_retries for stage in seq)
200 return DSLRule(check_retries, f"stages can be retried at most {max_retries} times")
203def create_required_stages_rule(required: Set[str]) -> DSLRule:
204 """
205 Creates a rule requiring certain stages to be present and passed.
207 Example:
208 security = create_required_stages_rule({"security_scan", "dependency_check"})
209 """
211 def check_required(seq: Sequence) -> bool:
212 completed = {
213 stage["name"]
214 for stage in seq
215 if stage["status"] == StageStatus.PASSED.value
216 }
217 return required.issubset(completed)
219 return DSLRule(check_required, f"stages {required} must pass")
222def create_environment_promotion_rule() -> DSLRule:
223 """
224 Creates a rule enforcing proper environment promotion order.
226 Example:
227 promotion = create_environment_promotion_rule() # dev -> staging -> prod
228 """
229 env_order = {
230 Environment.DEV.value: 0,
231 Environment.STAGING.value: 1,
232 Environment.PROD.value: 2,
233 }
235 def check_promotion(seq: Sequence) -> bool:
236 last_env_level = -1
237 for stage in seq:
238 if stage["environment"] is not None:
239 current_level = env_order[stage["environment"]]
240 if current_level < last_env_level:
241 return False
242 last_env_level = current_level
243 return True
245 return DSLRule(check_promotion, "environments must be promoted in order")
248def create_dependency_rule() -> DSLRule:
249 """
250 Creates a rule ensuring all stage dependencies are satisfied.
252 Example:
253 dependencies = create_dependency_rule()
254 """
256 def check_dependencies(seq: Sequence) -> bool:
257 completed_stages = {
258 stage["name"]
259 for stage in seq
260 if stage["status"] == StageStatus.PASSED.value
261 }
263 for stage in seq:
264 if stage["status"] not in {
265 StageStatus.PENDING.value,
266 StageStatus.BLOCKED.value,
267 StageStatus.SKIPPED.value,
268 }:
269 # Check if all dependencies are completed
270 if not stage["dependencies"].issubset(completed_stages):
271 return False
272 return True
274 return DSLRule(check_dependencies, "all stage dependencies must be satisfied")
277def create_resource_limit_rule(resource_type: ResourceType, limit: float) -> DSLRule:
278 """
279 Creates a rule limiting total resource usage across parallel stages.
281 Example:
282 cpu_limit = create_resource_limit_rule(ResourceType.CPU, limit=8)
283 """
285 def check_resources(seq: Sequence) -> bool:
286 # Group stages by parallel group
287 parallel_groups: Dict[Optional[str], List[AbstractObject]] = {}
288 for stage in seq:
289 if stage["status"] == StageStatus.RUNNING.value:
290 group = stage["parallel_group"]
291 if group not in parallel_groups:
292 parallel_groups[group] = []
293 parallel_groups[group].append(stage)
295 # Check resource usage in each group
296 for stages in parallel_groups.values():
297 total = sum(stage["resources"].get(resource_type, 0) for stage in stages)
298 if total > limit:
299 return False
300 return True
302 return DSLRule(
303 check_resources, f"total {resource_type.value} usage must not exceed {limit}"
304 )
307# Common pipeline rules
308tests_before_deploy = create_stage_order_rule("unit_tests", "deploy")
309staging_before_prod = create_stage_order_rule("staging_deploy", "prod_deploy")
310security_requirements = create_required_stages_rule(
311 {"security_scan", "dependency_check"}
312)
313prod_approval_rule = create_approval_rule("prod_deploy", min_approvals=2)
314pipeline_duration = create_duration_rule(max_minutes=100)
315retry_limit = create_retry_rule(max_retries=3)
316environment_promotion = create_environment_promotion_rule()
317dependency_check = create_dependency_rule()
318cpu_limit = create_resource_limit_rule(ResourceType.CPU, limit=8)
319memory_limit = create_resource_limit_rule(ResourceType.MEMORY, limit=32)
321# Example pipeline configuration with parallel stages and dependencies
322example_pipeline = [
323 PipelineStage(
324 "lint",
325 duration_mins=5,
326 parallel_group="static_analysis",
327 resources={ResourceType.CPU: 1, ResourceType.MEMORY: 2},
328 ),
329 PipelineStage(
330 "type_check",
331 duration_mins=5,
332 parallel_group="static_analysis",
333 resources={ResourceType.CPU: 1, ResourceType.MEMORY: 2},
334 ),
335 PipelineStage(
336 "unit_tests",
337 duration_mins=10,
338 dependencies={"lint", "type_check"},
339 resources={ResourceType.CPU: 4, ResourceType.MEMORY: 8},
340 ),
341 PipelineStage(
342 "security_scan",
343 duration_mins=15,
344 parallel_group="security",
345 resources={ResourceType.CPU: 2, ResourceType.MEMORY: 4},
346 ),
347 PipelineStage(
348 "dependency_check",
349 duration_mins=5,
350 parallel_group="security",
351 resources={ResourceType.CPU: 1, ResourceType.MEMORY: 2},
352 ),
353 PipelineStage(
354 "build",
355 duration_mins=8,
356 dependencies={"unit_tests", "security_scan", "dependency_check"},
357 resources={ResourceType.CPU: 4, ResourceType.MEMORY: 16},
358 ),
359 PipelineStage(
360 "staging_deploy",
361 duration_mins=10,
362 required_approvals=1,
363 environment=Environment.STAGING,
364 dependencies={"build"},
365 resources={ResourceType.DEPLOY_SLOT: 1},
366 ),
367 PipelineStage(
368 "integration_tests",
369 duration_mins=20,
370 dependencies={"staging_deploy"},
371 resources={ResourceType.CPU: 2, ResourceType.MEMORY: 4},
372 ),
373 PipelineStage(
374 "prod_deploy",
375 duration_mins=15,
376 required_approvals=2,
377 environment=Environment.PROD,
378 dependencies={"integration_tests"},
379 resources={ResourceType.DEPLOY_SLOT: 1},
380 ),
381]