Coverage for src/seqrule/rulesets/pipeline.py: 58%

113 statements  

« prev     ^ index     » next       coverage.py v7.6.12, created at 2025-02-27 10:39 -0600

1""" 

2Software Release Pipeline Rules. 

3 

4This module implements sequence rules for CI/CD pipelines, with support for: 

5- Stage sequencing and dependencies 

6- Approval requirements 

7- Duration constraints 

8- Retry limits 

9- Required security checks 

10- Environment promotion rules 

11- Resource constraints 

12- Parallel execution 

13- Stage dependencies 

14 

15Example pipeline: 

16 lint -> unit_tests -> security_scan -> build -> staging -> integration_tests -> prod 

17 

18Each stage has properties like duration, required approvals, status, and resource requirements. 

19""" 

20 

21from datetime import datetime 

22from enum import Enum 

23from typing import Dict, List, Optional, Set 

24 

25from ..core import AbstractObject, Sequence 

26from ..dsl import DSLRule 

27 

28 

29class StageStatus(Enum): 

30 """Status of a pipeline stage.""" 

31 

32 PENDING = "pending" 

33 RUNNING = "running" 

34 PASSED = "passed" 

35 FAILED = "failed" 

36 SKIPPED = "skipped" 

37 BLOCKED = "blocked" # Blocked by approval or dependency 

38 WAITING = "waiting" # Waiting for resources or parallel stages 

39 

40 

41class Environment(Enum): 

42 """Deployment environment.""" 

43 

44 DEV = "dev" 

45 STAGING = "staging" 

46 PROD = "prod" 

47 

48 

49class ResourceType(Enum): 

50 """Types of resources required by pipeline stages.""" 

51 

52 CPU = "cpu" # CPU cores 

53 MEMORY = "memory" # Memory in GB 

54 GPU = "gpu" # GPU units 

55 WORKER = "worker" # CI/CD workers 

56 DEPLOY_SLOT = "deploy_slot" # Deployment slots 

57 

58 

59class PipelineStage(AbstractObject): 

60 """A stage in the software release pipeline.""" 

61 

62 def __init__( 

63 self, 

64 name: str, 

65 duration_mins: int, 

66 required_approvals: int = 0, 

67 environment: Optional[Environment] = None, 

68 retry_count: int = 0, 

69 status: StageStatus = StageStatus.PENDING, 

70 started_at: Optional[datetime] = None, 

71 completed_at: Optional[datetime] = None, 

72 dependencies: Optional[Set[str]] = None, 

73 parallel_group: Optional[str] = None, 

74 resources: Optional[Dict[ResourceType, float]] = None, 

75 ): 

76 """ 

77 Initialize a pipeline stage. 

78 

79 Args: 

80 name: Stage name (e.g., "unit_tests", "deploy") 

81 duration_mins: Expected duration in minutes (must be positive) 

82 required_approvals: Number of approvals needed 

83 environment: Target environment for deployment stages 

84 retry_count: Number of times this stage has been retried 

85 status: Current stage status 

86 started_at: When the stage started 

87 completed_at: When the stage completed 

88 dependencies: Set of stage names that must complete before this stage 

89 parallel_group: Group name for stages that can run in parallel 

90 resources: Resource requirements {ResourceType: amount} 

91 """ 

92 if duration_mins <= 0: 

93 raise ValueError("Duration must be positive") 

94 

95 if retry_count < 0: 

96 raise ValueError("Retry count cannot be negative") 

97 

98 if required_approvals < 0: 

99 raise ValueError("Required approvals cannot be negative") 

100 

101 super().__init__( 

102 name=name, 

103 duration_mins=duration_mins, 

104 required_approvals=required_approvals, 

105 environment=environment.value if environment else None, 

106 retry_count=retry_count, 

107 status=status.value, 

108 started_at=started_at, 

109 completed_at=completed_at, 

110 dependencies=frozenset(dependencies) if dependencies else frozenset(), 

111 parallel_group=parallel_group, 

112 resources=resources or {}, 

113 ) 

114 

115 def __repr__(self) -> str: 

116 return ( 

117 f"PipelineStage(name={self['name']}, " 

118 f"status={self['status'].upper()}, " 

119 f"env={self['environment']})" 

120 ) 

121 

122 

123def create_stage_order_rule(before: str, after: str) -> DSLRule: 

124 """ 

125 Creates a rule requiring one stage to complete before another starts. 

126 

127 Example: 

128 tests_before_deploy = create_stage_order_rule("unit_tests", "deploy") 

129 """ 

130 

131 def check_order(seq: Sequence) -> bool: 

132 before_passed = False 

133 for stage in seq: 

134 if stage["name"] == before: 

135 before_passed = stage["status"] == StageStatus.PASSED.value 

136 elif stage["name"] == after and not before_passed: 

137 return False 

138 return True 

139 

140 return DSLRule(check_order, f"'{before}' must pass before '{after}' starts") 

141 

142 

143def create_approval_rule(stage_name: str, min_approvals: int) -> DSLRule: 

144 """ 

145 Creates a rule requiring a minimum number of approvals for a stage. 

146 

147 Example: 

148 prod_approval = create_approval_rule("prod_deploy", min_approvals=2) 

149 """ 

150 

151 def check_approvals(seq: Sequence) -> bool: 

152 for stage in seq: 

153 if ( 

154 stage["name"] == stage_name 

155 and stage["required_approvals"] >= min_approvals 

156 and stage["status"] != StageStatus.BLOCKED.value 

157 ): 

158 return False 

159 return True 

160 

161 return DSLRule( 

162 check_approvals, f"'{stage_name}' requires {min_approvals} approvals" 

163 ) 

164 

165 

166def create_duration_rule(max_minutes: int) -> DSLRule: 

167 """ 

168 Creates a rule limiting the total pipeline duration. 

169 Skipped and pending stages are excluded from the total duration. 

170 

171 Example: 

172 time_limit = create_duration_rule(max_minutes=60) 

173 """ 

174 

175 def check_duration(seq: Sequence) -> bool: 

176 excluded_statuses = {StageStatus.SKIPPED.value, StageStatus.PENDING.value} 

177 total = sum( 

178 stage["duration_mins"] 

179 for stage in seq 

180 if stage["status"] not in excluded_statuses 

181 ) 

182 return total <= max_minutes 

183 

184 return DSLRule( 

185 check_duration, f"pipeline must complete within {max_minutes} minutes" 

186 ) 

187 

188 

189def create_retry_rule(max_retries: int) -> DSLRule: 

190 """ 

191 Creates a rule limiting the number of retries for failed stages. 

192 

193 Example: 

194 retry_limit = create_retry_rule(max_retries=3) 

195 """ 

196 

197 def check_retries(seq: Sequence) -> bool: 

198 return all(stage["retry_count"] <= max_retries for stage in seq) 

199 

200 return DSLRule(check_retries, f"stages can be retried at most {max_retries} times") 

201 

202 

203def create_required_stages_rule(required: Set[str]) -> DSLRule: 

204 """ 

205 Creates a rule requiring certain stages to be present and passed. 

206 

207 Example: 

208 security = create_required_stages_rule({"security_scan", "dependency_check"}) 

209 """ 

210 

211 def check_required(seq: Sequence) -> bool: 

212 completed = { 

213 stage["name"] 

214 for stage in seq 

215 if stage["status"] == StageStatus.PASSED.value 

216 } 

217 return required.issubset(completed) 

218 

219 return DSLRule(check_required, f"stages {required} must pass") 

220 

221 

222def create_environment_promotion_rule() -> DSLRule: 

223 """ 

224 Creates a rule enforcing proper environment promotion order. 

225 

226 Example: 

227 promotion = create_environment_promotion_rule() # dev -> staging -> prod 

228 """ 

229 env_order = { 

230 Environment.DEV.value: 0, 

231 Environment.STAGING.value: 1, 

232 Environment.PROD.value: 2, 

233 } 

234 

235 def check_promotion(seq: Sequence) -> bool: 

236 last_env_level = -1 

237 for stage in seq: 

238 if stage["environment"] is not None: 

239 current_level = env_order[stage["environment"]] 

240 if current_level < last_env_level: 

241 return False 

242 last_env_level = current_level 

243 return True 

244 

245 return DSLRule(check_promotion, "environments must be promoted in order") 

246 

247 

248def create_dependency_rule() -> DSLRule: 

249 """ 

250 Creates a rule ensuring all stage dependencies are satisfied. 

251 

252 Example: 

253 dependencies = create_dependency_rule() 

254 """ 

255 

256 def check_dependencies(seq: Sequence) -> bool: 

257 completed_stages = { 

258 stage["name"] 

259 for stage in seq 

260 if stage["status"] == StageStatus.PASSED.value 

261 } 

262 

263 for stage in seq: 

264 if stage["status"] not in { 

265 StageStatus.PENDING.value, 

266 StageStatus.BLOCKED.value, 

267 StageStatus.SKIPPED.value, 

268 }: 

269 # Check if all dependencies are completed 

270 if not stage["dependencies"].issubset(completed_stages): 

271 return False 

272 return True 

273 

274 return DSLRule(check_dependencies, "all stage dependencies must be satisfied") 

275 

276 

277def create_resource_limit_rule(resource_type: ResourceType, limit: float) -> DSLRule: 

278 """ 

279 Creates a rule limiting total resource usage across parallel stages. 

280 

281 Example: 

282 cpu_limit = create_resource_limit_rule(ResourceType.CPU, limit=8) 

283 """ 

284 

285 def check_resources(seq: Sequence) -> bool: 

286 # Group stages by parallel group 

287 parallel_groups: Dict[Optional[str], List[AbstractObject]] = {} 

288 for stage in seq: 

289 if stage["status"] == StageStatus.RUNNING.value: 

290 group = stage["parallel_group"] 

291 if group not in parallel_groups: 

292 parallel_groups[group] = [] 

293 parallel_groups[group].append(stage) 

294 

295 # Check resource usage in each group 

296 for stages in parallel_groups.values(): 

297 total = sum(stage["resources"].get(resource_type, 0) for stage in stages) 

298 if total > limit: 

299 return False 

300 return True 

301 

302 return DSLRule( 

303 check_resources, f"total {resource_type.value} usage must not exceed {limit}" 

304 ) 

305 

306 

307# Common pipeline rules 

308tests_before_deploy = create_stage_order_rule("unit_tests", "deploy") 

309staging_before_prod = create_stage_order_rule("staging_deploy", "prod_deploy") 

310security_requirements = create_required_stages_rule( 

311 {"security_scan", "dependency_check"} 

312) 

313prod_approval_rule = create_approval_rule("prod_deploy", min_approvals=2) 

314pipeline_duration = create_duration_rule(max_minutes=100) 

315retry_limit = create_retry_rule(max_retries=3) 

316environment_promotion = create_environment_promotion_rule() 

317dependency_check = create_dependency_rule() 

318cpu_limit = create_resource_limit_rule(ResourceType.CPU, limit=8) 

319memory_limit = create_resource_limit_rule(ResourceType.MEMORY, limit=32) 

320 

321# Example pipeline configuration with parallel stages and dependencies 

322example_pipeline = [ 

323 PipelineStage( 

324 "lint", 

325 duration_mins=5, 

326 parallel_group="static_analysis", 

327 resources={ResourceType.CPU: 1, ResourceType.MEMORY: 2}, 

328 ), 

329 PipelineStage( 

330 "type_check", 

331 duration_mins=5, 

332 parallel_group="static_analysis", 

333 resources={ResourceType.CPU: 1, ResourceType.MEMORY: 2}, 

334 ), 

335 PipelineStage( 

336 "unit_tests", 

337 duration_mins=10, 

338 dependencies={"lint", "type_check"}, 

339 resources={ResourceType.CPU: 4, ResourceType.MEMORY: 8}, 

340 ), 

341 PipelineStage( 

342 "security_scan", 

343 duration_mins=15, 

344 parallel_group="security", 

345 resources={ResourceType.CPU: 2, ResourceType.MEMORY: 4}, 

346 ), 

347 PipelineStage( 

348 "dependency_check", 

349 duration_mins=5, 

350 parallel_group="security", 

351 resources={ResourceType.CPU: 1, ResourceType.MEMORY: 2}, 

352 ), 

353 PipelineStage( 

354 "build", 

355 duration_mins=8, 

356 dependencies={"unit_tests", "security_scan", "dependency_check"}, 

357 resources={ResourceType.CPU: 4, ResourceType.MEMORY: 16}, 

358 ), 

359 PipelineStage( 

360 "staging_deploy", 

361 duration_mins=10, 

362 required_approvals=1, 

363 environment=Environment.STAGING, 

364 dependencies={"build"}, 

365 resources={ResourceType.DEPLOY_SLOT: 1}, 

366 ), 

367 PipelineStage( 

368 "integration_tests", 

369 duration_mins=20, 

370 dependencies={"staging_deploy"}, 

371 resources={ResourceType.CPU: 2, ResourceType.MEMORY: 4}, 

372 ), 

373 PipelineStage( 

374 "prod_deploy", 

375 duration_mins=15, 

376 required_approvals=2, 

377 environment=Environment.PROD, 

378 dependencies={"integration_tests"}, 

379 resources={ResourceType.DEPLOY_SLOT: 1}, 

380 ), 

381]