63 lines
1.5 KiB
Python
63 lines
1.5 KiB
Python
"""Prefect Automations for ingestion pipeline monitoring and management."""
|
|
|
|
# Automation configurations as YAML-ready dictionaries
|
|
AUTOMATION_TEMPLATES = {
|
|
"cancel_long_running": """
|
|
name: Cancel Long Running Ingestion Flows
|
|
description: Cancels ingestion flows running longer than 30 minutes
|
|
trigger:
|
|
type: event
|
|
posture: Proactive
|
|
expect: [prefect.flow-run.Running]
|
|
match_related:
|
|
prefect.resource.role: flow
|
|
prefect.resource.name: ingestion_pipeline
|
|
threshold: 1
|
|
within: 1800
|
|
actions:
|
|
- type: cancel-flow-run
|
|
source: inferred
|
|
enabled: true
|
|
""",
|
|
|
|
"retry_failed": """
|
|
name: Retry Failed Ingestion Flows
|
|
description: Retries failed ingestion flows with original parameters
|
|
trigger:
|
|
type: event
|
|
posture: Reactive
|
|
expect: [prefect.flow-run.Failed]
|
|
match_related:
|
|
prefect.resource.role: flow
|
|
prefect.resource.name: ingestion_pipeline
|
|
threshold: 1
|
|
within: 0
|
|
actions:
|
|
- type: run-deployment
|
|
source: inferred
|
|
parameters:
|
|
validate_first: false
|
|
enabled: true
|
|
""",
|
|
|
|
"resource_monitoring": """
|
|
name: Manage Work Pool Based on Resources
|
|
description: Pauses work pool when system resources are constrained
|
|
trigger:
|
|
type: event
|
|
posture: Reactive
|
|
expect: [system.resource.high_usage]
|
|
threshold: 1
|
|
within: 120
|
|
actions:
|
|
- type: pause-work-pool
|
|
work_pool_name: default
|
|
enabled: true
|
|
""",
|
|
}
|
|
|
|
|
|
def get_automation_yaml_templates() -> dict[str, str]:
|
|
"""Get automation templates as YAML strings."""
|
|
return AUTOMATION_TEMPLATES.copy()
|