Repository navigation
Expand file tree
/
Copy pathpath-rules.yml
More file actions
100 lines (94 loc) · 4.01 KB
/
Copy pathpath-rules.yml
File metadata and controls
100 lines (94 loc) · 4.01 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
# Alerting and recording rules for Path.
#
# Load with:
# promtool check rules deploy/prometheus/path-rules.yml
# kubectl apply -f deploy/prometheus/path-rules.yml
#
# Or let the Helm chart render them: --set prometheusRule.enabled=true (the chart
# expects the Prometheus Operator CRDs).
#
# Every metric used here comes from /actuator/prometheus. The runbook links point at
# the section of docs/OPERATIONS.md that explains what the alert means and what to do.
groups:
- name: path-recording
interval: 30s
rules:
# Requests served from the response cache as a fraction of all requests. Recorded so
# the alert can compare today's ratio with its own 7-day average.
- record: path:cache_hit_ratio:rate30m
expr: |
sum(rate(path_cache_hits_total[30m]))
/
clamp_min(sum(rate(path_requests_total[30m])), 0.0001)
- name: path-alerts
rules:
- alert: PathHighFallbackRate
expr: |
sum(rate(path_fallbacks_total[5m]))
/
clamp_min(sum(rate(path_requests_total[5m])), 0.0001)
> 0.20
for: 10m
labels:
severity: warning
annotations:
summary: "More than 20% of requests are falling back to another route"
description: >-
A route is failing and every request is being retried on its fallback. Spend and
latency are rising even though requests still succeed.
runbook: "https://github.com/an1ndra/path/blob/Develop/docs/OPERATIONS.md#alerts"
- alert: PathCacheHitRatioLow
expr: |
path:cache_hit_ratio:rate30m
< 0.5 * avg_over_time(path:cache_hit_ratio:rate30m[7d])
for: 30m
labels:
severity: warning
annotations:
summary: "The response-cache hit ratio is below half its 7-day average"
description: >-
Cache hits collapsed against their own baseline. A changed prompt prefix, a
disabled cache_enabled route or a new client with temperature > 0 are the usual
causes, and all of them raise spend.
runbook: "https://github.com/an1ndra/path/blob/Develop/docs/OPERATIONS.md#alerts"
- alert: PathSpendRateHigh
expr: |
sum(increase(path_cost_usd_total[1h])) * 24
>
2 * sum(increase(path_cost_usd_total[1d] offset 1d))
for: 15m
labels:
severity: warning
annotations:
summary: "The current spend rate is on track to exceed twice yesterday"
description: >-
The last hour, extrapolated to a day, is more than double the whole of the
previous day. Check the top models and keys in /admin/stats before the invoice
does it for you.
runbook: "https://github.com/an1ndra/path/blob/Develop/docs/OPERATIONS.md#alerts"
- alert: PathErrorRateHigh
expr: |
sum(rate(path_errors_total{class="5xx"}[5m]))
/
clamp_min(sum(rate(path_requests_total[5m])), 0.0001)
> 0.02
for: 10m
labels:
severity: critical
annotations:
summary: "More than 2% of requests are failing with a 5xx"
description: >-
Fallback is exhausted or unavailable. The usage rows carry error_code and
error_message for the failing calls, and /actuator/health/providers shows which
providers are parked.
runbook: "https://github.com/an1ndra/path/blob/Develop/docs/OPERATIONS.md#alerts"
- alert: PathProviderParked
expr: min_over_time(path_provider_circuit_open[15m]) == 1
labels:
severity: critical
annotations:
summary: "Provider {{ $labels.provider_id }} has had its circuit open for 15 minutes"
description: >-
The breaker has been open continuously, so every route on this provider is being
skipped. Requests only succeed while a fallback exists.
runbook: "https://github.com/an1ndra/path/blob/Develop/docs/OPERATIONS.md#alerts"