Skip to content

Commit 9350311

Browse files
slokslok
committed
Port denominator corrected feature as SLO plugin
Signed-off-by: Xabier Larrakoetxea <me@slok.dev>
1 parent c47c6c1 commit 9350311

7 files changed

Lines changed: 1003 additions & 4 deletions

File tree

‎.github/CODEOWNERS‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,8 @@
55
/internal/plugin/slo/contrib/ @slok
66

77
# Specific contrib SLO owners.
8-
/internal/plugin/slo/contrib/info_labels_v1/ @cxdy @slok @wbollock
9-
/internal/plugin/slo/contrib/validate_victoria_metrics_v1/ @slok
10-
/internal/plugin/slo/contrib/rule_intervals_v1/ @cxdy @slok @wbollock
11-
/internal/plugin/slo/contrib/error_budget_exhausted_alert_v1/ @cxdy @slok @wbollock
8+
/internal/plugin/slo/contrib/error_budget_exhausted_alert_v1/ @cxdy @slok @wbollock
9+
/internal/plugin/slo/contrib/denominator_corrected_rules/v1/ @slok
10+
/internal/plugin/slo/contrib/info_labels_v1/ @cxdy @slok @wbollock
11+
/internal/plugin/slo/contrib/rule_intervals_v1/ @cxdy @slok @wbollock
12+
/internal/plugin/slo/contrib/validate_victoria_metrics_v1/ @slok

‎CHANGELOG.md‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,8 @@
99
- Allow `github.com/VictoriaMetrics/metricsql` module in SLO plugins.
1010
- Contrib plugin: `sloth.dev/contrib/validate_victoria_metrics/v1`.
1111
- Contrib plugin: `sloth.dev/contrib/rule_intervals/v1`.
12+
- Contrib plugin: `sloth.dev/contrib/error_budget_exhausted_alert/v1`.
13+
- Contrib plugin: `sloth.dev/contrib/denominator_corrected_rules/v1`.
1214

1315
## [v0.13.0] - 2025-09-10
1416

‎examples/_gen/contrib-denominator-corrected.yaml‎

Lines changed: 349 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,349 @@
1+
2+
---
3+
# Code generated by Sloth (dev): https://github.com/slok/sloth.
4+
# DO NOT EDIT.
5+
6+
apiVersion: monitoring.coreos.com/v1
7+
kind: PrometheusRule
8+
metadata:
9+
labels:
10+
app.kubernetes.io/component: SLO
11+
app.kubernetes.io/managed-by: sloth
12+
name: svc
13+
namespace: test-ns
14+
spec:
15+
groups:
16+
- name: sloth-slo-sli-recordings-svc01-slo1
17+
rules:
18+
- expr: |
19+
(
20+
slo:numerator_correction:ratio5m{sloth_id="svc01-slo1", sloth_service="svc01", sloth_slo="slo1"}
21+
* on()
22+
sum(rate(http_request_duration_seconds_count{job="myservice",code=~"(5..|429)"}[5m]))
23+
)
24+
/
25+
(sum(rate(http_request_duration_seconds_count{job="myservice"}[5m])))
26+
labels:
27+
cmd: examplesgen.sh
28+
global01k1: global01v1
29+
global02k1: global02v1
30+
sloth_id: svc01-slo1
31+
sloth_service: svc01
32+
sloth_slo: slo1
33+
sloth_window: 5m
34+
record: slo:sli_error:ratio_rate5m
35+
- expr: |
36+
(
37+
slo:numerator_correction:ratio30m{sloth_id="svc01-slo1", sloth_service="svc01", sloth_slo="slo1"}
38+
* on()
39+
sum(rate(http_request_duration_seconds_count{job="myservice",code=~"(5..|429)"}[30m]))
40+
)
41+
/
42+
(sum(rate(http_request_duration_seconds_count{job="myservice"}[30m])))
43+
labels:
44+
cmd: examplesgen.sh
45+
global01k1: global01v1
46+
global02k1: global02v1
47+
sloth_id: svc01-slo1
48+
sloth_service: svc01
49+
sloth_slo: slo1
50+
sloth_window: 30m
51+
record: slo:sli_error:ratio_rate30m
52+
- expr: |
53+
(
54+
slo:numerator_correction:ratio1h{sloth_id="svc01-slo1", sloth_service="svc01", sloth_slo="slo1"}
55+
* on()
56+
sum(rate(http_request_duration_seconds_count{job="myservice",code=~"(5..|429)"}[1h]))
57+
)
58+
/
59+
(sum(rate(http_request_duration_seconds_count{job="myservice"}[1h])))
60+
labels:
61+
cmd: examplesgen.sh
62+
global01k1: global01v1
63+
global02k1: global02v1
64+
sloth_id: svc01-slo1
65+
sloth_service: svc01
66+
sloth_slo: slo1
67+
sloth_window: 1h
68+
record: slo:sli_error:ratio_rate1h
69+
- expr: |
70+
(
71+
slo:numerator_correction:ratio2h{sloth_id="svc01-slo1", sloth_service="svc01", sloth_slo="slo1"}
72+
* on()
73+
sum(rate(http_request_duration_seconds_count{job="myservice",code=~"(5..|429)"}[2h]))
74+
)
75+
/
76+
(sum(rate(http_request_duration_seconds_count{job="myservice"}[2h])))
77+
labels:
78+
cmd: examplesgen.sh
79+
global01k1: global01v1
80+
global02k1: global02v1
81+
sloth_id: svc01-slo1
82+
sloth_service: svc01
83+
sloth_slo: slo1
84+
sloth_window: 2h
85+
record: slo:sli_error:ratio_rate2h
86+
- expr: |
87+
(
88+
slo:numerator_correction:ratio6h{sloth_id="svc01-slo1", sloth_service="svc01", sloth_slo="slo1"}
89+
* on()
90+
sum(rate(http_request_duration_seconds_count{job="myservice",code=~"(5..|429)"}[6h]))
91+
)
92+
/
93+
(sum(rate(http_request_duration_seconds_count{job="myservice"}[6h])))
94+
labels:
95+
cmd: examplesgen.sh
96+
global01k1: global01v1
97+
global02k1: global02v1
98+
sloth_id: svc01-slo1
99+
sloth_service: svc01
100+
sloth_slo: slo1
101+
sloth_window: 6h
102+
record: slo:sli_error:ratio_rate6h
103+
- expr: |
104+
(
105+
slo:numerator_correction:ratio1d{sloth_id="svc01-slo1", sloth_service="svc01", sloth_slo="slo1"}
106+
* on()
107+
sum(rate(http_request_duration_seconds_count{job="myservice",code=~"(5..|429)"}[1d]))
108+
)
109+
/
110+
(sum(rate(http_request_duration_seconds_count{job="myservice"}[1d])))
111+
labels:
112+
cmd: examplesgen.sh
113+
global01k1: global01v1
114+
global02k1: global02v1
115+
sloth_id: svc01-slo1
116+
sloth_service: svc01
117+
sloth_slo: slo1
118+
sloth_window: 1d
119+
record: slo:sli_error:ratio_rate1d
120+
- expr: |
121+
(
122+
slo:numerator_correction:ratio3d{sloth_id="svc01-slo1", sloth_service="svc01", sloth_slo="slo1"}
123+
* on()
124+
sum(rate(http_request_duration_seconds_count{job="myservice",code=~"(5..|429)"}[3d]))
125+
)
126+
/
127+
(sum(rate(http_request_duration_seconds_count{job="myservice"}[3d])))
128+
labels:
129+
cmd: examplesgen.sh
130+
global01k1: global01v1
131+
global02k1: global02v1
132+
sloth_id: svc01-slo1
133+
sloth_service: svc01
134+
sloth_slo: slo1
135+
sloth_window: 3d
136+
record: slo:sli_error:ratio_rate3d
137+
- expr: |
138+
(
139+
slo:numerator_correction:ratio30d{sloth_id="svc01-slo1", sloth_service="svc01", sloth_slo="slo1"}
140+
* on()
141+
sum(rate(http_request_duration_seconds_count{job="myservice",code=~"(5..|429)"}[30d]))
142+
)
143+
/
144+
(sum(rate(http_request_duration_seconds_count{job="myservice"}[30d])))
145+
labels:
146+
cmd: examplesgen.sh
147+
global01k1: global01v1
148+
global02k1: global02v1
149+
sloth_id: svc01-slo1
150+
sloth_service: svc01
151+
sloth_slo: slo1
152+
sloth_window: 30d
153+
record: slo:sli_error:ratio_rate30d
154+
- name: sloth-slo-meta-recordings-svc01-slo1
155+
rules:
156+
- expr: vector(0.9990000000000001)
157+
labels:
158+
cmd: examplesgen.sh
159+
global01k1: global01v1
160+
global02k1: global02v1
161+
sloth_id: svc01-slo1
162+
sloth_service: svc01
163+
sloth_slo: slo1
164+
record: slo:objective:ratio
165+
- expr: vector(1-0.9990000000000001)
166+
labels:
167+
cmd: examplesgen.sh
168+
global01k1: global01v1
169+
global02k1: global02v1
170+
sloth_id: svc01-slo1
171+
sloth_service: svc01
172+
sloth_slo: slo1
173+
record: slo:error_budget:ratio
174+
- expr: vector(30)
175+
labels:
176+
cmd: examplesgen.sh
177+
global01k1: global01v1
178+
global02k1: global02v1
179+
sloth_id: svc01-slo1
180+
sloth_service: svc01
181+
sloth_slo: slo1
182+
record: slo:time_period:days
183+
- expr: |
184+
slo:sli_error:ratio_rate5m{sloth_id="svc01-slo1", sloth_service="svc01", sloth_slo="slo1"}
185+
/ on(sloth_id, sloth_slo, sloth_service) group_left
186+
slo:error_budget:ratio{sloth_id="svc01-slo1", sloth_service="svc01", sloth_slo="slo1"}
187+
labels:
188+
cmd: examplesgen.sh
189+
global01k1: global01v1
190+
global02k1: global02v1
191+
sloth_id: svc01-slo1
192+
sloth_service: svc01
193+
sloth_slo: slo1
194+
record: slo:current_burn_rate:ratio
195+
- expr: |
196+
slo:sli_error:ratio_rate30d{sloth_id="svc01-slo1", sloth_service="svc01", sloth_slo="slo1"}
197+
/ on(sloth_id, sloth_slo, sloth_service) group_left
198+
slo:error_budget:ratio{sloth_id="svc01-slo1", sloth_service="svc01", sloth_slo="slo1"}
199+
labels:
200+
cmd: examplesgen.sh
201+
global01k1: global01v1
202+
global02k1: global02v1
203+
sloth_id: svc01-slo1
204+
sloth_service: svc01
205+
sloth_slo: slo1
206+
record: slo:period_burn_rate:ratio
207+
- expr: 1 - slo:period_burn_rate:ratio{sloth_id="svc01-slo1", sloth_service="svc01",
208+
sloth_slo="slo1"}
209+
labels:
210+
cmd: examplesgen.sh
211+
global01k1: global01v1
212+
global02k1: global02v1
213+
sloth_id: svc01-slo1
214+
sloth_service: svc01
215+
sloth_slo: slo1
216+
record: slo:period_error_budget_remaining:ratio
217+
- expr: vector(1)
218+
labels:
219+
cmd: examplesgen.sh
220+
global01k1: global01v1
221+
global02k1: global02v1
222+
sloth_id: svc01-slo1
223+
sloth_mode: cli-gen-k8s
224+
sloth_objective: "99.9"
225+
sloth_service: svc01
226+
sloth_slo: slo1
227+
sloth_spec: sloth.slok.dev/v1
228+
sloth_version: dev
229+
record: sloth_slo_info
230+
- expr: (sum(rate(http_request_duration_seconds_count{job="myservice"}[5m])))/(sum(rate(http_request_duration_seconds_count{job="myservice"}[30d])))
231+
labels:
232+
cmd: examplesgen.sh
233+
global01k1: global01v1
234+
global02k1: global02v1
235+
sloth_id: svc01-slo1
236+
sloth_service: svc01
237+
sloth_slo: slo1
238+
record: slo:numerator_correction:ratio5m
239+
- expr: (sum(rate(http_request_duration_seconds_count{job="myservice"}[30m])))/(sum(rate(http_request_duration_seconds_count{job="myservice"}[30d])))
240+
labels:
241+
cmd: examplesgen.sh
242+
global01k1: global01v1
243+
global02k1: global02v1
244+
sloth_id: svc01-slo1
245+
sloth_service: svc01
246+
sloth_slo: slo1
247+
record: slo:numerator_correction:ratio30m
248+
- expr: (sum(rate(http_request_duration_seconds_count{job="myservice"}[1h])))/(sum(rate(http_request_duration_seconds_count{job="myservice"}[30d])))
249+
labels:
250+
cmd: examplesgen.sh
251+
global01k1: global01v1
252+
global02k1: global02v1
253+
sloth_id: svc01-slo1
254+
sloth_service: svc01
255+
sloth_slo: slo1
256+
record: slo:numerator_correction:ratio1h
257+
- expr: (sum(rate(http_request_duration_seconds_count{job="myservice"}[2h])))/(sum(rate(http_request_duration_seconds_count{job="myservice"}[30d])))
258+
labels:
259+
cmd: examplesgen.sh
260+
global01k1: global01v1
261+
global02k1: global02v1
262+
sloth_id: svc01-slo1
263+
sloth_service: svc01
264+
sloth_slo: slo1
265+
record: slo:numerator_correction:ratio2h
266+
- expr: (sum(rate(http_request_duration_seconds_count{job="myservice"}[6h])))/(sum(rate(http_request_duration_seconds_count{job="myservice"}[30d])))
267+
labels:
268+
cmd: examplesgen.sh
269+
global01k1: global01v1
270+
global02k1: global02v1
271+
sloth_id: svc01-slo1
272+
sloth_service: svc01
273+
sloth_slo: slo1
274+
record: slo:numerator_correction:ratio6h
275+
- expr: (sum(rate(http_request_duration_seconds_count{job="myservice"}[1d])))/(sum(rate(http_request_duration_seconds_count{job="myservice"}[30d])))
276+
labels:
277+
cmd: examplesgen.sh
278+
global01k1: global01v1
279+
global02k1: global02v1
280+
sloth_id: svc01-slo1
281+
sloth_service: svc01
282+
sloth_slo: slo1
283+
record: slo:numerator_correction:ratio1d
284+
- expr: (sum(rate(http_request_duration_seconds_count{job="myservice"}[3d])))/(sum(rate(http_request_duration_seconds_count{job="myservice"}[30d])))
285+
labels:
286+
cmd: examplesgen.sh
287+
global01k1: global01v1
288+
global02k1: global02v1
289+
sloth_id: svc01-slo1
290+
sloth_service: svc01
291+
sloth_slo: slo1
292+
record: slo:numerator_correction:ratio3d
293+
- expr: (sum(rate(http_request_duration_seconds_count{job="myservice"}[30d])))/(sum(rate(http_request_duration_seconds_count{job="myservice"}[30d])))
294+
labels:
295+
cmd: examplesgen.sh
296+
global01k1: global01v1
297+
global02k1: global02v1
298+
sloth_id: svc01-slo1
299+
sloth_service: svc01
300+
sloth_slo: slo1
301+
record: slo:numerator_correction:ratio30d
302+
- name: sloth-slo-alerts-svc01-slo1
303+
rules:
304+
- alert: myServiceAlert
305+
annotations:
306+
alert02k1: alert02k2
307+
summary: '{{$labels.sloth_service}} {{$labels.sloth_slo}} SLO error budget
308+
burn rate is over expected.'
309+
title: (page) {{$labels.sloth_service}} {{$labels.sloth_slo}} SLO error budget
310+
burn rate is too fast.
311+
expr: |
312+
(
313+
max(slo:sli_error:ratio_rate5m{sloth_id="svc01-slo1", sloth_service="svc01", sloth_slo="slo1"} > (14.4 * 0.0009999999999999432)) without (sloth_window)
314+
and
315+
max(slo:sli_error:ratio_rate1h{sloth_id="svc01-slo1", sloth_service="svc01", sloth_slo="slo1"} > (14.4 * 0.0009999999999999432)) without (sloth_window)
316+
)
317+
or
318+
(
319+
max(slo:sli_error:ratio_rate30m{sloth_id="svc01-slo1", sloth_service="svc01", sloth_slo="slo1"} > (6 * 0.0009999999999999432)) without (sloth_window)
320+
and
321+
max(slo:sli_error:ratio_rate6h{sloth_id="svc01-slo1", sloth_service="svc01", sloth_slo="slo1"} > (6 * 0.0009999999999999432)) without (sloth_window)
322+
)
323+
labels:
324+
alert01k1: alert01v1
325+
alert03k1: alert03v1
326+
sloth_severity: page
327+
- alert: myServiceAlert
328+
annotations:
329+
alert02k1: alert02k2
330+
summary: '{{$labels.sloth_service}} {{$labels.sloth_slo}} SLO error budget
331+
burn rate is over expected.'
332+
title: (ticket) {{$labels.sloth_service}} {{$labels.sloth_slo}} SLO error
333+
budget burn rate is too fast.
334+
expr: |
335+
(
336+
max(slo:sli_error:ratio_rate2h{sloth_id="svc01-slo1", sloth_service="svc01", sloth_slo="slo1"} > (3 * 0.0009999999999999432)) without (sloth_window)
337+
and
338+
max(slo:sli_error:ratio_rate1d{sloth_id="svc01-slo1", sloth_service="svc01", sloth_slo="slo1"} > (3 * 0.0009999999999999432)) without (sloth_window)
339+
)
340+
or
341+
(
342+
max(slo:sli_error:ratio_rate6h{sloth_id="svc01-slo1", sloth_service="svc01", sloth_slo="slo1"} > (1 * 0.0009999999999999432)) without (sloth_window)
343+
and
344+
max(slo:sli_error:ratio_rate3d{sloth_id="svc01-slo1", sloth_service="svc01", sloth_slo="slo1"} > (1 * 0.0009999999999999432)) without (sloth_window)
345+
)
346+
labels:
347+
alert01k1: alert01v1
348+
alert04k1: alert04v1
349+
sloth_severity: ticket

0 commit comments

Comments
 (0)