Skip to content

Commit 448df41

Browse files
committed
Add optimized rules to denonimator corrected plugin
Signed-off-by: Xabier Larrakoetxea <me@slok.dev>
1 parent 390934f commit 448df41

5 files changed

Lines changed: 293 additions & 77 deletions

File tree

internal/plugin/slo/contrib/denominator_corrected_rules_v1/README.md

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ More details in the original [PR].
1717

1818
## Config
1919

20-
None
20+
- `disableOptimized`(**Optional**, `bool`): If `true`, disables optimized rule generation for long SLI windows. Optimized rules use short-window recording rules to derive long-window SLIs with lower Prometheus resource usage, at the cost of reduced accuracy. Defaults to `false`.
2121

2222
## Env vars
2323

@@ -37,4 +37,15 @@ This plugin should run after rule generation plugins.
3737
- id: "sloth.dev/contrib/denominator_corrected_rules/v1"
3838
```
3939
40+
### Disable optimization
41+
42+
```yaml
43+
sloPlugins:
44+
chain:
45+
- id: "sloth.dev/contrib/denominator_corrected_rules/v1"
46+
config:
47+
disableOptimized: true
48+
```
49+
50+
4051
[PR]: https://github.com/slok/sloth/pull/459

internal/plugin/slo/contrib/denominator_corrected_rules_v1/plugin.go

Lines changed: 52 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,6 @@ import (
55
"context"
66
"encoding/json"
77
"fmt"
8-
"sort"
98
"text/template"
109
"time"
1110

@@ -26,7 +25,9 @@ const (
2625
numeratorCorrectionMetric = "slo:numerator_correction:ratio" // The correction factor metric name.
2726
)
2827

29-
type PluginConfig struct{}
28+
type PluginConfig struct {
29+
DisableOptimized bool `json:"disableOptimized,omitempty"`
30+
}
3031

3132
func NewPlugin(c json.RawMessage, _ pluginslov1.AppUtils) (pluginslov1.Plugin, error) {
3233
cfg := &PluginConfig{}
@@ -49,36 +50,31 @@ func (p plugin) ProcessSLO(ctx context.Context, request *pluginslov1.Request, re
4950
}
5051

5152
// Generate and override SLI recordings.
52-
sliRules, err := generateSLIRecordingRules(ctx, request.SLO, request.MWMBAlertGroup)
53+
sliRules, err := p.generateSLIRecordingRules(ctx, request.SLO, request.MWMBAlertGroup)
5354
if err != nil {
5455
return err
5556
}
5657
result.SLORules.SLIErrorRecRules.Rules = sliRules
5758

5859
// Add required new metadata recordings with the correction factor.
59-
windows := getAlertGroupWindows(request.MWMBAlertGroup)
60-
windows = append(windows, request.SLO.TimeWindow) // Add the total time window as a handy helper.
61-
metadataLabels := utilsdata.MergeLabels(conventions.GetSLOIDPromLabels(request.SLO), request.SLO.Labels)
62-
for _, window := range windows {
63-
rule, err := createNumeratorCorrection(request.SLO, metadataLabels, window, request.SLO.TimeWindow)
64-
if err != nil {
65-
return fmt.Errorf("could not create numerator rule: %v", err)
66-
}
67-
result.SLORules.MetadataRecRules.Rules = append(result.SLORules.MetadataRecRules.Rules, *rule)
60+
metaRules, err := p.generateMetaRecordingRules(ctx, request.SLO, request.MWMBAlertGroup)
61+
if err != nil {
62+
return err
6863
}
64+
result.SLORules.MetadataRecRules.Rules = metaRules
6965

7066
return nil
7167
}
7268

73-
func generateSLIRecordingRules(ctx context.Context, slo model.PromSLO, alerts model.MWMBAlertGroup) ([]rulefmt.Rule, error) {
69+
func (p plugin) generateSLIRecordingRules(ctx context.Context, slo model.PromSLO, alerts model.MWMBAlertGroup) ([]rulefmt.Rule, error) {
7470
// Get the windows we need the recording rules.
75-
windows := getAlertGroupWindows(alerts)
71+
windows := alerts.TimeDurationWindows()
7672
windows = append(windows, slo.TimeWindow) // Add the total time window as a handy helper.
7773

7874
// Generate the rules
7975
rules := make([]rulefmt.Rule, 0, len(windows))
8076
for _, window := range windows {
81-
rule, err := denominatorCorrectedSLIRecordGenerator(slo, window, alerts)
77+
rule, err := p.denominatorCorrectedSLIRecordGenerator(slo, window, alerts)
8278
if err != nil {
8379
return nil, fmt.Errorf("could not create %q SLO rule for window %s: %w", slo.ID, window, err)
8480
}
@@ -88,41 +84,57 @@ func generateSLIRecordingRules(ctx context.Context, slo model.PromSLO, alerts mo
8884
return rules, nil
8985
}
9086

91-
// getAlertGroupWindows gets all the time windows from a multiwindow multiburn alert group.
92-
func getAlertGroupWindows(alerts model.MWMBAlertGroup) []time.Duration {
93-
// Use a map to avoid duplicated windows.
94-
windows := map[string]time.Duration{
95-
alerts.PageQuick.ShortWindow.String(): alerts.PageQuick.ShortWindow,
96-
alerts.PageQuick.LongWindow.String(): alerts.PageQuick.LongWindow,
97-
alerts.PageSlow.ShortWindow.String(): alerts.PageSlow.ShortWindow,
98-
alerts.PageSlow.LongWindow.String(): alerts.PageSlow.LongWindow,
99-
alerts.TicketQuick.ShortWindow.String(): alerts.TicketQuick.ShortWindow,
100-
alerts.TicketQuick.LongWindow.String(): alerts.TicketQuick.LongWindow,
101-
alerts.TicketSlow.ShortWindow.String(): alerts.TicketSlow.ShortWindow,
102-
alerts.TicketSlow.LongWindow.String(): alerts.TicketSlow.LongWindow,
103-
}
104-
105-
res := make([]time.Duration, 0, len(windows))
106-
for _, w := range windows {
107-
res = append(res, w)
87+
func (p plugin) generateMetaRecordingRules(ctx context.Context, slo model.PromSLO, alerts model.MWMBAlertGroup) ([]rulefmt.Rule, error) {
88+
metadataLabels := utilsdata.MergeLabels(conventions.GetSLOIDPromLabels(slo), slo.Labels)
89+
rules := []rulefmt.Rule{}
90+
for _, window := range alerts.TimeDurationWindows() {
91+
rule, err := createNumeratorCorrection(slo, metadataLabels, window, slo.TimeWindow)
92+
if err != nil {
93+
return nil, fmt.Errorf("could not create numerator rule: %v", err)
94+
}
95+
rules = append(rules, *rule)
10896
}
109-
sort.SliceStable(res, func(i, j int) bool { return res[i] < res[j] })
110-
111-
return res
97+
return rules, nil
11298
}
11399

114-
func denominatorCorrectedSLIRecordGenerator(slo model.PromSLO, window time.Duration, alerts model.MWMBAlertGroup) (*rulefmt.Rule, error) {
115-
const sliExprTplFmt = `(
100+
func (p plugin) denominatorCorrectedSLIRecordGenerator(slo model.PromSLO, window time.Duration, alerts model.MWMBAlertGroup) (*rulefmt.Rule, error) {
101+
const (
102+
sliExprTplFmt = `(
116103
{{.numeratorCorrectionMetric}}{{.window}}{{.filter}}
117104
* on()
118105
%s
119106
)
120107
/
121108
(%s)
122109
`
123-
124-
sliExprTpl := fmt.Sprintf(sliExprTplFmt, slo.SLI.Events.ErrorQuery, slo.SLI.Events.TotalQuery)
125-
110+
sliExprTotalWindowTplFmt = `(%s)
111+
/
112+
(%s)
113+
`
114+
// For more information about optimized SLI check `sloth.dev/core/sli_rules/v1` plugin.
115+
sliExprTotalWindowOptimizedTplFmt = `sum_over_time(%s[{{.window}}])
116+
/ ignoring ({{.windowKey}})
117+
count_over_time(%s[{{.window}}])
118+
`
119+
)
120+
121+
sliExprTpl := ""
122+
switch {
123+
// Last window (total window) when not optimized.
124+
case window == slo.TimeWindow && p.cfg.DisableOptimized:
125+
sliExprTpl = fmt.Sprintf(sliExprTotalWindowTplFmt, slo.SLI.Events.ErrorQuery, slo.SLI.Events.TotalQuery)
126+
127+
// Last window (total window) when optimized.
128+
case window == slo.TimeWindow && !p.cfg.DisableOptimized:
129+
shortWindowSLIRec := conventions.GetSLIErrorMetric(alerts.PageQuick.ShortWindow)
130+
filter := promutils.LabelsToPromFilter(conventions.GetSLOIDPromLabels(slo))
131+
metric := shortWindowSLIRec + filter
132+
sliExprTpl = fmt.Sprintf(sliExprTotalWindowOptimizedTplFmt, metric, metric)
133+
134+
// Regular SLI.
135+
default:
136+
sliExprTpl = fmt.Sprintf(sliExprTplFmt, slo.SLI.Events.ErrorQuery, slo.SLI.Events.TotalQuery)
137+
}
126138
// Render with our templated data.
127139
tpl, err := template.New("sliExpr").Option("missingkey=error").Parse(sliExprTpl)
128140
if err != nil {

0 commit comments

Comments
 (0)