Skip to content

Commit 28ae09d

Browse files
authored
Merge pull request #839 from metrico/feat/699
Support of the compound line filter expressions (#699)
2 parents 01a76f9 + 1562f2b commit 28ae09d

9 files changed

Lines changed: 231 additions & 97 deletions

File tree

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
([]*logql_parser.LogQLScript) (len=3) {
2+
(*logql_parser.LogQLScript)({app="x"}|~ "POST" or "GET"),
3+
(*logql_parser.LogQLScript)({app="x"}|= "a" and "b"),
4+
(*logql_parser.LogQLScript)({app="x"}|= ("foo" or "bar") and "baz")
5+
}

reader/logql/logql_parser/.snapshots/TestParser

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,11 @@
11
([]*logql_parser.LogQLScript) (len=39) {
22
(*logql_parser.LogQLScript)({test_id="${testID}"}),
33
(*logql_parser.LogQLScript)({test_id="${testID}",freq="2"}),
4-
(*logql_parser.LogQLScript)({test_id="${testID}",freq="2"} |~ "2[0-9]$"),
5-
(*logql_parser.LogQLScript)(rate ({test_id="${testID}",freq="2"} |~ "2[0-9]$"[1s])),
6-
(*logql_parser.LogQLScript)(sum by (test_id) (rate ({test_id="${testID}"} |~ "2[0-9]$"[1s]))),
7-
(*logql_parser.LogQLScript)(rate ({test_id="${testID}",freq="2"} |~ "2[0-9]$"[1s])),
8-
(*logql_parser.LogQLScript)(sum by (test_id) (rate ({test_id="${testID}"} |~ "2[0-9]$"[1s]))),
4+
(*logql_parser.LogQLScript)({test_id="${testID}",freq="2"}|~ "2[0-9]$"),
5+
(*logql_parser.LogQLScript)(rate ({test_id="${testID}",freq="2"}|~ "2[0-9]$"[1s])),
6+
(*logql_parser.LogQLScript)(sum by (test_id) (rate ({test_id="${testID}"}|~ "2[0-9]$"[1s]))),
7+
(*logql_parser.LogQLScript)(rate ({test_id="${testID}",freq="2"}|~ "2[0-9]$"[1s])),
8+
(*logql_parser.LogQLScript)(sum by (test_id) (rate ({test_id="${testID}"}|~ "2[0-9]$"[1s]))),
99
(*logql_parser.LogQLScript)({test_id="${testID}_json"}| json),
1010
(*logql_parser.LogQLScript)({test_id="${testID}_json"}| json lbl_repl = "new_lbl"),
1111
(*logql_parser.LogQLScript)({test_id="${testID}_json"}| json lbl_repl = "new_lbl" | lbl_repl = "new_val"),

reader/logql/logql_parser/model.go

Lines changed: 39 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -166,11 +166,48 @@ func (s *StrSelectorPipeline) String() string {
166166

167167
type LineFilter struct {
168168
Fn string `@("|="|"!="|"|~"|"!~"|"|>")`
169-
Val QuotedString `@@`
169+
Exp LineFilterExp `@@`
170170
}
171171

172172
func (l *LineFilter) String() string {
173-
return fmt.Sprintf(" %s %s", l.Fn, l.Val.String())
173+
return l.Fn + " " + l.Exp.String()
174+
}
175+
176+
type LineFilterExp struct {
177+
Head LineFilterHead `@@`
178+
Op string `@("and"|"or")?`
179+
Tail *LineFilterExp `@@?`
180+
}
181+
182+
func (l *LineFilterExp) String() string {
183+
res := l.Head.String()
184+
if l.Op != "" {
185+
res += " " + l.Op
186+
}
187+
if l.Tail != nil {
188+
res += " " + l.Tail.String()
189+
}
190+
return res
191+
}
192+
193+
type LineFilterHead struct {
194+
Complex *LineFilterExp `( "(" @@ ")" )`
195+
Simple *LineFilterSimple `| @@`
196+
}
197+
198+
func (h *LineFilterHead) String() string {
199+
if h.Complex != nil {
200+
return "(" + h.Complex.String() + ")"
201+
}
202+
return h.Simple.String()
203+
}
204+
205+
type LineFilterSimple struct {
206+
Val QuotedString `@@`
207+
}
208+
209+
func (s *LineFilterSimple) String() string {
210+
return fmt.Sprintf("%s", s.Val.String())
174211
}
175212

176213
type LabelFilter struct {

reader/logql/logql_parser/parser_test.go

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -79,6 +79,24 @@ func TestQuotedString_String(t *testing.T) {
7979
}
8080
}
8181

82+
func TestLineFilterBool(t *testing.T) {
83+
tests := []string{
84+
`{app="x"} |~ "POST" or "GET"`,
85+
`{app="x"} |= "a" and "b"`,
86+
`{app="x"} |= ("foo" or "bar") and "baz"`,
87+
}
88+
asts := make([]*LogQLScript, len(tests))
89+
for i, str := range tests {
90+
ast, err := Parse(str)
91+
if err != nil {
92+
fmt.Printf("[%d]: %s\n", i, str)
93+
t.Fatal(err)
94+
}
95+
asts[i] = ast
96+
}
97+
cupaloy.SnapshotT(t, asts)
98+
}
99+
82100
func TestParser2(t *testing.T) {
83101
ast, err := Parse(`{sender="logtest"} |= "GET"`)
84102
if err != nil {

reader/logql/logql_transpiler/clickhouse_planner/analyze.go

Lines changed: 23 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -118,16 +118,34 @@ func AnalyzeMetrics15sShortcut(script *logql_parser.LogQLScript) bool {
118118
if ppl.Drop != nil {
119119
return false
120120
}
121-
if ppl.LineFilter != nil {
122-
str, err := ppl.LineFilter.Val.Unquote()
123-
if str != "" || err != nil {
124-
return false
125-
}
121+
if ppl.LineFilter != nil && lineFilterHasContent(ppl.LineFilter) {
122+
return false
126123
}
127124
}
128125
return true
129126
}
130127

128+
func lineFilterHasContent(lf *logql_parser.LineFilter) bool {
129+
return lineFilterExpHasContent(&lf.Exp)
130+
}
131+
132+
func lineFilterExpHasContent(exp *logql_parser.LineFilterExp) bool {
133+
head := &exp.Head
134+
if head.Simple != nil {
135+
val, err := head.Simple.Val.Unquote()
136+
if val != "" || err != nil {
137+
return true
138+
}
139+
}
140+
if head.Complex != nil && lineFilterExpHasContent(head.Complex) {
141+
return true
142+
}
143+
if exp.Tail != nil {
144+
return lineFilterExpHasContent(exp.Tail)
145+
}
146+
return false
147+
}
148+
131149
func (p *planner) getFunctionOrder(script any) {
132150
maybeComparison := func(op *logql_parser.Comparison) {
133151
if op != nil {

reader/logql/logql_transpiler/clickhouse_planner/planner.go

Lines changed: 2 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -474,14 +474,9 @@ func (p *planner) planLabelFilter(ppl *logql_parser.StrSelectorPipeline, idx int
474474
}
475475

476476
func (p *planner) planLineFilter(ppl *logql_parser.StrSelectorPipeline) error {
477-
val, err := ppl.LineFilter.Val.Unquote()
478-
if err != nil {
479-
return err
480-
}
481477
p.samplesPlanner = &LineFilterPlanner{
482-
Op: ppl.LineFilter.Fn,
483-
Val: val,
484-
Main: p.samplesPlanner,
478+
Filter: ppl.LineFilter,
479+
Main: p.samplesPlanner,
485480
}
486481
return nil
487482
}

reader/logql/logql_transpiler/clickhouse_planner/planner_line_filter.go

Lines changed: 81 additions & 52 deletions
Original file line numberDiff line numberDiff line change
@@ -5,73 +5,109 @@ import (
55
"regexp/syntax"
66
"strings"
77

8+
log_parser "github.com/metrico/qryn/v4/reader/logql/logql_parser"
89
"github.com/metrico/qryn/v4/reader/logql/logql_transpiler/shared"
910
sql "github.com/metrico/qryn/v4/reader/utils/sql_select"
1011
)
1112

1213
type LineFilterPlanner struct {
13-
Op string
14-
Val string
15-
Main shared.SQLRequestPlanner
14+
Filter *log_parser.LineFilter
15+
Main shared.SQLRequestPlanner
1616
}
1717

1818
func (l *LineFilterPlanner) Process(ctx *shared.PlannerContext) (sql.ISelect, error) {
1919
req, err := l.Main.Process(ctx)
2020
if err != nil {
2121
return nil, err
2222
}
23+
clause, err := l.buildExpCondition(l.Filter.Fn, &l.Filter.Exp)
24+
if err != nil {
25+
return nil, err
26+
}
27+
return req.AndWhere(clause), nil
28+
}
2329

24-
var clause sql.SQLCondition
25-
switch l.Op {
30+
func (l *LineFilterPlanner) buildExpCondition(fn string, exp *log_parser.LineFilterExp) (sql.SQLCondition, error) {
31+
head, err := l.buildHeadCondition(fn, &exp.Head)
32+
if err != nil {
33+
return nil, err
34+
}
35+
if exp.Op == "" {
36+
return head, nil
37+
}
38+
tail, err := l.buildExpCondition(fn, exp.Tail)
39+
if err != nil {
40+
return nil, err
41+
}
42+
if exp.Op == "or" {
43+
return sql.Or(head, tail), nil
44+
}
45+
return sql.And(head, tail), nil
46+
}
47+
48+
func (l *LineFilterPlanner) buildHeadCondition(fn string, head *log_parser.LineFilterHead) (sql.SQLCondition, error) {
49+
if head.Complex != nil {
50+
return l.buildExpCondition(fn, head.Complex)
51+
}
52+
return l.buildSimpleCondition(fn, head.Simple)
53+
}
54+
55+
func (l *LineFilterPlanner) buildSimpleCondition(fn string, s *log_parser.LineFilterSimple) (sql.SQLCondition, error) {
56+
val, err := s.Val.Unquote()
57+
if err != nil {
58+
return nil, err
59+
}
60+
tmp := &lineFilterOps{val: val}
61+
switch fn {
2662
case "|=":
27-
clause, err = l.doLike("like")
63+
return tmp.doLike("like")
2864
case "!=":
29-
clause, err = l.doLike("notLike")
65+
return tmp.doLike("notLike")
3066
case "|~":
31-
likeStr, isInsensitive, isLike := l.re2Like()
67+
likeStr, isInsensitive, isLike := tmp.re2Like()
3268
if isLike {
33-
l.Val = likeStr
69+
tmp.val = likeStr
3470
like := "like"
3571
if isInsensitive {
3672
like = "ilike"
3773
}
38-
clause, err = l.doLike(like)
39-
} else {
40-
clause = sql.Eq(&SqlMatch{
41-
col: sql.NewRawObject("string"),
42-
pattern: l.Val,
43-
}, sql.NewIntVal(1))
74+
return tmp.doLike(like)
4475
}
76+
return sql.Eq(&SqlMatch{
77+
col: sql.NewRawObject("string"),
78+
pattern: val,
79+
}, sql.NewIntVal(1)), nil
4580
case "!~":
46-
likeStr, isInsensitive, isLike := l.re2Like()
81+
likeStr, isInsensitive, isLike := tmp.re2Like()
4782
if isLike {
48-
l.Val = likeStr
83+
tmp.val = likeStr
4984
like := "notLike"
5085
if isInsensitive {
5186
like = "notILike"
5287
}
53-
clause, err = l.doLike(like)
54-
} else {
55-
clause = sql.Eq(&SqlMatch{
56-
col: sql.NewRawObject("string"),
57-
pattern: l.Val,
58-
}, sql.NewIntVal(1))
88+
return tmp.doLike(like)
5989
}
90+
return sql.Eq(&SqlMatch{
91+
col: sql.NewRawObject("string"),
92+
pattern: val,
93+
}, sql.NewIntVal(1)), nil
6094
case "|>":
61-
likeOp := l.patternMatch(l.Val)
62-
clause = sql.BinaryLogicalOp("LIKE", sql.NewRawObject("string"), sql.NewStringVal(likeOp))
63-
default:
64-
err = &shared.NotSupportedError{Msg: fmt.Sprintf("%s not supported", l.Op)}
95+
likeOp := patternMatch(val)
96+
return sql.BinaryLogicalOp("LIKE", sql.NewRawObject("string"), sql.NewStringVal(likeOp)), nil
6597
}
98+
return nil, &shared.NotSupportedError{Msg: fmt.Sprintf("%s not supported", fn)}
99+
}
66100

67-
if err != nil {
68-
return nil, err
69-
}
70-
return req.AndWhere(clause), nil
101+
// lineFilterOps holds per-value helpers (extracted from the old LineFilterPlanner methods).
102+
type lineFilterOps struct {
103+
val string
71104
}
72105

73-
func (l *LineFilterPlanner) doLike(likeOp string) (sql.SQLCondition, error) {
74-
enqVal, err := l.enquoteStr(l.Val)
106+
func (o *lineFilterOps) doLike(likeOp string) (sql.SQLCondition, error) {
107+
enqVal, err := sql.NewStringVal(o.val).String(&sql.Ctx{
108+
Params: map[string]sql.SQLObject{},
109+
Result: map[string]sql.SQLObject{},
110+
})
75111
if err != nil {
76112
return nil, err
77113
}
@@ -83,7 +119,18 @@ func (l *LineFilterPlanner) doLike(likeOp string) (sql.SQLCondition, error) {
83119
), nil
84120
}
85121

86-
func (l *LineFilterPlanner) patternMatch(match string) string {
122+
func (o *lineFilterOps) re2Like() (string, bool, bool) {
123+
exp, err := syntax.Parse(o.val, syntax.PerlX)
124+
if err != nil {
125+
return "", false, false
126+
}
127+
if exp.Op != syntax.OpLiteral || exp.Flags& ^(syntax.PerlX|syntax.FoldCase) != 0 {
128+
return "", false, false
129+
}
130+
return string(exp.Rune), exp.Flags&syntax.FoldCase != 0, true
131+
}
132+
133+
func patternMatch(match string) string {
87134
match = strings.Replace(match, "%", "%%", -1)
88135
parts := strings.Split(match, "<_>")
89136
for i := len(parts) - 2; i >= 0; i-- {
@@ -102,21 +149,3 @@ func (l *LineFilterPlanner) patternMatch(match string) string {
102149
}
103150
return strings.Join(parts, "%")
104151
}
105-
106-
func (l *LineFilterPlanner) enquoteStr(str string) (string, error) {
107-
return sql.NewStringVal(str).String(&sql.Ctx{
108-
Params: map[string]sql.SQLObject{},
109-
Result: map[string]sql.SQLObject{},
110-
})
111-
}
112-
113-
func (l *LineFilterPlanner) re2Like() (string, bool, bool) {
114-
exp, err := syntax.Parse(l.Val, syntax.PerlX)
115-
if err != nil {
116-
return "", false, false
117-
}
118-
if exp.Op != syntax.OpLiteral || exp.Flags & ^(syntax.PerlX|syntax.FoldCase) != 0 {
119-
return "", false, false
120-
}
121-
return string(exp.Rune), exp.Flags&syntax.FoldCase != 0, true
122-
}

0 commit comments

Comments
 (0)