Skip to content

Commit a94ce5f

Browse files
ziflexziflex
committed
feat(compiler): expose syntax word metadata
1 parent 63ba629 commit a94ce5f

4 files changed

Lines changed: 386 additions & 1 deletion

File tree

‎pkg/compiler/semantic.go‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -64,6 +64,8 @@ type (
6464
// Span uses zero-based, half-open UTF-8 byte offsets.
6565
SyntaxToken struct {
6666
Kind SyntaxTokenKind
67+
// Word is the canonical language-word identity, or SyntaxWordUnknown.
68+
Word SyntaxWord
6769
Span source.Span
6870
}
6971

‎pkg/compiler/syntax_tokens.go‎

Lines changed: 141 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
11
package compiler
22

33
import (
4+
"strings"
5+
46
"github.com/antlr4-go/antlr/v4"
57

68
"github.com/MontFerret/ferret/v2/pkg/parser/fql"
@@ -28,12 +30,150 @@ func buildSyntaxTokens(src *source.Source, tokens []antlr.Token) []SyntaxToken {
2830
continue
2931
}
3032

31-
out = append(out, SyntaxToken{Kind: kind, Span: span})
33+
out = append(out, SyntaxToken{
34+
Kind: kind,
35+
Word: syntaxWord(token.GetTokenType(), token.GetText()),
36+
Span: span,
37+
})
3238
}
3339

3440
return out
3541
}
3642

43+
func syntaxWord(tokenType int, text string) SyntaxWord {
44+
switch tokenType {
45+
case fql.FqlLexerAggregate:
46+
return SyntaxWordAggregate
47+
case fql.FqlLexerAll:
48+
return SyntaxWordAll
49+
case fql.FqlLexerAnd:
50+
if strings.EqualFold(text, "AND") {
51+
return SyntaxWordAnd
52+
}
53+
case fql.FqlLexerAny:
54+
return SyntaxWordAny
55+
case fql.FqlLexerAs:
56+
return SyntaxWordAs
57+
case fql.FqlLexerAt:
58+
return SyntaxWordAt
59+
case fql.FqlLexerBackoff:
60+
return SyntaxWordBackoff
61+
case fql.FqlLexerCollect:
62+
return SyntaxWordCollect
63+
case fql.FqlLexerCount:
64+
return SyntaxWordCount
65+
case fql.FqlLexerDelete:
66+
return SyntaxWordDelete
67+
case fql.FqlLexerDispatch:
68+
return SyntaxWordDispatch
69+
case fql.FqlLexerDistinct:
70+
return SyntaxWordDistinct
71+
case fql.FqlLexerDo:
72+
return SyntaxWordDo
73+
case fql.FqlLexerEvent:
74+
return SyntaxWordEvent
75+
case fql.FqlLexerEvery:
76+
return SyntaxWordEvery
77+
case fql.FqlLexerExists:
78+
return SyntaxWordExists
79+
case fql.FqlLexerFilter:
80+
return SyntaxWordFilter
81+
case fql.FqlLexerFor:
82+
return SyntaxWordFor
83+
case fql.FqlLexerFunc:
84+
return SyntaxWordFunc
85+
case fql.FqlLexerIn:
86+
return SyntaxWordIn
87+
case fql.FqlLexerInto:
88+
return SyntaxWordInto
89+
case fql.FqlLexerJitter:
90+
return SyntaxWordJitter
91+
case fql.FqlLexerKeep:
92+
return SyntaxWordKeep
93+
case fql.FqlLexerLeast:
94+
return SyntaxWordLeast
95+
case fql.FqlLexerLet:
96+
return SyntaxWordLet
97+
case fql.FqlLexerLike:
98+
return SyntaxWordLike
99+
case fql.FqlLexerLimit:
100+
return SyntaxWordLimit
101+
case fql.FqlLexerMatch:
102+
return SyntaxWordMatch
103+
case fql.FqlLexerNone:
104+
return SyntaxWordNone
105+
case fql.FqlLexerNot:
106+
if strings.EqualFold(text, "NOT") {
107+
return SyntaxWordNot
108+
}
109+
case fql.FqlLexerNull:
110+
return SyntaxWordNull
111+
case fql.FqlLexerOne:
112+
return SyntaxWordOne
113+
case fql.FqlLexerOptions:
114+
return SyntaxWordOptions
115+
case fql.FqlLexerOr:
116+
if strings.EqualFold(text, "OR") {
117+
return SyntaxWordOr
118+
}
119+
case fql.FqlLexerQuery:
120+
return SyntaxWordQuery
121+
case fql.FqlLexerReturn:
122+
return SyntaxWordReturn
123+
case fql.FqlLexerSort:
124+
return SyntaxWordSort
125+
case fql.FqlLexerSortDirection:
126+
switch {
127+
case strings.EqualFold(text, "ASC"):
128+
return SyntaxWordAsc
129+
case strings.EqualFold(text, "DESC"):
130+
return SyntaxWordDesc
131+
}
132+
case fql.FqlLexerTimeout:
133+
return SyntaxWordTimeout
134+
case fql.FqlLexerTrigger:
135+
return SyntaxWordTrigger
136+
case fql.FqlLexerBooleanLiteral:
137+
switch {
138+
case strings.EqualFold(text, "TRUE"):
139+
return SyntaxWordTrue
140+
case strings.EqualFold(text, "FALSE"):
141+
return SyntaxWordFalse
142+
}
143+
case fql.FqlLexerUse:
144+
return SyntaxWordUse
145+
case fql.FqlLexerUsing:
146+
return SyntaxWordUsing
147+
case fql.FqlLexerValue:
148+
return SyntaxWordValue
149+
case fql.FqlLexerVar:
150+
return SyntaxWordVar
151+
case fql.FqlLexerWaitfor:
152+
return SyntaxWordWaitfor
153+
case fql.FqlLexerWhen:
154+
return SyntaxWordWhen
155+
case fql.FqlLexerWhile:
156+
return SyntaxWordWhile
157+
case fql.FqlLexerWith:
158+
return SyntaxWordWith
159+
case fql.FqlLexerIdentifier:
160+
switch {
161+
case strings.EqualFold(text, "ON"):
162+
return SyntaxWordOn
163+
case strings.EqualFold(text, "ERROR"):
164+
return SyntaxWordError
165+
case strings.EqualFold(text, "FAIL"):
166+
return SyntaxWordFail
167+
case strings.EqualFold(text, "RETRY"):
168+
return SyntaxWordRetry
169+
case strings.EqualFold(text, "DELAY"):
170+
return SyntaxWordDelay
171+
}
172+
}
173+
174+
return SyntaxWordUnknown
175+
}
176+
37177
func syntaxTokenKind(tokenType int) SyntaxTokenKind {
38178
switch tokenType {
39179
case fql.FqlLexerMultiLineComment, fql.FqlLexerSingleLineComment:

‎pkg/compiler/syntax_tokens_test.go‎

Lines changed: 93 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -130,6 +130,99 @@ func TestAnalysisSyntaxTokensReturnsDefensiveCopy(t *testing.T) {
130130
}
131131
}
132132

133+
func TestSyntaxWordsReturnCanonicalCategorizedDefensiveMetadata(t *testing.T) {
134+
words := SyntaxWords()
135+
if got, want := len(words), int(SyntaxWordWith); got != want {
136+
t.Fatalf("SyntaxWords length = %d, want %d", got, want)
137+
}
138+
139+
seen := make(map[SyntaxWord]SyntaxWordInfo, len(words))
140+
previous := ""
141+
142+
for _, word := range words {
143+
if word.Word == SyntaxWordUnknown {
144+
t.Fatalf("SyntaxWords contains unknown word: %+v", word)
145+
}
146+
147+
if _, ok := seen[word.Word]; ok {
148+
t.Fatalf("SyntaxWords contains duplicate identity %d", word.Word)
149+
}
150+
151+
if word.Spelling != strings.ToUpper(word.Spelling) {
152+
t.Fatalf("SyntaxWords spelling %q is not canonical uppercase", word.Spelling)
153+
}
154+
155+
if previous != "" && previous >= word.Spelling {
156+
t.Fatalf("SyntaxWords are not in spelling order: %q before %q", previous, word.Spelling)
157+
}
158+
159+
seen[word.Word] = word
160+
previous = word.Spelling
161+
}
162+
163+
for word := SyntaxWord(1); word <= SyntaxWordWith; word++ {
164+
if _, ok := seen[word]; !ok {
165+
t.Errorf("SyntaxWords omits identity %d", word)
166+
}
167+
}
168+
169+
wantCategories := map[SyntaxWord]SyntaxWordCategory{
170+
SyntaxWordLet: SyntaxWordCategoryKeyword,
171+
SyntaxWordAnd: SyntaxWordCategoryOperator,
172+
SyntaxWordTrue: SyntaxWordCategoryLiteral,
173+
SyntaxWordRetry: SyntaxWordCategoryContextual,
174+
}
175+
176+
for word, want := range wantCategories {
177+
if got := seen[word].Category; got != want {
178+
t.Errorf("SyntaxWords category for %q = %d, want %d", seen[word].Spelling, got, want)
179+
}
180+
}
181+
182+
words[0] = SyntaxWordInfo{}
183+
if next := SyntaxWords(); len(next) == 0 || next[0] == (SyntaxWordInfo{}) {
184+
t.Fatal("SyntaxWords exposed mutable metadata storage")
185+
}
186+
}
187+
188+
func TestAnalyzeSyntaxTokensExposeCanonicalWordIdentity(t *testing.T) {
189+
query := "let value = true\nRETURN value AND false ASC DESC && ! ON ERROR FAIL RETRY DELAY"
190+
analysis, _ := New().Analyze(source.NewAnonymous(query))
191+
if analysis == nil {
192+
t.Fatal("Analyze returned no partial snapshot")
193+
}
194+
195+
wantWords := map[string]SyntaxWord{
196+
"let": SyntaxWordLet,
197+
"true": SyntaxWordTrue,
198+
"RETURN": SyntaxWordReturn,
199+
"AND": SyntaxWordAnd,
200+
"false": SyntaxWordFalse,
201+
"ASC": SyntaxWordAsc,
202+
"DESC": SyntaxWordDesc,
203+
"ON": SyntaxWordOn,
204+
"ERROR": SyntaxWordError,
205+
"FAIL": SyntaxWordFail,
206+
"RETRY": SyntaxWordRetry,
207+
"DELAY": SyntaxWordDelay,
208+
}
209+
210+
tokens := analysis.SyntaxTokens()
211+
for text, want := range wantWords {
212+
token, ok := syntaxTokenWithText(tokens, query, text)
213+
if !ok || token.Word != want {
214+
t.Errorf("syntax word %q = %+v, %t, want identity %d", text, token, ok, want)
215+
}
216+
}
217+
218+
for _, text := range []string{"&&", "!"} {
219+
token, ok := syntaxTokenWithText(tokens, query, text)
220+
if !ok || token.Word != SyntaxWordUnknown {
221+
t.Errorf("symbolic operator %q word = %+v, %t", text, token, ok)
222+
}
223+
}
224+
}
225+
133226
func hasSyntaxTokenKind(tokens []SyntaxToken, kind SyntaxTokenKind) bool {
134227
for _, token := range tokens {
135228
if token.Kind == kind {

0 commit comments

Comments
 (0)