Skip to content

Commit eef908c

Browse files
committed
perf: speed up multi-dictionary searches
1 parent 92ca28f commit eef908c

15 files changed

Lines changed: 1890 additions & 251 deletions

.dockerignore

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
.git
2+
.github
3+
.env
4+
5+
frontend/node_modules
6+
frontend/dist
7+
8+
backend/data
9+
backend/web/dist
10+
11+
dist

backend/Dockerfile

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
FROM --platform=$BUILDPLATFORM node:22-alpine AS frontend
22
WORKDIR /build/frontend
3+
ENV CI=true
34
COPY frontend/package.json frontend/pnpm-lock.yaml ./
45
RUN corepack enable && pnpm install --frozen-lockfile
56
COPY frontend/ .

backend/go.mod

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,9 +7,10 @@ require (
77
github.com/JohannesKaufmann/html-to-markdown/v2 v2.5.0
88
github.com/go-sql-driver/mysql v1.9.3
99
github.com/golang-jwt/jwt/v5 v5.3.1
10+
github.com/google/jsonschema-go v0.4.2
1011
github.com/labstack/echo/v5 v5.1.0
1112
github.com/lib-x/entsqlite v0.1.13
12-
github.com/lib-x/mdx v0.1.15
13+
github.com/lib-x/mdx v0.1.20
1314
github.com/lib/pq v1.12.3
1415
github.com/modelcontextprotocol/go-sdk v1.5.0
1516
github.com/redis/go-redis/v9 v9.18.0
@@ -30,7 +31,6 @@ require (
3031
github.com/fatih/color v1.18.0 // indirect
3132
github.com/go-openapi/inflect v0.19.0 // indirect
3233
github.com/google/go-cmp v0.7.0 // indirect
33-
github.com/google/jsonschema-go v0.4.2 // indirect
3434
github.com/google/uuid v1.6.0 // indirect
3535
github.com/hashicorp/hcl/v2 v2.18.1 // indirect
3636
github.com/mattn/go-colorable v0.1.14 // indirect

backend/go.sum

Lines changed: 2 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -24,8 +24,6 @@ github.com/c0mm4nd/go-ripemd v0.0.0-20200326052756-bd1759ad7d10 h1:wJ2csnFApV9G1
2424
github.com/c0mm4nd/go-ripemd v0.0.0-20200326052756-bd1759ad7d10/go.mod h1:mYPR+a1fzjnHY3VFH5KL3PkEjMlVfGXP7c8rbWlkLJg=
2525
github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
2626
github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
27-
github.com/clipperhouse/displaywidth v0.6.2 h1:ZDpTkFfpHOKte4RG5O/BOyf3ysnvFswpyYrV7z2uAKo=
28-
github.com/clipperhouse/displaywidth v0.6.2/go.mod h1:R+kHuzaYWFkTm7xoMmK1lFydbci4X2CicfbGstSGg0o=
2927
github.com/clipperhouse/stringish v0.1.1 h1:+NSqMOr3GR6k1FdRhhnXrLfztGzuG+VuFDfatpWHKCs=
3028
github.com/clipperhouse/stringish v0.1.1/go.mod h1:v/WhFtE1q0ovMta2+m+UbpZ+2/HEXNWYXQgCt4hdOzA=
3129
github.com/clipperhouse/uax29/v2 v2.3.0 h1:SNdx9DVUqMoBuBoW3iLOj4FQv3dN5mDtuqwuhIGpJy4=
@@ -72,8 +70,8 @@ github.com/labstack/echo/v5 v5.1.0 h1:MvIRydoN+p9cx/zq8Lff6YXqUW2ZaEsOMISzEGSMrB
7270
github.com/labstack/echo/v5 v5.1.0/go.mod h1:SyvlSdObGjRXeQfCCXW/sybkZdOOQZBmpKF0bvALaeo=
7371
github.com/lib-x/entsqlite v0.1.13 h1:q7g5NKd4nJDWuLvBlLFwjkWG2rv3P7rwLKRXi2xkykI=
7472
github.com/lib-x/entsqlite v0.1.13/go.mod h1:3Bh+hMqf00TpJJM7ilnH5ofTbPUuY50i+JQEl4a93I8=
75-
github.com/lib-x/mdx v0.1.15 h1:kaQUT9xQycp7gUxMJexn8iJ3sqAAVsxsZ24EjbVSDTM=
76-
github.com/lib-x/mdx v0.1.15/go.mod h1:li7ykbhNjz2IxFEge0SOTycY/fBsyfNrpZbKVkMpplw=
73+
github.com/lib-x/mdx v0.1.20 h1:uhpUHDi8ty/CjHXqX5yOovF9CKh1z04s+2uMqLFd/Ms=
74+
github.com/lib-x/mdx v0.1.20/go.mod h1:li7ykbhNjz2IxFEge0SOTycY/fBsyfNrpZbKVkMpplw=
7775
github.com/lib/pq v1.12.3 h1:tTWxr2YLKwIvK90ZXEw8GP7UFHtcbTtty8zsI+YjrfQ=
7876
github.com/lib/pq v1.12.3/go.mod h1:/p+8NSbOcwzAEI7wiMXFlgydTwcgTr3OSKMsD2BitpA=
7977
github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHPsaIE=
@@ -93,14 +91,6 @@ github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOF
9391
github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls=
9492
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs=
9593
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
96-
github.com/olekukonko/cat v0.0.0-20250911104152-50322a0618f6 h1:zrbMGy9YXpIeTnGj4EljqMiZsIcE09mmF8XsD5AYOJc=
97-
github.com/olekukonko/cat v0.0.0-20250911104152-50322a0618f6/go.mod h1:rEKTHC9roVVicUIfZK7DYrdIoM0EOr8mK1Hj5s3JjH0=
98-
github.com/olekukonko/errors v1.1.0 h1:RNuGIh15QdDenh+hNvKrJkmxxjV4hcS50Db478Ou5sM=
99-
github.com/olekukonko/errors v1.1.0/go.mod h1:ppzxA5jBKcO1vIpCXQ9ZqgDh8iwODz6OXIGKU8r5m4Y=
100-
github.com/olekukonko/ll v0.1.4-0.20260115111900-9e59c2286df0 h1:jrYnow5+hy3WRDCBypUFvVKNSPPCdqgSXIE9eJDD8LM=
101-
github.com/olekukonko/ll v0.1.4-0.20260115111900-9e59c2286df0/go.mod h1:b52bVQRRPObe+yyBl0TxNfhesL0nedD4Cht0/zx55Ew=
102-
github.com/olekukonko/tablewriter v1.1.3 h1:VSHhghXxrP0JHl+0NnKid7WoEmd9/urKRJLysb70nnA=
103-
github.com/olekukonko/tablewriter v1.1.3/go.mod h1:9VU0knjhmMkXjnMKrZ3+L2JhhtsQ/L38BbL3CRNE8tM=
10494
github.com/op/go-logging v0.0.0-20160315200505-970db520ece7 h1:lDH9UUVJtmYCjyT0CI4q8xvlXPxeZ0gYCVvWbmPlp88=
10595
github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk=
10696
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
@@ -124,10 +114,6 @@ github.com/segmentio/encoding v0.5.4 h1:OW1VRern8Nw6ITAtwSZ7Idrl3MXCFwXHPgqESYfv
124114
github.com/segmentio/encoding v0.5.4/go.mod h1:HS1ZKa3kSN32ZHVZ7ZLPLXWvOVIiZtyJnO1gPH1sKt0=
125115
github.com/sergi/go-diff v1.4.0 h1:n/SP9D5ad1fORl+llWyN+D6qoUETXNZARKjyY2/KVCw=
126116
github.com/sergi/go-diff v1.4.0/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4=
127-
github.com/spf13/cobra v1.7.0 h1:hyqWnYt1ZQShIddO5kBpj3vu05/++x6tJ6dg8EC572I=
128-
github.com/spf13/cobra v1.7.0/go.mod h1:uLxZILRyS/50WlhOIKD7W6V5bgeIt+4sICxh6uRMrb0=
129-
github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
130-
github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
131117
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
132118
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
133119
github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=

backend/internal/api/mcp.go

Lines changed: 3 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -230,14 +230,12 @@ func (s *Server) resolveMCPDictionaryID(ctx context.Context, userID int, diction
230230
if name == "" {
231231
return 0, nil
232232
}
233-
items, err := s.dictionaries.ListAccessible(ctx, userID)
233+
id, found, err := s.dictionaries.ResolveAccessibleDictionaryID(ctx, userID, name)
234234
if err != nil {
235235
return 0, err
236236
}
237-
for _, item := range items {
238-
if strings.EqualFold(item.Name, name) || strings.EqualFold(item.Title, name) || strings.EqualFold(firstNonEmpty(item.Title, item.Name), name) {
239-
return item.ID, nil
240-
}
237+
if found {
238+
return id, nil
241239
}
242240
return 0, fmt.Errorf("dictionary %q is not available to this token", name)
243241
}
Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
package dictionary
2+
3+
import (
4+
"context"
5+
"strings"
6+
7+
entdict "owl/backend/ent/dictionary"
8+
entuser "owl/backend/ent/user"
9+
)
10+
11+
// ResolveAccessibleDictionaryID finds one enabled dictionary by name or title
12+
// without loading and materializing the caller's entire accessible library.
13+
func (s *Service) ResolveAccessibleDictionaryID(ctx context.Context, userID int, name string) (int, bool, error) {
14+
name = strings.TrimSpace(name)
15+
if name == "" {
16+
return 0, false, nil
17+
}
18+
19+
query := s.client.Dictionary.Query().
20+
Where(
21+
entdict.Enabled(true),
22+
entdict.Or(
23+
entdict.Public(true),
24+
entdict.HasOwnerWith(entuser.IDEQ(userID)),
25+
),
26+
).
27+
Order(entdict.ByTitle(), entdict.ByID()).
28+
Select(entdict.FieldID, entdict.FieldName, entdict.FieldTitle)
29+
items, err := query.All(ctx)
30+
if err != nil {
31+
return 0, false, err
32+
}
33+
for _, item := range items {
34+
if strings.EqualFold(item.Name, name) || strings.EqualFold(item.Title, name) {
35+
return item.ID, true, nil
36+
}
37+
}
38+
return 0, false, nil
39+
}
Lines changed: 192 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,192 @@
1+
package dictionary
2+
3+
import (
4+
"fmt"
5+
"path/filepath"
6+
"strings"
7+
"testing"
8+
9+
"owl/backend/ent"
10+
11+
_ "github.com/lib-x/entsqlite"
12+
)
13+
14+
func TestResolveAccessibleDictionaryIDHonorsVisibilityAndEnabledState(t *testing.T) {
15+
client, ownerID, otherUserID := newLookupTestClient(t, 0)
16+
ctx := t.Context()
17+
18+
owned, err := client.Dictionary.Create().
19+
SetName("owned-dictionary").
20+
SetTitle("Owned Dictionary").
21+
SetSlug("owned-dictionary").
22+
SetMdxPath(filepath.Join(t.TempDir(), "owned.mdx")).
23+
SetOwnerID(ownerID).
24+
Save(ctx)
25+
if err != nil {
26+
t.Fatal(err)
27+
}
28+
public, err := client.Dictionary.Create().
29+
SetName("public-dictionary").
30+
SetTitle("Public Dictionary").
31+
SetSlug("public-dictionary").
32+
SetMdxPath(filepath.Join(t.TempDir(), "public.mdx")).
33+
SetOwnerID(otherUserID).
34+
SetPublic(true).
35+
Save(ctx)
36+
if err != nil {
37+
t.Fatal(err)
38+
}
39+
unicode, err := client.Dictionary.Create().
40+
SetName("Kelvin").
41+
SetTitle("A Unicode Fold").
42+
SetSlug("unicode-fold").
43+
SetMdxPath(filepath.Join(t.TempDir(), "unicode.mdx")).
44+
SetOwnerID(otherUserID).
45+
SetPublic(true).
46+
Save(ctx)
47+
if err != nil {
48+
t.Fatal(err)
49+
}
50+
_, err = client.Dictionary.Create().
51+
SetName("kelvin").
52+
SetTitle("Z ASCII Fold").
53+
SetSlug("ascii-fold").
54+
SetMdxPath(filepath.Join(t.TempDir(), "ascii.mdx")).
55+
SetOwnerID(otherUserID).
56+
SetPublic(true).
57+
Save(ctx)
58+
if err != nil {
59+
t.Fatal(err)
60+
}
61+
_, err = client.Dictionary.Create().
62+
SetName("disabled-dictionary").
63+
SetTitle("Disabled Dictionary").
64+
SetSlug("disabled-dictionary").
65+
SetMdxPath(filepath.Join(t.TempDir(), "disabled.mdx")).
66+
SetOwnerID(ownerID).
67+
SetEnabled(false).
68+
Save(ctx)
69+
if err != nil {
70+
t.Fatal(err)
71+
}
72+
_, err = client.Dictionary.Create().
73+
SetName("private-dictionary").
74+
SetTitle("Private Dictionary").
75+
SetSlug("private-dictionary").
76+
SetMdxPath(filepath.Join(t.TempDir(), "private.mdx")).
77+
SetOwnerID(otherUserID).
78+
Save(ctx)
79+
if err != nil {
80+
t.Fatal(err)
81+
}
82+
83+
svc := NewService(client, "", "", nil, "", 0, "", false, 0, "", "")
84+
tests := []struct {
85+
name string
86+
query string
87+
want int
88+
found bool
89+
}{
90+
{name: "owned by internal name", query: " OWNED-DICTIONARY ", want: owned.ID, found: true},
91+
{name: "public by title", query: "public dictionary", want: public.ID, found: true},
92+
{name: "unicode simple case fold", query: "kelvin", want: unicode.ID, found: true},
93+
{name: "disabled", query: "disabled-dictionary"},
94+
{name: "other private", query: "private-dictionary"},
95+
{name: "empty", query: " "},
96+
}
97+
for _, tt := range tests {
98+
t.Run(tt.name, func(t *testing.T) {
99+
got, found, err := svc.ResolveAccessibleDictionaryID(t.Context(), ownerID, tt.query)
100+
if err != nil {
101+
t.Fatal(err)
102+
}
103+
if got != tt.want || found != tt.found {
104+
t.Fatalf("got (%d, %t), want (%d, %t)", got, found, tt.want, tt.found)
105+
}
106+
})
107+
}
108+
}
109+
110+
func BenchmarkResolveAccessibleDictionaryID256(b *testing.B) {
111+
client, ownerID, _ := newLookupTestClient(b, 256)
112+
svc := NewService(client, "", "", nil, "", 0, "", false, 0, "", "")
113+
const target = "Dictionary 255"
114+
115+
b.Run("full_list_and_match", func(b *testing.B) {
116+
b.ReportAllocs()
117+
for b.Loop() {
118+
items, err := svc.ListAccessible(b.Context(), ownerID)
119+
if err != nil {
120+
b.Fatal(err)
121+
}
122+
found := 0
123+
for _, item := range items {
124+
if strings.EqualFold(item.Name, target) || strings.EqualFold(item.Title, target) {
125+
found = item.ID
126+
break
127+
}
128+
}
129+
if found == 0 {
130+
b.Fatal("target dictionary not found")
131+
}
132+
}
133+
})
134+
135+
b.Run("select_target_id", func(b *testing.B) {
136+
b.ReportAllocs()
137+
for b.Loop() {
138+
id, found, err := svc.ResolveAccessibleDictionaryID(b.Context(), ownerID, target)
139+
if err != nil {
140+
b.Fatal(err)
141+
}
142+
if !found || id == 0 {
143+
b.Fatal("target dictionary not found")
144+
}
145+
}
146+
})
147+
}
148+
149+
func newLookupTestClient(tb testing.TB, dictionaryCount int) (*ent.Client, int, int) {
150+
tb.Helper()
151+
databasePath := filepath.Join(tb.TempDir(), "lookup.db")
152+
client, err := ent.Open("sqlite3", "file:"+databasePath+"?_pragma=foreign_keys(1)")
153+
if err != nil {
154+
tb.Fatal(err)
155+
}
156+
tb.Cleanup(func() { _ = client.Close() })
157+
ctx := tb.Context()
158+
if err := client.Schema.Create(ctx); err != nil {
159+
tb.Fatal(err)
160+
}
161+
owner, err := client.User.Create().
162+
SetUsername("owner").
163+
SetDisplayName("Owner").
164+
SetPasswordHash("test").
165+
Save(ctx)
166+
if err != nil {
167+
tb.Fatal(err)
168+
}
169+
other, err := client.User.Create().
170+
SetUsername("other").
171+
SetDisplayName("Other").
172+
SetPasswordHash("test").
173+
Save(ctx)
174+
if err != nil {
175+
tb.Fatal(err)
176+
}
177+
mdxPath := filepath.Join(tb.TempDir(), "dictionary.mdx")
178+
for i := range dictionaryCount {
179+
name := fmt.Sprintf("Dictionary %03d", i)
180+
_, err := client.Dictionary.Create().
181+
SetName(name).
182+
SetTitle(name).
183+
SetSlug(fmt.Sprintf("dictionary-%03d", i)).
184+
SetMdxPath(mdxPath).
185+
SetOwnerID(owner.ID).
186+
Save(ctx)
187+
if err != nil {
188+
tb.Fatal(err)
189+
}
190+
}
191+
return client, owner.ID, other.ID
192+
}

backend/internal/dictionary/managed_index_store.go

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ package dictionary
22

33
import (
44
"strings"
5+
"time"
56

67
"github.com/lib-x/mdx"
78
)
@@ -64,6 +65,33 @@ func (s *managedDictionaryIndexStore) DeleteDictionary(dictionaryName string) er
6465
return s.prefixStore.DeleteDictionary(dictionaryName)
6566
}
6667

68+
func (s *managedDictionaryIndexStore) HasDictionaryIndex(dictionaryName string) (bool, error) {
69+
// Use the method set instead of naming mdx.IndexHealthStore directly so
70+
// this adapter remains source-compatible with mdx releases predating the
71+
// optional health interface.
72+
healthStore, ok := s.prefixStore.(interface {
73+
HasDictionaryIndex(string) (bool, error)
74+
})
75+
if !ok {
76+
// Health checks are optional. Preserve the lifecycle behavior of a
77+
// ManagedIndexStore that cannot verify its underlying data.
78+
return true, nil
79+
}
80+
return healthStore.HasDictionaryIndex(sanitizeManagedDictionaryName(dictionaryName))
81+
}
82+
83+
func (s *managedDictionaryIndexStore) AcquireIndexBuildLease(dictionaryName string, ttl time.Duration) (func() error, bool, error) {
84+
leaseStore, ok := s.prefixStore.(interface {
85+
AcquireIndexBuildLease(string, time.Duration) (func() error, bool, error)
86+
})
87+
if !ok {
88+
// A missing optional lease capability means this process may proceed;
89+
// in-process namespace locking still serializes local rebuilds.
90+
return func() error { return nil }, true, nil
91+
}
92+
return leaseStore.AcquireIndexBuildLease(sanitizeManagedDictionaryName(dictionaryName), ttl)
93+
}
94+
6795
func sanitizeManagedDictionaryName(name string) string {
6896
name = strings.TrimSpace(name)
6997
if name == "" {

0 commit comments

Comments
 (0)