Skip to content

Commit 8181f93

Browse files
maparentmaparent
committed
ENG-1907-proposed changes
Add an original column to Content. Add a full index including original to Content; To allow multiple distinct format translations, set original to NULL instead of FALSE. Adjust the ForeignKey from FileReference to Content to use that index, instead of adding content_type to ForeignKey. That last change is worth discussing.
1 parent 5b1ca3e commit 8181f93

5 files changed

Lines changed: 94 additions & 50 deletions

File tree

‎packages/database/src/dbTypes.ts‎

Lines changed: 17 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -264,6 +264,7 @@ export type Database = {
264264
id: number
265265
last_modified: string
266266
metadata: Json
267+
original: boolean
267268
part_of_id: number | null
268269
scale: Database["public"]["Enums"]["Scale"]
269270
source_local_id: string | null
@@ -280,6 +281,7 @@ export type Database = {
280281
id?: number
281282
last_modified: string
282283
metadata?: Json
284+
original?: boolean
283285
part_of_id?: number | null
284286
scale: Database["public"]["Enums"]["Scale"]
285287
source_local_id?: string | null
@@ -296,6 +298,7 @@ export type Database = {
296298
id?: number
297299
last_modified?: string
298300
metadata?: Json
301+
original?: boolean
299302
part_of_id?: number | null
300303
scale?: Database["public"]["Enums"]["Scale"]
301304
source_local_id?: string | null
@@ -587,70 +590,70 @@ export type Database = {
587590
}
588591
FileReference: {
589592
Row: {
590-
content_type: string
591593
created: string
592594
filehash: string
593595
filepath: string
594596
last_modified: string
597+
original: boolean
595598
source_local_id: string
596599
space_id: number
597600
variant: Database["public"]["Enums"]["ContentVariant"] | null
598601
}
599602
Insert: {
600-
content_type?: string
601603
created: string
602604
filehash: string
603605
filepath: string
604606
last_modified: string
607+
original?: boolean
605608
source_local_id: string
606609
space_id: number
607610
variant?: Database["public"]["Enums"]["ContentVariant"] | null
608611
}
609612
Update: {
610-
content_type?: string
611613
created?: string
612614
filehash?: string
613615
filepath?: string
614616
last_modified?: string
617+
original?: boolean
615618
source_local_id?: string
616619
space_id?: number
617620
variant?: Database["public"]["Enums"]["ContentVariant"] | null
618621
}
619622
Relationships: [
620623
{
621624
foreignKeyName: "FileReference_content_fkey"
622-
columns: ["space_id", "source_local_id", "variant", "content_type"]
625+
columns: ["space_id", "source_local_id", "variant", "original"]
623626
isOneToOne: false
624627
referencedRelation: "Content"
625628
referencedColumns: [
626629
"space_id",
627630
"source_local_id",
628631
"variant",
629-
"content_type",
632+
"original",
630633
]
631634
},
632635
{
633636
foreignKeyName: "FileReference_content_fkey"
634-
columns: ["space_id", "source_local_id", "variant", "content_type"]
637+
columns: ["space_id", "source_local_id", "variant", "original"]
635638
isOneToOne: false
636639
referencedRelation: "my_contents"
637640
referencedColumns: [
638641
"space_id",
639642
"source_local_id",
640643
"variant",
641-
"content_type",
644+
"original",
642645
]
643646
},
644647
{
645648
foreignKeyName: "FileReference_content_fkey"
646-
columns: ["space_id", "source_local_id", "variant", "content_type"]
649+
columns: ["space_id", "source_local_id", "variant", "original"]
647650
isOneToOne: false
648651
referencedRelation: "my_contents_with_embedding_openai_text_embedding_3_small_1536"
649652
referencedColumns: [
650653
"space_id",
651654
"source_local_id",
652655
"variant",
653-
"content_type",
656+
"original",
654657
]
655658
},
656659
]
@@ -1064,6 +1067,7 @@ export type Database = {
10641067
id: number | null
10651068
last_modified: string | null
10661069
metadata: Json | null
1070+
original: boolean | null
10671071
part_of_id: number | null
10681072
scale: Database["public"]["Enums"]["Scale"] | null
10691073
source_local_id: string | null
@@ -1168,21 +1172,22 @@ export type Database = {
11681172
my_contents_with_embedding_openai_text_embedding_3_small_1536: {
11691173
Row: {
11701174
author_id: number | null
1175+
content_type: string | null
11711176
created: string | null
11721177
creator_id: number | null
11731178
document_id: number | null
11741179
id: number | null
11751180
last_modified: string | null
11761181
metadata: Json | null
11771182
model: Database["public"]["Enums"]["EmbeddingName"] | null
1183+
original: boolean | null
11781184
part_of_id: number | null
11791185
scale: Database["public"]["Enums"]["Scale"] | null
11801186
source_local_id: string | null
11811187
space_id: number | null
11821188
text: string | null
11831189
variant: Database["public"]["Enums"]["ContentVariant"] | null
11841190
vector: string | null
1185-
content_type: string | null
11861191
}
11871192
Relationships: [
11881193
{
@@ -1455,6 +1460,7 @@ export type Database = {
14551460
id: number
14561461
last_modified: string
14571462
metadata: Json
1463+
original: boolean
14581464
part_of_id: number | null
14591465
scale: Database["public"]["Enums"]["Scale"]
14601466
source_local_id: string | null
@@ -1628,6 +1634,7 @@ export type Database = {
16281634
id: number | null
16291635
last_modified: string | null
16301636
metadata: Json | null
1637+
original: boolean | null
16311638
part_of_id: number | null
16321639
scale: Database["public"]["Enums"]["Scale"] | null
16331640
source_local_id: string | null

‎packages/database/supabase/migrations/20260704120000_update_content_uniqueness.sql‎

Lines changed: 61 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -1,26 +1,26 @@
1+
ALTER TABLE ONLY public."Content"
2+
ADD COLUMN original BOOLEAN DEFAULT true;
3+
14
ALTER TABLE ONLY public."FileReference"
25
DROP CONSTRAINT IF EXISTS "FileReference_content_fkey";
36

47
ALTER TABLE public."FileReference"
5-
ADD COLUMN IF NOT EXISTS content_type character varying NOT NULL DEFAULT 'text/obsidian+markdown';
6-
7-
UPDATE public."FileReference" AS fr
8-
SET content_type = ct.content_type
9-
FROM public."Content" AS ct
10-
WHERE fr.space_id = ct.space_id
11-
AND fr.source_local_id = ct.source_local_id
12-
AND fr.variant = ct.variant;
8+
ADD COLUMN original BOOLEAN GENERATED ALWAYS AS (true) STORED;
139

1410
DROP INDEX IF EXISTS public.content_space_local_id_variant_idx;
1511

1612
CREATE UNIQUE INDEX IF NOT EXISTS content_space_local_id_variant_content_type_idx ON public."Content" USING btree (
1713
space_id, source_local_id, variant, content_type
1814
) NULLS DISTINCT;
1915

16+
CREATE UNIQUE INDEX IF NOT EXISTS content_space_local_id_variant_content_type_originals_idx ON public."Content" USING btree (
17+
space_id, source_local_id, variant, original
18+
);
19+
2020
ALTER TABLE ONLY public."FileReference"
2121
ADD CONSTRAINT "FileReference_content_fkey" FOREIGN KEY (
22-
space_id, source_local_id, variant, content_type
23-
) REFERENCES public."Content" (space_id, source_local_id, variant, content_type) ON DELETE CASCADE;
22+
space_id, source_local_id, variant, original
23+
) REFERENCES public."Content"(space_id, source_local_id, variant, original) ON DELETE CASCADE;
2424

2525
CREATE OR REPLACE FUNCTION public._local_content_to_db_content(data public.content_local_input)
2626
RETURNS public."Content" STABLE
@@ -74,7 +74,7 @@ $$;
7474

7575
COMMENT ON FUNCTION public._local_content_to_db_content IS 'utility function so we have the option to use platform identifiers for content upsert';
7676

77-
CREATE OR REPLACE FUNCTION public.upsert_content(v_space_id bigint, data jsonb, v_creator_id BIGINT, content_as_document boolean DEFAULT TRUE)
77+
CREATE OR REPLACE FUNCTION public.upsert_content(v_space_id bigint, data jsonb, v_creator_id BIGINT, content_as_document boolean DEFAULT true)
7878
RETURNS SETOF BIGINT
7979
SET search_path = ''
8080
LANGUAGE plpgsql
@@ -213,27 +213,56 @@ $$;
213213

214214
COMMENT ON FUNCTION public.upsert_content IS 'batch content upsert';
215215

216-
CREATE OR REPLACE VIEW public.my_contents_with_embedding_openai_text_embedding_3_small_1536 AS
216+
DROP VIEW public.my_contents_with_embedding_openai_text_embedding_3_small_1536;
217+
218+
CREATE VIEW public.my_contents_with_embedding_openai_text_embedding_3_small_1536 AS
217219
SELECT
218-
ct.id,
219-
ct.document_id,
220-
ct.source_local_id,
221-
ct.variant,
222-
ct.author_id,
223-
ct.creator_id,
224-
ct.created,
225-
ct.text,
226-
ct.metadata,
227-
ct.scale,
228-
ct.space_id,
229-
ct.last_modified,
230-
ct.part_of_id,
231-
emb.model,
232-
emb.vector,
233-
ct.content_type
220+
ct.id,
221+
ct.document_id,
222+
ct.source_local_id,
223+
ct.variant,
224+
ct.author_id,
225+
ct.creator_id,
226+
ct.created,
227+
ct.text,
228+
ct.metadata,
229+
ct.scale,
230+
ct.space_id,
231+
ct.last_modified,
232+
ct.part_of_id,
233+
ct.content_type,
234+
ct.original,
235+
emb.model,
236+
emb.vector
234237
FROM public."Content" AS ct
235-
JOIN public."ContentEmbedding_openai_text_embedding_3_small_1536" AS emb ON (ct.id = emb.target_id)
236-
LEFT OUTER JOIN public.my_accessible_resources () AS ra USING (space_id, source_local_id)
237-
WHERE (ct.space_id = any (public.my_space_ids ('reader'))
238-
OR (ct.space_id = any (public.my_space_ids ('partial')) AND ra.space_id IS NOT NULL))
238+
JOIN public."ContentEmbedding_openai_text_embedding_3_small_1536" AS emb ON (ct.id = emb.target_id)
239+
LEFT OUTER JOIN public.my_accessible_resources() AS ra USING (space_id, source_local_id)
240+
WHERE (
241+
ct.space_id = any(public.my_space_ids('reader'))
242+
OR (ct.space_id = any(public.my_space_ids('partial')) AND ra.space_id IS NOT null)
243+
)
239244
AND NOT emb.obsolete;
245+
246+
CREATE OR REPLACE VIEW public.my_contents AS
247+
SELECT
248+
id,
249+
document_id,
250+
source_local_id,
251+
variant,
252+
author_id,
253+
creator_id,
254+
created,
255+
text,
256+
metadata,
257+
scale,
258+
space_id,
259+
last_modified,
260+
part_of_id,
261+
content_type,
262+
original
263+
FROM public."Content"
264+
LEFT OUTER JOIN public.my_accessible_resources() AS ra USING (space_id, source_local_id)
265+
WHERE (
266+
space_id = any(public.my_space_ids('reader'))
267+
OR (space_id = any(public.my_space_ids('partial')) AND ra.space_id IS NOT null)
268+
);

‎packages/database/supabase/schemas/assets.sql‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -5,17 +5,17 @@ CREATE TABLE IF NOT EXISTS public."FileReference" (
55
filehash character varying NOT NULL, -- or binary?
66
"created" timestamp without time zone NOT NULL,
77
last_modified timestamp without time zone NOT NULL,
8-
content_type character varying NOT NULL DEFAULT 'text/obsidian+markdown',
98
-- not allowed virtual with user types
10-
variant public."ContentVariant" GENERATED ALWAYS AS ('full') STORED
9+
variant public."ContentVariant" GENERATED ALWAYS AS ('full') STORED,
10+
original BOOLEAN GENERATED ALWAYS AS (true) STORED
1111
);
1212
ALTER TABLE ONLY public."FileReference"
1313
ADD CONSTRAINT "FileReference_pkey" PRIMARY KEY (source_local_id, space_id, filepath);
1414

1515
ALTER TABLE ONLY public."FileReference"
1616
ADD CONSTRAINT "FileReference_content_fkey" FOREIGN KEY (
17-
space_id, source_local_id, variant, content_type
18-
) REFERENCES public."Content" (space_id, source_local_id, variant, content_type) ON DELETE CASCADE;
17+
space_id, source_local_id, variant, original
18+
) REFERENCES public."Content"(space_id, source_local_id, variant, original) ON DELETE CASCADE;
1919
-- note the absence of on update ; the generated column forbids cascade, so it will error
2020
-- However, update on those columns should never happen.
2121

‎packages/database/supabase/schemas/content.sql‎

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -88,7 +88,9 @@ CREATE TABLE IF NOT EXISTS public."Content" (
8888
space_id bigint,
8989
last_modified timestamp without time zone NOT NULL,
9090
part_of_id bigint,
91-
content_type character varying NOT NULL DEFAULT 'text/plain'
91+
content_type character varying NOT NULL DEFAULT 'text/plain',
92+
-- Use null, never false for the non-original rows.
93+
original BOOLEAN DEFAULT true
9294
);
9395

9496
ALTER TABLE ONLY public."Content"
@@ -135,6 +137,10 @@ CREATE UNIQUE INDEX content_space_local_id_variant_content_type_idx ON public."C
135137
space_id, source_local_id, variant, content_type
136138
) NULLS DISTINCT;
137139

140+
CREATE UNIQUE INDEX IF NOT EXISTS content_space_local_id_variant_content_type_originals_idx ON public."Content" USING btree (
141+
space_id, source_local_id, variant, original
142+
);
143+
138144
CREATE INDEX "Content_text" ON public."Content" USING pgroonga (text);
139145

140146
ALTER TABLE public."Content" OWNER TO "postgres";
@@ -265,7 +271,8 @@ SELECT
265271
space_id,
266272
last_modified,
267273
part_of_id,
268-
content_type
274+
content_type,
275+
original
269276
FROM public."Content"
270277
LEFT OUTER JOIN public.my_accessible_resources() AS ra USING (space_id, source_local_id)
271278
WHERE (

‎packages/database/supabase/schemas/embedding.sql‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -43,9 +43,10 @@ ct.scale,
4343
ct.space_id,
4444
ct.last_modified,
4545
ct.part_of_id,
46+
ct.content_type,
47+
ct.original,
4648
emb.model,
47-
emb.vector,
48-
ct.content_type
49+
emb.vector
4950
FROM public."Content" AS ct
5051
JOIN public."ContentEmbedding_openai_text_embedding_3_small_1536" AS emb ON (ct.id = emb.target_id)
5152
LEFT OUTER JOIN public.my_accessible_resources () AS ra USING (space_id, source_local_id)

0 commit comments

Comments
 (0)