Skip to content

Commit 0287faa

Browse files
maparentmaparent
committed
ENG-1907-proposed changes
Add an original column to Content. Add a full index including original to Content; To allow multiple distinct format translations, set original to NULL instead of FALSE. Adjust the ForeignKey from FileReference to Content to use that index, instead of adding content_type to ForeignKey. That last change is worth discussing.
1 parent 5b1ca3e commit 0287faa

6 files changed

Lines changed: 114 additions & 57 deletions

File tree

‎packages/database/features/addContent.feature‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -175,7 +175,8 @@ Feature: Content access
175175
"created": "2000/01/01",
176176
"last_modified": "2001/01/02",
177177
"text": "{\"type\":\"root\",\"children\":[]}",
178-
"content_type": "application/vnd.discourse-graph.atjson+json; version=1"
178+
"content_type": "application/vnd.discourse-graph.atjson+json; version=1",
179+
"original": false
179180
}
180181
]
181182
"""

‎packages/database/src/dbTypes.ts‎

Lines changed: 18 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -264,6 +264,7 @@ export type Database = {
264264
id: number
265265
last_modified: string
266266
metadata: Json
267+
original: boolean | null
267268
part_of_id: number | null
268269
scale: Database["public"]["Enums"]["Scale"]
269270
source_local_id: string | null
@@ -280,6 +281,7 @@ export type Database = {
280281
id?: number
281282
last_modified: string
282283
metadata?: Json
284+
original?: boolean | null
283285
part_of_id?: number | null
284286
scale: Database["public"]["Enums"]["Scale"]
285287
source_local_id?: string | null
@@ -296,6 +298,7 @@ export type Database = {
296298
id?: number
297299
last_modified?: string
298300
metadata?: Json
301+
original?: boolean | null
299302
part_of_id?: number | null
300303
scale?: Database["public"]["Enums"]["Scale"]
301304
source_local_id?: string | null
@@ -587,70 +590,70 @@ export type Database = {
587590
}
588591
FileReference: {
589592
Row: {
590-
content_type: string
591593
created: string
592594
filehash: string
593595
filepath: string
594596
last_modified: string
597+
original: boolean | null
595598
source_local_id: string
596599
space_id: number
597600
variant: Database["public"]["Enums"]["ContentVariant"] | null
598601
}
599602
Insert: {
600-
content_type?: string
601603
created: string
602604
filehash: string
603605
filepath: string
604606
last_modified: string
607+
original?: boolean | null
605608
source_local_id: string
606609
space_id: number
607610
variant?: Database["public"]["Enums"]["ContentVariant"] | null
608611
}
609612
Update: {
610-
content_type?: string
611613
created?: string
612614
filehash?: string
613615
filepath?: string
614616
last_modified?: string
617+
original?: boolean | null
615618
source_local_id?: string
616619
space_id?: number
617620
variant?: Database["public"]["Enums"]["ContentVariant"] | null
618621
}
619622
Relationships: [
620623
{
621624
foreignKeyName: "FileReference_content_fkey"
622-
columns: ["space_id", "source_local_id", "variant", "content_type"]
625+
columns: ["space_id", "source_local_id", "variant", "original"]
623626
isOneToOne: false
624627
referencedRelation: "Content"
625628
referencedColumns: [
626629
"space_id",
627630
"source_local_id",
628631
"variant",
629-
"content_type",
632+
"original",
630633
]
631634
},
632635
{
633636
foreignKeyName: "FileReference_content_fkey"
634-
columns: ["space_id", "source_local_id", "variant", "content_type"]
637+
columns: ["space_id", "source_local_id", "variant", "original"]
635638
isOneToOne: false
636639
referencedRelation: "my_contents"
637640
referencedColumns: [
638641
"space_id",
639642
"source_local_id",
640643
"variant",
641-
"content_type",
644+
"original",
642645
]
643646
},
644647
{
645648
foreignKeyName: "FileReference_content_fkey"
646-
columns: ["space_id", "source_local_id", "variant", "content_type"]
649+
columns: ["space_id", "source_local_id", "variant", "original"]
647650
isOneToOne: false
648651
referencedRelation: "my_contents_with_embedding_openai_text_embedding_3_small_1536"
649652
referencedColumns: [
650653
"space_id",
651654
"source_local_id",
652655
"variant",
653-
"content_type",
656+
"original",
654657
]
655658
},
656659
]
@@ -1064,6 +1067,7 @@ export type Database = {
10641067
id: number | null
10651068
last_modified: string | null
10661069
metadata: Json | null
1070+
original: boolean | null
10671071
part_of_id: number | null
10681072
scale: Database["public"]["Enums"]["Scale"] | null
10691073
source_local_id: string | null
@@ -1168,21 +1172,22 @@ export type Database = {
11681172
my_contents_with_embedding_openai_text_embedding_3_small_1536: {
11691173
Row: {
11701174
author_id: number | null
1175+
content_type: string | null
11711176
created: string | null
11721177
creator_id: number | null
11731178
document_id: number | null
11741179
id: number | null
11751180
last_modified: string | null
11761181
metadata: Json | null
11771182
model: Database["public"]["Enums"]["EmbeddingName"] | null
1183+
original: boolean | null
11781184
part_of_id: number | null
11791185
scale: Database["public"]["Enums"]["Scale"] | null
11801186
source_local_id: string | null
11811187
space_id: number | null
11821188
text: string | null
11831189
variant: Database["public"]["Enums"]["ContentVariant"] | null
11841190
vector: string | null
1185-
content_type: string | null
11861191
}
11871192
Relationships: [
11881193
{
@@ -1455,6 +1460,7 @@ export type Database = {
14551460
id: number
14561461
last_modified: string
14571462
metadata: Json
1463+
original: boolean | null
14581464
part_of_id: number | null
14591465
scale: Database["public"]["Enums"]["Scale"]
14601466
source_local_id: string | null
@@ -1628,6 +1634,7 @@ export type Database = {
16281634
id: number | null
16291635
last_modified: string | null
16301636
metadata: Json | null
1637+
original: boolean | null
16311638
part_of_id: number | null
16321639
scale: Database["public"]["Enums"]["Scale"] | null
16331640
source_local_id: string | null
@@ -2026,6 +2033,7 @@ export type Database = {
20262033
| null
20272034
variant: Database["public"]["Enums"]["ContentVariant"] | null
20282035
content_type: string | null
2036+
original: boolean | null
20292037
}
20302038
document_local_input: {
20312039
space_id: number | null

‎packages/database/supabase/migrations/20260704120000_update_content_uniqueness.sql‎

Lines changed: 70 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -1,26 +1,28 @@
1+
ALTER TABLE ONLY public."Content"
2+
ADD COLUMN original BOOLEAN DEFAULT true;
3+
4+
ALTER TYPE public."content_local_input" ADD ATTRIBUTE original BOOLEAN;
5+
16
ALTER TABLE ONLY public."FileReference"
27
DROP CONSTRAINT IF EXISTS "FileReference_content_fkey";
38

49
ALTER TABLE public."FileReference"
5-
ADD COLUMN IF NOT EXISTS content_type character varying NOT NULL DEFAULT 'text/obsidian+markdown';
6-
7-
UPDATE public."FileReference" AS fr
8-
SET content_type = ct.content_type
9-
FROM public."Content" AS ct
10-
WHERE fr.space_id = ct.space_id
11-
AND fr.source_local_id = ct.source_local_id
12-
AND fr.variant = ct.variant;
10+
ADD COLUMN original BOOLEAN GENERATED ALWAYS AS (true) STORED;
1311

1412
DROP INDEX IF EXISTS public.content_space_local_id_variant_idx;
1513

1614
CREATE UNIQUE INDEX IF NOT EXISTS content_space_local_id_variant_content_type_idx ON public."Content" USING btree (
1715
space_id, source_local_id, variant, content_type
1816
) NULLS DISTINCT;
1917

18+
CREATE UNIQUE INDEX IF NOT EXISTS content_space_local_id_variant_content_type_originals_idx ON public."Content" USING btree (
19+
space_id, source_local_id, variant, original
20+
);
21+
2022
ALTER TABLE ONLY public."FileReference"
2123
ADD CONSTRAINT "FileReference_content_fkey" FOREIGN KEY (
22-
space_id, source_local_id, variant, content_type
23-
) REFERENCES public."Content" (space_id, source_local_id, variant, content_type) ON DELETE CASCADE;
24+
space_id, source_local_id, variant, original
25+
) REFERENCES public."Content"(space_id, source_local_id, variant, original) ON DELETE CASCADE;
2426

2527
CREATE OR REPLACE FUNCTION public._local_content_to_db_content(data public.content_local_input)
2628
RETURNS public."Content" STABLE
@@ -61,6 +63,7 @@ BEGIN
6163
SELECT id FROM public."Space"
6264
WHERE url = data.space_url INTO content.space_id;
6365
END IF;
66+
content.original := CASE WHEN data.original IS false THEN NULL ELSE true END;
6467
-- now avoid null defaults
6568
IF content.metadata IS NULL then
6669
content.metadata := '{}';
@@ -74,7 +77,7 @@ $$;
7477

7578
COMMENT ON FUNCTION public._local_content_to_db_content IS 'utility function so we have the option to use platform identifiers for content upsert';
7679

77-
CREATE OR REPLACE FUNCTION public.upsert_content(v_space_id bigint, data jsonb, v_creator_id BIGINT, content_as_document boolean DEFAULT TRUE)
80+
CREATE OR REPLACE FUNCTION public.upsert_content(v_space_id bigint, data jsonb, v_creator_id BIGINT, content_as_document boolean DEFAULT true)
7881
RETURNS SETOF BIGINT
7982
SET search_path = ''
8083
LANGUAGE plpgsql
@@ -176,7 +179,8 @@ BEGIN
176179
space_id,
177180
last_modified,
178181
part_of_id,
179-
content_type
182+
content_type,
183+
original
180184
) VALUES (
181185
db_content.document_id,
182186
db_content.source_local_id,
@@ -190,7 +194,8 @@ BEGIN
190194
db_content.space_id,
191195
db_content.last_modified,
192196
db_content.part_of_id,
193-
db_content.content_type
197+
db_content.content_type,
198+
db_content.original
194199
)
195200
ON CONFLICT (space_id, source_local_id, variant, content_type) DO UPDATE SET
196201
document_id = COALESCE(db_content.document_id, EXCLUDED.document_id),
@@ -201,7 +206,8 @@ BEGIN
201206
metadata = COALESCE(db_content.metadata, EXCLUDED.metadata),
202207
scale = COALESCE(db_content.scale, EXCLUDED.scale),
203208
last_modified = COALESCE(db_content.last_modified, EXCLUDED.last_modified),
204-
part_of_id = COALESCE(db_content.part_of_id, EXCLUDED.part_of_id)
209+
part_of_id = COALESCE(db_content.part_of_id, EXCLUDED.part_of_id),
210+
original = db_content.original
205211
RETURNING id INTO STRICT upsert_id;
206212
IF model(embedding_inline(local_content)) IS NOT NULL THEN
207213
PERFORM public.upsert_content_embedding(upsert_id, model(embedding_inline(local_content)), vector(embedding_inline(local_content)));
@@ -213,27 +219,56 @@ $$;
213219

214220
COMMENT ON FUNCTION public.upsert_content IS 'batch content upsert';
215221

216-
CREATE OR REPLACE VIEW public.my_contents_with_embedding_openai_text_embedding_3_small_1536 AS
222+
DROP VIEW public.my_contents_with_embedding_openai_text_embedding_3_small_1536;
223+
224+
CREATE VIEW public.my_contents_with_embedding_openai_text_embedding_3_small_1536 AS
217225
SELECT
218-
ct.id,
219-
ct.document_id,
220-
ct.source_local_id,
221-
ct.variant,
222-
ct.author_id,
223-
ct.creator_id,
224-
ct.created,
225-
ct.text,
226-
ct.metadata,
227-
ct.scale,
228-
ct.space_id,
229-
ct.last_modified,
230-
ct.part_of_id,
231-
emb.model,
232-
emb.vector,
233-
ct.content_type
226+
ct.id,
227+
ct.document_id,
228+
ct.source_local_id,
229+
ct.variant,
230+
ct.author_id,
231+
ct.creator_id,
232+
ct.created,
233+
ct.text,
234+
ct.metadata,
235+
ct.scale,
236+
ct.space_id,
237+
ct.last_modified,
238+
ct.part_of_id,
239+
ct.content_type,
240+
ct.original,
241+
emb.model,
242+
emb.vector
234243
FROM public."Content" AS ct
235-
JOIN public."ContentEmbedding_openai_text_embedding_3_small_1536" AS emb ON (ct.id = emb.target_id)
236-
LEFT OUTER JOIN public.my_accessible_resources () AS ra USING (space_id, source_local_id)
237-
WHERE (ct.space_id = any (public.my_space_ids ('reader'))
238-
OR (ct.space_id = any (public.my_space_ids ('partial')) AND ra.space_id IS NOT NULL))
244+
JOIN public."ContentEmbedding_openai_text_embedding_3_small_1536" AS emb ON (ct.id = emb.target_id)
245+
LEFT OUTER JOIN public.my_accessible_resources() AS ra USING (space_id, source_local_id)
246+
WHERE (
247+
ct.space_id = any(public.my_space_ids('reader'))
248+
OR (ct.space_id = any(public.my_space_ids('partial')) AND ra.space_id IS NOT null)
249+
)
239250
AND NOT emb.obsolete;
251+
252+
CREATE OR REPLACE VIEW public.my_contents AS
253+
SELECT
254+
id,
255+
document_id,
256+
source_local_id,
257+
variant,
258+
author_id,
259+
creator_id,
260+
created,
261+
text,
262+
metadata,
263+
scale,
264+
space_id,
265+
last_modified,
266+
part_of_id,
267+
content_type,
268+
original
269+
FROM public."Content"
270+
LEFT OUTER JOIN public.my_accessible_resources() AS ra USING (space_id, source_local_id)
271+
WHERE (
272+
space_id = any(public.my_space_ids('reader'))
273+
OR (space_id = any(public.my_space_ids('partial')) AND ra.space_id IS NOT null)
274+
);

‎packages/database/supabase/schemas/assets.sql‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -5,17 +5,17 @@ CREATE TABLE IF NOT EXISTS public."FileReference" (
55
filehash character varying NOT NULL, -- or binary?
66
"created" timestamp without time zone NOT NULL,
77
last_modified timestamp without time zone NOT NULL,
8-
content_type character varying NOT NULL DEFAULT 'text/obsidian+markdown',
98
-- not allowed virtual with user types
10-
variant public."ContentVariant" GENERATED ALWAYS AS ('full') STORED
9+
variant public."ContentVariant" GENERATED ALWAYS AS ('full') STORED,
10+
original BOOLEAN GENERATED ALWAYS AS (true) STORED
1111
);
1212
ALTER TABLE ONLY public."FileReference"
1313
ADD CONSTRAINT "FileReference_pkey" PRIMARY KEY (source_local_id, space_id, filepath);
1414

1515
ALTER TABLE ONLY public."FileReference"
1616
ADD CONSTRAINT "FileReference_content_fkey" FOREIGN KEY (
17-
space_id, source_local_id, variant, content_type
18-
) REFERENCES public."Content" (space_id, source_local_id, variant, content_type) ON DELETE CASCADE;
17+
space_id, source_local_id, variant, original
18+
) REFERENCES public."Content"(space_id, source_local_id, variant, original) ON DELETE CASCADE;
1919
-- note the absence of on update ; the generated column forbids cascade, so it will error
2020
-- However, update on those columns should never happen.
2121

0 commit comments

Comments
 (0)