Skip to content

Commit e8e3dd2

Browse files
thedamodthedamod
committed
improve link ingestion and file card layout
1 parent 0102cca commit e8e3dd2

9 files changed

Lines changed: 149 additions & 56 deletions

File tree

‎apps/backend/src/ingestion-worker.ts‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,10 @@ import { getStorageUrl, uploadStorageFile } from "@avenire/storage";
3535
import { serve } from "@hono/node-server";
3636
import { config as loadEnv } from "dotenv";
3737
import { Hono } from "hono";
38-
import { publishWorkspaceStreamEvent } from "./workspace-event-stream";
38+
import {
39+
invalidateWorkspaceReadCaches,
40+
publishWorkspaceStreamEvent,
41+
} from "./workspace-event-stream";
3942

4043
// Prefer backend-local env; keep repo root as fallback.
4144
const here = fileURLToPath(new URL(".", import.meta.url));
@@ -285,6 +288,7 @@ async function persistLinkPreviewMetadata(input: {
285288
});
286289

287290
if (updated) {
291+
await invalidateWorkspaceReadCaches(input.workspaceId);
288292
await publishWorkspaceStreamEvent({
289293
workspaceUuid: input.workspaceId,
290294
type: "files.invalidate",

‎apps/backend/src/workspace-event-stream.ts‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,12 @@ import { createClient, type RedisClientType } from "redis";
33

44
const redisUrl = process.env.REDIS_URL;
55
const DEFAULT_MAX_LEN = 5000;
6+
const ROUTE_CACHE_VERSION_TTL_SECONDS = 60 * 60 * 24;
7+
const WORKSPACE_READ_CACHE_NAMESPACES = [
8+
"workspace:folder",
9+
"workspace:overview",
10+
"workspace:tree",
11+
] as const;
612

713
let publisher: RedisClientType | null = null;
814

@@ -46,6 +52,22 @@ async function getPublisherClient() {
4652
return publisher;
4753
}
4854

55+
export async function invalidateWorkspaceReadCaches(workspaceUuid: string) {
56+
if (!redisUrl) {
57+
return;
58+
}
59+
60+
const client = await getPublisherClient();
61+
const version = Date.now().toString();
62+
await Promise.all(
63+
WORKSPACE_READ_CACHE_NAMESPACES.map((namespace) =>
64+
client.set(`${namespace}:v1:version:${workspaceUuid}`, version, {
65+
EX: ROUTE_CACHE_VERSION_TTL_SECONDS,
66+
})
67+
)
68+
);
69+
}
70+
4971
export async function publishWorkspaceStreamEvent(input: {
5072
workspaceUuid: string;
5173
type: string;

‎apps/web/src/app/api/workspaces/[workspaceUuid]/links/workspace-links-route-post.ts‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
import { scheduleIngestionJob } from "@avenire/ingestion/queue";
22
import { after, NextResponse } from "next/server";
33
import { canStoreBytes } from "@/lib/billing";
4+
import { invalidateWorkspaceReadCaches } from "@/lib/domain-cache";
45
import {
56
createWorkspaceNoteFile,
67
isSharedFilesVirtualFolderId,
@@ -144,6 +145,8 @@ export async function handleWorkspaceLinksPost(input: {
144145
},
145146
});
146147

148+
await invalidateWorkspaceReadCaches(input.workspaceUuid);
149+
147150
scheduleLinkIngestionAfterUpload({
148151
workspaceUuid: input.workspaceUuid,
149152
fileId: file.id,

‎apps/web/src/components/files/explorer.tsx‎

Lines changed: 17 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -1596,7 +1596,7 @@ function createPendingLinkImport(input: {
15961596
faviconUrl: new URL("/favicon.ico", parsed.origin).toString(),
15971597
folderId: input.folderId,
15981598
id: `pending-link:${crypto.randomUUID()}`,
1599-
name: /\.mdx?$/i.test(name) ? name : `${name}.md`,
1599+
name: name.replace(/\.mdx?$/i, "") || "Imported link",
16001600
};
16011601
}
16021602

@@ -1626,14 +1626,14 @@ function PendingLinkImportCard({ item }: { item: PendingLinkImport }) {
16261626
loading
16271627
/>
16281628
</div>
1629-
<div className="mt-2 flex min-w-0 items-center gap-2">
1630-
<LinkSimple
1631-
aria-hidden="true"
1632-
className="size-4 shrink-0 text-muted-foreground"
1633-
/>
1634-
<span className="min-w-0 flex-1 truncate font-medium text-sm">
1629+
<div className="mt-2 grid w-full min-w-0 grid-cols-[auto_minmax(0,1fr)_auto] items-center gap-2">
1630+
<LinkResourceTitleIcon faviconUrl={item.faviconUrl} />
1631+
<span className="truncate font-medium text-sm" title={item.name}>
16351632
{item.name}
16361633
</span>
1634+
<span className="shrink-0 text-muted-foreground text-xs tabular-nums">
1635+
now
1636+
</span>
16371637
</div>
16381638
</CardContent>
16391639
</Card>
@@ -6765,18 +6765,16 @@ export function FileExplorer({
67656765
}
67666766
/>
67676767
</div>
6768-
<div className="flex w-full min-w-0 max-w-full items-center justify-between gap-2">
6769-
<div className="flex min-w-0 flex-1 items-center gap-2">
6770-
<span className="shrink-0 rounded bg-muted px-1.5 py-0.5 font-medium text-[10px] text-muted-foreground">
6771-
folder
6772-
</span>
6773-
<span
6774-
className="min-w-0 flex-1 truncate font-medium text-sm"
6775-
title={folder.name}
6776-
>
6777-
{folder.name}
6778-
</span>
6779-
</div>
6768+
<div className="grid w-full min-w-0 grid-cols-[auto_minmax(0,1fr)_auto] items-center gap-2">
6769+
<span className="shrink-0 rounded bg-muted px-1.5 py-0.5 font-medium text-[10px] text-muted-foreground">
6770+
folder
6771+
</span>
6772+
<span
6773+
className="truncate font-medium text-sm"
6774+
title={folder.name}
6775+
>
6776+
{folder.name}
6777+
</span>
67806778
<span className="shrink-0 text-muted-foreground text-xs tabular-nums">
67816779
{folderUpdatedLabel}
67826780
</span>

‎apps/web/src/components/files/file-card-thumbnail.tsx‎

Lines changed: 11 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -236,23 +236,18 @@ export function FileCard({
236236
<div className="pointer-events-none absolute inset-0 bg-black opacity-0 transition-opacity duration-300 group-hover:opacity-10" />
237237
) : null}
238238
</div>
239-
<div className="flex min-w-0 flex-col gap-2">
240-
<div className="flex w-full min-w-0 max-w-full items-center justify-between gap-2">
241-
<div className="flex min-w-0 flex-1 items-center gap-2">
242-
{titleIcon ? (
243-
titleIcon
244-
) : (
245-
<span className="shrink-0 text-muted-foreground">
246-
{getFileIcon(fileType)}
247-
</span>
248-
)}
249-
<span
250-
className="min-w-0 flex-1 truncate font-medium text-sm"
251-
title={name}
252-
>
253-
{name}
239+
<div className="flex w-full min-w-0 flex-col gap-2">
240+
<div className="grid w-full min-w-0 grid-cols-[auto_minmax(0,1fr)_auto] items-center gap-2">
241+
{titleIcon ? (
242+
titleIcon
243+
) : (
244+
<span className="shrink-0 text-muted-foreground">
245+
{getFileIcon(fileType)}
254246
</span>
255-
</div>
247+
)}
248+
<span className="truncate font-medium text-sm" title={name}>
249+
{name}
250+
</span>
256251
<span className="shrink-0 text-muted-foreground text-xs tabular-nums">
257252
{timeAgo}
258253
</span>

‎packages/ingestion/package.json‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@
2929
"abso-ai": "^0.0.9",
3030
"ai": "^7.0.2",
3131
"bullmq": "^5.70.4",
32+
"defuddle": "^0.19.1",
3233
"drizzle-orm": "0.45.2",
3334
"firecrawl": "^4.30.0",
3435
"officeparser": "7.2.3",

‎packages/ingestion/src/ingestion/link.test.ts‎

Lines changed: 29 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -28,14 +28,32 @@ describe("link ingestion", () => {
2828
scrapeMock.mockReset();
2929
});
3030

31-
it("gets reader markdown and a full-page screenshot in one Firecrawl run", async () => {
31+
it("cleans Firecrawl HTML with Defuddle and keeps the full-page screenshot", async () => {
3232
scrapeMock.mockResolvedValue({
33-
markdown:
34-
"## Introduction\n\nToday we're announcing Project Glasswing.\n\n**Mythos Preview** found critical vulnerabilities.",
33+
html: `<!doctype html>
34+
<html>
35+
<head>
36+
<title>Project Glasswing</title>
37+
<meta name="description" content="Securing critical software for the AI era">
38+
<link rel="icon" href="/favicon.ico">
39+
</head>
40+
<body>
41+
<nav>Products Pricing Company</nav>
42+
<main>
43+
<article>
44+
<h1>Project Glasswing</h1>
45+
<h2>Introduction</h2>
46+
<p>Today we're announcing Project Glasswing.</p>
47+
<p><strong>Mythos Preview</strong> found critical vulnerabilities.</p>
48+
</article>
49+
</main>
50+
<footer>Privacy Terms</footer>
51+
</body>
52+
</html>`,
3553
metadata: {
36-
description: "Securing critical software for the AI era",
37-
favicon: "https://anthropic.com/favicon.ico",
38-
title: "Project Glasswing",
54+
description: "Fallback description",
55+
favicon: "https://anthropic.com/fallback.ico",
56+
title: "Fallback title",
3957
},
4058
screenshot: "https://firecrawl.example/screenshots/glasswing.png",
4159
summary: "A new initiative for securing critical software.",
@@ -46,21 +64,23 @@ describe("link ingestion", () => {
4664
expect(preview.mode).toBe("firecrawl");
4765
expect(preview.provider).toBe("firecrawl");
4866
expect(preview.kind).toBe("article");
67+
expect(preview.title).toBe("Project Glasswing");
4968
expect(preview.imageUrl).toBe(
5069
"https://firecrawl.example/screenshots/glasswing.png"
5170
);
52-
expect(preview.readerMarkdown).toContain("Introduction");
71+
expect(preview.readerMarkdown).toContain("## Introduction");
5372
expect(preview.readerMarkdown).toContain("**Mythos Preview**");
73+
expect(preview.readerMarkdown).not.toContain("Products Pricing Company");
5474
expect(scrapeMock).toHaveBeenCalledTimes(1);
5575
expect(scrapeMock).toHaveBeenCalledWith(
5676
"https://anthropic.com/glasswing",
5777
expect.objectContaining({
5878
formats: expect.arrayContaining([
59-
"markdown",
79+
"html",
6080
"summary",
6181
expect.objectContaining({ fullPage: true, type: "screenshot" }),
6282
]),
63-
onlyMainContent: true,
83+
onlyMainContent: false,
6484
})
6585
);
6686
});

‎packages/ingestion/src/ingestion/link.ts‎

Lines changed: 25 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
import { Defuddle } from "defuddle/node";
12
import { Firecrawl } from "firecrawl";
23
import { z } from "zod";
34
import { config } from "../config";
@@ -35,7 +36,7 @@ export interface LinkPreview {
3536
}
3637

3738
const FirecrawlDocumentSchema = z.object({
38-
markdown: z.string().optional(),
39+
html: z.string().optional(),
3940
metadata: z
4041
.object({
4142
description: z.string().optional(),
@@ -92,7 +93,7 @@ const extractViaFirecrawl = async (url: string) => {
9293
const response = await getFirecrawlClient().scrape(url, {
9394
blockAds: true,
9495
formats: [
95-
"markdown",
96+
"html",
9697
"summary",
9798
{
9899
type: "screenshot",
@@ -101,24 +102,40 @@ const extractViaFirecrawl = async (url: string) => {
101102
viewport: { height: 900, width: 1440 },
102103
},
103104
],
104-
onlyMainContent: true,
105+
onlyMainContent: false,
105106
removeBase64Images: true,
106107
});
107108
const parsed = FirecrawlDocumentSchema.parse(response);
108-
const markdown = normalizeOptionalString(parsed.markdown);
109+
const html = normalizeOptionalString(parsed.html);
110+
if (!html) {
111+
throw new Error(`Firecrawl returned empty HTML for ${url}`);
112+
}
113+
114+
const defuddled = await Defuddle(html, url, {
115+
markdown: true,
116+
useAsync: false,
117+
});
118+
const markdown = normalizeOptionalString(defuddled.content);
109119
if (!markdown) {
110-
throw new Error(`Firecrawl returned empty markdown for ${url}`);
120+
throw new Error(`Defuddle returned empty Markdown for ${url}`);
111121
}
112122

113123
return {
114124
description:
125+
normalizeOptionalString(defuddled.description) ??
115126
normalizeOptionalString(parsed.summary) ??
116127
normalizeOptionalString(parsed.metadata?.description),
117-
favicon: normalizeOptionalString(parsed.metadata?.favicon),
128+
favicon:
129+
normalizeOptionalString(defuddled.favicon) ??
130+
normalizeOptionalString(parsed.metadata?.favicon),
118131
markdown,
119-
pageImageUrl: normalizeOptionalString(parsed.metadata?.ogImage),
132+
pageImageUrl:
133+
normalizeOptionalString(defuddled.image) ??
134+
normalizeOptionalString(parsed.metadata?.ogImage),
120135
screenshotUrl: normalizeOptionalString(parsed.screenshot),
121-
title: normalizeOptionalString(parsed.metadata?.title),
136+
title:
137+
normalizeOptionalString(defuddled.title) ??
138+
normalizeOptionalString(parsed.metadata?.title),
122139
};
123140
};
124141

0 commit comments

Comments
 (0)