Add lightweight PDF text extraction (#99)

* add lightweight PDF text extraction

* refine PDF parser loading and warnings

Author: ttoyekk1a

next/package.json

@@ -27,6 +27,7 @@
     "pptxgenjs": "^4.0.1",
     "react": "19.2.4",
     "react-dom": "19.2.4",
+    "unpdf": "^1.2.2",
     "xlsx": "^0.18.5",
     "zustand": "^5.0.13"
   },

next/src/components/ai-prompt-bar.tsx

@@ -7,7 +7,7 @@ import { useDraft } from "@/lib/use-draft";
 import { useUploadFile } from "@/lib/use-upload";
 
 const ACCEPT_TYPES =
-  ".md,.txt,.csv,.tsv,.xlsx,.xls,.json,.sql,.yaml,.yml,.png,.jpg,.jpeg,.gif,.webp,.svg,.html,.htm,.xml,.log";
+  ".md,.txt,.pdf,.csv,.tsv,.xlsx,.xls,.json,.sql,.yaml,.yml,.png,.jpg,.jpeg,.gif,.webp,.svg,.html,.htm,.xml,.log";
 
 /**
  * Sticky one-line input pinned to the bottom of the editor's Text tab. The

next/src/components/formats-gallery.tsx

@@ -8,7 +8,7 @@ import { useT } from "@/lib/i18n";
  * Format gallery — content-format snippets the user can one-click load into
  * the editor. Different from `samples-gallery.tsx` (which ships fully
  * pre-rendered HTML samples tied to specific skills): here every card is a
- * tiny example of an *input* shape (.md / .csv / .json / .sql / .yaml /
+ * tiny example of an *input* shape (.md / .pdf / .csv / .json / .sql / .yaml /
  * image, …). After loading, the template picker in the top toolbar decides
  * the *output* shape.
  */
@@ -106,6 +106,31 @@ Third, every release notes file ends with one customer-visible sentence.
 If anything blocks you on this, ping me directly.
 
 — Sam
+`,
+  },
+  {
+    id: "pdf-paper",
+    ext: ".pdf",
+    label: "PDF",
+    icon: "📄",
+    kind: "text",
+    description: "Text-layer PDFs — extracted locally into page sections.",
+    format: "pdf",
+    filename: "agentic-rl-paper.pdf",
+    content: `# PDF: agentic-rl-paper.pdf
+
+Source: PDF
+Pages: 3
+Extraction: embedded text
+
+## Page 1
+Agentic reinforcement learning focuses on agents that can plan, act, observe feedback, and improve policy behavior across multi-step environments.
+
+## Page 2
+LLM reinforcement learning often optimizes model outputs using preference data, reward models, or task-specific feedback over generated responses.
+
+## Page 3
+The practical distinction is workflow scope: agentic RL evaluates behavior across trajectories, while LLM RL usually evaluates individual or batched language outputs.
 `,
   },
   {

next/src/lib/i18n.ts

@@ -528,7 +528,7 @@ const en: Dict = {
   "editor.attach": "Attach",
   "editor.attachTooltip": "Drop a file or click to attach — appended to the editor below",
   "editor.dropTitle": "Drop to attach",
-  "editor.dropHint": ".md .txt .csv .tsv .xlsx .json .sql .yaml .png .jpg — appended to your editor",
+  "editor.dropHint": ".md .txt .pdf .csv .tsv .xlsx .json .sql .yaml .png .jpg — appended to your editor",
   "editor.backup": "⤓ Backup",
   "editor.backupTooltip": "Download the current content as a .md / .txt backup",
   "editor.restoring": "Restoring last content…",
@@ -889,7 +889,7 @@ const zhCN: Dict = {
   "editor.attach": "附加文件",
   "editor.attachTooltip": "拖文件进来或点这里上传 — 自动接到编辑器内容末尾",
   "editor.dropTitle": "松手附加",
-  "editor.dropHint": ".md .txt .csv .tsv .xlsx .json .sql .yaml .png .jpg — 内容会接到编辑器末尾",
+  "editor.dropHint": ".md .txt .pdf .csv .tsv .xlsx .json .sql .yaml .png .jpg — 内容会接到编辑器末尾",
   "editor.backup": "⤓ 备份",
   "editor.backupTooltip": "把当前内容下载为 .md / .txt 备份文件",
   "editor.restoring": "恢复上次内容…",

next/src/lib/parsers/__tests__/auto.test.ts

@@ -0,0 +1,24 @@
+import { describe, expect, it } from "vitest";
+
+import { detectFormat, summarizeForAgent } from "../auto";
+
+describe("PDF summaries", () => {
+  it("detects extracted PDF markdown as pdf content", () => {
+    const input = [
+      "# PDF: paper.pdf",
+      "",
+      "Source: PDF",
+      "Pages: 2",
+      "Extraction: embedded text",
+      "",
+      "## Page 1",
+      "First page",
+    ].join("\n");
+
+    expect(detectFormat(input)).toBe("pdf");
+
+    const summary = summarizeForAgent(input);
+    expect(summary.format).toBe("pdf");
+    expect(summary.preview).toBe("[PDF 文档, 2 页, 86 字符, extraction: embedded text]");
+  });
+});

next/src/lib/parsers/__tests__/file.test.ts

@@ -0,0 +1,93 @@
+import { describe, expect, it, vi } from "vitest";
+
+const unpdf = vi.hoisted(() => ({
+  extractText: vi.fn(),
+  getDocumentProxy: vi.fn(),
+}));
+
+vi.mock("unpdf", () => unpdf);
+
+import { parseFile } from "../file";
+
+describe("parseFile", () => {
+  it("extracts text-layer PDFs into markdown-style page sections", async () => {
+    const proxy = { destroy: vi.fn() };
+    unpdf.getDocumentProxy.mockResolvedValue(proxy);
+    unpdf.extractText.mockResolvedValue({
+      totalPages: 2,
+      text: [
+        "Agentic RL studies agents that learn through environment feedback.",
+        "LLM RL usually optimizes language model behavior from preferences.",
+      ],
+    });
+
+    const file = new File([new Uint8Array([37, 80, 68, 70])], "paper.pdf", {
+      type: "application/pdf",
+    });
+
+    const parsed = await parseFile(file);
+
+    expect(unpdf.getDocumentProxy).toHaveBeenCalledWith(expect.any(Uint8Array));
+    expect(unpdf.extractText).toHaveBeenCalledWith(proxy, { mergePages: false });
+    expect(proxy.destroy).toHaveBeenCalled();
+    expect(parsed).toEqual({
+      filename: "paper.pdf",
+      format: "pdf",
+      text: [
+        "# PDF: paper.pdf",
+        "",
+        "Source: PDF",
+        "Pages: 2",
+        "Extraction: embedded text",
+        "",
+        "## Page 1",
+        "Agentic RL studies agents that learn through environment feedback.",
+        "",
+        "## Page 2",
+        "LLM RL usually optimizes language model behavior from preferences.",
+      ].join("\n"),
+    });
+  });
+
+  it("marks image-heavy PDFs when little text is available", async () => {
+    const proxy = { destroy: vi.fn() };
+    unpdf.getDocumentProxy.mockResolvedValue(proxy);
+    unpdf.extractText.mockResolvedValue({
+      totalPages: 3,
+      text: ["", "  ", "Appendix"],
+    });
+
+    const file = new File([new Uint8Array([37, 80, 68, 70])], "scan.pdf", {
+      type: "application/pdf",
+    });
+
+    const parsed = await parseFile(file);
+
+    expect(parsed.format).toBe("pdf");
+    expect(parsed.text).toContain(
+      "> Note: This PDF appears to be scanned or image-heavy. Text extraction is limited.",
+    );
+    expect(parsed.text).toContain("Pages: 3");
+    expect(parsed.text).toContain("## Page 3\nAppendix");
+  });
+
+  it("does not mark concise text-layer PDFs as limited extraction", async () => {
+    const proxy = { destroy: vi.fn() };
+    unpdf.getDocumentProxy.mockResolvedValue(proxy);
+    unpdf.extractText.mockResolvedValue({
+      totalPages: 1,
+      text: ["Paid. Total: $42."],
+    });
+
+    const file = new File([new Uint8Array([37, 80, 68, 70])], "receipt.pdf", {
+      type: "application/pdf",
+    });
+
+    const parsed = await parseFile(file);
+
+    expect(parsed.format).toBe("pdf");
+    expect(parsed.text).toContain("Extraction: embedded text");
+    expect(parsed.text).not.toContain("Text extraction is limited");
+    expect(parsed.text).toContain("## Page 1\nPaid. Total: $42.");
+  });
+});

next/src/lib/parsers/auto.ts

@@ -6,6 +6,7 @@ export type DetectedFormat =
   | "json"
   | "csv"
   | "tsv"
+  | "pdf"
   | "sql"
   | "yaml"
   | "text";
@@ -14,6 +15,15 @@ export function detectFormat(input: string): DetectedFormat {
   const t = input.trim();
   if (!t) return "text";
 
+  // PDF text extracted by parseFile().
+  if (
+    /^# PDF:\s+\S/m.test(t) &&
+    /^Source:\s+PDF$/m.test(t) &&
+    /^Pages:\s+\d+$/m.test(t)
+  ) {
+    return "pdf";
+  }
+
   // HTML
   if (/^<!DOCTYPE\s+html/i.test(t) || /^<html[\s>]/i.test(t)) return "html";
 
@@ -126,6 +136,12 @@ export function summarizeForAgent(input: string): ParsedSummary {
     case "html":
       preview = `[HTML 文档, ${input.length} 字符]`;
       break;
+    case "pdf": {
+      const pages = input.match(/^Pages:\s+(\d+)$/m)?.[1] ?? "?";
+      const extraction = input.match(/^Extraction:\s+(.+)$/m)?.[1] ?? "embedded text";
+      preview = `[PDF 文档, ${pages} 页, ${input.length} 字符, extraction: ${extraction}]`;
+      break;
+    }
     case "sql":
       preview = `[SQL 查询/脚本]`;
       break;

next/src/lib/parsers/file.ts

@@ -17,15 +17,79 @@ const TEXT_EXTS = new Set([
 ]);
 const IMAGE_EXTS = new Set(["png", "jpg", "jpeg", "gif", "webp", "svg", "bmp"]);
 const SHEET_EXTS = new Set(["xlsx", "xls", "ods", "xlsm"]);
+const PDF_EXTS = new Set(["pdf"]);
+const LIMITED_PDF_TEXT_WARNING =
+  "> Note: This PDF appears to be scanned or image-heavy. Text extraction is limited.";
 
 function ext(name: string): string {
   const i = name.lastIndexOf(".");
   return i > -1 ? name.slice(i + 1).toLowerCase() : "";
 }
 
+function normalizePdfPageText(text: string): string {
+  return text
+    .replace(/\r\n?/g, "\n")
+    .split("\n")
+    .map((line) => line.trimEnd())
+    .join("\n")
+    .trim();
+}
+
+function hasLimitedPdfText(pages: string[]): boolean {
+  if (pages.length === 0) return true;
+  const emptyPages = pages.filter((page) => page.replace(/\s/g, "").length === 0).length;
+  return emptyPages === pages.length || emptyPages / pages.length > 0.5;
+}
+
+export function formatPdfText(
+  filename: string,
+  totalPages: number,
+  pages: string[],
+): string {
+  const pageCount = Math.max(totalPages, pages.length);
+  const normalizedPages = Array.from({ length: pageCount }, (_, i) =>
+    normalizePdfPageText(pages[i] ?? ""),
+  );
+  const isLimitedExtraction = hasLimitedPdfText(normalizedPages);
+
+  const out = [
+    `# PDF: ${filename}`,
+    "",
+    "Source: PDF",
+    `Pages: ${pageCount}`,
+    `Extraction: ${isLimitedExtraction ? "limited embedded text" : "embedded text"}`,
+  ];
+
+  if (isLimitedExtraction) {
+    out.push("", LIMITED_PDF_TEXT_WARNING);
+  }
+
+  for (const [i, pageText] of normalizedPages.entries()) {
+    out.push("", `## Page ${i + 1}`, pageText || "_No extractable text on this page._");
+  }
+
+  return out.join("\n");
+}
+
 export async function parseFile(file: File): Promise<FileParseResult> {
   const e = ext(file.name);
 
+  if (PDF_EXTS.has(e) || file.type === "application/pdf") {
+    const buf = await file.arrayBuffer();
+    const { extractText, getDocumentProxy } = await import("unpdf");
+    const pdf = await getDocumentProxy(new Uint8Array(buf));
+    try {
+      const { totalPages, text } = await extractText(pdf, { mergePages: false });
+      return {
+        filename: file.name,
+        format: "pdf",
+        text: formatPdfText(file.name, totalPages, text),
+      };
+    } finally {
+      await pdf.destroy?.();
+    }
+  }
+
   if (SHEET_EXTS.has(e)) {
     const buf = await file.arrayBuffer();
     const wb = XLSX.read(buf, { type: "array" });