feat: --since recency filter (drop sources older than a cutoff) (#62)

Builds on v0.14's published-date extraction. --since=<date|duration>
(env DEEPDIVE_SINCE) drops a fetched web source whose detected publication
date precedes the cutoff — absolute date (2024, 2024-06-15) or duration
meaning "that long ago" (30d, 2w). Dateless sources are kept (no penalty
for missing metadata); --include / continue sources are exempt. Emits a
new `stale` fetch.skipped reason. A supplied-but-unparseable value is a
hard error (exit 2). New pure resolveSince (exported); persistable as
`since` in the config file. 5 new tests (547 total green).

Author: askalf

CHANGELOG.md

@@ -6,6 +6,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
 
 ## [Unreleased]
 
+### Added — `--since` recency filter
+
+- **`--since=<date|duration>`** (env `DEEPDIVE_SINCE`) — drop fetched sources published before a cutoff, building on v0.14's published-date extraction. Accepts an absolute date (`2024`, `2024-06`, `2024-06-15`) or a relative duration meaning "that long ago" (`30d`, `12h`, `2w`). A web source whose detected publication date precedes the cutoff is skipped (new `stale` `fetch.skipped` reason); sources with no detectable date are kept (no penalty for missing metadata). Doesn't apply to `--include` / `continue` sources. New pure `resolveSince` (exported); persistable as `since` in the config file. A supplied-but-unparseable `--since` is a hard error (exit 2), not a silent no-op.
+
 ## [0.14.0] - 2026-06-09
 
 ### Added — config file, named profiles, shell completion

README.md

@@ -188,6 +188,12 @@ Two signals that help you read a report at a glance.
 
 **Published dates.** When deepdive fetches a page, it tries to recover the page's publication date from the rendered HTML — JSON-LD `datePublished`, `<meta property="article:published_time">` and friends, or a `<time datetime>` element. When it finds one, the source row shows it (`fetched 2026-05-07 · published 2024-03-15`), the HTML export shows it, the JSON carries it as `publishedAt`, **and** the synthesizer sees it — so when sources disagree it can prefer the more recent one and flag claims that come from an older page. Pages that don't expose a date (many SPAs) simply don't get the annotation; nothing breaks.
 
+**Recency filter.** Pass `--since` (or `DEEPDIVE_SINCE`) to drop stale sources outright — an absolute date (`--since=2024`, `--since=2024-06-15`) or a duration meaning "that long ago" (`--since=30d`, `--since=2w`). A fetched page whose detected publication date is before the cutoff is skipped (`stale` in `--verbose`); pages with no detectable date are kept, so a missing-metadata page is never penalized. Useful for fast-moving topics where a 2019 blog post is worse than no answer.
+
+```bash
+deepdive "best way to deploy a node app in 2026" --since=365d --deep
+```
+
 **Confidence.** After each run, alongside the cost line, deepdive prints a one-line coverage read:
 
 ```

src/agent.ts

@@ -106,6 +106,11 @@ export interface AgentConfig {
   // v0.14.0 — when true, the synthesizer leads with a one-paragraph TL;DR.
   // Opt-in (CLI --tldr); default off keeps output identical to v0.13.
   tldr?: boolean;
+  // v0.15.0 — recency filter. When set, a fetched web source whose extracted
+  // publication date is older than this epoch-ms cutoff is dropped (emits a
+  // `stale` skip). Sources with no extractable date are kept (not penalized
+  // for missing metadata). Does not apply to include[]/preKept sources.
+  sinceMs?: number;
   onEvent?: (event: AgentEvent) => void;
   // Fires for each SSE token emitted by the synthesizer. When set, the agent
   // uses the streaming LLM path for synthesize() calls. CLI callers enable
@@ -137,7 +142,8 @@ export type AgentEvent =
         | "pdf-no-extractor"
         | "pdf-extract-error"
         | "domain-deny"
-        | "domain-not-allowed";
+        | "domain-not-allowed"
+        | "stale";
     }
   | { type: "include.done"; ingested: number; skipped: number }
   | { type: "synthesize.start"; sourceCount: number; round: number }
@@ -443,6 +449,21 @@ export async function runAgent(
         // simply yield undefined — additive, never blocks keeping the source.
         const publishedAt = isPdf ? undefined : extractPublishedDate(f.page.html);
 
+        // Recency filter (--since): drop a source dated before the cutoff.
+        // Dateless sources pass (we don't penalize missing metadata).
+        if (
+          config.sinceMs !== undefined &&
+          publishedAt !== undefined &&
+          publishedAt < config.sinceMs
+        ) {
+          emit(config, {
+            type: "fetch.skipped",
+            url: f.page.finalUrl || f.page.url,
+            reason: "stale",
+          });
+          continue;
+        }
+
         keptSources.push({
           id: keptSources.length + 1,
           url: f.page.finalUrl || f.page.url,

src/cli.ts

@@ -132,6 +132,10 @@ Flags:
                                 exclusively (e.g. github.com,docs.anthropic.com).
   --deny-domain=<list>          Comma-separated hostname suffixes to drop
                                 (e.g. pinterest.com,quora.com).
+  --since=<date|duration>       Drop sources published before this — an absolute
+                                date (2024, 2024-06, 2024-06-15) or a duration
+                                (30d, 12h, 2w = that long ago). Sources with no
+                                detectable date are kept. Env: DEEPDIVE_SINCE.
   --api-format=<anthropic|openai>
                                 Wire format for the LLM endpoint. Default:
                                 auto-detected from --base-url (api.openai.com,
@@ -163,7 +167,7 @@ Environment:
   DEEPDIVE_NO_VERIFY_CITES, DEEPDIVE_STRICT_CITES, DEEPDIVE_CITE_MIN_RECALL,
   DEEPDIVE_NO_COST, DEEPDIVE_PRICE_INPUT_PER_MTOK, DEEPDIVE_PRICE_OUTPUT_PER_MTOK,
   DEEPDIVE_INCLUDE, DEEPDIVE_PDF_MAX_PAGES,
-  DEEPDIVE_ALLOW_DOMAIN, DEEPDIVE_DENY_DOMAIN, DEEPDIVE_API_FORMAT,
+  DEEPDIVE_ALLOW_DOMAIN, DEEPDIVE_DENY_DOMAIN, DEEPDIVE_SINCE, DEEPDIVE_API_FORMAT,
   DEEPDIVE_NO_SESSIONS, DEEPDIVE_SESSIONS_DIR, DEEPDIVE_CONFIG
 
 Config file:
@@ -373,6 +377,9 @@ export function parseArgs(argv: string[]): ParsedArgs {
         case "profile":
           flags.profile = value;
           break;
+        case "since":
+          flags.since = value;
+          break;
         case "format":
           flags.format = value.toLowerCase();
           break;
@@ -676,6 +683,15 @@ interface RunResearchOptions {
 
 async function runResearch(opts: RunResearchOptions): Promise<number> {
   const { question, parsed, config, preKept, parentId } = opts;
+  // A --since value that was supplied but didn't parse is a user error — fail
+  // loud rather than silently running with no recency filter.
+  if (config.sinceRaw && config.sinceMs === undefined) {
+    process.stderr.write(
+      `deepdive: --since must be a date (2024, 2024-06, 2024-06-15) or a duration ` +
+        `(30d, 12h, 2w); got: ${config.sinceRaw}\n`,
+    );
+    return 2;
+  }
   const search = await resolveSearchAdapter(config.searchAdapter, process.env);
   const cache = config.cache.enabled
     ? createCache({ dir: config.cache.dir, ttlMs: config.cache.ttlMs })
@@ -725,6 +741,7 @@ async function runResearch(opts: RunResearchOptions): Promise<number> {
         include: config.include,
         domainFilter: config.domainFilter,
         tldr: config.tldr,
+        sinceMs: config.sinceMs,
         env: process.env,
         onEvent: (e) => {
           if (config.verbose) process.stderr.write(renderEvent(e) + "\n");

src/config-file.ts

@@ -67,6 +67,7 @@ const KEY_MAP: Record<string, { env: string; kind: Kind }> = {
   sessionsDir: { env: "DEEPDIVE_SESSIONS_DIR", kind: "string" },
   allowDomain: { env: "DEEPDIVE_ALLOW_DOMAIN", kind: "list" },
   denyDomain: { env: "DEEPDIVE_DENY_DOMAIN", kind: "list" },
+  since: { env: "DEEPDIVE_SINCE", kind: "string" },
   include: { env: "DEEPDIVE_INCLUDE", kind: "list" },
   tldr: { env: "DEEPDIVE_TLDR", kind: "bool" },
   strictCites: { env: "DEEPDIVE_STRICT_CITES", kind: "bool" },

src/config.ts

@@ -9,6 +9,7 @@ import { parseDomainList, type DomainFilter } from "./domain-filter.js";
 import { detectApiFormat, type ApiFormat } from "./llm-format.js";
 import { defaultSessionsDir } from "./sessions.js";
 import { parseMaxCost } from "./budget.js";
+import { resolveSince } from "./dates.js";
 
 export interface RuntimeConfig {
   llm: LLMConfig;
@@ -41,6 +42,12 @@ export interface RuntimeConfig {
   maxCostUsd?: number;
   // v0.14.0 — lead the answer with a one-paragraph TL;DR. Opt-in.
   tldr: boolean;
+  // v0.15.0 — recency cutoff (epoch ms). Sources dated before this are
+  // dropped. Undefined = no recency filter. Set via --since / DEEPDIVE_SINCE.
+  sinceMs?: number;
+  // The raw --since value (if any), so the CLI can distinguish "not set" from
+  // "set but unparseable" and error on the latter.
+  sinceRaw?: string;
 }
 
 export interface CLIFlags {
@@ -79,6 +86,7 @@ export interface CLIFlags {
   noStream?: boolean;
   verbose?: boolean;
   tldr?: boolean;
+  since?: string;
   // v0.11.0 — already-parsed budget cap in USD. CLI parser converts
   // "--max-cost=$0.50" / "$5" / "0.25" into a number; resolveConfig
   // accepts the parsed value (parseMaxCost lives in budget.ts and the
@@ -251,6 +259,8 @@ export function resolveConfig(
   const streamEnabled = !streamOptOut && !jsonOutput;
   const verbose = flags.verbose ?? env.DEEPDIVE_VERBOSE === "1";
   const tldr = flags.tldr ?? env.DEEPDIVE_TLDR === "1";
+  const sinceRaw = flags.since ?? env.DEEPDIVE_SINCE;
+  const sinceMs = sinceRaw ? resolveSince(sinceRaw) : undefined;
 
   // v0.11.0 — budget cap. Flag takes a pre-parsed number from cli.ts
   // (which uses parseMaxCost on the raw string). Env var is parsed here.
@@ -298,6 +308,8 @@ export function resolveConfig(
     verbose,
     maxCostUsd,
     tldr,
+    sinceMs,
+    sinceRaw,
   };
 }
 

src/dates.ts

@@ -180,6 +180,29 @@ function walk(node: unknown, out: { published?: string; modified?: string }): vo
   }
 }
 
+// Exported for unit tests. Resolve a `--since` value to an absolute epoch-ms
+// cutoff. Accepts a relative duration with an explicit unit (`30d`, `2w`,
+// `12h`) — interpreted as "now minus that" — or an absolute date (`2024`,
+// `2024-06`, `2024-06-15`). A bare 4-digit number is treated as a YEAR, not a
+// day count, since `--since=2024` overwhelmingly means "since 2024". Returns
+// undefined for unparseable input.
+export function resolveSince(value: string, now: number = Date.now()): number | undefined {
+  const v = value.trim().toLowerCase();
+  const dur = /^(\d+)\s*(w|d|h|m|s)$/.exec(v);
+  if (dur) {
+    const mult: Record<string, number> = {
+      s: 1000,
+      m: 60_000,
+      h: 3_600_000,
+      d: 86_400_000,
+      w: 604_800_000,
+    };
+    return now - Number(dur[1]) * mult[dur[2]];
+  }
+  if (/^\d{4}$/.test(v)) return toEpoch(`${v}-01-01`, now);
+  return toEpoch(v, now);
+}
+
 // Exported for unit tests. Parse a date string to epoch ms, rejecting values
 // outside [1990-01-01, now + 2 days]. Bare YYYY / YYYY-MM are accepted.
 export function toEpoch(s: string, now: number = Date.now()): number | undefined {

src/index.ts

@@ -30,6 +30,7 @@ export {
   metaTags,
   jsonLdDates,
   toEpoch,
+  resolveSince,
 } from "./dates.js";
 export {
   assessConfidence,

test/agent-loop.test.mjs

@@ -1013,6 +1013,53 @@ test("agent: maxCostUsd aborts after the call that crosses the cap", async () =>
   }
 });
 
+test("agent: --since drops sources dated before the cutoff, keeps fresh + dateless", async () => {
+  const planJson = '{"queries":["q1"]}';
+  const synthText = "Answer [1][2].";
+  const { server } = makeLLMServer([planJson, synthText]);
+  const baseUrl = await startServer(server);
+
+  const meta = (d) =>
+    `<html><head><meta property="article:published_time" content="${d}"></head><body>${LOREM}</body></html>`;
+  const search = mockSearch({
+    q1: [
+      { url: "https://ex.com/old", title: "Old", snippet: "" },
+      { url: "https://ex.com/fresh", title: "Fresh", snippet: "" },
+      { url: "https://ex.com/nodate", title: "NoDate", snippet: "" },
+    ],
+  });
+  const pages = {
+    "https://ex.com/old": { text: LOREM, title: "Old", html: meta("2020-03-01") },
+    "https://ex.com/fresh": { text: LOREM, title: "Fresh", html: meta("2026-05-01") },
+    "https://ex.com/nodate": { text: LOREM, title: "NoDate" }, // html has no date
+  };
+  const skipped = [];
+  try {
+    const result = await runAgent("q", {
+      llm: { baseUrl, apiKey: "t", model: "test", maxTokens: 512 },
+      search,
+      browser: { headless: true, timeoutMs: 5000, maxBytes: 1_000_000 },
+      resultsPerQuery: 5,
+      maxSources: 12,
+      maxWordsPerSource: 2000,
+      deepRounds: 0,
+      concurrency: 2,
+      sinceMs: Date.UTC(2024, 0, 1),
+      browserFactory: mockBrowserFactory(pages),
+      onEvent: (e) => {
+        if (e.type === "fetch.skipped") skipped.push(e);
+      },
+    });
+    const urls = result.sources.map((s) => s.url).sort();
+    assert.deepEqual(urls, ["https://ex.com/fresh", "https://ex.com/nodate"]);
+    assert.equal(skipped.length, 1);
+    assert.equal(skipped[0].reason, "stale");
+    assert.equal(skipped[0].url, "https://ex.com/old");
+  } finally {
+    await stopServer(server);
+  }
+});
+
 test("agent: undefined maxCostUsd means no cap, run completes", async () => {
   // Same setup as above but no cap — should finish.
   const planJson =

test/dates.test.mjs

@@ -7,6 +7,7 @@ import {
   metaTags,
   jsonLdDates,
   toEpoch,
+  resolveSince,
 } from "../dist/dates.js";
 
 const NOW = Date.UTC(2026, 5, 1);
@@ -93,3 +94,24 @@ test("extractPublishedDate: a future/garbage meta date is rejected, not returned
   const html = `<meta name="date" content="3999-01-01">`;
   assert.equal(extractPublishedDate(html, NOW), undefined);
 });
+
+// ── resolveSince ─────────────────────────────────────────────────────────────
+
+test("resolveSince: a duration with a unit is relative to now", () => {
+  assert.equal(resolveSince("30d", NOW), NOW - 30 * 86_400_000);
+  assert.equal(resolveSince("2w", NOW), NOW - 14 * 86_400_000);
+  assert.equal(resolveSince("12h", NOW), NOW - 12 * 3_600_000);
+});
+
+test("resolveSince: a bare 4-digit value is a YEAR, not a day count", () => {
+  assert.equal(iso(resolveSince("2024", NOW)), "2024-01-01");
+});
+
+test("resolveSince: absolute dates parse", () => {
+  assert.equal(iso(resolveSince("2024-06-15", NOW)), "2024-06-15");
+});
+
+test("resolveSince: junk returns undefined", () => {
+  assert.equal(resolveSince("whenever", NOW), undefined);
+  assert.equal(resolveSince("3999", NOW), undefined); // out of range year
+});