night-shift: interest-trend detector over ingestion timestamps

README.md

@@ -1,7 +1,7 @@
 # Personal LLM
 
 [![CI](https://github.com/syzayd/personal-llm/actions/workflows/ci.yml/badge.svg)](https://github.com/syzayd/personal-llm/actions/workflows/ci.yml)
-![Tests](https://img.shields.io/badge/tests-107%20passed%20offline-brightgreen)
+![Tests](https://img.shields.io/badge/tests-118%20passed%20offline-brightgreen)
 ![Python](https://img.shields.io/badge/python-3.12-blue)
 [![License: MIT](https://img.shields.io/badge/license-MIT-green)](LICENSE)
 
@@ -32,6 +32,10 @@ Full design docs live in [`docs/`](docs/): [PRD](docs/PRD.md), [Technical Design
   important-but-forgotten items without being asked - and **verify answers**
   (`ask --verify`) across every available provider, flagging disagreement instead of
   silently picking one.
+- **Detect interest trends** (`trends`) by comparing keyword frequency in your recent
+  ingestion history against the equal-length window right before it - which topics
+  you're writing about more, and which have faded. Pure frequency counting, no model
+  call, fully offline.
 - **Ingest real Gmail/Drive content** (v1.0, `ingest-external`) fetched via Claude Code's
   own already-authenticated MCP connectors - `personal_llm` holds no Google credentials
   of its own; see [ADR 0005](docs/DECISIONS/0005-external-integrations-via-mcp-bridge.md).

src/personal_llm/interfaces/cli.py

@@ -16,6 +16,7 @@
 from personal_llm.integrations import ExternalItem, sync_external_items
 from personal_llm.memory.consolidate import consolidate as run_consolidate
 from personal_llm.memory.ingest import ingest_file
+from personal_llm.memory.interest_trends import detect_interest_trends_in_store
 from personal_llm.memory.retrieve import semantic_search
 from personal_llm.memory.types import MemoryRecord
 from personal_llm.rag.pipeline import ask as rag_ask
@@ -171,6 +172,23 @@ def review(days: int = typer.Option(7, help="How many days back counts as 'recen
             typer.echo(f"  - {item}")
 
 
+@app.command()
+def trends(
+    window_days: float = typer.Option(7.0, help="Size of the 'recent' and 'previous' comparison windows, in days."),
+    k: int = typer.Option(10, help="Number of keywords to show."),
+) -> None:
+    """Interest-trend detector: keywords said much more (or less) in the recent
+    window than the equal-length window right before it."""
+    engine = build_engine()
+    result = detect_interest_trends_in_store(engine.store, window_days=window_days, k=k)
+    if not result:
+        typer.echo("Not enough ingestion history yet to detect a trend.")
+        return
+    for trend in result:
+        arrow = "^" if trend.delta > 0 else "v" if trend.delta < 0 else "="
+        typer.echo(f"[{arrow}{abs(trend.delta)}] {trend.keyword} (recent {trend.recent_count}, previous {trend.previous_count})")
+
+
 @app.command()
 def eval() -> None:
     """Run the prompt-regression eval suite (offline, no engine/API key needed) and

src/personal_llm/memory/interest_trends.py

@@ -0,0 +1,116 @@
+"""Interest-trend detector: keyword-frequency drift over ingestion timestamps.
+
+PROJECT-GENESIS.md sec. 9 Tier 4 item 33 (aliased Tier 9 item 77): "personal-llm:
+interest-trend detector over ingestion timestamps". Every MemoryRecord already
+carries `created_at`; this buckets records into two adjacent, equal-length time
+windows - "recent" and the "previous" one right before it - and compares keyword
+frequency between them. A keyword said much more often in the recent window than
+the previous one is a rising interest; one said much less is a fading one. Pure
+frequency counting, not a topic model: no clustering, no embeddings, no model call
+- same "detect candidates, never interpret" contract as second-brain's near_dup.py
+and contradictions.py.
+
+No personal_llm.router import: this never calls a model, so it stays fully
+offline and fast to test.
+"""
+
+from __future__ import annotations
+
+import re
+from collections import Counter
+from dataclasses import dataclass
+from typing import Protocol, Sequence
+
+from personal_llm.memory.store import MemoryStore
+from personal_llm.memory.time_utils import days_since
+
+_WORD_RE = re.compile(r"[a-z]{3,}")
+
+_STOPWORDS = frozenset(
+    {
+        "the", "and", "for", "are", "was", "were", "with", "that", "this",
+        "have", "has", "had", "not", "but", "you", "your", "from", "they",
+        "will", "would", "could", "should", "about", "into", "over", "then",
+        "than", "just", "like", "also", "been", "being", "did", "does",
+        "doing", "when", "what", "which", "who", "why", "how", "there",
+        "here", "our", "their", "its", "his", "her", "she", "him", "them",
+        "today", "yesterday", "one", "two", "get", "got", "still",
+    }
+)
+
+
+class _TimedNote(Protocol):
+    content: str
+    created_at: str
+
+
+@dataclass(frozen=True)
+class TrendingKeyword:
+    keyword: str
+    recent_count: int
+    previous_count: int
+    delta: int
+
+
+def extract_keywords(content: str) -> Counter:
+    """Lowercased, stopword-filtered, 3+ letter word counts for one note's text."""
+    words = _WORD_RE.findall(content.lower())
+    return Counter(w for w in words if w not in _STOPWORDS)
+
+
+def _window_counts(notes: Sequence[_TimedNote], min_days: float, max_days: float) -> Counter:
+    counts: Counter = Counter()
+    for note in notes:
+        age = days_since(note.created_at)
+        if min_days <= age < max_days:
+            counts.update(extract_keywords(note.content))
+    return counts
+
+
+def detect_interest_trends(
+    notes: Sequence[_TimedNote],
+    *,
+    window_days: float = 7.0,
+    k: int = 10,
+    min_count: int = 2,
+) -> list[TrendingKeyword]:
+    """Rank keywords by how much more (or less) they appear in the recent window
+    than the equal-length window immediately before it.
+
+    "Recent" is notes aged [0, window_days) days; "previous" is [window_days,
+    2 x window_days) days - two adjacent slices of ingestion history, so the
+    comparison is always like-for-like (same span length, no seasonal skew from
+    comparing a week to a whole month). `delta = recent_count - previous_count`;
+    positive deltas are rising interests, negative ones are fading. A keyword is
+    only considered if it appears at least `min_count` times in EITHER window, to
+    keep single-mention noise out of the ranking. Ranked by absolute delta
+    descending (biggest swings first, rising or fading); ties break alphabetically
+    for a deterministic order.
+    """
+    recent = _window_counts(notes, 0.0, window_days)
+    previous = _window_counts(notes, window_days, 2 * window_days)
+
+    keywords = {kw for kw, count in recent.items() if count >= min_count} | {
+        kw for kw, count in previous.items() if count >= min_count
+    }
+
+    trends = [
+        TrendingKeyword(kw, recent.get(kw, 0), previous.get(kw, 0), recent.get(kw, 0) - previous.get(kw, 0))
+        for kw in keywords
+    ]
+    trends.sort(key=lambda t: (-abs(t.delta), t.keyword))
+    return trends[:k]
+
+
+def detect_interest_trends_in_store(
+    store: MemoryStore,
+    *,
+    window_days: float = 7.0,
+    k: int = 10,
+    min_count: int = 2,
+) -> list[TrendingKeyword]:
+    """Convenience wrapper: same as `detect_interest_trends`, reading directly from
+    a MemoryStore's non-archived memories."""
+    return detect_interest_trends(
+        store.list_memories(), window_days=window_days, k=k, min_count=min_count
+    )

tests/test_interest_trends.py

@@ -0,0 +1,110 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from datetime import datetime, timedelta, timezone
+
+from personal_llm.memory.interest_trends import (
+    TrendingKeyword,
+    detect_interest_trends,
+    detect_interest_trends_in_store,
+    extract_keywords,
+)
+from personal_llm.memory.types import MemoryRecord
+
+
+def _iso(days_ago: float) -> str:
+    return (datetime.now(timezone.utc) - timedelta(days=days_ago)).isoformat()
+
+
+@dataclass(frozen=True)
+class FakeNote:
+    content: str
+    created_at: str
+
+
+def test_extract_keywords_lowercases_and_filters_stopwords():
+    counts = extract_keywords("The Gateway auth token rotates and the token is new.")
+    assert counts["token"] == 2
+    assert counts["gateway"] == 1
+    assert counts["auth"] == 1
+    assert "the" not in counts
+    assert "and" not in counts
+    assert "is" not in counts  # shorter than 3 letters is already excluded by regex
+
+
+def test_extract_keywords_drops_words_shorter_than_three_letters():
+    assert extract_keywords("go to it") == {}
+
+
+def test_rising_keyword_ranks_by_positive_delta():
+    notes = [
+        FakeNote("gateway auth token rotation", _iso(1)),
+        FakeNote("gateway auth token again", _iso(2)),
+        FakeNote("gateway token mentioned once", _iso(10)),
+    ]
+    trends = detect_interest_trends(notes, window_days=7, min_count=1)
+    by_kw = {t.keyword: t for t in trends}
+    assert by_kw["gateway"].recent_count == 2
+    assert by_kw["gateway"].previous_count == 1
+    assert by_kw["gateway"].delta == 1
+
+
+def test_fading_keyword_has_negative_delta():
+    notes = [
+        FakeNote("legacy renderer bug", _iso(9)),
+        FakeNote("legacy renderer flicker", _iso(10)),
+        FakeNote("something unrelated", _iso(1)),
+    ]
+    trends = detect_interest_trends(notes, window_days=7, min_count=1)
+    by_kw = {t.keyword: t for t in trends}
+    assert by_kw["legacy"].recent_count == 0
+    assert by_kw["legacy"].previous_count == 2
+    assert by_kw["legacy"].delta == -2
+
+
+def test_min_count_filters_single_mentions():
+    notes = [FakeNote("obscure keyword appears once", _iso(1))]
+    trends = detect_interest_trends(notes, window_days=7, min_count=2)
+    assert trends == []
+
+
+def test_notes_older_than_two_windows_are_excluded():
+    notes = [FakeNote("ancient topic mentioned", _iso(30))]
+    trends = detect_interest_trends(notes, window_days=7, min_count=1)
+    assert trends == []
+
+
+def test_window_boundary_is_recent_exclusive_previous_inclusive():
+    # Exactly at window_days=7.0 days old must land in "previous", not "recent".
+    notes = [FakeNote("boundary keyword case", _iso(7.0))]
+    trends = detect_interest_trends(notes, window_days=7, min_count=1)
+    by_kw = {t.keyword: t for t in trends}
+    assert by_kw["boundary"].recent_count == 0
+    assert by_kw["boundary"].previous_count == 1
+
+
+def test_k_limits_result_count():
+    notes = [FakeNote(f"uniqueword{i} appears twice", _iso(1)) for i in range(5)] * 2
+    trends = detect_interest_trends(notes, window_days=7, min_count=1, k=2)
+    assert len(trends) == 2
+
+
+def test_ties_broken_alphabetically():
+    notes = [FakeNote("zeta alpha", _iso(1)), FakeNote("zeta alpha", _iso(2))]
+    trends = detect_interest_trends(notes, window_days=7, min_count=1)
+    assert [t.keyword for t in trends] == ["alpha", "zeta"]
+
+
+def test_empty_input_returns_empty():
+    assert detect_interest_trends([]) == []
+
+
+def test_detect_interest_trends_in_store_reads_real_memory_store(store):
+    store.add_memory(MemoryRecord(content="gateway auth token", created_at=_iso(1)))
+    store.add_memory(MemoryRecord(content="gateway auth token", created_at=_iso(2)))
+    store.add_memory(MemoryRecord(content="unrelated old note", created_at=_iso(20)))
+
+    trends = detect_interest_trends_in_store(store, window_days=7, min_count=1)
+    keywords = {t.keyword for t in trends}
+    assert "gateway" in keywords
+    assert "unrelated" not in keywords  # older than the 2x window, excluded