feat(lock): keep previous commit for repos whose pinned content is unchanged

When the --out file already exists, `gloam lock` now compares each
repository's pins against the previous snapshot and carries forward the
recorded commit/describe when every pin is byte-identical (same key set,
paths, and blobs).  Upstream commits that don't touch any pinned file no
longer churn the manifest — or anything regenerated from it — so
scheduled re-snapshot jobs stop producing no-op regeneration commits.

Preservation is per repository, not per file: pins from one repo share a
single resolved commit (cluster resolution), and both the generated
preamble and --version print one 'repo (describe)' line per repo, so
per-file preservation could make pins from the same repo disagree.  Any
content change (changed blob; file added, removed, or renamed) advances
the whole repo to the newly resolved commit.

A missing, unreadable, or schema-incompatible --out file is ignored and
the snapshot is taken fresh, so deleting the file forces a full
re-snapshot at current HEAD.

Signed-off-by: Steven Noonan <steven@uplinklabs.net>

Author: tycho

README.md

@@ -173,6 +173,10 @@ Commands:
         or at upstream HEAD with --fetch. Reuse it later with --lock.
         --out <FILE>
                   Output path for the snapshot [default: manifest.json].
+                  If the file already exists, a repo whose pinned files
+                  all match it keeps its previously recorded
+                  commit/describe; delete the file to force a full
+                  re-snapshot.
 ```
 
 ### Extension selection flags
@@ -228,6 +232,17 @@ This is the mechanism used to generate several loaders from one consistent
 upstream snapshot: take a single `gloam lock` snapshot, then drive each
 generation with `--lock`.
 
+When the `--out` file already exists from a previous snapshot, `gloam lock`
+compares against it per repository: a repo whose pinned files are all
+byte-identical (same key set, paths, and blobs) keeps its previously recorded
+commit/describe instead of advancing to the current HEAD. Upstream commits that
+don't touch any pinned file therefore leave the manifest — and everything
+regenerated from it — byte-identical, which keeps scheduled re-snapshot jobs
+from churning. Any content change (a changed blob, or a file added, removed, or
+renamed) advances the whole repo to the newly resolved commit, so pins from one
+repo never disagree about their snapshot. To force every repo to its current
+commit regardless, delete the existing snapshot file first.
+
 ## Generated output
 
 ### Context struct and dispatch

docs/manifest.md

@@ -249,6 +249,16 @@ any loader — covering every supported upstream source. This gives you a single
 point-in-time lock you can reuse across many different loader generations
 (`gloam lock` produces what `--lock` consumes).
 
+If the `--out` file already exists, the snapshot is **incremental per
+repository**: a repo whose pins are all byte-identical to the previous snapshot
+(same key set, `path_in_repo`s, and `blob`s) keeps its previous
+`commit`/`describe` rather than advancing to the current HEAD, so
+content-neutral upstream commits don't churn the manifest. Any content change
+advances every pin of that repo together — pins from one repo always share one
+recorded commit. A missing, unreadable, or schema-incompatible `--out` file is
+ignored and the snapshot is taken fresh, so deleting the file re-snapshots
+every repo at its current commit.
+
 ---
 
 ## Schema versioning

src/cli.rs

@@ -96,7 +96,11 @@ pub enum Generator {
 
 #[derive(Args, Debug)]
 pub struct LockArgs {
-    /// Output path for the snapshot manifest.
+    /// Output path for the snapshot manifest.  If the file already exists,
+    /// repos whose pinned files are all byte-identical to it keep their
+    /// previously recorded commit/describe, so upstream commits that don't
+    /// touch any pinned file don't churn the manifest.  Delete the file to
+    /// force a full re-snapshot.
     #[arg(long, default_value = "manifest.json")]
     pub out: String,
 }

src/lib.rs

@@ -151,14 +151,29 @@ fn write_lock_snapshot(
         .collect();
     pins.sort_keys();
 
+    let path = std::path::Path::new(&args.out);
+
+    // Carry forward commit/describe from an existing snapshot at --out for
+    // every repo whose pinned content is unchanged.  An upstream commit that
+    // doesn't touch any pinned file then leaves the manifest — and everything
+    // regenerated from it — byte-identical.  Deleting the file forces a full
+    // re-snapshot.
+    if let Some(prev) = read_snapshot(path) {
+        let kept = provenance::manifest::preserve_unchanged_repos(&mut pins, &prev.provenance);
+        if !quiet {
+            for repo in &kept {
+                eprintln!("gloam: {repo}: pinned content unchanged, keeping previous commit");
+            }
+        }
+    }
+
     let manifest = Manifest {
         schema_version: SCHEMA_VERSION,
         gloam: gloam_meta(command_line),
         provenance: pins,
         output: Vec::new(),
     };
 
-    let path = std::path::Path::new(&args.out);
     if let Some(parent) = path.parent().filter(|p| !p.as_os_str().is_empty()) {
         std::fs::create_dir_all(parent)?;
     }
@@ -169,6 +184,14 @@ fn write_lock_snapshot(
     Ok(())
 }
 
+/// Best-effort read of an existing snapshot manifest.  Missing, unreadable, or
+/// schema-mismatched files are ignored — the snapshot is simply taken fresh.
+fn read_snapshot(path: &std::path::Path) -> Option<Manifest> {
+    let text = std::fs::read_to_string(path).ok()?;
+    let m = Manifest::from_json(&text).ok()?;
+    (m.schema_version == SCHEMA_VERSION).then_some(m)
+}
+
 /// Write `.gloam/manifest.json` — the deterministic, pretty-printed bill of
 /// materials for the output tree.  No timestamps: identical inputs + gloam
 /// version produce a byte-identical manifest.

src/provenance/manifest.rs

@@ -94,6 +94,60 @@ impl Manifest {
     }
 }
 
+/// Carry forward `commit`/`describe` from a previous pin set for every
+/// repository whose pinned content is unchanged, returning the preserved repo
+/// slugs.
+///
+/// Granularity is per repository, not per file: pins from one repo share a
+/// single resolved commit (cluster resolution), and both the generated
+/// preamble and `--version` print one `repo (describe)` line per repo, so
+/// per-file preservation could make pins from the same repo disagree.  A repo
+/// is preserved only when its key set matches the previous snapshot exactly
+/// and every pin has the same `repo_url`, `path_in_repo`, and `blob`; any
+/// difference (changed blob, added/removed/renamed file) keeps the whole repo
+/// at the newly resolved commit.
+pub fn preserve_unchanged_repos(
+    pins: &mut IndexMap<String, ProvenancePin>,
+    prev: &IndexMap<String, ProvenancePin>,
+) -> Vec<String> {
+    let mut by_repo: IndexMap<String, Vec<String>> = IndexMap::new();
+    for (key, pin) in pins.iter() {
+        by_repo
+            .entry(pin.repo.clone())
+            .or_default()
+            .push(key.clone());
+    }
+
+    let mut preserved = Vec::new();
+    for (repo, keys) in &by_repo {
+        let prev_count = prev.values().filter(|p| &p.repo == repo).count();
+        if prev_count != keys.len() {
+            continue;
+        }
+        let unchanged = keys.iter().all(|k| {
+            prev.get(k).is_some_and(|old| {
+                let new = &pins[k];
+                old.repo == new.repo
+                    && old.repo_url == new.repo_url
+                    && old.path_in_repo == new.path_in_repo
+                    && old.blob == new.blob
+            })
+        });
+        if !unchanged {
+            continue;
+        }
+        for k in keys {
+            let old = &prev[k];
+            let (commit, describe) = (old.commit.clone(), old.describe.clone());
+            let pin = pins.get_mut(k).unwrap();
+            pin.commit = commit;
+            pin.describe = describe;
+        }
+        preserved.push(repo.clone());
+    }
+    preserved
+}
+
 /// Compute the git blob SHA-1 of `content` — identical to `git hash-object`,
 /// so manifest hashes equal the file's blob hash in any git repo.
 pub fn git_blob_sha1(content: &[u8]) -> String {
@@ -185,4 +239,127 @@ mod tests {
         let json2 = serde_json::to_string(&e2).unwrap();
         assert!(json2.contains("\"verbatim\":true"));
     }
+
+    // ---- preserve_unchanged_repos ----
+
+    fn pin(repo: &str, path: &str, commit: &str, blob: &str) -> ProvenancePin {
+        ProvenancePin {
+            repo: repo.to_string(),
+            repo_url: format!("https://github.com/{repo}"),
+            path_in_repo: path.to_string(),
+            commit: commit.to_string(),
+            describe: commit[..commit.len().min(7)].to_string(),
+            blob: blob.to_string(),
+        }
+    }
+
+    fn pins(entries: &[(&str, ProvenancePin)]) -> IndexMap<String, ProvenancePin> {
+        entries
+            .iter()
+            .map(|(k, p)| (k.to_string(), p.clone()))
+            .collect()
+    }
+
+    #[test]
+    fn preserve_keeps_previous_commit_when_blobs_unchanged() {
+        let prev = pins(&[
+            ("a.xml", pin("org/repo", "x/a.xml", "oldcommit", "blob-a")),
+            ("b.xml", pin("org/repo", "x/b.xml", "oldcommit", "blob-b")),
+        ]);
+        let mut new = pins(&[
+            ("a.xml", pin("org/repo", "x/a.xml", "newcommit", "blob-a")),
+            ("b.xml", pin("org/repo", "x/b.xml", "newcommit", "blob-b")),
+        ]);
+
+        let kept = preserve_unchanged_repos(&mut new, &prev);
+        assert_eq!(kept, vec!["org/repo"]);
+        assert_eq!(new["a.xml"].commit, "oldcommit");
+        assert_eq!(new["a.xml"].describe, "oldcomm");
+        assert_eq!(new["b.xml"].commit, "oldcommit");
+    }
+
+    #[test]
+    fn preserve_advances_whole_repo_when_any_blob_changed() {
+        let prev = pins(&[
+            ("a.xml", pin("org/repo", "x/a.xml", "oldcommit", "blob-a")),
+            ("b.xml", pin("org/repo", "x/b.xml", "oldcommit", "blob-b")),
+        ]);
+        let mut new = pins(&[
+            ("a.xml", pin("org/repo", "x/a.xml", "newcommit", "blob-a")),
+            ("b.xml", pin("org/repo", "x/b.xml", "newcommit", "blob-b2")),
+        ]);
+
+        let kept = preserve_unchanged_repos(&mut new, &prev);
+        assert!(kept.is_empty());
+        // Both pins advance together — no intra-repo commit divergence.
+        assert_eq!(new["a.xml"].commit, "newcommit");
+        assert_eq!(new["b.xml"].commit, "newcommit");
+    }
+
+    #[test]
+    fn preserve_advances_on_added_file() {
+        let prev = pins(&[("a.xml", pin("org/repo", "x/a.xml", "oldcommit", "blob-a"))]);
+        let mut new = pins(&[
+            ("a.xml", pin("org/repo", "x/a.xml", "newcommit", "blob-a")),
+            ("b.xml", pin("org/repo", "x/b.xml", "newcommit", "blob-b")),
+        ]);
+
+        assert!(preserve_unchanged_repos(&mut new, &prev).is_empty());
+        assert_eq!(new["a.xml"].commit, "newcommit");
+    }
+
+    #[test]
+    fn preserve_advances_on_removed_file() {
+        let prev = pins(&[
+            ("a.xml", pin("org/repo", "x/a.xml", "oldcommit", "blob-a")),
+            ("b.xml", pin("org/repo", "x/b.xml", "oldcommit", "blob-b")),
+        ]);
+        let mut new = pins(&[("a.xml", pin("org/repo", "x/a.xml", "newcommit", "blob-a"))]);
+
+        assert!(preserve_unchanged_repos(&mut new, &prev).is_empty());
+        assert_eq!(new["a.xml"].commit, "newcommit");
+    }
+
+    #[test]
+    fn preserve_advances_on_renamed_path() {
+        let prev = pins(&[("a.xml", pin("org/repo", "x/a.xml", "oldcommit", "blob-a"))]);
+        let mut new = pins(&[("a.xml", pin("org/repo", "y/a.xml", "newcommit", "blob-a"))]);
+
+        assert!(preserve_unchanged_repos(&mut new, &prev).is_empty());
+        assert_eq!(new["a.xml"].commit, "newcommit");
+    }
+
+    #[test]
+    fn preserve_ignores_repo_absent_from_previous() {
+        let prev = pins(&[("a.xml", pin("org/repo", "x/a.xml", "oldcommit", "blob-a"))]);
+        let mut new = pins(&[
+            ("a.xml", pin("org/repo", "x/a.xml", "newcommit", "blob-a")),
+            ("c.xml", pin("org/other", "x/c.xml", "newcommit", "blob-c")),
+        ]);
+
+        let kept = preserve_unchanged_repos(&mut new, &prev);
+        assert_eq!(kept, vec!["org/repo"]);
+        assert_eq!(new["a.xml"].commit, "oldcommit");
+        assert_eq!(new["c.xml"].commit, "newcommit");
+    }
+
+    #[test]
+    fn preserve_handles_repos_independently() {
+        let prev = pins(&[
+            ("a.xml", pin("org/stable", "x/a.xml", "oldcommit", "blob-a")),
+            ("b.xml", pin("org/churny", "x/b.xml", "oldcommit", "blob-b")),
+        ]);
+        let mut new = pins(&[
+            ("a.xml", pin("org/stable", "x/a.xml", "newcommit", "blob-a")),
+            (
+                "b.xml",
+                pin("org/churny", "x/b.xml", "newcommit", "blob-b2"),
+            ),
+        ]);
+
+        let kept = preserve_unchanged_repos(&mut new, &prev);
+        assert_eq!(kept, vec!["org/stable"]);
+        assert_eq!(new["a.xml"].commit, "oldcommit");
+        assert_eq!(new["b.xml"].commit, "newcommit");
+    }
 }