Merge pull request #1771 from entireio/fix/1770-git-refs-remote-list-discovery

feat(checkpoint): discover refs-native checkpoints on a second device via List remote enumeration

Author: suhaanthayyil

api/checkpoint/metadata.go

@@ -293,6 +293,14 @@ type CheckpointInfo struct {
 	// Imported is true when this checkpoint was imported from pre-existing
 	// agent history (Kind == "imported"): read-only and commit-less.
 	Imported bool
+
+	// ListedStub is true for names-only remote-discovery List entries that still
+	// need hydration (or have not yet failed a hydration attempt). It is cleared
+	// after a successful hydrate and also after a failed attempt (fail-once), so
+	// callers do not re-fetch forever. A local ref whose root metadata was
+	// unreadable has the same zero SessionID/SessionCount shape but ListedStub
+	// false — do not treat field zero-ness alone as stub-ness.
+	ListedStub bool `json:"-"`
 }
 
 // SessionContent contains the actual content for a session.

cmd/entire/cli/checkpoint/fetching_tree.go

@@ -23,6 +23,21 @@ type BlobFetchFunc func(ctx context.Context, hashes []plumbing.Hash) error
 // package cannot resolve the remote target itself, so the CLI layer injects it.
 type RefFetchFunc func(ctx context.Context, ref plumbing.ReferenceName) error
 
+// RemoteRefListFunc enumerates the per-checkpoint refs present on the configured
+// checkpoint remote (names only, via `ls-remote refs/entire/checkpoints/*` — no
+// object transfer), returning their full ref names. The git-refs store uses it
+// in List to discover checkpoints written on another machine that have no local
+// ref yet; each discovered checkpoint is then hydrated lazily on read via
+// RefFetchFunc. The checkpoint package cannot resolve the remote target itself,
+// so the CLI layer injects it.
+//
+// Scope is stricter than the on-demand read fetch: with no checkpoint_remote
+// configured the lister returns (nil, nil) and List stays local-only. The
+// on-demand fetch (FetchURL) falls back to origin in that case. When a
+// checkpoint_remote is configured the lister queries the resolved checkpoint
+// URL (which can still fall through to origin in FetchURL edge cases).
+type RemoteRefListFunc func(ctx context.Context) ([]plumbing.ReferenceName, error)
+
 // FetchingTree wraps a git tree to automatically fetch missing blobs on demand.
 // After a treeless fetch (--filter=blob:none), tree objects are available locally
 // but blob objects are not. Each File() call checks whether the target blob

cmd/entire/cli/checkpoint/id/id.go

@@ -8,6 +8,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"regexp"
+	"time"
 
 	ulid "github.com/oklog/ulid/v2"
 )
@@ -224,6 +225,24 @@ func (id CheckpointID) IsEmpty() bool {
 	return id == EmptyCheckpointID
 }
 
+// Time returns the creation time encoded in this ID and whether one is
+// available. A ULID embeds a millisecond Unix timestamp in its leading
+// characters, so the time is recoverable from the ID alone — no store read
+// required. This is what lets remote-ref discovery (which learns only ref names
+// via ls-remote) present and sort a not-yet-hydrated checkpoint by its real
+// creation time. Legacy 12-hex IDs carry no timestamp, so this returns
+// (zero, false) for them.
+func (id CheckpointID) Time() (time.Time, bool) {
+	if id.Kind() != KindULID {
+		return time.Time{}, false
+	}
+	u, err := ulid.ParseStrict(string(id))
+	if err != nil {
+		return time.Time{}, false
+	}
+	return ulid.Time(u.Time()), true
+}
+
 // Path returns the sharded path for this checkpoint ID on entire/checkpoints/v1.
 // Uses first 2 characters as shard (256 buckets), remaining as folder name.
 // Example: "a3b2c4d5e6f7" -> "a3/b2c4d5e6f7"

cmd/entire/cli/checkpoint/id/id_test.go

@@ -1,13 +1,53 @@
 package id
 
 import (
+	"bytes"
 	"encoding/json"
 	"testing"
+	"time"
+
+	ulid "github.com/oklog/ulid/v2"
 )
 
 // A representative ULID (Crockford base32, 26 chars) used across tests.
 const sampleULID = "01KVBJCWYA4YW6J5M9GP655HZN"
 
+func TestCheckpointID_Time(t *testing.T) {
+	t.Parallel()
+
+	// A ULID minted from a known instant recovers that instant (millisecond
+	// precision), so remote-ref discovery can sort/display a checkpoint by its
+	// real creation time from the ref name alone — no store read.
+	want := time.UnixMilli(1700000000000).UTC()
+	// Deterministic entropy (zeros); Time() only reads the timestamp prefix.
+	minted := ulid.MustNew(ulid.Timestamp(want), bytes.NewReader(make([]byte, 16)))
+	got, ok := CheckpointID(minted.String()).Time()
+	if !ok {
+		t.Fatalf("Time() ok = false for a valid ULID %q", minted)
+	}
+	if !got.Equal(want) {
+		t.Errorf("Time() = %v, want %v", got, want)
+	}
+
+	// The canonical sample ULID also yields a non-zero time.
+	if ts, ok := CheckpointID(sampleULID).Time(); !ok || ts.IsZero() {
+		t.Errorf("Time() for sample ULID = (%v, %v), want a non-zero time", ts, ok)
+	}
+
+	// A legacy hex ID carries no timestamp.
+	if _, ok := CheckpointID("a1b2c3d4e5f6").Time(); ok {
+		t.Errorf("Time() ok = true for a legacy hex ID; want false")
+	}
+
+	// Non-ID / empty strings report no time.
+	if _, ok := CheckpointID("").Time(); ok {
+		t.Errorf("Time() ok = true for empty ID; want false")
+	}
+	if _, ok := CheckpointID("not-an-id").Time(); ok {
+		t.Errorf("Time() ok = true for a non-ID string; want false")
+	}
+}
+
 func TestGenerateULID(t *testing.T) {
 	t.Parallel()
 	a, err := GenerateULID()

cmd/entire/cli/checkpoint/open.go

@@ -24,6 +24,13 @@ type OpenOptions struct {
 	// reads local-only; ignored by the git-branch backend.
 	RefFetcher RefFetchFunc
 
+	// RemoteRefLister is the CLI-level checkpoint-ref enumerator, used by the
+	// git-refs backend's List to discover checkpoints present on the checkpoint
+	// remote but not yet local (see RemoteRefListFunc). It only fires on a
+	// context marked by WithRemoteListDiscovery. nil (or an unmarked context)
+	// leaves List local-only; ignored by the git-branch backend.
+	RemoteRefLister RemoteRefListFunc
+
 	// Refs overrides the default committed-ref topology. A non-nil value wins,
 	// e.g. attach pins reads to Primary via PrimaryAsRead().
 	Refs *PersistentRefs
@@ -62,7 +69,7 @@ type Stores struct {
 // default git-branch backend with no mirrors, preserving default behavior.
 func Open(ctx context.Context, repo *git.Repository, opts OpenOptions) (*Stores, error) {
 	refs := resolveOpenRefs(ctx, opts)
-	env := OpenEnv{Repo: repo, BlobFetcher: opts.BlobFetcher, RefFetcher: opts.RefFetcher, Refs: refs}
+	env := OpenEnv{Repo: repo, BlobFetcher: opts.BlobFetcher, RefFetcher: opts.RefFetcher, RemoteRefLister: opts.RemoteRefLister, Refs: refs}
 
 	cfg, err := settings.LoadCheckpointsConfig(ctx)
 	if err != nil {

cmd/entire/cli/checkpoint/refs_naming.go

@@ -17,8 +17,8 @@ const CheckpointRefPrefix = "refs/entire/checkpoints/"
 
 // RefName returns the per-checkpoint git ref for a checkpoint ID:
 // refs/entire/checkpoints/<shard>/<id>, where <shard> is id.ShardFor() (the
-// first two chars for legacy hex IDs, the last two for ULIDs). The full ID is
-// always the leaf, so the ref round-trips through ParseRef.
+// last two characters of the ID for both legacy hex and ULID formats). The full
+// ID is always the leaf, so the ref round-trips through ParseRef.
 //
 // It errors on an empty or unrecognized checkpoint ID rather than returning a
 // malformed ref (e.g. "refs/entire/checkpoints//"), so callers at trust