Merge remote-tracking branch 'origin/main' into alisha/ent-1047-semantic-search-cli-multi-repo

Author: alishakawaguchi

.github/dependabot.yml

@@ -30,3 +30,11 @@ updates:
     schedule:
       interval: "daily"
     open-pull-requests-limit: 5
+    groups:
+      # CodeQL subactions (init, analyze, ...) share one monorepo release and
+      # must move together — a version mismatch fails the CodeQL job with
+      # "Loaded a configuration file for version X, but running version Y".
+      # Grouping bundles every github/codeql-action/* bump into one PR.
+      codeql-action:
+        patterns:
+          - "github/codeql-action/*"

.github/workflows/codeql-actions.yml

@@ -24,13 +24,13 @@ jobs:
           ref: ${{ github.event.pull_request.head.sha }}
 
       - name: Initialize CodeQL
-        uses: github/codeql-action/init@99df26d4f13ea111d4ec1a7dddef6063f76b97e9 # v4.37.0
+        uses: github/codeql-action/init@e4fba868fa4b1b91e1fdab776edc8cfbe6e9fb81 # v4.37.3
         with:
           languages: actions
           queries: security-extended,security-and-quality
 
       - name: Perform CodeQL analysis
-        uses: github/codeql-action/analyze@99df26d4f13ea111d4ec1a7dddef6063f76b97e9 # v4.37.0
+        uses: github/codeql-action/analyze@e4fba868fa4b1b91e1fdab776edc8cfbe6e9fb81 # v4.37.3
         with:
           category: /language:actions
           # Fork PRs receive a read-only GITHUB_TOKEN, so SARIF upload to the

api/checkpoint/metadata.go

@@ -37,6 +37,16 @@ type WriteOptions struct {
 	// Branch is the branch name where the checkpoint was created (empty if detached HEAD)
 	Branch string
 
+	// CommitSHA links this checkpoint to an existing commit without a trailer.
+	// It is an anchor — "imported at this point in time" — not attribution.
+	// Currently set only by `entire import`: imported history has no
+	// Entire-Checkpoint trailer (we never rewrite existing commits), so import
+	// stamps the resolved anchor commit here (the default branch head when
+	// resolvable; see resolveImportLinkCommitSHA for the fallback order).
+	// Empty for all other writers. This comment is the canonical description;
+	// Metadata.CommitSHA and CheckpointSummary.CommitSHA point back here.
+	CommitSHA string
+
 	// Transcript is the session transcript content (full.jsonl).
 	// Must be pre-redacted (via redact.JSONLBytes or redact.AlreadyRedacted for trusted sources).
 	Transcript redact.RedactedBytes
@@ -293,6 +303,14 @@ type CheckpointInfo struct {
 	// Imported is true when this checkpoint was imported from pre-existing
 	// agent history (Kind == "imported"): read-only and commit-less.
 	Imported bool
+
+	// ListedStub is true for names-only remote-discovery List entries that still
+	// need hydration (or have not yet failed a hydration attempt). It is cleared
+	// after a successful hydrate and also after a failed attempt (fail-once), so
+	// callers do not re-fetch forever. A local ref whose root metadata was
+	// unreadable has the same zero SessionID/SessionCount shape but ListedStub
+	// false — do not treat field zero-ness alone as stub-ness.
+	ListedStub bool `json:"-"`
 }
 
 // SessionContent contains the actual content for a session.
@@ -316,13 +334,17 @@ type SessionContent struct {
 
 // Metadata contains the metadata stored in metadata.json for each checkpoint.
 type Metadata struct {
-	CLIVersion       string          `json:"cli_version,omitempty"`
-	CheckpointID     id.CheckpointID `json:"checkpoint_id"`
-	SessionID        string          `json:"session_id"`
-	Strategy         string          `json:"strategy"`
-	CreatedAt        time.Time       `json:"created_at"`
-	Branch           string          `json:"branch,omitempty"` // Branch where checkpoint was created (empty if detached HEAD)
-	CheckpointsCount int             `json:"checkpoints_count"`
+	CLIVersion   string          `json:"cli_version,omitempty"`
+	CheckpointID id.CheckpointID `json:"checkpoint_id"`
+	SessionID    string          `json:"session_id"`
+	Strategy     string          `json:"strategy"`
+	CreatedAt    time.Time       `json:"created_at"`
+	Branch       string          `json:"branch,omitempty"` // Branch where checkpoint was created (empty if detached HEAD)
+	// CommitSHA anchors an imported checkpoint to an existing commit; empty for
+	// non-imported checkpoints, which link via the Entire-Checkpoint trailer.
+	// See WriteOptions.CommitSHA for the full semantics.
+	CommitSHA        string `json:"commit_sha,omitempty"`
+	CheckpointsCount int    `json:"checkpoints_count"`
 	// SaveStepCount is the number of SaveStep-recorded steps for this session.
 	// Honest "real checkpoint work happened" signal (0 = commit-only/fallback
 	// session), kept separate from the displayed CheckpointsCount prompt count.
@@ -474,10 +496,12 @@ type SessionFilePaths struct {
 //
 //nolint:revive // Named CheckpointSummary to avoid conflict with existing Summary struct
 type CheckpointSummary struct {
-	CLIVersion          string             `json:"cli_version,omitempty"`
-	CheckpointID        id.CheckpointID    `json:"checkpoint_id"`
-	Strategy            string             `json:"strategy"`
-	Branch              string             `json:"branch,omitempty"`
+	CLIVersion   string          `json:"cli_version,omitempty"`
+	CheckpointID id.CheckpointID `json:"checkpoint_id"`
+	Strategy     string          `json:"strategy"`
+	Branch       string          `json:"branch,omitempty"`
+	// CommitSHA: import-only anchor; see WriteOptions.CommitSHA.
+	CommitSHA           string             `json:"commit_sha,omitempty"`
 	CheckpointsCount    int                `json:"checkpoints_count"`
 	FilesTouched        []string           `json:"files_touched"`
 	Sessions            []SessionFilePaths `json:"sessions"`

cmd/entire/cli/agent/agent.go

@@ -236,6 +236,49 @@ type TextGenerator interface {
 	GenerateText(ctx context.Context, prompt string, model string) (string, error)
 }
 
+// ProgressPhase identifies a coarse stage in streaming text generation.
+type ProgressPhase string
+
+const (
+	// PhaseConnecting is emitted once when the CLI signals it is making the upstream request.
+	PhaseConnecting ProgressPhase = "connecting"
+	// PhaseFirstToken is emitted once when the upstream responds with the first event,
+	// carrying TTFT and input/cache token counts.
+	PhaseFirstToken ProgressPhase = "first-token"
+	// PhaseGenerating is emitted repeatedly as text or thinking deltas arrive.
+	// OutputTokens carries a running estimate based on delta sizes.
+	PhaseGenerating ProgressPhase = "generating"
+	// PhaseDone is emitted once when the final result event is received without error.
+	PhaseDone ProgressPhase = "done"
+)
+
+// GenerationProgress reports a snapshot of streaming text generation progress.
+// Fields not relevant to the current Phase may be zero-valued.
+type GenerationProgress struct {
+	Phase             ProgressPhase
+	OutputTokens      int // running estimate during PhaseGenerating; final at PhaseDone
+	InputTokens       int // populated at PhaseFirstToken
+	CachedInputTokens int // populated at PhaseFirstToken
+	TTFTms            int // time-to-first-token, populated at PhaseFirstToken
+	DurationMs        int // populated at PhaseDone (final result event)
+}
+
+// ProgressFn receives streaming progress updates. It must not block — invoke it
+// from the same goroutine that reads the stream and keep handlers fast.
+type ProgressFn func(GenerationProgress)
+
+// StreamingTextGenerator is an optional interface for text generators whose
+// underlying CLI exposes a streaming output mode. Callers can use AsStreamingTextGenerator
+// to detect support and fall back to plain GenerateText when unavailable.
+type StreamingTextGenerator interface {
+	Agent
+
+	// GenerateTextStreaming invokes the agent's streaming text generation and
+	// calls progress for each phase update. progress may be nil to suppress
+	// reporting. The returned string is the final response text.
+	GenerateTextStreaming(ctx context.Context, prompt, model string, progress ProgressFn) (string, error)
+}
+
 // CompactedTranscript contains the result of transcript compaction into Entire
 // Transcript Format. Assets are accepted in the protocol shape for forward
 // compatibility but may not yet be persisted by all call sites.

cmd/entire/cli/agent/capabilities.go

@@ -27,6 +27,7 @@ type DeclaredCaps struct {
 	TokenCalculator        bool `json:"token_calculator"`
 	CompactTranscript      bool `json:"compact_transcript"`
 	TextGenerator          bool `json:"text_generator"`
+	StreamingTextGenerator bool `json:"streaming_text_generator"`
 	HookResponseWriter     bool `json:"hook_response_writer"`
 	SubagentAwareExtractor bool `json:"subagent_aware_extractor"`
 }
@@ -100,6 +101,22 @@ func AsTextGenerator(ag Agent) (TextGenerator, bool) {
 	return declaredCapability[TextGenerator](ag, func(c DeclaredCaps) bool { return c.TextGenerator })
 }
 
+// AsStreamingTextGenerator returns the agent as StreamingTextGenerator if it both
+// implements the interface and (for CapabilityDeclarer agents) has declared the capability.
+func AsStreamingTextGenerator(ag Agent) (StreamingTextGenerator, bool) {
+	if ag == nil {
+		return nil, false
+	}
+	stg, ok := ag.(StreamingTextGenerator)
+	if !ok {
+		return nil, false
+	}
+	if cd, ok := ag.(CapabilityDeclarer); ok {
+		return stg, cd.DeclaredCapabilities().StreamingTextGenerator
+	}
+	return stg, true
+}
+
 // AsTranscriptCompactor returns the agent as TranscriptCompactor if it both
 // implements the interface and (for CapabilityDeclarer agents) has declared the capability.
 func AsTranscriptCompactor(ag Agent) (TranscriptCompactor, bool) {

cmd/entire/cli/agent/capabilities_test.go

@@ -86,6 +86,11 @@ func (m *mockFullAgent) GenerateText(context.Context, string, string) (string, e
 	return "", nil
 }
 
+// StreamingTextGenerator
+func (m *mockFullAgent) GenerateTextStreaming(context.Context, string, string, ProgressFn) (string, error) {
+	return "", nil
+}
+
 // TranscriptCompactor
 func (m *mockFullAgent) CompactTranscript(context.Context, string) (*CompactedTranscript, error) {
 	return &CompactedTranscript{}, nil
@@ -102,6 +107,15 @@ func (m *mockFullAgent) CalculateTotalTokenUsage([]byte, int, string) (*TokenUsa
 	return nil, nil //nolint:nilnil // test mock
 }
 
+// mockBuiltinStreamingAgent is a built-in agent that implements StreamingTextGenerator but NOT CapabilityDeclarer.
+type mockBuiltinStreamingAgent struct {
+	mockBaseAgent
+}
+
+func (m *mockBuiltinStreamingAgent) GenerateTextStreaming(context.Context, string, string, ProgressFn) (string, error) {
+	return "", nil
+}
+
 // mockBuiltinPromptAgent is a built-in agent that implements PromptExtractor but NOT CapabilityDeclarer.
 type mockBuiltinPromptAgent struct {
 	mockBaseAgent
@@ -439,3 +453,43 @@ func TestAsPromptExtractor(t *testing.T) {
 		}
 	})
 }
+
+func TestAsStreamingTextGenerator(t *testing.T) {
+	t.Parallel()
+
+	t.Run("not implemented", func(t *testing.T) {
+		t.Parallel()
+		ag := &mockBaseAgent{}
+		_, ok := AsStreamingTextGenerator(ag)
+		if ok {
+			t.Error("expected false for agent not implementing StreamingTextGenerator")
+		}
+	})
+
+	t.Run("builtin agent", func(t *testing.T) {
+		t.Parallel()
+		ag := &mockBuiltinStreamingAgent{}
+		stg, ok := AsStreamingTextGenerator(ag)
+		if !ok || stg == nil {
+			t.Error("expected true for built-in agent implementing StreamingTextGenerator")
+		}
+	})
+
+	t.Run("declared true", func(t *testing.T) {
+		t.Parallel()
+		ag := &mockFullAgent{caps: DeclaredCaps{StreamingTextGenerator: true}}
+		stg, ok := AsStreamingTextGenerator(ag)
+		if !ok || stg == nil {
+			t.Error("expected true when capability declared true")
+		}
+	})
+
+	t.Run("declared false", func(t *testing.T) {
+		t.Parallel()
+		ag := &mockFullAgent{caps: DeclaredCaps{StreamingTextGenerator: false}}
+		_, ok := AsStreamingTextGenerator(ag)
+		if ok {
+			t.Error("expected false when capability declared false")
+		}
+	})
+}

cmd/entire/cli/agent/claudecode/claude_test.go

@@ -110,6 +110,38 @@ func TestBuildGenerateArgs_PassesSettingsAsPath(t *testing.T) {
 	}
 }
 
+func TestBuildStreamingGenerateArgs_KeepsIsolationAndAuthContract(t *testing.T) {
+	t.Parallel()
+	// The streaming argv must keep the same isolation (--setting-sources "")
+	// and auth-injection (--settings <path>) contract as buildGenerateArgs;
+	// dropping the injection silently breaks apiKeyHelper (API-billing) auth
+	// on every streaming call.
+	args := buildStreamingGenerateArgs("haiku", "")
+	got, ok := flagValue(args, "--setting-sources")
+	if !ok {
+		t.Fatalf("--setting-sources flag missing from args: %v", args)
+	}
+	if got != "" {
+		t.Fatalf("--setting-sources = %q, want %q (must load no sources)", got, "")
+	}
+	if got, ok := flagValue(args, "--output-format"); !ok || got != "stream-json" {
+		t.Fatalf("--output-format = %q, want stream-json: %v", got, args)
+	}
+	if _, ok := flagValue(args, "--settings"); ok {
+		t.Fatalf("--settings must be absent when there is no settings path: %v", args)
+	}
+
+	path := "/tmp/entire-claude-auth-123.json"
+	args = buildStreamingGenerateArgs("haiku", path)
+	got, ok = flagValue(args, "--settings")
+	if !ok {
+		t.Fatalf("--settings flag missing: %v", args)
+	}
+	if got != path {
+		t.Fatalf("--settings = %q, want the file path %q", got, path)
+	}
+}
+
 func TestWriteAuthSettingsFile_WritesOnlyAPIKeyHelper0600(t *testing.T) {
 	t.Parallel()
 	helper := `echo "sk-ant-secret"` // could embed a literal key

cmd/entire/cli/agent/claudecode/generate.go

@@ -51,6 +51,26 @@ func buildGenerateArgs(model, settingsPath string) []string {
 	return args
 }
 
+// buildStreamingGenerateArgs is buildGenerateArgs for the stream-json path,
+// with the same isolation and auth-injection contract (see buildGenerateArgs).
+// --include-partial-messages enables the per-token stream_event envelopes
+// that PhaseFirstToken and PhaseGenerating are dispatched from, and
+// --verbose is required by the claude CLI for stream-json output.
+func buildStreamingGenerateArgs(model, settingsPath string) []string {
+	args := []string{
+		"--print",
+		"--output-format", "stream-json",
+		"--include-partial-messages",
+		"--verbose",
+		"--model", model,
+		"--setting-sources", "",
+	}
+	if settingsPath != "" {
+		args = append(args, "--settings", settingsPath)
+	}
+	return args
+}
+
 // writeAuthSettingsFile writes a minimal claude settings file containing only
 // the given apiKeyHelper and returns its path plus a cleanup func. The file is
 // created 0600 so the (possibly key-bearing) helper is no more exposed than the
@@ -183,12 +203,21 @@ func (c *ClaudeCodeAgent) GenerateText(ctx context.Context, prompt string, model
 			return "", classifyEnvelopeError(result, env.APIErrorStatus, exitCode)
 		}
 		// No structured signal on stdout — ctx cancellation is next most
-		// informative, since the rest is a guess.
-		if errors.Is(ctx.Err(), context.DeadlineExceeded) {
-			return "", context.DeadlineExceeded
-		}
-		if errors.Is(ctx.Err(), context.Canceled) {
-			return "", context.Canceled
+		// informative, since the rest is a guess. Wrap the sentinel in a
+		// *agent.TextGenerationError (like the streaming path and every other
+		// provider) so the explain timeout diagnostic still gets captured
+		// stderr and the stdout byte count when this path is reached via the
+		// old-CLI streaming fallback.
+		if ctxErr := ctx.Err(); ctxErr != nil && (errors.Is(ctxErr, context.DeadlineExceeded) || errors.Is(ctxErr, context.Canceled)) {
+			sentinel := context.Canceled
+			if errors.Is(ctxErr, context.DeadlineExceeded) {
+				sentinel = context.DeadlineExceeded
+			}
+			return "", &agent.TextGenerationError{
+				Err:         sentinel,
+				Stderr:      strings.TrimSpace(stderr.String()),
+				StdoutBytes: stdout.Len(),
+			}
 		}
 		if isExecNotFound(err) {
 			return "", &ClaudeError{Kind: ClaudeErrorCLIMissing, Cause: err}