Add basic tracing support for cloud agents. (warpdotdev#12363)

## Description

Adds opt-in OpenTelemetry tracing for cloud-agent runs so we can inspect
end-to-end execution when debugging latency and failures.

When `WARP_CLOUD_AGENT_OTLP_ENDPOINT` is configured, the native app
installs an OTLP/HTTP trace exporter; otherwise tracing remains a no-op.
Cloud-agent spans are explicitly marked, and filtering at the exporter
boundary prevents unrelated application spans from being sent. This also
adds spans and events across the cloud-agent lifecycle, server APIs, and
driver setup, and propagates the current span through asynchronous tasks
and streams.

The tracing provider is retained through the standard application
shutdown path, with a best-effort active-span registry that ends
still-reachable spans before shutting down the provider so long-lived
asynchronous work is less likely to be lost.

## Linked Issue

None.

## Testing

- [x] Manually tested locally with `./script/run -- agent run --prompt
"hello world"`.
- [x] Ran `./script/format`.
- [x] Ran `cargo check -p warp --features gui`.
- [x] Ran `cargo doc -p warp --features gui --no-deps`.

No automated tests were added; this is opt-in observability plumbing
that was validated through the cloud-agent command path and focused
compilation/documentation checks.

> [!NOTE]
> note from dstern: i ended up working with an agent to do an end-to-end
test of the fully-authenticated flow with a local cloud agent running
via a self-hosted worker host; everything appears to work.

## Agent Mode

- [x] Warp Agent Mode - This PR was created via Warp's AI Agent Mode

CHANGELOG-NONE

Co-Authored-By: Oz <oz-agent@warp.dev>

Author: vorporeal

Cargo.lock

@@ -198,6 +198,15 @@ notify-debouncer-full = { git = "https://github.com/warpdotdev/notify", rev = "9
 nom = "7.1.1"
 num-traits = "0.2"
 oauth2 = { version = "5.0.0", default-features = false }
+opentelemetry = { version = "0.32.0", default-features = false, features = ["trace"] }
+opentelemetry-otlp = { version = "0.32.0", default-features = false, features = [
+  "gzip-http",
+  "http-proto",
+  "reqwest-blocking-client",
+  "trace",
+  "zstd-http",
+] }
+opentelemetry_sdk = { version = "0.32.1", default-features = false, features = ["trace"] }
 openh264 = "0.8"
 static_assertions = "1.1.0"
 url = "2.5.4"
@@ -277,6 +286,12 @@ toml_edit = "0.25.5"
 tower = "0.5.2"
 tower-http = "0.6.6"
 tracing = "0.1.40"
+tracing-futures = "0.2.5"
+tracing-opentelemetry = { version = "0.33.0", default-features = false }
+tracing-subscriber = { version = "0.3.22", default-features = false, features = [
+  "registry",
+  "std",
+] }
 arborium = { version = "2", default-features = false, features = [
   "lang-rust",
   "lang-go",

Cargo.toml

@@ -213,6 +213,7 @@ tikv-jemallocator = { version = "0.6", optional = true, features = [
 toml = "0.8.13"
 toml_edit.workspace = true
 tracing.workspace = true
+tracing-futures = { workspace = true, features = ["futures-03"] }
 ui_components.workspace = true
 unicase = "2.7.0"
 unicode-general-category.workspace = true
@@ -316,8 +317,13 @@ http_server.workspace = true
 hyper.workspace = true
 libsqlite3-sys = { version = "0.33.0", features = ["bundled"] }
 mio = { version = "1.1.1", features = ["os-poll", "os-ext"] }
+opentelemetry.workspace = true
+opentelemetry-otlp.workspace = true
+opentelemetry_sdk.workspace = true
 tokio.workspace = true
 tokio-util.workspace = true
+tracing-opentelemetry.workspace = true
+tracing-subscriber.workspace = true
 
 # AWS SDK (loading credentials for BYO LLM)
 aws-config = { version = "1.8.16", features = ["credentials-login"] }

app/Cargo.toml

@@ -546,6 +546,12 @@ impl From<PrepareEnvironmentError> for AgentDriverError {
 }
 
 impl AgentDriver {
+    #[tracing::instrument(name = "AgentDriver::new", skip_all, err, fields(
+        tags.cloud_agent = true,
+        task_id = ?options.task_id,
+        parent_run_id = ?options.parent_run_id,
+        is_sandbox = tracing::field::Empty,
+    ))]
     pub fn new(
         options: AgentDriverOptions,
         ctx: &mut ModelContext<Self>,
@@ -626,6 +632,7 @@ impl AgentDriver {
         // so they allow root execution with permissive flags.
         if warp_isolation_platform::detect().is_some() {
             env_vars.insert(OsString::from("IS_SANDBOX"), OsString::from("1"));
+            tracing::Span::current().record("is_sandbox", true);
         }
 
         let resolved_env_vars = Arc::new(env_vars);
@@ -1782,6 +1789,7 @@ impl AgentDriver {
     /// Driving the agent mostly requires main-thread UI framework updates, but using `async` and
     /// a `ModelSpawner` lets us express the high-level process linearly rather than in a
     /// series of callbacks and state machine updates.
+    #[tracing::instrument(name = "AgentDriver::run_internal", skip_all, err, fields(tags.cloud_agent = true))]
     async fn run_internal(
         task: Task,
         foreground: ModelSpawner<Self>,
@@ -3278,6 +3286,11 @@ impl AgentDriver {
     ) {
         match event {
             TerminalDriverEvent::SlowBootstrap => {
+                tracing::event!(
+                    tracing::Level::WARN,
+                    tags.cloud_agent = true,
+                    "slow bootstrap"
+                );
                 eprintln!(
                     "Warning: Terminal session is slow to bootstrap. See https://docs.warp.dev/support-and-community/troubleshooting-and-support/known-issues#shells to troubleshoot."
                 );
@@ -3286,6 +3299,12 @@ impl AgentDriver {
                 session_id,
                 join_url,
             } => {
+                tracing::event!(
+                    tracing::Level::INFO,
+                    tags.cloud_agent = true,
+                    session_id = %*session_id,
+                    "shared session established",
+                );
                 write_session_joined(join_url, self.output_format);
 
                 // If running as part of a task, store the session-sharing link.
@@ -3369,6 +3388,7 @@ impl AgentDriver {
     /// Invoke the end-of-run snapshot upload pipeline if the feature flag is enabled and this
     /// driver is associated with a cloud task. Errors are logged internally; this helper always
     /// returns so cleanup can proceed.
+    #[tracing::instrument(skip_all, fields(tags.cloud_agent = true))]
     async fn run_snapshot_upload(spawner: &ModelSpawner<Self>) {
         if !FeatureFlag::OzHandoff.is_enabled() {
             return;

app/src/ai/agent_sdk/driver.rs

@@ -378,6 +378,7 @@ pub(super) async fn clone_repos(
 
 /// Clone a GitHub repository to `{working_dir}/{repo.repo}` if it does not already exist.
 /// This only performs the clone -- it does NOT register the repo with `DetectedRepositories`.
+#[tracing::instrument(skip_all, err, fields(tags.cloud_agent = true, repo = %repo))]
 pub(super) async fn clone_repo(
     repo: &GithubRepo,
     working_dir: &Path,
@@ -438,6 +439,7 @@ pub(super) async fn clone_repo(
 
 /// Register a cloned GitHub repository with `DetectedRepositories` so that the
 /// skill watcher and other repo-aware subsystems can discover it.
+#[tracing::instrument(skip_all, err, fields(tags.cloud_agent = true, repo = %repo, is_sandbox = is_sandbox))]
 pub(super) async fn register_cloned_repo(
     repo: &GithubRepo,
     working_dir: &Path,
@@ -546,6 +548,7 @@ async fn subscribe_to_codebase_index_events(
         .map_err(|_| PrepareEnvironmentError::InvalidRuntimeState)
 }
 
+#[tracing::instrument(skip_all, err, fields(tags.cloud_agent = true, repo = %repo_name))]
 async fn index_repo_codebase(
     repo_name: &str,
     working_dir: &Path,

app/src/ai/agent_sdk/driver/environment.rs

@@ -248,6 +248,10 @@ pub(crate) fn configure_git_identity(credentials: &[GitCredential]) {
 /// Returns `Ok(())` on success (including when the server returns no
 /// credentials). Returns `Err` when the workload-token issuance or the server
 /// API call fails — these are transient failures worth retrying.
+#[tracing::instrument(name = "git_credentials::try_refresh", skip_all, err, fields(
+    tags.cloud_agent = true,
+    task_id,
+))]
 async fn try_refresh(task_id: &str, ai_client: &Arc<dyn AIClient>) -> Result<()> {
     let workload_token =
         warp_isolation_platform::issue_workload_token(Some(Duration::from_secs(5 * 60)))

app/src/ai/agent_sdk/driver/git_credentials.rs

@@ -8,8 +8,10 @@ use std::task::{Context, Poll};
 use std::time::Duration;
 
 use futures::channel::oneshot;
+use futures::TryFutureExt as _;
 use session_sharing_protocol::common::{Role, SessionId};
 use session_sharing_protocol::sharer::SessionRetentionReason;
+use tracing::Instrument as _;
 use warp_cli::share::{ShareAccessLevel, ShareRequest, ShareSubject};
 use warp_completer::completer::CommandOutput;
 use warp_core::command::ExitCode;
@@ -547,11 +549,17 @@ impl TerminalDriver {
             session_bootstrapped
                 .wait()
                 .with_timeout(TERMINAL_SESSION_BOOTSTRAP_TIMEOUT)
-                .await
-                .map_err(|_| {
+                .map_err(|err| {
                     log::error!("Timed out waiting for session bootstrap");
+                    tracing::error!(error = %err);
+
                     AgentDriverError::BootstrapFailed
                 })
+                .instrument(tracing::info_span!(
+                    "wait_for_session_bootstrapped",
+                    tags.cloud_agent = true
+                ))
+                .await
         }
     }