fix(tracing): authenticate to remote viewers, and write traces where they survive

packages/nooa-bench/src/nooa_bench/runner.py

@@ -35,10 +35,15 @@
 
 logger = logging.getLogger("nooa_bench.runner")
 
-# Harbor container path conventions
+# Harbor container path conventions.
+#
+# Harbor bind-mounts ONLY /logs/agent and /logs/verifier from the host
+# (harbor.models.trial.paths). Everything else under /logs lives in the
+# container's own writable layer and is destroyed when the trial container is
+# removed -- which is the default. Traces therefore have to be written inside
+# /logs/agent to survive the run; /logs/artifacts silently discarded them.
 LOGS_DIR = Path("/logs/agent")
-ARTIFACTS_DIR = Path("/logs/artifacts")
-TRACES_DIR = ARTIFACTS_DIR / "traces"
+TRACES_DIR = LOGS_DIR / "traces"
 ANSWER_FILE = Path("/app/answer.txt")
 
 
@@ -60,7 +65,7 @@ def _setup_logging() -> None:
 def _setup_tracing(model: str, agent_type: str) -> None:
     """Enable OTel tracing, always on disk and additionally live when reachable.
 
-    JSONL files in the Harbor artifact directory (``/logs/artifacts/traces/``)
+    JSONL files under ``/logs/agent/traces/`` (a host-mounted directory)
     are written unconditionally — they are the record failure analysis runs on,
     and they are importable later via ``nemo-oo import-harbor``.  When
     ``OTLP_ENDPOINT`` (default ``http://localhost:5001``) is reachable the
@@ -135,11 +140,11 @@ def _write_result(result: dict[str, Any], model: str, agent_type: str) -> None:
 def _write_trajectory(agent: Any) -> None:
     """Dump the agent's full event history to LOGS_DIR/trajectory.json.
 
-    The OTLP spans under ``/logs/artifacts/traces/`` remain the canonical
-    record, but failure analysis starts in the per-task ``agent/`` directory —
-    which otherwise holds only ``nooa_bench.log`` and a ``result.json``
-    carrying just the final response.  Anyone looking there for the turn-by-turn
-    trajectory previously found nothing.
+    The OTLP spans under ``agent/traces/`` remain the canonical record, but
+    failure analysis starts in the per-task ``agent/`` directory — which
+    otherwise holds only ``nooa_bench.log`` and a ``result.json`` carrying just
+    the final response.  Anyone looking there for the turn-by-turn trajectory
+    previously found nothing.
     """
     manager = getattr(agent, "event_manager", None)
     if manager is None:

src/nooa/tracing/__init__.py

@@ -190,7 +190,9 @@ def probe_otlp_endpoint(endpoint: str, timeout: float | None = None) -> bool:
     base = endpoint.rstrip("/").removesuffix("/v1/traces").removesuffix("/v1")
     health_url = f"{base}/api/eval/health"
     try:
-        req = urllib.request.Request(health_url, method="GET")
+        from nooa.tracing._viewer_auth import apply_viewer_auth
+
+        req = urllib.request.Request(health_url, headers=apply_viewer_auth({}), method="GET")
         with urllib.request.urlopen(req, timeout=timeout):
             pass
         return True

src/nooa/tracing/_litellm_journal.py

@@ -205,7 +205,10 @@ def _send_impl() -> None:
                 n_items,
                 tag,
             )
-        headers = {"Content-Type": "application/json"}
+        from nooa.tracing._viewer_auth import apply_viewer_auth
+
+        # A remote viewer rejects unauthenticated writes with 403.
+        headers = apply_viewer_auth({"Content-Type": "application/json"})
         if session_id:
             # The receiver's /v1/journal/blocks route uses this header to
             # key blocks per session.  /v1/journal/calls reads session_id

src/nooa/tracing/_otlp_http_exporter.py

@@ -103,11 +103,14 @@ def _send_payload(self, spans: Sequence[ReadableSpan], payload: dict) -> SpanExp
             ``SpanExportResult.SUCCESS`` on HTTP 2xx, ``FAILURE`` otherwise.
         """
         try:
+            from nooa.tracing._viewer_auth import apply_viewer_auth
+
             data = json.dumps(payload, separators=(",", ":")).encode("utf-8")
             req = urllib.request.Request(
                 self._endpoint,
                 data=data,
-                headers={"Content-Type": "application/json"},
+                # A remote viewer rejects unauthenticated writes with 403.
+                headers=apply_viewer_auth({"Content-Type": "application/json"}),
                 method="POST",
             )
             with urllib.request.urlopen(req, timeout=10) as resp:

src/nooa/tracing/_viewer_auth.py

@@ -0,0 +1,44 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+"""Bearer-token auth for clients posting to a remote nooa viewer.
+
+The viewer refuses writes from non-loopback clients unless
+``NOOA_VIEWER_AUTH_TOKEN`` is configured on the server
+(:func:`nooa.viewer.main._require_viewer_authorization`), and it authenticates
+with ``Authorization: Bearer <token>``.
+
+Exporters read the same env var on the client side, so a remote run streams
+live by setting one variable at both ends::
+
+    # viewer host
+    NOOA_VIEWER_AUTH_TOKEN=<token> nooa start-dev
+
+    # machine running the agent
+    NOOA_VIEWER_AUTH_TOKEN=<token> OTLP_ENDPOINT=http://<host>:5001/v1/traces ...
+
+Unset (the default, loopback-only case) adds no header, so local development is
+unchanged.
+"""
+
+from __future__ import annotations
+
+import os
+
+_ENV_VAR = "NOOA_VIEWER_AUTH_TOKEN"
+
+
+def viewer_auth_token() -> str | None:
+    """Return the configured viewer token, or ``None`` when unset/blank."""
+    token = os.environ.get(_ENV_VAR, "").strip()
+    return token or None
+
+
+def apply_viewer_auth(headers: dict[str, str]) -> dict[str, str]:
+    """Add ``Authorization: Bearer`` to *headers* when a token is configured.
+
+    Mutates and returns *headers* so it can be used inline at call sites.
+    """
+    token = viewer_auth_token()
+    if token:
+        headers["Authorization"] = f"Bearer {token}"
+    return headers