misc rename continuations

Signed-off-by: shanaiabuggy <59746633+shanaiabuggy@users.noreply.github.com>

Author: shanaiabuggy

packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-experiments-upload/SKILL.md

@@ -194,8 +194,6 @@ If `run_count` is 0 after ingesting, the traces didn't associate — almost alwa
 ## Gotchas
 
 - **Create before you log.** The Evaluation entity must exist before any ingest referencing it — otherwise `400`.
-- **Evaluation and Experiment references use different fields.** `evaluation_name` associates telemetry with an Evaluation, while `experiment_ids` assigns that Evaluation to its parent Experiments.
-- **OTLP evaluation context:** set `nemo.evaluation.name` and, when applicable, `nemo.test_case.name` as root-span attributes. These correspond to `evaluation_name` and `test_case_name` in the JSON `evaluation_context` used by other ingest endpoints.
 - **The parent lives at `/experiments`; `/experiment-groups` is a deprecated hidden alias.** Prefer `/experiments`. Evaluations are created and logged under `/evaluations`.
 - **`metadata` is `dict[str, str]`** — stringify non-string values or you'll get a `422`.
 - **ATIF and chat-completions are `extra="forbid"`** (unknown keys → 422); `evaluation_context` itself is lenient (`extra="ignore"`).

packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-intake/references/ingest-formats.md

@@ -21,10 +21,6 @@ Full request schemas for the three intake ingest endpoints. All are under
 - `test_case_name` is optional and identifies which task or test case the run covers.
 - The referenced Evaluation **must already exist** (create it first) or the request is rejected with
   `400 "…must be created before it can be logged."`
-- The model is lenient (`extra="ignore"`): retired keys (`evaluation_sha`, `evaluation_run_id`,
-  `metadata`) are accepted but dropped.
-- A deprecated `experiment_context` `{experiment_id, test_case_id}` shape is still accepted;
-  `evaluation_context` wins if both are present. Use `evaluation_context`.
 
 ---
 

plugins/nemo-evaluator/src/nemo_evaluator/intake/mapping.py

@@ -84,7 +84,7 @@ def trial_to_atif_ingest(
     trial: AgentEvalTrial,
     *,
     run_id: str,
-    experiment_id: str,
+    evaluation_name: str,
     agent_name: str,
     started_at: datetime,
     agent_version: str = DEFAULT_AGENT_VERSION,
@@ -132,7 +132,7 @@ def trial_to_atif_ingest(
         "session_id": session_id_for(run_id, trial.id),
         "agent": agent,
         "steps": [step],
-        "evaluation_context": run_task_to_evaluation_context(trial, evaluation_name=experiment_id),
+        "evaluation_context": run_task_to_evaluation_context(trial, evaluation_name=evaluation_name),
     }
     if final_metrics is not None:
         body["final_metrics"] = final_metrics

plugins/nemo-evaluator/src/nemo_evaluator/intake/publish.py

@@ -186,7 +186,7 @@ async def _publish_trial(trial: AgentEvalTrial) -> PublishedTrial:
             body = mapping.trial_to_atif_ingest(
                 trial,
                 run_id=result.run_id,
-                experiment_id=experiment_id,
+                evaluation_name=experiment_id,
                 agent_name=agent_name,
                 started_at=started_at,
                 agent_version=agent_version,

plugins/nemo-evaluator/tests/intake/test_mapping.py

@@ -90,7 +90,7 @@ def test_trial_to_atif_ingest_shape() -> None:
     body = trial_to_atif_ingest(
         _trial(trial_id="t-1", task_id="task-1", output_text="final answer"),
         run_id="run-1",
-        experiment_id="exp-1",
+        evaluation_name="exp-1",
         agent_name="my-agent",
         started_at=STARTED_AT,
         model_name="gpt-4o",
@@ -104,14 +104,16 @@ def test_trial_to_atif_ingest_shape() -> None:
 
 
 def test_trial_to_atif_ingest_defaults_version_and_omits_model_name() -> None:
-    body = trial_to_atif_ingest(_trial(), run_id="run-1", experiment_id="exp-1", agent_name="a", started_at=STARTED_AT)
+    body = trial_to_atif_ingest(
+        _trial(), run_id="run-1", evaluation_name="exp-1", agent_name="a", started_at=STARTED_AT
+    )
     assert body["agent"] == {"name": "a", "version": "unknown"}
     assert "model_name" not in body["agent"]
 
 
 def test_trial_to_atif_ingest_handles_missing_output() -> None:
     body = trial_to_atif_ingest(
-        _trial(output_text=None), run_id="run-1", experiment_id="exp-1", agent_name="a", started_at=STARTED_AT
+        _trial(output_text=None), run_id="run-1", evaluation_name="exp-1", agent_name="a", started_at=STARTED_AT
     )
     assert body["steps"] == [{"source": "agent", "step_id": 1, "message": "", "timestamp": STARTED_AT}]
 
@@ -120,7 +122,7 @@ def test_trial_to_atif_ingest_includes_final_metrics_when_given() -> None:
     body = trial_to_atif_ingest(
         _trial(),
         run_id="run-1",
-        experiment_id="exp-1",
+        evaluation_name="exp-1",
         agent_name="a",
         started_at=STARTED_AT,
         final_metrics={"total_prompt_tokens": 10},
@@ -133,7 +135,7 @@ def test_trial_to_atif_ingest_adds_invocation_window_when_ended_at_given() -> No
     # root-span latency is the trial's runtime instead of 0.
     ended = STARTED_AT + timedelta(seconds=12.5)
     body = trial_to_atif_ingest(
-        _trial(), run_id="run-1", experiment_id="exp-1", agent_name="a", started_at=STARTED_AT, ended_at=ended
+        _trial(), run_id="run-1", evaluation_name="exp-1", agent_name="a", started_at=STARTED_AT, ended_at=ended
     )
     (step,) = body["steps"]
     assert step["extra"] == {
@@ -142,7 +144,9 @@ def test_trial_to_atif_ingest_adds_invocation_window_when_ended_at_given() -> No
 
 
 def test_trial_to_atif_ingest_omits_invocation_window_without_ended_at() -> None:
-    body = trial_to_atif_ingest(_trial(), run_id="run-1", experiment_id="exp-1", agent_name="a", started_at=STARTED_AT)
+    body = trial_to_atif_ingest(
+        _trial(), run_id="run-1", evaluation_name="exp-1", agent_name="a", started_at=STARTED_AT
+    )
     assert "extra" not in body["steps"][0]
 
 

sdk/python/nemo-platform/src/nemo_platform/skills/nemo-experiments-upload/SKILL.md

@@ -516,7 +516,7 @@ def _seed_sessions(
         prompt_tokens = max(10, int(rng.gauss(spec.prompt_tokens_mean, spec.prompt_tokens_mean * 0.25)))
         completion_tokens = max(5, int(rng.gauss(spec.completion_tokens_mean, spec.completion_tokens_mean * 0.3)))
 
-        test_case_id = f"case-{i:04d}"
+        test_case_name = f"case-{i:04d}"
         run_id = f"run-{i // 25:02d}"
         # Spread sessions across the ~5.5h prior to "now" so the Studio timeline looks varied.
         offset_seconds = (i / max(1, spec.n_sessions)) * 5.5 * 3600
@@ -531,9 +531,9 @@ def _seed_sessions(
         )
         atif_body = _demo_atif_body(
             base_started_at=base_started_at,
-            evaluation_id=spec.name,
+            evaluation_name=spec.name,
             run_id=run_id,
-            test_case_id=test_case_id,
+            test_case_name=test_case_name,
             cost_usd=cost_usd,
             latency_ms=latency_ms,
             offset_seconds=offset_seconds,
@@ -571,9 +571,9 @@ def _seed_sessions(
 def _demo_atif_body(
     *,
     base_started_at: datetime,
-    evaluation_id: str,
+    evaluation_name: str,
     run_id: str,
-    test_case_id: str,
+    test_case_name: str,
     cost_usd: float,
     latency_ms: int,
     offset_seconds: float,
@@ -585,20 +585,20 @@ def _demo_atif_body(
 ) -> dict[str, Any]:
     session_started_at = base_started_at + timedelta(seconds=offset_seconds)
     finished_at = session_started_at + timedelta(milliseconds=latency_ms)
-    session_id = f"{evaluation_id}-{run_id}-{test_case_id}"
+    session_id = f"{evaluation_name}-{run_id}-{test_case_name}"
     # `extra.verifier` carries the timing block (used by the rollup for session latency).
     # We omit `extra.verifier_result` so ATIF ingest doesn't auto-create a `harbor.verifier`
     # evaluator alongside our cleanly-named ones from POST /evaluator-results.
     return {
         "schema_version": "ATIF-v1.7",
         "session_id": session_id,
         "evaluation_context": {
-            "evaluation_name": evaluation_id,
-            "test_case_name": test_case_id,
+            "evaluation_name": evaluation_name,
+            "test_case_name": test_case_name,
         },
         "extra": {
-            "task_id": test_case_id,
-            "task_name": test_case_id,
+            "task_id": test_case_name,
+            "task_name": test_case_name,
             "verifier": {
                 "started_at": _iso(session_started_at),
                 "finished_at": _iso(finished_at),
@@ -614,14 +614,14 @@ def _demo_atif_body(
                 "step_id": 1,
                 "timestamp": _iso(session_started_at),
                 "source": "user",
-                "message": f"test case: {test_case_id}",
+                "message": f"test case: {test_case_name}",
             },
             {
                 "step_id": 2,
                 "timestamp": _iso(finished_at),
                 "source": "agent",
                 "model_name": model_name,
-                "message": f"solved {test_case_id}",
+                "message": f"solved {test_case_name}",
                 "metrics": {
                     "prompt_tokens": prompt_tokens,
                     "completion_tokens": completion_tokens,

sdk/python/nemo-platform/src/nemo_platform/skills/nemo-intake/references/ingest-formats.md

@@ -1385,7 +1385,7 @@ def _enqueue_stale_denormalization(
             or entity.agent_versions != rollup.agent_versions
             or entity.model_names != rollup.model_names
         ):
-            denormalizer.mark_dirty(workspace=workspace, evaluation_id=entity.name)
+            denormalizer.mark_dirty(workspace=workspace, evaluation_name=entity.name)
 
 
 async def _evaluation_response_with_rollup(

services/intake/scripts/spans/seed_experiments_demo.py

@@ -3,7 +3,7 @@
 
 """Denormalizes agent/model name fields from ClickHouse onto Evaluation entities.
 
-Ingest marks ``(workspace, evaluation_id)`` dirty; a background loop recomputes each touched
+Ingest marks ``(workspace, evaluation_name)`` dirty; a background loop recomputes each touched
 evaluation's rollup and writes the distinct ``agent_names``/``agent_versions``/``model_names`` sets onto
 the (system-managed) fields of its Evaluation entity. That lets the Evaluations list filter by
 agent/model name against the entity store (``$contains``) instead of scanning the ClickHouse session
@@ -35,7 +35,7 @@
 
 
 class EvaluationDenormalizer(BackgroundWorker):
-    """Coalesces dirty evaluation ids and refreshes their denormalized name fields on a fixed cadence.
+    """Coalesces dirty evaluation names and refreshes their denormalized name fields on a fixed cadence.
 
     A burst of :meth:`mark_dirty` calls for one evaluation within an interval collapses to a single
     refresh, and marking is a plain, non-blocking set add — so the ingest and read hot paths are never
@@ -55,12 +55,12 @@ def __init__(
         self._interval_seconds = interval_seconds
         self._dirty: set[tuple[str, str]] = set()
 
-    def mark_dirty(self, *, workspace: str, evaluation_id: str) -> None:
+    def mark_dirty(self, *, workspace: str, evaluation_name: str) -> None:
         """Queue an evaluation for refresh. Cheap and non-blocking; safe to call from the ingest path."""
-        self._dirty.add((workspace, evaluation_id))
+        self._dirty.add((workspace, evaluation_name))
 
     def pending(self) -> set[tuple[str, str]]:
-        """Return a copy of the currently-queued ``(workspace, evaluation_id)`` pairs (observability/tests)."""
+        """Return a copy of the currently-queued ``(workspace, evaluation_name)`` pairs (observability/tests)."""
         return set(self._dirty)
 
     async def _run(self) -> None:
@@ -99,28 +99,28 @@ async def flush(self) -> None:
         batch = self._dirty
         self._dirty = set()
         by_workspace: dict[str, list[str]] = {}
-        for workspace, evaluation_id in batch:
-            by_workspace.setdefault(workspace, []).append(evaluation_id)
-        for workspace, evaluation_ids in by_workspace.items():
+        for workspace, evaluation_name in batch:
+            by_workspace.setdefault(workspace, []).append(evaluation_name)
+        for workspace, evaluation_names in by_workspace.items():
             try:
-                await self._refresh_workspace(workspace, evaluation_ids)
+                await self._refresh_workspace(workspace, evaluation_names)
             except Exception:
                 # Re-queue the whole workspace batch for the next cycle (e.g. ClickHouse unavailable).
                 logger.exception("Failed to refresh evaluation names for workspace %s; re-queuing", workspace)
-                for evaluation_id in evaluation_ids:
-                    self.mark_dirty(workspace=workspace, evaluation_id=evaluation_id)
+                for evaluation_name in evaluation_names:
+                    self.mark_dirty(workspace=workspace, evaluation_name=evaluation_name)
 
-    async def _refresh_workspace(self, workspace: str, evaluation_ids: list[str]) -> None:
-        rollups = await self._rollup_repository.get_rollups(workspace=workspace, evaluation_ids=evaluation_ids)
-        for evaluation_id in evaluation_ids:
-            rollup = rollups.get(evaluation_id)
+    async def _refresh_workspace(self, workspace: str, evaluation_names: list[str]) -> None:
+        rollups = await self._rollup_repository.get_rollups(workspace=workspace, evaluation_ids=evaluation_names)
+        for evaluation_name in evaluation_names:
+            rollup = rollups.get(evaluation_name)
             if rollup is None:
                 continue
-            await self._write_names(workspace, evaluation_id, rollup)
+            await self._write_names(workspace, evaluation_name, rollup)
 
-    async def _write_names(self, workspace: str, evaluation_id: str, rollup: EvaluationRollup) -> None:
+    async def _write_names(self, workspace: str, evaluation_name: str, rollup: EvaluationRollup) -> None:
         try:
-            evaluation = await self._entity_client.get(Experiment, name=evaluation_id, workspace=workspace)
+            evaluation = await self._entity_client.get(Experiment, name=evaluation_name, workspace=workspace)
         except EntityNotFoundError:
             # Deleted between ingest and refresh; nothing to update.
             return
@@ -139,4 +139,4 @@ async def _write_names(self, workspace: str, evaluation_id: str, rollup: Evaluat
             await self._entity_client.update(evaluation)
         except EntityConflictError:
             # A concurrent user edit won the optimistic lock; re-queue for the next cycle.
-            self.mark_dirty(workspace=workspace, evaluation_id=evaluation_id)
+            self.mark_dirty(workspace=workspace, evaluation_name=evaluation_name)