Fix proxied Harbor Codex observations (#574)

* Fix proxied Harbor Codex observations

* Fix selected-task workflow validation

* Make Harbor observations runtime-authoritative

* Correlate Harbor MCP runtime telemetry

Author: N0zoM1z0

Makefile

@@ -4,6 +4,7 @@ UV_RUN := uv run --locked
 HARBOR_VERSION ?= 0.20.0
 HARBOR_RUNNER ?= uvx --from harbor==$(HARBOR_VERSION) harbor
 HARBOR_PYTHON ?= uvx --from harbor==$(HARBOR_VERSION) --with tomli-w==1.2.0 --with jsonschema python
+HARBOR_PROJECT_PYTHON ?= uv run --locked --with harbor==$(HARBOR_VERSION) --with tomli-w==1.2.0 --with jsonschema python
 # Validation pytest is process-isolated under xdist; Path monkeypatches stay
 # worker-local. Oracle/adapter Make targets remain serial.
 HARBOR_VALIDATION_WORKERS ?= 2
@@ -447,7 +448,7 @@ endif
 
 CODEX_WEB_SEARCH ?= disabled
 JACOBIAN_EVAL_PROXY ?= 0
-JACOBIAN_EVAL_CODEX_BINARY ?= $(shell command -v codex 2>/dev/null | xargs -r readlink -f)
+JACOBIAN_EVAL_CODEX_BINARY ?= $(shell command -v codex 2>/dev/null)
 define _jacobian_eval_container_proxy
 $(subst localhost,host.docker.internal,$(subst 127.0.0.1,host.docker.internal,$1))
 endef
@@ -487,6 +488,7 @@ endif
 
 agent-eval: ## Run a Harbor evaluation (JACOBIAN_ENABLED=0|1, JACOBIAN_EVAL_PROXY=0|1, DATASET=mathematical-benchmarks-v1, EVAL_EXECUTE=1).
 	@set -e; \
+	CODEX_BINARY="$(JACOBIAN_EVAL_CODEX_BINARY)"; \
 	if [ "$(EVAL_EXECUTE)" != "1" ]; then \
 		echo "Model execution is opt-in. Review the job, then run: make agent-eval DATASET=mathematical-benchmarks-v1 EVAL_EXECUTE=1"; \
 		exit 0; \
@@ -496,10 +498,7 @@ agent-eval: ## Run a Harbor evaluation (JACOBIAN_ENABLED=0|1, JACOBIAN_EVAL_PROX
 		exit 2; \
 	fi; \
 	if [ "$(JACOBIAN_EVAL_PROXY)" = "1" ]; then \
-		test -x "$(JACOBIAN_EVAL_CODEX_BINARY)" || { \
-			echo "JACOBIAN_EVAL_CODEX_BINARY must resolve to a Linux standalone Codex binary when JACOBIAN_EVAL_PROXY=1" >&2; \
-			exit 2; \
-		}; \
+		CODEX_BINARY="$$( $(UV_RUN) python -m benchmarks.tooling.codex_binary --candidate "$$CODEX_BINARY" )"; \
 		JACOBIAN_EVAL_UPSTREAM_PROXY="$(JACOBIAN_EVAL_UPSTREAM_PROXY)" \
 			$(UV_RUN) python -m benchmarks.tooling.harbor_proxy \
 			--output "$(JACOBIAN_EVAL_GOST_CONFIG)"; \
@@ -514,7 +513,7 @@ agent-eval: ## Run a Harbor evaluation (JACOBIAN_ENABLED=0|1, JACOBIAN_EVAL_PROX
 	JACOBIAN_EVAL_HTTPS_PROXY="$(JACOBIAN_EVAL_HTTPS_PROXY)" \
 	JACOBIAN_EVAL_ALL_PROXY="$(JACOBIAN_EVAL_ALL_PROXY)" \
 	JACOBIAN_EVAL_NO_PROXY="$(JACOBIAN_EVAL_NO_PROXY)" \
-	JACOBIAN_EVAL_CODEX_BINARY="$(JACOBIAN_EVAL_CODEX_BINARY)" \
+	JACOBIAN_EVAL_CODEX_BINARY="$$CODEX_BINARY" \
 	JACOBIAN_EVAL_GOST_CONFIG="$(JACOBIAN_EVAL_GOST_CONFIG)" \
 	$(HARBOR_RUNNER) run \
 		-c "$(EVAL_CONFIG)" \
@@ -527,7 +526,7 @@ agent-eval: ## Run a Harbor evaluation (JACOBIAN_ENABLED=0|1, JACOBIAN_EVAL_PROX
 
 agent-eval-validate: ## Normalize one observation (RESULTS=..., JOB=..., CONDITION=..., OUTPUT=...).
 	@test -n "$(RESULTS)" -a -n "$(JOB)" -a -n "$(CONDITION)" -a -n "$(OUTPUT)" || { echo "RESULTS, JOB, CONDITION, and OUTPUT are required" >&2; exit 2; }
-	$(UV_RUN) python -m benchmarks.tooling.observation_results validate \
+	$(HARBOR_PROJECT_PYTHON) -m benchmarks.tooling.observation_results validate \
 		--dataset "$(or $(DATASET),mathematical-benchmarks-v1)" --condition "$(CONDITION)" \
 		--job "$(JOB)" --jobs-dir "$(RESULTS)" --output "$(OUTPUT)" \
 		$(if $(RESULT),--result "$(RESULT)",) \

benchmarks/config/agent-eval-egress-proxy.compose.yaml

@@ -1,12 +1,12 @@
 services:
   main:
     environment:
-      HTTP_PROXY: ""
-      HTTPS_PROXY: ""
+      HTTP_PROXY: http://127.0.0.1:12346
+      HTTPS_PROXY: http://127.0.0.1:12346
       ALL_PROXY: ""
       NO_PROXY: ${JACOBIAN_EVAL_NO_PROXY:-localhost,127.0.0.1,jacobian}
-      http_proxy: ""
-      https_proxy: ""
+      http_proxy: http://127.0.0.1:12346
+      https_proxy: http://127.0.0.1:12346
       all_proxy: ""
       no_proxy: ${JACOBIAN_EVAL_NO_PROXY:-localhost,127.0.0.1,jacobian}
   harbor-docker-egress-control-sidecar:

benchmarks/config/mathematical-benchmarks-v1-control-proxy.json

@@ -2,6 +2,9 @@
   "jobs_dir": "benchmarks/results/mathematical-benchmarks-v1-control-proxy",
   "n_attempts": 3,
   "timeout_multiplier": 1,
+  "artifacts": [
+    "/logs/agent/trajectory.json"
+  ],
   "orchestrator": {
     "type": "local",
     "n_concurrent_trials": 1,

benchmarks/config/mathematical-benchmarks-v1-control.json

@@ -2,6 +2,9 @@
   "jobs_dir": "benchmarks/results/mathematical-benchmarks-v1-control",
   "n_attempts": 3,
   "timeout_multiplier": 1,
+  "artifacts": [
+    "/logs/agent/trajectory.json"
+  ],
   "orchestrator": {
     "type": "local",
     "n_concurrent_trials": 1,

benchmarks/datasets/mathematical-benchmarks-v1/README.md

@@ -92,6 +92,13 @@ Jacobian affordance, so include negative controls and tasks that discriminate
 tool adoption. Public-suite observations remain directional workflow evidence,
 not a causal performance claim.
 
+Jacobian-enabled jobs collect both Codex ATIF and the Jacobian sidecar's MCP
+runtime log. The runtime log is authoritative for `math.find`, `math.run`, and
+`reasoning.write` counts and failed capability attempts; the normalizer does
+not infer executions from JavaScript source text. A missing configured trace or
+sidecar log makes the observation incomplete. Control jobs collect ATIF only
+because they do not start the Jacobian sidecar.
+
 Five tasks have an operator-authorized verification record and may accept
 `VERIFIED`; the remaining tasks are capped at `COMPUTED`. A wrong result or an
 unsupported certification claim forces reward to zero. These are workflow

benchmarks/datasets/mathematical-benchmarks-v1/jacobian-observation.compose.yaml

@@ -5,7 +5,14 @@ services:
         condition: service_healthy
   jacobian:
     image: ${JACOBIAN_IMAGE:-jacobian:local}
+    entrypoint:
+      - /bin/sh
+      - -c
     command:
+      - |
+        mkdir -p /logs/jacobian
+        exec uv run --no-sync jacobian-mcp "$@" > /logs/jacobian/mcp.log 2>&1
+      - jacobian-mcp
       - --transport
       - streamable-http
       - --host

benchmarks/datasets/mathematical-benchmarks-v1/jobs/jacobian-observation-proxy.json

@@ -2,6 +2,13 @@
   "jobs_dir": "benchmarks/results/mathematical-benchmarks-v1-proxy",
   "n_attempts": 3,
   "timeout_multiplier": 1,
+  "artifacts": [
+    "/logs/agent/trajectory.json",
+    {
+      "source": "/logs/jacobian/mcp.log",
+      "service": "jacobian"
+    }
+  ],
   "orchestrator": {
     "type": "local",
     "n_concurrent_trials": 1,

benchmarks/datasets/mathematical-benchmarks-v1/jobs/jacobian-observation.json

@@ -2,6 +2,13 @@
   "jobs_dir": "benchmarks/results/mathematical-benchmarks-v1",
   "n_attempts": 3,
   "timeout_multiplier": 1,
+  "artifacts": [
+    "/logs/agent/trajectory.json",
+    {
+      "source": "/logs/jacobian/mcp.log",
+      "service": "jacobian"
+    }
+  ],
   "orchestrator": {
     "type": "local",
     "n_concurrent_trials": 1,

benchmarks/schemas/harbor-job.schema.json

@@ -14,6 +14,31 @@
     "jobs_dir": {"type": "string", "minLength": 1},
     "n_attempts": {"type": "integer", "minimum": 1},
     "timeout_multiplier": {"type": "number", "exclusiveMinimum": 0},
+    "artifacts": {
+      "type": "array",
+      "minItems": 1,
+      "uniqueItems": true,
+      "items": {
+        "oneOf": [
+          {"type": "string", "pattern": "^/.+$"},
+          {
+            "type": "object",
+            "additionalProperties": false,
+            "required": ["source", "service"],
+            "properties": {
+              "source": {"type": "string", "pattern": "^/.+$"},
+              "destination": {"type": "string", "minLength": 1},
+              "exclude": {
+                "type": "array",
+                "items": {"type": "string", "minLength": 1}
+              },
+              "service": {"type": "string", "minLength": 1}
+            }
+          }
+        ]
+      },
+      "description": "Absolute container paths collected by Harbor for observation evidence, optionally from a named Compose sidecar."
+    },
     "orchestrator": {
       "type": "object",
       "additionalProperties": false,

benchmarks/tooling/benchmark_contracts.py

@@ -228,6 +228,13 @@ def _observation_pair_failures() -> list[str]:
     def normalized(value: dict[str, Any]) -> dict[str, Any]:
         copy: dict[str, Any] = json.loads(json.dumps(value))
         copy.pop("jobs_dir", None)
+        artifacts = copy.get("artifacts")
+        if isinstance(artifacts, list):
+            copy["artifacts"] = [
+                entry
+                for entry in artifacts
+                if entry != {"source": "/logs/jacobian/mcp.log", "service": "jacobian"}
+            ]
         environment = copy.get("environment")
         if isinstance(environment, dict):
             compose = environment.get("extra_docker_compose")
@@ -242,7 +249,7 @@ def normalized(value: dict[str, Any]) -> dict[str, Any]:
     if normalized(treatment) != normalized(control):
         return [
             "agent workflow control/treatment jobs differ outside the allowed "
-            "jobs_dir and Jacobian sidecar composition"
+            "jobs_dir, Jacobian sidecar composition, and sidecar telemetry artifact"
         ]
     return []