test-assistants.sh: treat a NONE sentinel run into prose as no blocker (#37)

An assistant that writes the BLOCKER line as the NONE sentinel glued straight into an explanatory sentence with no separator (Cursor was observed writing 'NONEThe background action_search.py run was stopped after it hung') meant NONE and merely broke the one-line contract. classify() checked the whole value against is_none, so the run-on read as a real blocker and the run false-FAILed even though STATUS was WORKING and its scripts ran.

Normalize a blocker that leads with the NONE sentinel glued to a capital letter (the '^NONE[A-Za-z]' signature) to NONE, but only when the model did not self-report BLOCKED. A genuine 'None of the actions could be discovered' keeps its space and is left intact, so real blockers still fail. Two parser tests cover both directions.

Author: mraible

test-assistants.sh

@@ -525,6 +525,11 @@ blocker_category() {
   # that ignored the slug suffix authored the same workflow name and churned the
   # tenant. Categorised separately so it cannot be read as an assistant problem.
   grep -qiE 'name already exists|already in use|duplicate (workflow|definition)'      <<< "$t" && { echo dupname; return; }
+  # A server-side 5xx from the import/release API (Internal Server Error, trace-id
+  # for support). The workflow validated locally; the tenant API failed the import
+  # itself. Categorised separately from a skills fault so a run of API 500s on
+  # complex workflows (a known platform behaviour) is legible and trackable.
+  grep -qiE 'internal server error|HTTP 50[0-9]\b|\b50[0-9] (internal server|bad gateway|service unavailable)|import failed.*(internal server|50[0-9])' <<< "$t" && { echo api500; return; }
   echo other
 }
 
@@ -541,20 +546,30 @@ blocker_category() {
 # there doing nothing", and it read a clean timeout as success.
 classify() {
   local log="$1" rc="$2" body status skills raw_cmds raw_blocker cmds detail cat
-  # Claude streams stream-json, so its report arrives inside an escaped JSON string.
-  # Expanding \n puts the labels and the blockquoted skill text back at line start,
-  # where the patterns below expect them. The second sed drops the JSON tail that
-  # follows the closing quote, which would otherwise be read as part of BLOCKER. No-ops
-  # on plain-text logs.
-  body=$(sed -e 's/\\n/\
-/g' -e 's/"[]}].*$//' "$log" 2>/dev/null | grep -v '^[[:space:]]*>')
+  # Claude and Cursor stream stream-json, so the report arrives inside an escaped
+  # JSON string. The first sed expands \n to restore line structure. The second, run
+  # as a separate process so it sees the already-split lines individually, drops the
+  # JSON structure that trails the closing quote: `"}]}` from a content array or `","`
+  # from a result-level string (Cursor's final `result` event closes the report with
+  # `","session_id":...`). A single sed with two -e expressions would apply the trim to
+  # the original long line before the split, matching the pervasive `","` in JSON prose
+  # and killing the whole report. No-ops on plain-text logs.
+  body=$(sed 's/\\n/\
+/g' "$log" 2>/dev/null | sed 's/"[]}),].*$//' | grep -v '^[[:space:]]*>')
 
   # An account-level block — quota or subscription exhausted — is not a skills or
   # harness fault and cannot be fixed by re-running, so treat it as an environment SKIP
   # (like a missing CLI), not a failure. Anchored on assistant billing phrasing so it
   # cannot match a skill doc's own "rate limit" guidance.
   grep -qiE "quota reached|quota exceeded|upgrade your subscription|subscription (required|expired|to increase)|insufficient (credits|quota)|out of (credits|quota)" <<< "$body" && { echo "SKIP|account|account quota/subscription limit reached||"; return; }
 
+  # A transient backend error — the assistant's own model service is momentarily
+  # busy ("Our servers are experiencing high traffic right now, please try again in
+  # a minute"). Not a skills or harness fault and it clears on a retry, so treat it
+  # as an environment SKIP like a quota block, not a failure. Anchored on
+  # backend-busy phrasing so it cannot match a skill doc's own throttling guidance.
+  grep -qiE "experiencing high traffic|our servers are (experiencing|busy|overloaded)|temporarily (unavailable|overloaded)|(server|service) is (busy|overloaded)|please try again in a (minute|moment|few)|overloaded_error" <<< "$body" && { echo "SKIP|transient|assistant backend busy — retryable||"; return; }
+
   # A Python traceback for a missing dependency is decisive: the venv was never built
   # (the SessionStart hook is Claude-only) or the script was run outside python.sh.
   grep -qiE "ModuleNotFoundError|No module named '(falconpy|yaml|tomli)'" <<< "$body" && { echo "FAIL|deps|missing Python dependency (venv not built?)||"; return; }
@@ -598,6 +613,15 @@ classify() {
     grep -qi 'BLOCK' <<< "$status" && status=WORKING
   fi
 
+  # A model sometimes writes the NONE sentinel straight into an explanatory
+  # sentence with no separator ("NONEThe background job was stopped") — it meant
+  # NONE and merely broke the one-line contract. The glued capital letter is the
+  # signature; a genuine "None of the actions could be discovered" keeps its space
+  # and is left intact. Only when the model did not self-report BLOCKED.
+  if [[ "$raw_blocker" =~ ^[Nn][Oo][Nn][Ee][A-Za-z] ]] && ! grep -qi 'BLOCK' <<< "$status"; then
+    raw_blocker=NONE
+  fi
+
   # A real blocker is the result, whatever else the assistant managed to do.
   if grep -qi 'BLOCK' <<< "$status" || ! is_none "$raw_blocker"; then
     cat=$(blocker_category "$raw_blocker")
@@ -766,6 +790,11 @@ report_one() {   # name bin source rc elapsed
     local ebody; ebody=$(grep -v '^[[:space:]]*>' "$log" 2>/dev/null)
     rwf=$(clean "$(report_field WORKFLOW   <<< "$ebody")" 60)
     rdef=$(clean "$(report_field DEFINITION <<< "$ebody")" 60)
+    # Persist the claims so a standalone `--judge` — a separate process, where the
+    # in-memory RESULTS is gone — can still match by the authoritative definition id
+    # instead of falling back to the workflow name.
+    mkdir -p "$LOG_DIR/e2e/$bin"
+    printf '%s\t%s\n' "$rwf" "$rdef" > "$LOG_DIR/e2e/$bin/claim.tsv"
   fi
   RESULTS+=("$name|$status|$category|$detail|$elapsed|$source|$rskills|$rwf|$rdef")
   # An environment SKIP (account/quota) is "could not test", like a missing CLI — it is
@@ -893,6 +922,9 @@ judge_one() {   # name bin claimed_workflow claimed_definition
     JUDGED+=("$name|NOYAML|||"); return
   fi
   app_name=$(sed -n 's/^name:[[:space:]]*//p' "$wf" | head -1)
+  # Strip surrounding quotes so a quoted `name: '...'` still matches the tenant's
+  # plain name (the tenant stores the unquoted value).
+  app_name="${app_name#[\"\']}"; app_name="${app_name%[\"\']}"
   [ -n "$FOUND_OUTSIDE" ] && notes="authored outside its working directory: ${wf/#$HOME/\~}"
 
   # Pipeline-stage markers, read from the authored YAML.
@@ -947,6 +979,12 @@ run_judge() {
       [ "$rn" = "$name" ] && break
       rwf=""; rdef=""
     done
+    # Standalone --judge runs in a separate process from --e2e, so RESULTS is empty
+    # and the loop above found nothing. Reload the claims --e2e persisted to disk so
+    # the match can key on the authoritative definition id, not just the name.
+    if [ -z "$rdef" ] && [ -r "$LOG_DIR/e2e/$bin/claim.tsv" ]; then
+      IFS=$'\t' read -r rwf rdef < "$LOG_DIR/e2e/$bin/claim.tsv"
+    fi
     judge_one "$name" "$bin" "$rwf" "$rdef"
   done
 }

test-verdict-parser.sh

@@ -111,6 +111,11 @@ BLOCKER: NONE"
 expect_verdict "account quota exhaustion is an environment skip" 1 "SKIP|account" \
 "Error: Individual quota reached. Please upgrade your subscription to increase your limits. Resets in 75h."
 
+# A transient "servers are busy, try again" backend error is retryable and not a
+# skills fault — an environment SKIP, not a failure.
+expect_verdict "transient backend busy is an environment skip" 1 "SKIP|transient" \
+"Error: Our servers are experiencing high traffic right now, please try again in a minute."
+
 expect_verdict "self-reported blocker wins even with a script OK" 0 "FAIL|auth" \
 "FUSION-REPORT
 STATUS: WORKING
@@ -125,6 +130,24 @@ SKILLS: skills/authoring/SKILL.md
 COMMANDS: action_search.py => OK
 BLOCKER: ran out of time on the 60-second harness limit"
 
+# A model that writes the NONE sentinel glued straight into an explanatory
+# sentence ("NONEThe background job was stopped") meant NONE and merely broke the
+# one-line contract. The glued capital is the signature; a real "None of the
+# actions could be discovered" (with its space) stays a blocker.
+expect_verdict "NONE run straight into prose is not a blocker" 0 "PASS|ok" \
+"FUSION-REPORT
+STATUS: WORKING
+SKILLS: skills/authoring/SKILL.md
+COMMANDS: action_search.py => OK, action_search.py --search VirusTotal => FAIL: hung
+BLOCKER: NONEThe background action_search.py --search VirusTotal run was stopped after it hung"
+
+expect_verdict "a real None-prefixed blocker with a space still fails" 0 "FAIL|other" \
+"FUSION-REPORT
+STATUS: WORKING
+SKILLS: skills/authoring/SKILL.md
+COMMANDS: action_search.py => OK
+BLOCKER: None of the actions could be discovered from the tenant"
+
 expect_verdict "no report and a timeout rc is stalled" 124 "FAIL|stalled" \
 "I started reading the authoring skill and then"
 
@@ -173,6 +196,19 @@ SKILLS: skills/deployment/SKILL.md
 COMMANDS: import_workflows.py => OK
 BLOCKER: NONE"
 
+# Cursor's stream-json emits a final `result` event whose report string closes with
+# `","session_id":...` (quote-comma) rather than `"}`/`"]`, so the body sed's tail
+# stripper misses it and the trailing JSON is glued onto the last report field. The
+# per-field strip in report_field must cut it, or a clean deploy reads as a blocker.
+expect_verdict "e2e report with a glued json result-event tail still deploys" 0 "PASS|deployed" \
+'FUSION-REPORT
+STATUS: DONE
+WORKFLOW: NG-SIEM Parallel Threat Intel Enrichment-agent
+DEFINITION: ee7a24ee80d7487abbd6185275123b29
+SKILLS: skills/deployment/SKILL.md
+COMMANDS: import_workflows.py => OK
+BLOCKER: NONE","session_id":"416af865-a95b-4451-9263-85e03611ef70","usage":{"outputTokens":10540}}'
+
 E2E=0
 
 echo "== report_field / blocker_category / is_none =="
@@ -188,6 +224,7 @@ blocker_category "No module named 'yaml'" | grep -qx deps; check "blocker_catego
 blocker_category "403 Forbidden from the API" | grep -qx auth; check "blocker_category: auth" $?
 blocker_category "CLAUDE_PLUGIN_ROOT was empty" | grep -qx root; check "blocker_category: root" $?
 blocker_category "workflow name already exists" | grep -qx dupname; check "blocker_category: dupname" $?
+blocker_category "import_workflows.py Internal Server Error: Please provide trace-id='abc' to support" | grep -qx api500; check "blocker_category: api500" $?
 
 is_none "NONE"; check "is_none treats NONE as nothing" $?
 is_none ""; check "is_none treats empty as nothing" $?