Merge remote-tracking branch 'origin/main' into issue5826/empty-opena… #13

Workflow file for this run

.github/workflows/npx-server-smoke.yml at 058bedb

	# Smoke job for `npx @langwatch/server`. Builds the workspace package as a
	# tarball, then runs it from a fresh sandbox dir on every supported platform.
	# Asserts the full stack (postgres + redis + clickhouse + nlpgo + langevals
	# + ai-gateway + langwatch app) reaches healthy state and that the critical
	# user paths — workflow execution, evaluator run — return 200.
	#
	# Trigger policy:
	# - workflow_dispatch (manual, with optional `port_base` input)
	# - nightly schedule (04:00 UTC)
	# - push to non-main branches when key files change (PR validation)
	#
	# This workflow is intentionally not in the langwatch-app-complete required
	# checks — postgres/clickhouse/uv downloads are heavy and flaky-prone. Treat
	# failures as signal, not gate, until the workflow has hardened.

	name: npx-server-smoke

	on:
	workflow_dispatch:
	inputs:
	port_base:
	description: "Port base to test (default 5560)"
	required: false
	default: "5560"
	schedule:
	- cron: "0 4 * * *"
	push:
	branches-ignore:
	- main
	paths:
	- "package.json"
	- "pnpm-workspace.yaml"
	- "pnpm-lock.yaml"
	- "packages/server/**"
	- "langevals/pyproject.toml"
	- "langevals/uv.lock"
	- "services/aigateway/**"
	- "cmd/service/**"
	- "langwatch/package.json"
	- "langwatch/scripts/start.sh"
	- "langwatch/scripts/check-ports.sh"
	- ".github/workflows/npx-server-smoke.yml"

	concurrency:
	group: ${{ github.workflow }}-${{ github.ref }}
	cancel-in-progress: true

	permissions:
	contents: read

	jobs:
	smoke:
	strategy:
	fail-fast: false
	matrix:
	runner:
	- macos-latest
	- ubuntu-latest
	- ubuntu-24.04-arm
	runs-on: ${{ matrix.runner }}
	timeout-minutes: 35
	env:
	LANGWATCH_HOME: ${{ github.workspace }}/.langwatch-smoke
	PORT_BASE: ${{ inputs.port_base \|\| '5560' }}
	CI: "true"

	steps:
	- name: Checkout
	uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v6

	- name: Identify runner
	run: \|
	uname -a
	node --version \|\| true
	go version \|\| true

	- uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v4
	with:
	node-version: "24"

	- name: Setup pnpm
	uses: pnpm/action-setup@0ebf47130e4866e96fce0953f49152a61190b271 # v6.0.9
	with:
	run_install: false

	# No Go setup. The npx flow ships only the prebuilt aigateway monobinary
	# download path; LANGWATCH_AIGATEWAY_DEV_BUILD is intentionally unset
	# so the env-gated local-build fallback can never trigger. We don't
	# try to remove Go from PATH (every GitHub-hosted runner ships it
	# baked in) — the env-var gate plus the binary-identity assertions
	# below (version match + stripped check) are what catch a regression.

	# Install ONLY tools that aren't predeps but the CLI needs to operate
	# (lsof for port-collision detection on linux). Postgres, redis,
	# clickhouse, uv, and goose all flow through their respective predeps,
	# which is the actual user experience we want to validate end-to-end —
	# download from embeds.langwatch.ai (pg/redis), clickhouse.com one-liner
	# (clickhouse), astral.sh installer (uv), github releases (goose).
	- name: Install non-predep tooling
	shell: bash
	run: \|
	set -e
	if [[ "$RUNNER_OS" == "Linux" ]]; then
	sudo apt-get update
	sudo apt-get install -y --no-install-recommends lsof file
	fi

	- name: Install pnpm dependencies (workspace)
	run: pnpm install --frozen-lockfile

	- name: Build @langwatch/server CLI
	run: pnpm --filter @langwatch/server-cli build

	- name: Pack the npm tarball
	id: pack
	# Use the same wrapper as npx-server-publish.yml so the smoke tarball
	# matches what end users get from npm (Apache LICENSE swap, exact
	# files[] from package.json — no go.mod/cmd/, etc).
	run: \|
	mkdir -p "$GITHUB_WORKSPACE/_pack"
	bash scripts/pack-npm.sh --pack-destination "$GITHUB_WORKSPACE/_pack"
	tarball="$(ls "$GITHUB_WORKSPACE/_pack"/langwatch-server-*.tgz \| head -n1)"
	echo "tarball=$tarball" >> "$GITHUB_OUTPUT"
	echo "Tarball: $tarball"
	ls -lh "$tarball"
	# Belt-and-suspenders: tarball must NOT ship Go source (silent
	# local-build foot-gun from beta.8 era).
	if tar -tzf "$tarball" \| grep -qE "^package/(go\.mod\|cmd/\|pkg/\|services/)"; then
	echo "✗ tarball ships Go source dirs — local-build fallback regression."
	exit 1
	fi

	- name: Run npx @langwatch/server in a sandbox
	id: start
	env:
	OPENAI_API_KEY: ${{ secrets.NPX_SERVER_OPENAI_API_KEY }}
	run: \|
	set -e
	mkdir -p "$LANGWATCH_HOME"
	# `node packages/server/dist/cli.cjs` reproduces what `npx <tarball>` would
	# do, but doesn't depend on a working npm registry. Faster + more stable.
	node packages/server/dist/cli.cjs start \
	--yes \
	--no-open \
	--port-base "$PORT_BASE" \
	> "$GITHUB_WORKSPACE/_smoke.log" 2>&1 &
	echo "$!" > "$GITHUB_WORKSPACE/_smoke.pid"
	echo "Started CLI in background (pid $(cat $GITHUB_WORKSPACE/_smoke.pid))"

	- name: Wait for /api/health (up to 18 minutes)
	run: \|
	set -e
	base=${PORT_BASE}
	# CI runners have nothing cached — predep download (postgres ~80MB,
	# clickhouse ~100MB, redis tarball, uv installer) + uv venv install
	# for langevals + aigateway/nlpgo monobinary download + pnpm
	# install + vite build of the langwatch app together can take 8-12
	# minutes on a cold runner. Be generous so the smoke isn't flaky on
	# the initial provisioning window. Subsequent runs hit the cached
	# ~/.langwatch and finish in <1 min.
	#
	# Auto-shift is unlikely on a fresh CI runner (5560 should be free)
	# so the inner timeout is what matters; outer shift is a safety net.
	for shift in 0 10 20; do
	url="http://127.0.0.1:$((base + shift))/api/health"
	echo "==> probing $url for up to 18min"
	for i in $(seq 1 216); do
	if curl -fsS "$url" >/dev/null 2>&1; then
	echo "✓ $url is healthy after $((i*5))s"
	echo "RESOLVED_BASE=$((base + shift))" >> "$GITHUB_ENV"
	exit 0
	fi
	if [ $((i % 12)) -eq 0 ]; then
	echo " …still waiting at $((i*5))s — last cli.log lines:"
	tail -n 10 "$GITHUB_WORKSPACE/_smoke.log" 2>/dev/null \| sed 's/^/ /' \|\| true
	fi
	sleep 5
	done
	done
	echo "✗ langwatch never reported healthy on any port-shift"
	exit 1

	- name: Probe service health endpoints
	run: \|
	# pipefail catches curl exit codes when piped through tee —
	# without it, `curl … \| tee` always exits 0 because tee
	# always succeeds, masking real probe failures (the original
	# /health vs /healthz misnomer slipped past CI for weeks
	# because of this).
	set -eo pipefail
	base=${RESOLVED_BASE}
	# The Wait step only gates on langwatch /api/health (~13s on a
	# cold runner), but langevals normally finishes its fastapi boot
	# + evaluator discovery ~1-2s later (~14-15s). One-shot probes
	# raced langevals and intermittently failed on all runners.
	# Retry each endpoint for up to 60s — same pattern the CLI
	# orchestrator uses internally.
	probe() {
	local name="$1" url="$2" out="$3"
	for i in $(seq 1 60); do
	# --connect-timeout caps TCP setup so a single network hang
	# can't eat the whole 60s budget; --max-time caps the full
	# request so a slow-bodied response can't either.
	if curl -fsS --connect-timeout 2 --max-time 5 "$url" > "$out" 2>/dev/null; then
	echo "✓ $name healthy after ${i}s"
	cat "$out"; echo
	return 0
	fi
	sleep 1
	done
	echo "✗ $name never responded at $url"
	return 1
	}
	probe app "http://127.0.0.1:$base/api/health" /tmp/health-app.json
	# nlpgo only serves /healthz (chi-routed liveness). /health
	# falls through to goOnlyModeFallback OR the proxypass
	# 502-after-hang loop and is not a valid probe.
	probe nlpgo "http://127.0.0.1:$((base + 1))/healthz" /tmp/health-nlp.json
	probe langevals "http://127.0.0.1:$((base + 2))/" /tmp/health-langevals.json
	probe gateway "http://127.0.0.1:$((base + 3))/healthz" /tmp/health-gateway.json

	# The assistant has exactly two legitimate outcomes, and the CLI's own
	# log says which one happened: `langyagent` reached healthy, or it was
	# deliberately skipped because the downloaded release binary predates
	# the langyagent service (the smoke tests an unreleased CLI against the
	# PREVIOUS release's binary, so the skip is the expected path until the
	# next release publishes fresh binaries). Anything else — crash-looping,
	# or a "healthy" verdict without a listener actually answering on the
	# port — fails the smoke. The port re-probe exists because a stray
	# listener on the assistant's port once let a dead langyagent pass as
	# healthy on ubuntu runners.
	- name: Assert the assistant either booted or declared itself skipped
	run: \|
	set -eo pipefail
	base=${RESOLVED_BASE}
	log="$GITHUB_WORKSPACE/_smoke.log"
	if grep -q "langy assistant disabled" "$log"; then
	echo "✓ assistant skipped with the release-binary notice (expected until the next release's binaries exist)"
	exit 0
	fi
	if grep -qE "langyagent +healthy in" "$log"; then
	curl -fsS --connect-timeout 2 --max-time 5 "http://127.0.0.1:$((base + 4))/health" > /tmp/health-langy.json
	echo "✓ assistant healthy and answering on port $((base + 4))"
	cat /tmp/health-langy.json; echo
	exit 0
	fi
	echo "✗ cli.log shows neither a healthy langyagent nor the deliberate-skip notice"
	grep -i "langyagent" "$log" \| tail -n 5 \|\| true
	exit 1

	# Belt-and-suspenders for the beta.8 → beta.9 regression: the smoke
	# matrix happily passed for weeks while aigateway was being silently
	# `go build`-ed on the runner. Two assertions to make a future
	# regression loud:
	# (1) aigateway --version returns the expected npm version (the
	# prebuilt is built with -ldflags "-X main.Version=$VERSION");
	# a stale build or local fallback would print "dev" or "unknown".
	# (2) the binary is stripped (file output ≠ "not stripped"). The
	# prebuilt uses `-ldflags "-s -w"`; an in-tree `go build` does
	# not, so this trivially distinguishes the two paths.
	- name: Assert aigateway came from the prebuilt GH release download
	run: \|
	set -e
	bin="$LANGWATCH_HOME/bin/aigateway"
	if [ ! -f "$bin" ]; then
	echo "✗ no aigateway binary at $bin — predep install must have failed silently"
	exit 1
	fi
	expected_version="$(node -p "require('$GITHUB_WORKSPACE/package.json').version")"
	actual_version="$("$bin" --version 2>&1 \| head -n1 \| tr -d '\r')"
	echo "aigateway --version: $actual_version"
	echo "expected: $expected_version"
	if [ "$actual_version" != "$expected_version" ]; then
	echo "✗ aigateway version mismatch — likely a stale or locally-built binary"
	exit 1
	fi
	file_out="$(file "$bin")"
	echo "$file_out"
	if echo "$file_out" \| grep -q "not stripped"; then
	echo "✗ aigateway binary is NOT stripped — looks like a local Go build, not the prebuilt download."
	echo " The prebuilt monobinary uploaded to v\$VERSION on GH is built with"
	echo " '-ldflags \"-s -w\"'. If you see this in CI, the download path silently"
	echo " fell through to LANGWATCH_AIGATEWAY_DEV_BUILD=1 (which should be unset)."
	exit 1
	fi
	echo "✓ aigateway is stripped + version matches — prebuilt download path confirmed"

	# Also verify the cli.log shows the download progress message and
	# NOT the local-build message. Defence in depth — a future
	# refactor that strips the download stripping flags would still
	# fail the (1) and (2) checks above, but a refactor that also
	# strips the local-build wouldn't trigger any of them. The
	# log-line check pins the runtime path itself.
	if [ -f "$GITHUB_WORKSPACE/_smoke.log" ]; then
	if grep -q "building from local checkout" "$GITHUB_WORKSPACE/_smoke.log"; then
	echo "✗ cli.log contains 'building from local checkout' — local-build path took effect."
	exit 1
	fi
	if ! grep -q "downloading langwatch ai-gateway" "$GITHUB_WORKSPACE/_smoke.log"; then
	echo "⚠ cli.log does not mention 'downloading langwatch ai-gateway' — predep may have skipped install entirely (cached?)."
	echo " This is OK for re-runs against an existing $LANGWATCH_HOME/bin, but flag for visibility."
	fi
	fi

	- name: Probe a workflow execution end-to-end
	# `secrets.*` is not available in step-level `if:` expressions
	# (only `env`, `inputs`, `github`, `runner`, etc. are valid there).
	# We expose the secret as a step-env var, then short-circuit inside
	# the script when it's empty. Net effect matches the original
	# "skip if no key" intent without invalid YAML.
	env:
	OPENAI_API_KEY: ${{ secrets.NPX_SERVER_OPENAI_API_KEY }}
	run: \|
	set -e
	if [[ -z "$OPENAI_API_KEY" ]]; then
	echo "NPX_SERVER_OPENAI_API_KEY not set — skipping workflow execution probe."
	exit 0
	fi
	# Placeholder until the real workflow-execution fixture lands.
	# We hit nlpgo's /healthz (chi-routed liveness) — the only path
	# that is actually wired. Earlier drafts hit /health and either
	# 502'd (POST) or hung 5+min (GET → proxypass self-loop).
	base=${RESOLVED_BASE}
	curl -fsS --max-time 10 "http://127.0.0.1:$((base + 1))/healthz" >/dev/null
	echo "(placeholder workflow probe — real fixture pending)"

	- name: Tear down
	if: always()
	run: \|
	if [ -f "$GITHUB_WORKSPACE/_smoke.pid" ]; then
	pid="$(cat $GITHUB_WORKSPACE/_smoke.pid)"
	echo "Killing CLI pid $pid"
	kill -TERM "$pid" 2>/dev/null \|\| true
	for i in $(seq 1 10); do
	if ! kill -0 "$pid" 2>/dev/null; then break; fi
	sleep 1
	done
	kill -KILL "$pid" 2>/dev/null \|\| true
	fi

	- name: Capture logs
	if: always()
	run: \|
	mkdir -p _logs
	cp "$GITHUB_WORKSPACE/_smoke.log" _logs/cli.log 2>/dev/null \|\| true
	if [ -d "$LANGWATCH_HOME/logs" ]; then
	cp -r "$LANGWATCH_HOME/logs" _logs/services/
	fi
	ls -la _logs/

	- name: Upload logs on failure or cancellation
	# The `failure()` filter doesn't fire on timeout-cancellation, so we
	# widen to `!success()` — that catches both genuine failures and
	# 35-minute-job-cap cancellations, which is when we most want logs.
	if: ${{ !success() }}
	uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
	with:
	name: npx-server-smoke-logs-${{ matrix.runner }}-${{ github.sha }}
	path: _logs
	retention-days: 7

	alls-green:
	if: always()
	needs: [smoke]
	runs-on: ubuntu-latest
	steps:
	- name: Decide on aggregate status
	run: \|
	# Smoke is informational until it's been running clean for a week —
	# only fail the aggregator on workflow_dispatch + scheduled runs.
	status='${{ needs.smoke.result }}'
	echo "smoke status: $status"
	if [ "${{ github.event_name }}" = "schedule" ] && [ "$status" != "success" ]; then
	exit 1
	fi
	if [ "${{ github.event_name }}" = "workflow_dispatch" ] && [ "$status" != "success" ]; then
	exit 1
	fi
	# On push, never fail — let humans decide.
	exit 0

Navigation Menu

Merge remote-tracking branch 'origin/main' into issue5826/empty-opena… #13

Workflow file

Merge remote-tracking branch 'origin/main' into issue5826/empty-opena… #13

Uh oh!

Workflow file for this run