Skip to content

Commit bbf0a3e

Browse files
yacosta738yacosta738
authored
test(skills): add Phase 1 CLI acceptance harness
1 parent 97df9d2 commit bbf0a3e

4 files changed

Lines changed: 372 additions & 1 deletion

File tree

‎Makefile‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,8 @@ JS_WORKSPACE := $(PNPM) --filter agentsync
1111

1212
.PHONY: help all install js-install js-test js-build \
1313
rust-build rust-test rust-run e2e-test fmt docs-dev docs-build docs-preview \
14-
agents-sync agents-sync-clean clean verify-all check-all check-rust check-js check-docs check-e2e
14+
agents-sync agents-sync-clean clean verify-all check-all check-rust check-js check-docs \
15+
check-e2e acceptance-phase1
1516

1617
help:
1718
@echo "Makefile for agentsync"
@@ -38,6 +39,7 @@ help:
3839
@echo "Examples:"
3940
@echo " make js-test"
4041
@echo " make rust-test"
42+
@echo " make acceptance-phase1 AGENTSYNC_BIN=target/release/agentsync"
4143

4244
all: install js-build
4345

@@ -116,6 +118,13 @@ e2e-test:
116118
docker compose -f tests/e2e/docker-compose.yml down --volumes --remove-orphans; \
117119
exit $$status
118120

121+
# Narrow black-box acceptance for the Phase 1 catalog migration. The script requires an external
122+
# release-like binary and copies only the three approved skill directories into temporary fixtures.
123+
acceptance-phase1:
124+
@AGENTSYNC_BIN="$(AGENTSYNC_BIN)" \
125+
AGENTSYNC_SOURCE_REPO="$(AGENTSYNC_SOURCE_REPO)" \
126+
tests/acceptance/phase1_catalog.sh
127+
119128
# Formatting
120129
fmt:
121130
@echo "Formatting Rust + JS..."

‎tests/acceptance/phase1_catalog.sh‎

Lines changed: 319 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,319 @@
1+
#!/usr/bin/env bash
2+
set -euo pipefail
3+
4+
# Reproducible Phase 1 black-box acceptance target.
5+
#
6+
# Run against an already-built external executable:
7+
# cargo build --release
8+
# AGENTSYNC_BIN=target/release/agentsync \
9+
# AGENTSYNC_SOURCE_REPO=../agents-skills \
10+
# tests/acceptance/phase1_catalog.sh
11+
#
12+
# The harness copies only the three approved Phase 1 skill directories into temporary
13+
# source fixtures. It never calls Rust modules directly and never changes either checkout.
14+
15+
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
16+
REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
17+
CALLER_DIR="$PWD"
18+
AGENTSYNC_BIN="${AGENTSYNC_BIN:-$REPO_ROOT/target/release/agentsync}"
19+
SOURCE_REPO="${AGENTSYNC_SOURCE_REPO:-$REPO_ROOT/../agents-skills}"
20+
PHASE1_SKILLS=(drizzle-orm pydantic sqlalchemy)
21+
22+
if [[ "$AGENTSYNC_BIN" != /* ]]; then
23+
AGENTSYNC_BIN="$CALLER_DIR/$AGENTSYNC_BIN"
24+
fi
25+
26+
fail() {
27+
printf '❌ %s\n' "$*" >&2
28+
exit 1
29+
}
30+
31+
log_step() {
32+
printf '• %s\n' "$*"
33+
}
34+
35+
if [ ! -x "$AGENTSYNC_BIN" ]; then
36+
printf 'AGENTSYNC_BIN target does not exist or is not executable: %s\n' "$AGENTSYNC_BIN" >&2
37+
printf 'Build a release-like target first with: cargo build --release\n' >&2
38+
exit 1
39+
fi
40+
41+
if [ ! -d "$SOURCE_REPO/skills" ]; then
42+
fail "AGENTSYNC_SOURCE_REPO has no skills directory: $SOURCE_REPO"
43+
fi
44+
45+
for skill_id in "${PHASE1_SKILLS[@]}"; do
46+
[ -d "$SOURCE_REPO/skills/$skill_id" ] ||
47+
fail "AGENTSYNC_SOURCE_REPO is missing migrated skill: skills/$skill_id"
48+
done
49+
50+
TMP_ROOT="$(mktemp -d "${TMPDIR:-/tmp}/agentsync-phase1-acceptance.XXXXXX")"
51+
trap 'rm -rf "$TMP_ROOT"' EXIT
52+
53+
COMMAND_CWD="$TMP_ROOT/command-cwd"
54+
SIBLING_LAYOUT="$TMP_ROOT/sibling-layout"
55+
SIBLING_REPO="$SIBLING_LAYOUT/agents-skills"
56+
OVERRIDE_REPO="$TMP_ROOT/override-repo"
57+
EMPTY_REPO="$TMP_ROOT/empty-repo"
58+
mkdir -p "$COMMAND_CWD" "$SIBLING_REPO/skills" "$OVERRIDE_REPO/skills" "$EMPTY_REPO/skills"
59+
60+
copy_phase1_skills() {
61+
local destination="$1"
62+
local skill_id
63+
64+
for skill_id in "${PHASE1_SKILLS[@]}"; do
65+
cp -R "$SOURCE_REPO/skills/$skill_id" "$destination/skills/"
66+
done
67+
}
68+
69+
copy_phase1_skills "$SIBLING_REPO"
70+
copy_phase1_skills "$OVERRIDE_REPO"
71+
72+
for skill_id in "${PHASE1_SKILLS[@]}"; do
73+
printf '\n<!-- phase1 acceptance sibling source: %s -->\n' "$skill_id" \
74+
>> "$SIBLING_REPO/skills/$skill_id/SKILL.md"
75+
done
76+
printf '\n<!-- phase1 acceptance override source: drizzle-orm -->\n' \
77+
>> "$OVERRIDE_REPO/skills/drizzle-orm/SKILL.md"
78+
79+
DIRECT_PROJECT="$SIBLING_LAYOUT/direct-project"
80+
OVERRIDE_PROJECT="$SIBLING_LAYOUT/override-project"
81+
SUGGEST_PROJECT="$SIBLING_LAYOUT/suggest-project"
82+
MISSING_PROJECT="$TMP_ROOT/missing-project"
83+
mkdir -p "$DIRECT_PROJECT" "$OVERRIDE_PROJECT" "$SUGGEST_PROJECT" "$MISSING_PROJECT"
84+
85+
run_cli() {
86+
local project_root="$1"
87+
local stdout_path="$2"
88+
local stderr_path="$3"
89+
shift 3
90+
91+
(
92+
cd "$COMMAND_CWD"
93+
unset AGENTSYNC_LOCAL_SKILLS_REPO AGENTSYNC_TEST_SKILL_SOURCE_DIR
94+
export AGENTSYNC_NO_UPDATE_CHECK=1
95+
export HOME="$TMP_ROOT/home"
96+
export RUST_LOG=off
97+
"$AGENTSYNC_BIN" --log-level warn skill --project-root "$project_root" "$@"
98+
) >"$stdout_path" 2>"$stderr_path"
99+
}
100+
101+
run_cli_with_source() {
102+
local source_root="$1"
103+
local project_root="$2"
104+
local stdout_path="$3"
105+
local stderr_path="$4"
106+
shift 4
107+
108+
(
109+
cd "$COMMAND_CWD"
110+
unset AGENTSYNC_TEST_SKILL_SOURCE_DIR
111+
export AGENTSYNC_LOCAL_SKILLS_REPO="$source_root"
112+
export AGENTSYNC_NO_UPDATE_CHECK=1
113+
export HOME="$TMP_ROOT/home"
114+
export RUST_LOG=off
115+
"$AGENTSYNC_BIN" --log-level warn skill --project-root "$project_root" "$@"
116+
) >"$stdout_path" 2>"$stderr_path"
117+
}
118+
119+
assert_file() {
120+
local path="$1"
121+
[ -f "$path" ] || fail "expected file to exist: $path"
122+
}
123+
124+
assert_absent() {
125+
local path="$1"
126+
[ ! -e "$path" ] || fail "expected path to be absent: $path"
127+
}
128+
129+
assert_contains() {
130+
local path="$1"
131+
local expected="$2"
132+
133+
python3 - "$path" "$expected" <<'PY'
134+
import pathlib
135+
import sys
136+
137+
path = pathlib.Path(sys.argv[1])
138+
expected = sys.argv[2]
139+
if expected not in path.read_text():
140+
raise SystemExit(f"expected {expected!r} in {path}")
141+
PY
142+
}
143+
144+
assert_not_contains() {
145+
local path="$1"
146+
local unexpected="$2"
147+
148+
python3 - "$path" "$unexpected" <<'PY'
149+
import pathlib
150+
import sys
151+
152+
path = pathlib.Path(sys.argv[1])
153+
unexpected = sys.argv[2]
154+
if unexpected in path.read_text():
155+
raise SystemExit(f"did not expect {unexpected!r} in {path}")
156+
PY
157+
}
158+
159+
assert_json_status() {
160+
local path="$1"
161+
local expected_status="$2"
162+
local skill_id="$3"
163+
164+
python3 - "$path" "$expected_status" "$skill_id" <<'PY'
165+
import json
166+
import pathlib
167+
import sys
168+
169+
path = pathlib.Path(sys.argv[1])
170+
expected_status = sys.argv[2]
171+
skill_id = sys.argv[3]
172+
payload = json.loads(path.read_text())
173+
actual_status = payload.get("status")
174+
if actual_status != expected_status:
175+
raise SystemExit(
176+
f"{skill_id}: expected JSON status {expected_status!r}, got {actual_status!r}"
177+
)
178+
PY
179+
}
180+
181+
assert_registry_entry() {
182+
local project_root="$1"
183+
local skill_id="$2"
184+
local registry="$project_root/.agents/skills/registry.json"
185+
186+
assert_file "$registry"
187+
python3 - "$registry" "$skill_id" <<'PY'
188+
import json
189+
import pathlib
190+
import sys
191+
192+
registry = pathlib.Path(sys.argv[1])
193+
skill_id = sys.argv[2]
194+
payload = json.loads(registry.read_text())
195+
skills = payload.get("skills", {})
196+
if skill_id not in skills:
197+
raise SystemExit(f"registry {registry} is missing local key {skill_id!r}")
198+
PY
199+
}
200+
201+
log_step "Installing all three migrated skills through the external CLI from sibling fixtures"
202+
for skill_id in "${PHASE1_SKILLS[@]}"; do
203+
stdout_path="$TMP_ROOT/direct-$skill_id.stdout"
204+
stderr_path="$TMP_ROOT/direct-$skill_id.stderr"
205+
run_cli "$DIRECT_PROJECT" "$stdout_path" "$stderr_path" install "$skill_id" --json
206+
assert_json_status "$stdout_path" installed "$skill_id"
207+
assert_file "$DIRECT_PROJECT/.agents/skills/$skill_id/SKILL.md"
208+
assert_registry_entry "$DIRECT_PROJECT" "$skill_id"
209+
assert_contains \
210+
"$DIRECT_PROJECT/.agents/skills/$skill_id/SKILL.md" \
211+
"phase1 acceptance sibling source: $skill_id"
212+
done
213+
214+
assert_file "$DIRECT_PROJECT/.agents/skills/drizzle-orm/references/advanced-schemas.md"
215+
assert_file "$DIRECT_PROJECT/.agents/skills/drizzle-orm/references/performance.md"
216+
assert_file "$DIRECT_PROJECT/.agents/skills/drizzle-orm/references/query-patterns.md"
217+
assert_file "$DIRECT_PROJECT/.agents/skills/drizzle-orm/references/vs-prisma.md"
218+
assert_file "$DIRECT_PROJECT/.agents/skills/pydantic/references/full-source.md"
219+
assert_file "$DIRECT_PROJECT/.agents/skills/sqlalchemy/references/full-source.md"
220+
assert_file \
221+
"$DIRECT_PROJECT/.agents/skills/sqlalchemy/references/sql-quality-antipatterns.md"
222+
223+
log_step "Checking AGENTSYNC_LOCAL_SKILLS_REPO override precedence"
224+
override_stdout="$TMP_ROOT/override.stdout"
225+
override_stderr="$TMP_ROOT/override.stderr"
226+
run_cli_with_source \
227+
"$OVERRIDE_REPO" \
228+
"$OVERRIDE_PROJECT" \
229+
"$override_stdout" \
230+
"$override_stderr" \
231+
install drizzle-orm --json
232+
assert_json_status "$override_stdout" installed drizzle-orm
233+
assert_contains \
234+
"$OVERRIDE_PROJECT/.agents/skills/drizzle-orm/SKILL.md" \
235+
"phase1 acceptance override source: drizzle-orm"
236+
assert_not_contains \
237+
"$OVERRIDE_PROJECT/.agents/skills/drizzle-orm/SKILL.md" \
238+
"phase1 acceptance sibling source: drizzle-orm"
239+
240+
log_step "Checking missing curated content fails closed without external fallback"
241+
missing_stdout="$TMP_ROOT/missing.stdout"
242+
missing_stderr="$TMP_ROOT/missing.stderr"
243+
if run_cli_with_source \
244+
"$EMPTY_REPO" \
245+
"$MISSING_PROJECT" \
246+
"$missing_stdout" \
247+
"$missing_stderr" \
248+
install pydantic; then
249+
fail "missing curated pydantic source unexpectedly installed"
250+
fi
251+
assert_contains "$missing_stderr" "refusing external fallback"
252+
assert_absent "$MISSING_PROJECT/.agents/skills/pydantic"
253+
254+
log_step "Checking suggestion install propagates the supplied project root"
255+
cat > "$SUGGEST_PROJECT/pyproject.toml" <<'EOF'
256+
[project]
257+
name = "phase1-suggestion-fixture"
258+
version = "0.0.0"
259+
dependencies = ["pydantic>=2"]
260+
EOF
261+
262+
# The Python detector also recommends three generic Python skills. Mark those paths as already
263+
# installed so the black-box install selects only the migrated pydantic source; no unrelated source
264+
# fixture or external resolution is used by this acceptance path.
265+
mkdir -p \
266+
"$SUGGEST_PROJECT/.agents/skills/best-practices" \
267+
"$SUGGEST_PROJECT/.agents/skills/python-executor" \
268+
"$SUGGEST_PROJECT/.agents/skills/python-testing-patterns"
269+
270+
suggest_stdout="$TMP_ROOT/suggest.stdout"
271+
suggest_stderr="$TMP_ROOT/suggest.stderr"
272+
run_cli \
273+
"$SUGGEST_PROJECT" \
274+
"$suggest_stdout" \
275+
"$suggest_stderr" \
276+
suggest --install --all --json
277+
278+
python3 - "$suggest_stdout" <<'PY'
279+
import json
280+
import pathlib
281+
import sys
282+
283+
payload = json.loads(pathlib.Path(sys.argv[1]).read_text())
284+
recommendation = next(
285+
(item for item in payload["recommendations"] if item["skill_id"] == "pydantic"),
286+
None,
287+
)
288+
if recommendation is None:
289+
raise SystemExit("suggestion output did not include pydantic")
290+
if recommendation["provider_skill_id"] != "dallay/agents-skills/pydantic":
291+
raise SystemExit("pydantic suggestion lost its qualified provider identity")
292+
293+
allowed = {
294+
"best-practices",
295+
"python-executor",
296+
"python-testing-patterns",
297+
"pydantic",
298+
}
299+
results = payload["results"]
300+
unexpected = {item["skill_id"] for item in results} - allowed
301+
if unexpected:
302+
raise SystemExit(f"suggestion acceptance attempted unrelated skills: {sorted(unexpected)}")
303+
304+
pydantic_result = next(item for item in results if item["skill_id"] == "pydantic")
305+
if pydantic_result["status"] != "installed":
306+
raise SystemExit(f"pydantic suggestion did not install: {pydantic_result}")
307+
if any(item["status"] == "failed" for item in results):
308+
raise SystemExit(f"suggestion acceptance reported a failed install: {results}")
309+
PY
310+
311+
assert_file "$SUGGEST_PROJECT/.agents/skills/pydantic/SKILL.md"
312+
assert_file "$SUGGEST_PROJECT/.agents/skills/pydantic/references/full-source.md"
313+
assert_registry_entry "$SUGGEST_PROJECT" pydantic
314+
assert_contains \
315+
"$SUGGEST_PROJECT/.agents/skills/pydantic/SKILL.md" \
316+
"phase1 acceptance sibling source: pydantic"
317+
assert_absent "$COMMAND_CWD/.agents/skills/pydantic"
318+
319+
printf '✅ Phase 1 black-box acceptance passed against %s\n' "$AGENTSYNC_BIN"

‎tests/acceptance/test_phase1_catalog_harness.sh‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
#!/usr/bin/env bash
2+
set -euo pipefail
3+
4+
REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
5+
HARNESS="$REPO_ROOT/tests/acceptance/phase1_catalog.sh"
6+
7+
if [ ! -x "$HARNESS" ]; then
8+
printf 'phase1 acceptance harness is missing or not executable: %s\n' "$HARNESS" >&2
9+
exit 1
10+
fi
11+
12+
set +e
13+
output=$(AGENTSYNC_BIN="$REPO_ROOT/target/does-not-exist" "$HARNESS" 2>&1)
14+
status=$?
15+
set -e
16+
17+
if [ "$status" -eq 0 ]; then
18+
printf 'expected the acceptance harness to reject a missing binary\n' >&2
19+
exit 1
20+
fi
21+
22+
case "$output" in
23+
*"AGENTSYNC_BIN target does not exist or is not executable"*)
24+
printf 'phase1 acceptance harness missing-binary contract passed\n'
25+
;;
26+
*)
27+
printf 'unexpected missing-binary failure:\n%s\n' "$output" >&2
28+
exit 1
29+
;;
30+
esac

‎tests/acceptance/test_phase1_provenance.sh‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
#!/usr/bin/env bash
2+
set -euo pipefail
3+
4+
REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
5+
SOURCE_REPO="${AGENTSYNC_SOURCE_REPO:-$REPO_ROOT/../agents-skills}"
6+
VALIDATOR="$SOURCE_REPO/scripts/validate_provenance.py"
7+
8+
if [ ! -f "$VALIDATOR" ]; then
9+
printf 'provenance validator is missing: %s\n' "$VALIDATOR" >&2
10+
exit 1
11+
fi
12+
13+
python3 "$VALIDATOR" --root "$SOURCE_REPO"

0 commit comments

Comments
 (0)