Mutation Testing #5

Workflow file for this run

.github/workflows/mutation-testing.yml at c9bf0b1

	name: Mutation Testing

	on:
	# Run on PRs to main/develop so mutation scores are visible before merge
	pull_request:
	branches: [main, develop]
	paths:
	- 'contracts/stellar-save/src/**'
	- 'frontend/src/**'
	- '.github/workflows/mutation-testing.yml'
	- 'contracts/stellar-save/mutants.toml'
	- 'frontend/stryker.config.mjs'

	# Weekly scheduled run (Sunday 03:00 UTC) for a full baseline report
	schedule:
	- cron: '0 3 * * 0'

	# Allow manual trigger with optional scope selection
	workflow_dispatch:
	inputs:
	scope:
	description: 'Which mutation suite to run'
	required: false
	default: 'all'
	type: choice
	options:
	- all
	- contracts
	- frontend

	# Cancel in-progress runs on the same branch to save CI minutes
	concurrency:
	group: mutation-${{ github.ref }}
	cancel-in-progress: true

	jobs:
	# ─── Rust Contract Mutation Testing (cargo-mutants) ─────────────────────────
	contract-mutation:
	name: Contract Mutation Tests (cargo-mutants)
	runs-on: ubuntu-latest
	if: >
	github.event_name != 'workflow_dispatch' \|\|
	github.event.inputs.scope == 'all' \|\|
	github.event.inputs.scope == 'contracts'
	timeout-minutes: 60

	steps:
	- uses: actions/checkout@v4

	- name: Install Rust toolchain
	uses: dtolnay/rust-toolchain@stable

	- name: Cache Rust dependencies
	uses: Swatinem/rust-cache@v2
	with:
	workspaces: ". -> target"
	# Include cargo-mutants binary in the cache key
	key: mutants-${{ hashFiles('contracts/stellar-save/Cargo.lock') }}

	- name: Install cargo-mutants
	# Pin to a specific version for reproducibility
	run: cargo install cargo-mutants --version 24.11.1 --locked

	- name: Run mutation tests
	working-directory: contracts/stellar-save
	run: \|
	cargo mutants \
	--manifest-path Cargo.toml \
	--jobs 2 \
	--timeout 120 \
	--output mutants-out \
	--json \
	-- --test-threads=1
	# Continue even if mutants survive so we can upload the report
	continue-on-error: true

	- name: Parse and display mutation score
	working-directory: contracts/stellar-save
	run: \|
	if [ -f mutants-out/mutants.json ]; then
	python3 - <<'EOF'
	import json, sys

	with open("mutants-out/mutants.json") as f:
	data = json.load(f)

	total = len(data)
	caught = sum(1 for m in data if m.get("status") == "caught")
	missed = sum(1 for m in data if m.get("status") == "missed")
	timeout = sum(1 for m in data if m.get("status") == "timeout")
	unviable = sum(1 for m in data if m.get("status") == "unviable")

	tested = total - unviable
	score = (caught / tested * 100) if tested > 0 else 0.0

	print(f"=== Contract Mutation Score ===")
	print(f" Total mutants : {total}")
	print(f" Caught : {caught}")
	print(f" Missed : {missed}")
	print(f" Timeout : {timeout}")
	print(f" Unviable : {unviable}")
	print(f" Score : {score:.1f}%")

	if missed > 0:
	print("\n--- Surviving mutants (missed) ---")
	for m in data:
	if m.get("status") == "missed":
	print(f" [{m.get('file','?')}:{m.get('line','?')}] {m.get('mutation','?')}")

	# Enforce minimum threshold
	THRESHOLD = 60.0
	if score < THRESHOLD:
	print(f"\nFAIL: mutation score {score:.1f}% is below the {THRESHOLD}% threshold")
	sys.exit(1)
	else:
	print(f"\nPASS: mutation score {score:.1f}% meets the {THRESHOLD}% threshold")
	EOF
	else
	echo "mutants.json not found — cargo-mutants may have failed to produce output"
	exit 1
	fi

	- name: Upload mutation report
	uses: actions/upload-artifact@v4
	if: always()
	with:
	name: contract-mutation-report
	path: contracts/stellar-save/mutants-out/
	retention-days: 30

	- name: Comment mutation score on PR
	if: github.event_name == 'pull_request' && always()
	uses: actions/github-script@v7
	with:
	script: \|
	const fs = require('fs');
	const path = 'contracts/stellar-save/mutants-out/mutants.json';

	if (!fs.existsSync(path)) {
	console.log('No mutants.json found, skipping PR comment');
	return;
	}

	const data = JSON.parse(fs.readFileSync(path, 'utf8'));
	const total = data.length;
	const caught = data.filter(m => m.status === 'caught').length;
	const missed = data.filter(m => m.status === 'missed').length;
	const unviable = data.filter(m => m.status === 'unviable').length;
	const tested = total - unviable;
	const score = tested > 0 ? (caught / tested * 100).toFixed(1) : '0.0';

	const emoji = parseFloat(score) >= 80 ? '🟢' : parseFloat(score) >= 60 ? '🟡' : '🔴';

	let body = `## ${emoji} Contract Mutation Score: ${score}%\n\n`;
	body += `\| Metric \| Count \|\n\|--------\|-------\|\n`;
	body += `\| Total mutants \| ${total} \|\n`;
	body += `\| Caught (killed) \| ${caught} \|\n`;
	body += `\| Missed (survived) \| ${missed} \|\n`;
	body += `\| Unviable (skipped) \| ${unviable} \|\n`;
	body += `\| Score \| ${score}% \|\n\n`;

	if (missed > 0) {
	body += `<details><summary>Surviving mutants (${missed})</summary>\n\n`;
	data.filter(m => m.status === 'missed').forEach(m => {
	body += `- \`${m.file}:${m.line}\` — ${m.mutation}\n`;
	});
	body += `\n</details>\n`;
	}

	body += `\n> Full report available in the [workflow artifacts](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}).`;

	// Find and update existing comment, or create a new one
	const { data: comments } = await github.rest.issues.listComments({
	owner: context.repo.owner,
	repo: context.repo.repo,
	issue_number: context.issue.number,
	});

	const existing = comments.find(c =>
	c.user.login === 'github-actions[bot]' &&
	c.body.includes('Contract Mutation Score')
	);

	if (existing) {
	await github.rest.issues.updateComment({
	owner: context.repo.owner,
	repo: context.repo.repo,
	comment_id: existing.id,
	body,
	});
	} else {
	await github.rest.issues.createComment({
	owner: context.repo.owner,
	repo: context.repo.repo,
	issue_number: context.issue.number,
	body,
	});
	}

	# ─── Frontend Mutation Testing (Stryker) ────────────────────────────────────
	frontend-mutation:
	name: Frontend Mutation Tests (Stryker)
	runs-on: ubuntu-latest
	if: >
	github.event_name != 'workflow_dispatch' \|\|
	github.event.inputs.scope == 'all' \|\|
	github.event.inputs.scope == 'frontend'
	timeout-minutes: 45
	defaults:
	run:
	working-directory: frontend

	steps:
	- uses: actions/checkout@v4

	- name: Setup Node.js
	uses: actions/setup-node@v4
	with:
	node-version: '20'
	cache: 'npm'
	cache-dependency-path: frontend/package-lock.json

	- name: Install dependencies
	run: npm ci

	- name: Run Stryker mutation tests
	run: npm run test:mutation:ci
	# Continue so we can always upload the report
	continue-on-error: true

	- name: Parse and enforce mutation score threshold
	run: \|
	REPORT="reports/mutation/mutation.json"
	if [ ! -f "$REPORT" ]; then
	echo "mutation.json not found — Stryker may have failed"
	exit 1
	fi

	python3 - <<'EOF'
	import json, sys

	with open("reports/mutation/mutation.json") as f:
	data = json.load(f)

	metrics = data.get("metrics", {})
	score = metrics.get("mutationScore", 0.0)
	total = metrics.get("totalMutants", 0)
	killed = metrics.get("killed", 0)
	survived = metrics.get("survived", 0)
	timeout = metrics.get("timedOut", 0)
	no_cov = metrics.get("noCoverage", 0)

	print(f"=== Frontend Mutation Score ===")
	print(f" Total mutants : {total}")
	print(f" Killed : {killed}")
	print(f" Survived : {survived}")
	print(f" Timeout : {timeout}")
	print(f" No coverage : {no_cov}")
	print(f" Score : {score:.1f}%")

	THRESHOLD = 50.0
	if score < THRESHOLD:
	print(f"\nFAIL: mutation score {score:.1f}% is below the {THRESHOLD}% threshold")
	sys.exit(1)
	else:
	print(f"\nPASS: mutation score {score:.1f}% meets the {THRESHOLD}% threshold")
	EOF

	- name: Upload Stryker mutation report
	uses: actions/upload-artifact@v4
	if: always()
	with:
	name: frontend-mutation-report
	path: frontend/reports/mutation/
	retention-days: 30

	- name: Comment Stryker score on PR
	if: github.event_name == 'pull_request' && always()
	uses: actions/github-script@v7
	with:
	script: \|
	const fs = require('fs');
	const path = 'frontend/reports/mutation/mutation.json';

	if (!fs.existsSync(path)) {
	console.log('No mutation.json found, skipping PR comment');
	return;
	}

	const data = JSON.parse(fs.readFileSync(path, 'utf8'));
	const m = data.metrics \|\| {};
	const score = (m.mutationScore \|\| 0).toFixed(1);
	const total = m.totalMutants \|\| 0;
	const killed = m.killed \|\| 0;
	const survived = m.survived \|\| 0;
	const noCov = m.noCoverage \|\| 0;

	const emoji = parseFloat(score) >= 80 ? '🟢' : parseFloat(score) >= 60 ? '🟡' : '🔴';

	let body = `## ${emoji} Frontend Mutation Score: ${score}%\n\n`;
	body += `\| Metric \| Count \|\n\|--------\|-------\|\n`;
	body += `\| Total mutants \| ${total} \|\n`;
	body += `\| Killed \| ${killed} \|\n`;
	body += `\| Survived \| ${survived} \|\n`;
	body += `\| No coverage \| ${noCov} \|\n`;
	body += `\| Score \| ${score}% \|\n\n`;
	body += `> Full HTML report available in the [workflow artifacts](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}).`;

	const { data: comments } = await github.rest.issues.listComments({
	owner: context.repo.owner,
	repo: context.repo.repo,
	issue_number: context.issue.number,
	});

	const existing = comments.find(c =>
	c.user.login === 'github-actions[bot]' &&
	c.body.includes('Frontend Mutation Score')
	);

	if (existing) {
	await github.rest.issues.updateComment({
	owner: context.repo.owner,
	repo: context.repo.repo,
	comment_id: existing.id,
	body,
	});
	} else {
	await github.rest.issues.createComment({
	owner: context.repo.owner,
	repo: context.repo.repo,
	issue_number: context.issue.number,
	body,
	});
	}

	# ─── Summary gate ────────────────────────────────────────────────────────────
	mutation-summary:
	name: Mutation Testing Summary
	runs-on: ubuntu-latest
	needs: [contract-mutation, frontend-mutation]
	if: always()

	steps:
	- name: Check job results
	run: \|
	CONTRACT="${{ needs.contract-mutation.result }}"
	FRONTEND="${{ needs.frontend-mutation.result }}"

	echo "Contract mutation result : $CONTRACT"
	echo "Frontend mutation result : $FRONTEND"

	# Treat 'skipped' (workflow_dispatch scope filter) as success
	if [[ "$CONTRACT" == "failure" \|\| "$FRONTEND" == "failure" ]]; then
	echo "One or more mutation suites failed the score threshold."
	exit 1
	fi

	echo "All mutation suites passed."

Navigation Menu

Mutation Testing #5

Workflow file

Mutation Testing #5

Uh oh!

Workflow file for this run