Sound star-set softmax-attention reach + VNN-COMP 2023 ViT verifier #305
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Regression Tests Workflow | |
| # This workflow runs the NNV test suite with baseline comparison to detect regressions | |
| # It runs separately from the main CI workflow to provide detailed regression analysis | |
| name: Regression Tests | |
| on: | |
| # Run on pull requests to master | |
| pull_request: | |
| branches: [ "master" ] | |
| # Allow manual triggering | |
| workflow_dispatch: | |
| inputs: | |
| update_baselines: | |
| description: 'Update baselines after tests' | |
| required: false | |
| default: false | |
| type: boolean | |
| run_cp: | |
| type: boolean | |
| required: false | |
| default: false | |
| env: | |
| # Opt JS actions into Node 24 ahead of GitHub's forced 2026-06-16 default flip (Node 20 | |
| # EOL on the runners), so any incompatibility surfaces in CI now rather than mid-competition. | |
| FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: 'true' | |
| jobs: | |
| regression-test: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| with: | |
| submodules: 'true' | |
| - name: Reclaim runner disk space | |
| # Free ~20-30 GB of preinstalled tooling so the ~5 GB MATLAB toolbox download | |
| # can't exhaust the runner disk ("No space left on device"). See test-matrix.yml. | |
| run: | | |
| echo "Disk before reclaim:"; df -h / | |
| sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/.ghcup /usr/local/lib/android \ | |
| /usr/share/swift /usr/local/share/powershell \ | |
| /opt/hostedtoolcache/CodeQL /usr/local/share/chromium /usr/local/share/boost || true | |
| sudo docker image prune --all --force >/dev/null 2>&1 || true | |
| echo "Disk after reclaim:"; df -h / | |
| - name: Setup MATLAB | |
| uses: matlab-actions/setup-matlab@v3 | |
| with: | |
| release: R2024b | |
| products: > | |
| Computer_Vision_Toolbox | |
| Control_System_Toolbox | |
| Deep_Learning_Toolbox | |
| Image_Processing_Toolbox | |
| Optimization_Toolbox | |
| Parallel_Computing_Toolbox | |
| Statistics_and_Machine_Learning_Toolbox | |
| Symbolic_Math_Toolbox | |
| System_Identification_Toolbox | |
| Deep_Learning_Toolbox_Converter_for_ONNX_Model_Format | |
| Deep_Learning_Toolbox_Converter_for_PyTorch_Model_Format | |
| Deep_Learning_Toolbox_Converter_for_TensorFlow_models | |
| - name: Setup Python for CP | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.12' | |
| - name: Install Python requirements | |
| if: github.event.inputs.run_cp == 'true' | |
| run: | | |
| python3 -m venv .venv | |
| source .venv/bin/activate | |
| pip install --upgrade pip | |
| pip install -r requirement.txt | |
| continue-on-error: true | |
| # Cache the tbxmanager toolboxes (MPT/glpk/sedumi/etc.) install.m downloads from the | |
| # ETH/tbxmanager server. OS/arch-keyed, MATLAB-release-independent; shared with the | |
| # matrix + legacy CI. startup_nnv's `tbxmanager restorepath` re-adds them, so a warm | |
| # cache turns install.m's downloads into no-ops. Restore/save split so we only persist a | |
| # COMPLETE install (gated on the last toolbox, sedumi), never a partial flaky download. | |
| # Force-refresh: run "CI (matrix-sharded)" with rebuild_cache=true (deletes | |
| # the repo-wide nnv-tbxmanager-* cache). | |
| - name: Restore NNV tbxmanager toolboxes | |
| id: tbxcache | |
| uses: actions/cache/restore@v4 | |
| with: | |
| path: code/nnv/tbxmanager | |
| key: nnv-tbxmanager-${{ runner.os }}-${{ hashFiles('code/nnv/install.m') }} | |
| restore-keys: | | |
| nnv-tbxmanager-${{ runner.os }}- | |
| - name: Run regression tests | |
| uses: matlab-actions/run-command@v2 | |
| env: | |
| NNV_TEST_COMPARE_BASELINES: '1' | |
| NNV_TEST_FAIL_ON_REGRESSION: '1' | |
| with: | |
| command: | | |
| disp('=== NNV Regression Test Suite ==='); | |
| cd("code/nnv"); | |
| install; | |
| % Add test utilities to path | |
| addpath(fullfile(pwd, 'tests', 'test_utils')); | |
| % Run tests with regression detection | |
| try | |
| [test_results, regression_results] = run_tests_with_regression("compare", true, "verbose", true); | |
| % Check for failures | |
| if ~isempty(test_results) | |
| n_failed = sum([test_results.Failed]); | |
| if n_failed > 0 | |
| error('Test failures detected: %d tests failed', n_failed); | |
| end | |
| end | |
| % Check for regressions | |
| if isfield(regression_results, 'regressions') && ~isempty(regression_results.regressions) | |
| error('Regressions detected: %d baseline mismatches', length(regression_results.regressions)); | |
| end | |
| disp('All tests passed, no regressions detected.'); | |
| catch ME | |
| disp(['Error: ' ME.message]); | |
| rethrow(ME); | |
| end | |
| - name: Save NNV tbxmanager cache (only a COMPLETE install, only on a miss) | |
| if: always() && steps.tbxcache.outputs.cache-hit != 'true' && hashFiles('code/nnv/tbxmanager/toolboxes/sedumi/**') != '' | |
| uses: actions/cache/save@v4 | |
| with: | |
| path: code/nnv/tbxmanager | |
| key: nnv-tbxmanager-${{ runner.os }}-${{ hashFiles('code/nnv/install.m') }} | |
| - name: Upload test results | |
| if: always() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: test-results | |
| path: | | |
| results/tests/data/ | |
| results/tests/figures/ | |
| retention-days: 30 | |
| - name: Update baselines (manual trigger only) | |
| if: github.event_name == 'workflow_dispatch' && github.event.inputs.update_baselines == 'true' | |
| uses: matlab-actions/run-command@v2 | |
| with: | |
| command: | | |
| cd("code/nnv"); | |
| install; | |
| addpath(fullfile(pwd, 'tests', 'test_utils')); | |
| manage_baselines('save'); | |
| disp('Baselines updated successfully.'); | |
| - name: Commit updated baselines | |
| if: github.event_name == 'workflow_dispatch' && github.event.inputs.update_baselines == 'true' | |
| run: | | |
| git config --local user.email "action@github.com" | |
| git config --local user.name "GitHub Action" | |
| git add results/tests/baselines/ | |
| git diff --staged --quiet || git commit -m "Update test baselines [skip ci]" | |
| git push |