[wip][feature] Add --recursive-extensions option to configure files t… #1756
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Tests | |
| on: | |
| push: | |
| branches: '**' | |
| tags-ignore: '**' | |
| pull_request: | |
| jobs: | |
| Static-Code-Checks: | |
| runs-on: ubuntu-slim | |
| env: | |
| RATARMOUNT_CHECK_OVERRIDES: 1 | |
| steps: | |
| - uses: actions/checkout@v5 | |
| - name: Setup Shared Test Environment | |
| uses: ./.github/actions/setup-test-env | |
| with: | |
| os: ${{ matrix.os }} | |
| python-version: 3.12 | |
| linux-packages: 'liblzo2-dev shellcheck' | |
| - name: Install pip Dependencies | |
| run: | | |
| uv pip install pytest fsspec ruff~=0.14 black~=26.0 codespell flake8 pylint mypy | |
| ( cd core && uv pip install .[full,ext4,fsspec,sqlar] ) | |
| uv pip install . | |
| - name: Style Check With Ruff | |
| run: | | |
| # I hate when the CI fails after not pushing for a while because ruff has new default rules. | |
| ruff check --config tests/.ruff.toml -- $( git ls-tree -r --name-only HEAD | 'grep' -E '[.]py$' | 'grep' -v '/_external/' ) | |
| - name: Style Check With Black | |
| run: | | |
| black -q --diff --line-length 120 --skip-string-normalization --target-version=py39 \ | |
| $( git ls-tree -r --name-only HEAD | 'grep' '[.]py$' | 'grep' -v '/_external/' ) > black.diff | |
| if [ -s black.diff ]; then | |
| cat black.diff | |
| exit 123 | |
| fi | |
| - name: Lint With Codespell | |
| run: | | |
| codespell $( git ls-tree -r --name-only HEAD | 'grep' -E '[.](py|md|txt|sh|yml)$' | 'grep' -v '/_external/' ) | |
| - name: Lint With Flake8 | |
| run: | | |
| flake8 --config tests/.flake8 {ratarmount,tests}/[^_]*.py core/{ratarmountcore,tests}/[^_]*.py | |
| - name: Lint With Pylint | |
| run: | | |
| # ubuntu-slim runs on one core, so parallelism with -j would only add overhead. | |
| pylint --rcfile tests/.pylintrc ratarmount core/ratarmountcore tests/*.py core/tests/*.py | tee pylint.log | |
| ! 'egrep' ': E[0-9]{4}: ' pylint.log | |
| - name: Lint With Mypy | |
| run: | | |
| yes | uv pip install --upgrade types-dataclasses | |
| mypy --config-file tests/.mypy.ini ratarmount core/ratarmountcore core/tests | |
| - name: Lint With ShellCheck | |
| run: | | |
| shellcheck -e SC2064 tests/*.sh | |
| Longer-Running-Checks: | |
| runs-on: ubuntu-latest | |
| needs: [Static-Code-Checks] | |
| env: | |
| RATARMOUNT_CHECK_OVERRIDES: 1 | |
| steps: | |
| - uses: actions/checkout@v5 | |
| - name: Setup Shared Test Environment | |
| uses: ./.github/actions/setup-test-env | |
| with: | |
| os: ${{ matrix.os }} | |
| # Pytype does not support Python 3.13 as of 2025-06. | |
| # https://news.ycombinator.com/item?id=44963724 | |
| # https://github.com/google/pytype | |
| # > The last supported Python version for Pytype will be 3.12. | |
| python-version: 3.12 | |
| linux-packages: 'liblzo2-dev' | |
| - name: Lint With Pytype | |
| run: | | |
| uv pip install pytype | |
| pytype -j auto -d import-error -P$( cd core && pwd ):$( pwd ) --exclude=core/ratarmountcore/_external ratarmount core/ratarmountcore core/tests | |
| Micro-Tests: | |
| runs-on: ${{ matrix.os }} | |
| strategy: | |
| matrix: | |
| os: ['macos-15-intel', 'ubuntu-latest'] | |
| # ubuntu-slim does not work because the simple startup tests require FUSE. | |
| # macos-13 / macos-latest does not work anymore because the dependencies don't have any wheels, | |
| # probably because it is M1 based. | |
| # ToDo: Add windows-latest but it requires a lot of setup of the dependencies! | |
| # Maybe only test ratarmount-core without most dependencies after I have split that off. | |
| # Oldest and newest versions should be enough. Python versions are supported 5 years from release date. | |
| # https://endoflife.date/python | |
| # 3.5 was released 2015-09-13 and end-of-life was 2020-09-13 | |
| # 3.6 was released 2016-12-23 and end-of-life was 2021-12-23 | |
| # 3.7 was released 2018-06-27 and end-of-life was 2023-06-27 | |
| # 3.8 was released 2019-10-14 and end-of-life was 2024-10-14 | |
| # 3.9 was released 2020-10-05 and end-of-life will be 2025-10-25 | |
| # 3.10 was released 2021-10-04 and end-of-life will be 2026-10-04 | |
| # 3.11 was released 2022-10-24 and end-of-life will be 2027-10 | |
| # 3.12 was released 2023-10-02 and end-of-life will be 2028-10 | |
| # 3.13 was released 2024-10-07 and end-of-life will be 2029-10 | |
| # 3.14 was released 2025-10-07 and end-of-life will be 2030-10 | |
| # Supported versions: https://github.com/actions/python-versions/blob/main/versions-manifest.json | |
| python-version: ['3.9', '3.12', '3.14'] | |
| libfuse-version: ['system'] | |
| include: | |
| - os: ubuntu-latest | |
| python-version: '3.12' | |
| libfuse-version: '3.17.2' | |
| # Windows won't install easily in the full version because python-lzo has been missing Python 3.12+ | |
| # wheels since at least 2025-04: https://github.com/jd-boyd/python-lzo/issues/90 | |
| # WinFsp installation does not fully work yet and therefore mfusepy cannot be imported. | |
| # The WinFsp installation seems to hang. Seems like it does not work fully unattended. | |
| # Maybe some command line options are missing. | |
| #- os: windows-latest | |
| # python-version: '3.9' | |
| # libfuse-version: 'system' | |
| #- os: windows-latest | |
| # python-version: '3.11' | |
| # libfuse-version: 'system' | |
| defaults: | |
| run: | |
| # This is especially important for windows because it seems to default to powershell | |
| shell: bash | |
| env: | |
| LIBFUSE_VERSION: ${{ matrix.libfuse-version }} | |
| RATARMOUNT_CHECK_OVERRIDES: 1 | |
| steps: | |
| - uses: actions/checkout@v5 | |
| - name: Setup Shared Test Environment | |
| uses: ./.github/actions/setup-test-env | |
| with: | |
| os: ${{ matrix.os }} | |
| python-version: ${{ matrix.python-version }} | |
| install-winfsp: 'true' | |
| libfuse-version: ${{ matrix.libfuse-version }} | |
| - name: Install Dependencies (Linux) | |
| if: startsWith( matrix.os, 'ubuntu' ) | |
| run: | | |
| # It seems like this is necessary because the cached apt install actions does not work. | |
| # Maybe because of post-install actions not being run? | |
| sudo apt-get -y update | |
| sudo apt-get -y install libfuse2 fuse3 | |
| - name: Test ratarmountcore Installation From Tarball | |
| working-directory: core | |
| run: | | |
| python3 -m build | |
| twine check dist/* | |
| uv pip install "$( find dist -name '*.tar.gz' | head -1 )" | |
| uv pip install mfusepy | |
| - name: Test Startup Without Compression Dependencies | |
| if: ${{ startsWith( matrix.os, 'ubuntu' ) }} | |
| run: | | |
| # Segfaults (139) are not allowed but other exit codes are valid! | |
| python3 -m ratarmount tests/simple.bz2 || [ $? != 139 ] | |
| - name: Test Startup With Only One Compression Dependency | |
| if: ${{ !startsWith( matrix.os, 'windows' ) }} | |
| run: | | |
| for module in indexed_gzip indexed_zstd lzmaffi python-xz; do | |
| if [[ $module == lzmaffi ]]; then | |
| # --use-pep517 to fix deprecation warning: https://github.com/pypa/pip/issues/6334 | |
| uv pip install --no-build-isolation --upgrade "$module" | |
| elif [[ $module == indexed_gzip ]]; then | |
| uv pip install --upgrade "indexed_gzip >= 1.6.3, < 1.9.3" | |
| fi | |
| # Segfaults (139) are not allowed but other exit codes are valid! | |
| # indexed_zstd=1.2.0 did segfault here! | |
| python3 -m ratarmount README.md || [ $? != 139 ] | |
| uv pip uninstall --system "$module" | |
| done | |
| uv pip install --upgrade 'git+https://github.com/mxmlnkn/indexed_bzip2.git@master#egginfo=rapidgzip&subdirectory=python/rapidgzip' | |
| - name: Test Installation From Tarball | |
| run: | | |
| python3 -m build | |
| # Around commit 08ddc8fb3679faacd6e54eb5c54104995b045dc4, I had some very weird twine check failures | |
| # because .tar.gz tarballs were wrongly recognized as ZIP files by CPython's zipfile.is_zipfile. | |
| # Therefore, try recompressing the tarballs with different compression levels until it does not randomly | |
| # look like a ZIP anymore. It is an ugly hack, but so is zipfile.is_zipfile. | |
| tarball=$( find dist -name '*.tar.gz' ) | |
| if python3 -c 'import sys, zipfile; sys.exit(0 if zipfile.is_zipfile(sys.argv[1]) else 1)' "$tarball"; then | |
| gzip -c -d "$tarball" > "$tarball.tar" | |
| for (( i=9; i>0; --i )); do | |
| cat "$tarball.tar" | gzip -$i > "$tarball" | |
| if ! python3 -c 'import sys, zipfile; sys.exit(0 if zipfile.is_zipfile(sys.argv[1]) else 1)' "$tarball" | |
| then break; fi | |
| done | |
| fi | |
| twine check dist/* | |
| uv pip install "$( find dist -name '*.tar.gz' | head -1 )"[full] | |
| - name: Test Simple Startup | |
| run: | | |
| ratarmount --help | |
| ratarmount --version | |
| - name: Test Simple Mount | |
| # macOS 11+ is too uptight about "security" and is not able to fully load the macfuse kernel extension. | |
| # https://github.com/actions/runner-images/issues/4731 | |
| if: ${{ startsWith( matrix.os, 'ubuntu' ) }} | |
| run: | | |
| ratarmount -d 3 tests/single-file.tar mimi | |
| ls -la mimi | |
| sleep 1s | |
| # MacOS does not have fusermount! | |
| ratarmount -u mimi | |
| - name: CLI Tests (Linux) | |
| if: ${{ startsWith( matrix.os, 'ubuntu' ) }} | |
| run: | | |
| uv pip install pytest pytest-order | |
| # First off, n=auto seems to use the physical cores and ignores virtual ones. | |
| # Secondly, these tests scale much better than the others because most time is spent waiting for | |
| # the FUSE mount point to appear or disappear, which doesn't seem to be bottlenecked by CPU usage. | |
| # There is a new problem in the Github Actions CI: | |
| # fusermount: too many FUSE filesystems mounted; mount_max=N can be set in /etc/fuse.conf | |
| # The output of 'mount' shows me only 4 ratarmount mounts! | |
| # And now, with 4 mounts, it simply fails without an error message ... WTF is going on. | |
| # Remove any multi-threaded execution seems to work, but the error occurred sporadically, so maybe it's that. | |
| python3 -X dev -W ignore::DeprecationWarning:fuse -u \ | |
| -c "import pytest, sys; sys.exit(pytest.console_main())" \ | |
| --disable-warnings -rs -v tests/test_cli.py | |
| - name: Module tests without fusepy | |
| run: | | |
| uv pip install pytest | |
| uv pip uninstall --system fusepy mfusepy | |
| pytest -rs -v tests/tests.py | |
| Unit-Tests: | |
| runs-on: ${{ matrix.os }} | |
| needs: [Micro-Tests, Static-Code-Checks] | |
| strategy: | |
| # Disable because it makes no sense. Most often, they run in parallel, i.e., they would fail in roughly the | |
| # same time anyway if the error persists on every system, and if not, it would be nice to know which systems | |
| # work and which don't. If multiple systems are affected it might only extend debugging because I would only | |
| # notice the error on the next CI run. | |
| fail-fast: false | |
| matrix: | |
| os: ['macos-15-intel', 'ubuntu-latest'] | |
| # ubuntu-slim is limited to 15 min jobs, which is insufficient for Python 3.9 and 3.14 tests. | |
| # The Python 3.12 tests only need ~12.5 min for some reason. | |
| # macos-13 / macos-latest does not work anymore because the dependencies don't have any wheels, | |
| # probably because it is M1 based. | |
| # https://endoflife.date/python | |
| # Oldest and newest versions should be enough. Python versions are supported 5 years from release date. | |
| # 3.n is released October of 2011+n and EOL in 2016+n for n>=8. | |
| # Supported versions: https://github.com/actions/python-versions/blob/main/versions-manifest.json | |
| python-version: ['3.9', '3.12', '3.14'] | |
| include: | |
| # Windows won't install easily in the full version because python-lzo has been missing Python 3.12+ | |
| # wheels since at least 2025-04: https://github.com/jd-boyd/python-lzo/issues/90 | |
| - os: windows-latest | |
| python-version: '3.9' | |
| - os: windows-latest | |
| python-version: '3.11' | |
| defaults: | |
| run: | |
| # This is especially important for windows because it seems to default to powershell | |
| shell: bash | |
| steps: | |
| - uses: actions/checkout@v5 | |
| - name: Setup Shared Test Environment | |
| uses: ./.github/actions/setup-test-env | |
| with: | |
| os: ${{ matrix.os }} | |
| python-version: ${{ matrix.python-version }} | |
| install-winfsp: 'false' | |
| linux-packages: bzip2 pbzip2 pixz zstd unar lrzip lzop gcc liblzo2-dev liblzma-dev* | |
| macos-packages: coreutils pixz pbzip2 zstd unar libarchive lrzip lzop lzo | |
| - name: Unit Tests | |
| run: | | |
| uv pip install pytest pytest-xdist pytest-order pandas | |
| pytest -rs -v -m 'not parallel' core/tests/test_*.py | |
| pytest -rs -v -n auto -m parallel core/tests/test_*.py | |
| pytest -rs -v tests/tests.py | |
| Regression-Tests: | |
| runs-on: ${{ matrix.os }} | |
| needs: [Micro-Tests, Static-Code-Checks] | |
| strategy: | |
| # Disable because it makes no sense. Most often, they run in parallel, i.e., they would fail in roughly the | |
| # same time anyway if the error persists on every system, and if not, it would be nice to know which systems | |
| # work and which don't. If multiple systems are affected it might only extend debugging because I would only | |
| # notice the error on the next CI run. | |
| fail-fast: false | |
| matrix: | |
| os: ['ubuntu-latest'] | |
| # macos-13 / macos-latest does not work anymore because the dependencies don't have any wheels, | |
| # probably because it is M1 based and because of FUSE permissions lockdown. | |
| # https://endoflife.date/python | |
| # Oldest and newest versions should be enough. Python versions are supported 5 years from release date. | |
| # 3.n is released October of 2011+n and EOL in 2016+n for n>=8. | |
| # Supported versions: https://github.com/actions/python-versions/blob/main/versions-manifest.json | |
| python-version: ['3.9', '3.12', '3.14'] | |
| libfuse-version: ['system'] | |
| parallelization: [1, 2] | |
| include: | |
| - os: ubuntu-latest | |
| python-version: '3.12' | |
| libfuse-version: '3.17.2' | |
| parallelization: 2 | |
| defaults: | |
| run: | |
| # This is especially important for windows because it seems to default to powershell | |
| shell: bash | |
| env: | |
| LIBFUSE_VERSION: ${{ matrix.libfuse-version }} | |
| RATARMOUNT_CHECK_OVERRIDES: 1 | |
| steps: | |
| - uses: actions/checkout@v5 | |
| - name: Setup Shared Test Environment | |
| uses: ./.github/actions/setup-test-env | |
| with: | |
| os: ${{ matrix.os }} | |
| python-version: ${{ matrix.python-version }} | |
| libfuse-version: ${{ matrix.libfuse-version }} | |
| - name: Fetch tag for tests | |
| run: git fetch origin refs/tags/v0.15.2:refs/tags/v0.15.2 | |
| - name: Install pip Test Dependencies | |
| run: | | |
| uv pip install -r tests/requirements-tests.txt | |
| - name: Install Test Dependencies (Linux) | |
| run: | | |
| sudo bash tests/install-smbd.sh | |
| - name: Regression Tests (FUSE 3) | |
| if: ${{ matrix.libfuse-version != 'system' }} | |
| env: | |
| DROPBOX_TOKEN: ${{ secrets.DROPBOX_TOKEN }} | |
| PARALLELIZATIONS: ${{ matrix.parallelization }} | |
| run: | | |
| if [[ $LIBFUSE_VERSION == system ]]; then | |
| export FUSE_LIBRARY_PATH=$( dpkg -L libfuse3-3 | 'grep' -F .so | head -1 ) | |
| else | |
| export FUSE_LIBRARY_PATH=/usr/local/lib/x86_64-linux-gnu/libfuse3.so.$LIBFUSE_VERSION | |
| fi | |
| ratarmount --version | grep -i fuse | |
| bash tests/runtests.sh | |
| - name: Regression Tests (FUSE 2) | |
| if: ${{ matrix.libfuse-version == 'system' }} | |
| env: | |
| DROPBOX_TOKEN: ${{ secrets.DROPBOX_TOKEN }} | |
| PARALLELIZATIONS: ${{ matrix.parallelization }} | |
| run: | | |
| ratarmount --version | grep -i fuse | |
| bash tests/runtests.sh |