tests.yml

name: Tests

on:
  push:
    branches: '**'
    tags-ignore: '**'
  pull_request:

jobs:
  Static-Code-Checks:
    runs-on: ubuntu-latest

    steps:
    - uses: actions/checkout@v4

    - name: Set up Python
      uses: actions/setup-python@v5
      with:
        python-version: '3.9'

    - name: Install pip Dependencies
      run: |
        python3 -m pip install --upgrade pip
        python3 -m pip install --user fusepy pytest lz4 PySquashfsImage asyncssh fsspec

    - name: Style Check With Black
      run: |
        python3 -m pip install black
        black -q --diff --line-length 120 --skip-string-normalization ratarmount.py tests/*.py core/ratarmountcore/*.py > black.diff
        if [ -s black.diff ]; then
            cat black.diff
            exit 123
        fi

    - name: Lint With Codespell
      run: |
        python3 -m pip install codespell
        codespell $( git ls-tree -r --name-only HEAD | 'grep' -E '[.](py|md|txt|sh|yml)' )

    - name: Lint With Flake8
      run: |
        python3 -m pip install flake8
        flake8 --config tests/.flake8 *.py tests/*.py core/ratarmountcore/[^_]*.py

    - name: Lint With Pylint
      run: |
        python3 -m pip install pylint
        pylint --rcfile tests/.pylintrc *.py tests/*.py core/ratarmountcore/*.py | tee pylint.log
        ! 'egrep' ': E[0-9]{4}: ' pylint.log

    - name: Lint With Pytype
      run: |
        python3 -m pip install pytype
        pytype -d import-error -P$( cd core && pwd ):$( pwd ) ratarmount.py core/ratarmountcore/*.py

    - name: Lint With Mypy
      run: |
        yes | python3 -m pip install --upgrade-strategy eager --upgrade types-dataclasses mypy
        mypy --config-file tests/.mypy.ini *.py core/ratarmountcore/*.py
        yes | python3 -m pip uninstall types-dataclasses

    - name: Lint With ShellCheck
      run: |
        sudo apt-get -y install shellcheck
        shellcheck -e SC2064 tests/*.sh

  Tests:
    runs-on: ${{ matrix.os }}

    strategy:
      matrix:
        os: ['macos-13', 'ubuntu-latest']
        # macos-13 / macos-latest does not work anymore because the dependencies don't have any wheels,
        # probably because it is M1 based.
        # ToDo: Add windows-latest but it requires a lot of setup of the dependencies!
        #       Maybe only test ratarmount-core without most dependencies after I have split that off.
        # Oldest and newest versions should be enough. Python versions are supported 5 years from release date.
        # https://endoflife.date/python
        # 3.5 was released 2015-09-13 and end-of-life was 2020-09-13
        # 3.6 was released 2016-12-23 and end-of-life was 2021-12-23
        # 3.7 was released 2018-06-27 and end-of-life was 2023-06-27
        # 3.8 was released 2019-10-14 and end-of-life will be 2024-10-14
        # 3.9 was released 2020-10-05 and end-of-life will be 2025-10-25
        # 3.10 was released 2021-10-04 and end-of-life will be 2026-10-04
        # 3.11 was released 2022-10-24 and end-of-life will be 2027-10
        # 3.12 was released 2023-10-02 and end-of-life will be 2028-10
        # 3.13 is to be released 2024-10
        # 3.14 is to be released 2025-10
        python-version: ['3.9', '3.12', '3.13']
        include:
        - os: ubuntu-latest
          python-version: '3.14.0-alpha.0'

    defaults:
      run:
        # This is especially important for windows because it seems to default to powershell
        shell: bash

    steps:
    - uses: actions/checkout@v4
      with:
        # We need one tag for testing the git mount.
        # This is BROKEN! God damn it. Is anything working at all...
        # https://github.com/actions/checkout/issues/1781
        fetch-tags: true

    - name: Fetch tag for tests
      run: git fetch origin refs/tags/v0.15.2:refs/tags/v0.15.2

    - name: Set up Python ${{ matrix.python-version }}
      uses: actions/setup-python@v5
      with:
        python-version: ${{ matrix.python-version }}

    - name: Print System Information
      run: |
        echo "uname -a: $( uname -a )"
        echo "Shell: $SHELL"
        echo "Cores: $( nproc )"
        echo "Mount points:"; mount

    - uses: msys2/setup-msys2@v2
      if: startsWith( matrix.os, 'windows' )
      with:
        install: gcc make liblzma-devel libzstd-devel zlib-devel

    - name: Install Dependencies (Linux)
      if: startsWith( matrix.os, 'ubuntu' )
      run: |
        # Libarchive calls the grzip, lrzip, lzop binaries for lrzip support. Others, such as bzip2, gzip, lz4, lzma,
        # zstd, may also call external binaries depending on how libarchive was compiled!
        # https://github.com/libarchive/libarchive/blob/ad5a0b542c027883d7069f6844045e6788c7d70c/libarchive/
        #   archive_read_support_filter_lrzip.c#L68
        sudo apt-get -y install libfuse2 fuse3 bzip2 pbzip2 pixz zstd unar lrzip lzop gcc liblzo2-dev ruby-webrick

    - name: Install Dependencies For Unreleased Python Versions (Linux)
      if: >
        startsWith( matrix.os, 'ubuntu' ) && (
          matrix.python-version == '3.13' ||
          matrix.python-version == '3.14.0-alpha.0')
      run: |
        #libgit2-dev is too old on Ubuntu 22.04. Leads to error about missing git2/sys/errors.h
        #sudo apt-get -y install libgit2-dev
        sudo apt-get -y install cmake
        git clone --branch v1.8.1 --depth 1 https://github.com/libgit2/libgit2.git
        ( cd libgit2 && mkdir build && cd build && cmake .. && cmake --build . && sudo cmake --build . -- install )
        echo "LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib" >> "$GITHUB_ENV"
        # Expects exactly libgit2 1.8.x! Therefore install it here instead via the non-pinned ratarmount dependencies.
        python3 -m pip install pygit2==1.15

    - name: Install Dependencies (MacOS)
      if: startsWith( matrix.os, 'macos' )
      run: |
        # coreutils is required for the tests written in shell, e.g., for the 'realpath' command
        # unar is required for rar tests with passwords. By default, bsdtar is installed but that is the only
        # one of the three supported tools (the third is unrar) that does not support passwords.
        # And the error message is atrocious:
        #   cmdline.extend(args)
        #   TypeError: 'NoneType' object is not iterable
        brew install macfuse coreutils pixz pbzip2 zstd unar libarchive lrzip lzop lzo
        # Add brew installation binary folder to PATH so that command line tools like zstd can be found
        echo PATH="$PATH:/usr/local/bin" >> "$GITHUB_ENV"

    - name: Install Dependencies For Unreleased Python Versions (MacOS)
      if: >
        startsWith( matrix.os, 'macos' ) && (
          matrix.python-version == '3.13' ||
          matrix.python-version == '3.14.0-alpha.0')
      run: |
        brew install libgit2@1.8
        brew link libgit2@1.8 --force
        # Expects exactly libgit2 1.8.x! Therefore install it here instead via the non-pinned ratarmount dependencies.
        python3 -m pip install pygit2==1.15

    - name: Install pip Dependencies
      run: |
        python3 -m pip install --upgrade pip
        python3 -m pip install --upgrade wheel
        python3 -m pip install --upgrade setuptools
        python3 -m pip install --upgrade-strategy eager --upgrade twine build fusepy cffi

    - name: Test Startup With Only One Compression Dependency
      run: |
        for module in indexed_gzip indexed_zstd lzmaffi python-xz; do
          python3 -m pip install --upgrade "$module"
          # Segfaults (139) are not allowed but other exit codes are valid!
          # indexed_zstd=1.2.0 did segfault here!
          python3 ratarmount.py README.md || [ $? != 139 ]
          python3 -m pip uninstall --yes "$module"
        done
        python3 -m pip install --upgrade 'git+https://github.com/mxmlnkn/indexed_bzip2.git@master#egginfo=rapidgzip&subdirectory=python/rapidgzip'

    - name: Test ratarmountcore Installation From Tarball
      working-directory: core
      run: |
        python3 -m build
        twine check dist/*
        python3 -m pip install "$( find dist -name '*.tar.gz' | head -1 )"[full]

    - name: Test Installation From Tarball
      run: |
        python3 -m build
        # Around commit 08ddc8fb3679faacd6e54eb5c54104995b045dc4, I had some very weird twine check failures
        # because .tar.gz tarballs were wrongly recognized as ZIP files by CPython's zipfile.is_zipfile.
        # Therefore, try recompressing the tarballs with different compression levels until it does not randomly
        # look like a ZIP anymore. It is an ugly hack, but so is zipfile.is_zipfile.
        tarball=$( find dist -name '*.tar.gz' )
        if python3 -c 'import sys, zipfile; sys.exit(0 if zipfile.is_zipfile(sys.argv[1]) else 1)' "$tarball"; then
            gzip -c -d "$tarball" > "$tarball.tar"
            for (( i=9; i>0; --i )); do
                cat "$tarball.tar" | gzip -$i > "$tarball"
                if ! python3 -c 'import sys, zipfile; sys.exit(0 if zipfile.is_zipfile(sys.argv[1]) else 1)' "$tarball"
                then break; fi
            done
        fi
        twine check dist/*
        python3 -m pip install "$( find dist -name '*.tar.gz' | head -1 )"[full]

    - name: Test Installation From Source
      run: |
        python3 -m pip install .[full]

    - name: Test Simple Startup
      run: |
        ratarmount --help
        ratarmount --version

    - name: Test Simple Mount
      # macOS 11+ is too uptight about "security" and is not able to fully load the macfuse kernel extension.
      # https://github.com/actions/runner-images/issues/4731
      if: ${{ !startsWith( matrix.os, 'macos' ) }}
      run: |
        ratarmount tests/single-file.tar mimi
        ls -la mimi
        sleep 1s
        # MacOS does not have fusermount!
        ratarmount -u mimi

    - name: Test Startup Without Compression Dependencies
      if: ${{ !startsWith( matrix.os, 'macos' ) }}
      run: |
        # Segfaults (139) are not allowed but other exit codes are valid!
        python3 ratarmount.py tests/simple.bz2 || [ $? != 139 ]

    - name: Unit Tests
      run: |
        python3 -m pip install pytest pytest-xdist pandas
        for file in core/tests/test_*.py tests/test_*.py; do
            case "$file" in
              "tests/test_cli.py")
                  if ! uname | 'grep' -q -i darwin; then
                      # First off, n=auto seems to use the physical cores and ignores virtual ones.
                      # Secondly, these tests scale much better than the others because most time is spent waiting for
                      # the FUSE mount point to appear or disappear, which doesn't seem to be bottlenecked by CPU usage.
                      python3 -X dev -W ignore::DeprecationWarning:fuse -u \
                          -c "import pytest, re, sys; sys.exit(pytest.console_main())" \
                          -n 16 --disable-warnings "$testFile"
                  fi
                  ;;
                "core/tests/test_AutoMountLayer.py"\
                |"core/tests/test_BlockParallelReaders.py"\
                |"core/tests/test_LibarchiveMountSource.py"\
                |"core/tests/test_SQLiteIndexedTar.py")
                    echo "$file"  # pytest-xdist seems to omit the test file name
                    pytest -n auto --disable-warnings "$file"
                    ;;
                *)
                  # Fusepy warns about usage of use_ns because the implicit behavior is deprecated.
                  # But there has been no development to fusepy for 4 years, so I think it should be fine to ignore.
                  pytest --disable-warnings "$file"
            esac
        done
        python3 tests/tests.py

    - name: Install pip Test Dependencies
      run: |
        python3 -m pip install -r tests/requirements-tests.txt

    - name: Install Test Dependencies (Linux)
      if: ${{ !startsWith( matrix.os, 'macos' ) }}
      run: |
        sudo bash tests/install-smbd.sh

    - name: Regression Tests (FUSE 3)
      if: ${{ !startsWith( matrix.os, 'macos' ) }}
      env:
        DROPBOX_TOKEN: ${{ secrets.DROPBOX_TOKEN }}
      run: |
        export FUSE_LIBRARY_PATH=$( dpkg -L libfuse3-3 | 'grep' -F .so | head -1 )
        ratarmount --version
        bash tests/runtests.sh

    - name: Regression Tests (FUSE 2)
      if: ${{ !startsWith( matrix.os, 'macos' ) }}
      env:
        DROPBOX_TOKEN: ${{ secrets.DROPBOX_TOKEN }}
      run: |
        bash tests/runtests.sh
        python3 -c 'import pygit2'

    - name: Module tests without fusepy
      run: |
        python3 -m pip uninstall -y fusepy
        python3 tests/tests.py