Add support for converting Curated Transfomer state dicts Hugging Face compatible state dicts #1179
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Test | |
| on: [push, pull_request, workflow_call] | |
| jobs: | |
| validate: | |
| name: Validate | |
| if: github.repository_owner == 'explosion' | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Check out repo | |
| uses: actions/checkout@v3 | |
| - name: Configure Python version | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: "3.8" | |
| architecture: x64 | |
| - name: black | |
| run: | | |
| python -m pip install black -c requirements.txt | |
| python -m black curated_transformers examples --check | |
| - name: isort | |
| run: | | |
| python -m pip install isort -c requirements.txt | |
| python -m isort curated_transformers examples --check | |
| - name: flake8 | |
| run: | | |
| python -m pip install flake8==5.0.4 | |
| python -m flake8 curated_transformers examples --count --select=E901,E999,F821,F822,F823,W605 --show-source --statistics | |
| test: | |
| name: Run tests | |
| needs: Validate | |
| runs-on: ${{ matrix.os }} | |
| strategy: | |
| matrix: | |
| os: [ubuntu-latest, macos-latest, windows-latest] | |
| python-version: ["3.8", "3.11"] | |
| steps: | |
| - uses: actions/checkout@v1 | |
| with: | |
| submodules: true | |
| - uses: actions/setup-python@v4 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| # Disabled as this seems to randomly break builds on Windows. | |
| # cache: "pip" | |
| - name: Install requirements | |
| run: | | |
| python -m pip install --upgrade pip setuptools wheel build | |
| python -m pip install -r requirements.txt | |
| - name: Build sdist | |
| run: python -m build --sdist | |
| - name: Run mypy | |
| run: mypy curated_transformers | |
| - name: Delete source directory | |
| run: | | |
| cp curated_transformers/tests/conftest.py . | |
| rm -rf curated_transformers | |
| shell: bash | |
| - name: Uninstall all packages | |
| run: | | |
| python -m pip freeze | |
| python -m pip freeze --exclude pywin32 > installed.txt | |
| python -m pip uninstall -y -r installed.txt | |
| - name: Install from sdist | |
| run: | | |
| SDIST=$(python -c "import os;print(os.listdir('./dist')[-1])" 2>&1) | |
| python -m pip install dist/$SDIST | |
| shell: bash | |
| - name: Install test dependencies | |
| run: | | |
| python -m pip install -r requirements.txt | |
| - name: Run pytest | |
| run: python -m pytest --pyargs curated_transformers | |
| - name: Install HF transformers | |
| run: | | |
| python -m pip install transformers | |
| shell: bash | |
| # Upstream Falcon imports this, but doesn't use it. 😭 | |
| - name: Install einops | |
| run: | | |
| python -m pip install einops | |
| # For testing the Llama tokenizer against the HF slow tokenizer. | |
| - name: Install sentencepiece | |
| run: | | |
| python -m pip install sentencepiece | |
| # For testing safetensor checkpoints. | |
| - name: Install safetensors | |
| run: | | |
| python -m pip install safetensors | |
| - name: Run pytest (w/t HF transformers) | |
| run: python -m pytest --pyargs curated_transformers |