PyPI release #6

	# https://github.com/huggingface/datatrove/pull/159
	name: PyPI release
	on:
	workflow_dispatch:

	jobs:
	testing:
	uses: ./.github/workflows/testing.yml
	release:
	needs: testing
	runs-on: ubuntu-latest
	env:
	TWINE_USERNAME: __token__

	permissions:
	contents: write # needed to create a tag

	steps:
	- name: Checkout Repo
	uses: actions/checkout@v3

	- name: Setup Python
	uses: actions/setup-python@v4
	with:
	python-version: "3.10"

	- name: Install build dependencies
	run: \|
	python -m pip install --upgrade pip
	pip install -U twine build

	- name: Build the dist files
	run: python -m build .

	- name: Publish to the test PyPI
	env:
	TWINE_PASSWORD: ${{ secrets.TEST_PYPI_TOKEN }}
	run: twine upload dist/* --skip-existing --repository=testpypi

	- name: Test installing from test PyPI and running tests
	run: \|
	python -m pip install uv
	uv pip install -i https://testpypi.python.org/pypi --extra-index-url https://pypi.org/simple datatrove[testing]
	python -m nltk.downloader punkt
	make test

	- name: Get tag name
	id: get_tag_name
	run: \|
	echo TAG_NAME=$(grep '^version' pyproject.toml \| head -1 \| cut -d '"' -f 2) >> $GITHUB_OUTPUT

	- name: Tag the release
	uses: actions/github-script@v7
	with:
	script: \|
	github.rest.git.createRef({
	owner: context.repo.owner,
	repo: context.repo.repo,
	ref: 'refs/tags/v${{ steps.get_tag_name.outputs.TAG_NAME }}',
	sha: context.sha
	})
	env:
	GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # needed to tag

	- name: Publish to PyPI
	env:
	TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }}
	run: twine upload dist/* --repository=pypi

Provide feedback