From 1e45fa5a7f2a357280b77052a30f2f52c7ecb36a Mon Sep 17 00:00:00 2001 From: Jed Laundry Date: Sat, 5 Oct 2024 23:10:09 +0000 Subject: [PATCH 1/2] Create optional dependency groups for build and cli --- .github/workflows/python-tests.yml | 7 +++-- build.sh | 2 +- docs/source/installation.md | 2 +- pyproject.toml | 27 ++++++++++++++---- requirements.txt | 46 ------------------------------ 5 files changed, 27 insertions(+), 57 deletions(-) delete mode 100644 requirements.txt diff --git a/.github/workflows/python-tests.yml b/.github/workflows/python-tests.yml index d5946904..1fbd2164 100644 --- a/.github/workflows/python-tests.yml +++ b/.github/workflows/python-tests.yml @@ -39,7 +39,7 @@ jobs: - name: Install Python dependencies run: | python -m pip install --upgrade pip - if [ -f requirements.txt ]; then pip install -r requirements.txt; fi + pip install .[build] - name: Test building documentation run: | cd docs @@ -49,8 +49,7 @@ jobs: ruff check . - name: Run unit tests run: | - coverage run tests.py - coverage json + pytest --cov --cov-report=xml tests.py - name: Test sample DMARC reports run: | pip install -e . @@ -61,3 +60,5 @@ jobs: hatch build - name: Upload coverage to Codecov uses: codecov/codecov-action@v4 + with: + token: ${{ secrets.CODECOV_TOKEN }} diff --git a/build.sh b/build.sh index ce37510d..34e9d84c 100755 --- a/build.sh +++ b/build.sh @@ -7,7 +7,7 @@ if [ ! -d "venv" ]; then fi . venv/bin/activate -pip install -U -r requirements.txt +pip install .[build] ruff format . cd docs make clean diff --git a/docs/source/installation.md b/docs/source/installation.md index aaea4587..f2039ba3 100644 --- a/docs/source/installation.md +++ b/docs/source/installation.md @@ -177,7 +177,7 @@ source /opt/parsedmarc/venv/bin/activate To install or upgrade `parsedmarc` inside the virtualenv, run: ```bash -sudo -u parsedmarc /opt/parsedmarc/venv/bin/pip install -U parsedmarc +sudo -u parsedmarc /opt/parsedmarc/venv/bin/pip install -U parsedmarc[cli] ``` ## Optional dependencies diff --git a/pyproject.toml b/pyproject.toml index d6d1d719..5850d487 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -30,12 +30,8 @@ classifiers = [ ] dependencies = [ "azure-identity>=1.8.0", - "azure-monitor-ingestion>=1.0.0", - "boto3>=1.16.63", "dateparser>=1.1.1", "dnspython>=2.0.0", - "elasticsearch-dsl==7.4.0", - "elasticsearch<7.14.0", "expiringdict>=1.1.4", "geoip2>=3.0.0", "google-api-core>=2.4.0", @@ -44,16 +40,35 @@ dependencies = [ "google-auth-oauthlib>=0.4.6", "google-auth>=2.3.3", "imapclient>=2.1.0", - "kafka-python-ng>=2.2.2", "lxml>=4.4.0", "mailsuite>=1.9.17", "msgraph-core==0.2.2", - "opensearch-py>=2.4.2,<=3.0.0", "publicsuffixlist>=0.10.0", "requests>=2.22.0", "tqdm>=4.31.1", "urllib3>=1.25.7", "xmltodict>=0.12.0", +] + +[project.optional-dependencies] +build = [ + "hatch", + "myst-parser[linkify]", + "nose", + "pytest", + "pytest-cov", + "ruff", + "sphinx", + "sphinx_rtd_theme", +] +cli = [ + "azure-identity>=1.8.0", + "azure-monitor-ingestion>=1.0.0", + "boto3>=1.16.63", + "elasticsearch-dsl==7.4.0", + "elasticsearch<7.14.0", + "kafka-python-ng>=2.2.2", + "opensearch-py>=2.4.2,<=3.0.0", "pygelf>=0.4.2", ] diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index ba292bb6..00000000 --- a/requirements.txt +++ /dev/null @@ -1,46 +0,0 @@ -tqdm>=4.31.1 -pygments>=2.11.1 -dnspython>=2.0.0 -expiringdict>=1.1.4 -urllib3>=1.25.7 -requests>=2.22.0 -publicsuffixlist>=0.10.0 -xmltodict>=0.12.0 -geoip2>=3.0.0 -imapclient>=2.1.0 -dateparser>=1.1.1 -elasticsearch<7.14.0 -elasticsearch-dsl>=7.4.0 -opensearch-py>=2.4.2,<=3.0.0 -kafka-python-ng>=2.2.2 -mailsuite>=1.9.17 -pygelf -nose>=1.3.7 -wheel>=0.37.0 -ruff -jinja2>=2.10.1 -packaging>=19.1 -imagesize>=1.1.0 -alabaster>=0.7.12 -Babel>=2.7.0 -docutils<0.18,>=0.14 -sphinx>=1.0.5 -sphinx_rtd_theme>=0.4.3 -codecov>=2.0.15 -lxml>=4.4.0 -boto3>=1.16.63 -msgraph-core==0.2.2 -azure-identity>=1.8.0 -azure-monitor-ingestion>=1.0.0 -google-api-core>=2.4.0 -google-api-python-client>=2.35.0 -google-auth>=2.3.3 -google-auth-httplib2>=0.1.0 -google-auth-oauthlib>=0.4.6 -hatch>=1.5.0 -myst-parser>=0.18.0 -myst-parser[linkify] -requests -bs4 -pytest - From b33abae13a00cc89bb1744b5c1bc40c60c537ffa Mon Sep 17 00:00:00 2001 From: Jed Laundry Date: Mon, 7 Oct 2024 11:51:12 +1300 Subject: [PATCH 2/2] revert cli optional-dependencies group --- docs/source/installation.md | 2 +- pyproject.toml | 17 +++++++---------- 2 files changed, 8 insertions(+), 11 deletions(-) diff --git a/docs/source/installation.md b/docs/source/installation.md index f2039ba3..aaea4587 100644 --- a/docs/source/installation.md +++ b/docs/source/installation.md @@ -177,7 +177,7 @@ source /opt/parsedmarc/venv/bin/activate To install or upgrade `parsedmarc` inside the virtualenv, run: ```bash -sudo -u parsedmarc /opt/parsedmarc/venv/bin/pip install -U parsedmarc[cli] +sudo -u parsedmarc /opt/parsedmarc/venv/bin/pip install -U parsedmarc ``` ## Optional dependencies diff --git a/pyproject.toml b/pyproject.toml index 5850d487..7ffc954b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -30,8 +30,12 @@ classifiers = [ ] dependencies = [ "azure-identity>=1.8.0", + "azure-monitor-ingestion>=1.0.0", + "boto3>=1.16.63", "dateparser>=1.1.1", "dnspython>=2.0.0", + "elasticsearch-dsl==7.4.0", + "elasticsearch<7.14.0", "expiringdict>=1.1.4", "geoip2>=3.0.0", "google-api-core>=2.4.0", @@ -40,10 +44,13 @@ dependencies = [ "google-auth-oauthlib>=0.4.6", "google-auth>=2.3.3", "imapclient>=2.1.0", + "kafka-python-ng>=2.2.2", "lxml>=4.4.0", "mailsuite>=1.9.17", "msgraph-core==0.2.2", + "opensearch-py>=2.4.2,<=3.0.0", "publicsuffixlist>=0.10.0", + "pygelf>=0.4.2", "requests>=2.22.0", "tqdm>=4.31.1", "urllib3>=1.25.7", @@ -61,16 +68,6 @@ build = [ "sphinx", "sphinx_rtd_theme", ] -cli = [ - "azure-identity>=1.8.0", - "azure-monitor-ingestion>=1.0.0", - "boto3>=1.16.63", - "elasticsearch-dsl==7.4.0", - "elasticsearch<7.14.0", - "kafka-python-ng>=2.2.2", - "opensearch-py>=2.4.2,<=3.0.0", - "pygelf>=0.4.2", -] [project.scripts] parsedmarc = "parsedmarc.cli:_main"