From 49cc224994002982338ee7da41643a21e0378c59 Mon Sep 17 00:00:00 2001 From: Andrew Player Date: Fri, 28 Jun 2024 14:08:02 -0500 Subject: [PATCH 01/27] changes for upgraded python version --- .github/workflows/deploy.yml | 2 +- api/requirements.txt | 1 + api/src/api.py | 2 +- find_new/requirements.txt | 2 +- harvest_products/requirements.txt | 2 +- tests/conftest.py | 4 ++-- tests/test_find_new.py | 18 +++++++++++------- 7 files changed, 18 insertions(+), 13 deletions(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 6caa3a6..bb7269d 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -44,7 +44,7 @@ jobs: - uses: actions/setup-python@v1 with: - python-version: 3.8 + python-version: 3.12 - name: install dependencies and render templates run: | diff --git a/api/requirements.txt b/api/requirements.txt index 377078b..6b16147 100644 --- a/api/requirements.txt +++ b/api/requirements.txt @@ -1,5 +1,6 @@ flask flask_api +werkzeug<3 Flask-Cors serverless_wsgi ./database diff --git a/api/src/api.py b/api/src/api.py index fbf1fb2..1425a12 100644 --- a/api/src/api.py +++ b/api/src/api.py @@ -1,8 +1,8 @@ from datetime import datetime, timedelta, timezone from decimal import Decimal +from json import JSONEncoder from flask import Flask, abort, jsonify -from flask.json import JSONEncoder from flask_api.status import HTTP_404_NOT_FOUND from flask_cors import CORS from serverless_wsgi import handle_request diff --git a/find_new/requirements.txt b/find_new/requirements.txt index 582b272..9d136fb 100644 --- a/find_new/requirements.txt +++ b/find_new/requirements.txt @@ -1,5 +1,5 @@ asf-search==3.1.3 -hyp3-sdk>=1.3.2 +hyp3-sdk<6.1.0 python-dateutil requests ./database diff --git a/harvest_products/requirements.txt b/harvest_products/requirements.txt index 3e980bc..66ae3e9 100644 --- a/harvest_products/requirements.txt +++ b/harvest_products/requirements.txt @@ -1,3 +1,3 @@ -hyp3-sdk>=1.3.2 +hyp3-sdk<6.1.0 requests ./database diff --git a/tests/conftest.py b/tests/conftest.py index 341b939..a21df1c 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -5,7 +5,7 @@ import pytest import yaml from botocore.stub import Stubber -from moto import mock_dynamodb2 +from moto import mock_aws import api import harvest_products @@ -23,7 +23,7 @@ def get_table_properties_from_template(resource_name): @pytest.fixture def tables(): - with mock_dynamodb2(): + with mock_aws(): database.DB = boto3.resource('dynamodb') class Tables: diff --git a/tests/test_find_new.py b/tests/test_find_new.py index 39f7e2f..b1edb91 100644 --- a/tests/test_find_new.py +++ b/tests/test_find_new.py @@ -338,7 +338,7 @@ def test_get_neighbors_max_neighbors_error(): @responses.activate def test_submit_jobs_for_granule(tables): - responses.add(responses.GET, AUTH_URL) + responses.add(responses.GET, AUTH_URL, json.dumps({})) mock_hyp3_response = { 'jobs': [ { @@ -393,6 +393,10 @@ def test_submit_jobs_for_granule(tables): } event_id = 'event_id1' + mock_user = {"user_id": "some_user"} + responses.add(responses.GET, environ['HYP3_URL'] + '/user', json.dumps(mock_user)) + + hyp3 = HyP3(environ['HYP3_URL'], username=environ['EDL_USERNAME'], password=environ['EDL_PASSWORD']) with patch('find_new.get_neighbors', lambda x: mock_neighbors): find_new.submit_jobs_for_granule(hyp3, event_id, granule) @@ -590,13 +594,13 @@ def test_lambda_handler(tables): assert len(products) == 5 - assert products[2]['job_type'] == 'RTC_GAMMA' + assert products[4]['job_type'] == 'RTC_GAMMA' + assert products[4]['granules'][0]['granule_name'] == 'granule3' + + assert products[2]['job_type'] == 'INSAR_GAMMA' assert products[2]['granules'][0]['granule_name'] == 'granule3' + assert products[2]['granules'][1]['granule_name'] == 'neighbor1' assert products[3]['job_type'] == 'INSAR_GAMMA' assert products[3]['granules'][0]['granule_name'] == 'granule3' - assert products[3]['granules'][1]['granule_name'] == 'neighbor1' - - assert products[4]['job_type'] == 'INSAR_GAMMA' - assert products[4]['granules'][0]['granule_name'] == 'granule3' - assert products[4]['granules'][1]['granule_name'] == 'neighbor2' + assert products[3]['granules'][1]['granule_name'] == 'neighbor2' From a33692c380ab45833c95656a41d0676c27460c29 Mon Sep 17 00:00:00 2001 From: Andrew Player Date: Fri, 28 Jun 2024 14:10:36 -0500 Subject: [PATCH 02/27] changes for python version upgrade --- .github/workflows/test.yml | 2 +- README.md | 2 +- database/setup.py | 2 +- find_new/cloudformation.yml | 2 +- harvest_products/cloudformation.yml | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 93920e0..3ad6749 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -10,7 +10,7 @@ jobs: - uses: actions/setup-python@v1 with: - python-version: 3.8 + python-version: 3.12 - run: | python -m pip install --upgrade pip diff --git a/README.md b/README.md index 6e0e0f5..55c3a12 100644 --- a/README.md +++ b/README.md @@ -53,7 +53,7 @@ Review the parameters in [cloudformation.yml](cloudformation.yml) for deploy tim ### Deploy with CloudFormation -- Install dependencies for each component (requires pip for python 3.8) +- Install dependencies for each component (requires pip for python 3.12) ```sh python -m pip install -r find_new/requirements.txt -t find_new/src diff --git a/database/setup.py b/database/setup.py index 732713b..f7776f0 100644 --- a/database/setup.py +++ b/database/setup.py @@ -6,7 +6,7 @@ license='BSD', include_package_data=True, - python_requires='~=3.8', + python_requires='3.12', packages=find_packages(), ) diff --git a/find_new/cloudformation.yml b/find_new/cloudformation.yml index fea02e5..47f08e6 100644 --- a/find_new/cloudformation.yml +++ b/find_new/cloudformation.yml @@ -62,7 +62,7 @@ Resources: Handler: find_new.lambda_handler MemorySize: 128 Role: !GetAtt Role.Arn - Runtime: python3.8 + Runtime: python3.12 Timeout: 900 EventInvokeConfig: diff --git a/harvest_products/cloudformation.yml b/harvest_products/cloudformation.yml index 71fa6bf..5ae65cf 100644 --- a/harvest_products/cloudformation.yml +++ b/harvest_products/cloudformation.yml @@ -61,7 +61,7 @@ Resources: Handler: harvest_products.lambda_handler MemorySize: 2048 Role: !GetAtt Role.Arn - Runtime: python3.8 + Runtime: python3.12 Timeout: 900 EventInvokeConfig: From 0786c0b3dc282d7880bb3d39ad66cbb7d8bbc649 Mon Sep 17 00:00:00 2001 From: Andrew Player Date: Fri, 28 Jun 2024 14:21:48 -0500 Subject: [PATCH 03/27] set version correctly --- database/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/database/setup.py b/database/setup.py index f7776f0..6d33efe 100644 --- a/database/setup.py +++ b/database/setup.py @@ -6,7 +6,7 @@ license='BSD', include_package_data=True, - python_requires='3.12', + python_requires='~=3.12', packages=find_packages(), ) From e47d76a2c31ed4ccfa4d46a7d7038066c2386c2c Mon Sep 17 00:00:00 2001 From: Andrew Player Date: Fri, 28 Jun 2024 14:24:54 -0500 Subject: [PATCH 04/27] updated changelog --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index f601ea6..3d991aa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [PEP 440](https://www.python.org/dev/peps/pep-0440/) and uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [0.1.0] +### Removed +- Support for Python 3.8 has been removed. Python 3.12 is now supported. ## [0.0.13] ### Changed From 653c23a3f85d5ec586207b1472303786999f1bf3 Mon Sep 17 00:00:00 2001 From: Andrew Player Date: Fri, 28 Jun 2024 14:26:34 -0500 Subject: [PATCH 05/27] flake8 --- tests/test_find_new.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_find_new.py b/tests/test_find_new.py index b1edb91..89bb529 100644 --- a/tests/test_find_new.py +++ b/tests/test_find_new.py @@ -396,7 +396,6 @@ def test_submit_jobs_for_granule(tables): mock_user = {"user_id": "some_user"} responses.add(responses.GET, environ['HYP3_URL'] + '/user', json.dumps(mock_user)) - hyp3 = HyP3(environ['HYP3_URL'], username=environ['EDL_USERNAME'], password=environ['EDL_PASSWORD']) with patch('find_new.get_neighbors', lambda x: mock_neighbors): find_new.submit_jobs_for_granule(hyp3, event_id, granule) From 2ed44916364a26919df8afe87e2fd06a93b2e4ef Mon Sep 17 00:00:00 2001 From: Andrew Player Date: Fri, 28 Jun 2024 14:35:36 -0500 Subject: [PATCH 06/27] Update find_new/requirements.txt Co-authored-by: Jake Herrmann --- find_new/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/find_new/requirements.txt b/find_new/requirements.txt index 9d136fb..760b280 100644 --- a/find_new/requirements.txt +++ b/find_new/requirements.txt @@ -1,5 +1,5 @@ asf-search==3.1.3 -hyp3-sdk<6.1.0 +hyp3-sdk==6.1.0 python-dateutil requests ./database From 3dd60b6a3abf02dff26374b25d641f7d21040a11 Mon Sep 17 00:00:00 2001 From: Andrew Player Date: Fri, 28 Jun 2024 14:36:21 -0500 Subject: [PATCH 07/27] Update harvest_products/requirements.txt Co-authored-by: Jake Herrmann --- harvest_products/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/harvest_products/requirements.txt b/harvest_products/requirements.txt index 66ae3e9..cc30bdb 100644 --- a/harvest_products/requirements.txt +++ b/harvest_products/requirements.txt @@ -1,3 +1,3 @@ -hyp3-sdk<6.1.0 +hyp3-sdk==6.1.0 requests ./database From 57906af0b528e9fa419cab6f7e787d3432bbbc3b Mon Sep 17 00:00:00 2001 From: Andrew Player Date: Fri, 28 Jun 2024 14:43:24 -0500 Subject: [PATCH 08/27] reverted unnecessary changes --- tests/test_find_new.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/tests/test_find_new.py b/tests/test_find_new.py index 89bb529..3ca0e8a 100644 --- a/tests/test_find_new.py +++ b/tests/test_find_new.py @@ -338,7 +338,7 @@ def test_get_neighbors_max_neighbors_error(): @responses.activate def test_submit_jobs_for_granule(tables): - responses.add(responses.GET, AUTH_URL, json.dumps({})) + responses.add(responses.GET, AUTH_URL) mock_hyp3_response = { 'jobs': [ { @@ -393,9 +393,6 @@ def test_submit_jobs_for_granule(tables): } event_id = 'event_id1' - mock_user = {"user_id": "some_user"} - responses.add(responses.GET, environ['HYP3_URL'] + '/user', json.dumps(mock_user)) - hyp3 = HyP3(environ['HYP3_URL'], username=environ['EDL_USERNAME'], password=environ['EDL_PASSWORD']) with patch('find_new.get_neighbors', lambda x: mock_neighbors): find_new.submit_jobs_for_granule(hyp3, event_id, granule) From b38959784600ebac0b80645c52a19b6298396dea Mon Sep 17 00:00:00 2001 From: Andrew Player Date: Fri, 28 Jun 2024 14:53:49 -0500 Subject: [PATCH 09/27] pinning versions --- api/requirements.txt | 10 +++++----- find_new/requirements.txt | 4 ++-- harvest_products/requirements.txt | 2 +- requirements-all.txt | 18 +++++++++--------- 4 files changed, 17 insertions(+), 17 deletions(-) diff --git a/api/requirements.txt b/api/requirements.txt index 6b16147..e541306 100644 --- a/api/requirements.txt +++ b/api/requirements.txt @@ -1,6 +1,6 @@ -flask -flask_api -werkzeug<3 -Flask-Cors -serverless_wsgi +flask==2.3.3 +flask_api==3.1 +werkzeug==2.3.8 +Flask-Cors==4.0.1 +serverless_wsgi==3.0.4 ./database diff --git a/find_new/requirements.txt b/find_new/requirements.txt index 760b280..9c5d913 100644 --- a/find_new/requirements.txt +++ b/find_new/requirements.txt @@ -1,5 +1,5 @@ asf-search==3.1.3 hyp3-sdk==6.1.0 -python-dateutil -requests +python-dateutil==2.9.0.post0 +requests==2.32.3 ./database diff --git a/harvest_products/requirements.txt b/harvest_products/requirements.txt index cc30bdb..696d548 100644 --- a/harvest_products/requirements.txt +++ b/harvest_products/requirements.txt @@ -1,3 +1,3 @@ hyp3-sdk==6.1.0 -requests +requests==2.32.3 ./database diff --git a/requirements-all.txt b/requirements-all.txt index 30a8ac6..598518a 100644 --- a/requirements-all.txt +++ b/requirements-all.txt @@ -1,13 +1,13 @@ -boto3 -cfn-lint -flake8 -flake8-blind-except -flake8-builtins -flake8-import-order +boto3==1.34.135 +cfn-lint==1.4.2 +flake8==7.1.0 +flake8-blind-except==0.2.1 +flake8-builtins==2.5.0 +flake8-import-order==0.18.2 pytest -moto>=1.3.17dev240 -responses>=0.12.1 -pyYAML +moto==5.0.10.dev36 +responses==0.25.3 +pyYAML==6.0.1 -r find_new/requirements.txt -r api/requirements.txt -r harvest_products/requirements.txt From 59abc5580809551262cb6ddd5e5c4de4d8a72ec7 Mon Sep 17 00:00:00 2001 From: Andrew Player Date: Fri, 28 Jun 2024 14:55:06 -0500 Subject: [PATCH 10/27] pin pytest --- requirements-all.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-all.txt b/requirements-all.txt index 598518a..3f9f74d 100644 --- a/requirements-all.txt +++ b/requirements-all.txt @@ -4,7 +4,7 @@ flake8==7.1.0 flake8-blind-except==0.2.1 flake8-builtins==2.5.0 flake8-import-order==0.18.2 -pytest +pytest==8.2.2 moto==5.0.10.dev36 responses==0.25.3 pyYAML==6.0.1 From c6d2f8ddc2ea6f9e66090bf3b829220e3a2c3eb7 Mon Sep 17 00:00:00 2001 From: Jake Herrmann Date: Fri, 28 Jun 2024 11:56:07 -0800 Subject: [PATCH 11/27] add dependabot --- .github/dependabot.yml | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 .github/dependabot.yml diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..f95d8a5 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,19 @@ +# To get started with Dependabot version updates, you'll need to specify which +# package ecosystems to update and where the package manifests are located. +# Please see the documentation for all configuration options: +# https://docs.github.com/en/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file + +version: 2 +updates: + - package-ecosystem: pip + directory: / + schedule: + interval: weekly + labels: + - bumpless + - package-ecosystem: github-actions + directory: / + schedule: + interval: weekly + labels: + - bumpless From 60a1dfa1512602503eb6970b47a2d239daac44ee Mon Sep 17 00:00:00 2001 From: Jake Herrmann Date: Fri, 28 Jun 2024 12:00:55 -0800 Subject: [PATCH 12/27] rename requirements files --- .github/workflows/deploy.yml | 6 +++--- README.md | 6 +++--- requirements-all.txt | 6 +++--- api/requirements.txt => requirements-api.txt | 0 find_new/requirements.txt => requirements-find-new.txt | 0 .../requirements.txt => requirements-harvest-products.txt | 0 6 files changed, 9 insertions(+), 9 deletions(-) rename api/requirements.txt => requirements-api.txt (100%) rename find_new/requirements.txt => requirements-find-new.txt (100%) rename harvest_products/requirements.txt => requirements-harvest-products.txt (100%) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index bb7269d..6f581fb 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -49,9 +49,9 @@ jobs: - name: install dependencies and render templates run: | python -m pip install --upgrade pip - python -m pip install -r find_new/requirements.txt -t find_new/src - python -m pip install -r api/requirements.txt -t api/src - python -m pip install -r harvest_products/requirements.txt -t harvest_products/src + python -m pip install -r requirements-find-new.txt -t find_new/src + python -m pip install -r requirements-api.txt -t api/src + python -m pip install -r requirements-harvest-products.txt -t harvest_products/src - name: Package and deploy run: | diff --git a/README.md b/README.md index 55c3a12..e5794a3 100644 --- a/README.md +++ b/README.md @@ -56,9 +56,9 @@ Review the parameters in [cloudformation.yml](cloudformation.yml) for deploy tim - Install dependencies for each component (requires pip for python 3.12) ```sh -python -m pip install -r find_new/requirements.txt -t find_new/src -python -m pip install -r api/requirements.txt -t api/src -python -m pip install -r harvest_products/requirements.txt -t harvest_products/src +python -m pip install -r requirements-find-new.txt -t find_new/src +python -m pip install -r requirements-api.txt -t api/src +python -m pip install -r requirements-harvest-products.txt -t harvest_products/src ``` - Package the CloudFormation template diff --git a/requirements-all.txt b/requirements-all.txt index 3f9f74d..fb4b078 100644 --- a/requirements-all.txt +++ b/requirements-all.txt @@ -8,6 +8,6 @@ pytest==8.2.2 moto==5.0.10.dev36 responses==0.25.3 pyYAML==6.0.1 --r find_new/requirements.txt --r api/requirements.txt --r harvest_products/requirements.txt +-r requirements-find-new.txt +-r requirements-api.txt +-r requirements-harvest-products.txt diff --git a/api/requirements.txt b/requirements-api.txt similarity index 100% rename from api/requirements.txt rename to requirements-api.txt diff --git a/find_new/requirements.txt b/requirements-find-new.txt similarity index 100% rename from find_new/requirements.txt rename to requirements-find-new.txt diff --git a/harvest_products/requirements.txt b/requirements-harvest-products.txt similarity index 100% rename from harvest_products/requirements.txt rename to requirements-harvest-products.txt From 34dc35e744c0c87a43ec5fc4937c389e8a89a9f0 Mon Sep 17 00:00:00 2001 From: Andrew Player Date: Fri, 28 Jun 2024 15:17:31 -0500 Subject: [PATCH 13/27] Add Default Bucket OwnershipControls --- cloudformation.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cloudformation.yml b/cloudformation.yml index 378d08e..0bb24fb 100644 --- a/cloudformation.yml +++ b/cloudformation.yml @@ -20,6 +20,9 @@ Resources: Type: AWS::S3::Bucket Properties: AccessControl: LogDeliveryWrite + OwnershipControls: + Rules: + - ObjectOwnership: BucketOwnerEnforced PublicAccessBlockConfiguration: BlockPublicAcls: True IgnorePublicAcls: True From daca88196d43843da58d7c22dfe5ce0277af6afb Mon Sep 17 00:00:00 2001 From: Jake Herrmann Date: Fri, 28 Jun 2024 15:29:25 -0800 Subject: [PATCH 14/27] Update github workflows and delete pull request template --- .github/PULL_REQUEST_TEMPLATE/release.md | 37 ------------------- .github/pull_request_template.md | 8 ---- .github/workflows/changelog.yml | 4 +- .github/workflows/create-jira-issue.yml | 15 ++++++++ .github/workflows/deploy.yml | 3 +- .github/workflows/labeled-pr.yml | 15 ++++++++ .../workflows/release-template-comment.yml | 14 +++++++ .github/workflows/release.yml | 2 +- .github/workflows/static-analysis.yml | 14 ++++--- 9 files changed, 56 insertions(+), 56 deletions(-) delete mode 100644 .github/PULL_REQUEST_TEMPLATE/release.md delete mode 100644 .github/pull_request_template.md create mode 100644 .github/workflows/create-jira-issue.yml create mode 100644 .github/workflows/labeled-pr.yml create mode 100644 .github/workflows/release-template-comment.yml diff --git a/.github/PULL_REQUEST_TEMPLATE/release.md b/.github/PULL_REQUEST_TEMPLATE/release.md deleted file mode 100644 index da7cffe..0000000 --- a/.github/PULL_REQUEST_TEMPLATE/release.md +++ /dev/null @@ -1,37 +0,0 @@ -## Description of this release - - - - - -### Developer checklist - -- [ ] Assigned a reviewer - -- [ ] Indicated the level of changes to this package by affixing one of these labels: - * major -- Major changes to the API that may break current workflows - * minor -- Minor changes to the API that do not break current workflows - * patch -- Patches and bugfixes for the current version that do not break current workflows - * bumpless -- Changes to documentation, CI/CD pipelines, etc. that don't affect the software's version - -- [ ] (If applicable) Updated the dependencies and indicated any downstream changes that are required - -- [ ] Updated `CHANGELOG.md` -- [ ] Added/updated documentation for these changes -- [ ] Added/updated tests for these changes - -### Reviewer checklist - -- [ ] Have all dependencies been updated? -- [ ] Is the level of changes labeled appropriately? -- [ ] Are all the changes described appropriately in `CHANGELOG.md`? -- [ ] Has the documentation been adequately updated? -- [ ] Are the tests adequate? diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md deleted file mode 100644 index 21f719f..0000000 --- a/.github/pull_request_template.md +++ /dev/null @@ -1,8 +0,0 @@ - \ No newline at end of file diff --git a/.github/workflows/changelog.yml b/.github/workflows/changelog.yml index 57ae156..7740de9 100644 --- a/.github/workflows/changelog.yml +++ b/.github/workflows/changelog.yml @@ -13,6 +13,4 @@ on: jobs: call-changelog-check-workflow: - uses: ASFHyP3/actions/.github/workflows/reusable-changelog-check.yml@main - secrets: - USER_TOKEN: ${{ secrets.GITHUB_TOKEN }} + uses: ASFHyP3/actions/.github/workflows/reusable-changelog-check.yml@v0.11.2 diff --git a/.github/workflows/create-jira-issue.yml b/.github/workflows/create-jira-issue.yml new file mode 100644 index 0000000..99489d5 --- /dev/null +++ b/.github/workflows/create-jira-issue.yml @@ -0,0 +1,15 @@ +name: Create Jira issue + +on: + issues: + types: [labeled] + +jobs: + call-create-jira-issue-workflow: + uses: ASFHyP3/actions/.github/workflows/reusable-create-jira-issue.yml@v0.11.2 + secrets: + JIRA_BASE_URL: ${{ secrets.JIRA_BASE_URL }} + JIRA_USER_EMAIL: ${{ secrets.JIRA_USER_EMAIL }} + JIRA_API_TOKEN: ${{ secrets.JIRA_API_TOKEN }} + JIRA_PROJECT: ${{ secrets.JIRA_PROJECT }} + JIRA_FIELDS: ${{ secrets.JIRA_FIELDS }} diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 6f581fb..7bbe625 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -1,4 +1,3 @@ - name: Deploy to AWS on: @@ -73,6 +72,6 @@ jobs: call-bump-version-workflow: if: github.ref == 'refs/heads/main' needs: deploy - uses: ASFHyP3/actions/.github/workflows/reusable-bump-version.yml@main + uses: ASFHyP3/actions/.github/workflows/reusable-bump-version.yml@v0.11.2 secrets: USER_TOKEN: ${{ secrets.TOOLS_BOT_PAK }} diff --git a/.github/workflows/labeled-pr.yml b/.github/workflows/labeled-pr.yml new file mode 100644 index 0000000..f89f3e3 --- /dev/null +++ b/.github/workflows/labeled-pr.yml @@ -0,0 +1,15 @@ +name: Is PR labeled? + +on: + pull_request: + types: + - opened + - labeled + - unlabeled + - synchronize + branches: + - main + +jobs: + call-labeled-pr-check-workflow: + uses: ASFHyP3/actions/.github/workflows/reusable-labeled-pr-check.yml@v0.11.2 diff --git a/.github/workflows/release-template-comment.yml b/.github/workflows/release-template-comment.yml new file mode 100644 index 0000000..782d773 --- /dev/null +++ b/.github/workflows/release-template-comment.yml @@ -0,0 +1,14 @@ +on: + pull_request: + types: + - opened + branches: + - main + +jobs: + call-release-checklist-workflow: + uses: ASFHyP3/actions/.github/workflows/reusable-release-checklist-comment.yml@v0.11.2 + permissions: + pull-requests: write + secrets: + USER_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 1f5cabd..41586ee 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -7,7 +7,7 @@ on: jobs: call-release-workflow: - uses: ASFHyP3/actions/.github/workflows/reusable-release.yml@main + uses: ASFHyP3/actions/.github/workflows/reusable-release.yml@v0.11.2 with: release_prefix: HyP3 Event Monitoring secrets: diff --git a/.github/workflows/static-analysis.yml b/.github/workflows/static-analysis.yml index 8741ea6..e321a71 100644 --- a/.github/workflows/static-analysis.yml +++ b/.github/workflows/static-analysis.yml @@ -4,15 +4,19 @@ on: push jobs: call-flake8-workflow: - uses: ASFHyP3/actions/.github/workflows/reusable-flake8.yml@main + uses: ASFHyP3/actions/.github/workflows/reusable-flake8.yml@v0.11.2 with: local_package_names: api,database,find_new,harvest_products cfn-lint: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 - - - uses: scottbrenner/cfn-lint-action@master + - uses: actions/checkout@v4.1.7 + - uses: actions/setup-python@v5 with: - args: "--ignore-checks W3002 --template cloudformation.yml **/cloudformation.yml" + python-version: 3.12 + - run: | + python -m pip install --upgrade pip + python -m pip install -r requirements-all.txt + - run: | + cfn-lint --info --ignore-checks W3002 --template cloudformation.yml **/cloudformation.yml From 908b418522588eb69d7c46a034c393f19283799e Mon Sep 17 00:00:00 2001 From: Andrew Player Date: Mon, 1 Jul 2024 12:52:39 -0500 Subject: [PATCH 15/27] Updated S3 Logging --- cloudformation.yml | 46 +++++++++++++++++++++++++++++++++++++++------- 1 file changed, 39 insertions(+), 7 deletions(-) diff --git a/cloudformation.yml b/cloudformation.yml index 0bb24fb..752bdde 100644 --- a/cloudformation.yml +++ b/cloudformation.yml @@ -16,24 +16,53 @@ Parameters: Type: CommaDelimitedList Resources: - LogBucket: - Type: AWS::S3::Bucket + LoggingBucket: + Type: 'AWS::S3::Bucket' Properties: - AccessControl: LogDeliveryWrite - OwnershipControls: - Rules: - - ObjectOwnership: BucketOwnerEnforced PublicAccessBlockConfiguration: BlockPublicAcls: True IgnorePublicAcls: True BlockPublicPolicy: True RestrictPublicBuckets: True + BucketEncryption: + ServerSideEncryptionConfiguration: + - ServerSideEncryptionByDefault: + SSEAlgorithm: AES256 + BucketKeyEnabled: true + OwnershipControls: + Rules: + - ObjectOwnership: BucketOwnerEnforced + + LoggingBucketPolicy: + Type: 'AWS::S3::BucketPolicy' + Properties: + Bucket: !Ref LoggingBucket + PolicyDocument: + Version: 2012-10-17 + Statement: + - Action: + - 's3:PutObject' + Effect: Allow + Principal: + Service: logging.s3.amazonaws.com + Resource: !Join + - '' + - - 'arn:aws:s3:::' + - !Ref LoggingBucket + - /* + Condition: + ArnLike: + 'aws:SourceArn': !GetAtt + - ProductBucket + - Arn + StringEquals: + 'aws:SourceAccount': !Sub '${AWS::AccountId}' ProductBucket: Type: AWS::S3::Bucket Properties: LoggingConfiguration: - DestinationBucketName: !Ref LogBucket + DestinationBucketName: !Ref LoggingBucket LogFilePrefix: s3-access-logs/product-bucket/ LifecycleConfiguration: Rules: @@ -58,6 +87,9 @@ Resources: - HEAD AllowedOrigins: - "*.asf.alaska.edu" + OwnershipControls: + Rules: + - ObjectOwnership: BucketOwnerEnforced BucketPolicy: Type: AWS::S3::BucketPolicy From 2ead7f86639e0a7bb435b42f60bd4756b1121c85 Mon Sep 17 00:00:00 2001 From: Andrew Player Date: Mon, 1 Jul 2024 12:56:44 -0500 Subject: [PATCH 16/27] renamed back to LogBucket --- cloudformation.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/cloudformation.yml b/cloudformation.yml index 752bdde..1683074 100644 --- a/cloudformation.yml +++ b/cloudformation.yml @@ -16,7 +16,7 @@ Parameters: Type: CommaDelimitedList Resources: - LoggingBucket: + LogBucket: Type: 'AWS::S3::Bucket' Properties: PublicAccessBlockConfiguration: @@ -33,10 +33,10 @@ Resources: Rules: - ObjectOwnership: BucketOwnerEnforced - LoggingBucketPolicy: + LogBucketPolicy: Type: 'AWS::S3::BucketPolicy' Properties: - Bucket: !Ref LoggingBucket + Bucket: !Ref LogBucket PolicyDocument: Version: 2012-10-17 Statement: @@ -48,7 +48,7 @@ Resources: Resource: !Join - '' - - 'arn:aws:s3:::' - - !Ref LoggingBucket + - !Ref LogBucket - /* Condition: ArnLike: @@ -62,7 +62,7 @@ Resources: Type: AWS::S3::Bucket Properties: LoggingConfiguration: - DestinationBucketName: !Ref LoggingBucket + DestinationBucketName: !Ref LogBucket LogFilePrefix: s3-access-logs/product-bucket/ LifecycleConfiguration: Rules: From e59e1692deb8ac54f4379f521a9378cc289e455b Mon Sep 17 00:00:00 2001 From: Andrew Player Date: Mon, 1 Jul 2024 13:04:47 -0500 Subject: [PATCH 17/27] cleaning --- cloudformation.yml | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/cloudformation.yml b/cloudformation.yml index 1683074..24af86c 100644 --- a/cloudformation.yml +++ b/cloudformation.yml @@ -17,7 +17,7 @@ Parameters: Resources: LogBucket: - Type: 'AWS::S3::Bucket' + Type: "AWS::S3::Bucket" Properties: PublicAccessBlockConfiguration: BlockPublicAcls: True @@ -34,29 +34,23 @@ Resources: - ObjectOwnership: BucketOwnerEnforced LogBucketPolicy: - Type: 'AWS::S3::BucketPolicy' + Type: "AWS::S3::BucketPolicy" Properties: Bucket: !Ref LogBucket PolicyDocument: Version: 2012-10-17 Statement: - Action: - - 's3:PutObject' + - "s3:PutObject" Effect: Allow Principal: Service: logging.s3.amazonaws.com - Resource: !Join - - '' - - - 'arn:aws:s3:::' - - !Ref LogBucket - - /* + Resource: !Sub "${LogBucket.Arn}/*" Condition: ArnLike: - 'aws:SourceArn': !GetAtt - - ProductBucket - - Arn + "aws:SourceArn": !GetAtt ContentBucket.Arn StringEquals: - 'aws:SourceAccount': !Sub '${AWS::AccountId}' + "aws:SourceAccount": !Sub "${AWS::AccountId}" ProductBucket: Type: AWS::S3::Bucket From d92f4ea3d3f0a90fe79577f216dadee6054319d1 Mon Sep 17 00:00:00 2001 From: Andrew Player Date: Mon, 1 Jul 2024 13:05:21 -0500 Subject: [PATCH 18/27] cleaning --- cloudformation.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cloudformation.yml b/cloudformation.yml index 24af86c..599d330 100644 --- a/cloudformation.yml +++ b/cloudformation.yml @@ -48,7 +48,7 @@ Resources: Resource: !Sub "${LogBucket.Arn}/*" Condition: ArnLike: - "aws:SourceArn": !GetAtt ContentBucket.Arn + "aws:SourceArn": !GetAtt ProductBucket.Arn StringEquals: "aws:SourceAccount": !Sub "${AWS::AccountId}" From ee2c60599f8293fecf647544dcbb08c38ea2bbb6 Mon Sep 17 00:00:00 2001 From: Andrew Player Date: Mon, 1 Jul 2024 13:23:10 -0500 Subject: [PATCH 19/27] Update cloudformation.yml Co-authored-by: Jake Herrmann --- cloudformation.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/cloudformation.yml b/cloudformation.yml index 599d330..61434a0 100644 --- a/cloudformation.yml +++ b/cloudformation.yml @@ -40,8 +40,7 @@ Resources: PolicyDocument: Version: 2012-10-17 Statement: - - Action: - - "s3:PutObject" + - Action: "s3:PutObject" Effect: Allow Principal: Service: logging.s3.amazonaws.com From f3d901797122c18078fabb66683a356a218606a1 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 1 Jul 2024 18:31:01 +0000 Subject: [PATCH 20/27] Bump actions/setup-python from 1 to 5 Bumps [actions/setup-python](https://github.com/actions/setup-python) from 1 to 5. - [Release notes](https://github.com/actions/setup-python/releases) - [Commits](https://github.com/actions/setup-python/compare/v1...v5) --- updated-dependencies: - dependency-name: actions/setup-python dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/deploy.yml | 2 +- .github/workflows/test.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 7bbe625..012c4ee 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -41,7 +41,7 @@ jobs: aws-secret-access-key: ${{ env.AWS_SECRET_ACCESS_KEY }} aws-region: ${{ env.AWS_REGION }} - - uses: actions/setup-python@v1 + - uses: actions/setup-python@v5 with: python-version: 3.12 diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 3ad6749..151810c 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -8,7 +8,7 @@ jobs: steps: - uses: actions/checkout@v2 - - uses: actions/setup-python@v1 + - uses: actions/setup-python@v5 with: python-version: 3.12 From d4811460a12a0810aabe58136742cacd7cdb0a18 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 1 Jul 2024 18:31:03 +0000 Subject: [PATCH 21/27] Bump aws-actions/configure-aws-credentials from 1 to 4 Bumps [aws-actions/configure-aws-credentials](https://github.com/aws-actions/configure-aws-credentials) from 1 to 4. - [Release notes](https://github.com/aws-actions/configure-aws-credentials/releases) - [Changelog](https://github.com/aws-actions/configure-aws-credentials/blob/main/CHANGELOG.md) - [Commits](https://github.com/aws-actions/configure-aws-credentials/compare/v1...v4) --- updated-dependencies: - dependency-name: aws-actions/configure-aws-credentials dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/deploy.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 7bbe625..f3acf4e 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -35,7 +35,7 @@ jobs: echo "STACK_NAME=hyp3-event-monitoring" >> $GITHUB_ENV echo "HYP3_URL=https://hyp3-api.asf.alaska.edu" >> $GITHUB_ENV - - uses: aws-actions/configure-aws-credentials@v1 + - uses: aws-actions/configure-aws-credentials@v4 with: aws-access-key-id: ${{ env.AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ env.AWS_SECRET_ACCESS_KEY }} From be44b95ab63e320fe0c6d051e115f295c95a5806 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 1 Jul 2024 18:31:05 +0000 Subject: [PATCH 22/27] Bump actions/checkout from 2 to 4 Bumps [actions/checkout](https://github.com/actions/checkout) from 2 to 4. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v2...v4) --- updated-dependencies: - dependency-name: actions/checkout dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/deploy.yml | 2 +- .github/workflows/static-analysis.yml | 2 +- .github/workflows/test.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 7bbe625..9f0d527 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -20,7 +20,7 @@ jobs: deploy: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 with: fetch-depth: 0 token: ${{ secrets.TOOLS_BOT_PAK }} diff --git a/.github/workflows/static-analysis.yml b/.github/workflows/static-analysis.yml index e321a71..9c4a36e 100644 --- a/.github/workflows/static-analysis.yml +++ b/.github/workflows/static-analysis.yml @@ -11,7 +11,7 @@ jobs: cfn-lint: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4.1.7 + - uses: actions/checkout@v4 - uses: actions/setup-python@v5 with: python-version: 3.12 diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 3ad6749..92bbfd5 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -6,7 +6,7 @@ jobs: pytest: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - uses: actions/setup-python@v1 with: From 80b411fb31be08f97bf43506bca4a7c24927259c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 1 Jul 2024 18:31:20 +0000 Subject: [PATCH 23/27] Bump moto from 5.0.10.dev36 to 5.0.11.dev2 Bumps [moto](https://github.com/getmoto/moto) from 5.0.10.dev36 to 5.0.11.dev2. - [Release notes](https://github.com/getmoto/moto/releases) - [Changelog](https://github.com/getmoto/moto/blob/master/CHANGELOG.md) - [Commits](https://github.com/getmoto/moto/commits) --- updated-dependencies: - dependency-name: moto dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements-all.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-all.txt b/requirements-all.txt index fb4b078..2af1200 100644 --- a/requirements-all.txt +++ b/requirements-all.txt @@ -5,7 +5,7 @@ flake8-blind-except==0.2.1 flake8-builtins==2.5.0 flake8-import-order==0.18.2 pytest==8.2.2 -moto==5.0.10.dev36 +moto==5.0.11.dev2 responses==0.25.3 pyYAML==6.0.1 -r requirements-find-new.txt From a770c7117919b0baedbcd67eb4f647449bec7702 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 1 Jul 2024 18:31:22 +0000 Subject: [PATCH 24/27] Bump boto3 from 1.34.135 to 1.34.136 Bumps [boto3](https://github.com/boto/boto3) from 1.34.135 to 1.34.136. - [Release notes](https://github.com/boto/boto3/releases) - [Commits](https://github.com/boto/boto3/compare/1.34.135...1.34.136) --- updated-dependencies: - dependency-name: boto3 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements-all.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-all.txt b/requirements-all.txt index fb4b078..06facbb 100644 --- a/requirements-all.txt +++ b/requirements-all.txt @@ -1,4 +1,4 @@ -boto3==1.34.135 +boto3==1.34.136 cfn-lint==1.4.2 flake8==7.1.0 flake8-blind-except==0.2.1 From 4fc85397d4dc22b1402926a2534905fa94a3c264 Mon Sep 17 00:00:00 2001 From: Jake Herrmann Date: Mon, 1 Jul 2024 10:41:39 -0800 Subject: [PATCH 25/27] Update requirements-all.txt --- requirements-all.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-all.txt b/requirements-all.txt index 2af1200..44d4f79 100644 --- a/requirements-all.txt +++ b/requirements-all.txt @@ -5,7 +5,7 @@ flake8-blind-except==0.2.1 flake8-builtins==2.5.0 flake8-import-order==0.18.2 pytest==8.2.2 -moto==5.0.11.dev2 +moto==5.0.10 responses==0.25.3 pyYAML==6.0.1 -r requirements-find-new.txt From c5293b26b070f6b96541f4a904a6cb3e10cbdeca Mon Sep 17 00:00:00 2001 From: Jake Herrmann Date: Mon, 1 Jul 2024 10:58:57 -0800 Subject: [PATCH 26/27] add deploy concurrency --- .github/workflows/deploy.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 4897629..2aea3cc 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -6,6 +6,8 @@ on: - main - develop +concurrency: ${{ github.workflow }}-${{ github.ref }} + env: AWS_REGION: us-west-2 TEMPLATE_BUCKET: cf-templates-aubvn3i9olmk-us-west-2 From 415d9a600e3ee63c517128a6d25ffb051b92d71e Mon Sep 17 00:00:00 2001 From: Andrew Player Date: Mon, 1 Jul 2024 14:03:50 -0500 Subject: [PATCH 27/27] fix flask versioning issues --- api/src/api.py | 3 +-- requirements-api.txt | 4 +--- tests/test_api.py | 18 ++++++++---------- 3 files changed, 10 insertions(+), 15 deletions(-) diff --git a/api/src/api.py b/api/src/api.py index 1425a12..d0131b7 100644 --- a/api/src/api.py +++ b/api/src/api.py @@ -3,7 +3,6 @@ from json import JSONEncoder from flask import Flask, abort, jsonify -from flask_api.status import HTTP_404_NOT_FOUND from flask_cors import CORS from serverless_wsgi import handle_request @@ -35,7 +34,7 @@ def get_event_by_id(event_id): try: event = database.get_event(event_id) except ValueError: - abort(HTTP_404_NOT_FOUND) + abort(404) event['products'] = database.get_products_for_event(event_id, status_code='SUCCEEDED') return jsonify(event) diff --git a/requirements-api.txt b/requirements-api.txt index e541306..5256949 100644 --- a/requirements-api.txt +++ b/requirements-api.txt @@ -1,6 +1,4 @@ -flask==2.3.3 -flask_api==3.1 -werkzeug==2.3.8 +flask==3.0.3 Flask-Cors==4.0.1 serverless_wsgi==3.0.4 ./database diff --git a/tests/test_api.py b/tests/test_api.py index 421f7b4..0034489 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -1,8 +1,6 @@ from datetime import datetime, timedelta, timezone from decimal import Decimal -from flask_api import status - from api import lambda_handler @@ -52,29 +50,29 @@ def seed_data(tables): def test_events(api_client, tables): response = api_client.get('/events') - assert response.status_code == status.HTTP_200_OK + assert response.status_code == 200 assert response.get_json() == [] seed_data(tables) response = api_client.get('/events') - assert response.status_code == status.HTTP_200_OK + assert response.status_code == 200 assert len(response.get_json()) == 2 def test_event_by_id(api_client, tables): response = api_client.get('/events/event1') - assert response.status_code == status.HTTP_404_NOT_FOUND + assert response.status_code == 404 seed_data(tables) response = api_client.get('/events/event1') - assert response.status_code == status.HTTP_200_OK + assert response.status_code == 200 assert response.get_json()['event_id'] == 'event1' assert response.get_json()['products'] == [] response = api_client.get('/events/event2') - assert response.status_code == status.HTTP_200_OK + assert response.status_code == 200 assert response.get_json()['event_id'] == 'event2' product_ids = [p['product_id'] for p in response.get_json()['products']] assert sorted(product_ids) == ['product1', 'product3', 'product4'] @@ -82,13 +80,13 @@ def test_event_by_id(api_client, tables): def test_recent_products(api_client, tables): response = api_client.get('/recent_products') - assert response.status_code == status.HTTP_200_OK + assert response.status_code == 200 assert response.get_json() == [] seed_data(tables) response = api_client.get('/recent_products') - assert response.status_code == status.HTTP_200_OK + assert response.status_code == 200 product_ids = [p['product_id'] for p in response.get_json()] assert sorted(product_ids) == ['product1', 'product3'] @@ -113,6 +111,6 @@ def test_lambda_handler(): 'headers': {}, } response = lambda_handler(event, None) - assert response['statusCode'] == status.HTTP_404_NOT_FOUND + assert response['statusCode'] == 404 assert response['headers']['Content-Type'] == 'text/html; charset=utf-8' assert response['isBase64Encoded'] is False