diff --git a/.github/PULL_REQUEST_TEMPLATE/release.md b/.github/PULL_REQUEST_TEMPLATE/release.md deleted file mode 100644 index da7cffe..0000000 --- a/.github/PULL_REQUEST_TEMPLATE/release.md +++ /dev/null @@ -1,37 +0,0 @@ -## Description of this release - - - - - -### Developer checklist - -- [ ] Assigned a reviewer - -- [ ] Indicated the level of changes to this package by affixing one of these labels: - * major -- Major changes to the API that may break current workflows - * minor -- Minor changes to the API that do not break current workflows - * patch -- Patches and bugfixes for the current version that do not break current workflows - * bumpless -- Changes to documentation, CI/CD pipelines, etc. that don't affect the software's version - -- [ ] (If applicable) Updated the dependencies and indicated any downstream changes that are required - -- [ ] Updated `CHANGELOG.md` -- [ ] Added/updated documentation for these changes -- [ ] Added/updated tests for these changes - -### Reviewer checklist - -- [ ] Have all dependencies been updated? -- [ ] Is the level of changes labeled appropriately? -- [ ] Are all the changes described appropriately in `CHANGELOG.md`? -- [ ] Has the documentation been adequately updated? -- [ ] Are the tests adequate? diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..f95d8a5 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,19 @@ +# To get started with Dependabot version updates, you'll need to specify which +# package ecosystems to update and where the package manifests are located. +# Please see the documentation for all configuration options: +# https://docs.github.com/en/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file + +version: 2 +updates: + - package-ecosystem: pip + directory: / + schedule: + interval: weekly + labels: + - bumpless + - package-ecosystem: github-actions + directory: / + schedule: + interval: weekly + labels: + - bumpless diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md deleted file mode 100644 index 21f719f..0000000 --- a/.github/pull_request_template.md +++ /dev/null @@ -1,8 +0,0 @@ - \ No newline at end of file diff --git a/.github/workflows/changelog.yml b/.github/workflows/changelog.yml index 57ae156..7740de9 100644 --- a/.github/workflows/changelog.yml +++ b/.github/workflows/changelog.yml @@ -13,6 +13,4 @@ on: jobs: call-changelog-check-workflow: - uses: ASFHyP3/actions/.github/workflows/reusable-changelog-check.yml@main - secrets: - USER_TOKEN: ${{ secrets.GITHUB_TOKEN }} + uses: ASFHyP3/actions/.github/workflows/reusable-changelog-check.yml@v0.11.2 diff --git a/.github/workflows/create-jira-issue.yml b/.github/workflows/create-jira-issue.yml new file mode 100644 index 0000000..99489d5 --- /dev/null +++ b/.github/workflows/create-jira-issue.yml @@ -0,0 +1,15 @@ +name: Create Jira issue + +on: + issues: + types: [labeled] + +jobs: + call-create-jira-issue-workflow: + uses: ASFHyP3/actions/.github/workflows/reusable-create-jira-issue.yml@v0.11.2 + secrets: + JIRA_BASE_URL: ${{ secrets.JIRA_BASE_URL }} + JIRA_USER_EMAIL: ${{ secrets.JIRA_USER_EMAIL }} + JIRA_API_TOKEN: ${{ secrets.JIRA_API_TOKEN }} + JIRA_PROJECT: ${{ secrets.JIRA_PROJECT }} + JIRA_FIELDS: ${{ secrets.JIRA_FIELDS }} diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 6caa3a6..2aea3cc 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -1,4 +1,3 @@ - name: Deploy to AWS on: @@ -7,6 +6,8 @@ on: - main - develop +concurrency: ${{ github.workflow }}-${{ github.ref }} + env: AWS_REGION: us-west-2 TEMPLATE_BUCKET: cf-templates-aubvn3i9olmk-us-west-2 @@ -21,7 +22,7 @@ jobs: deploy: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 with: fetch-depth: 0 token: ${{ secrets.TOOLS_BOT_PAK }} @@ -36,22 +37,22 @@ jobs: echo "STACK_NAME=hyp3-event-monitoring" >> $GITHUB_ENV echo "HYP3_URL=https://hyp3-api.asf.alaska.edu" >> $GITHUB_ENV - - uses: aws-actions/configure-aws-credentials@v1 + - uses: aws-actions/configure-aws-credentials@v4 with: aws-access-key-id: ${{ env.AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ env.AWS_SECRET_ACCESS_KEY }} aws-region: ${{ env.AWS_REGION }} - - uses: actions/setup-python@v1 + - uses: actions/setup-python@v5 with: - python-version: 3.8 + python-version: 3.12 - name: install dependencies and render templates run: | python -m pip install --upgrade pip - python -m pip install -r find_new/requirements.txt -t find_new/src - python -m pip install -r api/requirements.txt -t api/src - python -m pip install -r harvest_products/requirements.txt -t harvest_products/src + python -m pip install -r requirements-find-new.txt -t find_new/src + python -m pip install -r requirements-api.txt -t api/src + python -m pip install -r requirements-harvest-products.txt -t harvest_products/src - name: Package and deploy run: | @@ -73,6 +74,6 @@ jobs: call-bump-version-workflow: if: github.ref == 'refs/heads/main' needs: deploy - uses: ASFHyP3/actions/.github/workflows/reusable-bump-version.yml@main + uses: ASFHyP3/actions/.github/workflows/reusable-bump-version.yml@v0.11.2 secrets: USER_TOKEN: ${{ secrets.TOOLS_BOT_PAK }} diff --git a/.github/workflows/labeled-pr.yml b/.github/workflows/labeled-pr.yml new file mode 100644 index 0000000..f89f3e3 --- /dev/null +++ b/.github/workflows/labeled-pr.yml @@ -0,0 +1,15 @@ +name: Is PR labeled? + +on: + pull_request: + types: + - opened + - labeled + - unlabeled + - synchronize + branches: + - main + +jobs: + call-labeled-pr-check-workflow: + uses: ASFHyP3/actions/.github/workflows/reusable-labeled-pr-check.yml@v0.11.2 diff --git a/.github/workflows/release-template-comment.yml b/.github/workflows/release-template-comment.yml new file mode 100644 index 0000000..782d773 --- /dev/null +++ b/.github/workflows/release-template-comment.yml @@ -0,0 +1,14 @@ +on: + pull_request: + types: + - opened + branches: + - main + +jobs: + call-release-checklist-workflow: + uses: ASFHyP3/actions/.github/workflows/reusable-release-checklist-comment.yml@v0.11.2 + permissions: + pull-requests: write + secrets: + USER_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 1f5cabd..41586ee 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -7,7 +7,7 @@ on: jobs: call-release-workflow: - uses: ASFHyP3/actions/.github/workflows/reusable-release.yml@main + uses: ASFHyP3/actions/.github/workflows/reusable-release.yml@v0.11.2 with: release_prefix: HyP3 Event Monitoring secrets: diff --git a/.github/workflows/static-analysis.yml b/.github/workflows/static-analysis.yml index 8741ea6..9c4a36e 100644 --- a/.github/workflows/static-analysis.yml +++ b/.github/workflows/static-analysis.yml @@ -4,15 +4,19 @@ on: push jobs: call-flake8-workflow: - uses: ASFHyP3/actions/.github/workflows/reusable-flake8.yml@main + uses: ASFHyP3/actions/.github/workflows/reusable-flake8.yml@v0.11.2 with: local_package_names: api,database,find_new,harvest_products cfn-lint: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 - - - uses: scottbrenner/cfn-lint-action@master + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 with: - args: "--ignore-checks W3002 --template cloudformation.yml **/cloudformation.yml" + python-version: 3.12 + - run: | + python -m pip install --upgrade pip + python -m pip install -r requirements-all.txt + - run: | + cfn-lint --info --ignore-checks W3002 --template cloudformation.yml **/cloudformation.yml diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 93920e0..a86beef 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -6,11 +6,11 @@ jobs: pytest: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - - uses: actions/setup-python@v1 + - uses: actions/setup-python@v5 with: - python-version: 3.8 + python-version: 3.12 - run: | python -m pip install --upgrade pip diff --git a/CHANGELOG.md b/CHANGELOG.md index f601ea6..3d991aa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [PEP 440](https://www.python.org/dev/peps/pep-0440/) and uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [0.1.0] +### Removed +- Support for Python 3.8 has been removed. Python 3.12 is now supported. ## [0.0.13] ### Changed diff --git a/README.md b/README.md index 6e0e0f5..e5794a3 100644 --- a/README.md +++ b/README.md @@ -53,12 +53,12 @@ Review the parameters in [cloudformation.yml](cloudformation.yml) for deploy tim ### Deploy with CloudFormation -- Install dependencies for each component (requires pip for python 3.8) +- Install dependencies for each component (requires pip for python 3.12) ```sh -python -m pip install -r find_new/requirements.txt -t find_new/src -python -m pip install -r api/requirements.txt -t api/src -python -m pip install -r harvest_products/requirements.txt -t harvest_products/src +python -m pip install -r requirements-find-new.txt -t find_new/src +python -m pip install -r requirements-api.txt -t api/src +python -m pip install -r requirements-harvest-products.txt -t harvest_products/src ``` - Package the CloudFormation template diff --git a/api/requirements.txt b/api/requirements.txt deleted file mode 100644 index 377078b..0000000 --- a/api/requirements.txt +++ /dev/null @@ -1,5 +0,0 @@ -flask -flask_api -Flask-Cors -serverless_wsgi -./database diff --git a/api/src/api.py b/api/src/api.py index fbf1fb2..d0131b7 100644 --- a/api/src/api.py +++ b/api/src/api.py @@ -1,9 +1,8 @@ from datetime import datetime, timedelta, timezone from decimal import Decimal +from json import JSONEncoder from flask import Flask, abort, jsonify -from flask.json import JSONEncoder -from flask_api.status import HTTP_404_NOT_FOUND from flask_cors import CORS from serverless_wsgi import handle_request @@ -35,7 +34,7 @@ def get_event_by_id(event_id): try: event = database.get_event(event_id) except ValueError: - abort(HTTP_404_NOT_FOUND) + abort(404) event['products'] = database.get_products_for_event(event_id, status_code='SUCCEEDED') return jsonify(event) diff --git a/cloudformation.yml b/cloudformation.yml index 378d08e..61434a0 100644 --- a/cloudformation.yml +++ b/cloudformation.yml @@ -17,14 +17,39 @@ Parameters: Resources: LogBucket: - Type: AWS::S3::Bucket + Type: "AWS::S3::Bucket" Properties: - AccessControl: LogDeliveryWrite PublicAccessBlockConfiguration: BlockPublicAcls: True IgnorePublicAcls: True BlockPublicPolicy: True RestrictPublicBuckets: True + BucketEncryption: + ServerSideEncryptionConfiguration: + - ServerSideEncryptionByDefault: + SSEAlgorithm: AES256 + BucketKeyEnabled: true + OwnershipControls: + Rules: + - ObjectOwnership: BucketOwnerEnforced + + LogBucketPolicy: + Type: "AWS::S3::BucketPolicy" + Properties: + Bucket: !Ref LogBucket + PolicyDocument: + Version: 2012-10-17 + Statement: + - Action: "s3:PutObject" + Effect: Allow + Principal: + Service: logging.s3.amazonaws.com + Resource: !Sub "${LogBucket.Arn}/*" + Condition: + ArnLike: + "aws:SourceArn": !GetAtt ProductBucket.Arn + StringEquals: + "aws:SourceAccount": !Sub "${AWS::AccountId}" ProductBucket: Type: AWS::S3::Bucket @@ -55,6 +80,9 @@ Resources: - HEAD AllowedOrigins: - "*.asf.alaska.edu" + OwnershipControls: + Rules: + - ObjectOwnership: BucketOwnerEnforced BucketPolicy: Type: AWS::S3::BucketPolicy diff --git a/database/setup.py b/database/setup.py index 732713b..6d33efe 100644 --- a/database/setup.py +++ b/database/setup.py @@ -6,7 +6,7 @@ license='BSD', include_package_data=True, - python_requires='~=3.8', + python_requires='~=3.12', packages=find_packages(), ) diff --git a/find_new/cloudformation.yml b/find_new/cloudformation.yml index fea02e5..47f08e6 100644 --- a/find_new/cloudformation.yml +++ b/find_new/cloudformation.yml @@ -62,7 +62,7 @@ Resources: Handler: find_new.lambda_handler MemorySize: 128 Role: !GetAtt Role.Arn - Runtime: python3.8 + Runtime: python3.12 Timeout: 900 EventInvokeConfig: diff --git a/find_new/requirements.txt b/find_new/requirements.txt deleted file mode 100644 index 582b272..0000000 --- a/find_new/requirements.txt +++ /dev/null @@ -1,5 +0,0 @@ -asf-search==3.1.3 -hyp3-sdk>=1.3.2 -python-dateutil -requests -./database diff --git a/harvest_products/cloudformation.yml b/harvest_products/cloudformation.yml index 71fa6bf..5ae65cf 100644 --- a/harvest_products/cloudformation.yml +++ b/harvest_products/cloudformation.yml @@ -61,7 +61,7 @@ Resources: Handler: harvest_products.lambda_handler MemorySize: 2048 Role: !GetAtt Role.Arn - Runtime: python3.8 + Runtime: python3.12 Timeout: 900 EventInvokeConfig: diff --git a/harvest_products/requirements.txt b/harvest_products/requirements.txt deleted file mode 100644 index 3e980bc..0000000 --- a/harvest_products/requirements.txt +++ /dev/null @@ -1,3 +0,0 @@ -hyp3-sdk>=1.3.2 -requests -./database diff --git a/requirements-all.txt b/requirements-all.txt index 30a8ac6..7d4feb1 100644 --- a/requirements-all.txt +++ b/requirements-all.txt @@ -1,13 +1,13 @@ -boto3 -cfn-lint -flake8 -flake8-blind-except -flake8-builtins -flake8-import-order -pytest -moto>=1.3.17dev240 -responses>=0.12.1 -pyYAML --r find_new/requirements.txt --r api/requirements.txt --r harvest_products/requirements.txt +boto3==1.34.136 +cfn-lint==1.4.2 +flake8==7.1.0 +flake8-blind-except==0.2.1 +flake8-builtins==2.5.0 +flake8-import-order==0.18.2 +pytest==8.2.2 +moto==5.0.10 +responses==0.25.3 +pyYAML==6.0.1 +-r requirements-find-new.txt +-r requirements-api.txt +-r requirements-harvest-products.txt diff --git a/requirements-api.txt b/requirements-api.txt new file mode 100644 index 0000000..5256949 --- /dev/null +++ b/requirements-api.txt @@ -0,0 +1,4 @@ +flask==3.0.3 +Flask-Cors==4.0.1 +serverless_wsgi==3.0.4 +./database diff --git a/requirements-find-new.txt b/requirements-find-new.txt new file mode 100644 index 0000000..9c5d913 --- /dev/null +++ b/requirements-find-new.txt @@ -0,0 +1,5 @@ +asf-search==3.1.3 +hyp3-sdk==6.1.0 +python-dateutil==2.9.0.post0 +requests==2.32.3 +./database diff --git a/requirements-harvest-products.txt b/requirements-harvest-products.txt new file mode 100644 index 0000000..696d548 --- /dev/null +++ b/requirements-harvest-products.txt @@ -0,0 +1,3 @@ +hyp3-sdk==6.1.0 +requests==2.32.3 +./database diff --git a/tests/conftest.py b/tests/conftest.py index 341b939..a21df1c 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -5,7 +5,7 @@ import pytest import yaml from botocore.stub import Stubber -from moto import mock_dynamodb2 +from moto import mock_aws import api import harvest_products @@ -23,7 +23,7 @@ def get_table_properties_from_template(resource_name): @pytest.fixture def tables(): - with mock_dynamodb2(): + with mock_aws(): database.DB = boto3.resource('dynamodb') class Tables: diff --git a/tests/test_api.py b/tests/test_api.py index 421f7b4..0034489 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -1,8 +1,6 @@ from datetime import datetime, timedelta, timezone from decimal import Decimal -from flask_api import status - from api import lambda_handler @@ -52,29 +50,29 @@ def seed_data(tables): def test_events(api_client, tables): response = api_client.get('/events') - assert response.status_code == status.HTTP_200_OK + assert response.status_code == 200 assert response.get_json() == [] seed_data(tables) response = api_client.get('/events') - assert response.status_code == status.HTTP_200_OK + assert response.status_code == 200 assert len(response.get_json()) == 2 def test_event_by_id(api_client, tables): response = api_client.get('/events/event1') - assert response.status_code == status.HTTP_404_NOT_FOUND + assert response.status_code == 404 seed_data(tables) response = api_client.get('/events/event1') - assert response.status_code == status.HTTP_200_OK + assert response.status_code == 200 assert response.get_json()['event_id'] == 'event1' assert response.get_json()['products'] == [] response = api_client.get('/events/event2') - assert response.status_code == status.HTTP_200_OK + assert response.status_code == 200 assert response.get_json()['event_id'] == 'event2' product_ids = [p['product_id'] for p in response.get_json()['products']] assert sorted(product_ids) == ['product1', 'product3', 'product4'] @@ -82,13 +80,13 @@ def test_event_by_id(api_client, tables): def test_recent_products(api_client, tables): response = api_client.get('/recent_products') - assert response.status_code == status.HTTP_200_OK + assert response.status_code == 200 assert response.get_json() == [] seed_data(tables) response = api_client.get('/recent_products') - assert response.status_code == status.HTTP_200_OK + assert response.status_code == 200 product_ids = [p['product_id'] for p in response.get_json()] assert sorted(product_ids) == ['product1', 'product3'] @@ -113,6 +111,6 @@ def test_lambda_handler(): 'headers': {}, } response = lambda_handler(event, None) - assert response['statusCode'] == status.HTTP_404_NOT_FOUND + assert response['statusCode'] == 404 assert response['headers']['Content-Type'] == 'text/html; charset=utf-8' assert response['isBase64Encoded'] is False diff --git a/tests/test_find_new.py b/tests/test_find_new.py index 39f7e2f..3ca0e8a 100644 --- a/tests/test_find_new.py +++ b/tests/test_find_new.py @@ -590,13 +590,13 @@ def test_lambda_handler(tables): assert len(products) == 5 - assert products[2]['job_type'] == 'RTC_GAMMA' + assert products[4]['job_type'] == 'RTC_GAMMA' + assert products[4]['granules'][0]['granule_name'] == 'granule3' + + assert products[2]['job_type'] == 'INSAR_GAMMA' assert products[2]['granules'][0]['granule_name'] == 'granule3' + assert products[2]['granules'][1]['granule_name'] == 'neighbor1' assert products[3]['job_type'] == 'INSAR_GAMMA' assert products[3]['granules'][0]['granule_name'] == 'granule3' - assert products[3]['granules'][1]['granule_name'] == 'neighbor1' - - assert products[4]['job_type'] == 'INSAR_GAMMA' - assert products[4]['granules'][0]['granule_name'] == 'granule3' - assert products[4]['granules'][1]['granule_name'] == 'neighbor2' + assert products[3]['granules'][1]['granule_name'] == 'neighbor2'