Merge pull request #2055 from ASFHyP3/develop

Release v6.0.0
ASFHyP3 · Feb 2, 2024 · 78b1766 · 78b1766
2 parents 6018fac + 6852c75
commit 78b1766
Show file tree

Hide file tree

Showing 25 changed files with 740 additions and 490 deletions.
diff --git a/.github/actions/deploy-hyp3/action.yml b/.github/actions/deploy-hyp3/action.yml
@@ -35,11 +35,14 @@ inputs:
   CLOUDFORMATION_ROLE_ARN:
     description: "The CloudFormation role to use for this deployment"
     required: true
-  MONTHLY_JOB_QUOTA_PER_USER:
-    description: "The default number of jobs any user with an Earthdata Login can run per month"
+  DEFAULT_CREDITS_PER_USER:
+    description: "The default number of credits given to a new user"
+    required: true
+  RESET_CREDITS_MONTHLY:
+    description: "Whether to reset each user's remaining credits each month"
     required: true
   JOB_FILES:
-    description: "Space seperated list of job spec YAMLs to include"
+    description: "Space separated list of job spec YAMLs to include"
     required: true
   DEFAULT_MAX_VCPUS:
     description: "Default maximum size for the AWS Batch compute environment"
@@ -119,7 +122,8 @@ runs:
                 $CERTIFICATE_ARN \
                 $ORIGIN_ACCESS_IDENTITY_ID \
                 $DISTRIBUTION_URL \
-                MonthlyJobQuotaPerUser='${{ inputs.MONTHLY_JOB_QUOTA_PER_USER }}' \
+                DefaultCreditsPerUser='${{ inputs.DEFAULT_CREDITS_PER_USER }}' \
+                ResetCreditsMonthly='${{ inputs.RESET_CREDITS_MONTHLY }}' \
                 DefaultMaxvCpus='${{ inputs.DEFAULT_MAX_VCPUS }}' \
                 ExpandedMaxvCpus='${{ inputs.EXPANDED_MAX_VCPUS }}' \
                 MonthlyBudget='${{ inputs.MONTHLY_BUDGET }}' \

diff --git a/.github/workflows/deploy-credits-sandbox.yml b/.github/workflows/deploy-credits-sandbox.yml
@@ -0,0 +1,77 @@
+name: Deploy Credits Sandbox Stack to AWS
+
+on:
+  push:
+    branches:
+      - credits-sandbox
+
+concurrency: ${{ github.workflow }}-${{ github.ref }}
+
+jobs:
+  deploy:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - environment: hyp3-credits-sandbox
+            domain: hyp3-credits-sandbox.asf.alaska.edu
+            template_bucket: cf-templates-1hz9ldhhl4ahu-us-west-2
+            image_tag: test
+            product_lifetime_in_days: 14
+            default_credits_per_user: 0
+            reset_credits_monthly: true
+            deploy_ref: refs/heads/credits-sandbox
+            job_files: job_spec/AUTORIFT.yml job_spec/INSAR_GAMMA.yml job_spec/RTC_GAMMA.yml job_spec/INSAR_ISCE_BURST.yml
+            instance_types: r6id.xlarge,r6id.2xlarge,r6id.4xlarge,r6id.8xlarge,r6idn.xlarge,r6idn.2xlarge,r6idn.4xlarge,r6idn.8xlarge
+            default_max_vcpus: 640
+            expanded_max_vcpus: 640
+            required_surplus: 0
+            security_environment: ASF
+            ami_id: /aws/service/ecs/optimized-ami/amazon-linux-2023/recommended/image_id
+            distribution_url: ''
+
+    environment:
+      name: ${{ matrix.environment }}
+      url: https://${{ matrix.domain }}
+
+    steps:
+      - uses: actions/[email protected]
+
+      - uses: aws-actions/configure-aws-credentials@v4
+        with:
+          aws-access-key-id: ${{ secrets.V2_AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.V2_AWS_SECRET_ACCESS_KEY }}
+          aws-session-token: ${{ secrets.V2_AWS_SESSION_TOKEN }}
+          aws-region: ${{ secrets.AWS_REGION }}
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: 3.9
+
+      - uses: ./.github/actions/deploy-hyp3
+        with:
+          TEMPLATE_BUCKET:  ${{ matrix.template_bucket }}
+          STACK_NAME: ${{ matrix.environment }}
+          DOMAIN_NAME: ${{ matrix.domain }}
+          API_NAME: ${{ matrix.environment }}
+          CERTIFICATE_ARN:  ${{ secrets.CERTIFICATE_ARN }}
+          IMAGE_TAG: ${{ matrix.image_tag }}
+          PRODUCT_LIFETIME: ${{ matrix.product_lifetime_in_days }}
+          VPC_ID: ${{ secrets.VPC_ID }}
+          SUBNET_IDS: ${{ secrets.SUBNET_IDS }}
+          SECRET_ARN: ${{ secrets.SECRET_ARN }}
+          CLOUDFORMATION_ROLE_ARN: ${{ secrets.CLOUDFORMATION_ROLE_ARN }}
+          DEFAULT_CREDITS_PER_USER: ${{ matrix.default_credits_per_user }}
+          RESET_CREDITS_MONTHLY: ${{ matrix.reset_credits_monthly }}
+          JOB_FILES: ${{ matrix.job_files }}
+          DEFAULT_MAX_VCPUS: ${{ matrix.default_max_vcpus }}
+          EXPANDED_MAX_VCPUS: ${{ matrix.expanded_max_vcpus }}
+          MONTHLY_BUDGET: ${{ secrets.MONTHLY_BUDGET }}
+          REQUIRED_SURPLUS: ${{ matrix.required_surplus }}
+          ORIGIN_ACCESS_IDENTITY_ID: ${{ secrets.ORIGIN_ACCESS_IDENTITY_ID }}
+          SECURITY_ENVIRONMENT: ${{ matrix.security_environment }}
+          AMI_ID: ${{ matrix.ami_id }}
+          INSTANCE_TYPES: ${{ matrix.instance_types }}
+          DISTRIBUTION_URL: ${{ matrix.distribution_url }}
+          AUTH_PUBLIC_KEY: ${{ secrets.AUTH_PUBLIC_KEY }}
diff --git a/.github/workflows/deploy-daac.yml b/.github/workflows/deploy-daac.yml
@@ -21,7 +21,8 @@ jobs:
             template_bucket: cf-templates-118mtzosmrltk-us-west-2
             image_tag: latest
             product_lifetime_in_days: 14
-            quota: 1000
+            default_credits_per_user: 1000
+            reset_credits_monthly: true
             deploy_ref: refs/heads/main
             job_files: job_spec/AUTORIFT.yml job_spec/INSAR_GAMMA.yml job_spec/RTC_GAMMA.yml job_spec/INSAR_ISCE_BURST.yml
             instance_types: r6id.xlarge,r6id.2xlarge,r6id.4xlarge,r6id.8xlarge,r6idn.xlarge,r6idn.2xlarge,r6idn.4xlarge,r6idn.8xlarge
@@ -38,7 +39,8 @@ jobs:
             template_bucket: cf-templates-118ylv0o6jp2n-us-west-2
             image_tag: test
             product_lifetime_in_days: 14
-            quota: 1000
+            default_credits_per_user: 1000
+            reset_credits_monthly: true
             deploy_ref: refs/heads/develop
             job_files: >-
               job_spec/AUTORIFT.yml
@@ -86,7 +88,8 @@ jobs:
           SUBNET_IDS: ${{ secrets.SUBNET_IDS }}
           SECRET_ARN: ${{ secrets.SECRET_ARN }}
           CLOUDFORMATION_ROLE_ARN: ${{ secrets.CLOUDFORMATION_ROLE_ARN }}
-          MONTHLY_JOB_QUOTA_PER_USER: ${{ matrix.quota }}
+          DEFAULT_CREDITS_PER_USER: ${{ matrix.default_credits_per_user }}
+          RESET_CREDITS_MONTHLY: ${{ matrix.reset_credits_monthly }}
           JOB_FILES: ${{ matrix.job_files }}
           DEFAULT_MAX_VCPUS: ${{ matrix.default_max_vcpus }}
           EXPANDED_MAX_VCPUS: ${{ matrix.expanded_max_vcpus }}

diff --git a/.github/workflows/deploy-enterprise-test.yml b/.github/workflows/deploy-enterprise-test.yml
@@ -19,7 +19,8 @@ jobs:
             template_bucket: cf-templates-1iw894v4yzqya-us-west-2
             image_tag: test
             product_lifetime_in_days: 14
-            quota: 0
+            default_credits_per_user: 0
+            reset_credits_monthly: false
             deploy_ref: refs/heads/develop
             job_files: >-
               job_spec/AUTORIFT_ITS_LIVE.yml
@@ -69,7 +70,8 @@ jobs:
           SUBNET_IDS: ${{ secrets.SUBNET_IDS }}
           SECRET_ARN: ${{ secrets.SECRET_ARN }}
           CLOUDFORMATION_ROLE_ARN: ${{ secrets.CLOUDFORMATION_ROLE_ARN }}
-          MONTHLY_JOB_QUOTA_PER_USER: ${{ matrix.quota }}
+          DEFAULT_CREDITS_PER_USER: ${{ matrix.default_credits_per_user }}
+          RESET_CREDITS_MONTHLY: ${{ matrix.reset_credits_monthly }}
           JOB_FILES: ${{ matrix.job_files }}
           DEFAULT_MAX_VCPUS: ${{ matrix.default_max_vcpus }}
           EXPANDED_MAX_VCPUS: ${{ matrix.expanded_max_vcpus }}

diff --git a/.github/workflows/deploy-enterprise.yml b/.github/workflows/deploy-enterprise.yml
@@ -19,7 +19,8 @@ jobs:
             template_bucket: cf-templates-3o5lnspmwmzg-us-west-2
             image_tag: latest
             product_lifetime_in_days: 45
-            quota: 0
+            default_credits_per_user: 0
+            reset_credits_monthly: true
             job_files: >-
               job_spec/AUTORIFT_ITS_LIVE.yml
               job_spec/AUTORIFT_ITS_LIVE_TEST.yml
@@ -37,7 +38,8 @@ jobs:
             template_bucket: cf-templates-v4pvone059de-us-west-2
             image_tag: latest
             product_lifetime_in_days: 180
-            quota: 0
+            default_credits_per_user: 0
+            reset_credits_monthly: true
             job_files: job_spec/INSAR_ISCE.yml job_spec/INSAR_ISCE_TEST.yml
             instance_types: c6id.xlarge,c6id.2xlarge,c6id.4xlarge,c6id.8xlarge
             default_max_vcpus: 10000
@@ -52,7 +54,8 @@ jobs:
             template_bucket: cf-templates-1or0efwqffkgd-us-west-2
             image_tag: latest
             product_lifetime_in_days: 60
-            quota: 0
+            default_credits_per_user: 0
+            reset_credits_monthly: true
             job_files: job_spec/INSAR_ISCE.yml job_spec/INSAR_ISCE_TEST.yml
             instance_types: c6id.xlarge,c6id.2xlarge,c6id.4xlarge,c6id.8xlarge
             default_max_vcpus: 0
@@ -67,7 +70,8 @@ jobs:
             template_bucket: cf-templates-gdeyr9hh8rzs-us-west-2
             image_tag: latest
             product_lifetime_in_days: 14
-            quota: 0
+            default_credits_per_user: 0
+            reset_credits_monthly: true
             job_files: job_spec/INSAR_ISCE.yml job_spec/INSAR_ISCE_TEST.yml
             instance_types: c6id.xlarge,c6id.2xlarge,c6id.4xlarge,c6id.8xlarge
             default_max_vcpus: 1600
@@ -82,7 +86,8 @@ jobs:
             template_bucket: cf-templates-1x4a21iq1cba7-us-west-2
             image_tag: latest
             product_lifetime_in_days: 365000
-            quota: 0
+            default_credits_per_user: 0
+            reset_credits_monthly: true
             job_files: job_spec/INSAR_GAMMA.yml
             instance_types: r6id.xlarge,r6id.2xlarge,r6id.4xlarge,r6id.8xlarge,r6idn.xlarge,r6idn.2xlarge,r6idn.4xlarge,r6idn.8xlarge
             default_max_vcpus: 640
@@ -97,7 +102,8 @@ jobs:
             template_bucket: cf-templates-1217di08q7vwl-us-west-2
             image_tag: latest
             product_lifetime_in_days: 14
-            quota: 0
+            default_credits_per_user: 0
+            reset_credits_monthly: true
             job_files: job_spec/RTC_GAMMA.yml job_spec/WATER_MAP.yml job_spec/WATER_MAP_EQ.yml
             instance_types: r6id.xlarge,r6id.2xlarge,r6id.4xlarge,r6id.8xlarge,r6idn.xlarge,r6idn.2xlarge,r6idn.4xlarge,r6idn.8xlarge
             default_max_vcpus: 640
@@ -112,7 +118,8 @@ jobs:
             template_bucket: cf-templates-15gmiot9prm67-us-west-2
             image_tag: latest
             product_lifetime_in_days: 90
-            quota: 0
+            default_credits_per_user: 0
+            reset_credits_monthly: true
             job_files: job_spec/RTC_GAMMA.yml job_spec/WATER_MAP.yml job_spec/WATER_MAP_EQ.yml
             instance_types: r6id.xlarge,r6id.2xlarge,r6id.4xlarge,r6id.8xlarge,r6idn.xlarge,r6idn.2xlarge,r6idn.4xlarge,r6idn.8xlarge
             default_max_vcpus: 1600
@@ -127,7 +134,8 @@ jobs:
             template_bucket: cf-templates-xlga17noink6-us-west-2
             image_tag: latest
             product_lifetime_in_days: 30
-            quota: 0
+            default_credits_per_user: 0
+            reset_credits_monthly: true
             job_files: job_spec/INSAR_GAMMA.yml job_spec/INSAR_ISCE_BURST.yml
             instance_types: r6id.xlarge,r6id.2xlarge,r6id.4xlarge,r6id.8xlarge,r6idn.xlarge,r6idn.2xlarge,r6idn.4xlarge,r6idn.8xlarge
             default_max_vcpus: 640
@@ -142,7 +150,8 @@ jobs:
             template_bucket: cf-templates-j4kd746vpsuv-us-east-1
             image_tag: latest
             product_lifetime_in_days: 14
-            quota: 0
+            default_credits_per_user: 0
+            reset_credits_monthly: true
             job_files: job_spec/INSAR_GAMMA.yml job_spec/RTC_GAMMA.yml
             instance_types: r6id.xlarge,r6id.2xlarge,r6id.4xlarge,r6id.8xlarge,r6idn.xlarge,r6idn.2xlarge,r6idn.4xlarge,r6idn.8xlarge
             default_max_vcpus: 640
@@ -157,7 +166,8 @@ jobs:
             template_bucket: cf-templates-ez0805f6vy20-us-west-2
             image_tag: latest
             product_lifetime_in_days: 14
-            quota: 0
+            default_credits_per_user: 0
+            reset_credits_monthly: true
             job_files: job_spec/INSAR_GAMMA.yml job_spec/RTC_GAMMA.yml
             instance_types: r6id.xlarge,r6id.2xlarge,r6id.4xlarge,r6id.8xlarge,r6idn.xlarge,r6idn.2xlarge,r6idn.4xlarge,r6idn.8xlarge
             default_max_vcpus: 640
@@ -172,7 +182,8 @@ jobs:
             template_bucket: cf-templates-1qx2mwia5g4kh-us-west-2
             image_tag: latest
             product_lifetime_in_days: 30
-            quota: 0
+            default_credits_per_user: 0
+            reset_credits_monthly: true
             job_files: job_spec/INSAR_GAMMA.yml job_spec/RTC_GAMMA.yml
             instance_types: r6id.xlarge,r6id.2xlarge,r6id.4xlarge,r6id.8xlarge,r6idn.xlarge,r6idn.2xlarge,r6idn.4xlarge,r6idn.8xlarge
             default_max_vcpus: 640
@@ -189,7 +200,8 @@ jobs:
             # TODO product lifetime could be much shorter since they all get transferred to a separate
             # S3 bucket, but maybe we want to allow for a backlog of products-to-be-transferred?
             product_lifetime_in_days: 14
-            quota: 0
+            default_credits_per_user: 0
+            reset_credits_monthly: true
             job_files: job_spec/WATER_MAP.yml
             instance_types: r6id.xlarge,r6id.2xlarge,r6id.4xlarge,r6id.8xlarge,r6idn.xlarge,r6idn.2xlarge,r6idn.4xlarge,r6idn.8xlarge
             default_max_vcpus: 640
@@ -230,7 +242,8 @@ jobs:
           SUBNET_IDS: ${{ secrets.SUBNET_IDS }}
           SECRET_ARN: ${{ secrets.SECRET_ARN }}
           CLOUDFORMATION_ROLE_ARN: ${{ secrets.CLOUDFORMATION_ROLE_ARN }}
-          MONTHLY_JOB_QUOTA_PER_USER: ${{ matrix.quota }}
+          DEFAULT_CREDITS_PER_USER: ${{ matrix.default_credits_per_user }}
+          RESET_CREDITS_MONTHLY: ${{ matrix.reset_credits_monthly }}
           JOB_FILES: ${{ matrix.job_files }}
           DEFAULT_MAX_VCPUS: ${{ matrix.default_max_vcpus }}
           EXPANDED_MAX_VCPUS: ${{ matrix.expanded_max_vcpus }}

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -4,6 +4,21 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [6.0.0]
+
+HyP3's monthly quota system has been replaced by a credits system. Previously, HyP3 provided each user with a certain number of jobs per month. Now, each job costs a particular number of credits, and users spend credits when they submit jobs. This release assigns every job a cost of 1 credit, but future releases will assign a different credit cost to each job type. Additionally, the main production deployment (`https://hyp3-api.asf.alaska.edu`) resets each user's balance to 1,000 credits each month, effectively granting each user 1,000 jobs per month. Therefore, users should not notice any difference when ordering jobs via ASF's On Demand service at <https://search.asf.alaska.edu>.
+
+### Added
+- The `job` object returned by the `/jobs` API endpoint now includes a `credit_cost` attribute, which represents the job's cost in credits.
+- A `DAR` tag is now included in Earthdata Cloud deployments for each S3 bucket to communicate which contain objects
+  that required to be encrypted at rest.
+
+### Changed
+- The `quota` attribute of the `user` object returned by the `/user` API endpoint has been replaced by a `remaining_credits` attribute, which represents the user's remaining credits.
+
+### Removed
+- The non-functional CloudWatch alarm for API 5xx errors has been removed from the `monitoring` module. See [#2044](https://github.com/ASFHyP3/hyp3/issues/2044).
+
 ## [5.0.4]
 ### Added
 - `INSAR_ISCE_BURST` jobs are now available in the azdwr-hyp3 deployment.

diff --git a/apps/api/api-cf.yml.j2 b/apps/api/api-cf.yml.j2
@@ -12,9 +12,12 @@ Parameters:
   AuthAlgorithm:
     Type: String
 
-  MonthlyJobQuotaPerUser:
+  DefaultCreditsPerUser:
     Type: Number
 
+  ResetCreditsMonthly:
+    Type: String
+
   SystemAvailable:
     Type: String
 
@@ -41,9 +44,6 @@ Outputs:
   Url:
     Value: {{ '!Sub "https://${RestApi}.execute-api.${AWS::Region}.amazonaws.com/${Stage}/ui"' if security_environment == 'EDC' else '!Sub "https://${CustomDomainName}/ui"' }}
 
-  ApiId:
-    Value: !Ref RestApi
-
 Resources:
 
   RestApi:
@@ -161,13 +161,15 @@ Resources:
             Resource: !Sub "arn:aws:logs:${AWS::Region}:${AWS::AccountId}:log-group:/aws/lambda/*"
           - Effect: Allow
             Action:
-              - dynamodb:PutItem
+              - dynamodb:BatchWriteItem
               - dynamodb:Query
               - dynamodb:GetItem
             Resource: !Sub "arn:aws:dynamodb:${AWS::Region}:${AWS::AccountId}:table/${JobsTable}*"
           - Effect: Allow
             Action:
               - dynamodb:GetItem
+              - dynamodb:PutItem
+              - dynamodb:UpdateItem
             Resource: !Sub "arn:aws:dynamodb:${AWS::Region}:${AWS::AccountId}:table/${UsersTable}*"
 
   Lambda:
@@ -179,7 +181,8 @@ Resources:
           USERS_TABLE_NAME: !Ref UsersTable
           AUTH_PUBLIC_KEY: !Ref AuthPublicKey
           AUTH_ALGORITHM: !Ref AuthAlgorithm
-          MONTHLY_JOB_QUOTA_PER_USER: !Ref MonthlyJobQuotaPerUser
+          DEFAULT_CREDITS_PER_USER: !Ref DefaultCreditsPerUser
+          RESET_CREDITS_MONTHLY: !Ref ResetCreditsMonthly
           SYSTEM_AVAILABLE: !Ref SystemAvailable
       Code: src/
       Handler: hyp3_api.lambda_handler.handler