EDSC-4265: Adds support for generateNotebook Lambda

nasa · Nov 21, 2024 · cdcd1cf · cdcd1cf
1 parent cd4f064
commit cdcd1cf
Show file tree

Hide file tree

Showing 12 changed files with 3,623 additions and 627 deletions.
diff --git a/package-lock.json b/package-lock.json
diff --git a/package.json b/package.json
@@ -89,6 +89,7 @@
     "@aws-sdk/client-secrets-manager": "^3.352.0",
     "@aws-sdk/client-sfn": "^3.354.0",
     "@aws-sdk/client-sqs": "^3.352.0",
+    "@aws-sdk/s3-request-presigner": "^3.693.0",
     "@babel/eslint-parser": "^7.22.15",
     "@babel/plugin-syntax-dynamic-import": "^7.8.3",
     "@babel/plugin-transform-class-properties": "^7.22.5",
@@ -143,6 +144,7 @@
     "formdata-node": "^2.5.0",
     "formik": "^2.4.6",
     "geojson": "^0.5.0",
+    "handlebars": "^4.7.8",
     "hex-to-rgba": "^2.0.1",
     "history": "^4.10.1",
     "jest-canvas-mock": "^2.5.2",
@@ -216,6 +218,7 @@
     "serverless-plugin-log-subscription": "^2.1.5",
     "serverless-plugin-split-stacks": "^1.12.0",
     "serverless-step-functions": "^3.11.0",
+    "serverless-s3-local": "^0.8.5",
     "serverless-webpack": "^5.10.0",
     "sharp": "^0.33.2",
     "simple-oauth2": "^4.3.0",

diff --git a/serverless-configs/aws-functions.yml b/serverless-configs/aws-functions.yml
@@ -51,6 +51,24 @@
     timeout: 900
 
   #
+  # Generate Jupyter Notebook Lambda
+  #
+  generateNotebook:
+    handler: serverless/src/generateNotebook/handler.default
+    timeout: ${env:LAMBDA_TIMEOUT, '30'}
+    events:
+      - http:
+          method: post
+          cors: ${file(./serverless-configs/${self:provider.name}-cors-configuration.yml)}
+          path: generateNotebook
+          authorizer:
+            name: edlOptionalAuthorizer
+            type: request
+            resultTtlInSeconds: 0
+    package:
+      patterns:
+        - 'serverless/src/generateNotebook/*.ipynb'
+
   # SQS Lambdas
   #
   processColorMap:

diff --git a/serverless-configs/aws-infrastructure-resources.yml b/serverless-configs/aws-infrastructure-resources.yml
@@ -140,6 +140,12 @@ Resources:
                 Action:
                   - states:*
                 Resource: '*'
+              - Effect: Allow
+                Action:
+                  - "s3:GetBucketLocation"
+                  - "s3:GetObject"
+                  - "s3:PutObject"
+                Resource: '*'
 
   # Redis Cache for browse-scaler/image-resizing
   # The CIDR notation 0.0. 0.0/0 defines an IP block containing all possible IP addresses

diff --git a/serverless-configs/aws-resources.yml b/serverless-configs/aws-resources.yml
@@ -174,11 +174,59 @@ Resources:
               - "s3:ListBucket"
               - "s3:ListAllMyBuckets"
               - "s3:GetObject"
+              - "s3:PutObject"
             Effect: "Allow"
             Resource: "*"
       PolicyName: ${self:provider.stage}-S3CloudfrontLogToCloudwatchPolicy
       Roles:
         - Ref: IamRoleCustomResourcesLambdaExecution
+
+  GenerateNotebooksBucket:
+    Type: AWS::S3::Bucket
+    Properties:
+      BucketName: ${self:custom.generateNotebooksBucketName}
+
+  GenerateNotebooksBucketPolicy:
+    Type: AWS::S3::BucketPolicy
+    Properties:
+      Bucket: ${self:custom.generateNotebooksBucketName}
+      PolicyDocument:
+        Version: '2012-10-17'
+        Statement:
+          - Effect: Allow
+            Principal:
+              AWS: 'arn:aws:iam::cloudfront:user/CloudFront Origin Access Identity E384JW5C6G2RZR'
+            Action:
+              - s3:GetObject*
+              - s3:ListBucket
+            Resource:
+              - arn:aws:s3:::${self:custom.generateNotebooksBucketName}/*
+              - arn:aws:s3:::${self:custom.generateNotebooksBucketName}
+          - Sid: Internet-Services-East-VPC-Access
+            Effect: Allow
+            Principal: '*'
+            Action:
+              - s3:GetObject*
+              - s3:ListBucket
+            Resource:
+              - arn:aws:s3:::${self:custom.generateNotebooksBucketName}/*
+              - arn:aws:s3:::${self:custom.generateNotebooksBucketName}
+            Condition:
+              StringEquals:
+          - Sid: Internet-Services-West-VPC-Access
+            Effect: Allow
+            Principal: '*'
+            Action:
+              - s3:GetObject*
+              - s3:ListBucket
+            Resource:
+              - arn:aws:s3:::${self:custom.generateNotebooksBucketName}/*
+              - arn:aws:s3:::${self:custom.generateNotebooksBucketName}
+            Condition:
+              StringEquals:
+                aws:sourceVpc: ${env:VPC_ID}
+
+
 Outputs:
   UpdateOrderStatusWorkflow:
     Description: ARN of the order status step function workflow

diff --git a/serverless.yml b/serverless.yml
@@ -10,6 +10,8 @@ provider:
   memorySize: 128
   environment:
     NODE_ENV: ${env:NODE_ENV, 'development'}
+
+    GENERATE_NOTEBOOKS_BUCKET_NAME: ${self:custom.generateNotebooksBucketName}
     # Variables for new Encrypted database
     databaseEndpoint:
       Fn::ImportValue: ${self:provider.stage}-EncryptedDatabaseEndpoint
@@ -47,6 +49,8 @@ provider:
 
     NODE_OPTIONS: '--enable-source-maps'
 
+    stage: ${self:provider.stage}
+
     # Redis cache configuration
     cacheHost:
       Fn::ImportValue: ${self:provider.stage}-ElastiCacheEndpoint
@@ -85,6 +89,7 @@ plugins:
   - serverless-plugin-log-subscription
   - serverless-plugin-ifelse
   - serverless-offline
+  - serverless-s3-local
 
 #
 # Lambda Functions
@@ -116,6 +121,8 @@ custom:
 
   infrastructureStackName: earthdata-search-infrastructure-${self:provider.stage}
 
+  generateNotebooksBucketName: ${self:custom.siteName}-generate-notebooks
+
   serverlessIfElse:
     # When invoking an offline lambda with `npm run invoke-local` this condition will disable serverless components that need to import or reference cloudformation values
     - If: '"${self:provider.stage}" == "invokeLocal"'
@@ -216,3 +223,7 @@ custom:
   logSubscription:
     enabled: true
     destinationArn: ${env:LOG_DESTINATION_ARN, ''}
+
+  s3:
+    host: localhost
+    directory: tmp
diff --git a/serverless/src/edlOptionalAuthorizer/handler.js b/serverless/src/edlOptionalAuthorizer/handler.js
@@ -32,7 +32,8 @@ const edlOptionalAuthorizer = async (event) => {
     const authOptionalPaths = [
       '/autocomplete',
       '/opensearch/granules',
-      '/collections/export'
+      '/collections/export',
+      '/generateNotebook'
     ]
 
     // Allow for optional authentication