EDSC-4265: Develop API endpoint to dynamically create jupyter notebook

bin/deploy-bamboo.sh

@@ -88,6 +88,8 @@ dockerRun() {
         -e "DB_ALLOCATED_STORAGE=$bamboo_DB_ALLOCATED_STORAGE" \
         -e "DB_INSTANCE_CLASS=$bamboo_DB_INSTANCE_CLASS" \
         -e "GIBS_JOB_ENABLED=$bamboo_GIBS_JOB_ENABLED" \
+        -e "Internet_Services_East_VPC=$bamboo_Internet_Services_East_VPC" \
+        -e "Internet_Services_West_VPC=$bamboo_Internet_Services_West_VPC" \
         -e "LAMBDA_TIMEOUT=$bamboo_LAMBDA_TIMEOUT" \
         -e "LOG_DESTINATION_ARN=$bamboo_LOG_DESTINATION_ARN" \
         -e "NODE_ENV=production" \

package.json

@@ -89,6 +89,7 @@
     "@aws-sdk/client-secrets-manager": "^3.352.0",
     "@aws-sdk/client-sfn": "^3.354.0",
     "@aws-sdk/client-sqs": "^3.352.0",
+    "@aws-sdk/s3-request-presigner": "^3.693.0",
     "@babel/eslint-parser": "^7.22.15",
     "@babel/plugin-syntax-dynamic-import": "^7.8.3",
     "@babel/plugin-transform-class-properties": "^7.22.5",
@@ -143,6 +144,7 @@
     "formdata-node": "^2.5.0",
     "formik": "^2.4.6",
     "geojson": "^0.5.0",
+    "handlebars": "^4.7.8",
     "hex-to-rgba": "^2.0.1",
     "history": "^4.10.1",
     "jest-canvas-mock": "^2.5.2",
@@ -216,6 +218,7 @@
     "serverless-plugin-log-subscription": "^2.1.5",
     "serverless-plugin-split-stacks": "^1.12.0",
     "serverless-step-functions": "^3.11.0",
+    "serverless-s3-local": "^0.8.5",
     "serverless-webpack": "^5.10.0",
     "sharp": "^0.33.2",
     "simple-oauth2": "^4.3.0",

serverless-configs/aws-functions.yml

@@ -51,6 +51,24 @@
     timeout: 900
 
   #
+  # Generate Jupyter Notebook Lambda
+  #
+  generateNotebook:
+    handler: serverless/src/generateNotebook/handler.default
+    timeout: ${env:LAMBDA_TIMEOUT, '30'}
+    events:
+      - http:
+          method: post
+          cors: ${file(./serverless-configs/${self:provider.name}-cors-configuration.yml)}
+          path: generateNotebook
+          authorizer:
+            name: edlOptionalAuthorizer
+            type: request
+            resultTtlInSeconds: 0
+    package:
+      patterns:
+        - 'serverless/src/generateNotebook/*.ipynb'
+
   # SQS Lambdas
   #
   processColorMap:

serverless-configs/aws-infrastructure-resources.yml

@@ -140,6 +140,12 @@ Resources:
                 Action:
                   - states:*
                 Resource: '*'
+              - Effect: Allow
+                Action:
+                  - "s3:GetBucketLocation"
+                  - "s3:GetObject"
+                  - "s3:PutObject"
+                Resource: '*'
 
   # Redis Cache for browse-scaler/image-resizing
   # The CIDR notation 0.0. 0.0/0 defines an IP block containing all possible IP addresses

serverless-configs/aws-resources.yml

@@ -174,11 +174,60 @@ Resources:
               - "s3:ListBucket"
               - "s3:ListAllMyBuckets"
               - "s3:GetObject"
+              - "s3:PutObject"
             Effect: "Allow"
             Resource: "*"
       PolicyName: ${self:provider.stage}-S3CloudfrontLogToCloudwatchPolicy
       Roles:
         - Ref: IamRoleCustomResourcesLambdaExecution
+
+  GenerateNotebooksBucket:
+    Type: AWS::S3::Bucket
+    Properties:
+      BucketName: ${self:custom.generateNotebooksBucketName}
+
+  GenerateNotebooksBucketPolicy:
+    Type: AWS::S3::BucketPolicy
+    Properties:
+      Bucket: ${self:custom.generateNotebooksBucketName}
+      PolicyDocument:
+        Version: '2012-10-17'
+        Statement:
+          - Effect: Allow
+            Principal:
+              AWS: 'arn:aws:iam::cloudfront:user/CloudFront Origin Access Identity E384JW5C6G2RZR'
+            Action:
+              - s3:GetObject*
+              - s3:ListBucket
+            Resource:
+              - arn:aws:s3:::${self:custom.generateNotebooksBucketName}/*
+              - arn:aws:s3:::${self:custom.generateNotebooksBucketName}
+          - Sid: Internet-Services-East-VPC-Access
+            Effect: Allow
+            Principal: '*'
+            Action:
+              - s3:GetObject*
+              - s3:ListBucket
+            Resource:
+              - arn:aws:s3:::${self:custom.generateNotebooksBucketName}/*
+              - arn:aws:s3:::${self:custom.generateNotebooksBucketName}
+            Condition:
+              StringEquals:
+                aws:sourceVpc: ${env:Internet_Services_East_VPC}
+          - Sid: Internet-Services-West-VPC-Access
+            Effect: Allow
+            Principal: '*'
+            Action:
+              - s3:GetObject*
+              - s3:ListBucket
+            Resource:
+              - arn:aws:s3:::${self:custom.generateNotebooksBucketName}/*
+              - arn:aws:s3:::${self:custom.generateNotebooksBucketName}
+            Condition:
+              StringEquals:
+                aws:sourceVpc: ${env:Internet_Services_West_VPC}
+
+
 Outputs:
   UpdateOrderStatusWorkflow:
     Description: ARN of the order status step function workflow

serverless.yml

@@ -10,6 +10,8 @@ provider:
   memorySize: 128
   environment:
     NODE_ENV: ${env:NODE_ENV, 'development'}
+
+    GENERATE_NOTEBOOKS_BUCKET_NAME: ${self:custom.generateNotebooksBucketName}
     # Variables for new Encrypted database
     databaseEndpoint:
       Fn::ImportValue: ${self:provider.stage}-EncryptedDatabaseEndpoint
@@ -47,6 +49,8 @@ provider:
 
     NODE_OPTIONS: '--enable-source-maps'
 
+    stage: ${self:provider.stage}
+
     # Redis cache configuration
     cacheHost:
       Fn::ImportValue: ${self:provider.stage}-ElastiCacheEndpoint
@@ -85,6 +89,7 @@ plugins:
   - serverless-plugin-log-subscription
   - serverless-plugin-ifelse
   - serverless-offline
+  - serverless-s3-local
 
 #
 # Lambda Functions
@@ -116,6 +121,8 @@ custom:
 
   infrastructureStackName: earthdata-search-infrastructure-${self:provider.stage}
 
+  generateNotebooksBucketName: ${self:custom.siteName}-generate-notebooks
+
   serverlessIfElse:
     # When invoking an offline lambda with `npm run invoke-local` this condition will disable serverless components that need to import or reference cloudformation values
     - If: '"${self:provider.stage}" == "invokeLocal"'
@@ -216,3 +223,7 @@ custom:
   logSubscription:
     enabled: true
     destinationArn: ${env:LOG_DESTINATION_ARN, ''}
+
+  s3:
+    host: localhost
+    directory: tmp

serverless/src/edlOptionalAuthorizer/handler.js

@@ -32,7 +32,8 @@ const edlOptionalAuthorizer = async (event) => {
     const authOptionalPaths = [
       '/autocomplete',
       '/opensearch/granules',
-      '/collections/export'
+      '/collections/export',
+      '/generateNotebook'
     ]
 
     // Allow for optional authentication