Merge branch 'master' into DOP-3977

.github/workflows/clean-feature-branch.yml

@@ -0,0 +1,31 @@
+on:
+  pull_request:
+    branches:
+      - master
+    types:
+      - closed
+
+name: Cleanup Feature Branch Infrastructure
+jobs: 
+  destroy:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-node@v1
+        with:
+          node-version: '18.x'
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v1
+        with:
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: us-east-2
+      - name: Cleanup Feature Branch Infrastructure
+        run: |
+          cd cdk-infra/
+          npm ci
+          npm run cdk destroy -- -c customFeatureName=enhancedApp-stg-${{github.head_ref}} --force \
+            auto-builder-stack-enhancedApp-stg-${{github.head_ref}}-worker \
+            auto-builder-stack-enhancedApp-stg-${{github.head_ref}}-queues \
+            auto-builder-stack-enhancedApp-stg-${{github.head_ref}}-webhooks

.github/workflows/deploy-feature-branch.yml

@@ -0,0 +1,32 @@
+on:
+  pull_request:
+    branches:
+    - master
+    types:
+      - opened
+      - reopened
+
+name: Initial Feature Branch Deploy
+jobs: 
+  deploy:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-node@v1
+        with:
+          node-version: '18.x'
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v1
+        with:
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: us-east-2
+      - name: Deploy Feature Branch Infrastructure
+        run: |
+          npm ci
+          cd cdk-infra/
+          npm ci
+          npm run deploy:feature -- -c env=stg -c customFeatureName=enhancedApp-stg-${{github.head_ref}} --outputs-file outputs.json
+      - name: Display resource URLs
+        run: cat cdk-infra/outputs.json

.github/workflows/update-feature-branch.yml

@@ -0,0 +1,58 @@
+on:
+  pull_request:
+    branches:
+      - master
+    types:
+      - synchronize
+
+concurrency:
+  group: environment-stg-feature-${{ github.ref }}
+  cancel-in-progress: true
+name: Update Feature Branch Infrastructure
+jobs: 
+  deploy:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-node@v1
+        with:
+          node-version: '18.x'
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v1
+        with:
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: us-east-2
+
+      - uses: actions/checkout@v2
+      - uses: dorny/paths-filter@v2
+        id: filter
+        with:
+          filters: |
+            webhooks:
+              - 'api/v2/**'
+              - 'cdk-infra/lib/constructs/api/**'
+              - 'cdk-infra/utils/**'
+            worker:
+              - 'src/**'
+              - 'cdk-infra/lib/constructs/worker/**'
+              - 'Dockerfile.enhanced'
+              - 'modules/**'
+      - name: Update Webhook Stack
+        if: steps.filter.outputs.webhooks == 'true'
+        run: |
+          npm ci
+          cd cdk-infra/
+          npm ci
+          npm run deploy:feature:stack -- -c env=stg -c customFeatureName=enhancedApp-stg-${{github.head_ref}} \
+            auto-builder-stack-enhancedApp-stg-${{github.head_ref}}-webhook
+  
+      - name: Update Worker Stack
+        if: steps.filter.outputs.worker == 'true'
+        run: |
+          npm ci
+          cd cdk-infra/
+          npm ci
+          npm run deploy:feature:stack -- -c env=stg -c customFeatureName=enhancedApp-stg-${{github.head_ref}} \
+            auto-builder-stack-enhancedApp-stg-${{github.head_ref}}-worker

cdk-infra/lib/constructs/api/webhook-env-construct.ts

@@ -21,6 +21,7 @@ export class WebhookEnvConstruct extends Construct {
 
     const dbName = StringParameter.valueFromLookup(this, `${ssmPrefix}/atlas/dbname`);
     const snootyDbName = StringParameter.valueFromLookup(this, `${ssmPrefix}/atlas/collections/snooty`);
+    const repoBranchesCollection = StringParameter.valueFromLookup(this, `${ssmPrefix}/atlas/collections/repo`);
     const dbUsername = StringParameter.valueFromLookup(this, `${ssmPrefix}/atlas/username`);
     const dbHost = StringParameter.valueFromLookup(this, `${ssmPrefix}/atlas/host`);
     const jobCollection = StringParameter.valueFromLookup(this, `${ssmPrefix}/atlas/collections/job/queue`);
@@ -37,6 +38,7 @@ export class WebhookEnvConstruct extends Construct {
       MONGO_ATLAS_URL: `mongodb+srv://${dbUsername}:${dbPassword}@${dbHost}/admin?retryWrites=true`,
       DB_NAME: dbName,
       SNOOTY_DB_NAME: snootyDbName,
+      REPO_BRANCHES_COL_NAME: repoBranchesCollection,
       JOB_QUEUE_COL_NAME: jobCollection,
       NODE_CONFIG_DIR: './config',
       JOBS_QUEUE_URL: jobsQueue.queueUrl,

cdk-infra/package.json

@@ -11,7 +11,7 @@
     "cdk": "cdk",
     "deploy:enhanced": "cdk deploy -c enhanced=true --require-approval never --all",
     "deploy:enhanced:worker": "cdk deploy auto-builder-stack-enhancedApp-worker -c enhanced=true  --require-approval never --exclusively",
-    "deploy:enhanced:webhooks": "cdk deploy auto-builder-stack-enhancedApp-webhooks -c enhanced=true--require-approval never --exclusively",
+    "deploy:enhanced:webhooks": "cdk deploy auto-builder-stack-enhancedApp-webhooks -c enhanced=true --require-approval never --exclusively",
     "deploy:enhanced:queues": "cdk deploy auto-builder-stack-enhancedApp-queues -c enhanced=true --require-approval never --exclusively",
     "deploy:feature": "cdk deploy -c enhanced=true --require-approval never --all",
     "deploy:feature:stack": "cdk deploy  -c enhanced=true --require-approval never --exclusively"

modules/oas-page-builder/src/services/pageBuilder.ts

@@ -45,6 +45,7 @@ interface AtlasSpecUrlParams {
   apiKeyword: string;
   apiVersion?: string;
   resourceVersion?: string;
+  latestResourceVersion?: string;
 }
 
 const ensureSavedVersionDataMatches = (versions: VersionData, apiVersion?: string, resourceVersion?: string) => {
@@ -57,7 +58,12 @@ const ensureSavedVersionDataMatches = (versions: VersionData, apiVersion?: strin
   }
 };
 
-const getAtlasSpecUrl = async ({ apiKeyword, apiVersion, resourceVersion }: AtlasSpecUrlParams) => {
+const getAtlasSpecUrl = async ({
+  apiKeyword,
+  apiVersion,
+  resourceVersion,
+  latestResourceVersion,
+}: AtlasSpecUrlParams) => {
   // Currently, the only expected API fetched programmatically is the Cloud Admin API,
   // but it's possible to have more in the future with varying processes.
   const keywords = ['cloud'];
@@ -66,7 +72,11 @@ const getAtlasSpecUrl = async ({ apiKeyword, apiVersion, resourceVersion }: Atla
   }
 
   const versionExtension = `${apiVersion ? `-v${apiVersion.split('.')[0]}` : ''}${
-    apiVersion && resourceVersion ? `-${resourceVersion}` : ''
+    apiVersion && resourceVersion
+      ? `-${resourceVersion}`
+      : apiVersion && latestResourceVersion && !resourceVersion
+      ? `-${latestResourceVersion}`
+      : ''
   }`;
 
   let oasFileURL;
@@ -169,6 +179,17 @@ async function getOASpec({
           resourceVersions.length - 1
         ];
 
+        /* Build the latest Resource Version spec if on a base API Version that has multiple Resource Versions
+         * Do not build the base API VERSION, since it may have out of order resources*/
+        const { oasFileURL, successfulGitHash } = await getAtlasSpecUrl({
+          apiKeyword: source,
+          apiVersion,
+          resourceVersion,
+          latestResourceVersion,
+        });
+        spec = oasFileURL;
+        isSuccessfulBuild = successfulGitHash;
+
         versionOptions = {
           active: {
             apiVersion,

modules/oas-page-builder/tests/unit/services/pageBuilder.test.ts

@@ -192,7 +192,7 @@ describe('pageBuilder', () => {
 
     await buildOpenAPIPages(testEntries, testOptions);
 
-    expect(mockExecute).toBeCalledTimes(testEntries.length * 2);
+    expect(mockExecute).toBeCalledTimes(4);
     // Local
     expect(mockExecute).toBeCalledWith(
       `${testOptions.repo}/source${testEntries[0][1].source}`,
@@ -201,12 +201,6 @@ describe('pageBuilder', () => {
       getExpectedVersionOptions(`${SITE_URL}/${testEntries[0][0]}`)
     );
 
-    expect(mockExecute).toBeCalledWith(
-      `${testOptions.repo}/source${testEntries[0][1].source}`,
-      `${testOptions.output}/${testEntries[0][0]}/index.html`,
-      expectedDefaultBuildOptions,
-      getExpectedVersionOptions(`${SITE_URL}/${testEntries[0][0]}`)
-    );
     // Url
     expect(mockExecute).toBeCalledWith(
       `${testEntries[1][1].source}`,
@@ -215,12 +209,6 @@ describe('pageBuilder', () => {
       getExpectedVersionOptions(`${SITE_URL}/${testEntries[1][0]}`)
     );
 
-    expect(mockExecute).toBeCalledWith(
-      `${testEntries[1][1].source}`,
-      getExpectedOutputPath(testOptions.output, testEntries[1][0], API_VERSION),
-      expectedDefaultBuildOptions,
-      getExpectedVersionOptions(`${SITE_URL}/${testEntries[1][0]}`)
-    );
     // Atlas
     expect(mockExecute).toBeCalledWith(
       `https://mongodb-mms-prod-build-server.s3.amazonaws.com/openapi/${MOCKED_GIT_HASH}-v2-${RESOURCE_VERSION}.json`,
@@ -230,8 +218,8 @@ describe('pageBuilder', () => {
     );
 
     expect(mockExecute).toBeCalledWith(
-      `https://mongodb-mms-prod-build-server.s3.amazonaws.com/openapi/${MOCKED_GIT_HASH}-v2.json`,
-      getExpectedOutputPath(testOptions.output, testEntries[2][0], API_VERSION),
+      `https://mongodb-mms-prod-build-server.s3.amazonaws.com/openapi/${MOCKED_GIT_HASH}-v2-${RESOURCE_VERSION}.json`,
+      getExpectedOutputPath(testOptions.output, testEntries[2][0], API_VERSION, RESOURCE_VERSION),
       expectedAtlasBuildOptions,
       getExpectedVersionOptions(`${SITE_URL}/${testEntries[2][0]}`)
     );
@@ -280,8 +268,8 @@ describe('pageBuilder', () => {
     );
 
     expect(mockExecute).toBeCalledWith(
-      `https://mongodb-mms-prod-build-server.s3.amazonaws.com/openapi/${MOCKED_GIT_HASH}-v2.json`,
-      getExpectedOutputPath(testOptions.output, testEntries[0][0], API_VERSION),
+      `https://mongodb-mms-prod-build-server.s3.amazonaws.com/openapi/${MOCKED_GIT_HASH}-v2-${LATEST_RESOURCE_VERSION}.json`,
+      getExpectedOutputPath(testOptions.output, testEntries[0][0], API_VERSION, LATEST_RESOURCE_VERSION),
       expectedAtlasBuildOptions,
       getExpectedVersionOptions(`${SITE_URL}/${testEntries[0][0]}`, LATEST_RESOURCE_VERSION)
     );

src/enhanced/job/enhancedJobHandlerFactory.ts

@@ -4,6 +4,7 @@ import { JobHandler } from '../../job/jobHandler';
 import { IJobValidator } from '../../job/jobValidator';
 import { JobRepository } from '../../repositories/jobRepository';
 import { RepoBranchesRepository } from '../../repositories/repoBranchesRepository';
+import { RepoEntitlementsRepository } from '../../repositories/repoEntitlementsRepository';
 import { ICDNConnector } from '../../services/cdn';
 import { IJobCommandExecutor } from '../../services/commandExecutor';
 import { IFileSystemServices } from '../../services/fileServices';
@@ -35,7 +36,8 @@ export class EnhancedJobHandlerFactory {
     repoConnector: IRepoConnector,
     logger: IJobRepoLogger,
     validator: IJobValidator,
-    repoBranchesRepo: RepoBranchesRepository
+    repoBranchesRepo: RepoBranchesRepository,
+    repoEntitlementsRepo: RepoEntitlementsRepository
   ): JobHandler {
     const jt = job.payload?.jobType;
     if (!(jt in enhancedJobHandlerMap)) throw new InvalidJobError('Job type not supported');
@@ -50,7 +52,8 @@ export class EnhancedJobHandlerFactory {
       repoConnector,
       logger,
       validator,
-      repoBranchesRepo
+      repoBranchesRepo,
+      repoEntitlementsRepo
     );
   }
 }