diff --git a/.github/workflows/deploy-stg-ecs.yml b/.github/workflows/deploy-stg-ecs.yml index 2e0cfb462..ca799a2e3 100644 --- a/.github/workflows/deploy-stg-ecs.yml +++ b/.github/workflows/deploy-stg-ecs.yml @@ -3,6 +3,7 @@ on: branches: - "main" - "integration" + - "DOP-4451" concurrency: group: environment-stg-${{ github.ref }} cancel-in-progress: true diff --git a/.github/workflows/deploy-stg-enhanced-webhooks.yml b/.github/workflows/deploy-stg-enhanced-webhooks.yml index c69e736b6..46ebbd0d6 100644 --- a/.github/workflows/deploy-stg-enhanced-webhooks.yml +++ b/.github/workflows/deploy-stg-enhanced-webhooks.yml @@ -1,9 +1,10 @@ on: push: - paths: ['api/**', 'cdk-infra/lib/constructs/api/**', 'cdk-infra/utils/**'] + # paths: [api/**', 'cdk-infra/lib/constructs/api/**', 'cdk-infra/utils/**'] branches: - - 'main' - - 'integration' + - "main" + - "integration" + - "DOP-4451" concurrency: group: environment-stg-enhanced-webhooks-${{ github.ref }} cancel-in-progress: true diff --git a/.github/workflows/deploy-stg-enhanced-worker.yml b/.github/workflows/deploy-stg-enhanced-worker.yml index 24000195c..0381f6860 100644 --- a/.github/workflows/deploy-stg-enhanced-worker.yml +++ b/.github/workflows/deploy-stg-enhanced-worker.yml @@ -1,9 +1,10 @@ on: push: - paths: ['src/**', 'cdk-infra/lib/constructs/worker/**', 'Dockerfile', 'modules/**'] + # paths: ['src/**', 'cdk-infra/lib/constructs/worker/**', 'Dockerfile', 'modules/**'] branches: - 'main' - 'integration' + - 'DOP-4451' concurrency: group: environment-stg-enhanced-worker-${{ github.ref }} cancel-in-progress: true @@ -27,5 +28,8 @@ jobs: npm ci cd cdk-infra/ npm ci - npm run deploy:feature:stack -- -c env=dotcomstg -c customFeatureName=enhancedApp-dotcomstg auto-builder-stack-enhancedApp-dotcomstg-worker - npm run deploy:feature:stack -- -c env=stg -c customFeatureName=enhancedApp-stg auto-builder-stack-enhancedApp-stg-worker + npm run cdk destroy -- -c customFeatureName=enhancedApp-dotcomstg --force auto-builder-stack-enhancedApp-dotcomstg-webhooks + npm run deploy:feature:stack -- -c isFeatureBranch=true -c env=dotcomstg -c customFeatureName=enhancedApp-dotcomstg \ + auto-builder-stack-enhancedApp-dotcomstg-worker + npm run deploy:feature:stack -- -c isFeatureBranch=true -c env=dotcomstg -c customFeatureName=enhancedApp-dotcomstg \ + auto-builder-stack-enhancedApp-dotcomstg-webhooks diff --git a/.github/workflows/update-feature-branch.yml b/.github/workflows/update-feature-branch.yml index 1f16579d8..cf9ff4df6 100644 --- a/.github/workflows/update-feature-branch.yml +++ b/.github/workflows/update-feature-branch.yml @@ -41,47 +41,6 @@ jobs: cdk-infra/node_modules key: ${{ github.head_ref }} build-webhooks: - needs: prep-build - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@v1 - with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws-region: us-east-2 - - uses: actions/setup-node@v4 - with: - node-version: '18.x' - - uses: actions/cache/restore@v3 - id: cache-restore - with: - path: | - node_modules - cdk-infra/node_modules - key: ${{ github.head_ref }} - - name: Install Dependencies - if: steps.cache-restore.outputs.cache-hit != 'true' - run: | - npm ci - cd cdk-infra/ - npm ci - - uses: dorny/paths-filter@v2 - id: filter - with: - filters: | - webhooks: - - 'api/**' - - 'cdk-infra/lib/constructs/api/**' - - 'cdk-infra/utils/**' - - name: Update Webhook Stack - if: steps.filter.outputs.webhooks == 'true' - run: | - cd cdk-infra/ - npm run deploy:feature:stack -- -c isFeatureBranch=true -c env=stg -c customFeatureName=enhancedApp-stg-${{github.head_ref}} \ - auto-builder-stack-enhancedApp-stg-${{github.head_ref}}-webhooks - build-worker: needs: prep-build runs-on: ubuntu-latest steps: @@ -117,9 +76,15 @@ jobs: - 'cdk-infra/lib/constructs/worker/**' - 'Dockerfile' - 'modules/**' + webhooks: + - 'api/**' + - 'cdk-infra/lib/constructs/api/**' + - 'cdk-infra/utils/**' - name: Update Worker Stack - if: steps.filter.outputs.worker == 'true' run: | cd cdk-infra/ + npm run cdk destroy -- -c customFeatureName=enhancedApp-stg-${{github.head_ref}} --force auto-builder-stack-enhancedApp-stg-${{github.head_ref}}-webhooks npm run deploy:feature:stack -- -c isFeatureBranch=true -c env=stg -c customFeatureName=enhancedApp-stg-${{github.head_ref}} \ - auto-builder-stack-enhancedApp-stg-${{github.head_ref}}-worker + auto-builder-stack-enhancedApp-stg-${{github.head_ref}}-worker + npm run deploy:feature:stack -- -c isFeatureBranch=true -c env=stg -c customFeatureName=enhancedApp-stg-${{github.head_ref}} \ + auto-builder-stack-enhancedApp-stg-${{github.head_ref}}-webhooks diff --git a/Dockerfile b/Dockerfile index 65be09e59..cc75bf9e5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -75,7 +75,7 @@ ENV PATH="${WORK_DIRECTORY}/.cargo/bin:${PATH}" RUN curl https://raw.githubusercontent.com/mongodb/docs-worker-pool/meta/makefiles/shared.mk -o shared.mk # install snooty frontend and docs-tools -RUN git clone -b v${SNOOTY_FRONTEND_VERSION} --depth 1 https://github.com/mongodb/snooty.git \ +RUN git clone -b main --depth 1 https://github.com/mongodb/snooty.git \ && cd snooty \ # Need to remove omit dev as the filter functionality for the frontend depends on a dev dependency. && npm ci --legacy-peer-deps \ diff --git a/api/controllers/v1/slack.ts b/api/controllers/v1/slack.ts index 3b2f71864..f8d9e6c34 100644 --- a/api/controllers/v1/slack.ts +++ b/api/controllers/v1/slack.ts @@ -50,7 +50,7 @@ export const DisplayRepoOptions = async (event: APIGatewayEvent): Promise => { const decoded = decodeURIComponent(event.body).split('=')[1]; const parsed = JSON.parse(decoded); const stateValues = parsed.view.state.values; + console.info('testing', 'logging here'); + console.log(JSON.stringify(parsed.view)); //TODO: create an interface for slack view_submission payloads if (parsed.type !== 'view_submission') { diff --git a/api/controllers/v2/github.ts b/api/controllers/v2/github.ts index c97823694..0d00f47ad 100644 --- a/api/controllers/v2/github.ts +++ b/api/controllers/v2/github.ts @@ -2,7 +2,7 @@ import * as c from 'config'; import * as mongodb from 'mongodb'; import { APIGatewayEvent, APIGatewayProxyResult } from 'aws-lambda'; import { PushEvent, WorkflowRunCompletedEvent } from '@octokit/webhooks-types'; - +import { ECSClient, RunTaskCommand } from '@aws-sdk/client-ecs'; import { JobRepository } from '../../../src/repositories/jobRepository'; import { ConsoleLogger } from '../../../src/services/logger'; import { RepoBranchesRepository } from '../../../src/repositories/repoBranchesRepository'; @@ -86,7 +86,7 @@ async function createPayload({ if (!githubEvent) { throw new Error(`Non SmokeTest Deploy jobs must have a github Event`); } - action = 'push'; + action = 'automatedTest'; jobType = 'githubPush'; branchName = githubEvent.ref.split('/')[2]; url = githubEvent.repository?.clone_url; @@ -168,19 +168,51 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr body: `Workflow ${body.workflow_run.name} completed successfully but only Deploy Staging ECS workflow completion will trigger smoke test site deployments`, }; - // if the build was not building main branch, no need for smoke test sites - if (body.workflow_run.head_branch != 'main' || body.repository.fork) { - console.log('Build was not on master branch in main repo, sites will not deploy as no smoke tests are needed'); - return { - statusCode: 202, - headers: { 'Content-Type': 'text/plain' }, - body: `Build on branch ${body.workflow_run.head_branch} will not trigger site deployments as it was not on main branch in upstream repo`, - }; - } + // if the build was not building the main branch, no need for smoke test sites + + // if (body.workflow_run.head_branch != 'main' || body.repository.fork) { + // console.log('Build was not on master branch in main repo, sites will not deploy as no smoke tests are needed'); + // return { + // statusCode: 202, + // headers: { 'Content-Type': 'text/plain' }, + // body: `Build on branch ${body.workflow_run.head_branch} will not trigger site deployments as it was not on main branch in upstream repo`, + // }; + // } //automated test builds will always deploy in dotcomstg const env = 'dotcomstg'; + async function runAdditionalECSTasks() { + const { TASK_DEFINITION, CLUSTER, SUBNETS } = process.env; + console.log('RUN'); + + if (!TASK_DEFINITION) throw new Error('ERROR! process.env.TASK_DEFINITION is not defined'); + if (!CLUSTER) throw new Error('ERROR! process.env.CLUSTER is not defined'); + if (!SUBNETS) throw new Error('ERROR! process.env.SUBNETS is not defined'); + + console.log('Env Vars exist for run'); + + const client = new ECSClient({ + region: 'us-east-2', + }); + + const command = new RunTaskCommand({ + taskDefinition: TASK_DEFINITION, + cluster: CLUSTER, + launchType: 'FARGATE', + overrides: { + containerOverrides: [{ environment: [{ name: 'INDPENDENT_TASK', value: 'true' }] }], + }, + networkConfiguration: { + awsvpcConfiguration: { + subnets: JSON.parse(SUBNETS), + }, + }, + }); + + await client.send(command); + } + async function createAndInsertJob() { return await Promise.all( SMOKETEST_SITES.map(async (repoName): Promise => { @@ -208,10 +240,10 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr repoOwner, }); - //add logic for getting master branch, latest stable branch const job = await prepGithubPushPayload(body, payload, jobTitle); try { + // await runAdditionalECSTasks(); consoleLogger.info(job.title, 'Creating Job'); const jobId = await jobRepository.insertJob(job, c.get('jobsQueueUrl')); jobRepository.notify(jobId, c.get('jobUpdatesQueueUrl'), JobStatus.inQueue, 0); diff --git a/api/handlers/slack.ts b/api/handlers/slack.ts index 83d42c086..1cb90cdd6 100644 --- a/api/handlers/slack.ts +++ b/api/handlers/slack.ts @@ -23,45 +23,48 @@ export async function buildEntitledGroupsList(entitlement: any, repoBranchesRepo const repoOptions: any[] = []; for (const repo of entitlement.repos) { const [repoOwner, repoName, directoryPath] = repo.split('/'); - const branches = await repoBranchesRepository.getRepoBranches(repoName, directoryPath); - const options: any[] = []; - for (const branch of branches) { - const buildWithSnooty = branch['buildsWithSnooty']; - if (buildWithSnooty) { - const active = branch['active']; - const branchName = `${directoryPath ? `${directoryPath}/` : ''}${branch['gitBranchName']}`; - const repoPath = `${repoOwner}/${repoName}/${branchName}`; - let txt: string; - if (!active) { - txt = `(!inactive) ${repoPath}`; - } else { - txt = repoPath; + if (branches.length) { + const options: any[] = []; + for (const branch of branches) { + const buildWithSnooty = branch['buildsWithSnooty']; + if (buildWithSnooty) { + const active = branch['active']; + const branchName = `${directoryPath ? `${directoryPath}/` : ''}${branch['gitBranchName']}`; + const repoPath = `${repoOwner}/${repoName}/${branchName}`; + let txt: string; + if (!active) { + txt = `(!inactive) ${repoPath}`; + } else { + txt = repoPath; + } + options.push({ + text: { + type: 'plain_text', + text: txt, + }, + value: repoPath, + }); } - options.push({ - text: { - type: 'plain_text', - text: txt, - }, - value: repoPath, - }); } - } - const repoOption = { - label: { - type: 'plain_text', - text: repoName, - }, - //sort the options by version number - options: options.sort((branchOne, branchTwo) => - branchTwo.text.text - .toString() - .replace(/\d+/g, (n) => +n + 100000) - .localeCompare(branchOne.text.text.toString().replace(/\d+/g, (n) => +n + 100000)) - ), - }; - repoOptions.push(repoOption); + const repoOption = { + label: { + type: 'plain_text', + text: repoName, + }, + //sort the options by version number + options: options.sort((branchOne, branchTwo) => + branchTwo.text.text + .toString() + .replace(/\d+/g, (n) => +n + 100000) + .localeCompare(branchOne.text.text.toString().replace(/\d+/g, (n) => +n + 100000)) + ), + }; + repoOptions.push(repoOption); + } else { + console.log(repo, branches); + } } return repoOptions.sort((repoOne, repoTwo) => repoOne.label.text.localeCompare(repoTwo.label.text)); } diff --git a/cdk-infra/bin/cdk-infra.ts b/cdk-infra/bin/cdk-infra.ts index 5a7737137..952c89b69 100644 --- a/cdk-infra/bin/cdk-infra.ts +++ b/cdk-infra/bin/cdk-infra.ts @@ -38,12 +38,22 @@ async function main() { const stackName = `auto-builder-stack-${getFeatureName()}`; const queues = new AutoBuilderQueueStack(app, `${stackName}-queues`, { env }); - const { clusterName } = new WorkerStack(app, `${stackName}-worker`, { queues, workerSecureStrings, vpc, env }); + const { clusterName, taskDefinition } = new WorkerStack(app, `${stackName}-worker`, { + queues, + workerSecureStrings, + vpc, + env, + }); + + // TODO: Pass the VPC as a prop here like the queues or clusterName + // TODO: Pass the task definition that will be added from the Worker stack new WebhookStack(app, `${stackName}-webhooks`, { queues, clusterName, webhookSecureStrings, env, + vpc, + taskDefinition, }); new CacheUpdaterStack(app, `${stackName}-cache`, { diff --git a/cdk-infra/lib/constructs/api/webhook-api-construct.ts b/cdk-infra/lib/constructs/api/webhook-api-construct.ts index f892f20ea..c0855349d 100644 --- a/cdk-infra/lib/constructs/api/webhook-api-construct.ts +++ b/cdk-infra/lib/constructs/api/webhook-api-construct.ts @@ -7,6 +7,8 @@ import { IQueue } from 'aws-cdk-lib/aws-sqs'; import { Construct } from 'constructs'; import path from 'path'; import { getFeatureName } from '../../../utils/env'; +import { IVpc } from 'aws-cdk-lib/aws-ec2'; +import { TaskDefinition } from 'aws-cdk-lib/aws-ecs'; const HANDLERS_PATH = path.join(__dirname, '/../../../../api/controllers/v2'); @@ -31,10 +33,17 @@ interface WebhookApiConstructProps { jobsQueue: IQueue; jobUpdatesQueue: IQueue; environment: Record; + vpc: IVpc; + taskDefinition: TaskDefinition; + clusterName: string; } export class WebhookApiConstruct extends Construct { - constructor(scope: Construct, id: string, { jobsQueue, jobUpdatesQueue, environment }: WebhookApiConstructProps) { + constructor( + scope: Construct, + id: string, + { jobsQueue, jobUpdatesQueue, environment, vpc, taskDefinition, clusterName }: WebhookApiConstructProps + ) { super(scope, id); const timeout = Duration.minutes(2); @@ -79,10 +88,17 @@ export class WebhookApiConstruct extends Construct { runtime, handler: 'triggerSmokeTestAutomatedBuild', bundling, - environment, + environment: { + TASK_DEFINITION: taskDefinition.taskDefinitionArn, + SUBNETS: JSON.stringify(vpc.privateSubnets.map((subnet) => subnet.subnetId)), + CLUSTER: clusterName, + ...environment, + }, timeout, }); + taskDefinition.grantRun(githubSmokeTestBuildLambda); + const githubDeleteArtifactsLambda = new NodejsFunction(this, 'githubDeleteArtifactsLambda', { entry: `${HANDLERS_PATH}/github.ts`, runtime, diff --git a/cdk-infra/lib/constructs/worker/worker-construct.ts b/cdk-infra/lib/constructs/worker/worker-construct.ts index 63f550305..b24e8baa6 100644 --- a/cdk-infra/lib/constructs/worker/worker-construct.ts +++ b/cdk-infra/lib/constructs/worker/worker-construct.ts @@ -6,6 +6,7 @@ import { FargateService, FargateTaskDefinition, LogDrivers, + TaskDefinition, } from 'aws-cdk-lib/aws-ecs'; import { Effect, IRole, PolicyStatement, Role, ServicePrincipal } from 'aws-cdk-lib/aws-iam'; import { LogGroup } from 'aws-cdk-lib/aws-logs'; @@ -23,6 +24,7 @@ interface WorkerConstructProps { export class WorkerConstruct extends Construct { readonly ecsTaskRole: IRole; readonly clusterName: string; + readonly taskDefinition: TaskDefinition; constructor( scope: Construct, @@ -112,6 +114,7 @@ export class WorkerConstruct extends Construct { maxHealthyPercent: 200, }); + this.taskDefinition = taskDefinition; this.clusterName = cluster.clusterName; this.ecsTaskRole = taskRole; } diff --git a/cdk-infra/lib/stacks/webhook-stack.ts b/cdk-infra/lib/stacks/webhook-stack.ts index 41367a41b..303e1642d 100644 --- a/cdk-infra/lib/stacks/webhook-stack.ts +++ b/cdk-infra/lib/stacks/webhook-stack.ts @@ -1,5 +1,7 @@ import { Stack, StackProps } from 'aws-cdk-lib'; import { Construct } from 'constructs'; +import { Vpc } from 'aws-cdk-lib/aws-ec2'; +import { TaskDefinition } from 'aws-cdk-lib/aws-ecs'; import { AutoBuilderQueues } from './auto-builder-queue-stack'; import { WebhookApiConstruct } from '../constructs/api/webhook-api-construct'; import { WebhookEnvConstruct } from '../constructs/api/webhook-env-construct'; @@ -8,12 +10,16 @@ interface WebhookStackProps extends StackProps { webhookSecureStrings: Record; queues: AutoBuilderQueues; clusterName: string; + vpc: Vpc; + taskDefinition: TaskDefinition; } + +//TODO: use taskDefition and vpc somewhere here export class WebhookStack extends Stack { constructor( scope: Construct, id: string, - { queues, webhookSecureStrings, clusterName, ...props }: WebhookStackProps + { queues, webhookSecureStrings, clusterName, vpc, taskDefinition, ...props }: WebhookStackProps ) { super(scope, id, props); @@ -24,6 +30,9 @@ export class WebhookStack extends Stack { new WebhookApiConstruct(this, 'api', { ...queues, + vpc, + clusterName, + taskDefinition, environment: { ...webhookEnvironment, TASK_DEFINITION_FAMILY: clusterName }, }); } diff --git a/cdk-infra/lib/stacks/worker-stack.ts b/cdk-infra/lib/stacks/worker-stack.ts index a6c0ea401..1cb605d61 100644 --- a/cdk-infra/lib/stacks/worker-stack.ts +++ b/cdk-infra/lib/stacks/worker-stack.ts @@ -4,6 +4,7 @@ import { WorkerConstruct } from '../constructs/worker/worker-construct'; import { WorkerEnvConstruct } from '../constructs/worker/worker-env-construct'; import { WorkerBucketsConstruct } from '../constructs/worker/buckets-construct'; import { AutoBuilderQueues } from './auto-builder-queue-stack'; +import { TaskDefinition } from 'aws-cdk-lib/aws-ecs'; import { IVpc } from 'aws-cdk-lib/aws-ec2'; interface WorkerStackProps extends StackProps { @@ -13,7 +14,10 @@ interface WorkerStackProps extends StackProps { } export class WorkerStack extends Stack { + // TODO: Create the task definition as properties here so + // that they are accessible to the webhook stack public readonly clusterName: string; + public readonly taskDefinition: TaskDefinition; constructor(scope: Construct, id: string, { queues, workerSecureStrings, vpc, ...props }: WorkerStackProps) { super(scope, id, props); @@ -23,7 +27,9 @@ export class WorkerStack extends Stack { secureStrings: workerSecureStrings, }); - const { clusterName, ecsTaskRole } = new WorkerConstruct(this, 'worker', { + // TODO: retrieve the task definition from this stack as it is required + // for the RunTask in the smoke test build + const { clusterName, ecsTaskRole, taskDefinition } = new WorkerConstruct(this, 'worker', { vpc, dockerEnvironment: environment, ...queues, @@ -34,6 +40,9 @@ export class WorkerStack extends Stack { bucket.grantReadWrite(ecsTaskRole); }); + // TODO: Assign the task definition as properties here so + // that they are accessible to the webhook stack + this.taskDefinition = taskDefinition; this.clusterName = clusterName; } } diff --git a/src/enhanced/utils/queue/index.ts b/src/enhanced/utils/queue/index.ts index 289ea5306..977d8ea37 100644 --- a/src/enhanced/utils/queue/index.ts +++ b/src/enhanced/utils/queue/index.ts @@ -39,7 +39,9 @@ export async function listenToJobQueue(): Promise { // NOTE: Intentionally not catching here, as this throw should be handled by the method listening to the queue. // We don't want to continue listening to the queue, as there is something wrong with the protect task mechanism. // We can let the task end, as it is unsafe to let an unprotected task process a job. - await protectTask(); + if (process.env.INDEPENDENT_TASK !== 'yes') { + await protectTask(); + } console.log('[listenToJobQueue]: Deleting message...'); diff --git a/src/repositories/repoBranchesRepository.ts b/src/repositories/repoBranchesRepository.ts index 20bce0aeb..34f59d170 100644 --- a/src/repositories/repoBranchesRepository.ts +++ b/src/repositories/repoBranchesRepository.ts @@ -32,7 +32,7 @@ export class RepoBranchesRepository extends BaseRepository { return repo?.['branches'] ?? []; } - async getProdDeployableRepoBranches(): Promise { + async getProdDeployableRepos(): Promise { const reposArray = await this._collection .aggregate([{ $match: { prodDeployable: true, internalOnly: false } }, { $project: { _id: 0, repoName: 1 } }]) .toArray(); diff --git a/src/services/slack.ts b/src/services/slack.ts index 229646f1e..4a50f5235 100644 --- a/src/services/slack.ts +++ b/src/services/slack.ts @@ -72,7 +72,7 @@ export class SlackConnector implements ISlackConnector { } values['deploy_option'] = 'deploy_all'; - values['repo_option'] = await repoBranchesRepository.getProdDeployableRepoBranches(); + values['repo_option'] = await repoBranchesRepository.getProdDeployableRepos(); return values; } @@ -120,6 +120,7 @@ export class SlackConnector implements ISlackConnector { async displayRepoOptions(repos: string[], triggerId: string, isAdmin: boolean): Promise { const repoOptView = this._getDropDownView(triggerId, repos, isAdmin); + console.log('got dropdown view'); const slackToken = this._config.get('slackAuthToken'); const slackUrl = this._config.get('slackViewOpenUrl'); return await axiosApi.post(slackUrl, repoOptView, {