From 4db21c2505c11cac00cfdc7a3ce83a554ea38444 Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Wed, 6 Mar 2024 14:31:34 -0800 Subject: [PATCH 001/254] DOP-4414 pushing to preprd --- .github/workflows/deploy-stg-ecs.yml | 1 + api/controllers/v2/github.ts | 6 ++++++ 2 files changed, 7 insertions(+) diff --git a/.github/workflows/deploy-stg-ecs.yml b/.github/workflows/deploy-stg-ecs.yml index 2e0cfb462..1b5c34109 100644 --- a/.github/workflows/deploy-stg-ecs.yml +++ b/.github/workflows/deploy-stg-ecs.yml @@ -3,6 +3,7 @@ on: branches: - "main" - "integration" + - "DOP-4414" concurrency: group: environment-stg-${{ github.ref }} cancel-in-progress: true diff --git a/api/controllers/v2/github.ts b/api/controllers/v2/github.ts index a761e742b..832257f34 100644 --- a/api/controllers/v2/github.ts +++ b/api/controllers/v2/github.ts @@ -115,6 +115,12 @@ export const TriggerBuild = async (event: APIGatewayEvent): Promise Date: Wed, 6 Mar 2024 17:20:40 -0800 Subject: [PATCH 002/254] DOP-4414 logging more --- api/controllers/v2/slack.ts | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/api/controllers/v2/slack.ts b/api/controllers/v2/slack.ts index 2cb5307bc..11d434e86 100644 --- a/api/controllers/v2/slack.ts +++ b/api/controllers/v2/slack.ts @@ -81,6 +81,7 @@ const deployHelper = (deployable, payload, jobTitle, jobUserName, jobUserEmail) // For every repo/branch selected to be deployed, return an array of jobs with the payload data // needed for a successful build. export const getDeployableJobs = async ( + consoleLogger, values, entitlement, repoBranchesRepository: RepoBranchesRepository, @@ -106,6 +107,10 @@ export const getDeployableJobs = async ( const jobUserEmail = entitlement?.email ?? ''; const repoInfo = await docsetsRepository.getRepo(repoName, directory); + consoleLogger.info(jobTitle, 'Creating Job'); + consoleLogger.info(jobTitle, 'JOB INFO :' + repoOwner); + consoleLogger.info(jobTitle, 'REPO INFO:' + repoInfo); + consoleLogger.info(jobTitle, 'REPO INFO PREFIX:' + repoInfo?.prefix); const non_versioned = repoInfo.branches.length === 1; const branchObject = await repoBranchesRepository.getRepoBranchAliases(repoName, branchName, repoInfo.project); @@ -211,7 +216,13 @@ export const DeployRepo = async (event: APIGatewayEvent): Promise 0) { await deployRepo(deployable, consoleLogger, jobRepository, c.get('jobsQueueUrl')); } From 51c62aad829a19b86584ac4b7a6864defa2e1959 Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Thu, 7 Mar 2024 16:55:56 -0800 Subject: [PATCH 003/254] DOP-4414 logging --- .github/workflows/deploy-stg-ecs.yml | 1 - api/controllers/v2/github.ts | 11 +++++++---- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/.github/workflows/deploy-stg-ecs.yml b/.github/workflows/deploy-stg-ecs.yml index 1b5c34109..2e0cfb462 100644 --- a/.github/workflows/deploy-stg-ecs.yml +++ b/.github/workflows/deploy-stg-ecs.yml @@ -3,7 +3,6 @@ on: branches: - "main" - "integration" - - "DOP-4414" concurrency: group: environment-stg-${{ github.ref }} cancel-in-progress: true diff --git a/api/controllers/v2/github.ts b/api/controllers/v2/github.ts index 832257f34..589afa123 100644 --- a/api/controllers/v2/github.ts +++ b/api/controllers/v2/github.ts @@ -107,19 +107,22 @@ export const TriggerBuild = async (event: APIGatewayEvent): Promise('env'); async function createAndInsertJob(path?: string) { + //hardcode bodd.repository.name + //when is path used? const repoInfo = await docsetsRepository.getRepo(body.repository.name, path); const jobPrefix = repoInfo?.prefix ? repoInfo['prefix'][env] : ''; const job = await prepGithubPushPayload(body, repoBranchesRepository, jobPrefix, repoInfo, path); consoleLogger.info(job.title, 'Creating Job'); - consoleLogger.info(job.title, 'JOB INFO :' + body?.repository); - consoleLogger.info(job.title, 'REPO INFO:' + repoInfo); - consoleLogger.info(job.title, 'REPO INFO PREFIX:' + repoInfo?.prefix); + consoleLogger.info(job.title, 'JOB INFO :' + body?.repository.name); + consoleLogger.info(job.title, 'JOB INFO :' + body.head_commit + body.head_commit?.id); + consoleLogger.info(job.title, 'REPO INFO:' + repoInfo[0]); + consoleLogger.info(job.title, 'REPO INFO PREFIX:' + repoInfo?.prefix[1]); consoleLogger.info(job.title, 'REPO INFO PREFIX ENV:' + jobPrefix); - consoleLogger.info(job.title, 'PATH:' + path); const jobId = await jobRepository.insertJob(job, c.get('jobsQueueUrl')); jobRepository.notify(jobId, c.get('jobUpdatesQueueUrl'), JobStatus.inQueue, 0); From b68c8700eb51d728be0dfa21432b15270f0ac7d0 Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Fri, 8 Mar 2024 09:09:23 -0500 Subject: [PATCH 004/254] DOP-4414 logging --- api/controllers/v2/github.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/api/controllers/v2/github.ts b/api/controllers/v2/github.ts index 589afa123..bb4a2d53f 100644 --- a/api/controllers/v2/github.ts +++ b/api/controllers/v2/github.ts @@ -118,10 +118,10 @@ export const TriggerBuild = async (event: APIGatewayEvent): Promise Date: Fri, 8 Mar 2024 15:44:15 -0500 Subject: [PATCH 005/254] DOP-4414 logging --- api/controllers/v2/github.ts | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/api/controllers/v2/github.ts b/api/controllers/v2/github.ts index bb4a2d53f..53cf49a79 100644 --- a/api/controllers/v2/github.ts +++ b/api/controllers/v2/github.ts @@ -107,22 +107,20 @@ export const TriggerBuild = async (event: APIGatewayEvent): Promise('env'); async function createAndInsertJob(path?: string) { - //hardcode bodd.repository.name - //when is path used? const repoInfo = await docsetsRepository.getRepo(body.repository.name, path); const jobPrefix = repoInfo?.prefix ? repoInfo['prefix'][env] : ''; const job = await prepGithubPushPayload(body, repoBranchesRepository, jobPrefix, repoInfo, path); consoleLogger.info(job.title, 'Creating Job'); consoleLogger.info(job.title, 'JOB repo name :' + body?.repository.name); - consoleLogger.info(job.title, 'job info:' + Object.keys(body)); - consoleLogger.info(job.title, 'commit info:' + (body.head_commit ? Object.keys(body.head_commit) : '')); - consoleLogger.info(job.title, 'REPO INFO:' + Object.keys(repoInfo)); - consoleLogger.info(job.title, 'REPO INFO PREFIX ENV:' + jobPrefix); + consoleLogger.info(job.title, 'commits info:' + (body.head_commit ? Object.keys(body.commits) : '')); + consoleLogger.info(job.title, 'commits info:' + (body.base_ref ? Object.keys(body.base_ref) : '')); + consoleLogger.info(job.title, 'commits info:' + (body.ref ? Object.keys(body.ref) : '')); + consoleLogger.info(job.title, 'commits info:' + (body.before ? Object.keys(body.before) : '')); + consoleLogger.info(job.title, 'env' + env); const jobId = await jobRepository.insertJob(job, c.get('jobsQueueUrl')); jobRepository.notify(jobId, c.get('jobUpdatesQueueUrl'), JobStatus.inQueue, 0); From 8cdff970c5b90241f720b9b1a629ad0ce3ae3349 Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Fri, 8 Mar 2024 15:44:39 -0500 Subject: [PATCH 006/254] DOP-4414 logging --- api/controllers/v2/github.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/api/controllers/v2/github.ts b/api/controllers/v2/github.ts index 53cf49a79..383db0cdb 100644 --- a/api/controllers/v2/github.ts +++ b/api/controllers/v2/github.ts @@ -117,9 +117,9 @@ export const TriggerBuild = async (event: APIGatewayEvent): Promise Date: Fri, 8 Mar 2024 18:51:16 -0500 Subject: [PATCH 007/254] DOP-4414 initial changes --- api/controllers/v2/github.ts | 129 ++++++++++++++++-- .../constructs/api/webhook-api-construct.ts | 12 ++ 2 files changed, 132 insertions(+), 9 deletions(-) diff --git a/api/controllers/v2/github.ts b/api/controllers/v2/github.ts index 383db0cdb..7f2c70b71 100644 --- a/api/controllers/v2/github.ts +++ b/api/controllers/v2/github.ts @@ -14,11 +14,24 @@ import { getUpdatedFilePaths } from '../../../src/monorepo/utils/path-utils'; import { ReposBranchesDocsetsDocument } from '../../../modules/persistence/src/services/metadata/repos_branches'; import { MONOREPO_NAME } from '../../../src/monorepo/utils/monorepo-constants'; +const SMOKETEST_SITES = [ + 'docs-landing', + 'cloud-docs', + 'docs-realm', + 'docs', + 'docs-atlas-cli', + 'docs-ecosystem', + 'docs-node', + 'docs-landing', + 'docs-app-services', +]; + async function prepGithubPushPayload( githubEvent: PushEvent, repoBranchesRepository: RepoBranchesRepository, prefix: string, repoInfo: ReposBranchesDocsetsDocument, + title: string, directory?: string ): Promise> { const branch_name = githubEvent.ref.split('/')[2]; @@ -31,7 +44,7 @@ async function prepGithubPushPayload( const project = repoInfo?.project ?? githubEvent.repository.name; return { - title: githubEvent.repository.full_name, + title: title, user: githubEvent.pusher.name, email: githubEvent.pusher.email ?? '', status: JobStatus.inQueue, @@ -60,6 +73,110 @@ async function prepGithubPushPayload( }; } +/** + * 1st, we want to define a list of the sites we want to build as a new collection. + * validate credentials + * Next, we want to create each job + * - should I create a new prepGithubPayload method, a new githubEvent interface, or amend the existing one to be able to pass in a different title, branchName and it's own githash, + * - i could also alter body attributes, although I don't know it that's a good idea + * - could also create a createpayload function + * create and insert the jobs using bulk insert + */ +export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Promise => { + // this code can be updated to references values pertaining to the set list of sites + const client = new mongodb.MongoClient(c.get('dbUrl')); + await client.connect(); + const db = client.db(c.get('dbName')); + const consoleLogger = new ConsoleLogger(); + const jobRepository = new JobRepository(db, c, consoleLogger); + const repoBranchesRepository = new RepoBranchesRepository(db, c, consoleLogger); + const docsetsRepository = new DocsetsRepository(db, c, consoleLogger); + + if (!event.body) { + const err = 'Trigger build does not have a body in event payload'; + consoleLogger.error('TriggerBuildError', err); + return { + statusCode: 400, + headers: { 'Content-Type': 'text/plain' }, + body: err, + }; + } + + // validate credentials here + + let body: PushEvent; + try { + body = JSON.parse(event.body) as PushEvent; + } catch (e) { + console.log('[TriggerBuild]: ERROR! Could not parse event.body', e); + return { + statusCode: 502, + headers: { 'Content-Type': 'text/plain' }, + body: ' ERROR! Could not parse event.body', + }; + } + + //env could be hardcoded? + const env = c.get('env'); + + async function createAndInsertJob(path?: string) { + //should this array be typed more specifically + const deployable: Array = []; + + for (const s in SMOKETEST_SITES) { + //ensure repotitle is consistent with other type of title + const jobTitle = 'Smoke Test' + s; + const repoInfo = await docsetsRepository.getRepo(s, path); + + //add commit hash- how do you get commit hash?? + //local-build/index.ts line 53? + const jobPrefix = repoInfo?.prefix ? repoInfo['prefix'][env] : ''; + const ammendedJobPrefix = body.after ? jobPrefix + body.after : jobPrefix; + + //change this so that can pass in specific branch + //add logic for getting master branch, latest stable branch + const job = await prepGithubPushPayload( + body, + repoBranchesRepository, + ammendedJobPrefix, + repoInfo, + jobTitle, + path + ); + deployable.push(job); + } + + try { + await jobRepository.insertBulkJobs(deployable, c.get('jobsQueueUrl')); + + // notify the jobUpdatesQueue + await Promise.all( + deployable.map(async ({ jobId }) => { + await jobRepository.notify(jobId, c.get('jobUpdatesQueueUrl'), JobStatus.inQueue, 0); + consoleLogger.info(jobId, `Created Job ${jobId}`); + }) + ); + } catch (err) { + consoleLogger.error('deployRepo', err); + } + } + + try { + await createAndInsertJob(); + } catch (err) { + return { + statusCode: 500, + headers: { 'Content-Type': 'text/plain' }, + body: err, + }; + } + return { + statusCode: 202, + headers: { 'Content-Type': 'text/plain' }, + body: 'Jobs Queued', + }; +}; + export const TriggerBuild = async (event: APIGatewayEvent): Promise => { const client = new mongodb.MongoClient(c.get('dbUrl')); await client.connect(); @@ -112,16 +229,10 @@ export const TriggerBuild = async (event: APIGatewayEvent): Promise Date: Mon, 11 Mar 2024 18:31:18 -0400 Subject: [PATCH 008/254] DOP-4414 majority of ticket --- api/controllers/v2/github.ts | 143 ++++++++++++++++--------- api/controllers/v2/slack.ts | 4 - src/repositories/projectsRepository.ts | 28 +++++ 3 files changed, 123 insertions(+), 52 deletions(-) create mode 100644 src/repositories/projectsRepository.ts diff --git a/api/controllers/v2/github.ts b/api/controllers/v2/github.ts index 7f2c70b71..3ddc4c801 100644 --- a/api/controllers/v2/github.ts +++ b/api/controllers/v2/github.ts @@ -6,12 +6,17 @@ import { PushEvent } from '@octokit/webhooks-types'; import { JobRepository } from '../../../src/repositories/jobRepository'; import { ConsoleLogger } from '../../../src/services/logger'; import { RepoBranchesRepository } from '../../../src/repositories/repoBranchesRepository'; +import { ProjectsRepository } from '../../../src/repositories/projectsRepository'; import { EnhancedJob, JobStatus } from '../../../src/entities/job'; import { markBuildArtifactsForDeletion, validateJsonWebhook } from '../../handlers/github'; import { DocsetsRepository } from '../../../src/repositories/docsetsRepository'; import { getMonorepoPaths } from '../../../src/monorepo'; +x; import { getUpdatedFilePaths } from '../../../src/monorepo/utils/path-utils'; -import { ReposBranchesDocsetsDocument } from '../../../modules/persistence/src/services/metadata/repos_branches'; +import { + ReposBranchesDocsetsDocument, + ReposBranchesDocument, +} from '../../../modules/persistence/src/services/metadata/repos_branches'; import { MONOREPO_NAME } from '../../../src/monorepo/utils/monorepo-constants'; const SMOKETEST_SITES = [ @@ -22,27 +27,14 @@ const SMOKETEST_SITES = [ 'docs-atlas-cli', 'docs-ecosystem', 'docs-node', - 'docs-landing', 'docs-app-services', ]; async function prepGithubPushPayload( githubEvent: PushEvent, - repoBranchesRepository: RepoBranchesRepository, - prefix: string, - repoInfo: ReposBranchesDocsetsDocument, - title: string, - directory?: string + payload: any, + title: string ): Promise> { - const branch_name = githubEvent.ref.split('/')[2]; - const branch_info = await repoBranchesRepository.getRepoBranchAliases( - githubEvent.repository.name, - branch_name, - repoInfo.project - ); - const urlSlug = branch_info.aliasObject?.urlSlug ?? branch_name; - const project = repoInfo?.project ?? githubEvent.repository.name; - return { title: title, user: githubEvent.pusher.name, @@ -54,41 +46,88 @@ async function prepGithubPushPayload( priority: 1, error: {}, result: null, - payload: { - jobType: 'githubPush', - source: 'github', - action: 'push', - repoName: githubEvent.repository.name, - branchName: githubEvent.ref.split('/')[2], - isFork: githubEvent.repository.fork, - repoOwner: githubEvent.repository.owner.login, - url: githubEvent.repository.clone_url, - newHead: githubEvent.after, - urlSlug: urlSlug, - prefix: prefix, - project: project, - directory: directory, - }, + payload: payload, logs: [], }; } +async function createPayload( + repoName: string, + isSmokeTestDeploy = false, + prefix: string, + repoBranchesRepository: RepoBranchesRepository, + repoInfo: ReposBranchesDocsetsDocument, + githubEvent?: PushEvent, + repoOwner?: string, + directory?: string +) { + const jobType = 'githubPush'; + const source = 'github'; + const action = 'push'; + const project = repoInfo?.project ?? repoName; + + let branch_name = 'master'; + let isFork = false; + let url: string | undefined; + let newHead; + + if (isSmokeTestDeploy) { + branch_name = 'master'; + url = 'https://github.com/' + repoOwner + '/' + repoName; + newHead = null; + } else { + try { + if (!githubEvent) { + return false; + } + branch_name = githubEvent.ref.split('/')[2]; + isFork = githubEvent?.repository.fork; + url = githubEvent?.repository.clone_url; + newHead = githubEvent?.after; + repoOwner = githubEvent.repository.owner.login; + } catch (e) { + console.log('Error! No Github Push Event provided, payload could not be constructed'); + } + } + + const branch_info = await repoBranchesRepository.getRepoBranchAliases(repoName, branch_name, repoInfo.project); + const urlSlug = branch_info.aliasObject?.urlSlug ?? branch_name; + + return { + jobType, + source, + action, + repoName, + repoOwner, + branchName: branch_name, + project, + prefix, + urlSlug, + isFork, + url, + //can we keep newHead as null for smokeTest Deploys + newHead, + directory, + }; +} + /** * 1st, we want to define a list of the sites we want to build as a new collection. * validate credentials * Next, we want to create each job - * - should I create a new prepGithubPayload method, a new githubEvent interface, or amend the existing one to be able to pass in a different title, branchName and it's own githash, + * - should I create a new prepGithubPayload method, a new githubEvent interface, or amend the existing one to be able to pass in a different title, branchName and it's own githash * - i could also alter body attributes, although I don't know it that's a good idea * - could also create a createpayload function * create and insert the jobs using bulk insert */ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Promise => { - // this code can be updated to references values pertaining to the set list of sites const client = new mongodb.MongoClient(c.get('dbUrl')); await client.connect(); const db = client.db(c.get('dbName')); const consoleLogger = new ConsoleLogger(); const jobRepository = new JobRepository(db, c, consoleLogger); + //add docs_metadata to config + const projectsRepository = new ProjectsRepository(client.db('docs_metadata'), c, consoleLogger); const repoBranchesRepository = new RepoBranchesRepository(db, c, consoleLogger); const docsetsRepository = new DocsetsRepository(db, c, consoleLogger); @@ -103,6 +142,14 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr } // validate credentials here + if (!validateJsonWebhook(event, c.get('githubSecret'))) { + const errMsg = "X-Hub-Signature incorrect. Github webhook token doesn't match"; + return { + statusCode: 401, + headers: { 'Content-Type': 'text/plain' }, + body: errMsg, + }; + } let body: PushEvent; try { @@ -116,33 +163,30 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr }; } - //env could be hardcoded? - const env = c.get('env'); + //automated test builds will always deploy in dotcomstg + const env = 'dotcomstg'; async function createAndInsertJob(path?: string) { //should this array be typed more specifically const deployable: Array = []; for (const s in SMOKETEST_SITES) { - //ensure repotitle is consistent with other type of title + //ensure repoTitle is consistent with other type of title const jobTitle = 'Smoke Test' + s; const repoInfo = await docsetsRepository.getRepo(s, path); + const repoName = s; + const projectEntry = await projectsRepository.getProjectEntry(s); + const repoOwner = projectEntry.github.organization; //add commit hash- how do you get commit hash?? //local-build/index.ts line 53? const jobPrefix = repoInfo?.prefix ? repoInfo['prefix'][env] : ''; const ammendedJobPrefix = body.after ? jobPrefix + body.after : jobPrefix; + const prefix = ammendedJobPrefix; - //change this so that can pass in specific branch + const payload = await createPayload(repoName, true, prefix, repoBranchesRepository, repoInfo, repoOwner); //add logic for getting master branch, latest stable branch - const job = await prepGithubPushPayload( - body, - repoBranchesRepository, - ammendedJobPrefix, - repoInfo, - jobTitle, - path - ); + const job = await prepGithubPushPayload(body, payload, jobTitle); deployable.push(job); } @@ -227,10 +271,13 @@ export const TriggerBuild = async (event: APIGatewayEvent): Promise('env'); async function createAndInsertJob(path?: string) { - const repoInfo = await docsetsRepository.getRepo(body.repository.name, path); + const repo = body.repository; + const repoInfo = await docsetsRepository.getRepo(repo.name, path); const jobPrefix = repoInfo?.prefix ? repoInfo['prefix'][env] : ''; - const jobTitle = body.repository.full_name; - const job = await prepGithubPushPayload(body, repoBranchesRepository, jobPrefix, repoInfo, jobTitle, path); + const jobTitle = repo.full_name; + + // \ const payload = createPayload(repo.owner.login, repo.name, body.ref.split('/')[2], body.after, project, isFork: repo.fork, url: repo.clone_url, urlSlug: urlSlug, prefix: prefix, directory: directory,) + const job = await prepGithubPushPayload(body, repoBranchesRepository, jobPrefix, repoInfo, payload, jobTitle, path); consoleLogger.info(job.title, 'Creating Job'); const jobId = await jobRepository.insertJob(job, c.get('jobsQueueUrl')); diff --git a/api/controllers/v2/slack.ts b/api/controllers/v2/slack.ts index 11d434e86..e8e6ce8b0 100644 --- a/api/controllers/v2/slack.ts +++ b/api/controllers/v2/slack.ts @@ -107,10 +107,6 @@ export const getDeployableJobs = async ( const jobUserEmail = entitlement?.email ?? ''; const repoInfo = await docsetsRepository.getRepo(repoName, directory); - consoleLogger.info(jobTitle, 'Creating Job'); - consoleLogger.info(jobTitle, 'JOB INFO :' + repoOwner); - consoleLogger.info(jobTitle, 'REPO INFO:' + repoInfo); - consoleLogger.info(jobTitle, 'REPO INFO PREFIX:' + repoInfo?.prefix); const non_versioned = repoInfo.branches.length === 1; const branchObject = await repoBranchesRepository.getRepoBranchAliases(repoName, branchName, repoInfo.project); diff --git a/src/repositories/projectsRepository.ts b/src/repositories/projectsRepository.ts new file mode 100644 index 000000000..af54fbfb7 --- /dev/null +++ b/src/repositories/projectsRepository.ts @@ -0,0 +1,28 @@ +import { Db } from 'mongodb'; +import { IConfig } from 'config'; +import { BaseRepository } from './baseRepository'; +import { ILogger } from '../services/logger'; + +/** + * Manages metadata for parser builds. + */ +export class ProjectsRepository extends BaseRepository { + constructor(db: Db, config: IConfig, logger: ILogger) { + super(config, logger, 'ProjectsRepository', db.collection('projects')); + } + + async getProjectEntry(name: string): Promise { + const query = { name: name }; + const projectEntry = await this.findOne( + query, + `Mongo Timeout Error: Timedout while retrieving branches for ${name} + }` + ); + // if user has specific entitlements + return projectEntry; + } + + static getGithuRepoUrl(repoOwner: string, repoName: string): string { + return 'https://github.com/' + repoOwner + '/' + repoName; + } +} From 4f784e23d7adae8108f4104a608f9d7bba49e715 Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Mon, 11 Mar 2024 18:40:47 -0400 Subject: [PATCH 009/254] DOP-4414 cleaning up --- api/controllers/v2/github.ts | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/api/controllers/v2/github.ts b/api/controllers/v2/github.ts index 3ddc4c801..c043a3dd1 100644 --- a/api/controllers/v2/github.ts +++ b/api/controllers/v2/github.ts @@ -11,12 +11,8 @@ import { EnhancedJob, JobStatus } from '../../../src/entities/job'; import { markBuildArtifactsForDeletion, validateJsonWebhook } from '../../handlers/github'; import { DocsetsRepository } from '../../../src/repositories/docsetsRepository'; import { getMonorepoPaths } from '../../../src/monorepo'; -x; import { getUpdatedFilePaths } from '../../../src/monorepo/utils/path-utils'; -import { - ReposBranchesDocsetsDocument, - ReposBranchesDocument, -} from '../../../modules/persistence/src/services/metadata/repos_branches'; +import { ReposBranchesDocsetsDocument } from '../../../modules/persistence/src/services/metadata/repos_branches'; import { MONOREPO_NAME } from '../../../src/monorepo/utils/monorepo-constants'; const SMOKETEST_SITES = [ @@ -63,20 +59,22 @@ async function createPayload( ) { const jobType = 'githubPush'; const source = 'github'; - const action = 'push'; const project = repoInfo?.project ?? repoName; - let branch_name = 'master'; + let branch_name = githubEvent?.ref.split('/')[2]; + let action = 'push'; let isFork = false; - let url: string | undefined; + let url = githubEvent?.repository.clone_url; let newHead; if (isSmokeTestDeploy) { branch_name = 'master'; url = 'https://github.com/' + repoOwner + '/' + repoName; newHead = null; + action = 'automatedTest'; } else { try { + action = 'push'; if (!githubEvent) { return false; } @@ -276,8 +274,8 @@ export const TriggerBuild = async (event: APIGatewayEvent): Promise Date: Tue, 12 Mar 2024 10:20:27 -0400 Subject: [PATCH 010/254] DOP-4414 refactoring --- api/controllers/v2/github.ts | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/api/controllers/v2/github.ts b/api/controllers/v2/github.ts index c043a3dd1..f8b017cc8 100644 --- a/api/controllers/v2/github.ts +++ b/api/controllers/v2/github.ts @@ -61,15 +61,23 @@ async function createPayload( const source = 'github'; const project = repoInfo?.project ?? repoName; - let branch_name = githubEvent?.ref.split('/')[2]; - let action = 'push'; + let branch_name = ''; + let action = ''; let isFork = false; - let url = githubEvent?.repository.clone_url; + let url; let newHead; if (isSmokeTestDeploy) { branch_name = 'master'; - url = 'https://github.com/' + repoOwner + '/' + repoName; + try { + if (!repoOwner) { + return false; + } + url = 'https://github.com/' + repoOwner + '/' + repoName; + } catch (e) { + console.log('Error! repoOwner is must be configured for an automated smoke test deploy'); + } + newHead = null; action = 'automatedTest'; } else { From 7d6edd62f2c927379205cd3ee1b15c5418f00c80 Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Tue, 12 Mar 2024 11:28:33 -0400 Subject: [PATCH 011/254] DOP-4414 webhook api construct --- cdk-infra/lib/constructs/api/webhook-api-construct.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/cdk-infra/lib/constructs/api/webhook-api-construct.ts b/cdk-infra/lib/constructs/api/webhook-api-construct.ts index e662965fe..ee9469a6c 100644 --- a/cdk-infra/lib/constructs/api/webhook-api-construct.ts +++ b/cdk-infra/lib/constructs/api/webhook-api-construct.ts @@ -170,7 +170,9 @@ export class WebhookApiConstruct extends Construct { .addMethod('POST', new LambdaIntegration(githubTriggerLambda)); // add endpoint for automated testing - githubEndpointTrigger.addResource('smoke-test-build', { defaultCorsPreflightOptions }); + githubEndpointTrigger + .addResource('smoke-test-build', { defaultCorsPreflightOptions }) + .addMethod('POST', new LambdaIntegration(githubSmokeTestBuildLambda)); githubEndpointTrigger .addResource('delete', { defaultCorsPreflightOptions }) @@ -188,6 +190,7 @@ export class WebhookApiConstruct extends Construct { // grant permission for lambdas to enqueue messages to the jobs queue jobsQueue.grantSendMessages(slackTriggerLambda); + jobsQueue.grantSendMessages(githubSmokeTestBuildLambda); jobsQueue.grantSendMessages(githubTriggerLambda); jobsQueue.grantSendMessages(triggerLocalBuildLambda); From f9d4616d1ade739925045e24eb5aa9460c46f4ef Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Tue, 12 Mar 2024 16:36:48 -0400 Subject: [PATCH 012/254] DOP-4414 checks build on main --- api/controllers/v2/github.ts | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/api/controllers/v2/github.ts b/api/controllers/v2/github.ts index f8b017cc8..a97bb92fd 100644 --- a/api/controllers/v2/github.ts +++ b/api/controllers/v2/github.ts @@ -126,7 +126,7 @@ async function createPayload( * - could also create a createpayload function * create and insert the jobs using bulk insert */ -export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Promise => { +export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Promise => { const client = new mongodb.MongoClient(c.get('dbUrl')); await client.connect(); const db = client.db(c.get('dbName')); @@ -169,6 +169,12 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr }; } + //if the build was not building master, no need for smoke test sites + if (body.ref.split('/')[2] != 'main') { + console.log('Build was not on master branch, sites will not deploy as no smoke tests are needed'); + return null; + } + //automated test builds will always deploy in dotcomstg const env = 'dotcomstg'; From 63b183f50273a5adc70af0833a563805eee52ef0 Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Tue, 12 Mar 2024 16:56:24 -0400 Subject: [PATCH 013/254] DOP-4414 update validation --- api/controllers/v2/github.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/controllers/v2/github.ts b/api/controllers/v2/github.ts index a97bb92fd..21558bb8c 100644 --- a/api/controllers/v2/github.ts +++ b/api/controllers/v2/github.ts @@ -148,7 +148,7 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr } // validate credentials here - if (!validateJsonWebhook(event, c.get('githubSecret'))) { + if (!validateJsonWebhook(event, 'mongodbsmoketesting')) { const errMsg = "X-Hub-Signature incorrect. Github webhook token doesn't match"; return { statusCode: 401, From 295f27f1b6e74d15813b8ce34908d8a79d405c1d Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Tue, 12 Mar 2024 16:59:52 -0400 Subject: [PATCH 014/254] DOP-4414 deploy to preprd --- .github/workflows/deploy-stg-ecs.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/deploy-stg-ecs.yml b/.github/workflows/deploy-stg-ecs.yml index 2e0cfb462..1b5c34109 100644 --- a/.github/workflows/deploy-stg-ecs.yml +++ b/.github/workflows/deploy-stg-ecs.yml @@ -3,6 +3,7 @@ on: branches: - "main" - "integration" + - "DOP-4414" concurrency: group: environment-stg-${{ github.ref }} cancel-in-progress: true From 8d0965056e01643ad14bed93579c1580a6c38324 Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Tue, 12 Mar 2024 17:19:39 -0400 Subject: [PATCH 015/254] DOP-4414 testing webhook --- api/controllers/v2/github.ts | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/api/controllers/v2/github.ts b/api/controllers/v2/github.ts index 21558bb8c..cc07eaa90 100644 --- a/api/controllers/v2/github.ts +++ b/api/controllers/v2/github.ts @@ -127,6 +127,22 @@ async function createPayload( * create and insert the jobs using bulk insert */ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Promise => { + // validate credentials here + if (!validateJsonWebhook(event, 'mongodbsmoketesting')) { + const errMsg = "X-Hub-Signature incorrect. Github webhook token doesn't match"; + return { + statusCode: 401, + headers: { 'Content-Type': 'text/plain' }, + body: errMsg, + }; + } + + return { + statusCode: 202, + headers: { 'Content-Type': 'text/plain' }, + body: 'Jobs Queued', + }; + const client = new mongodb.MongoClient(c.get('dbUrl')); await client.connect(); const db = client.db(c.get('dbName')); @@ -147,16 +163,6 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr }; } - // validate credentials here - if (!validateJsonWebhook(event, 'mongodbsmoketesting')) { - const errMsg = "X-Hub-Signature incorrect. Github webhook token doesn't match"; - return { - statusCode: 401, - headers: { 'Content-Type': 'text/plain' }, - body: errMsg, - }; - } - let body: PushEvent; try { body = JSON.parse(event.body) as PushEvent; From 807be13050e5c9450f678af0787f2bbc302a73e4 Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Tue, 12 Mar 2024 17:43:10 -0400 Subject: [PATCH 016/254] DOP-4414 testing lambda --- api/controllers/v2/github.ts | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/api/controllers/v2/github.ts b/api/controllers/v2/github.ts index cc07eaa90..d93447f4f 100644 --- a/api/controllers/v2/github.ts +++ b/api/controllers/v2/github.ts @@ -137,12 +137,6 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr }; } - return { - statusCode: 202, - headers: { 'Content-Type': 'text/plain' }, - body: 'Jobs Queued', - }; - const client = new mongodb.MongoClient(c.get('dbUrl')); await client.connect(); const db = client.db(c.get('dbName')); @@ -175,6 +169,12 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr }; } + return { + statusCode: 202, + headers: { 'Content-Type': 'text/plain' }, + body: 'Jobs Queued', + }; + //if the build was not building master, no need for smoke test sites if (body.ref.split('/')[2] != 'main') { console.log('Build was not on master branch, sites will not deploy as no smoke tests are needed'); From 26712508b72081b1f422f90af0febd2bdc914247 Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Tue, 12 Mar 2024 17:47:45 -0400 Subject: [PATCH 017/254] DOP-4414 testing lambda --- api/controllers/v2/github.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/controllers/v2/github.ts b/api/controllers/v2/github.ts index d93447f4f..be2d5d043 100644 --- a/api/controllers/v2/github.ts +++ b/api/controllers/v2/github.ts @@ -172,7 +172,7 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr return { statusCode: 202, headers: { 'Content-Type': 'text/plain' }, - body: 'Jobs Queued', + body: 'Jobs Queued 2', }; //if the build was not building master, no need for smoke test sites From 01dc7b8260f9980cc978f05c421a3e9a2e08a74c Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Tue, 12 Mar 2024 17:53:14 -0400 Subject: [PATCH 018/254] DOP-4414 testing lambda --- api/controllers/v2/github.ts | 32 +++++++++++++++----------------- 1 file changed, 15 insertions(+), 17 deletions(-) diff --git a/api/controllers/v2/github.ts b/api/controllers/v2/github.ts index be2d5d043..26d66fa39 100644 --- a/api/controllers/v2/github.ts +++ b/api/controllers/v2/github.ts @@ -127,16 +127,6 @@ async function createPayload( * create and insert the jobs using bulk insert */ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Promise => { - // validate credentials here - if (!validateJsonWebhook(event, 'mongodbsmoketesting')) { - const errMsg = "X-Hub-Signature incorrect. Github webhook token doesn't match"; - return { - statusCode: 401, - headers: { 'Content-Type': 'text/plain' }, - body: errMsg, - }; - } - const client = new mongodb.MongoClient(c.get('dbUrl')); await client.connect(); const db = client.db(c.get('dbName')); @@ -157,6 +147,16 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr }; } + // validate credentials here + if (!validateJsonWebhook(event, 'mongodbsmoketesting')) { + const errMsg = "X-Hub-Signature incorrect. Github webhook token doesn't match"; + return { + statusCode: 401, + headers: { 'Content-Type': 'text/plain' }, + body: errMsg, + }; + } + let body: PushEvent; try { body = JSON.parse(event.body) as PushEvent; @@ -169,16 +169,14 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr }; } - return { - statusCode: 202, - headers: { 'Content-Type': 'text/plain' }, - body: 'Jobs Queued 2', - }; - //if the build was not building master, no need for smoke test sites if (body.ref.split('/')[2] != 'main') { console.log('Build was not on master branch, sites will not deploy as no smoke tests are needed'); - return null; + return { + statusCode: 202, + headers: { 'Content-Type': 'text/plain' }, + body: 'No Jobs queued, build was not on master branch', + }; } //automated test builds will always deploy in dotcomstg From e60b48232da386f39a071ee28281be2d966fa6ec Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Tue, 12 Mar 2024 18:05:56 -0400 Subject: [PATCH 019/254] DOP-4414 testing lambda --- api/controllers/v2/github.ts | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/api/controllers/v2/github.ts b/api/controllers/v2/github.ts index 26d66fa39..9339fcc9e 100644 --- a/api/controllers/v2/github.ts +++ b/api/controllers/v2/github.ts @@ -175,13 +175,19 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr return { statusCode: 202, headers: { 'Content-Type': 'text/plain' }, - body: 'No Jobs queued, build was not on master branch', + body: 'No Jobs queued, build was not on master branch' + body.ref, }; } //automated test builds will always deploy in dotcomstg const env = 'dotcomstg'; + return { + statusCode: 202, + headers: { 'Content-Type': 'text/plain' }, + body: 'Return early' + body.ref, + }; + async function createAndInsertJob(path?: string) { //should this array be typed more specifically const deployable: Array = []; From b73845dbe26655dbcca3019611971e392900b9a7 Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Tue, 12 Mar 2024 18:26:28 -0400 Subject: [PATCH 020/254] DOP-4414 testing lambda --- api/controllers/v2/github.ts | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/api/controllers/v2/github.ts b/api/controllers/v2/github.ts index 9339fcc9e..73dfaab53 100644 --- a/api/controllers/v2/github.ts +++ b/api/controllers/v2/github.ts @@ -169,6 +169,12 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr }; } + return { + statusCode: 202, + headers: { 'Content-Type': 'text/plain' }, + body: 'Return early' + body.ref, + }; + //if the build was not building master, no need for smoke test sites if (body.ref.split('/')[2] != 'main') { console.log('Build was not on master branch, sites will not deploy as no smoke tests are needed'); @@ -182,12 +188,6 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr //automated test builds will always deploy in dotcomstg const env = 'dotcomstg'; - return { - statusCode: 202, - headers: { 'Content-Type': 'text/plain' }, - body: 'Return early' + body.ref, - }; - async function createAndInsertJob(path?: string) { //should this array be typed more specifically const deployable: Array = []; From c44d6bf7f395fea1aa1f78162080c358fe41e1f1 Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Tue, 12 Mar 2024 18:34:58 -0400 Subject: [PATCH 021/254] DOP-4414 testing lambda --- api/controllers/v2/github.ts | 27 ++++++++++----------------- 1 file changed, 10 insertions(+), 17 deletions(-) diff --git a/api/controllers/v2/github.ts b/api/controllers/v2/github.ts index 73dfaab53..00b52725b 100644 --- a/api/controllers/v2/github.ts +++ b/api/controllers/v2/github.ts @@ -68,7 +68,6 @@ async function createPayload( let newHead; if (isSmokeTestDeploy) { - branch_name = 'master'; try { if (!repoOwner) { return false; @@ -77,7 +76,7 @@ async function createPayload( } catch (e) { console.log('Error! repoOwner is must be configured for an automated smoke test deploy'); } - + branch_name = 'master'; newHead = null; action = 'automatedTest'; } else { @@ -169,21 +168,15 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr }; } - return { - statusCode: 202, - headers: { 'Content-Type': 'text/plain' }, - body: 'Return early' + body.ref, - }; - - //if the build was not building master, no need for smoke test sites - if (body.ref.split('/')[2] != 'main') { - console.log('Build was not on master branch, sites will not deploy as no smoke tests are needed'); - return { - statusCode: 202, - headers: { 'Content-Type': 'text/plain' }, - body: 'No Jobs queued, build was not on master branch' + body.ref, - }; - } + // //if the build was not building master, no need for smoke test sites + // if (body.ref.split('/')[2] != 'main') { + // console.log('Build was not on master branch, sites will not deploy as no smoke tests are needed'); + // return { + // statusCode: 202, + // headers: { 'Content-Type': 'text/plain' }, + // body: 'No Jobs queued, build was not on master branch' + body.ref, + // }; + // } //automated test builds will always deploy in dotcomstg const env = 'dotcomstg'; From 9b9b25d3b6d3c04f10f242a765dffe91d3f077ab Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Tue, 12 Mar 2024 18:41:07 -0400 Subject: [PATCH 022/254] DOP-4414 testing lambda --- api/controllers/v2/github.ts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/api/controllers/v2/github.ts b/api/controllers/v2/github.ts index 00b52725b..8e3c95f9b 100644 --- a/api/controllers/v2/github.ts +++ b/api/controllers/v2/github.ts @@ -204,6 +204,11 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr const job = await prepGithubPushPayload(body, payload, jobTitle); deployable.push(job); } + return { + statusCode: 202, + headers: { 'Content-Type': 'text/plain' }, + body: 'Jobs Queued', + }; try { await jobRepository.insertBulkJobs(deployable, c.get('jobsQueueUrl')); From 9d6d0d5a331b4f2bc61ef1485146e495d7d4a3d9 Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Tue, 12 Mar 2024 18:43:27 -0400 Subject: [PATCH 023/254] DOP-4414 testing lambda --- api/controllers/v2/github.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/api/controllers/v2/github.ts b/api/controllers/v2/github.ts index 8e3c95f9b..4426257bd 100644 --- a/api/controllers/v2/github.ts +++ b/api/controllers/v2/github.ts @@ -196,10 +196,10 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr //add commit hash- how do you get commit hash?? //local-build/index.ts line 53? const jobPrefix = repoInfo?.prefix ? repoInfo['prefix'][env] : ''; - const ammendedJobPrefix = body.after ? jobPrefix + body.after : jobPrefix; - const prefix = ammendedJobPrefix; + // const ammendedJobPrefix = body.after ? jobPrefix + body.after : jobPrefix; + // const prefix = ammendedJobPrefix; - const payload = await createPayload(repoName, true, prefix, repoBranchesRepository, repoInfo, repoOwner); + const payload = await createPayload(repoName, true, jobPrefix, repoBranchesRepository, repoInfo, repoOwner); //add logic for getting master branch, latest stable branch const job = await prepGithubPushPayload(body, payload, jobTitle); deployable.push(job); From e9dd2a1b39317659358a3698e42cb911e9a4c99e Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Tue, 12 Mar 2024 18:46:58 -0400 Subject: [PATCH 024/254] DOP-4414 testing lambda --- api/controllers/v2/github.ts | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/api/controllers/v2/github.ts b/api/controllers/v2/github.ts index 4426257bd..16184ee65 100644 --- a/api/controllers/v2/github.ts +++ b/api/controllers/v2/github.ts @@ -185,6 +185,12 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr //should this array be typed more specifically const deployable: Array = []; + return { + statusCode: 202, + headers: { 'Content-Type': 'text/plain' }, + body: 'Jobs Queued', + }; + for (const s in SMOKETEST_SITES) { //ensure repoTitle is consistent with other type of title const jobTitle = 'Smoke Test' + s; From 47eee25d8812df0c39f6bc11299fd33664a2089c Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Tue, 12 Mar 2024 18:50:19 -0400 Subject: [PATCH 025/254] DOP-4414 testing lambda --- api/controllers/v2/github.ts | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/api/controllers/v2/github.ts b/api/controllers/v2/github.ts index 16184ee65..c0fb73114 100644 --- a/api/controllers/v2/github.ts +++ b/api/controllers/v2/github.ts @@ -181,6 +181,12 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr //automated test builds will always deploy in dotcomstg const env = 'dotcomstg'; + return { + statusCode: 202, + headers: { 'Content-Type': 'text/plain' }, + body: 'Jobs Queued', + }; + async function createAndInsertJob(path?: string) { //should this array be typed more specifically const deployable: Array = []; From aa1904e0b174a31de55384b3f6ada9f4bf4a5983 Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Tue, 12 Mar 2024 18:58:14 -0400 Subject: [PATCH 026/254] DOP-4414 testing lambda --- api/controllers/v2/github.ts | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/api/controllers/v2/github.ts b/api/controllers/v2/github.ts index c0fb73114..f98971b82 100644 --- a/api/controllers/v2/github.ts +++ b/api/controllers/v2/github.ts @@ -181,13 +181,12 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr //automated test builds will always deploy in dotcomstg const env = 'dotcomstg'; - return { - statusCode: 202, - headers: { 'Content-Type': 'text/plain' }, - body: 'Jobs Queued', - }; - async function createAndInsertJob(path?: string) { + return { + statusCode: 202, + headers: { 'Content-Type': 'text/plain' }, + body: 'Jobs Queued', + }; //should this array be typed more specifically const deployable: Array = []; From c85b210d2d402be15b93a6ddb0c4373aed8f6e36 Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Tue, 12 Mar 2024 19:03:37 -0400 Subject: [PATCH 027/254] DOP-4414 testing lambda --- api/controllers/v2/github.ts | 5 ----- 1 file changed, 5 deletions(-) diff --git a/api/controllers/v2/github.ts b/api/controllers/v2/github.ts index f98971b82..16184ee65 100644 --- a/api/controllers/v2/github.ts +++ b/api/controllers/v2/github.ts @@ -182,11 +182,6 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr const env = 'dotcomstg'; async function createAndInsertJob(path?: string) { - return { - statusCode: 202, - headers: { 'Content-Type': 'text/plain' }, - body: 'Jobs Queued', - }; //should this array be typed more specifically const deployable: Array = []; From 12989719f062e312f0fa7e13e6868d859aebe38d Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Tue, 12 Mar 2024 19:06:26 -0400 Subject: [PATCH 028/254] DOP-4414 testing lambda --- api/controllers/v2/github.ts | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/api/controllers/v2/github.ts b/api/controllers/v2/github.ts index 16184ee65..0d053455a 100644 --- a/api/controllers/v2/github.ts +++ b/api/controllers/v2/github.ts @@ -185,16 +185,15 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr //should this array be typed more specifically const deployable: Array = []; - return { - statusCode: 202, - headers: { 'Content-Type': 'text/plain' }, - body: 'Jobs Queued', - }; - for (const s in SMOKETEST_SITES) { //ensure repoTitle is consistent with other type of title const jobTitle = 'Smoke Test' + s; const repoInfo = await docsetsRepository.getRepo(s, path); + return { + statusCode: 202, + headers: { 'Content-Type': 'text/plain' }, + body: 'Jobs Queued', + }; const repoName = s; const projectEntry = await projectsRepository.getProjectEntry(s); const repoOwner = projectEntry.github.organization; From b76af20f6829a0c2c4f390b19dd97e66c3c4b6ec Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Tue, 12 Mar 2024 19:12:08 -0400 Subject: [PATCH 029/254] DOP-4414 testing lambda --- api/controllers/v2/github.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/api/controllers/v2/github.ts b/api/controllers/v2/github.ts index 0d053455a..674a950c4 100644 --- a/api/controllers/v2/github.ts +++ b/api/controllers/v2/github.ts @@ -189,12 +189,13 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr //ensure repoTitle is consistent with other type of title const jobTitle = 'Smoke Test' + s; const repoInfo = await docsetsRepository.getRepo(s, path); + const repoName = s; + return { statusCode: 202, headers: { 'Content-Type': 'text/plain' }, - body: 'Jobs Queued', + body: 'Jobs Queued' + body, }; - const repoName = s; const projectEntry = await projectsRepository.getProjectEntry(s); const repoOwner = projectEntry.github.organization; From 46cf92e5ca7d9e9724c9e13911337b8409068f9d Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Tue, 12 Mar 2024 19:13:49 -0400 Subject: [PATCH 030/254] DOP-4414 testing lambda --- api/controllers/v2/github.ts | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/api/controllers/v2/github.ts b/api/controllers/v2/github.ts index 674a950c4..081813a95 100644 --- a/api/controllers/v2/github.ts +++ b/api/controllers/v2/github.ts @@ -194,7 +194,7 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr return { statusCode: 202, headers: { 'Content-Type': 'text/plain' }, - body: 'Jobs Queued' + body, + body: 'Jobs Queued 3' + body, }; const projectEntry = await projectsRepository.getProjectEntry(s); const repoOwner = projectEntry.github.organization; @@ -210,11 +210,6 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr const job = await prepGithubPushPayload(body, payload, jobTitle); deployable.push(job); } - return { - statusCode: 202, - headers: { 'Content-Type': 'text/plain' }, - body: 'Jobs Queued', - }; try { await jobRepository.insertBulkJobs(deployable, c.get('jobsQueueUrl')); From aba0f43cab3b1d8119e71c3b93cb85fc26f50211 Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Tue, 12 Mar 2024 19:15:06 -0400 Subject: [PATCH 031/254] DOP-4414 testing lambda --- api/controllers/v2/github.ts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/api/controllers/v2/github.ts b/api/controllers/v2/github.ts index 081813a95..bbf2b4bca 100644 --- a/api/controllers/v2/github.ts +++ b/api/controllers/v2/github.ts @@ -228,6 +228,11 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr try { await createAndInsertJob(); + return { + statusCode: 202, + headers: { 'Content-Type': 'text/plain' }, + body: 'Jobs Queued 3' + body, + }; } catch (err) { return { statusCode: 500, From dcebb4fe153e9047342046a3c2a32e57b23f0bbb Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Tue, 12 Mar 2024 19:17:46 -0400 Subject: [PATCH 032/254] DOP-4414 testing lambda --- api/controllers/v2/github.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/controllers/v2/github.ts b/api/controllers/v2/github.ts index bbf2b4bca..40da52b93 100644 --- a/api/controllers/v2/github.ts +++ b/api/controllers/v2/github.ts @@ -348,7 +348,7 @@ export const TriggerBuild = async (event: APIGatewayEvent): Promise Date: Tue, 12 Mar 2024 19:20:14 -0400 Subject: [PATCH 033/254] DOP-4414 testing lambda --- api/controllers/v2/github.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/api/controllers/v2/github.ts b/api/controllers/v2/github.ts index 40da52b93..f2d0011b2 100644 --- a/api/controllers/v2/github.ts +++ b/api/controllers/v2/github.ts @@ -231,7 +231,7 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr return { statusCode: 202, headers: { 'Content-Type': 'text/plain' }, - body: 'Jobs Queued 3' + body, + body: 'Jobs Queued 2' + body, }; } catch (err) { return { @@ -348,7 +348,7 @@ export const TriggerBuild = async (event: APIGatewayEvent): Promise Date: Tue, 12 Mar 2024 19:21:30 -0400 Subject: [PATCH 034/254] DOP-4414 testing lambda --- api/controllers/v2/github.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/controllers/v2/github.ts b/api/controllers/v2/github.ts index f2d0011b2..05ee1c87a 100644 --- a/api/controllers/v2/github.ts +++ b/api/controllers/v2/github.ts @@ -194,7 +194,7 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr return { statusCode: 202, headers: { 'Content-Type': 'text/plain' }, - body: 'Jobs Queued 3' + body, + body: 'Jobs Queued 4' + body, }; const projectEntry = await projectsRepository.getProjectEntry(s); const repoOwner = projectEntry.github.organization; From e80cdabcfa9956971946308d2d97c3beef6f6d9e Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Tue, 12 Mar 2024 19:25:42 -0400 Subject: [PATCH 035/254] DOP-4414 testing lambda --- api/controllers/v2/github.ts | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/api/controllers/v2/github.ts b/api/controllers/v2/github.ts index 05ee1c87a..e11cc86a8 100644 --- a/api/controllers/v2/github.ts +++ b/api/controllers/v2/github.ts @@ -190,15 +190,10 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr const jobTitle = 'Smoke Test' + s; const repoInfo = await docsetsRepository.getRepo(s, path); const repoName = s; - - return { - statusCode: 202, - headers: { 'Content-Type': 'text/plain' }, - body: 'Jobs Queued 4' + body, - }; const projectEntry = await projectsRepository.getProjectEntry(s); const repoOwner = projectEntry.github.organization; + return true; //add commit hash- how do you get commit hash?? //local-build/index.ts line 53? const jobPrefix = repoInfo?.prefix ? repoInfo['prefix'][env] : ''; From c187a3697ab61f1172284d52ddbbcdc6e2feee62 Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Tue, 12 Mar 2024 19:27:44 -0400 Subject: [PATCH 036/254] DOP-4414 testing lambda --- api/controllers/v2/github.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/controllers/v2/github.ts b/api/controllers/v2/github.ts index e11cc86a8..37e94a61e 100644 --- a/api/controllers/v2/github.ts +++ b/api/controllers/v2/github.ts @@ -226,7 +226,7 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr return { statusCode: 202, headers: { 'Content-Type': 'text/plain' }, - body: 'Jobs Queued 2' + body, + body: 'Jobs Queued 5' + body, }; } catch (err) { return { From c0887048777bb7f164634c8d5a5c7ae37ae3623f Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Tue, 12 Mar 2024 22:24:41 -0400 Subject: [PATCH 037/254] DOP-4414 testing lambda --- api/controllers/v2/github.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/api/controllers/v2/github.ts b/api/controllers/v2/github.ts index 37e94a61e..a4955cc39 100644 --- a/api/controllers/v2/github.ts +++ b/api/controllers/v2/github.ts @@ -191,6 +191,7 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr const repoInfo = await docsetsRepository.getRepo(s, path); const repoName = s; const projectEntry = await projectsRepository.getProjectEntry(s); + return true; const repoOwner = projectEntry.github.organization; return true; @@ -226,7 +227,7 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr return { statusCode: 202, headers: { 'Content-Type': 'text/plain' }, - body: 'Jobs Queued 5' + body, + body: 'Jobs Queued 1' + body, }; } catch (err) { return { From e8863315880ec47c6585572828e5e967b546d099 Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Tue, 12 Mar 2024 22:26:17 -0400 Subject: [PATCH 038/254] DOP-4414 testing lambda --- api/controllers/v2/github.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/api/controllers/v2/github.ts b/api/controllers/v2/github.ts index a4955cc39..61696cec4 100644 --- a/api/controllers/v2/github.ts +++ b/api/controllers/v2/github.ts @@ -190,11 +190,10 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr const jobTitle = 'Smoke Test' + s; const repoInfo = await docsetsRepository.getRepo(s, path); const repoName = s; - const projectEntry = await projectsRepository.getProjectEntry(s); return true; + const projectEntry = await projectsRepository.getProjectEntry(s); const repoOwner = projectEntry.github.organization; - return true; //add commit hash- how do you get commit hash?? //local-build/index.ts line 53? const jobPrefix = repoInfo?.prefix ? repoInfo['prefix'][env] : ''; From 9acfb29c7787309c1c5f9a507ccc820000c15f3e Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Tue, 12 Mar 2024 22:27:48 -0400 Subject: [PATCH 039/254] DOP-4414 testing lambda --- api/controllers/v2/github.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/api/controllers/v2/github.ts b/api/controllers/v2/github.ts index 61696cec4..e00d96537 100644 --- a/api/controllers/v2/github.ts +++ b/api/controllers/v2/github.ts @@ -190,7 +190,11 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr const jobTitle = 'Smoke Test' + s; const repoInfo = await docsetsRepository.getRepo(s, path); const repoName = s; - return true; + return { + statusCode: 202, + headers: { 'Content-Type': 'text/plain' }, + body: 'Jobs Queued 1' + body, + }; const projectEntry = await projectsRepository.getProjectEntry(s); const repoOwner = projectEntry.github.organization; From 93a360a57ba2aa7aea0433feb08c5cd42c7083e8 Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Tue, 12 Mar 2024 22:29:01 -0400 Subject: [PATCH 040/254] DOP-4414 testing lambda --- api/controllers/v2/github.ts | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/api/controllers/v2/github.ts b/api/controllers/v2/github.ts index e00d96537..c0fddfe30 100644 --- a/api/controllers/v2/github.ts +++ b/api/controllers/v2/github.ts @@ -190,11 +190,12 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr const jobTitle = 'Smoke Test' + s; const repoInfo = await docsetsRepository.getRepo(s, path); const repoName = s; - return { - statusCode: 202, - headers: { 'Content-Type': 'text/plain' }, - body: 'Jobs Queued 1' + body, - }; + return true; + // return { + // statusCode: 202, + // headers: { 'Content-Type': 'text/plain' }, + // body: 'Jobs Queued 1' + body, + // }; const projectEntry = await projectsRepository.getProjectEntry(s); const repoOwner = projectEntry.github.organization; @@ -230,7 +231,7 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr return { statusCode: 202, headers: { 'Content-Type': 'text/plain' }, - body: 'Jobs Queued 1' + body, + body: 'Jobs Queued 2' + body, }; } catch (err) { return { From ca3814c6dd528cdafd849e6848c96b8e29334359 Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Tue, 12 Mar 2024 23:05:27 -0400 Subject: [PATCH 041/254] DOP-4414 testing lambda --- api/controllers/v2/github.ts | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/api/controllers/v2/github.ts b/api/controllers/v2/github.ts index c0fddfe30..55e00a304 100644 --- a/api/controllers/v2/github.ts +++ b/api/controllers/v2/github.ts @@ -190,13 +190,8 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr const jobTitle = 'Smoke Test' + s; const repoInfo = await docsetsRepository.getRepo(s, path); const repoName = s; - return true; - // return { - // statusCode: 202, - // headers: { 'Content-Type': 'text/plain' }, - // body: 'Jobs Queued 1' + body, - // }; const projectEntry = await projectsRepository.getProjectEntry(s); + return true; const repoOwner = projectEntry.github.organization; //add commit hash- how do you get commit hash?? @@ -231,7 +226,7 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr return { statusCode: 202, headers: { 'Content-Type': 'text/plain' }, - body: 'Jobs Queued 2' + body, + body: 'Jobs Queued 1' + body, }; } catch (err) { return { From 0bf36a5d6478d592fc44711216ac800713a66e99 Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Tue, 12 Mar 2024 23:09:54 -0400 Subject: [PATCH 042/254] DOP-4414 testing lambda --- api/controllers/v2/github.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/controllers/v2/github.ts b/api/controllers/v2/github.ts index 55e00a304..368e7cb4d 100644 --- a/api/controllers/v2/github.ts +++ b/api/controllers/v2/github.ts @@ -226,7 +226,7 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr return { statusCode: 202, headers: { 'Content-Type': 'text/plain' }, - body: 'Jobs Queued 1' + body, + body: 'Jobs Queued 3' + body, }; } catch (err) { return { From 1ce574b6e01326c6dea7bdba6e0ab069e9e65789 Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Tue, 12 Mar 2024 23:14:05 -0400 Subject: [PATCH 043/254] DOP-4414 testing lambda --- api/controllers/v2/github.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/api/controllers/v2/github.ts b/api/controllers/v2/github.ts index 368e7cb4d..53164db16 100644 --- a/api/controllers/v2/github.ts +++ b/api/controllers/v2/github.ts @@ -191,7 +191,7 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr const repoInfo = await docsetsRepository.getRepo(s, path); const repoName = s; const projectEntry = await projectsRepository.getProjectEntry(s); - return true; + return projectEntry; const repoOwner = projectEntry.github.organization; //add commit hash- how do you get commit hash?? @@ -222,11 +222,11 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr } try { - await createAndInsertJob(); + const projectEntry = await createAndInsertJob(); return { statusCode: 202, headers: { 'Content-Type': 'text/plain' }, - body: 'Jobs Queued 3' + body, + body: 'Jobs Queued 4' + projectEntry, }; } catch (err) { return { From c51a0e0d701be42b5d2376d3b3fbf30577b8a1e7 Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Tue, 12 Mar 2024 23:36:43 -0400 Subject: [PATCH 044/254] DOP-4414 testing lambda --- api/controllers/v2/github.ts | 2 +- src/repositories/projectsRepository.ts | 5 ----- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/api/controllers/v2/github.ts b/api/controllers/v2/github.ts index 53164db16..ea25b2e02 100644 --- a/api/controllers/v2/github.ts +++ b/api/controllers/v2/github.ts @@ -16,7 +16,7 @@ import { ReposBranchesDocsetsDocument } from '../../../modules/persistence/src/s import { MONOREPO_NAME } from '../../../src/monorepo/utils/monorepo-constants'; const SMOKETEST_SITES = [ - 'docs-landing', + 'landing', 'cloud-docs', 'docs-realm', 'docs', diff --git a/src/repositories/projectsRepository.ts b/src/repositories/projectsRepository.ts index af54fbfb7..29a9cd24a 100644 --- a/src/repositories/projectsRepository.ts +++ b/src/repositories/projectsRepository.ts @@ -18,11 +18,6 @@ export class ProjectsRepository extends BaseRepository { `Mongo Timeout Error: Timedout while retrieving branches for ${name} }` ); - // if user has specific entitlements return projectEntry; } - - static getGithuRepoUrl(repoOwner: string, repoName: string): string { - return 'https://github.com/' + repoOwner + '/' + repoName; - } } From a43b0db5ff891e69444745508d03ef358e081299 Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Tue, 12 Mar 2024 23:48:11 -0400 Subject: [PATCH 045/254] DOP-4414 testing lambda --- src/repositories/projectsRepository.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/repositories/projectsRepository.ts b/src/repositories/projectsRepository.ts index 29a9cd24a..af8a6fc2c 100644 --- a/src/repositories/projectsRepository.ts +++ b/src/repositories/projectsRepository.ts @@ -18,6 +18,6 @@ export class ProjectsRepository extends BaseRepository { `Mongo Timeout Error: Timedout while retrieving branches for ${name} }` ); - return projectEntry; + return name; } } From 285f88ee4ced185e8edb214e55d66b56fe1a3435 Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Tue, 12 Mar 2024 23:55:41 -0400 Subject: [PATCH 046/254] DOP-4414 testing lambda --- api/controllers/v2/github.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/controllers/v2/github.ts b/api/controllers/v2/github.ts index ea25b2e02..afb439809 100644 --- a/api/controllers/v2/github.ts +++ b/api/controllers/v2/github.ts @@ -226,7 +226,7 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr return { statusCode: 202, headers: { 'Content-Type': 'text/plain' }, - body: 'Jobs Queued 4' + projectEntry, + body: 'Jobs Queued 5' + projectEntry, }; } catch (err) { return { From b420a803232e53a0941800904e7cec2c36844575 Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Wed, 13 Mar 2024 00:07:16 -0400 Subject: [PATCH 047/254] DOP-4414 testing lambda --- api/controllers/v2/github.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/controllers/v2/github.ts b/api/controllers/v2/github.ts index afb439809..540dd3881 100644 --- a/api/controllers/v2/github.ts +++ b/api/controllers/v2/github.ts @@ -191,7 +191,7 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr const repoInfo = await docsetsRepository.getRepo(s, path); const repoName = s; const projectEntry = await projectsRepository.getProjectEntry(s); - return projectEntry; + return s; const repoOwner = projectEntry.github.organization; //add commit hash- how do you get commit hash?? From 14280a6dcad3a01a3b8914474a9cc21441c1a100 Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Wed, 13 Mar 2024 00:14:40 -0400 Subject: [PATCH 048/254] DOP-4414 testing lambda --- api/controllers/v2/github.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/api/controllers/v2/github.ts b/api/controllers/v2/github.ts index 540dd3881..17314013c 100644 --- a/api/controllers/v2/github.ts +++ b/api/controllers/v2/github.ts @@ -187,11 +187,11 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr for (const s in SMOKETEST_SITES) { //ensure repoTitle is consistent with other type of title - const jobTitle = 'Smoke Test' + s; + const repoName = SMOKETEST_SITES[s]; + const jobTitle = 'Smoke Test' + repoName; const repoInfo = await docsetsRepository.getRepo(s, path); - const repoName = s; const projectEntry = await projectsRepository.getProjectEntry(s); - return s; + return repoName; const repoOwner = projectEntry.github.organization; //add commit hash- how do you get commit hash?? @@ -226,7 +226,7 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr return { statusCode: 202, headers: { 'Content-Type': 'text/plain' }, - body: 'Jobs Queued 5' + projectEntry, + body: 'Jobs Queued 1' + projectEntry, }; } catch (err) { return { From 8844c5ab48047d14fca10e48476f4c81889559e6 Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Wed, 13 Mar 2024 00:18:46 -0400 Subject: [PATCH 049/254] DOP-4414 testing lambda --- api/controllers/v2/github.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/api/controllers/v2/github.ts b/api/controllers/v2/github.ts index 17314013c..f5fef4332 100644 --- a/api/controllers/v2/github.ts +++ b/api/controllers/v2/github.ts @@ -189,9 +189,9 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr //ensure repoTitle is consistent with other type of title const repoName = SMOKETEST_SITES[s]; const jobTitle = 'Smoke Test' + repoName; - const repoInfo = await docsetsRepository.getRepo(s, path); - const projectEntry = await projectsRepository.getProjectEntry(s); - return repoName; + const repoInfo = await docsetsRepository.getRepo(repoName, path); + const projectEntry = await projectsRepository.getProjectEntry(repoName); + return projectEntry; const repoOwner = projectEntry.github.organization; //add commit hash- how do you get commit hash?? From dfa6854bd4670470fb944122454cf125627c680d Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Wed, 13 Mar 2024 00:25:57 -0400 Subject: [PATCH 050/254] DOP-4414 testing lambda --- src/repositories/projectsRepository.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/repositories/projectsRepository.ts b/src/repositories/projectsRepository.ts index af8a6fc2c..29a9cd24a 100644 --- a/src/repositories/projectsRepository.ts +++ b/src/repositories/projectsRepository.ts @@ -18,6 +18,6 @@ export class ProjectsRepository extends BaseRepository { `Mongo Timeout Error: Timedout while retrieving branches for ${name} }` ); - return name; + return projectEntry; } } From 81461c234c9e1c4b2b5470765469d4ebcf4c7b8d Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Wed, 13 Mar 2024 00:33:42 -0400 Subject: [PATCH 051/254] DOP-4414 testing lambda --- api/controllers/v2/github.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/api/controllers/v2/github.ts b/api/controllers/v2/github.ts index f5fef4332..0d0320145 100644 --- a/api/controllers/v2/github.ts +++ b/api/controllers/v2/github.ts @@ -191,7 +191,7 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr const jobTitle = 'Smoke Test' + repoName; const repoInfo = await docsetsRepository.getRepo(repoName, path); const projectEntry = await projectsRepository.getProjectEntry(repoName); - return projectEntry; + return projectEntry.github; const repoOwner = projectEntry.github.organization; //add commit hash- how do you get commit hash?? @@ -226,7 +226,7 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr return { statusCode: 202, headers: { 'Content-Type': 'text/plain' }, - body: 'Jobs Queued 1' + projectEntry, + body: 'Jobs Queued 3' + projectEntry, }; } catch (err) { return { From 3a6c9181c5e58a4754b4fc762b3c1ed5ba0cff8b Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Wed, 13 Mar 2024 00:43:59 -0400 Subject: [PATCH 052/254] DOP-4414 testing lambda --- api/controllers/v2/github.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/api/controllers/v2/github.ts b/api/controllers/v2/github.ts index 0d0320145..0fecd976d 100644 --- a/api/controllers/v2/github.ts +++ b/api/controllers/v2/github.ts @@ -191,7 +191,7 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr const jobTitle = 'Smoke Test' + repoName; const repoInfo = await docsetsRepository.getRepo(repoName, path); const projectEntry = await projectsRepository.getProjectEntry(repoName); - return projectEntry.github; + return projectEntry.github.organization; const repoOwner = projectEntry.github.organization; //add commit hash- how do you get commit hash?? @@ -226,7 +226,7 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr return { statusCode: 202, headers: { 'Content-Type': 'text/plain' }, - body: 'Jobs Queued 3' + projectEntry, + body: 'Jobs Queued 1' + projectEntry, }; } catch (err) { return { From c9a40eae0fb6f4c7aa55188fff856e5e93ea422b Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Wed, 13 Mar 2024 00:50:05 -0400 Subject: [PATCH 053/254] DOP-4414 testing lambda --- api/controllers/v2/github.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/controllers/v2/github.ts b/api/controllers/v2/github.ts index 0fecd976d..14f4525e6 100644 --- a/api/controllers/v2/github.ts +++ b/api/controllers/v2/github.ts @@ -191,7 +191,6 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr const jobTitle = 'Smoke Test' + repoName; const repoInfo = await docsetsRepository.getRepo(repoName, path); const projectEntry = await projectsRepository.getProjectEntry(repoName); - return projectEntry.github.organization; const repoOwner = projectEntry.github.organization; //add commit hash- how do you get commit hash?? @@ -202,6 +201,7 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr const payload = await createPayload(repoName, true, jobPrefix, repoBranchesRepository, repoInfo, repoOwner); //add logic for getting master branch, latest stable branch + return payload; const job = await prepGithubPushPayload(body, payload, jobTitle); deployable.push(job); } From d6035296002b982bc779ef64efab6c35ba10b9d3 Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Wed, 13 Mar 2024 00:56:12 -0400 Subject: [PATCH 054/254] DOP-4414 testing lambda --- api/controllers/v2/github.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/controllers/v2/github.ts b/api/controllers/v2/github.ts index 14f4525e6..222f923d2 100644 --- a/api/controllers/v2/github.ts +++ b/api/controllers/v2/github.ts @@ -70,7 +70,7 @@ async function createPayload( if (isSmokeTestDeploy) { try { if (!repoOwner) { - return false; + return 'no repo owner'; } url = 'https://github.com/' + repoOwner + '/' + repoName; } catch (e) { From e650b1536fafd6e28b0d5d483650ecc736bd61c3 Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Wed, 13 Mar 2024 00:59:09 -0400 Subject: [PATCH 055/254] DOP-4414 testing lambda --- api/controllers/v2/github.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/controllers/v2/github.ts b/api/controllers/v2/github.ts index 222f923d2..9c05743c9 100644 --- a/api/controllers/v2/github.ts +++ b/api/controllers/v2/github.ts @@ -226,7 +226,7 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr return { statusCode: 202, headers: { 'Content-Type': 'text/plain' }, - body: 'Jobs Queued 1' + projectEntry, + body: 'Jobs Queued 2' + projectEntry, }; } catch (err) { return { From 26f648bd32c202e55d457226ddd9cc9e4e1077d7 Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Wed, 13 Mar 2024 01:03:20 -0400 Subject: [PATCH 056/254] DOP-4414 testing lambda --- api/controllers/v2/github.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/controllers/v2/github.ts b/api/controllers/v2/github.ts index 9c05743c9..52f159aaf 100644 --- a/api/controllers/v2/github.ts +++ b/api/controllers/v2/github.ts @@ -53,8 +53,8 @@ async function createPayload( prefix: string, repoBranchesRepository: RepoBranchesRepository, repoInfo: ReposBranchesDocsetsDocument, - githubEvent?: PushEvent, repoOwner?: string, + githubEvent?: PushEvent, directory?: string ) { const jobType = 'githubPush'; From 0fb91a196397b895ec6b765e86d4636099f27ee7 Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Wed, 13 Mar 2024 01:05:38 -0400 Subject: [PATCH 057/254] DOP-4414 testing lambda --- api/controllers/v2/github.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/api/controllers/v2/github.ts b/api/controllers/v2/github.ts index 52f159aaf..dafa598cf 100644 --- a/api/controllers/v2/github.ts +++ b/api/controllers/v2/github.ts @@ -69,7 +69,7 @@ async function createPayload( if (isSmokeTestDeploy) { try { - if (!repoOwner) { + if (repoOwner == null) { return 'no repo owner'; } url = 'https://github.com/' + repoOwner + '/' + repoName; @@ -226,7 +226,7 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr return { statusCode: 202, headers: { 'Content-Type': 'text/plain' }, - body: 'Jobs Queued 2' + projectEntry, + body: 'Jobs Queued 3' + projectEntry, }; } catch (err) { return { From 33f91cf5c9c2795431057783885d76c570463142 Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Wed, 13 Mar 2024 01:07:30 -0400 Subject: [PATCH 058/254] DOP-4414 testing lambda --- api/controllers/v2/github.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/controllers/v2/github.ts b/api/controllers/v2/github.ts index dafa598cf..0b300e96f 100644 --- a/api/controllers/v2/github.ts +++ b/api/controllers/v2/github.ts @@ -201,7 +201,7 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr const payload = await createPayload(repoName, true, jobPrefix, repoBranchesRepository, repoInfo, repoOwner); //add logic for getting master branch, latest stable branch - return payload; + return 'true'; const job = await prepGithubPushPayload(body, payload, jobTitle); deployable.push(job); } From d895253d400d05a2d3fbf63994fdb7da08ab8fc5 Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Wed, 13 Mar 2024 01:11:56 -0400 Subject: [PATCH 059/254] DOP-4414 testing lambda --- api/controllers/v2/github.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/controllers/v2/github.ts b/api/controllers/v2/github.ts index 0b300e96f..3093cdf27 100644 --- a/api/controllers/v2/github.ts +++ b/api/controllers/v2/github.ts @@ -198,10 +198,10 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr const jobPrefix = repoInfo?.prefix ? repoInfo['prefix'][env] : ''; // const ammendedJobPrefix = body.after ? jobPrefix + body.after : jobPrefix; // const prefix = ammendedJobPrefix; + return true; const payload = await createPayload(repoName, true, jobPrefix, repoBranchesRepository, repoInfo, repoOwner); //add logic for getting master branch, latest stable branch - return 'true'; const job = await prepGithubPushPayload(body, payload, jobTitle); deployable.push(job); } From b1c8c27678bbb399e8908f6f37f4bd7b656da585 Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Wed, 13 Mar 2024 10:24:38 -0400 Subject: [PATCH 060/254] DOP-4414 testing lambda --- api/controllers/v2/github.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/api/controllers/v2/github.ts b/api/controllers/v2/github.ts index 3093cdf27..1b3463a64 100644 --- a/api/controllers/v2/github.ts +++ b/api/controllers/v2/github.ts @@ -66,7 +66,7 @@ async function createPayload( let isFork = false; let url; let newHead; - + return repoOwner; if (isSmokeTestDeploy) { try { if (repoOwner == null) { @@ -198,9 +198,9 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr const jobPrefix = repoInfo?.prefix ? repoInfo['prefix'][env] : ''; // const ammendedJobPrefix = body.after ? jobPrefix + body.after : jobPrefix; // const prefix = ammendedJobPrefix; - return true; const payload = await createPayload(repoName, true, jobPrefix, repoBranchesRepository, repoInfo, repoOwner); + return payload; //add logic for getting master branch, latest stable branch const job = await prepGithubPushPayload(body, payload, jobTitle); deployable.push(job); @@ -226,7 +226,7 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr return { statusCode: 202, headers: { 'Content-Type': 'text/plain' }, - body: 'Jobs Queued 3' + projectEntry, + body: 'Jobs Queued 1' + projectEntry, }; } catch (err) { return { From 64e1c34e46fc902056f7e25ce47afb62b3d429d8 Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Wed, 13 Mar 2024 10:28:17 -0400 Subject: [PATCH 061/254] DOP-4414 testing lambda --- api/controllers/v2/github.ts | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/api/controllers/v2/github.ts b/api/controllers/v2/github.ts index 1b3463a64..026d05569 100644 --- a/api/controllers/v2/github.ts +++ b/api/controllers/v2/github.ts @@ -66,11 +66,10 @@ async function createPayload( let isFork = false; let url; let newHead; - return repoOwner; if (isSmokeTestDeploy) { try { if (repoOwner == null) { - return 'no repo owner'; + return 'no repo owner' + repoOwner; } url = 'https://github.com/' + repoOwner + '/' + repoName; } catch (e) { @@ -226,7 +225,7 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr return { statusCode: 202, headers: { 'Content-Type': 'text/plain' }, - body: 'Jobs Queued 1' + projectEntry, + body: 'Jobs Queued 2' + projectEntry, }; } catch (err) { return { From 0a837741a1ef9c9e2a8a21c63d3d35397d3a8e63 Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Wed, 13 Mar 2024 10:32:29 -0400 Subject: [PATCH 062/254] DOP-4414 testing lambda --- api/controllers/v2/github.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/api/controllers/v2/github.ts b/api/controllers/v2/github.ts index 026d05569..b538b031f 100644 --- a/api/controllers/v2/github.ts +++ b/api/controllers/v2/github.ts @@ -67,6 +67,7 @@ async function createPayload( let url; let newHead; if (isSmokeTestDeploy) { + return repoOwner; try { if (repoOwner == null) { return 'no repo owner' + repoOwner; @@ -225,7 +226,7 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr return { statusCode: 202, headers: { 'Content-Type': 'text/plain' }, - body: 'Jobs Queued 2' + projectEntry, + body: 'Jobs Queued 3' + projectEntry, }; } catch (err) { return { From bddde4c05dde5196b6f01a38448e7cf39f753442 Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Wed, 13 Mar 2024 10:51:15 -0400 Subject: [PATCH 063/254] DOP-4414 testing lambda --- api/controllers/v2/github.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/api/controllers/v2/github.ts b/api/controllers/v2/github.ts index b538b031f..90549021d 100644 --- a/api/controllers/v2/github.ts +++ b/api/controllers/v2/github.ts @@ -67,12 +67,12 @@ async function createPayload( let url; let newHead; if (isSmokeTestDeploy) { - return repoOwner; try { if (repoOwner == null) { return 'no repo owner' + repoOwner; } url = 'https://github.com/' + repoOwner + '/' + repoName; + return url; } catch (e) { console.log('Error! repoOwner is must be configured for an automated smoke test deploy'); } @@ -226,7 +226,7 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr return { statusCode: 202, headers: { 'Content-Type': 'text/plain' }, - body: 'Jobs Queued 3' + projectEntry, + body: 'Jobs Queued 1' + projectEntry, }; } catch (err) { return { From e3f4151dd9a40700f4c266a4a2bb62dc29a103b7 Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Wed, 13 Mar 2024 11:04:36 -0400 Subject: [PATCH 064/254] DOP-4414 testing lambda --- api/controllers/v2/github.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/controllers/v2/github.ts b/api/controllers/v2/github.ts index 90549021d..55abad835 100644 --- a/api/controllers/v2/github.ts +++ b/api/controllers/v2/github.ts @@ -72,7 +72,6 @@ async function createPayload( return 'no repo owner' + repoOwner; } url = 'https://github.com/' + repoOwner + '/' + repoName; - return url; } catch (e) { console.log('Error! repoOwner is must be configured for an automated smoke test deploy'); } @@ -97,6 +96,7 @@ async function createPayload( const branch_info = await repoBranchesRepository.getRepoBranchAliases(repoName, branch_name, repoInfo.project); const urlSlug = branch_info.aliasObject?.urlSlug ?? branch_name; + return branch_info; return { jobType, From ba4dbae2900069be7a90373910184919458d4a02 Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Wed, 13 Mar 2024 11:11:44 -0400 Subject: [PATCH 065/254] DOP-4414 testing lambda --- api/controllers/v2/github.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/api/controllers/v2/github.ts b/api/controllers/v2/github.ts index 55abad835..bf613a821 100644 --- a/api/controllers/v2/github.ts +++ b/api/controllers/v2/github.ts @@ -95,8 +95,8 @@ async function createPayload( } const branch_info = await repoBranchesRepository.getRepoBranchAliases(repoName, branch_name, repoInfo.project); - const urlSlug = branch_info.aliasObject?.urlSlug ?? branch_name; return branch_info; + const urlSlug = branch_info.aliasObject?.urlSlug ?? branch_name; return { jobType, @@ -226,7 +226,7 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr return { statusCode: 202, headers: { 'Content-Type': 'text/plain' }, - body: 'Jobs Queued 1' + projectEntry, + body: 'Jobs Queued 2' + projectEntry, }; } catch (err) { return { From 666debf135ba84ad1cbd535e2f0a1fa333b17777 Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Wed, 13 Mar 2024 11:25:13 -0400 Subject: [PATCH 066/254] DOP-4414 testing lambda --- api/controllers/v2/github.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/controllers/v2/github.ts b/api/controllers/v2/github.ts index bf613a821..e33835901 100644 --- a/api/controllers/v2/github.ts +++ b/api/controllers/v2/github.ts @@ -94,8 +94,8 @@ async function createPayload( } } + return repoName + branch_name + repoInfo.project; const branch_info = await repoBranchesRepository.getRepoBranchAliases(repoName, branch_name, repoInfo.project); - return branch_info; const urlSlug = branch_info.aliasObject?.urlSlug ?? branch_name; return { From 97f61fdcd8d8479d27b789ff8397ba280a9d0f73 Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Wed, 13 Mar 2024 11:31:07 -0400 Subject: [PATCH 067/254] DOP-4414 testing lambda --- api/controllers/v2/github.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/controllers/v2/github.ts b/api/controllers/v2/github.ts index e33835901..e44090998 100644 --- a/api/controllers/v2/github.ts +++ b/api/controllers/v2/github.ts @@ -78,6 +78,7 @@ async function createPayload( branch_name = 'master'; newHead = null; action = 'automatedTest'; + return repoName + branch_name + repoInfo.project; } else { try { action = 'push'; @@ -94,7 +95,6 @@ async function createPayload( } } - return repoName + branch_name + repoInfo.project; const branch_info = await repoBranchesRepository.getRepoBranchAliases(repoName, branch_name, repoInfo.project); const urlSlug = branch_info.aliasObject?.urlSlug ?? branch_name; From b1a04236458712d51a0e78a5570ca8565efac6d7 Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Wed, 13 Mar 2024 11:36:48 -0400 Subject: [PATCH 068/254] DOP-4414 testing lambda --- api/controllers/v2/github.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/controllers/v2/github.ts b/api/controllers/v2/github.ts index e44090998..5232fc2ac 100644 --- a/api/controllers/v2/github.ts +++ b/api/controllers/v2/github.ts @@ -78,7 +78,7 @@ async function createPayload( branch_name = 'master'; newHead = null; action = 'automatedTest'; - return repoName + branch_name + repoInfo.project; + return repoName + branch_name + repoInfo; } else { try { action = 'push'; From c81dce11711c908beb42cdef84805e7c7bec085d Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Wed, 13 Mar 2024 11:42:42 -0400 Subject: [PATCH 069/254] DOP-4414 testing lambda --- api/controllers/v2/github.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/api/controllers/v2/github.ts b/api/controllers/v2/github.ts index 5232fc2ac..3815e05e5 100644 --- a/api/controllers/v2/github.ts +++ b/api/controllers/v2/github.ts @@ -181,7 +181,7 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr //automated test builds will always deploy in dotcomstg const env = 'dotcomstg'; - async function createAndInsertJob(path?: string) { + async function createAndInsertJob() { //should this array be typed more specifically const deployable: Array = []; @@ -189,7 +189,7 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr //ensure repoTitle is consistent with other type of title const repoName = SMOKETEST_SITES[s]; const jobTitle = 'Smoke Test' + repoName; - const repoInfo = await docsetsRepository.getRepo(repoName, path); + const repoInfo = await docsetsRepository.getRepo(repoName); const projectEntry = await projectsRepository.getProjectEntry(repoName); const repoOwner = projectEntry.github.organization; @@ -226,7 +226,7 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr return { statusCode: 202, headers: { 'Content-Type': 'text/plain' }, - body: 'Jobs Queued 2' + projectEntry, + body: 'Jobs Queued 1' + projectEntry, }; } catch (err) { return { From b4bca0593a79817a186d1d786a0d81801fc28237 Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Wed, 13 Mar 2024 12:22:44 -0400 Subject: [PATCH 070/254] DOP-4414 testing lambda --- api/controllers/v2/github.ts | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/api/controllers/v2/github.ts b/api/controllers/v2/github.ts index 3815e05e5..78e6b4405 100644 --- a/api/controllers/v2/github.ts +++ b/api/controllers/v2/github.ts @@ -16,7 +16,7 @@ import { ReposBranchesDocsetsDocument } from '../../../modules/persistence/src/s import { MONOREPO_NAME } from '../../../src/monorepo/utils/monorepo-constants'; const SMOKETEST_SITES = [ - 'landing', + 'docs-landing', 'cloud-docs', 'docs-realm', 'docs', @@ -188,9 +188,10 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr for (const s in SMOKETEST_SITES) { //ensure repoTitle is consistent with other type of title const repoName = SMOKETEST_SITES[s]; + const jobTitle = 'Smoke Test' + repoName; const repoInfo = await docsetsRepository.getRepo(repoName); - const projectEntry = await projectsRepository.getProjectEntry(repoName); + const projectEntry = await projectsRepository.getProjectEntry(repoInfo.project); const repoOwner = projectEntry.github.organization; //add commit hash- how do you get commit hash?? @@ -226,7 +227,7 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr return { statusCode: 202, headers: { 'Content-Type': 'text/plain' }, - body: 'Jobs Queued 1' + projectEntry, + body: 'Jobs Queued 2' + projectEntry, }; } catch (err) { return { From 783dd3b800c338aa1f7bb8caed4ab41e34ff6cd7 Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Wed, 13 Mar 2024 13:26:02 -0400 Subject: [PATCH 071/254] DOP-4414 testing lambda --- api/controllers/v2/github.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/api/controllers/v2/github.ts b/api/controllers/v2/github.ts index 78e6b4405..50507b5ab 100644 --- a/api/controllers/v2/github.ts +++ b/api/controllers/v2/github.ts @@ -201,10 +201,10 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr // const prefix = ammendedJobPrefix; const payload = await createPayload(repoName, true, jobPrefix, repoBranchesRepository, repoInfo, repoOwner); - return payload; //add logic for getting master branch, latest stable branch const job = await prepGithubPushPayload(body, payload, jobTitle); deployable.push(job); + return job; } try { @@ -227,7 +227,7 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr return { statusCode: 202, headers: { 'Content-Type': 'text/plain' }, - body: 'Jobs Queued 2' + projectEntry, + body: 'Jobs Queued 1' + projectEntry, }; } catch (err) { return { From ad1bd54318a113a6717ea0d4ed8bbc1953b8d092 Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Wed, 13 Mar 2024 13:31:34 -0400 Subject: [PATCH 072/254] DOP-4414 testing lambda --- api/controllers/v2/github.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/api/controllers/v2/github.ts b/api/controllers/v2/github.ts index 50507b5ab..f3edbde08 100644 --- a/api/controllers/v2/github.ts +++ b/api/controllers/v2/github.ts @@ -30,7 +30,8 @@ async function prepGithubPushPayload( githubEvent: PushEvent, payload: any, title: string -): Promise> { +): Promise | boolean> { + return true; return { title: title, user: githubEvent.pusher.name, From a574352ad1bd48dac8c906a393fe50170f7b733c Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Wed, 13 Mar 2024 13:48:45 -0400 Subject: [PATCH 073/254] DOP-4414 testing lambda --- api/controllers/v2/github.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/api/controllers/v2/github.ts b/api/controllers/v2/github.ts index f3edbde08..c428e03ee 100644 --- a/api/controllers/v2/github.ts +++ b/api/controllers/v2/github.ts @@ -30,8 +30,8 @@ async function prepGithubPushPayload( githubEvent: PushEvent, payload: any, title: string -): Promise | boolean> { - return true; +): Promise | string> { + return 'here' + githubEvent.pusher; return { title: title, user: githubEvent.pusher.name, From 622f914ee8988236c9fdf3b7c58368ff356bc2b9 Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Wed, 13 Mar 2024 13:55:25 -0400 Subject: [PATCH 074/254] DOP-4414 testing lambda --- api/controllers/v2/github.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/controllers/v2/github.ts b/api/controllers/v2/github.ts index c428e03ee..aa027b58b 100644 --- a/api/controllers/v2/github.ts +++ b/api/controllers/v2/github.ts @@ -31,7 +31,7 @@ async function prepGithubPushPayload( payload: any, title: string ): Promise | string> { - return 'here' + githubEvent.pusher; + return 'here' + JSON.stringify(githubEvent); return { title: title, user: githubEvent.pusher.name, From 65572bbd2a5dee2d0500fce8c1b8513504f43b26 Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Wed, 13 Mar 2024 14:10:59 -0400 Subject: [PATCH 075/254] DOP-4414 testing lambda --- api/controllers/v2/github.ts | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/api/controllers/v2/github.ts b/api/controllers/v2/github.ts index aa027b58b..5e0967f92 100644 --- a/api/controllers/v2/github.ts +++ b/api/controllers/v2/github.ts @@ -31,11 +31,11 @@ async function prepGithubPushPayload( payload: any, title: string ): Promise | string> { - return 'here' + JSON.stringify(githubEvent); return { title: title, - user: githubEvent.pusher.name, - email: githubEvent.pusher.email ?? '', + user: githubEvent.sender.login, + // email: githubEvent.pusher.email ?? '', + email: '', status: JobStatus.inQueue, createdTime: new Date(), startTime: null, @@ -170,6 +170,7 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr } // //if the build was not building master, no need for smoke test sites + //check that fork is false // if (body.ref.split('/')[2] != 'main') { // console.log('Build was not on master branch, sites will not deploy as no smoke tests are needed'); // return { From 23cab1fa532026e03d7ceaf49d77584d0bdb9ce9 Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Wed, 13 Mar 2024 14:20:22 -0400 Subject: [PATCH 076/254] DOP-4414 testing lambda --- api/controllers/v2/github.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/controllers/v2/github.ts b/api/controllers/v2/github.ts index 5e0967f92..1085d8c7d 100644 --- a/api/controllers/v2/github.ts +++ b/api/controllers/v2/github.ts @@ -229,7 +229,7 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr return { statusCode: 202, headers: { 'Content-Type': 'text/plain' }, - body: 'Jobs Queued 1' + projectEntry, + body: 'Jobs Queued 2' + projectEntry, }; } catch (err) { return { From 81ff14ff6fe46569825817b450f7d7d5f808346c Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Wed, 13 Mar 2024 15:03:47 -0400 Subject: [PATCH 077/254] DOP-4414 testing lambda --- api/controllers/v2/github.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/controllers/v2/github.ts b/api/controllers/v2/github.ts index 1085d8c7d..2d57dc4d1 100644 --- a/api/controllers/v2/github.ts +++ b/api/controllers/v2/github.ts @@ -229,7 +229,7 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr return { statusCode: 202, headers: { 'Content-Type': 'text/plain' }, - body: 'Jobs Queued 2' + projectEntry, + body: 'Jobs Queued 3' + projectEntry, }; } catch (err) { return { From 452bd57cc0a0120ce86dc764090e059f03b03c06 Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Wed, 13 Mar 2024 15:58:53 -0400 Subject: [PATCH 078/254] DOP-4414 testing lambda --- api/controllers/v2/github.ts | 30 ++++++++++++++++++++++-------- 1 file changed, 22 insertions(+), 8 deletions(-) diff --git a/api/controllers/v2/github.ts b/api/controllers/v2/github.ts index 2d57dc4d1..644c94f2c 100644 --- a/api/controllers/v2/github.ts +++ b/api/controllers/v2/github.ts @@ -1,7 +1,7 @@ import * as c from 'config'; import * as mongodb from 'mongodb'; import { APIGatewayEvent, APIGatewayProxyResult } from 'aws-lambda'; -import { PushEvent } from '@octokit/webhooks-types'; +import { PushEvent, WorkflowRunCompletedEvent } from '@octokit/webhooks-types'; import { JobRepository } from '../../../src/repositories/jobRepository'; import { ConsoleLogger } from '../../../src/services/logger'; @@ -27,7 +27,7 @@ const SMOKETEST_SITES = [ ]; async function prepGithubPushPayload( - githubEvent: PushEvent, + githubEvent: PushEvent | WorkflowRunCompletedEvent, payload: any, title: string ): Promise | string> { @@ -157,9 +157,9 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr }; } - let body: PushEvent; + let body: WorkflowRunCompletedEvent; try { - body = JSON.parse(event.body) as PushEvent; + body = JSON.parse(event.body) as WorkflowRunCompletedEvent; } catch (e) { console.log('[TriggerBuild]: ERROR! Could not parse event.body', e); return { @@ -169,14 +169,27 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr }; } + // if (body.workflow_run.conclusion != 'success') + // return { + // statusCode: 202, + // headers: { 'Content-Type': 'text/plain' }, + // body: 'Build on branch' + body.workflow_run.head_branch + ' failed and will not trigger smoke test site deployments ' , + // }; + + // if (body.workflow_run.name != 'Deploy Staging ECS') + // return { + // statusCode: 202, + // headers: { 'Content-Type': 'text/plain' }, + // body: 'Workflow' + body.workflow_run.name + 'completed successfully. Will not trigger smoke test site deployments, only Deploy Staging ECS workflow completion will.' , + // }; + // //if the build was not building master, no need for smoke test sites - //check that fork is false - // if (body.ref.split('/')[2] != 'main') { - // console.log('Build was not on master branch, sites will not deploy as no smoke tests are needed'); + // if (body.workflow_run.head_branch != 'main' || body.repository.fork) { + // console.log('Build was not on master branch in main repo, sites will not deploy as no smoke tests are needed'); // return { // statusCode: 202, // headers: { 'Content-Type': 'text/plain' }, - // body: 'No Jobs queued, build was not on master branch' + body.ref, + // body: 'Build on branch' + body.workflow_run.head_branch + ' will not trigger site deployments as no smoke tests are needed' , // }; // } @@ -204,6 +217,7 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr const payload = await createPayload(repoName, true, jobPrefix, repoBranchesRepository, repoInfo, repoOwner); //add logic for getting master branch, latest stable branch + return true; const job = await prepGithubPushPayload(body, payload, jobTitle); deployable.push(job); return job; From ca67083659ff6610293dbe28df7b0dac2d35a620 Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Wed, 13 Mar 2024 15:59:34 -0400 Subject: [PATCH 079/254] DOP-4414 testing lambda --- .github/workflows/deploy-stg-ecs.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/deploy-stg-ecs.yml b/.github/workflows/deploy-stg-ecs.yml index 1b5c34109..2e0cfb462 100644 --- a/.github/workflows/deploy-stg-ecs.yml +++ b/.github/workflows/deploy-stg-ecs.yml @@ -3,7 +3,6 @@ on: branches: - "main" - "integration" - - "DOP-4414" concurrency: group: environment-stg-${{ github.ref }} cancel-in-progress: true From e1c7d11ec7510d052a9afccdb71dade834466408 Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Wed, 13 Mar 2024 16:19:05 -0400 Subject: [PATCH 080/254] DOP-4414 testing lambda --- api/controllers/v2/github.ts | 47 ++++++++++++++---------------------- 1 file changed, 18 insertions(+), 29 deletions(-) diff --git a/api/controllers/v2/github.ts b/api/controllers/v2/github.ts index 644c94f2c..b44e17ab1 100644 --- a/api/controllers/v2/github.ts +++ b/api/controllers/v2/github.ts @@ -34,7 +34,6 @@ async function prepGithubPushPayload( return { title: title, user: githubEvent.sender.login, - // email: githubEvent.pusher.email ?? '', email: '', status: JobStatus.inQueue, createdTime: new Date(), @@ -62,38 +61,29 @@ async function createPayload( const source = 'github'; const project = repoInfo?.project ?? repoName; - let branch_name = ''; - let action = ''; - let isFork = false; - let url; - let newHead; + let branch_name: string; + let action: string; + let url: any; + let isFork: boolean; + let newHead: any; + if (isSmokeTestDeploy) { - try { - if (repoOwner == null) { - return 'no repo owner' + repoOwner; - } - url = 'https://github.com/' + repoOwner + '/' + repoName; - } catch (e) { - console.log('Error! repoOwner is must be configured for an automated smoke test deploy'); - } + url = 'https://github.com/' + repoOwner + '/' + repoName; branch_name = 'master'; newHead = null; + isFork = false; action = 'automatedTest'; return repoName + branch_name + repoInfo; } else { - try { - action = 'push'; - if (!githubEvent) { - return false; - } - branch_name = githubEvent.ref.split('/')[2]; - isFork = githubEvent?.repository.fork; - url = githubEvent?.repository.clone_url; - newHead = githubEvent?.after; - repoOwner = githubEvent.repository.owner.login; - } catch (e) { - console.log('Error! No Github Push Event provided, payload could not be constructed'); + if (!githubEvent) { + return false; } + action = 'push'; + branch_name = githubEvent.ref.split('/')[2]; + isFork = githubEvent?.repository.fork; + url = githubEvent?.repository.clone_url; + newHead = githubEvent?.after; + repoOwner = githubEvent.repository.owner.login; } const branch_info = await repoBranchesRepository.getRepoBranchAliases(repoName, branch_name, repoInfo.project); @@ -217,10 +207,9 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr const payload = await createPayload(repoName, true, jobPrefix, repoBranchesRepository, repoInfo, repoOwner); //add logic for getting master branch, latest stable branch - return true; const job = await prepGithubPushPayload(body, payload, jobTitle); deployable.push(job); - return job; + return 'job' + job; } try { @@ -243,7 +232,7 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr return { statusCode: 202, headers: { 'Content-Type': 'text/plain' }, - body: 'Jobs Queued 3' + projectEntry, + body: 'Jobs Queued 4' + projectEntry, }; } catch (err) { return { From 8dd1e0cb3a5a5fcf88610d6b7dc2fce96550a5c4 Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Wed, 13 Mar 2024 16:26:31 -0400 Subject: [PATCH 081/254] DOP-4414 testing lambda --- api/controllers/v2/github.ts | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/api/controllers/v2/github.ts b/api/controllers/v2/github.ts index b44e17ab1..79e4124b5 100644 --- a/api/controllers/v2/github.ts +++ b/api/controllers/v2/github.ts @@ -159,12 +159,13 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr }; } - // if (body.workflow_run.conclusion != 'success') - // return { - // statusCode: 202, - // headers: { 'Content-Type': 'text/plain' }, - // body: 'Build on branch' + body.workflow_run.head_branch + ' failed and will not trigger smoke test site deployments ' , - // }; + if (body.workflow_run.conclusion != 'success') + return { + statusCode: 202, + headers: { 'Content-Type': 'text/plain' }, + body: + 'Build on branch' + body.workflow_run.head_branch + ' failed and will not trigger smoke test site deployments ', + }; // if (body.workflow_run.name != 'Deploy Staging ECS') // return { From 521092b34f7087148f0be2d316d4020da105c16e Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Wed, 13 Mar 2024 16:30:40 -0400 Subject: [PATCH 082/254] DOP-4414 testing lambda --- api/controllers/v2/github.ts | 38 +++++++++++++++++++++--------------- 1 file changed, 22 insertions(+), 16 deletions(-) diff --git a/api/controllers/v2/github.ts b/api/controllers/v2/github.ts index 79e4124b5..3b9ad4e38 100644 --- a/api/controllers/v2/github.ts +++ b/api/controllers/v2/github.ts @@ -167,22 +167,28 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr 'Build on branch' + body.workflow_run.head_branch + ' failed and will not trigger smoke test site deployments ', }; - // if (body.workflow_run.name != 'Deploy Staging ECS') - // return { - // statusCode: 202, - // headers: { 'Content-Type': 'text/plain' }, - // body: 'Workflow' + body.workflow_run.name + 'completed successfully. Will not trigger smoke test site deployments, only Deploy Staging ECS workflow completion will.' , - // }; - - // //if the build was not building master, no need for smoke test sites - // if (body.workflow_run.head_branch != 'main' || body.repository.fork) { - // console.log('Build was not on master branch in main repo, sites will not deploy as no smoke tests are needed'); - // return { - // statusCode: 202, - // headers: { 'Content-Type': 'text/plain' }, - // body: 'Build on branch' + body.workflow_run.head_branch + ' will not trigger site deployments as no smoke tests are needed' , - // }; - // } + if (body.workflow_run.name != 'Deploy Staging ECS') + return { + statusCode: 202, + headers: { 'Content-Type': 'text/plain' }, + body: + 'Workflow' + + body.workflow_run.name + + 'completed successfully. Will not trigger smoke test site deployments, only Deploy Staging ECS workflow completion will.', + }; + + //if the build was not building master, no need for smoke test sites + if (body.workflow_run.head_branch != 'main' || body.repository.fork) { + console.log('Build was not on master branch in main repo, sites will not deploy as no smoke tests are needed'); + return { + statusCode: 202, + headers: { 'Content-Type': 'text/plain' }, + body: + 'Build on branch' + + body.workflow_run.head_branch + + ' will not trigger site deployments as no smoke tests are needed', + }; + } //automated test builds will always deploy in dotcomstg const env = 'dotcomstg'; From ea6eb17fdbfd5ec36a8f49ea131bec70e083664d Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Wed, 13 Mar 2024 16:40:08 -0400 Subject: [PATCH 083/254] DOP-4414 testing lambda --- api/controllers/v2/github.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/controllers/v2/github.ts b/api/controllers/v2/github.ts index 3b9ad4e38..19894de30 100644 --- a/api/controllers/v2/github.ts +++ b/api/controllers/v2/github.ts @@ -216,7 +216,7 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr //add logic for getting master branch, latest stable branch const job = await prepGithubPushPayload(body, payload, jobTitle); deployable.push(job); - return 'job' + job; + return 'New' + job; } try { From 386434446299fb843d2220ac52a3a466b09c24ca Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Wed, 13 Mar 2024 17:15:23 -0400 Subject: [PATCH 084/254] DOP-4414 more changes --- api/controllers/v2/github.ts | 7 ++++--- src/entities/job.ts | 2 +- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/api/controllers/v2/github.ts b/api/controllers/v2/github.ts index 19894de30..8250cb701 100644 --- a/api/controllers/v2/github.ts +++ b/api/controllers/v2/github.ts @@ -34,7 +34,6 @@ async function prepGithubPushPayload( return { title: title, user: githubEvent.sender.login, - email: '', status: JobStatus.inQueue, createdTime: new Date(), startTime: null, @@ -164,7 +163,9 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr statusCode: 202, headers: { 'Content-Type': 'text/plain' }, body: - 'Build on branch' + body.workflow_run.head_branch + ' failed and will not trigger smoke test site deployments ', + 'Build on branch' + + body.workflow_run.head_branch + + ' is not complete and will not trigger smoke test site deployments ', }; if (body.workflow_run.name != 'Deploy Staging ECS') @@ -174,7 +175,7 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr body: 'Workflow' + body.workflow_run.name + - 'completed successfully. Will not trigger smoke test site deployments, only Deploy Staging ECS workflow completion will.', + 'completed successfully but only Deploy Staging ECS workflow completion will trigger smoke test site deployments', }; //if the build was not building master, no need for smoke test sites diff --git a/src/entities/job.ts b/src/entities/job.ts index eaceaba59..00611f1b4 100644 --- a/src/entities/job.ts +++ b/src/entities/job.ts @@ -131,7 +131,7 @@ export type EnhancedJob = { buildCommands?: string[]; deployCommands?: string[]; invalidationStatusURL?: string | null; - email: string | null; // probably can be removed + email?: string | null; // probably can be removed comMessage?: string[] | null; purgedUrls?: string[] | null; shouldGenerateSearchManifest?: boolean; From 61eb93e4f0974fead12a98b3c664ba590cf3015f Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Wed, 13 Mar 2024 17:38:32 -0400 Subject: [PATCH 085/254] DOP-4414 cleaning up --- api/controllers/v2/github.ts | 16 +++------------- 1 file changed, 3 insertions(+), 13 deletions(-) diff --git a/api/controllers/v2/github.ts b/api/controllers/v2/github.ts index 8250cb701..5b71a84ad 100644 --- a/api/controllers/v2/github.ts +++ b/api/controllers/v2/github.ts @@ -69,7 +69,6 @@ async function createPayload( if (isSmokeTestDeploy) { url = 'https://github.com/' + repoOwner + '/' + repoName; branch_name = 'master'; - newHead = null; isFork = false; action = 'automatedTest'; return repoName + branch_name + repoInfo; @@ -100,7 +99,6 @@ async function createPayload( urlSlug, isFork, url, - //can we keep newHead as null for smokeTest Deploys newHead, directory, }; @@ -199,7 +197,6 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr const deployable: Array = []; for (const s in SMOKETEST_SITES) { - //ensure repoTitle is consistent with other type of title const repoName = SMOKETEST_SITES[s]; const jobTitle = 'Smoke Test' + repoName; @@ -207,8 +204,7 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr const projectEntry = await projectsRepository.getProjectEntry(repoInfo.project); const repoOwner = projectEntry.github.organization; - //add commit hash- how do you get commit hash?? - //local-build/index.ts line 53? + //add commit hash here const jobPrefix = repoInfo?.prefix ? repoInfo['prefix'][env] : ''; // const ammendedJobPrefix = body.after ? jobPrefix + body.after : jobPrefix; // const prefix = ammendedJobPrefix; @@ -217,7 +213,6 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr //add logic for getting master branch, latest stable branch const job = await prepGithubPushPayload(body, payload, jobTitle); deployable.push(job); - return 'New' + job; } try { @@ -236,12 +231,7 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr } try { - const projectEntry = await createAndInsertJob(); - return { - statusCode: 202, - headers: { 'Content-Type': 'text/plain' }, - body: 'Jobs Queued 4' + projectEntry, - }; + await createAndInsertJob(); } catch (err) { return { statusCode: 500, @@ -252,7 +242,7 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr return { statusCode: 202, headers: { 'Content-Type': 'text/plain' }, - body: 'Jobs Queued', + body: 'Jobs Queued 2', }; }; From 62d9ee362087eff3dfd2650736d25e936dcf2b1d Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Wed, 13 Mar 2024 17:42:09 -0400 Subject: [PATCH 086/254] DOP-4414 cleaning up --- api/controllers/v2/github.ts | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/api/controllers/v2/github.ts b/api/controllers/v2/github.ts index 5b71a84ad..db9d21ae5 100644 --- a/api/controllers/v2/github.ts +++ b/api/controllers/v2/github.ts @@ -60,7 +60,7 @@ async function createPayload( const source = 'github'; const project = repoInfo?.project ?? repoName; - let branch_name: string; + let branchName: string; let action: string; let url: any; let isFork: boolean; @@ -68,24 +68,24 @@ async function createPayload( if (isSmokeTestDeploy) { url = 'https://github.com/' + repoOwner + '/' + repoName; - branch_name = 'master'; + branchName = 'master'; isFork = false; action = 'automatedTest'; - return repoName + branch_name + repoInfo; + return repoName + branchName + repoInfo; } else { if (!githubEvent) { return false; } action = 'push'; - branch_name = githubEvent.ref.split('/')[2]; + branchName = githubEvent.ref.split('/')[2]; isFork = githubEvent?.repository.fork; url = githubEvent?.repository.clone_url; newHead = githubEvent?.after; repoOwner = githubEvent.repository.owner.login; } - const branch_info = await repoBranchesRepository.getRepoBranchAliases(repoName, branch_name, repoInfo.project); - const urlSlug = branch_info.aliasObject?.urlSlug ?? branch_name; + const branchInfo = await repoBranchesRepository.getRepoBranchAliases(repoName, branchName, repoInfo.project); + const urlSlug = branchInfo.aliasObject?.urlSlug ?? branchName; return { jobType, @@ -93,7 +93,7 @@ async function createPayload( action, repoName, repoOwner, - branchName: branch_name, + branchName, project, prefix, urlSlug, From 7d55d9df88330ac3062f2e6f0507458d822e78db Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Thu, 14 Mar 2024 10:17:11 -0400 Subject: [PATCH 087/254] DOP-4414 push to preprd --- .github/workflows/deploy-stg-ecs.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/deploy-stg-ecs.yml b/.github/workflows/deploy-stg-ecs.yml index 2e0cfb462..1b5c34109 100644 --- a/.github/workflows/deploy-stg-ecs.yml +++ b/.github/workflows/deploy-stg-ecs.yml @@ -3,6 +3,7 @@ on: branches: - "main" - "integration" + - "DOP-4414" concurrency: group: environment-stg-${{ github.ref }} cancel-in-progress: true From 3cddb7f79885f1348f459ff0d79024f8f41ba06c Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Thu, 14 Mar 2024 10:32:05 -0400 Subject: [PATCH 088/254] DOP-4414 push to preprd --- api/controllers/v2/github.ts | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/api/controllers/v2/github.ts b/api/controllers/v2/github.ts index db9d21ae5..675959538 100644 --- a/api/controllers/v2/github.ts +++ b/api/controllers/v2/github.ts @@ -171,23 +171,23 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr statusCode: 202, headers: { 'Content-Type': 'text/plain' }, body: - 'Workflow' + + 'Workflow ' + body.workflow_run.name + - 'completed successfully but only Deploy Staging ECS workflow completion will trigger smoke test site deployments', + ' completed successfully but only Deploy Staging ECS workflow completion will trigger smoke test site deployments', }; //if the build was not building master, no need for smoke test sites - if (body.workflow_run.head_branch != 'main' || body.repository.fork) { - console.log('Build was not on master branch in main repo, sites will not deploy as no smoke tests are needed'); - return { - statusCode: 202, - headers: { 'Content-Type': 'text/plain' }, - body: - 'Build on branch' + - body.workflow_run.head_branch + - ' will not trigger site deployments as no smoke tests are needed', - }; - } + // if (body.workflow_run.head_branch != 'main' || body.repository.fork) { + // console.log('Build was not on master branch in main repo, sites will not deploy as no smoke tests are needed'); + // return { + // statusCode: 202, + // headers: { 'Content-Type': 'text/plain' }, + // body: + // 'Build on branch' + + // body.workflow_run.head_branch + + // ' will not trigger site deployments as it was not on master branch in upstream repo', + // }; + // } //automated test builds will always deploy in dotcomstg const env = 'dotcomstg'; From 5dc7ff368d5527bffc3ab9e4a7be5a2dfea2e549 Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Thu, 14 Mar 2024 10:32:56 -0400 Subject: [PATCH 089/254] DOP-4414 push to preprd --- api/controllers/v2/github.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/api/controllers/v2/github.ts b/api/controllers/v2/github.ts index 675959538..f0956b7ba 100644 --- a/api/controllers/v2/github.ts +++ b/api/controllers/v2/github.ts @@ -216,6 +216,7 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr } try { + return true; await jobRepository.insertBulkJobs(deployable, c.get('jobsQueueUrl')); // notify the jobUpdatesQueue From 81002445ee425ccc6fee92290c892f09cc3b9b41 Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Thu, 14 Mar 2024 10:45:08 -0400 Subject: [PATCH 090/254] DOP-4414 push to preprd --- api/controllers/v2/github.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/controllers/v2/github.ts b/api/controllers/v2/github.ts index f0956b7ba..718290b34 100644 --- a/api/controllers/v2/github.ts +++ b/api/controllers/v2/github.ts @@ -213,10 +213,10 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr //add logic for getting master branch, latest stable branch const job = await prepGithubPushPayload(body, payload, jobTitle); deployable.push(job); + return deployable; } try { - return true; await jobRepository.insertBulkJobs(deployable, c.get('jobsQueueUrl')); // notify the jobUpdatesQueue From 49e106aeb28ce85187865c9677e054e0186c72cd Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Thu, 14 Mar 2024 11:13:08 -0400 Subject: [PATCH 091/254] DOP-4414 push to preprd --- api/controllers/v2/github.ts | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/api/controllers/v2/github.ts b/api/controllers/v2/github.ts index 718290b34..e699eafcd 100644 --- a/api/controllers/v2/github.ts +++ b/api/controllers/v2/github.ts @@ -18,12 +18,12 @@ import { MONOREPO_NAME } from '../../../src/monorepo/utils/monorepo-constants'; const SMOKETEST_SITES = [ 'docs-landing', 'cloud-docs', - 'docs-realm', - 'docs', - 'docs-atlas-cli', - 'docs-ecosystem', - 'docs-node', - 'docs-app-services', + // 'docs-realm', + // 'docs', + // 'docs-atlas-cli', + // 'docs-ecosystem', + // 'docs-node', + // 'docs-app-services', ]; async function prepGithubPushPayload( @@ -195,6 +195,7 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr async function createAndInsertJob() { //should this array be typed more specifically const deployable: Array = []; + let names = ''; for (const s in SMOKETEST_SITES) { const repoName = SMOKETEST_SITES[s]; @@ -213,8 +214,9 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr //add logic for getting master branch, latest stable branch const job = await prepGithubPushPayload(body, payload, jobTitle); deployable.push(job); - return deployable; + names = names + s; } + return names; try { await jobRepository.insertBulkJobs(deployable, c.get('jobsQueueUrl')); @@ -231,8 +233,9 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr } } + let returnVal; try { - await createAndInsertJob(); + returnVal = await createAndInsertJob(); } catch (err) { return { statusCode: 500, @@ -243,7 +246,7 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr return { statusCode: 202, headers: { 'Content-Type': 'text/plain' }, - body: 'Jobs Queued 2', + body: 'Jobs Queued ' + returnVal, }; }; From 33c1d31c64b467349d0f30c5909c0b3e570e0d2e Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Thu, 14 Mar 2024 11:32:25 -0400 Subject: [PATCH 092/254] DOP-4414 push to preprd --- api/controllers/v2/github.ts | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/api/controllers/v2/github.ts b/api/controllers/v2/github.ts index e699eafcd..b7cab96c3 100644 --- a/api/controllers/v2/github.ts +++ b/api/controllers/v2/github.ts @@ -18,8 +18,8 @@ import { MONOREPO_NAME } from '../../../src/monorepo/utils/monorepo-constants'; const SMOKETEST_SITES = [ 'docs-landing', 'cloud-docs', - // 'docs-realm', - // 'docs', + 'docs-realm', + 'docs', // 'docs-atlas-cli', // 'docs-ecosystem', // 'docs-node', @@ -30,7 +30,7 @@ async function prepGithubPushPayload( githubEvent: PushEvent | WorkflowRunCompletedEvent, payload: any, title: string -): Promise | string> { +): Promise> { return { title: title, user: githubEvent.sender.login, @@ -214,7 +214,7 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr //add logic for getting master branch, latest stable branch const job = await prepGithubPushPayload(body, payload, jobTitle); deployable.push(job); - names = names + s; + names = names + repoName; } return names; @@ -305,7 +305,7 @@ export const TriggerBuild = async (event: APIGatewayEvent): Promise Date: Thu, 14 Mar 2024 11:44:53 -0400 Subject: [PATCH 093/254] DOP-4414 push to preprd --- api/controllers/v2/github.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/api/controllers/v2/github.ts b/api/controllers/v2/github.ts index b7cab96c3..22b65a6d9 100644 --- a/api/controllers/v2/github.ts +++ b/api/controllers/v2/github.ts @@ -20,10 +20,10 @@ const SMOKETEST_SITES = [ 'cloud-docs', 'docs-realm', 'docs', - // 'docs-atlas-cli', - // 'docs-ecosystem', - // 'docs-node', - // 'docs-app-services', + 'docs-atlas-cli', + 'docs-ecosystem', + 'docs-node', + 'docs-app-services', ]; async function prepGithubPushPayload( From 221187ac63f262d58418187d0d4fdbb00f645298 Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Thu, 14 Mar 2024 11:59:39 -0400 Subject: [PATCH 094/254] DOP-4414 push to preprd --- api/controllers/v2/github.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/api/controllers/v2/github.ts b/api/controllers/v2/github.ts index 22b65a6d9..0f5a69401 100644 --- a/api/controllers/v2/github.ts +++ b/api/controllers/v2/github.ts @@ -21,9 +21,9 @@ const SMOKETEST_SITES = [ 'docs-realm', 'docs', 'docs-atlas-cli', - 'docs-ecosystem', - 'docs-node', - 'docs-app-services', + // 'docs-ecosystem', + // 'docs-node', + // 'docs-app-services', ]; async function prepGithubPushPayload( From 56e63cf72ed375dddd860eca57cffbc5e9ba47b1 Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Thu, 14 Mar 2024 12:12:56 -0400 Subject: [PATCH 095/254] DOP-4414 push to preprd --- api/controllers/v2/github.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/api/controllers/v2/github.ts b/api/controllers/v2/github.ts index 0f5a69401..4c30f1c72 100644 --- a/api/controllers/v2/github.ts +++ b/api/controllers/v2/github.ts @@ -16,10 +16,10 @@ import { ReposBranchesDocsetsDocument } from '../../../modules/persistence/src/s import { MONOREPO_NAME } from '../../../src/monorepo/utils/monorepo-constants'; const SMOKETEST_SITES = [ - 'docs-landing', - 'cloud-docs', - 'docs-realm', - 'docs', + // 'docs-landing', + // 'cloud-docs', + // 'docs-realm', + // 'docs', 'docs-atlas-cli', // 'docs-ecosystem', // 'docs-node', From 099bbbed8cf8afbeb0bf727c34a5655c4b91cacc Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Thu, 14 Mar 2024 12:20:51 -0400 Subject: [PATCH 096/254] DOP-4414 push to preprd --- api/controllers/v2/github.ts | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/api/controllers/v2/github.ts b/api/controllers/v2/github.ts index 4c30f1c72..37f30d0ce 100644 --- a/api/controllers/v2/github.ts +++ b/api/controllers/v2/github.ts @@ -201,9 +201,14 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr const repoName = SMOKETEST_SITES[s]; const jobTitle = 'Smoke Test' + repoName; - const repoInfo = await docsetsRepository.getRepo(repoName); - const projectEntry = await projectsRepository.getProjectEntry(repoInfo.project); - const repoOwner = projectEntry.github.organization; + let repoInfo, projectEntry, repoOwner; + try { + repoInfo = await docsetsRepository.getRepo(repoName); + projectEntry = await projectsRepository.getProjectEntry(repoInfo.project); + repoOwner = projectEntry.github.organization; + } catch { + return 'repoInfo, projectEntry, or repoOwner not found ' + repoInfo + projectEntry + repoOwner; + } //add commit hash here const jobPrefix = repoInfo?.prefix ? repoInfo['prefix'][env] : ''; @@ -211,6 +216,7 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr // const prefix = ammendedJobPrefix; const payload = await createPayload(repoName, true, jobPrefix, repoBranchesRepository, repoInfo, repoOwner); + //add logic for getting master branch, latest stable branch const job = await prepGithubPushPayload(body, payload, jobTitle); deployable.push(job); From 16a40578cd475bd7a63a3dca590a07639428a4a0 Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Thu, 14 Mar 2024 13:31:00 -0400 Subject: [PATCH 097/254] DOP-4414 push to preprd --- api/controllers/v2/github.ts | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/api/controllers/v2/github.ts b/api/controllers/v2/github.ts index 37f30d0ce..f43425e4e 100644 --- a/api/controllers/v2/github.ts +++ b/api/controllers/v2/github.ts @@ -16,11 +16,11 @@ import { ReposBranchesDocsetsDocument } from '../../../modules/persistence/src/s import { MONOREPO_NAME } from '../../../src/monorepo/utils/monorepo-constants'; const SMOKETEST_SITES = [ - // 'docs-landing', - // 'cloud-docs', - // 'docs-realm', - // 'docs', - 'docs-atlas-cli', + 'docs-landing', + 'cloud-docs', + 'docs-realm', + 'docs', + // 'docs-atlas-cli', // 'docs-ecosystem', // 'docs-node', // 'docs-app-services', @@ -198,16 +198,24 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr let names = ''; for (const s in SMOKETEST_SITES) { + const repoBranches = await repoBranchesRepository.getRepoBranches(SMOKETEST_SITES[s]); const repoName = SMOKETEST_SITES[s]; - const jobTitle = 'Smoke Test' + repoName; let repoInfo, projectEntry, repoOwner; try { repoInfo = await docsetsRepository.getRepo(repoName); + return repoBranches.project + repoInfo.project; projectEntry = await projectsRepository.getProjectEntry(repoInfo.project); repoOwner = projectEntry.github.organization; } catch { - return 'repoInfo, projectEntry, or repoOwner not found ' + repoInfo + projectEntry + repoOwner; + return ( + 'repoInfo, projectEntry, or repoOwner not found. Repo info: ' + + repoInfo.project + + 'projectEntry ' + + projectEntry + + ' repoOwner ' + + repoOwner + ); } //add commit hash here From ac84af3a3bcb9871992592c1b7c6927d63f46f74 Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Thu, 14 Mar 2024 13:37:27 -0400 Subject: [PATCH 098/254] DOP-4414 push to preprd --- api/controllers/v2/github.ts | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/api/controllers/v2/github.ts b/api/controllers/v2/github.ts index f43425e4e..b0e4ce129 100644 --- a/api/controllers/v2/github.ts +++ b/api/controllers/v2/github.ts @@ -21,9 +21,9 @@ const SMOKETEST_SITES = [ 'docs-realm', 'docs', // 'docs-atlas-cli', - // 'docs-ecosystem', - // 'docs-node', - // 'docs-app-services', + 'docs-ecosystem', + 'docs-node', + 'docs-app-services', ]; async function prepGithubPushPayload( @@ -198,13 +198,11 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr let names = ''; for (const s in SMOKETEST_SITES) { - const repoBranches = await repoBranchesRepository.getRepoBranches(SMOKETEST_SITES[s]); const repoName = SMOKETEST_SITES[s]; const jobTitle = 'Smoke Test' + repoName; let repoInfo, projectEntry, repoOwner; try { repoInfo = await docsetsRepository.getRepo(repoName); - return repoBranches.project + repoInfo.project; projectEntry = await projectsRepository.getProjectEntry(repoInfo.project); repoOwner = projectEntry.github.organization; } catch { @@ -218,8 +216,8 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr ); } - //add commit hash here const jobPrefix = repoInfo?.prefix ? repoInfo['prefix'][env] : ''; + //add commit hash to jobPrefix here? // const ammendedJobPrefix = body.after ? jobPrefix + body.after : jobPrefix; // const prefix = ammendedJobPrefix; From c3c32aa8d1d8747bc0ad3d709b411d60257d3ce6 Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Thu, 14 Mar 2024 13:49:42 -0400 Subject: [PATCH 099/254] DOP-4414 push to preprd --- api/controllers/v2/github.ts | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/api/controllers/v2/github.ts b/api/controllers/v2/github.ts index b0e4ce129..fdae05e3e 100644 --- a/api/controllers/v2/github.ts +++ b/api/controllers/v2/github.ts @@ -195,7 +195,7 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr async function createAndInsertJob() { //should this array be typed more specifically const deployable: Array = []; - let names = ''; + const names = ''; for (const s in SMOKETEST_SITES) { const repoName = SMOKETEST_SITES[s]; @@ -226,9 +226,7 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr //add logic for getting master branch, latest stable branch const job = await prepGithubPushPayload(body, payload, jobTitle); deployable.push(job); - names = names + repoName; } - return names; try { await jobRepository.insertBulkJobs(deployable, c.get('jobsQueueUrl')); @@ -240,6 +238,7 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr consoleLogger.info(jobId, `Created Job ${jobId}`); }) ); + return true; } catch (err) { consoleLogger.error('deployRepo', err); } From 0c7b2692817068a385cd65b906bbe81bcca9464a Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Thu, 14 Mar 2024 15:32:17 -0400 Subject: [PATCH 100/254] DOP-4414 push to preprd --- api/controllers/v2/github.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/controllers/v2/github.ts b/api/controllers/v2/github.ts index fdae05e3e..27cda89ce 100644 --- a/api/controllers/v2/github.ts +++ b/api/controllers/v2/github.ts @@ -20,7 +20,7 @@ const SMOKETEST_SITES = [ 'cloud-docs', 'docs-realm', 'docs', - // 'docs-atlas-cli', + 'docs-atlas-cli', 'docs-ecosystem', 'docs-node', 'docs-app-services', From c15f5bb50cf7f2ce29e72732144963a96c6366bc Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Thu, 14 Mar 2024 15:32:59 -0400 Subject: [PATCH 101/254] DOP-4414 push to preprd --- api/controllers/v2/github.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/api/controllers/v2/github.ts b/api/controllers/v2/github.ts index 27cda89ce..8a0fdcdd0 100644 --- a/api/controllers/v2/github.ts +++ b/api/controllers/v2/github.ts @@ -195,7 +195,6 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr async function createAndInsertJob() { //should this array be typed more specifically const deployable: Array = []; - const names = ''; for (const s in SMOKETEST_SITES) { const repoName = SMOKETEST_SITES[s]; @@ -238,7 +237,6 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr consoleLogger.info(jobId, `Created Job ${jobId}`); }) ); - return true; } catch (err) { consoleLogger.error('deployRepo', err); } From 39f729d54bddb703d1575889ff4eb3ed7c412525 Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Thu, 14 Mar 2024 15:55:29 -0400 Subject: [PATCH 102/254] DOP-4414 push to preprd --- api/controllers/v2/github.ts | 52 +++++++++++++++++++++--------------- 1 file changed, 31 insertions(+), 21 deletions(-) diff --git a/api/controllers/v2/github.ts b/api/controllers/v2/github.ts index 8a0fdcdd0..80b48daa6 100644 --- a/api/controllers/v2/github.ts +++ b/api/controllers/v2/github.ts @@ -18,12 +18,12 @@ import { MONOREPO_NAME } from '../../../src/monorepo/utils/monorepo-constants'; const SMOKETEST_SITES = [ 'docs-landing', 'cloud-docs', - 'docs-realm', - 'docs', - 'docs-atlas-cli', - 'docs-ecosystem', - 'docs-node', - 'docs-app-services', + // 'docs-realm', + // 'docs', + // 'docs-atlas-cli', + // 'docs-ecosystem', + // 'docs-node', + // 'docs-app-services', ]; async function prepGithubPushPayload( @@ -224,22 +224,32 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr //add logic for getting master branch, latest stable branch const job = await prepGithubPushPayload(body, payload, jobTitle); - deployable.push(job); - } - - try { - await jobRepository.insertBulkJobs(deployable, c.get('jobsQueueUrl')); - - // notify the jobUpdatesQueue - await Promise.all( - deployable.map(async ({ jobId }) => { - await jobRepository.notify(jobId, c.get('jobUpdatesQueueUrl'), JobStatus.inQueue, 0); - consoleLogger.info(jobId, `Created Job ${jobId}`); - }) - ); - } catch (err) { - consoleLogger.error('deployRepo', err); + try { + consoleLogger.info(job.title, 'Creating Job'); + const jobId = await jobRepository.insertJob(job, c.get('jobsQueueUrl')); + jobRepository.notify(jobId, c.get('jobUpdatesQueueUrl'), JobStatus.inQueue, 0); + consoleLogger.info(job.title, `Created Job ${jobId}`); + deployable.push(jobId); + } catch (err) { + consoleLogger.error('TriggerBuildError', err + repoName); + } + // deployable.push(job); } + return deployable; + + // try { + // await jobRepository.insertBulkJobs(deployable, c.get('jobsQueueUrl')); + + // // notify the jobUpdatesQueue + // await Promise.all( + // deployable.map(async ({ jobId }) => { + // await jobRepository.notify(jobId, c.get('jobUpdatesQueueUrl'), JobStatus.inQueue, 0); + // consoleLogger.info(jobId, `Created Job ${jobId}`); + // }) + // ); + // } catch (err) { + // consoleLogger.error('TriggerBuildError', err+ ); + // } } let returnVal; From 8e378d2492e960a7bb4fbb6aeda683b0b9042202 Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Thu, 14 Mar 2024 16:09:56 -0400 Subject: [PATCH 103/254] DOP-4414 push to preprd --- api/controllers/v2/github.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/api/controllers/v2/github.ts b/api/controllers/v2/github.ts index 80b48daa6..19a429f9b 100644 --- a/api/controllers/v2/github.ts +++ b/api/controllers/v2/github.ts @@ -231,6 +231,7 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr consoleLogger.info(job.title, `Created Job ${jobId}`); deployable.push(jobId); } catch (err) { + return err + repoName; consoleLogger.error('TriggerBuildError', err + repoName); } // deployable.push(job); From ab213457050d6b0380520819c9a4a513fd52d04a Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Thu, 14 Mar 2024 16:29:42 -0400 Subject: [PATCH 104/254] DOP-4414 return true early --- api/controllers/v2/github.ts | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/api/controllers/v2/github.ts b/api/controllers/v2/github.ts index 19a429f9b..33aee01b8 100644 --- a/api/controllers/v2/github.ts +++ b/api/controllers/v2/github.ts @@ -224,16 +224,17 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr //add logic for getting master branch, latest stable branch const job = await prepGithubPushPayload(body, payload, jobTitle); - try { - consoleLogger.info(job.title, 'Creating Job'); - const jobId = await jobRepository.insertJob(job, c.get('jobsQueueUrl')); - jobRepository.notify(jobId, c.get('jobUpdatesQueueUrl'), JobStatus.inQueue, 0); - consoleLogger.info(job.title, `Created Job ${jobId}`); - deployable.push(jobId); - } catch (err) { - return err + repoName; - consoleLogger.error('TriggerBuildError', err + repoName); - } + return true; + // try { + // consoleLogger.info(job.title, 'Creating Job'); + // const jobId = await jobRepository.insertJob(job, c.get('jobsQueueUrl')); + // jobRepository.notify(jobId, c.get('jobUpdatesQueueUrl'), JobStatus.inQueue, 0); + // consoleLogger.info(job.title, `Created Job ${jobId}`); + // deployable.push(jobId); + // } catch (err) { + // return err + repoName; + // consoleLogger.error('TriggerBuildError', err + repoName); + // } // deployable.push(job); } return deployable; From d3a7632a28ae427028169ce9df7fd0c2b07d45b2 Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Thu, 14 Mar 2024 16:46:29 -0400 Subject: [PATCH 105/254] DOP-4414 return true early --- api/controllers/v2/github.ts | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/api/controllers/v2/github.ts b/api/controllers/v2/github.ts index 33aee01b8..461cecfab 100644 --- a/api/controllers/v2/github.ts +++ b/api/controllers/v2/github.ts @@ -224,18 +224,18 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr //add logic for getting master branch, latest stable branch const job = await prepGithubPushPayload(body, payload, jobTitle); - return true; - // try { - // consoleLogger.info(job.title, 'Creating Job'); - // const jobId = await jobRepository.insertJob(job, c.get('jobsQueueUrl')); - // jobRepository.notify(jobId, c.get('jobUpdatesQueueUrl'), JobStatus.inQueue, 0); - // consoleLogger.info(job.title, `Created Job ${jobId}`); - // deployable.push(jobId); - // } catch (err) { - // return err + repoName; - // consoleLogger.error('TriggerBuildError', err + repoName); - // } - // deployable.push(job); + try { + consoleLogger.info(job.title, 'Creating Job'); + const jobId = await jobRepository.insertJob(job, c.get('jobsQueueUrl')); + return jobId; + jobRepository.notify(jobId, c.get('jobUpdatesQueueUrl'), JobStatus.inQueue, 0); + consoleLogger.info(job.title, `Created Job ${jobId}`); + deployable.push(jobId); + } catch (err) { + return err + repoName; + consoleLogger.error('TriggerBuildError', err + repoName); + } + deployable.push(job); } return deployable; From cdc2a8029626f3290dcdd418decb5ee70acf61b2 Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Thu, 14 Mar 2024 17:00:18 -0400 Subject: [PATCH 106/254] DOP-4414 return true early --- api/controllers/v2/github.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/controllers/v2/github.ts b/api/controllers/v2/github.ts index 461cecfab..2a1895f6d 100644 --- a/api/controllers/v2/github.ts +++ b/api/controllers/v2/github.ts @@ -227,8 +227,8 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr try { consoleLogger.info(job.title, 'Creating Job'); const jobId = await jobRepository.insertJob(job, c.get('jobsQueueUrl')); - return jobId; jobRepository.notify(jobId, c.get('jobUpdatesQueueUrl'), JobStatus.inQueue, 0); + return jobId; consoleLogger.info(job.title, `Created Job ${jobId}`); deployable.push(jobId); } catch (err) { From 49262ccdd3c178875ed20a24e9bf0abe7ffc3b0d Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Thu, 14 Mar 2024 17:25:33 -0400 Subject: [PATCH 107/254] DOP-4414 return true early --- api/controllers/v2/github.ts | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/api/controllers/v2/github.ts b/api/controllers/v2/github.ts index 2a1895f6d..42405ffdf 100644 --- a/api/controllers/v2/github.ts +++ b/api/controllers/v2/github.ts @@ -134,15 +134,15 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr }; } - // validate credentials here - if (!validateJsonWebhook(event, 'mongodbsmoketesting')) { - const errMsg = "X-Hub-Signature incorrect. Github webhook token doesn't match"; - return { - statusCode: 401, - headers: { 'Content-Type': 'text/plain' }, - body: errMsg, - }; - } + // // validate credentials here + // if (!validateJsonWebhook(event, 'mongodbsmoketesting')) { + // const errMsg = "X-Hub-Signature incorrect. Github webhook token doesn't match"; + // return { + // statusCode: 401, + // headers: { 'Content-Type': 'text/plain' }, + // body: errMsg, + // }; + // } let body: WorkflowRunCompletedEvent; try { @@ -161,7 +161,7 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr statusCode: 202, headers: { 'Content-Type': 'text/plain' }, body: - 'Build on branch' + + 'Build on branch ' + body.workflow_run.head_branch + ' is not complete and will not trigger smoke test site deployments ', }; From 4eb40e05a84a901747f9efc653d204bae4ffb889 Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Thu, 14 Mar 2024 17:47:17 -0400 Subject: [PATCH 108/254] DOP-4414 return true early --- api/controllers/v2/github.ts | 3 --- 1 file changed, 3 deletions(-) diff --git a/api/controllers/v2/github.ts b/api/controllers/v2/github.ts index 42405ffdf..7e281ddb1 100644 --- a/api/controllers/v2/github.ts +++ b/api/controllers/v2/github.ts @@ -71,7 +71,6 @@ async function createPayload( branchName = 'master'; isFork = false; action = 'automatedTest'; - return repoName + branchName + repoInfo; } else { if (!githubEvent) { return false; @@ -228,9 +227,7 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr consoleLogger.info(job.title, 'Creating Job'); const jobId = await jobRepository.insertJob(job, c.get('jobsQueueUrl')); jobRepository.notify(jobId, c.get('jobUpdatesQueueUrl'), JobStatus.inQueue, 0); - return jobId; consoleLogger.info(job.title, `Created Job ${jobId}`); - deployable.push(jobId); } catch (err) { return err + repoName; consoleLogger.error('TriggerBuildError', err + repoName); From 4c27e3af33ee43555419fd7209911d2eb674c44a Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Thu, 14 Mar 2024 18:03:00 -0400 Subject: [PATCH 109/254] DOP-4414 return true early --- api/controllers/v2/github.ts | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/api/controllers/v2/github.ts b/api/controllers/v2/github.ts index 7e281ddb1..7b620d450 100644 --- a/api/controllers/v2/github.ts +++ b/api/controllers/v2/github.ts @@ -17,9 +17,9 @@ import { MONOREPO_NAME } from '../../../src/monorepo/utils/monorepo-constants'; const SMOKETEST_SITES = [ 'docs-landing', - 'cloud-docs', - // 'docs-realm', - // 'docs', + // 'cloud-docs', + 'docs-realm', + 'docs', // 'docs-atlas-cli', // 'docs-ecosystem', // 'docs-node', @@ -195,9 +195,10 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr //should this array be typed more specifically const deployable: Array = []; + let names = ''; for (const s in SMOKETEST_SITES) { const repoName = SMOKETEST_SITES[s]; - const jobTitle = 'Smoke Test' + repoName; + const jobTitle = 'Smoke Test ' + repoName; let repoInfo, projectEntry, repoOwner; try { repoInfo = await docsetsRepository.getRepo(repoName); @@ -228,13 +229,14 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr const jobId = await jobRepository.insertJob(job, c.get('jobsQueueUrl')); jobRepository.notify(jobId, c.get('jobUpdatesQueueUrl'), JobStatus.inQueue, 0); consoleLogger.info(job.title, `Created Job ${jobId}`); + names = names + repoName; } catch (err) { return err + repoName; consoleLogger.error('TriggerBuildError', err + repoName); } - deployable.push(job); + // deployable.push(job); } - return deployable; + return names; // try { // await jobRepository.insertBulkJobs(deployable, c.get('jobsQueueUrl')); @@ -251,9 +253,8 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr // } } - let returnVal; try { - returnVal = await createAndInsertJob(); + await createAndInsertJob(); } catch (err) { return { statusCode: 500, @@ -264,7 +265,7 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr return { statusCode: 202, headers: { 'Content-Type': 'text/plain' }, - body: 'Jobs Queued ' + returnVal, + body: 'Jobs Queued ', }; }; From debac8f434753d9f2e74b8a92fb7ffe9f3fbf5b9 Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Fri, 15 Mar 2024 10:38:43 -0400 Subject: [PATCH 110/254] DOP-4414 return true early --- api/controllers/v2/github.ts | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/api/controllers/v2/github.ts b/api/controllers/v2/github.ts index 7b620d450..21008247f 100644 --- a/api/controllers/v2/github.ts +++ b/api/controllers/v2/github.ts @@ -17,13 +17,13 @@ import { MONOREPO_NAME } from '../../../src/monorepo/utils/monorepo-constants'; const SMOKETEST_SITES = [ 'docs-landing', - // 'cloud-docs', + 'cloud-docs', 'docs-realm', 'docs', - // 'docs-atlas-cli', - // 'docs-ecosystem', - // 'docs-node', - // 'docs-app-services', + 'docs-atlas-cli', + 'docs-ecosystem', + 'docs-node', + 'docs-app-services', ]; async function prepGithubPushPayload( @@ -133,15 +133,15 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr }; } - // // validate credentials here - // if (!validateJsonWebhook(event, 'mongodbsmoketesting')) { - // const errMsg = "X-Hub-Signature incorrect. Github webhook token doesn't match"; - // return { - // statusCode: 401, - // headers: { 'Content-Type': 'text/plain' }, - // body: errMsg, - // }; - // } + // validate credentials here + if (!validateJsonWebhook(event, 'mongodbsmoketesting')) { + const errMsg = "X-Hub-Signature incorrect. Github webhook token doesn't match"; + return { + statusCode: 401, + headers: { 'Content-Type': 'text/plain' }, + body: errMsg, + }; + } let body: WorkflowRunCompletedEvent; try { From fbb69873ddd226ba0f9970e016f8c5f103d6370b Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Fri, 15 Mar 2024 11:19:11 -0400 Subject: [PATCH 111/254] DOP-4414 return true early --- api/controllers/v2/github.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/api/controllers/v2/github.ts b/api/controllers/v2/github.ts index 21008247f..53a93645e 100644 --- a/api/controllers/v2/github.ts +++ b/api/controllers/v2/github.ts @@ -16,8 +16,8 @@ import { ReposBranchesDocsetsDocument } from '../../../modules/persistence/src/s import { MONOREPO_NAME } from '../../../src/monorepo/utils/monorepo-constants'; const SMOKETEST_SITES = [ - 'docs-landing', - 'cloud-docs', + // 'docs-landing', + // 'cloud-docs', 'docs-realm', 'docs', 'docs-atlas-cli', From 0dd6ecc66449dd377bb11b2f013af2069034a2de Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Fri, 15 Mar 2024 11:32:51 -0400 Subject: [PATCH 112/254] DOP-4414 remove validation --- api/controllers/v2/github.ts | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/api/controllers/v2/github.ts b/api/controllers/v2/github.ts index 53a93645e..125a3ff6a 100644 --- a/api/controllers/v2/github.ts +++ b/api/controllers/v2/github.ts @@ -95,7 +95,7 @@ async function createPayload( branchName, project, prefix, - urlSlug, + urlSlug: '', isFork, url, newHead, @@ -134,14 +134,14 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr } // validate credentials here - if (!validateJsonWebhook(event, 'mongodbsmoketesting')) { - const errMsg = "X-Hub-Signature incorrect. Github webhook token doesn't match"; - return { - statusCode: 401, - headers: { 'Content-Type': 'text/plain' }, - body: errMsg, - }; - } + // if (!validateJsonWebhook(event, 'mongodbsmoketesting')) { + // const errMsg = "X-Hub-Signature incorrect. Github webhook token doesn't match"; + // return { + // statusCode: 401, + // headers: { 'Content-Type': 'text/plain' }, + // body: errMsg, + // }; + // } let body: WorkflowRunCompletedEvent; try { From 8280c1b6092408ed07840ca230d716346cf932af Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Fri, 15 Mar 2024 11:48:56 -0400 Subject: [PATCH 113/254] DOP-4414 fewer sites --- api/controllers/v2/github.ts | 20 +++++++++----------- src/job/jobHandler.ts | 1 + 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/api/controllers/v2/github.ts b/api/controllers/v2/github.ts index 125a3ff6a..711bcbd86 100644 --- a/api/controllers/v2/github.ts +++ b/api/controllers/v2/github.ts @@ -20,10 +20,10 @@ const SMOKETEST_SITES = [ // 'cloud-docs', 'docs-realm', 'docs', - 'docs-atlas-cli', - 'docs-ecosystem', - 'docs-node', - 'docs-app-services', + // 'docs-atlas-cli', + // 'docs-ecosystem', + // 'docs-node', + // 'docs-app-services', ]; async function prepGithubPushPayload( @@ -95,7 +95,7 @@ async function createPayload( branchName, project, prefix, - urlSlug: '', + urlSlug, isFork, url, newHead, @@ -192,9 +192,6 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr const env = 'dotcomstg'; async function createAndInsertJob() { - //should this array be typed more specifically - const deployable: Array = []; - let names = ''; for (const s in SMOKETEST_SITES) { const repoName = SMOKETEST_SITES[s]; @@ -253,19 +250,20 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr // } } + let returnVal; try { - await createAndInsertJob(); + returnVal = await createAndInsertJob(); } catch (err) { return { statusCode: 500, headers: { 'Content-Type': 'text/plain' }, - body: err, + body: returnVal + err, }; } return { statusCode: 202, headers: { 'Content-Type': 'text/plain' }, - body: 'Jobs Queued ', + body: 'Jobs Queued ' + returnVal, }; }; diff --git a/src/job/jobHandler.ts b/src/job/jobHandler.ts index dbd9b6718..fc92964b0 100644 --- a/src/job/jobHandler.ts +++ b/src/job/jobHandler.ts @@ -152,6 +152,7 @@ export abstract class JobHandler { const pathPrefix = await this.getPathPrefix(); // TODO: Can empty string check be removed? if (pathPrefix || pathPrefix === '') { + this._logger.save(this.currJob._id, 'Path prefix raw' + pathPrefix); this.currJob.payload.pathPrefix = pathPrefix; const mutPrefix = pathPrefix.split(`/${server_user}`)[0]; this.currJob.payload.mutPrefix = mutPrefix; From 84768ee44eeb1750cf759446b1629314579d394f Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Fri, 15 Mar 2024 12:05:40 -0400 Subject: [PATCH 114/254] DOP-4414 fewer sites --- api/controllers/v2/github.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/api/controllers/v2/github.ts b/api/controllers/v2/github.ts index 711bcbd86..7c7f9b52c 100644 --- a/api/controllers/v2/github.ts +++ b/api/controllers/v2/github.ts @@ -221,6 +221,10 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr //add logic for getting master branch, latest stable branch const job = await prepGithubPushPayload(body, payload, jobTitle); + + if (repoName == 'docs') { + return job; + } try { consoleLogger.info(job.title, 'Creating Job'); const jobId = await jobRepository.insertJob(job, c.get('jobsQueueUrl')); From a324aa93ef2f04b38a875fd52f04ec624d902e80 Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Fri, 15 Mar 2024 12:34:28 -0400 Subject: [PATCH 115/254] DOP-4414 fewer sites --- api/controllers/v2/github.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/controllers/v2/github.ts b/api/controllers/v2/github.ts index 7c7f9b52c..5b3fc0e25 100644 --- a/api/controllers/v2/github.ts +++ b/api/controllers/v2/github.ts @@ -236,8 +236,8 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr consoleLogger.error('TriggerBuildError', err + repoName); } // deployable.push(job); + return names; } - return names; // try { // await jobRepository.insertBulkJobs(deployable, c.get('jobsQueueUrl')); From d1aa50888267cb6b709adc0e231ec54268d7011c Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Fri, 15 Mar 2024 12:56:56 -0400 Subject: [PATCH 116/254] DOP-4414 fewer sites --- api/controllers/v2/github.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/api/controllers/v2/github.ts b/api/controllers/v2/github.ts index 5b3fc0e25..43956ab0b 100644 --- a/api/controllers/v2/github.ts +++ b/api/controllers/v2/github.ts @@ -219,12 +219,12 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr const payload = await createPayload(repoName, true, jobPrefix, repoBranchesRepository, repoInfo, repoOwner); + if (repoName != 'docs-realm') { + return payload; + } //add logic for getting master branch, latest stable branch const job = await prepGithubPushPayload(body, payload, jobTitle); - if (repoName == 'docs') { - return job; - } try { consoleLogger.info(job.title, 'Creating Job'); const jobId = await jobRepository.insertJob(job, c.get('jobsQueueUrl')); From c3f01ffc1ff28b75679f3925acf70a40aad523db Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Fri, 15 Mar 2024 13:36:43 -0400 Subject: [PATCH 117/254] DOP-4414 fewer sites --- api/controllers/v2/github.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/api/controllers/v2/github.ts b/api/controllers/v2/github.ts index 43956ab0b..8e52c9211 100644 --- a/api/controllers/v2/github.ts +++ b/api/controllers/v2/github.ts @@ -56,6 +56,9 @@ async function createPayload( githubEvent?: PushEvent, directory?: string ) { + if (repoName != 'docs-realm') { + return repoName; + } const jobType = 'githubPush'; const source = 'github'; const project = repoInfo?.project ?? repoName; @@ -219,9 +222,6 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr const payload = await createPayload(repoName, true, jobPrefix, repoBranchesRepository, repoInfo, repoOwner); - if (repoName != 'docs-realm') { - return payload; - } //add logic for getting master branch, latest stable branch const job = await prepGithubPushPayload(body, payload, jobTitle); From 51bc7919ad7b1e746257e0bbc06c7281fc627792 Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Fri, 15 Mar 2024 13:57:55 -0400 Subject: [PATCH 118/254] DOP-4414 fewer sites --- api/controllers/v2/github.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/api/controllers/v2/github.ts b/api/controllers/v2/github.ts index 8e52c9211..e059d4707 100644 --- a/api/controllers/v2/github.ts +++ b/api/controllers/v2/github.ts @@ -56,9 +56,6 @@ async function createPayload( githubEvent?: PushEvent, directory?: string ) { - if (repoName != 'docs-realm') { - return repoName; - } const jobType = 'githubPush'; const source = 'github'; const project = repoInfo?.project ?? repoName; @@ -200,6 +197,9 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr const repoName = SMOKETEST_SITES[s]; const jobTitle = 'Smoke Test ' + repoName; let repoInfo, projectEntry, repoOwner; + if (repoName != 'docs-realm') { + return repoName; + } try { repoInfo = await docsetsRepository.getRepo(repoName); projectEntry = await projectsRepository.getProjectEntry(repoInfo.project); From 7035d967477eebfa55990ad1f7932296cbec8f93 Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Fri, 15 Mar 2024 14:12:24 -0400 Subject: [PATCH 119/254] DOP-4414 fewer sites --- api/controllers/v2/github.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/api/controllers/v2/github.ts b/api/controllers/v2/github.ts index e059d4707..06359b632 100644 --- a/api/controllers/v2/github.ts +++ b/api/controllers/v2/github.ts @@ -236,7 +236,6 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr consoleLogger.error('TriggerBuildError', err + repoName); } // deployable.push(job); - return names; } // try { From bed862c52644e3b1a2d960ad8de56010399f96ff Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Fri, 15 Mar 2024 15:00:25 -0400 Subject: [PATCH 120/254] DOP-4414 fewer sites --- api/controllers/v2/github.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/controllers/v2/github.ts b/api/controllers/v2/github.ts index 06359b632..6b4b60893 100644 --- a/api/controllers/v2/github.ts +++ b/api/controllers/v2/github.ts @@ -18,7 +18,7 @@ import { MONOREPO_NAME } from '../../../src/monorepo/utils/monorepo-constants'; const SMOKETEST_SITES = [ // 'docs-landing', // 'cloud-docs', - 'docs-realm', + // 'docs-realm', 'docs', // 'docs-atlas-cli', // 'docs-ecosystem', From 304cc33bfd9b9024a9c23419535b86dfaa7bc0e9 Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Fri, 15 Mar 2024 15:23:17 -0400 Subject: [PATCH 121/254] DOP-4414 testing job prefix --- api/controllers/v2/github.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/api/controllers/v2/github.ts b/api/controllers/v2/github.ts index 6b4b60893..2ad67acb7 100644 --- a/api/controllers/v2/github.ts +++ b/api/controllers/v2/github.ts @@ -215,10 +215,10 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr ); } - const jobPrefix = repoInfo?.prefix ? repoInfo['prefix'][env] : ''; + const jobPrefix = 'testing'; + // repoInfo?.prefix ? repoInfo['prefix'][env] : ''; //add commit hash to jobPrefix here? - // const ammendedJobPrefix = body.after ? jobPrefix + body.after : jobPrefix; - // const prefix = ammendedJobPrefix; + // const prefix = jobPrefix + "/" + body.workflow_run.head_sha; const payload = await createPayload(repoName, true, jobPrefix, repoBranchesRepository, repoInfo, repoOwner); From 511d6d7dda0b2eeb8b575ed72c4af7b071541c08 Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Fri, 15 Mar 2024 15:37:54 -0400 Subject: [PATCH 122/254] DOP-4414 testing job prefix --- api/controllers/v2/github.ts | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/api/controllers/v2/github.ts b/api/controllers/v2/github.ts index 2ad67acb7..71ac5cce4 100644 --- a/api/controllers/v2/github.ts +++ b/api/controllers/v2/github.ts @@ -197,9 +197,7 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr const repoName = SMOKETEST_SITES[s]; const jobTitle = 'Smoke Test ' + repoName; let repoInfo, projectEntry, repoOwner; - if (repoName != 'docs-realm') { - return repoName; - } + try { repoInfo = await docsetsRepository.getRepo(repoName); projectEntry = await projectsRepository.getProjectEntry(repoInfo.project); From f50970796e93d289b70cddd0b8fbfb0def4130c0 Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Fri, 15 Mar 2024 16:11:50 -0400 Subject: [PATCH 123/254] DOP-4414 testing job prefix --- api/controllers/v2/github.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/controllers/v2/github.ts b/api/controllers/v2/github.ts index 71ac5cce4..e781151f8 100644 --- a/api/controllers/v2/github.ts +++ b/api/controllers/v2/github.ts @@ -369,7 +369,7 @@ export const TriggerBuild = async (event: APIGatewayEvent): Promise Date: Fri, 15 Mar 2024 16:25:53 -0400 Subject: [PATCH 124/254] DOP-4414 testing job prefix --- api/controllers/v2/github.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/api/controllers/v2/github.ts b/api/controllers/v2/github.ts index e781151f8..eaf394540 100644 --- a/api/controllers/v2/github.ts +++ b/api/controllers/v2/github.ts @@ -16,7 +16,7 @@ import { ReposBranchesDocsetsDocument } from '../../../modules/persistence/src/s import { MONOREPO_NAME } from '../../../src/monorepo/utils/monorepo-constants'; const SMOKETEST_SITES = [ - // 'docs-landing', + 'docs-landing', // 'cloud-docs', // 'docs-realm', 'docs', @@ -234,6 +234,7 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr consoleLogger.error('TriggerBuildError', err + repoName); } // deployable.push(job); + return names; } // try { From 2b2c10825ac88fd4be47e183d9eb10f8fb1d78ab Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Fri, 15 Mar 2024 17:11:46 -0400 Subject: [PATCH 125/254] DOP-4414 testing job prefix --- api/controllers/v2/github.ts | 21 ++++++++++++++------- src/commands/src/shared/next-gen-stage.ts | 9 ++++++++- 2 files changed, 22 insertions(+), 8 deletions(-) diff --git a/api/controllers/v2/github.ts b/api/controllers/v2/github.ts index eaf394540..c032993e8 100644 --- a/api/controllers/v2/github.ts +++ b/api/controllers/v2/github.ts @@ -52,6 +52,7 @@ async function createPayload( prefix: string, repoBranchesRepository: RepoBranchesRepository, repoInfo: ReposBranchesDocsetsDocument, + newHead: string, repoOwner?: string, githubEvent?: PushEvent, directory?: string @@ -64,7 +65,6 @@ async function createPayload( let action: string; let url: any; let isFork: boolean; - let newHead: any; if (isSmokeTestDeploy) { url = 'https://github.com/' + repoOwner + '/' + repoName; @@ -213,12 +213,19 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr ); } - const jobPrefix = 'testing'; - // repoInfo?.prefix ? repoInfo['prefix'][env] : ''; + const jobPrefix = repoInfo?.prefix ? repoInfo['prefix'][env] : ''; //add commit hash to jobPrefix here? - // const prefix = jobPrefix + "/" + body.workflow_run.head_sha; - - const payload = await createPayload(repoName, true, jobPrefix, repoBranchesRepository, repoInfo, repoOwner); + const newHead = body.workflow_run.head_sha; + + const payload = await createPayload( + repoName, + true, + jobPrefix, + repoBranchesRepository, + repoInfo, + newHead, + repoOwner + ); //add logic for getting master branch, latest stable branch const job = await prepGithubPushPayload(body, payload, jobTitle); @@ -230,7 +237,7 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr consoleLogger.info(job.title, `Created Job ${jobId}`); names = names + repoName; } catch (err) { - return err + repoName; + return false; consoleLogger.error('TriggerBuildError', err + repoName); } // deployable.push(job); diff --git a/src/commands/src/shared/next-gen-stage.ts b/src/commands/src/shared/next-gen-stage.ts index 9c12dc66a..2a31b5282 100644 --- a/src/commands/src/shared/next-gen-stage.ts +++ b/src/commands/src/shared/next-gen-stage.ts @@ -10,7 +10,7 @@ interface StageParams { } export async function nextGenStage({ job, bucket, url }: StageParams): Promise { - const { mutPrefix, branchName, project, newHead, patchId } = job.payload; + const { mutPrefix, branchName, project, newHead, patchId, action } = job.payload; if (!bucket) { console.log(`nextGenStage has failed. Variable for S3 bucket address was undefined.`); @@ -29,16 +29,23 @@ export async function nextGenStage({ job, bucket, url }: StageParams): Promise Date: Fri, 15 Mar 2024 20:49:34 -0400 Subject: [PATCH 126/254] DOP-4414 testing job prefix --- api/controllers/v2/github.ts | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) diff --git a/api/controllers/v2/github.ts b/api/controllers/v2/github.ts index c032993e8..72ef84a62 100644 --- a/api/controllers/v2/github.ts +++ b/api/controllers/v2/github.ts @@ -215,18 +215,10 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr const jobPrefix = repoInfo?.prefix ? repoInfo['prefix'][env] : ''; //add commit hash to jobPrefix here? - const newHead = body.workflow_run.head_sha; - - const payload = await createPayload( - repoName, - true, - jobPrefix, - repoBranchesRepository, - repoInfo, - newHead, - repoOwner - ); + //const newHead = body.workflow_run.head_sha; + const payload = await createPayload(repoName, true, jobPrefix, repoBranchesRepository, repoInfo, '', repoOwner); + names = names + repoName; //add logic for getting master branch, latest stable branch const job = await prepGithubPushPayload(body, payload, jobTitle); @@ -235,7 +227,6 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr const jobId = await jobRepository.insertJob(job, c.get('jobsQueueUrl')); jobRepository.notify(jobId, c.get('jobUpdatesQueueUrl'), JobStatus.inQueue, 0); consoleLogger.info(job.title, `Created Job ${jobId}`); - names = names + repoName; } catch (err) { return false; consoleLogger.error('TriggerBuildError', err + repoName); From f09d8e176b287f8a9dd38a5dc1c72df496938c1e Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Fri, 15 Mar 2024 21:58:51 -0400 Subject: [PATCH 127/254] DOP-4414 testing job prefix --- api/controllers/v2/github.ts | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/api/controllers/v2/github.ts b/api/controllers/v2/github.ts index 72ef84a62..460449e10 100644 --- a/api/controllers/v2/github.ts +++ b/api/controllers/v2/github.ts @@ -134,14 +134,14 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr } // validate credentials here - // if (!validateJsonWebhook(event, 'mongodbsmoketesting')) { - // const errMsg = "X-Hub-Signature incorrect. Github webhook token doesn't match"; - // return { - // statusCode: 401, - // headers: { 'Content-Type': 'text/plain' }, - // body: errMsg, - // }; - // } + if (!validateJsonWebhook(event, 'mongodbsmoketesting')) { + const errMsg = "X-Hub-Signature incorrect. Github webhook token doesn't match"; + return { + statusCode: 401, + headers: { 'Content-Type': 'text/plain' }, + body: errMsg, + }; + } let body: WorkflowRunCompletedEvent; try { From 4a37c8d83444ccbdf206c9019a7c076f113371c5 Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Sat, 16 Mar 2024 14:19:49 -0400 Subject: [PATCH 128/254] DOP-4414 testing job prefix --- api/controllers/v2/github.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/api/controllers/v2/github.ts b/api/controllers/v2/github.ts index 460449e10..2de45d10d 100644 --- a/api/controllers/v2/github.ts +++ b/api/controllers/v2/github.ts @@ -194,6 +194,7 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr async function createAndInsertJob() { let names = ''; for (const s in SMOKETEST_SITES) { + names += s; const repoName = SMOKETEST_SITES[s]; const jobTitle = 'Smoke Test ' + repoName; let repoInfo, projectEntry, repoOwner; @@ -218,7 +219,6 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr //const newHead = body.workflow_run.head_sha; const payload = await createPayload(repoName, true, jobPrefix, repoBranchesRepository, repoInfo, '', repoOwner); - names = names + repoName; //add logic for getting master branch, latest stable branch const job = await prepGithubPushPayload(body, payload, jobTitle); @@ -228,12 +228,12 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr jobRepository.notify(jobId, c.get('jobUpdatesQueueUrl'), JobStatus.inQueue, 0); consoleLogger.info(job.title, `Created Job ${jobId}`); } catch (err) { - return false; + return repoName; consoleLogger.error('TriggerBuildError', err + repoName); } // deployable.push(job); - return names; } + return names; // try { // await jobRepository.insertBulkJobs(deployable, c.get('jobsQueueUrl')); From 68bd960fa8622b1a64ff63a8c6c25c6da9c1f71b Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Sat, 16 Mar 2024 14:52:16 -0400 Subject: [PATCH 129/254] DOP-4414 testing job prefix --- api/controllers/v2/github.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/api/controllers/v2/github.ts b/api/controllers/v2/github.ts index 2de45d10d..72f1709be 100644 --- a/api/controllers/v2/github.ts +++ b/api/controllers/v2/github.ts @@ -194,8 +194,9 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr async function createAndInsertJob() { let names = ''; for (const s in SMOKETEST_SITES) { - names += s; + names = names + s; const repoName = SMOKETEST_SITES[s]; + if (repoName == 'docs') return true; const jobTitle = 'Smoke Test ' + repoName; let repoInfo, projectEntry, repoOwner; @@ -233,7 +234,6 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr } // deployable.push(job); } - return names; // try { // await jobRepository.insertBulkJobs(deployable, c.get('jobsQueueUrl')); From bb47d6043f7eb358048d64179ec08bd9dd098655 Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Sat, 16 Mar 2024 15:50:11 -0400 Subject: [PATCH 130/254] DOP-4414 testing job prefix --- api/controllers/v2/github.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/controllers/v2/github.ts b/api/controllers/v2/github.ts index 72f1709be..b8c4de4d7 100644 --- a/api/controllers/v2/github.ts +++ b/api/controllers/v2/github.ts @@ -196,7 +196,6 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr for (const s in SMOKETEST_SITES) { names = names + s; const repoName = SMOKETEST_SITES[s]; - if (repoName == 'docs') return true; const jobTitle = 'Smoke Test ' + repoName; let repoInfo, projectEntry, repoOwner; @@ -204,6 +203,7 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr repoInfo = await docsetsRepository.getRepo(repoName); projectEntry = await projectsRepository.getProjectEntry(repoInfo.project); repoOwner = projectEntry.github.organization; + if (repoName == 'docs') return true; } catch { return ( 'repoInfo, projectEntry, or repoOwner not found. Repo info: ' + From 48722fe6e2bcce0c22c8d35535285afb1093f4b3 Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Sat, 16 Mar 2024 16:09:03 -0400 Subject: [PATCH 131/254] DOP-4414 testing job prefix --- api/controllers/v2/github.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/controllers/v2/github.ts b/api/controllers/v2/github.ts index b8c4de4d7..6e8cb5364 100644 --- a/api/controllers/v2/github.ts +++ b/api/controllers/v2/github.ts @@ -201,9 +201,9 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr try { repoInfo = await docsetsRepository.getRepo(repoName); + if (repoName == 'docs') return true; projectEntry = await projectsRepository.getProjectEntry(repoInfo.project); repoOwner = projectEntry.github.organization; - if (repoName == 'docs') return true; } catch { return ( 'repoInfo, projectEntry, or repoOwner not found. Repo info: ' + From 8edf3481d977672956a10fbcead9012ac6e609be Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Sat, 16 Mar 2024 16:29:17 -0400 Subject: [PATCH 132/254] DOP-4414 testing job prefix --- api/controllers/v2/github.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/controllers/v2/github.ts b/api/controllers/v2/github.ts index 6e8cb5364..cfa620171 100644 --- a/api/controllers/v2/github.ts +++ b/api/controllers/v2/github.ts @@ -198,10 +198,10 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr const repoName = SMOKETEST_SITES[s]; const jobTitle = 'Smoke Test ' + repoName; let repoInfo, projectEntry, repoOwner; + if (repoName == 'docs') return true; try { repoInfo = await docsetsRepository.getRepo(repoName); - if (repoName == 'docs') return true; projectEntry = await projectsRepository.getProjectEntry(repoInfo.project); repoOwner = projectEntry.github.organization; } catch { From 48e8e72472348066497db7b578b6f061cf82a708 Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Sat, 16 Mar 2024 17:05:00 -0400 Subject: [PATCH 133/254] DOP-4414 testing job prefix --- api/controllers/v2/github.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/controllers/v2/github.ts b/api/controllers/v2/github.ts index cfa620171..53a8741e1 100644 --- a/api/controllers/v2/github.ts +++ b/api/controllers/v2/github.ts @@ -198,7 +198,7 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr const repoName = SMOKETEST_SITES[s]; const jobTitle = 'Smoke Test ' + repoName; let repoInfo, projectEntry, repoOwner; - if (repoName == 'docs') return true; + if (repoName == 'docs') return docsetsRepository.getRepo(repoName); try { repoInfo = await docsetsRepository.getRepo(repoName); From 623746387f317b4f4473d67d4350077e4ca11ddf Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Sun, 17 Mar 2024 14:23:35 -0400 Subject: [PATCH 134/254] DOP-4414 testing job prefix --- api/controllers/v2/github.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/controllers/v2/github.ts b/api/controllers/v2/github.ts index 53a8741e1..7935b3999 100644 --- a/api/controllers/v2/github.ts +++ b/api/controllers/v2/github.ts @@ -121,7 +121,6 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr //add docs_metadata to config const projectsRepository = new ProjectsRepository(client.db('docs_metadata'), c, consoleLogger); const repoBranchesRepository = new RepoBranchesRepository(db, c, consoleLogger); - const docsetsRepository = new DocsetsRepository(db, c, consoleLogger); if (!event.body) { const err = 'Trigger build does not have a body in event payload'; @@ -198,6 +197,7 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr const repoName = SMOKETEST_SITES[s]; const jobTitle = 'Smoke Test ' + repoName; let repoInfo, projectEntry, repoOwner; + const docsetsRepository = new DocsetsRepository(db, c, consoleLogger); if (repoName == 'docs') return docsetsRepository.getRepo(repoName); try { From a2f7f1f1e8d115298e78bd628325a31744eb228e Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Sun, 17 Mar 2024 23:31:53 -0400 Subject: [PATCH 135/254] DOP-4414 testing --- api/controllers/v2/github.ts | 10 +++++----- src/repositories/docsetsRepository.ts | 1 + 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/api/controllers/v2/github.ts b/api/controllers/v2/github.ts index 7935b3999..e32d0405f 100644 --- a/api/controllers/v2/github.ts +++ b/api/controllers/v2/github.ts @@ -16,13 +16,13 @@ import { ReposBranchesDocsetsDocument } from '../../../modules/persistence/src/s import { MONOREPO_NAME } from '../../../src/monorepo/utils/monorepo-constants'; const SMOKETEST_SITES = [ - 'docs-landing', + // 'docs-landing', // 'cloud-docs', // 'docs-realm', 'docs', // 'docs-atlas-cli', // 'docs-ecosystem', - // 'docs-node', + 'docs-node', // 'docs-app-services', ]; @@ -192,9 +192,9 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr async function createAndInsertJob() { let names = ''; - for (const s in SMOKETEST_SITES) { - names = names + s; - const repoName = SMOKETEST_SITES[s]; + for (let i = 0; i < 1; i++) { + names = names + i; + const repoName = SMOKETEST_SITES[i]; const jobTitle = 'Smoke Test ' + repoName; let repoInfo, projectEntry, repoOwner; const docsetsRepository = new DocsetsRepository(db, c, consoleLogger); diff --git a/src/repositories/docsetsRepository.ts b/src/repositories/docsetsRepository.ts index 9a04c1740..c483bfe89 100644 --- a/src/repositories/docsetsRepository.ts +++ b/src/repositories/docsetsRepository.ts @@ -78,6 +78,7 @@ export class DocsetsRepository extends BaseRepository { directory ? `with directory: /${directory}` : '' }`; this._logger.info(this._repoName, msg); + return this._repoName; } return res?.[0]; } From 489efa14e6c9ddedf1a087dc5cd19d3ce4cd908f Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Mon, 18 Mar 2024 00:24:00 -0400 Subject: [PATCH 136/254] DOP-4414 testing --- api/controllers/v2/github.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/api/controllers/v2/github.ts b/api/controllers/v2/github.ts index e32d0405f..f65aca756 100644 --- a/api/controllers/v2/github.ts +++ b/api/controllers/v2/github.ts @@ -192,13 +192,13 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr async function createAndInsertJob() { let names = ''; - for (let i = 0; i < 1; i++) { + for (let i = 0; i < 2; i++) { names = names + i; const repoName = SMOKETEST_SITES[i]; const jobTitle = 'Smoke Test ' + repoName; let repoInfo, projectEntry, repoOwner; const docsetsRepository = new DocsetsRepository(db, c, consoleLogger); - if (repoName == 'docs') return docsetsRepository.getRepo(repoName); + // if (repoName == 'docs') return docsetsRepository.getRepo(repoName); try { repoInfo = await docsetsRepository.getRepo(repoName); @@ -234,6 +234,7 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr } // deployable.push(job); } + return names; // try { // await jobRepository.insertBulkJobs(deployable, c.get('jobsQueueUrl')); From 7a9b1ae0f3f22512bc2f793a04b8c7a9494eb28b Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Mon, 18 Mar 2024 00:38:34 -0400 Subject: [PATCH 137/254] DOP-4414 testing --- api/controllers/v2/github.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/controllers/v2/github.ts b/api/controllers/v2/github.ts index f65aca756..77adad5c3 100644 --- a/api/controllers/v2/github.ts +++ b/api/controllers/v2/github.ts @@ -22,7 +22,7 @@ const SMOKETEST_SITES = [ 'docs', // 'docs-atlas-cli', // 'docs-ecosystem', - 'docs-node', + // 'docs-node', // 'docs-app-services', ]; From cc362929a3d6a54206d3ce17760bfee76add5771 Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Mon, 18 Mar 2024 00:52:08 -0400 Subject: [PATCH 138/254] DOP-4414 testing --- api/controllers/v2/github.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/api/controllers/v2/github.ts b/api/controllers/v2/github.ts index 77adad5c3..83576d57e 100644 --- a/api/controllers/v2/github.ts +++ b/api/controllers/v2/github.ts @@ -19,8 +19,8 @@ const SMOKETEST_SITES = [ // 'docs-landing', // 'cloud-docs', // 'docs-realm', - 'docs', - // 'docs-atlas-cli', + // 'docs', + 'docs-atlas-cli', // 'docs-ecosystem', // 'docs-node', // 'docs-app-services', From fd61bfcd04b50ec47dabc3a82d935691ffc78778 Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Mon, 18 Mar 2024 01:05:24 -0400 Subject: [PATCH 139/254] DOP-4414 testing --- api/controllers/v2/github.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/api/controllers/v2/github.ts b/api/controllers/v2/github.ts index 83576d57e..0c8ef849d 100644 --- a/api/controllers/v2/github.ts +++ b/api/controllers/v2/github.ts @@ -16,11 +16,11 @@ import { ReposBranchesDocsetsDocument } from '../../../modules/persistence/src/s import { MONOREPO_NAME } from '../../../src/monorepo/utils/monorepo-constants'; const SMOKETEST_SITES = [ - // 'docs-landing', + 'docs-landing', // 'cloud-docs', // 'docs-realm', // 'docs', - 'docs-atlas-cli', + // 'docs-atlas-cli', // 'docs-ecosystem', // 'docs-node', // 'docs-app-services', From bc14ddf492e43ae41eaca41a035606d71b4ab3d3 Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Mon, 18 Mar 2024 01:06:31 -0400 Subject: [PATCH 140/254] DOP-4414 testing --- api/controllers/v2/github.ts | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/api/controllers/v2/github.ts b/api/controllers/v2/github.ts index 0c8ef849d..764b35d9b 100644 --- a/api/controllers/v2/github.ts +++ b/api/controllers/v2/github.ts @@ -16,13 +16,12 @@ import { ReposBranchesDocsetsDocument } from '../../../modules/persistence/src/s import { MONOREPO_NAME } from '../../../src/monorepo/utils/monorepo-constants'; const SMOKETEST_SITES = [ - 'docs-landing', + // 'docs-landing', // 'cloud-docs', // 'docs-realm', // 'docs', // 'docs-atlas-cli', - // 'docs-ecosystem', - // 'docs-node', + 'docs-node', // 'docs-app-services', ]; From 3bc7982e1147b819e1785961ee4dab97bad51a60 Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Mon, 18 Mar 2024 01:07:03 -0400 Subject: [PATCH 141/254] DOP-4414 testing --- api/controllers/v2/github.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/api/controllers/v2/github.ts b/api/controllers/v2/github.ts index 764b35d9b..281cf11a9 100644 --- a/api/controllers/v2/github.ts +++ b/api/controllers/v2/github.ts @@ -19,9 +19,9 @@ const SMOKETEST_SITES = [ // 'docs-landing', // 'cloud-docs', // 'docs-realm', - // 'docs', + 'docs', // 'docs-atlas-cli', - 'docs-node', + // 'docs-node', // 'docs-app-services', ]; From 484d6a1c8dedadab41da275fa5880bea041e3436 Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Mon, 18 Mar 2024 01:24:01 -0400 Subject: [PATCH 142/254] DOP-4414 testing --- api/controllers/v2/github.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/controllers/v2/github.ts b/api/controllers/v2/github.ts index 281cf11a9..1feba9982 100644 --- a/api/controllers/v2/github.ts +++ b/api/controllers/v2/github.ts @@ -20,7 +20,7 @@ const SMOKETEST_SITES = [ // 'cloud-docs', // 'docs-realm', 'docs', - // 'docs-atlas-cli', + 'docs-atlas-cli', // 'docs-node', // 'docs-app-services', ]; From d5aa217019d31a2501178218f4f737ac83fa1e02 Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Mon, 18 Mar 2024 01:39:38 -0400 Subject: [PATCH 143/254] DOP-4414 testing --- api/controllers/v2/github.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/controllers/v2/github.ts b/api/controllers/v2/github.ts index 1feba9982..629786332 100644 --- a/api/controllers/v2/github.ts +++ b/api/controllers/v2/github.ts @@ -228,7 +228,7 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr jobRepository.notify(jobId, c.get('jobUpdatesQueueUrl'), JobStatus.inQueue, 0); consoleLogger.info(job.title, `Created Job ${jobId}`); } catch (err) { - return repoName; + return err + repoName; consoleLogger.error('TriggerBuildError', err + repoName); } // deployable.push(job); From 69a0086f959a7c80fb258124b8550e0309f59fc8 Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Mon, 18 Mar 2024 10:08:41 -0400 Subject: [PATCH 144/254] DOP-4414 testing --- api/controllers/v2/github.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/controllers/v2/github.ts b/api/controllers/v2/github.ts index 629786332..945219813 100644 --- a/api/controllers/v2/github.ts +++ b/api/controllers/v2/github.ts @@ -225,7 +225,7 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr try { consoleLogger.info(job.title, 'Creating Job'); const jobId = await jobRepository.insertJob(job, c.get('jobsQueueUrl')); - jobRepository.notify(jobId, c.get('jobUpdatesQueueUrl'), JobStatus.inQueue, 0); + // jobRepository.notify(jobId, c.get('jobUpdatesQueueUrl'), JobStatus.inQueue, 0); consoleLogger.info(job.title, `Created Job ${jobId}`); } catch (err) { return err + repoName; From 25f19b88e84dc390a1869fc302eed6c9867be7a6 Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Mon, 18 Mar 2024 10:22:19 -0400 Subject: [PATCH 145/254] DOP-4414 testing --- api/controllers/v2/github.ts | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/api/controllers/v2/github.ts b/api/controllers/v2/github.ts index 945219813..fc43e0d3b 100644 --- a/api/controllers/v2/github.ts +++ b/api/controllers/v2/github.ts @@ -16,13 +16,13 @@ import { ReposBranchesDocsetsDocument } from '../../../modules/persistence/src/s import { MONOREPO_NAME } from '../../../src/monorepo/utils/monorepo-constants'; const SMOKETEST_SITES = [ - // 'docs-landing', - // 'cloud-docs', - // 'docs-realm', + 'docs-landing', + 'cloud-docs', + 'docs-realm', 'docs', 'docs-atlas-cli', - // 'docs-node', - // 'docs-app-services', + 'docs-node', + 'docs-app-services', ]; async function prepGithubPushPayload( @@ -191,9 +191,9 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr async function createAndInsertJob() { let names = ''; - for (let i = 0; i < 2; i++) { - names = names + i; - const repoName = SMOKETEST_SITES[i]; + for (const s in SMOKETEST_SITES) { + names = names + s; + const repoName = SMOKETEST_SITES[s]; const jobTitle = 'Smoke Test ' + repoName; let repoInfo, projectEntry, repoOwner; const docsetsRepository = new DocsetsRepository(db, c, consoleLogger); @@ -237,7 +237,6 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr // try { // await jobRepository.insertBulkJobs(deployable, c.get('jobsQueueUrl')); - // // notify the jobUpdatesQueue // await Promise.all( // deployable.map(async ({ jobId }) => { From 50a685ab7613d85c8fd1aec36533b12ea441daf2 Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Mon, 18 Mar 2024 11:26:39 -0400 Subject: [PATCH 146/254] DOP-4414 testing jobupdates queue --- api/controllers/v2/github.ts | 6 +++--- cdk-infra/lib/constructs/api/webhook-api-construct.ts | 1 + src/repositories/docsetsRepository.ts | 1 - 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/api/controllers/v2/github.ts b/api/controllers/v2/github.ts index fc43e0d3b..2156c6234 100644 --- a/api/controllers/v2/github.ts +++ b/api/controllers/v2/github.ts @@ -120,6 +120,7 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr //add docs_metadata to config const projectsRepository = new ProjectsRepository(client.db('docs_metadata'), c, consoleLogger); const repoBranchesRepository = new RepoBranchesRepository(db, c, consoleLogger); + const docsetsRepository = new DocsetsRepository(db, c, consoleLogger); if (!event.body) { const err = 'Trigger build does not have a body in event payload'; @@ -196,8 +197,6 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr const repoName = SMOKETEST_SITES[s]; const jobTitle = 'Smoke Test ' + repoName; let repoInfo, projectEntry, repoOwner; - const docsetsRepository = new DocsetsRepository(db, c, consoleLogger); - // if (repoName == 'docs') return docsetsRepository.getRepo(repoName); try { repoInfo = await docsetsRepository.getRepo(repoName); @@ -215,7 +214,8 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr } const jobPrefix = repoInfo?.prefix ? repoInfo['prefix'][env] : ''; - //add commit hash to jobPrefix here? + + //add commit hash here //const newHead = body.workflow_run.head_sha; const payload = await createPayload(repoName, true, jobPrefix, repoBranchesRepository, repoInfo, '', repoOwner); diff --git a/cdk-infra/lib/constructs/api/webhook-api-construct.ts b/cdk-infra/lib/constructs/api/webhook-api-construct.ts index ee9469a6c..10862f6b8 100644 --- a/cdk-infra/lib/constructs/api/webhook-api-construct.ts +++ b/cdk-infra/lib/constructs/api/webhook-api-construct.ts @@ -196,6 +196,7 @@ export class WebhookApiConstruct extends Construct { // grant permission for lambdas to enqueue messages to the job updates queue jobUpdatesQueue.grantSendMessages(slackTriggerLambda); + jobUpdatesQueue.grantSendMessages(githubSmokeTestBuildLambda); jobUpdatesQueue.grantSendMessages(githubTriggerLambda); jobUpdatesQueue.grantSendMessages(triggerLocalBuildLambda); diff --git a/src/repositories/docsetsRepository.ts b/src/repositories/docsetsRepository.ts index c483bfe89..9a04c1740 100644 --- a/src/repositories/docsetsRepository.ts +++ b/src/repositories/docsetsRepository.ts @@ -78,7 +78,6 @@ export class DocsetsRepository extends BaseRepository { directory ? `with directory: /${directory}` : '' }`; this._logger.info(this._repoName, msg); - return this._repoName; } return res?.[0]; } From a0088eb02e94ee02dc94368a46080156629ac324 Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Mon, 18 Mar 2024 13:53:01 -0400 Subject: [PATCH 147/254] DOP-4414 testing refactor --- api/controllers/v2/github.ts | 71 ++++++++++++++++++++++-------------- 1 file changed, 43 insertions(+), 28 deletions(-) diff --git a/api/controllers/v2/github.ts b/api/controllers/v2/github.ts index 2156c6234..86daae71b 100644 --- a/api/controllers/v2/github.ts +++ b/api/controllers/v2/github.ts @@ -45,38 +45,49 @@ async function prepGithubPushPayload( }; } -async function createPayload( - repoName: string, +interface CreatePayloadProps { + repoName: string; + githubEvent?: PushEvent; + isSmokeTestDeploy?: boolean; + prefix: string; + repoBranchesRepository: RepoBranchesRepository; + repoInfo: ReposBranchesDocsetsDocument; + newHead?: string; + repoOwner?: string; + directory?: string; +} + +async function createPayload({ + repoName, isSmokeTestDeploy = false, - prefix: string, - repoBranchesRepository: RepoBranchesRepository, - repoInfo: ReposBranchesDocsetsDocument, - newHead: string, - repoOwner?: string, - githubEvent?: PushEvent, - directory?: string -) { - const jobType = 'githubPush'; + prefix, + repoBranchesRepository, + repoInfo, + newHead, + repoOwner, + githubEvent, + directory, +}: CreatePayloadProps) { const source = 'github'; const project = repoInfo?.project ?? repoName; let branchName: string; + let jobType: string; let action: string; - let url: any; - let isFork: boolean; + let url: string; if (isSmokeTestDeploy) { url = 'https://github.com/' + repoOwner + '/' + repoName; - branchName = 'master'; - isFork = false; action = 'automatedTest'; + jobType = 'productionDeploy'; + branchName = 'master'; } else { if (!githubEvent) { return false; } action = 'push'; + jobType = 'githubPush'; branchName = githubEvent.ref.split('/')[2]; - isFork = githubEvent?.repository.fork; url = githubEvent?.repository.clone_url; newHead = githubEvent?.after; repoOwner = githubEvent.repository.owner.login; @@ -95,7 +106,6 @@ async function createPayload( project, prefix, urlSlug, - isFork, url, newHead, directory, @@ -103,13 +113,7 @@ async function createPayload( } /** - * 1st, we want to define a list of the sites we want to build as a new collection. - * validate credentials - * Next, we want to create each job - * - should I create a new prepGithubPayload method, a new githubEvent interface, or amend the existing one to be able to pass in a different title, branchName and it's own githash - * - i could also alter body attributes, although I don't know it that's a good idea - * - could also create a createpayload function - * create and insert the jobs using bulk insert + * */ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Promise => { const client = new mongodb.MongoClient(c.get('dbUrl')); @@ -117,7 +121,6 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr const db = client.db(c.get('dbName')); const consoleLogger = new ConsoleLogger(); const jobRepository = new JobRepository(db, c, consoleLogger); - //add docs_metadata to config const projectsRepository = new ProjectsRepository(client.db('docs_metadata'), c, consoleLogger); const repoBranchesRepository = new RepoBranchesRepository(db, c, consoleLogger); const docsetsRepository = new DocsetsRepository(db, c, consoleLogger); @@ -218,7 +221,14 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr //add commit hash here //const newHead = body.workflow_run.head_sha; - const payload = await createPayload(repoName, true, jobPrefix, repoBranchesRepository, repoInfo, '', repoOwner); + const payload = await createPayload({ + repoName, + isSmokeTestDeploy: true, + prefix: jobPrefix, + repoBranchesRepository, + repoInfo, + repoOwner, + }); //add logic for getting master branch, latest stable branch const job = await prepGithubPushPayload(body, payload, jobTitle); @@ -277,7 +287,6 @@ export const TriggerBuild = async (event: APIGatewayEvent): Promise Date: Mon, 18 Mar 2024 14:53:07 -0400 Subject: [PATCH 148/254] DOP-4414 testing refactor --- src/entities/job.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/entities/job.ts b/src/entities/job.ts index 00611f1b4..fcbbd5233 100644 --- a/src/entities/job.ts +++ b/src/entities/job.ts @@ -64,7 +64,7 @@ export type EnhancedPayload = { action: string; repoName: string; branchName: string; - isFork: boolean; + isFork?: boolean; isXlarge?: boolean | null; repoOwner: string; url: string; From 41787bfdc4e6f409c0582b2d62f7e64b8da34130 Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Mon, 18 Mar 2024 14:58:37 -0400 Subject: [PATCH 149/254] DOP-4414 testing jobupdatesqueue --- api/controllers/v2/github.ts | 48 +++++++++--------------------------- 1 file changed, 12 insertions(+), 36 deletions(-) diff --git a/api/controllers/v2/github.ts b/api/controllers/v2/github.ts index 86daae71b..c08fc47cb 100644 --- a/api/controllers/v2/github.ts +++ b/api/controllers/v2/github.ts @@ -7,7 +7,7 @@ import { JobRepository } from '../../../src/repositories/jobRepository'; import { ConsoleLogger } from '../../../src/services/logger'; import { RepoBranchesRepository } from '../../../src/repositories/repoBranchesRepository'; import { ProjectsRepository } from '../../../src/repositories/projectsRepository'; -import { EnhancedJob, JobStatus } from '../../../src/entities/job'; +import { EnhancedJob, EnhancedPayload, JobStatus } from '../../../src/entities/job'; import { markBuildArtifactsForDeletion, validateJsonWebhook } from '../../handlers/github'; import { DocsetsRepository } from '../../../src/repositories/docsetsRepository'; import { getMonorepoPaths } from '../../../src/monorepo'; @@ -25,9 +25,10 @@ const SMOKETEST_SITES = [ 'docs-app-services', ]; +//EnhancedPayload and EnhancedJob are used here for both githubPush (feature branch) events as well as productionDeploy(smoke test deploy) events for typing purposes async function prepGithubPushPayload( githubEvent: PushEvent | WorkflowRunCompletedEvent, - payload: any, + payload: EnhancedPayload, title: string ): Promise> { return { @@ -47,7 +48,6 @@ async function prepGithubPushPayload( interface CreatePayloadProps { repoName: string; - githubEvent?: PushEvent; isSmokeTestDeploy?: boolean; prefix: string; repoBranchesRepository: RepoBranchesRepository; @@ -55,6 +55,7 @@ interface CreatePayloadProps { newHead?: string; repoOwner?: string; directory?: string; + githubEvent?: PushEvent; } async function createPayload({ @@ -64,10 +65,10 @@ async function createPayload({ repoBranchesRepository, repoInfo, newHead, - repoOwner, + repoOwner = '', githubEvent, directory, -}: CreatePayloadProps) { +}: CreatePayloadProps): Promise { const source = 'github'; const project = repoInfo?.project ?? repoName; @@ -83,7 +84,7 @@ async function createPayload({ branchName = 'master'; } else { if (!githubEvent) { - return false; + throw 'Non SmokeTest Deploy jobs must have a github Event'; } action = 'push'; jobType = 'githubPush'; @@ -194,9 +195,7 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr const env = 'dotcomstg'; async function createAndInsertJob() { - let names = ''; for (const s in SMOKETEST_SITES) { - names = names + s; const repoName = SMOKETEST_SITES[s]; const jobTitle = 'Smoke Test ' + repoName; let repoInfo, projectEntry, repoOwner; @@ -206,20 +205,12 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr projectEntry = await projectsRepository.getProjectEntry(repoInfo.project); repoOwner = projectEntry.github.organization; } catch { - return ( - 'repoInfo, projectEntry, or repoOwner not found. Repo info: ' + - repoInfo.project + - 'projectEntry ' + - projectEntry + - ' repoOwner ' + - repoOwner - ); + return 'repoInfo, projectEntry, or repoOwner not found for docs site ' + repoName; } const jobPrefix = repoInfo?.prefix ? repoInfo['prefix'][env] : ''; - //add commit hash here - //const newHead = body.workflow_run.head_sha; + //add commit hash here: const newHead = body.workflow_run.head_sha; const payload = await createPayload({ repoName, @@ -235,28 +226,13 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr try { consoleLogger.info(job.title, 'Creating Job'); const jobId = await jobRepository.insertJob(job, c.get('jobsQueueUrl')); - // jobRepository.notify(jobId, c.get('jobUpdatesQueueUrl'), JobStatus.inQueue, 0); + jobRepository.notify(jobId, c.get('jobUpdatesQueueUrl'), JobStatus.inQueue, 0); consoleLogger.info(job.title, `Created Job ${jobId}`); } catch (err) { - return err + repoName; consoleLogger.error('TriggerBuildError', err + repoName); + return err; } - // deployable.push(job); } - return names; - - // try { - // await jobRepository.insertBulkJobs(deployable, c.get('jobsQueueUrl')); - // // notify the jobUpdatesQueue - // await Promise.all( - // deployable.map(async ({ jobId }) => { - // await jobRepository.notify(jobId, c.get('jobUpdatesQueueUrl'), JobStatus.inQueue, 0); - // consoleLogger.info(jobId, `Created Job ${jobId}`); - // }) - // ); - // } catch (err) { - // consoleLogger.error('TriggerBuildError', err+ ); - // } } let returnVal; @@ -329,7 +305,7 @@ export const TriggerBuild = async (event: APIGatewayEvent): Promise Date: Mon, 18 Mar 2024 15:18:01 -0400 Subject: [PATCH 150/254] DOP-4414 cleaning up --- api/controllers/v2/github.ts | 7 +++---- api/controllers/v2/slack.ts | 13 +++---------- src/commands/src/shared/next-gen-stage.ts | 7 +------ src/job/jobHandler.ts | 1 - src/repositories/projectsRepository.ts | 2 +- 5 files changed, 8 insertions(+), 22 deletions(-) diff --git a/api/controllers/v2/github.ts b/api/controllers/v2/github.ts index c08fc47cb..20f2bfadc 100644 --- a/api/controllers/v2/github.ts +++ b/api/controllers/v2/github.ts @@ -235,20 +235,19 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr } } - let returnVal; try { - returnVal = await createAndInsertJob(); + await createAndInsertJob(); } catch (err) { return { statusCode: 500, headers: { 'Content-Type': 'text/plain' }, - body: returnVal + err, + body: err, }; } return { statusCode: 202, headers: { 'Content-Type': 'text/plain' }, - body: 'Jobs Queued ' + returnVal, + body: 'Smoke Test Jobs Queued ', }; }; diff --git a/api/controllers/v2/slack.ts b/api/controllers/v2/slack.ts index e8e6ce8b0..b6d76f74a 100644 --- a/api/controllers/v2/slack.ts +++ b/api/controllers/v2/slack.ts @@ -6,7 +6,7 @@ import { ConsoleLogger, ILogger } from '../../../src/services/logger'; import { SlackConnector } from '../../../src/services/slack'; import { JobRepository } from '../../../src/repositories/jobRepository'; import { APIGatewayEvent, APIGatewayProxyResult } from 'aws-lambda'; -import { JobStatus } from '../../../src/entities/job'; +import { EnhancedPayload, JobStatus } from '../../../src/entities/job'; import { buildEntitledBranchList, getQSString, @@ -81,7 +81,6 @@ const deployHelper = (deployable, payload, jobTitle, jobUserName, jobUserEmail) // For every repo/branch selected to be deployed, return an array of jobs with the payload data // needed for a successful build. export const getDeployableJobs = async ( - consoleLogger, values, entitlement, repoBranchesRepository: RepoBranchesRepository, @@ -212,13 +211,7 @@ export const DeployRepo = async (event: APIGatewayEvent): Promise 0) { await deployRepo(deployable, consoleLogger, jobRepository, c.get('jobsQueueUrl')); } @@ -263,7 +256,7 @@ function createPayload( }; } -function createJob(payload: any, jobTitle: string, jobUserName: string, jobUserEmail: string) { +function createJob(payload: EnhancedPayload, jobTitle: string, jobUserName: string, jobUserEmail: string) { return { title: jobTitle, user: jobUserName, diff --git a/src/commands/src/shared/next-gen-stage.ts b/src/commands/src/shared/next-gen-stage.ts index 2a31b5282..a2406118b 100644 --- a/src/commands/src/shared/next-gen-stage.ts +++ b/src/commands/src/shared/next-gen-stage.ts @@ -10,7 +10,7 @@ interface StageParams { } export async function nextGenStage({ job, bucket, url }: StageParams): Promise { - const { mutPrefix, branchName, project, newHead, patchId, action } = job.payload; + const { mutPrefix, branchName, project, newHead, patchId } = job.payload; if (!bucket) { console.log(`nextGenStage has failed. Variable for S3 bucket address was undefined.`); @@ -29,10 +29,7 @@ export async function nextGenStage({ job, bucket, url }: StageParams): Promise Date: Mon, 18 Mar 2024 16:19:03 -0400 Subject: [PATCH 151/254] DOP-4414 webhook update and cleaning --- api/controllers/v2/github.ts | 32 ++++++++++------------- src/commands/src/shared/next-gen-stage.ts | 2 -- 2 files changed, 14 insertions(+), 20 deletions(-) diff --git a/api/controllers/v2/github.ts b/api/controllers/v2/github.ts index 20f2bfadc..9f74d7107 100644 --- a/api/controllers/v2/github.ts +++ b/api/controllers/v2/github.ts @@ -113,9 +113,6 @@ async function createPayload({ }; } -/** - * - */ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Promise => { const client = new mongodb.MongoClient(c.get('dbUrl')); await client.connect(); @@ -136,8 +133,7 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr }; } - // validate credentials here - if (!validateJsonWebhook(event, 'mongodbsmoketesting')) { + if (!validateJsonWebhook(event, c.get('githubSecret'))) { const errMsg = "X-Hub-Signature incorrect. Github webhook token doesn't match"; return { statusCode: 401, @@ -178,18 +174,18 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr ' completed successfully but only Deploy Staging ECS workflow completion will trigger smoke test site deployments', }; - //if the build was not building master, no need for smoke test sites - // if (body.workflow_run.head_branch != 'main' || body.repository.fork) { - // console.log('Build was not on master branch in main repo, sites will not deploy as no smoke tests are needed'); - // return { - // statusCode: 202, - // headers: { 'Content-Type': 'text/plain' }, - // body: - // 'Build on branch' + - // body.workflow_run.head_branch + - // ' will not trigger site deployments as it was not on master branch in upstream repo', - // }; - // } + // if the build was not building master, no need for smoke test sites + if (body.workflow_run.head_branch != 'main' || body.repository.fork) { + console.log('Build was not on master branch in main repo, sites will not deploy as no smoke tests are needed'); + return { + statusCode: 202, + headers: { 'Content-Type': 'text/plain' }, + body: + 'Build on branch' + + body.workflow_run.head_branch + + ' will not trigger site deployments as it was not on master branch in upstream repo', + }; + } //automated test builds will always deploy in dotcomstg const env = 'dotcomstg'; @@ -357,7 +353,7 @@ export const TriggerBuild = async (event: APIGatewayEvent): Promise Date: Mon, 18 Mar 2024 16:33:12 -0400 Subject: [PATCH 152/254] DOP-4414 secret --- api/controllers/v2/github.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/controllers/v2/github.ts b/api/controllers/v2/github.ts index 9f74d7107..f3d3dca3a 100644 --- a/api/controllers/v2/github.ts +++ b/api/controllers/v2/github.ts @@ -265,7 +265,7 @@ export const TriggerBuild = async (event: APIGatewayEvent): Promise('githubSecret'))) { + if (!validateJsonWebhook(event, process.env.GITHUB_SECRET)) { const errMsg = "X-Hub-Signature incorrect. Github webhook token doesn't match"; return { statusCode: 401, From 635025b5a94546016a2d93eed128e08abf3d86bd Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Mon, 18 Mar 2024 17:03:42 -0400 Subject: [PATCH 153/254] DOP-4414 checking main --- api/controllers/v2/github.ts | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/api/controllers/v2/github.ts b/api/controllers/v2/github.ts index f3d3dca3a..2328acf58 100644 --- a/api/controllers/v2/github.ts +++ b/api/controllers/v2/github.ts @@ -174,6 +174,14 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr ' completed successfully but only Deploy Staging ECS workflow completion will trigger smoke test site deployments', }; + return { + statusCode: 202, + headers: { 'Content-Type': 'text/plain' }, + body: + 'Build on branch' + + body.workflow_run.head_branch + + ' will not trigger site deployments as it was not on master branch in upstream repo', + }; // if the build was not building master, no need for smoke test sites if (body.workflow_run.head_branch != 'main' || body.repository.fork) { console.log('Build was not on master branch in main repo, sites will not deploy as no smoke tests are needed'); From 98c4a156550f73190b3b8161fc7881fea58a048f Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Mon, 18 Mar 2024 17:10:41 -0400 Subject: [PATCH 154/254] DOP-4414 checking main --- api/controllers/v2/github.ts | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/api/controllers/v2/github.ts b/api/controllers/v2/github.ts index 2328acf58..a5c2b855d 100644 --- a/api/controllers/v2/github.ts +++ b/api/controllers/v2/github.ts @@ -174,14 +174,6 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr ' completed successfully but only Deploy Staging ECS workflow completion will trigger smoke test site deployments', }; - return { - statusCode: 202, - headers: { 'Content-Type': 'text/plain' }, - body: - 'Build on branch' + - body.workflow_run.head_branch + - ' will not trigger site deployments as it was not on master branch in upstream repo', - }; // if the build was not building master, no need for smoke test sites if (body.workflow_run.head_branch != 'main' || body.repository.fork) { console.log('Build was not on master branch in main repo, sites will not deploy as no smoke tests are needed'); @@ -189,7 +181,7 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr statusCode: 202, headers: { 'Content-Type': 'text/plain' }, body: - 'Build on branch' + + 'Build on branch ' + body.workflow_run.head_branch + ' will not trigger site deployments as it was not on master branch in upstream repo', }; From aee5cd49ab37e7404585cdc83b3f48df4be7c638 Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Tue, 19 Mar 2024 10:44:31 -0400 Subject: [PATCH 155/254] DOP-4414 types and env vars --- api/controllers/v2/github.ts | 23 +++++++++++++------ .../lib/constructs/worker/worker-construct.ts | 2 +- src/repositories/baseRepository.ts | 6 ++++- src/repositories/jobRepository.ts | 2 +- src/repositories/projectsRepository.ts | 4 ++-- 5 files changed, 25 insertions(+), 12 deletions(-) diff --git a/api/controllers/v2/github.ts b/api/controllers/v2/github.ts index a5c2b855d..0f92f7eea 100644 --- a/api/controllers/v2/github.ts +++ b/api/controllers/v2/github.ts @@ -41,7 +41,7 @@ async function prepGithubPushPayload( priority: 1, error: {}, result: null, - payload: payload, + payload, logs: [], }; } @@ -89,8 +89,8 @@ async function createPayload({ action = 'push'; jobType = 'githubPush'; branchName = githubEvent.ref.split('/')[2]; - url = githubEvent?.repository.clone_url; - newHead = githubEvent?.after; + url = githubEvent.repository.clone_url; + newHead = githubEvent.after; repoOwner = githubEvent.repository.owner.login; } @@ -191,11 +191,9 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr const env = 'dotcomstg'; async function createAndInsertJob() { - for (const s in SMOKETEST_SITES) { - const repoName = SMOKETEST_SITES[s]; + for (const repoName of SMOKETEST_SITES) { const jobTitle = 'Smoke Test ' + repoName; let repoInfo, projectEntry, repoOwner; - try { repoInfo = await docsetsRepository.getRepo(repoName); projectEntry = await projectsRepository.getProjectEntry(repoInfo.project); @@ -265,7 +263,17 @@ export const TriggerBuild = async (event: APIGatewayEvent): Promise { + protected async findOne( + query: any, + errorMsg: string, + options: mongodb.FindOptions = {} + ): Promise | null> { try { return await this.promiseTimeoutS( this._config.get('MONGO_TIMEOUT_S'), diff --git a/src/repositories/jobRepository.ts b/src/repositories/jobRepository.ts index 7712f15b3..d2e27da19 100644 --- a/src/repositories/jobRepository.ts +++ b/src/repositories/jobRepository.ts @@ -79,7 +79,7 @@ export class JobRepository extends BaseRepository { return jobIds; } - async getJobById(id: string): Promise { + async getJobById(id: string): Promise { const query = { _id: new objectId(id), }; diff --git a/src/repositories/projectsRepository.ts b/src/repositories/projectsRepository.ts index d03866227..c8486a255 100644 --- a/src/repositories/projectsRepository.ts +++ b/src/repositories/projectsRepository.ts @@ -1,4 +1,4 @@ -import { Db } from 'mongodb'; +import mongodb, { Db } from 'mongodb'; import { IConfig } from 'config'; import { BaseRepository } from './baseRepository'; import { ILogger } from '../services/logger'; @@ -11,7 +11,7 @@ export class ProjectsRepository extends BaseRepository { super(config, logger, 'ProjectsRepository', db.collection('projects')); } - async getProjectEntry(name: string): Promise { + async getProjectEntry(name: string): Promise | null> { const query = { name: name }; const projectEntry = await this.findOne( query, From bce69c38757ce6d3e7ec3cef5b143a77c9842c0a Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Tue, 19 Mar 2024 10:45:53 -0400 Subject: [PATCH 156/254] DOP-4414 comment out check for main branch --- api/controllers/v2/github.ts | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/api/controllers/v2/github.ts b/api/controllers/v2/github.ts index 0f92f7eea..61035ae2b 100644 --- a/api/controllers/v2/github.ts +++ b/api/controllers/v2/github.ts @@ -88,10 +88,10 @@ async function createPayload({ } action = 'push'; jobType = 'githubPush'; - branchName = githubEvent.ref.split('/')[2]; - url = githubEvent.repository.clone_url; + branchName = githubEvent.ref?.split('/')[2]; + url = githubEvent.repository?.clone_url; newHead = githubEvent.after; - repoOwner = githubEvent.repository.owner.login; + repoOwner = githubEvent.repository?.owner.login; } const branchInfo = await repoBranchesRepository.getRepoBranchAliases(repoName, branchName, repoInfo.project); @@ -174,18 +174,18 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr ' completed successfully but only Deploy Staging ECS workflow completion will trigger smoke test site deployments', }; - // if the build was not building master, no need for smoke test sites - if (body.workflow_run.head_branch != 'main' || body.repository.fork) { - console.log('Build was not on master branch in main repo, sites will not deploy as no smoke tests are needed'); - return { - statusCode: 202, - headers: { 'Content-Type': 'text/plain' }, - body: - 'Build on branch ' + - body.workflow_run.head_branch + - ' will not trigger site deployments as it was not on master branch in upstream repo', - }; - } + // // if the build was not building master, no need for smoke test sites + // if (body.workflow_run.head_branch != 'main' || body.repository.fork) { + // console.log('Build was not on master branch in main repo, sites will not deploy as no smoke tests are needed'); + // return { + // statusCode: 202, + // headers: { 'Content-Type': 'text/plain' }, + // body: + // 'Build on branch ' + + // body.workflow_run.head_branch + + // ' will not trigger site deployments as it was not on master branch in upstream repo', + // }; + // } //automated test builds will always deploy in dotcomstg const env = 'dotcomstg'; From c86c4665f8d052fec06d3c0e867ce4623e92c128 Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Tue, 19 Mar 2024 11:40:26 -0400 Subject: [PATCH 157/254] DOP-4414 entitlements repos --- src/repositories/repoEntitlementsRepository.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/repositories/repoEntitlementsRepository.ts b/src/repositories/repoEntitlementsRepository.ts index f9cdcde26..e878b7458 100644 --- a/src/repositories/repoEntitlementsRepository.ts +++ b/src/repositories/repoEntitlementsRepository.ts @@ -73,7 +73,7 @@ export class RepoEntitlementsRepository extends BaseRepository { `Mongo Timeout Error: Timedout while retrieving entitlements for ${slackUserId}` ); // if user has specific entitlements - if ((entitlementsObject?.repos?.length ?? 0) > 0) { + if (entitlementsObject?.repos && entitlementsObject.repos.length > 0) { return { repos: entitlementsObject.repos, github_username: entitlementsObject.github_username, From c4ac8373134709786a22d3d7a859f9a819b1bcc7 Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Tue, 19 Mar 2024 12:15:39 -0400 Subject: [PATCH 158/254] DOP-4414 testing --- api/controllers/v2/github.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/api/controllers/v2/github.ts b/api/controllers/v2/github.ts index 61035ae2b..cd0525825 100644 --- a/api/controllers/v2/github.ts +++ b/api/controllers/v2/github.ts @@ -174,7 +174,7 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr ' completed successfully but only Deploy Staging ECS workflow completion will trigger smoke test site deployments', }; - // // if the build was not building master, no need for smoke test sites + // // if the build was not building main branch, no need for smoke test sites // if (body.workflow_run.head_branch != 'main' || body.repository.fork) { // console.log('Build was not on master branch in main repo, sites will not deploy as no smoke tests are needed'); // return { @@ -196,6 +196,7 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr let repoInfo, projectEntry, repoOwner; try { repoInfo = await docsetsRepository.getRepo(repoName); + return repoName + repoInfo; projectEntry = await projectsRepository.getProjectEntry(repoInfo.project); repoOwner = projectEntry.github.organization; } catch { From 2c3693a08f9ce3578d9d2485b4200ee3aff14ebd Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Tue, 19 Mar 2024 13:43:53 -0400 Subject: [PATCH 159/254] revert "DOP-4414 testing" This reverts commit c4ac8373134709786a22d3d7a859f9a819b1bcc7. reverting changes --- api/controllers/v2/github.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/api/controllers/v2/github.ts b/api/controllers/v2/github.ts index cd0525825..61035ae2b 100644 --- a/api/controllers/v2/github.ts +++ b/api/controllers/v2/github.ts @@ -174,7 +174,7 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr ' completed successfully but only Deploy Staging ECS workflow completion will trigger smoke test site deployments', }; - // // if the build was not building main branch, no need for smoke test sites + // // if the build was not building master, no need for smoke test sites // if (body.workflow_run.head_branch != 'main' || body.repository.fork) { // console.log('Build was not on master branch in main repo, sites will not deploy as no smoke tests are needed'); // return { @@ -196,7 +196,6 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr let repoInfo, projectEntry, repoOwner; try { repoInfo = await docsetsRepository.getRepo(repoName); - return repoName + repoInfo; projectEntry = await projectsRepository.getProjectEntry(repoInfo.project); repoOwner = projectEntry.github.organization; } catch { From 242cf10b655417bc95ca1e41f98f90e169a855c8 Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Tue, 19 Mar 2024 13:45:26 -0400 Subject: [PATCH 160/254] Revert "DOP-4414 entitlements repos" This reverts commit c86c4665f8d052fec06d3c0e867ce4623e92c128. reverting commit --- src/repositories/repoEntitlementsRepository.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/repositories/repoEntitlementsRepository.ts b/src/repositories/repoEntitlementsRepository.ts index e878b7458..f9cdcde26 100644 --- a/src/repositories/repoEntitlementsRepository.ts +++ b/src/repositories/repoEntitlementsRepository.ts @@ -73,7 +73,7 @@ export class RepoEntitlementsRepository extends BaseRepository { `Mongo Timeout Error: Timedout while retrieving entitlements for ${slackUserId}` ); // if user has specific entitlements - if (entitlementsObject?.repos && entitlementsObject.repos.length > 0) { + if ((entitlementsObject?.repos?.length ?? 0) > 0) { return { repos: entitlementsObject.repos, github_username: entitlementsObject.github_username, From 40a1194da573a95d9248efc75736304b52f7c69b Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Tue, 19 Mar 2024 13:45:41 -0400 Subject: [PATCH 161/254] Revert "DOP-4414 comment out check for main branch" This reverts commit bce69c38757ce6d3e7ec3cef5b143a77c9842c0a. reverting commit --- api/controllers/v2/github.ts | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/api/controllers/v2/github.ts b/api/controllers/v2/github.ts index 61035ae2b..0f92f7eea 100644 --- a/api/controllers/v2/github.ts +++ b/api/controllers/v2/github.ts @@ -88,10 +88,10 @@ async function createPayload({ } action = 'push'; jobType = 'githubPush'; - branchName = githubEvent.ref?.split('/')[2]; - url = githubEvent.repository?.clone_url; + branchName = githubEvent.ref.split('/')[2]; + url = githubEvent.repository.clone_url; newHead = githubEvent.after; - repoOwner = githubEvent.repository?.owner.login; + repoOwner = githubEvent.repository.owner.login; } const branchInfo = await repoBranchesRepository.getRepoBranchAliases(repoName, branchName, repoInfo.project); @@ -174,18 +174,18 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr ' completed successfully but only Deploy Staging ECS workflow completion will trigger smoke test site deployments', }; - // // if the build was not building master, no need for smoke test sites - // if (body.workflow_run.head_branch != 'main' || body.repository.fork) { - // console.log('Build was not on master branch in main repo, sites will not deploy as no smoke tests are needed'); - // return { - // statusCode: 202, - // headers: { 'Content-Type': 'text/plain' }, - // body: - // 'Build on branch ' + - // body.workflow_run.head_branch + - // ' will not trigger site deployments as it was not on master branch in upstream repo', - // }; - // } + // if the build was not building master, no need for smoke test sites + if (body.workflow_run.head_branch != 'main' || body.repository.fork) { + console.log('Build was not on master branch in main repo, sites will not deploy as no smoke tests are needed'); + return { + statusCode: 202, + headers: { 'Content-Type': 'text/plain' }, + body: + 'Build on branch ' + + body.workflow_run.head_branch + + ' will not trigger site deployments as it was not on master branch in upstream repo', + }; + } //automated test builds will always deploy in dotcomstg const env = 'dotcomstg'; From 6ccdc00d2dce9d3b35bd13a4340d326e3a51a60e Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Tue, 19 Mar 2024 13:46:01 -0400 Subject: [PATCH 162/254] reverting commit Revert "DOP-4414 types and env vars" This reverts commit aee5cd49ab37e7404585cdc83b3f48df4be7c638. reverting commit --- api/controllers/v2/github.ts | 23 ++++++------------- .../lib/constructs/worker/worker-construct.ts | 2 +- src/repositories/baseRepository.ts | 6 +---- src/repositories/jobRepository.ts | 2 +- src/repositories/projectsRepository.ts | 4 ++-- 5 files changed, 12 insertions(+), 25 deletions(-) diff --git a/api/controllers/v2/github.ts b/api/controllers/v2/github.ts index 0f92f7eea..a5c2b855d 100644 --- a/api/controllers/v2/github.ts +++ b/api/controllers/v2/github.ts @@ -41,7 +41,7 @@ async function prepGithubPushPayload( priority: 1, error: {}, result: null, - payload, + payload: payload, logs: [], }; } @@ -89,8 +89,8 @@ async function createPayload({ action = 'push'; jobType = 'githubPush'; branchName = githubEvent.ref.split('/')[2]; - url = githubEvent.repository.clone_url; - newHead = githubEvent.after; + url = githubEvent?.repository.clone_url; + newHead = githubEvent?.after; repoOwner = githubEvent.repository.owner.login; } @@ -191,9 +191,11 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr const env = 'dotcomstg'; async function createAndInsertJob() { - for (const repoName of SMOKETEST_SITES) { + for (const s in SMOKETEST_SITES) { + const repoName = SMOKETEST_SITES[s]; const jobTitle = 'Smoke Test ' + repoName; let repoInfo, projectEntry, repoOwner; + try { repoInfo = await docsetsRepository.getRepo(repoName); projectEntry = await projectsRepository.getProjectEntry(repoInfo.project); @@ -263,17 +265,7 @@ export const TriggerBuild = async (event: APIGatewayEvent): Promise | null> { + protected async findOne(query: any, errorMsg: string, options: mongodb.FindOptions = {}): Promise { try { return await this.promiseTimeoutS( this._config.get('MONGO_TIMEOUT_S'), diff --git a/src/repositories/jobRepository.ts b/src/repositories/jobRepository.ts index d2e27da19..7712f15b3 100644 --- a/src/repositories/jobRepository.ts +++ b/src/repositories/jobRepository.ts @@ -79,7 +79,7 @@ export class JobRepository extends BaseRepository { return jobIds; } - async getJobById(id: string): Promise { + async getJobById(id: string): Promise { const query = { _id: new objectId(id), }; diff --git a/src/repositories/projectsRepository.ts b/src/repositories/projectsRepository.ts index c8486a255..d03866227 100644 --- a/src/repositories/projectsRepository.ts +++ b/src/repositories/projectsRepository.ts @@ -1,4 +1,4 @@ -import mongodb, { Db } from 'mongodb'; +import { Db } from 'mongodb'; import { IConfig } from 'config'; import { BaseRepository } from './baseRepository'; import { ILogger } from '../services/logger'; @@ -11,7 +11,7 @@ export class ProjectsRepository extends BaseRepository { super(config, logger, 'ProjectsRepository', db.collection('projects')); } - async getProjectEntry(name: string): Promise | null> { + async getProjectEntry(name: string): Promise { const query = { name: name }; const projectEntry = await this.findOne( query, From c2b1f57ad79e148fb217f55c247583138e5eb0dc Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Tue, 19 Mar 2024 13:50:10 -0400 Subject: [PATCH 163/254] DOP-4414 comment out main check for testing --- api/controllers/v2/github.ts | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/api/controllers/v2/github.ts b/api/controllers/v2/github.ts index a5c2b855d..f7515ecd6 100644 --- a/api/controllers/v2/github.ts +++ b/api/controllers/v2/github.ts @@ -175,17 +175,17 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr }; // if the build was not building master, no need for smoke test sites - if (body.workflow_run.head_branch != 'main' || body.repository.fork) { - console.log('Build was not on master branch in main repo, sites will not deploy as no smoke tests are needed'); - return { - statusCode: 202, - headers: { 'Content-Type': 'text/plain' }, - body: - 'Build on branch ' + - body.workflow_run.head_branch + - ' will not trigger site deployments as it was not on master branch in upstream repo', - }; - } + // if (body.workflow_run.head_branch != 'main' || body.repository.fork) { + // console.log('Build was not on master branch in main repo, sites will not deploy as no smoke tests are needed'); + // return { + // statusCode: 202, + // headers: { 'Content-Type': 'text/plain' }, + // body: + // 'Build on branch ' + + // body.workflow_run.head_branch + + // ' will not trigger site deployments as it was not on main branch in upstream repo', + // }; + // } //automated test builds will always deploy in dotcomstg const env = 'dotcomstg'; From abdd426470bd4ddd57abf7a3253ef0335e63f7e0 Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Tue, 19 Mar 2024 14:26:56 -0400 Subject: [PATCH 164/254] DOP-4414 testing --- api/controllers/v2/github.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/controllers/v2/github.ts b/api/controllers/v2/github.ts index f7515ecd6..b71da0981 100644 --- a/api/controllers/v2/github.ts +++ b/api/controllers/v2/github.ts @@ -174,7 +174,7 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr ' completed successfully but only Deploy Staging ECS workflow completion will trigger smoke test site deployments', }; - // if the build was not building master, no need for smoke test sites + // if the build was not building main branch, no need for smoke test sites // if (body.workflow_run.head_branch != 'main' || body.repository.fork) { // console.log('Build was not on master branch in main repo, sites will not deploy as no smoke tests are needed'); // return { From 1a79d7bc56ce091cb5f6286b4dfa836d55c48351 Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Tue, 19 Mar 2024 16:58:08 -0400 Subject: [PATCH 165/254] DOP-4414 add branch to deploy enhanced worker wq --- .github/workflows/deploy-stg-enhanced-worker.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/deploy-stg-enhanced-worker.yml b/.github/workflows/deploy-stg-enhanced-worker.yml index c779ff49b..0c067587b 100644 --- a/.github/workflows/deploy-stg-enhanced-worker.yml +++ b/.github/workflows/deploy-stg-enhanced-worker.yml @@ -4,6 +4,7 @@ on: branches: - 'main' - 'integration' + - 'DOP-4414' concurrency: group: environment-stg-enhanced-worker-${{ github.ref }} From 273da47f56328070688dc8963de4341a0facc72d Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Wed, 20 Mar 2024 10:39:30 -0400 Subject: [PATCH 166/254] DOP-4414 add branch to deploy enhanced webhook --- .github/workflows/deploy-stg-enhanced-webhooks.yml | 1 + .github/workflows/deploy-stg-enhanced-worker.yml | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/deploy-stg-enhanced-webhooks.yml b/.github/workflows/deploy-stg-enhanced-webhooks.yml index f4d4dd938..630599bf3 100644 --- a/.github/workflows/deploy-stg-enhanced-webhooks.yml +++ b/.github/workflows/deploy-stg-enhanced-webhooks.yml @@ -4,6 +4,7 @@ on: branches: - "main" - "integration" + - "DOP-4414" concurrency: group: environment-stg-enhanced-webhooks-${{ github.ref }} diff --git a/.github/workflows/deploy-stg-enhanced-worker.yml b/.github/workflows/deploy-stg-enhanced-worker.yml index 0c067587b..c779ff49b 100644 --- a/.github/workflows/deploy-stg-enhanced-worker.yml +++ b/.github/workflows/deploy-stg-enhanced-worker.yml @@ -4,7 +4,6 @@ on: branches: - 'main' - 'integration' - - 'DOP-4414' concurrency: group: environment-stg-enhanced-worker-${{ github.ref }} From ca9bb9f0fe92f17e0b8579248b32bbc095d02cc0 Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Wed, 20 Mar 2024 11:23:10 -0400 Subject: [PATCH 167/254] DOP-4414 add branch to deploy enhanced webhook --- .github/workflows/deploy-stg-enhanced-worker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/deploy-stg-enhanced-worker.yml b/.github/workflows/deploy-stg-enhanced-worker.yml index c779ff49b..8388d7d34 100644 --- a/.github/workflows/deploy-stg-enhanced-worker.yml +++ b/.github/workflows/deploy-stg-enhanced-worker.yml @@ -1,6 +1,6 @@ on: push: - paths: ['src/**', 'cdk-infra/lib/constructs/worker/**', 'Dockerfile', 'modules/**'] + # paths: ['src/**', 'cdk-infra/lib/constructs/worker/**', 'Dockerfile', 'modules/**'] branches: - 'main' - 'integration' From dccda6681e7d96805aa23f322619b1ac2f115bdc Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Wed, 20 Mar 2024 11:26:53 -0400 Subject: [PATCH 168/254] DOP-4414 add branch to deploy enhanced webhook --- .github/workflows/deploy-stg-enhanced-webhooks.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/deploy-stg-enhanced-webhooks.yml b/.github/workflows/deploy-stg-enhanced-webhooks.yml index 630599bf3..40c4347f1 100644 --- a/.github/workflows/deploy-stg-enhanced-webhooks.yml +++ b/.github/workflows/deploy-stg-enhanced-webhooks.yml @@ -1,6 +1,6 @@ on: push: - paths: ["api/v2/**", "cdk-infra/lib/constructs/api/**", "cdk-infra/utils/**"] + # paths: ["api/v2/**", "cdk-infra/lib/constructs/api/**", "cdk-infra/utils/**"] branches: - "main" - "integration" From d953ccd9f839ca20e0bfd32d20e3ebacc5cd413d Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Wed, 20 Mar 2024 11:27:00 -0400 Subject: [PATCH 169/254] DOP-4414 add branch to deploy enhanced webhook --- .github/workflows/deploy-stg-enhanced-worker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/deploy-stg-enhanced-worker.yml b/.github/workflows/deploy-stg-enhanced-worker.yml index 8388d7d34..c779ff49b 100644 --- a/.github/workflows/deploy-stg-enhanced-worker.yml +++ b/.github/workflows/deploy-stg-enhanced-worker.yml @@ -1,6 +1,6 @@ on: push: - # paths: ['src/**', 'cdk-infra/lib/constructs/worker/**', 'Dockerfile', 'modules/**'] + paths: ['src/**', 'cdk-infra/lib/constructs/worker/**', 'Dockerfile', 'modules/**'] branches: - 'main' - 'integration' From c0fe254e4df7e2645fc72a4685a92ac52d889e95 Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Wed, 20 Mar 2024 12:40:26 -0400 Subject: [PATCH 170/254] DOP-4414 nits --- api/controllers/v2/github.ts | 25 +++++++++---------- src/repositories/baseRepository.ts | 6 ++++- src/repositories/projectsRepository.ts | 4 +-- .../repoEntitlementsRepository.ts | 2 +- 4 files changed, 20 insertions(+), 17 deletions(-) diff --git a/api/controllers/v2/github.ts b/api/controllers/v2/github.ts index b71da0981..ea7218574 100644 --- a/api/controllers/v2/github.ts +++ b/api/controllers/v2/github.ts @@ -41,7 +41,7 @@ async function prepGithubPushPayload( priority: 1, error: {}, result: null, - payload: payload, + payload, logs: [], }; } @@ -89,9 +89,9 @@ async function createPayload({ action = 'push'; jobType = 'githubPush'; branchName = githubEvent.ref.split('/')[2]; - url = githubEvent?.repository.clone_url; - newHead = githubEvent?.after; - repoOwner = githubEvent.repository.owner.login; + url = githubEvent.repository?.clone_url; + newHead = githubEvent.after; + repoOwner = githubEvent.repository?.owner?.login; } const branchInfo = await repoBranchesRepository.getRepoBranchAliases(repoName, branchName, repoInfo.project); @@ -154,15 +154,14 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr }; } - if (body.workflow_run.conclusion != 'success') - return { - statusCode: 202, - headers: { 'Content-Type': 'text/plain' }, - body: - 'Build on branch ' + - body.workflow_run.head_branch + - ' is not complete and will not trigger smoke test site deployments ', - }; + return { + statusCode: 202, + headers: { 'Content-Type': 'text/plain' }, + body: + 'Build on branch ' + + body.workflow_run.head_branch + + ' is not complete and will not trigger smoke test site deployments ', + }; if (body.workflow_run.name != 'Deploy Staging ECS') return { diff --git a/src/repositories/baseRepository.ts b/src/repositories/baseRepository.ts index 750a1f83f..564e9b268 100644 --- a/src/repositories/baseRepository.ts +++ b/src/repositories/baseRepository.ts @@ -71,7 +71,11 @@ export abstract class BaseRepository { ); } - protected async findOne(query: any, errorMsg: string, options: mongodb.FindOptions = {}): Promise { + protected async findOne( + query: any, + errorMsg: string, + options: mongodb.FindOptions = {} + ): Promise | null> { try { return await this.promiseTimeoutS( this._config.get('MONGO_TIMEOUT_S'), diff --git a/src/repositories/projectsRepository.ts b/src/repositories/projectsRepository.ts index d03866227..c8486a255 100644 --- a/src/repositories/projectsRepository.ts +++ b/src/repositories/projectsRepository.ts @@ -1,4 +1,4 @@ -import { Db } from 'mongodb'; +import mongodb, { Db } from 'mongodb'; import { IConfig } from 'config'; import { BaseRepository } from './baseRepository'; import { ILogger } from '../services/logger'; @@ -11,7 +11,7 @@ export class ProjectsRepository extends BaseRepository { super(config, logger, 'ProjectsRepository', db.collection('projects')); } - async getProjectEntry(name: string): Promise { + async getProjectEntry(name: string): Promise | null> { const query = { name: name }; const projectEntry = await this.findOne( query, diff --git a/src/repositories/repoEntitlementsRepository.ts b/src/repositories/repoEntitlementsRepository.ts index f9cdcde26..e878b7458 100644 --- a/src/repositories/repoEntitlementsRepository.ts +++ b/src/repositories/repoEntitlementsRepository.ts @@ -73,7 +73,7 @@ export class RepoEntitlementsRepository extends BaseRepository { `Mongo Timeout Error: Timedout while retrieving entitlements for ${slackUserId}` ); // if user has specific entitlements - if ((entitlementsObject?.repos?.length ?? 0) > 0) { + if (entitlementsObject?.repos && entitlementsObject.repos.length > 0) { return { repos: entitlementsObject.repos, github_username: entitlementsObject.github_username, From b28d519e83ffdb714589d36a40ae67e5c7b02bdd Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Wed, 20 Mar 2024 13:42:07 -0400 Subject: [PATCH 171/254] DOP-4414 fixing small error --- src/repositories/jobRepository.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/repositories/jobRepository.ts b/src/repositories/jobRepository.ts index 7712f15b3..d2e27da19 100644 --- a/src/repositories/jobRepository.ts +++ b/src/repositories/jobRepository.ts @@ -79,7 +79,7 @@ export class JobRepository extends BaseRepository { return jobIds; } - async getJobById(id: string): Promise { + async getJobById(id: string): Promise { const query = { _id: new objectId(id), }; From 61b2c33a10968f90f42701bfcc37d36200032a88 Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Wed, 20 Mar 2024 14:17:01 -0400 Subject: [PATCH 172/254] DOP-4414 refactoring promises --- api/controllers/v2/github.ts | 94 ++++++++++++++++++------------------ 1 file changed, 48 insertions(+), 46 deletions(-) diff --git a/api/controllers/v2/github.ts b/api/controllers/v2/github.ts index ea7218574..733742474 100644 --- a/api/controllers/v2/github.ts +++ b/api/controllers/v2/github.ts @@ -154,14 +154,15 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr }; } - return { - statusCode: 202, - headers: { 'Content-Type': 'text/plain' }, - body: - 'Build on branch ' + - body.workflow_run.head_branch + - ' is not complete and will not trigger smoke test site deployments ', - }; + if (body.workflow_run.conclusion != 'success') + return { + statusCode: 202, + headers: { 'Content-Type': 'text/plain' }, + body: + 'Build on branch ' + + body.workflow_run.head_branch + + ' is not complete and will not trigger smoke test site deployments ', + }; if (body.workflow_run.name != 'Deploy Staging ECS') return { @@ -190,44 +191,45 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr const env = 'dotcomstg'; async function createAndInsertJob() { - for (const s in SMOKETEST_SITES) { - const repoName = SMOKETEST_SITES[s]; - const jobTitle = 'Smoke Test ' + repoName; - let repoInfo, projectEntry, repoOwner; - - try { - repoInfo = await docsetsRepository.getRepo(repoName); - projectEntry = await projectsRepository.getProjectEntry(repoInfo.project); - repoOwner = projectEntry.github.organization; - } catch { - return 'repoInfo, projectEntry, or repoOwner not found for docs site ' + repoName; - } - - const jobPrefix = repoInfo?.prefix ? repoInfo['prefix'][env] : ''; - - //add commit hash here: const newHead = body.workflow_run.head_sha; - - const payload = await createPayload({ - repoName, - isSmokeTestDeploy: true, - prefix: jobPrefix, - repoBranchesRepository, - repoInfo, - repoOwner, - }); - //add logic for getting master branch, latest stable branch - const job = await prepGithubPushPayload(body, payload, jobTitle); - - try { - consoleLogger.info(job.title, 'Creating Job'); - const jobId = await jobRepository.insertJob(job, c.get('jobsQueueUrl')); - jobRepository.notify(jobId, c.get('jobUpdatesQueueUrl'), JobStatus.inQueue, 0); - consoleLogger.info(job.title, `Created Job ${jobId}`); - } catch (err) { - consoleLogger.error('TriggerBuildError', err + repoName); - return err; - } - } + const jobs = await Promise.all( + SMOKETEST_SITES.map(async (repoName): Promise => { + const jobTitle = 'Smoke Test ' + repoName; + let repoInfo, projectEntry, repoOwner; + try { + repoInfo = docsetsRepository.getRepo(repoName); + projectEntry = projectsRepository.getProjectEntry(repoInfo.project); + repoOwner = projectEntry.github.organization; + } catch { + return 'repoInfo, projectEntry, or repoOwner not found for docs site ' + repoName; + } + + const jobPrefix = repoInfo?.prefix ? repoInfo['prefix'][env] : ''; + + //add commit hash here: const newHead = body.workflow_run.head_sha; + + const payload = await createPayload({ + repoName, + isSmokeTestDeploy: true, + prefix: jobPrefix, + repoBranchesRepository, + repoInfo, + repoOwner, + }); + //add logic for getting master branch, latest stable branch + const job = await prepGithubPushPayload(body, payload, jobTitle); + + try { + consoleLogger.info(job.title, 'Creating Job'); + const jobId = await jobRepository.insertJob(job, c.get('jobsQueueUrl')); + jobRepository.notify(jobId, c.get('jobUpdatesQueueUrl'), JobStatus.inQueue, 0); + consoleLogger.info(job.title, `Created Job ${jobId}`); + return jobId; + } catch (err) { + consoleLogger.error('TriggerBuildError', err + repoName); + return err; + } + }) + ); } try { From 47d06009be56a970d6e2802e5d69ca84b12dc0cb Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Wed, 20 Mar 2024 14:26:37 -0400 Subject: [PATCH 173/254] DOP-4414 testing new promise format --- api/controllers/v2/github.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/controllers/v2/github.ts b/api/controllers/v2/github.ts index 733742474..48557ce2d 100644 --- a/api/controllers/v2/github.ts +++ b/api/controllers/v2/github.ts @@ -191,7 +191,7 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr const env = 'dotcomstg'; async function createAndInsertJob() { - const jobs = await Promise.all( + return await Promise.all( SMOKETEST_SITES.map(async (repoName): Promise => { const jobTitle = 'Smoke Test ' + repoName; let repoInfo, projectEntry, repoOwner; From 654f59b2ea1f8b9a8639b5ea0845426ec0b2d74d Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Wed, 20 Mar 2024 14:32:04 -0400 Subject: [PATCH 174/254] DOP-4414 testing new promise format --- api/controllers/v2/github.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/api/controllers/v2/github.ts b/api/controllers/v2/github.ts index 48557ce2d..74f3d2607 100644 --- a/api/controllers/v2/github.ts +++ b/api/controllers/v2/github.ts @@ -232,8 +232,9 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr ); } + let returnVal; try { - await createAndInsertJob(); + returnVal = await createAndInsertJob(); } catch (err) { return { statusCode: 500, @@ -244,7 +245,7 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr return { statusCode: 202, headers: { 'Content-Type': 'text/plain' }, - body: 'Smoke Test Jobs Queued ', + body: 'Smoke Test Jobs Queued ' + returnVal, }; }; From 3fefba128a870fb1b2e183f5ae25345d7b2601d7 Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Wed, 20 Mar 2024 14:37:17 -0400 Subject: [PATCH 175/254] DOP-4414 testing new promise format --- api/controllers/v2/github.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/api/controllers/v2/github.ts b/api/controllers/v2/github.ts index 74f3d2607..6d02af7b9 100644 --- a/api/controllers/v2/github.ts +++ b/api/controllers/v2/github.ts @@ -196,8 +196,8 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr const jobTitle = 'Smoke Test ' + repoName; let repoInfo, projectEntry, repoOwner; try { - repoInfo = docsetsRepository.getRepo(repoName); - projectEntry = projectsRepository.getProjectEntry(repoInfo.project); + repoInfo = await docsetsRepository.getRepo(repoName); + projectEntry = await projectsRepository.getProjectEntry(repoInfo.project); repoOwner = projectEntry.github.organization; } catch { return 'repoInfo, projectEntry, or repoOwner not found for docs site ' + repoName; From 025a8722b5aa69da322b4fbbc6cdee05f3d510e7 Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Wed, 20 Mar 2024 16:28:09 -0400 Subject: [PATCH 176/254] DOP-4414 refactoring and cleaning --- .github/workflows/deploy-stg-ecs.yml | 1 - api/controllers/v2/github.ts | 39 ++++++++++++++-------------- 2 files changed, 19 insertions(+), 21 deletions(-) diff --git a/.github/workflows/deploy-stg-ecs.yml b/.github/workflows/deploy-stg-ecs.yml index 1b5c34109..2e0cfb462 100644 --- a/.github/workflows/deploy-stg-ecs.yml +++ b/.github/workflows/deploy-stg-ecs.yml @@ -3,7 +3,6 @@ on: branches: - "main" - "integration" - - "DOP-4414" concurrency: group: environment-stg-${{ github.ref }} cancel-in-progress: true diff --git a/api/controllers/v2/github.ts b/api/controllers/v2/github.ts index 6d02af7b9..7c479ac29 100644 --- a/api/controllers/v2/github.ts +++ b/api/controllers/v2/github.ts @@ -84,7 +84,7 @@ async function createPayload({ branchName = 'master'; } else { if (!githubEvent) { - throw 'Non SmokeTest Deploy jobs must have a github Event'; + throw new Error(`Non SmokeTest Deploy jobs must have a github Event`); } action = 'push'; jobType = 'githubPush'; @@ -158,20 +158,17 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr return { statusCode: 202, headers: { 'Content-Type': 'text/plain' }, - body: - 'Build on branch ' + - body.workflow_run.head_branch + - ' is not complete and will not trigger smoke test site deployments ', + body: `Build on branch + ${body.workflow_run.head_branch} + is not complete and will not trigger smoke test site deployments`, }; if (body.workflow_run.name != 'Deploy Staging ECS') return { statusCode: 202, headers: { 'Content-Type': 'text/plain' }, - body: - 'Workflow ' + - body.workflow_run.name + - ' completed successfully but only Deploy Staging ECS workflow completion will trigger smoke test site deployments', + body: `Workflow + ${body.workflow_run.name} completed successfully but only Deploy Staging ECS workflow completion will trigger smoke test site deployments`, }; // if the build was not building main branch, no need for smoke test sites @@ -181,9 +178,9 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr // statusCode: 202, // headers: { 'Content-Type': 'text/plain' }, // body: - // 'Build on branch ' + - // body.workflow_run.head_branch + - // ' will not trigger site deployments as it was not on main branch in upstream repo', + // `Build on branch + // ${body.workflow_run.head_branch} + // will not trigger site deployments as it was not on main branch in upstream repo`, // }; // } @@ -199,14 +196,15 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr repoInfo = await docsetsRepository.getRepo(repoName); projectEntry = await projectsRepository.getProjectEntry(repoInfo.project); repoOwner = projectEntry.github.organization; - } catch { - return 'repoInfo, projectEntry, or repoOwner not found for docs site ' + repoName; + } catch (err) { + consoleLogger.error( + `Atlas Repo Information Error`, + `RepoInfo, projectEntry, or repoOwner not found for docs site ${repoName}. RepoInfo: ${repoInfo}, projectEntry: ${projectEntry}, repoOwner: ${repoOwner}` + ); + return err; } const jobPrefix = repoInfo?.prefix ? repoInfo['prefix'][env] : ''; - - //add commit hash here: const newHead = body.workflow_run.head_sha; - const payload = await createPayload({ repoName, isSmokeTestDeploy: true, @@ -215,6 +213,7 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr repoInfo, repoOwner, }); + //add logic for getting master branch, latest stable branch const job = await prepGithubPushPayload(body, payload, jobTitle); @@ -225,7 +224,7 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr consoleLogger.info(job.title, `Created Job ${jobId}`); return jobId; } catch (err) { - consoleLogger.error('TriggerBuildError', err + repoName); + consoleLogger.error('TriggerBuildError', `${err} Error inserting job for ${repoName}`); return err; } }) @@ -245,7 +244,7 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr return { statusCode: 202, headers: { 'Content-Type': 'text/plain' }, - body: 'Smoke Test Jobs Queued ' + returnVal, + body: 'Smoke Test Jobs Queued with the following Job Ids' + returnVal, }; }; @@ -267,7 +266,7 @@ export const TriggerBuild = async (event: APIGatewayEvent): Promise('githubSecret'))) { const errMsg = "X-Hub-Signature incorrect. Github webhook token doesn't match"; return { statusCode: 401, From 2797e7155f0a862d8b7ab3a56c7e0256e340e3e4 Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Wed, 20 Mar 2024 16:42:15 -0400 Subject: [PATCH 177/254] DOP-4414 refactoring and cleaning --- .github/workflows/deploy-stg-ecs.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/deploy-stg-ecs.yml b/.github/workflows/deploy-stg-ecs.yml index 2e0cfb462..1b5c34109 100644 --- a/.github/workflows/deploy-stg-ecs.yml +++ b/.github/workflows/deploy-stg-ecs.yml @@ -3,6 +3,7 @@ on: branches: - "main" - "integration" + - "DOP-4414" concurrency: group: environment-stg-${{ github.ref }} cancel-in-progress: true From b82af1a2bc7b076161cf0692c6bff674b6cdf2de Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Thu, 21 Mar 2024 11:37:19 -0400 Subject: [PATCH 178/254] DOP-4414 adding projects collection as constant --- api/config/custom-environment-variables.json | 1 + api/config/default.json | 3 +++ cdk-infra/lib/constructs/api/webhook-env-construct.ts | 1 + .../lib/constructs/worker/worker-env-construct.ts | 1 + .../api/config/custom-environment-variables.json | 1 + cdk-infra/static/api/config/default.json | 1 + config/default.json | 1 + config/test.json | 1 + src/repositories/projectsRepository.ts | 11 ++++++----- 9 files changed, 16 insertions(+), 5 deletions(-) diff --git a/api/config/custom-environment-variables.json b/api/config/custom-environment-variables.json index 4199fbce2..3e49afbd0 100644 --- a/api/config/custom-environment-variables.json +++ b/api/config/custom-environment-variables.json @@ -17,6 +17,7 @@ "slackViewOpenUrl": "https://slack.com/api/views.open", "snootySecret": "SNOOTY_SECRET", "jobQueueCollection": "JOB_QUEUE_COL_NAME", + "projectsCollection": "PROJECTS_COL_NAME", "entitlementCollection": "USER_ENTITLEMENT_COL_NAME", "dashboardUrl": "DASHBOARD_URL", "repoBranchesCollection": "REPO_BRANCHES_COL_NAME", diff --git a/api/config/default.json b/api/config/default.json index 16d56ce57..a7988e98c 100644 --- a/api/config/default.json +++ b/api/config/default.json @@ -17,6 +17,7 @@ "slackViewOpenUrl": "https://slack.com/api/views.open", "snootySecret": "SNOOTY_SECRET", "jobQueueCollection": "JOB_QUEUE_COL_NAME", + "projectsCollection": "PROJECTS_COL_NAME", "entitlementCollection": "USER_ENTITLEMENT_COL_NAME", "repoBranchesCollection": "REPO_BRANCHES_COL_NAME", "docsetsCollection": "DOCSETS_COL_NAME", @@ -37,6 +38,7 @@ "stg": { "dbName": "pool_test", "jobQueueCollection": "dotcom_queue", + "projectsCollection": "PROJECTS_COL_NAME", "jobsQueueUrl": "https://sqs.us-east-2.amazonaws.com/216656347858/autobuilder-jobs-queue-dotcomstg", "jobUpdatesQueueUrl": "https://sqs.us-east-2.amazonaws.com/216656347858/autobuilder-job-updates-queue-dotcomstg", "parallelStgName": "dotcomstg" @@ -44,6 +46,7 @@ "prd": { "dbName": "pool", "jobQueueCollection": "dotcom_queue", + "projectsCollection": "PROJECTS_COL_NAME", "jobsQueueUrl": "https://sqs.us-east-2.amazonaws.com/216656347858/autobuilder-jobs-queue-dotcomprd", "jobUpdatesQueueUrl": "https://sqs.us-east-2.amazonaws.com/216656347858/autobuilder-job-updates-queue-dotcomprd", "parallelStgName": "dotcomprd" diff --git a/cdk-infra/lib/constructs/api/webhook-env-construct.ts b/cdk-infra/lib/constructs/api/webhook-env-construct.ts index 800b9277d..fd8f9cd01 100644 --- a/cdk-infra/lib/constructs/api/webhook-env-construct.ts +++ b/cdk-infra/lib/constructs/api/webhook-env-construct.ts @@ -42,6 +42,7 @@ export class WebhookEnvConstruct extends Construct { SNOOTY_DB_NAME: snootyDbName, REPO_BRANCHES_COL_NAME: repoBranchesCollection, DOCSETS_COL_NAME: docsetsCollection, + PROJECTS_COL_NAME: 'projects', JOB_QUEUE_COL_NAME: jobCollection, NODE_CONFIG_DIR: './config', JOBS_QUEUE_URL: jobsQueue.queueUrl, diff --git a/cdk-infra/lib/constructs/worker/worker-env-construct.ts b/cdk-infra/lib/constructs/worker/worker-env-construct.ts index 1b532a16c..54d9737ac 100644 --- a/cdk-infra/lib/constructs/worker/worker-env-construct.ts +++ b/cdk-infra/lib/constructs/worker/worker-env-construct.ts @@ -92,6 +92,7 @@ export class WorkerEnvConstruct extends Construct { REPO_BRANCHES_COL_NAME: repoBranchesCollection, DOCSETS_COL_NAME: docsetsCollection, JOB_QUEUE_COL_NAME: jobCollection, + PROJECTS_COL_NAME: 'projects', CDN_INVALIDATOR_SERVICE_URL: getCdnInvalidatorUrl(env), SEARCH_INDEX_BUCKET: 'docs-search-indexes-test', SEARCH_INDEX_FOLDER: getSearchIndexFolder(env), diff --git a/cdk-infra/static/api/config/custom-environment-variables.json b/cdk-infra/static/api/config/custom-environment-variables.json index 8d554d1b5..cf328ded6 100644 --- a/cdk-infra/static/api/config/custom-environment-variables.json +++ b/cdk-infra/static/api/config/custom-environment-variables.json @@ -17,6 +17,7 @@ "slackViewOpenUrl": "https://slack.com/api/views.open", "snootySecret": "SNOOTY_SECRET", "jobQueueCollection": "JOB_QUEUE_COL_NAME", + "projectsCollection": "PROJECTS_COL_NAME", "entitlementCollection": "USER_ENTITLEMENT_COL_NAME", "dashboardUrl": "DASHBOARD_URL", "repoBranchesCollection": "REPO_BRANCHES_COL_NAME", diff --git a/cdk-infra/static/api/config/default.json b/cdk-infra/static/api/config/default.json index 3758dea37..481f695d3 100644 --- a/cdk-infra/static/api/config/default.json +++ b/cdk-infra/static/api/config/default.json @@ -17,6 +17,7 @@ "slackViewOpenUrl": "https://slack.com/api/views.open", "snootySecret": "SNOOTY_SECRET", "jobQueueCollection": "JOB_QUEUE_COL_NAME", + "projectsCollection": "PROJECTS_COL_NAME", "entitlementCollection": "USER_ENTITLEMENT_COL_NAME", "repoBranchesCollection": "REPO_BRANCHES_COL_NAME", "docsetsCollection": "DOCSETS_COL_NAME", diff --git a/config/default.json b/config/default.json index 8a089b421..ae9b5f9a7 100644 --- a/config/default.json +++ b/config/default.json @@ -12,6 +12,7 @@ "githubBotPW": "q1w", "fastlyDochubMap": "devfslydochubmap", "entitlementCollection": "entitlements", + "projectsCollection": "projects", "docsetsCollection": "docsets", "MONGO_TIMEOUT_S": 1500, "JOB_TIMEOUT_S": 900, diff --git a/config/test.json b/config/test.json index 5a17611ea..aded7d904 100644 --- a/config/test.json +++ b/config/test.json @@ -12,6 +12,7 @@ "githubBotPW": "q1w", "fastlyDochubMap": "devfslydochubmap", "entitlementCollection": "entitlements", + "projectsCollection": "projects", "docsetsCollection": "docsets", "MONGO_TIMEOUT_S": 1, "JOB_TIMEOUT_S": 10, diff --git a/src/repositories/projectsRepository.ts b/src/repositories/projectsRepository.ts index c8486a255..ea5085549 100644 --- a/src/repositories/projectsRepository.ts +++ b/src/repositories/projectsRepository.ts @@ -3,12 +3,13 @@ import { IConfig } from 'config'; import { BaseRepository } from './baseRepository'; import { ILogger } from '../services/logger'; -/** - * Project information from docs_metadata.projects for parser builds. - */ +//Project information from docs_metadata.projects for parser builds. + +const projectsCollectionName = process.env.PROJECTS_COL_NAME || 'projects'; + export class ProjectsRepository extends BaseRepository { - constructor(db: Db, config: IConfig, logger: ILogger) { - super(config, logger, 'ProjectsRepository', db.collection('projects')); + constructor(db: mongodb.Db, config: IConfig, logger: ILogger) { + super(config, logger, 'ProjectsRepository', db.collection(projectsCollectionName)); } async getProjectEntry(name: string): Promise | null> { From 70c67bdcdf1db611fac0eb23b02a3e559bc9ba45 Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Thu, 21 Mar 2024 12:22:23 -0400 Subject: [PATCH 179/254] DOP-4414 add docs_metadata as constant --- api/config/custom-environment-variables.json | 1 + api/config/default.json | 1 + cdk-infra/lib/constructs/worker/worker-env-construct.ts | 1 + cdk-infra/static/api/config/custom-environment-variables.json | 1 + cdk-infra/static/api/config/default.json | 1 + config/custom-environment-variables.json | 2 ++ src/app.ts | 2 +- 7 files changed, 8 insertions(+), 1 deletion(-) diff --git a/api/config/custom-environment-variables.json b/api/config/custom-environment-variables.json index 3e49afbd0..ee59dd577 100644 --- a/api/config/custom-environment-variables.json +++ b/api/config/custom-environment-variables.json @@ -7,6 +7,7 @@ "dbPassword": "MONGO_ATLAS_PASSWORD", "dbName": "DB_NAME", "snootyDbName": "SNOOTY_DB_NAME", + "metadataDbName": "METADATA_DB_NAME", "dbUrl": "MONGO_ATLAS_URL", "fastlyDochubMap": "FASTLY_DOCHUB_MAP", "githubSecret": "GITHUB_SECRET", diff --git a/api/config/default.json b/api/config/default.json index a7988e98c..66ee2ed48 100644 --- a/api/config/default.json +++ b/api/config/default.json @@ -7,6 +7,7 @@ "dbPassword": "password", "dbName": "db_name", "snootyDbName": "snooty_db_name", + "metadataDbName": "METADATA_DB_NAME", "dbUrl": "url", "fastlyDochubMap": "FASTLY_DOCHUB_MAP", "githubSecret": "GITHUB_SECRET", diff --git a/cdk-infra/lib/constructs/worker/worker-env-construct.ts b/cdk-infra/lib/constructs/worker/worker-env-construct.ts index 54d9737ac..e45065886 100644 --- a/cdk-infra/lib/constructs/worker/worker-env-construct.ts +++ b/cdk-infra/lib/constructs/worker/worker-env-construct.ts @@ -80,6 +80,7 @@ export class WorkerEnvConstruct extends Construct { MONGO_ATLAS_URL: `mongodb+srv://${dbUsername}:${dbPassword}@${dbHost}/admin?retryWrites=true`, DB_NAME: dbName, SNOOTY_DB_NAME: snootyDbName, + METADATA_DB_NAME: 'docs_metadata', JOBS_QUEUE_URL: jobsQueue.queueUrl, JOB_UPDATES_QUEUE_URL: jobUpdatesQueue.queueUrl, GITHUB_BOT_USERNAME: githubBotUsername, diff --git a/cdk-infra/static/api/config/custom-environment-variables.json b/cdk-infra/static/api/config/custom-environment-variables.json index cf328ded6..849fd6a7a 100644 --- a/cdk-infra/static/api/config/custom-environment-variables.json +++ b/cdk-infra/static/api/config/custom-environment-variables.json @@ -8,6 +8,7 @@ "githubBotPW": "GITHUB_BOT_PASSWORD", "dbName": "DB_NAME", "snootyDbName": "SNOOTY_DB_NAME", + "metadataDbName": "METADATA_DB_NAME", "dbUrl": "MONGO_ATLAS_URL", "fastlyDochubMap": "FASTLY_DOCHUB_MAP", "githubSecret": "GITHUB_SECRET", diff --git a/cdk-infra/static/api/config/default.json b/cdk-infra/static/api/config/default.json index 481f695d3..621fb830e 100644 --- a/cdk-infra/static/api/config/default.json +++ b/cdk-infra/static/api/config/default.json @@ -7,6 +7,7 @@ "dbPassword": "password", "dbName": "db_name", "snootyDbName": "snooty_db_name", + "metadataDbName": "METADATA_DB_NAME", "dbUrl": "url", "githubBotPW": "GITHUB_BOT_PASSWORD", "fastlyDochubMap": "FASTLY_DOCHUB_MAP", diff --git a/config/custom-environment-variables.json b/config/custom-environment-variables.json index 6093f03ee..73bd24144 100644 --- a/config/custom-environment-variables.json +++ b/config/custom-environment-variables.json @@ -6,8 +6,10 @@ "dbHost": "MONGO_ATLAS_HOST", "dbName": "DB_NAME", "snootyDbName": "SNOOTY_DB_NAME", + "metadataDbName": "METADATA_DB_NAME", "dbUrl": "MONGO_ATLAS_URL", "jobQueueCollection": "JOB_QUEUE_COL_NAME", + "projectsCollection": "PROJECTS_COL_NAME", "awsKey": "AWS_ACCESS_KEY_ID", "awsSecret": "AWS_SECRET_ACCESS_KEY", "githubBotUserName": "GITHUB_BOT_USERNAME", diff --git a/src/app.ts b/src/app.ts index 8edcd8023..8ecf93001 100644 --- a/src/app.ts +++ b/src/app.ts @@ -40,7 +40,7 @@ async function init(): Promise { )}/?retryWrites=true&w=majority`; const client = new mongodb.MongoClient(atlasURL); await client.connect(); - db = client.db(c.get('dbName')); + db = client.db(c.get('')); consoleLogger = new ConsoleLogger(); fileSystemServices = new FileSystemServices(); jobCommandExecutor = new JobSpecificCommandExecutor(); From 73ce79edef4382e7e0dee35361704af4da72d594 Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Mon, 25 Mar 2024 10:09:24 -0400 Subject: [PATCH 180/254] DOP-4414 remove unecessary vars from the configs --- api/config/custom-environment-variables.json | 2 -- api/config/default.json | 4 ---- api/controllers/v2/github.ts | 2 +- cdk-infra/static/api/config/custom-environment-variables.json | 2 -- cdk-infra/static/api/config/default.json | 2 -- config/custom-environment-variables.json | 2 -- config/test.json | 1 - 7 files changed, 1 insertion(+), 14 deletions(-) diff --git a/api/config/custom-environment-variables.json b/api/config/custom-environment-variables.json index ee59dd577..4199fbce2 100644 --- a/api/config/custom-environment-variables.json +++ b/api/config/custom-environment-variables.json @@ -7,7 +7,6 @@ "dbPassword": "MONGO_ATLAS_PASSWORD", "dbName": "DB_NAME", "snootyDbName": "SNOOTY_DB_NAME", - "metadataDbName": "METADATA_DB_NAME", "dbUrl": "MONGO_ATLAS_URL", "fastlyDochubMap": "FASTLY_DOCHUB_MAP", "githubSecret": "GITHUB_SECRET", @@ -18,7 +17,6 @@ "slackViewOpenUrl": "https://slack.com/api/views.open", "snootySecret": "SNOOTY_SECRET", "jobQueueCollection": "JOB_QUEUE_COL_NAME", - "projectsCollection": "PROJECTS_COL_NAME", "entitlementCollection": "USER_ENTITLEMENT_COL_NAME", "dashboardUrl": "DASHBOARD_URL", "repoBranchesCollection": "REPO_BRANCHES_COL_NAME", diff --git a/api/config/default.json b/api/config/default.json index 66ee2ed48..16d56ce57 100644 --- a/api/config/default.json +++ b/api/config/default.json @@ -7,7 +7,6 @@ "dbPassword": "password", "dbName": "db_name", "snootyDbName": "snooty_db_name", - "metadataDbName": "METADATA_DB_NAME", "dbUrl": "url", "fastlyDochubMap": "FASTLY_DOCHUB_MAP", "githubSecret": "GITHUB_SECRET", @@ -18,7 +17,6 @@ "slackViewOpenUrl": "https://slack.com/api/views.open", "snootySecret": "SNOOTY_SECRET", "jobQueueCollection": "JOB_QUEUE_COL_NAME", - "projectsCollection": "PROJECTS_COL_NAME", "entitlementCollection": "USER_ENTITLEMENT_COL_NAME", "repoBranchesCollection": "REPO_BRANCHES_COL_NAME", "docsetsCollection": "DOCSETS_COL_NAME", @@ -39,7 +37,6 @@ "stg": { "dbName": "pool_test", "jobQueueCollection": "dotcom_queue", - "projectsCollection": "PROJECTS_COL_NAME", "jobsQueueUrl": "https://sqs.us-east-2.amazonaws.com/216656347858/autobuilder-jobs-queue-dotcomstg", "jobUpdatesQueueUrl": "https://sqs.us-east-2.amazonaws.com/216656347858/autobuilder-job-updates-queue-dotcomstg", "parallelStgName": "dotcomstg" @@ -47,7 +44,6 @@ "prd": { "dbName": "pool", "jobQueueCollection": "dotcom_queue", - "projectsCollection": "PROJECTS_COL_NAME", "jobsQueueUrl": "https://sqs.us-east-2.amazonaws.com/216656347858/autobuilder-jobs-queue-dotcomprd", "jobUpdatesQueueUrl": "https://sqs.us-east-2.amazonaws.com/216656347858/autobuilder-job-updates-queue-dotcomprd", "parallelStgName": "dotcomprd" diff --git a/api/controllers/v2/github.ts b/api/controllers/v2/github.ts index 7c479ac29..db4880496 100644 --- a/api/controllers/v2/github.ts +++ b/api/controllers/v2/github.ts @@ -119,7 +119,7 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr const db = client.db(c.get('dbName')); const consoleLogger = new ConsoleLogger(); const jobRepository = new JobRepository(db, c, consoleLogger); - const projectsRepository = new ProjectsRepository(client.db('docs_metadata'), c, consoleLogger); + const projectsRepository = new ProjectsRepository(client.db(process.env.METADATA_DB_NAME), c, consoleLogger); const repoBranchesRepository = new RepoBranchesRepository(db, c, consoleLogger); const docsetsRepository = new DocsetsRepository(db, c, consoleLogger); diff --git a/cdk-infra/static/api/config/custom-environment-variables.json b/cdk-infra/static/api/config/custom-environment-variables.json index 849fd6a7a..8d554d1b5 100644 --- a/cdk-infra/static/api/config/custom-environment-variables.json +++ b/cdk-infra/static/api/config/custom-environment-variables.json @@ -8,7 +8,6 @@ "githubBotPW": "GITHUB_BOT_PASSWORD", "dbName": "DB_NAME", "snootyDbName": "SNOOTY_DB_NAME", - "metadataDbName": "METADATA_DB_NAME", "dbUrl": "MONGO_ATLAS_URL", "fastlyDochubMap": "FASTLY_DOCHUB_MAP", "githubSecret": "GITHUB_SECRET", @@ -18,7 +17,6 @@ "slackViewOpenUrl": "https://slack.com/api/views.open", "snootySecret": "SNOOTY_SECRET", "jobQueueCollection": "JOB_QUEUE_COL_NAME", - "projectsCollection": "PROJECTS_COL_NAME", "entitlementCollection": "USER_ENTITLEMENT_COL_NAME", "dashboardUrl": "DASHBOARD_URL", "repoBranchesCollection": "REPO_BRANCHES_COL_NAME", diff --git a/cdk-infra/static/api/config/default.json b/cdk-infra/static/api/config/default.json index 621fb830e..3758dea37 100644 --- a/cdk-infra/static/api/config/default.json +++ b/cdk-infra/static/api/config/default.json @@ -7,7 +7,6 @@ "dbPassword": "password", "dbName": "db_name", "snootyDbName": "snooty_db_name", - "metadataDbName": "METADATA_DB_NAME", "dbUrl": "url", "githubBotPW": "GITHUB_BOT_PASSWORD", "fastlyDochubMap": "FASTLY_DOCHUB_MAP", @@ -18,7 +17,6 @@ "slackViewOpenUrl": "https://slack.com/api/views.open", "snootySecret": "SNOOTY_SECRET", "jobQueueCollection": "JOB_QUEUE_COL_NAME", - "projectsCollection": "PROJECTS_COL_NAME", "entitlementCollection": "USER_ENTITLEMENT_COL_NAME", "repoBranchesCollection": "REPO_BRANCHES_COL_NAME", "docsetsCollection": "DOCSETS_COL_NAME", diff --git a/config/custom-environment-variables.json b/config/custom-environment-variables.json index 73bd24144..6093f03ee 100644 --- a/config/custom-environment-variables.json +++ b/config/custom-environment-variables.json @@ -6,10 +6,8 @@ "dbHost": "MONGO_ATLAS_HOST", "dbName": "DB_NAME", "snootyDbName": "SNOOTY_DB_NAME", - "metadataDbName": "METADATA_DB_NAME", "dbUrl": "MONGO_ATLAS_URL", "jobQueueCollection": "JOB_QUEUE_COL_NAME", - "projectsCollection": "PROJECTS_COL_NAME", "awsKey": "AWS_ACCESS_KEY_ID", "awsSecret": "AWS_SECRET_ACCESS_KEY", "githubBotUserName": "GITHUB_BOT_USERNAME", diff --git a/config/test.json b/config/test.json index aded7d904..5a17611ea 100644 --- a/config/test.json +++ b/config/test.json @@ -12,7 +12,6 @@ "githubBotPW": "q1w", "fastlyDochubMap": "devfslydochubmap", "entitlementCollection": "entitlements", - "projectsCollection": "projects", "docsetsCollection": "docsets", "MONGO_TIMEOUT_S": 1, "JOB_TIMEOUT_S": 10, From 2563973c905e6e86245b92599d7830605756c92c Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Mon, 25 Mar 2024 10:32:04 -0400 Subject: [PATCH 181/254] DOP-4414 readded name --- src/app.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/app.ts b/src/app.ts index 8ecf93001..8edcd8023 100644 --- a/src/app.ts +++ b/src/app.ts @@ -40,7 +40,7 @@ async function init(): Promise { )}/?retryWrites=true&w=majority`; const client = new mongodb.MongoClient(atlasURL); await client.connect(); - db = client.db(c.get('')); + db = client.db(c.get('dbName')); consoleLogger = new ConsoleLogger(); fileSystemServices = new FileSystemServices(); jobCommandExecutor = new JobSpecificCommandExecutor(); From bfb75a9b7a5179ca3c82db25f86ca1f32c967f84 Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Mon, 25 Mar 2024 10:46:29 -0400 Subject: [PATCH 182/254] DOP-4414 testing --- api/controllers/v2/github.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/controllers/v2/github.ts b/api/controllers/v2/github.ts index db4880496..7c479ac29 100644 --- a/api/controllers/v2/github.ts +++ b/api/controllers/v2/github.ts @@ -119,7 +119,7 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr const db = client.db(c.get('dbName')); const consoleLogger = new ConsoleLogger(); const jobRepository = new JobRepository(db, c, consoleLogger); - const projectsRepository = new ProjectsRepository(client.db(process.env.METADATA_DB_NAME), c, consoleLogger); + const projectsRepository = new ProjectsRepository(client.db('docs_metadata'), c, consoleLogger); const repoBranchesRepository = new RepoBranchesRepository(db, c, consoleLogger); const docsetsRepository = new DocsetsRepository(db, c, consoleLogger); From b48eff15795d8794195f7dbca3c05352b4f717f5 Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Mon, 25 Mar 2024 11:12:34 -0400 Subject: [PATCH 183/254] DOP-4414 testing env vars --- api/controllers/v2/github.ts | 2 +- infrastructure/ecs-main/ecs_service.yml | 2 ++ serverless.yml | 2 ++ src/repositories/projectsRepository.ts | 2 +- 4 files changed, 6 insertions(+), 2 deletions(-) diff --git a/api/controllers/v2/github.ts b/api/controllers/v2/github.ts index 7c479ac29..db4880496 100644 --- a/api/controllers/v2/github.ts +++ b/api/controllers/v2/github.ts @@ -119,7 +119,7 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr const db = client.db(c.get('dbName')); const consoleLogger = new ConsoleLogger(); const jobRepository = new JobRepository(db, c, consoleLogger); - const projectsRepository = new ProjectsRepository(client.db('docs_metadata'), c, consoleLogger); + const projectsRepository = new ProjectsRepository(client.db(process.env.METADATA_DB_NAME), c, consoleLogger); const repoBranchesRepository = new RepoBranchesRepository(db, c, consoleLogger); const docsetsRepository = new DocsetsRepository(db, c, consoleLogger); diff --git a/infrastructure/ecs-main/ecs_service.yml b/infrastructure/ecs-main/ecs_service.yml index 2fdad6e1d..1bb6a5333 100644 --- a/infrastructure/ecs-main/ecs_service.yml +++ b/infrastructure/ecs-main/ecs_service.yml @@ -50,6 +50,8 @@ Resources: Value: ${self:custom.repoBranchesCollection} - Name: DOCSETS_COL_NAME Value: ${self:custom.docsetsCollection} + - Name: PROJECTS_COL_NAME + Value: ${self:custom.projectsCollection} - Name: NPM_BASE_64_AUTH Value: ${self:custom.npmBase64Auth} - Name: NPM_EMAIL diff --git a/serverless.yml b/serverless.yml index 66a398034..3299feee3 100644 --- a/serverless.yml +++ b/serverless.yml @@ -57,6 +57,7 @@ custom: entitlementCollection: ${ssm:/env/${self:provider.stage}/docs/worker_pool/atlas/collections/user/entitlements} repoBranchesCollection: ${ssm:/env/${self:provider.stage}/docs/worker_pool/atlas/collections/repo} docsetsCollection: ${ssm:/env/${self:provider.stage}/docs/worker_pool/atlas/collections/docsets} + projectsCollection: ${ssm:/env/${self:provider.stage}/docs/worker_pool/atlas/collections/projects} githubSecret: ${ssm:/env/${self:provider.stage}/docs/worker_pool/github/webhook/secret} githubDeletionSecret: ${ssm:/env/${self:provider.stage}/docs/worker_pool/github/webhook/deletionSecret} githubBotPW: ${ssm:/env/${self:provider.stage}/docs/worker_pool/github/bot/password} @@ -110,6 +111,7 @@ webhook-env-core: &webhook-env-core GITHUB_BOT_PASSWORD: ${self:custom.githubBotPW} REPO_BRANCHES_COL_NAME: ${self:custom.repoBranchesCollection} DOCSETS_COL_NAME: ${self:custom.docsetsCollection} + PROJECTS_COL_NAME: ${self:custom.projectsCollection} SLACK_SECRET: ${self:custom.slackSecret} SLACK_TOKEN: ${self:custom.slackAuthToken} SNOOTY_SECRET: ${self:custom.snootySecret} diff --git a/src/repositories/projectsRepository.ts b/src/repositories/projectsRepository.ts index ea5085549..f51912835 100644 --- a/src/repositories/projectsRepository.ts +++ b/src/repositories/projectsRepository.ts @@ -5,7 +5,7 @@ import { ILogger } from '../services/logger'; //Project information from docs_metadata.projects for parser builds. -const projectsCollectionName = process.env.PROJECTS_COL_NAME || 'projects'; +const projectsCollectionName = process.env.PROJECTS_COL_NAME || ''; export class ProjectsRepository extends BaseRepository { constructor(db: mongodb.Db, config: IConfig, logger: ILogger) { From b0fcb258d86eddb6e730efb68dbc3eb88c22ba6f Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Mon, 25 Mar 2024 11:18:41 -0400 Subject: [PATCH 184/254] DOP-4414 testing env vars --- api/config/custom-environment-variables.json | 1 + cdk-infra/static/api/config/custom-environment-variables.json | 1 + config/custom-environment-variables.json | 4 +++- src/commands/src/scripts/local-build/utils/get-env-vars.ts | 2 ++ 4 files changed, 7 insertions(+), 1 deletion(-) diff --git a/api/config/custom-environment-variables.json b/api/config/custom-environment-variables.json index 4199fbce2..3937e8174 100644 --- a/api/config/custom-environment-variables.json +++ b/api/config/custom-environment-variables.json @@ -28,5 +28,6 @@ "cdnClientSecret": "CDN_CLIENT_SECRET", "cdnInvalidatorServiceURL": "CDN_INVALIDATOR_SERVICE_URL", "searchIndexBucket": "SEARCH_INDEX_BUCKET", + "projectsCollection": "PROJECTS_COL_NAME", "searchIndexFolder": "SEARCH_INDEX_FOLDER" } diff --git a/cdk-infra/static/api/config/custom-environment-variables.json b/cdk-infra/static/api/config/custom-environment-variables.json index 8d554d1b5..ef39f1c7a 100644 --- a/cdk-infra/static/api/config/custom-environment-variables.json +++ b/cdk-infra/static/api/config/custom-environment-variables.json @@ -28,5 +28,6 @@ "cdnClientSecret": "CDN_CLIENT_SECRET", "cdnInvalidatorServiceURL": "CDN_INVALIDATOR_SERVICE_URL", "searchIndexBucket": "SEARCH_INDEX_BUCKET", + "projectsCollection": "PROJECTS_COL_NAME", "searchIndexFolder": "SEARCH_INDEX_FOLDER" } diff --git a/config/custom-environment-variables.json b/config/custom-environment-variables.json index 6093f03ee..bd40d2a4b 100644 --- a/config/custom-environment-variables.json +++ b/config/custom-environment-variables.json @@ -35,5 +35,7 @@ "cdnClientID": "CDN_CLIENT_ID", "cdnClientSecret": "CDN_CLIENT_SECRET", "cdnInvalidatorServiceURL": "CDN_INVALIDATOR_SERVICE_URL", - "searchIndexBucket": "SEARCH_INDEX_BUCKET" + "searchIndexBucket": "SEARCH_INDEX_BUCKET", + "projectsCollection": "PROJECTS_COL_NAME" + } diff --git a/src/commands/src/scripts/local-build/utils/get-env-vars.ts b/src/commands/src/scripts/local-build/utils/get-env-vars.ts index dcc3aef46..e151ee08e 100644 --- a/src/commands/src/scripts/local-build/utils/get-env-vars.ts +++ b/src/commands/src/scripts/local-build/utils/get-env-vars.ts @@ -37,6 +37,7 @@ const workerParamStrings = [ '/flag/monorepo_path', '/atlas/collections/repo', '/atlas/collections/docsets', + '/atlas/collections/projects', '/atlas/collections/job/queue', '/atlas/collections/user/entitlements', ] as const; @@ -61,6 +62,7 @@ const workerParamPathToEnvName: Record = { '/cdn/client/id': 'CDN_CLIENT_ID', '/cdn/client/secret': 'CDN_CLIENT_SECRET', '/atlas/collections/docsets': 'DOCSETS_COL_NAME', + '/atlas/collections/projects': 'PROJECTS_COL_NAME', '/atlas/dbname': 'DB_NAME', '/atlas/collections/snooty': 'SNOOTY_DB_NAME', '/atlas/username': 'MONGO_ATLAS_USERNAME', From 98be6891d3922132e84d60077aad0724b83b06fd Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Mon, 25 Mar 2024 11:33:26 -0400 Subject: [PATCH 185/254] DOP-4414 testing env vars --- api/config/default.json | 1 + cdk-infra/static/api/config/default.json | 1 + config/custom-environment-variables.json | 1 - src/repositories/projectsRepository.ts | 6 +++--- 4 files changed, 5 insertions(+), 4 deletions(-) diff --git a/api/config/default.json b/api/config/default.json index 16d56ce57..2b9121ae1 100644 --- a/api/config/default.json +++ b/api/config/default.json @@ -20,6 +20,7 @@ "entitlementCollection": "USER_ENTITLEMENT_COL_NAME", "repoBranchesCollection": "REPO_BRANCHES_COL_NAME", "docsetsCollection": "DOCSETS_COL_NAME", + "projectsCollection": "PROJECTS_COL_NAME", "MONGO_TIMEOUT_S": 15, "JOB_TIMEOUT_S": 900, "RETRY_TIMEOUT_MS": 5000, diff --git a/cdk-infra/static/api/config/default.json b/cdk-infra/static/api/config/default.json index 3758dea37..28d596fe5 100644 --- a/cdk-infra/static/api/config/default.json +++ b/cdk-infra/static/api/config/default.json @@ -20,6 +20,7 @@ "entitlementCollection": "USER_ENTITLEMENT_COL_NAME", "repoBranchesCollection": "REPO_BRANCHES_COL_NAME", "docsetsCollection": "DOCSETS_COL_NAME", + "projectsCollection": "PROJECTS_COL_NAME", "MONGO_TIMEOUT_S": 15, "JOB_TIMEOUT_S": 900, "RETRY_TIMEOUT_MS": 5000, diff --git a/config/custom-environment-variables.json b/config/custom-environment-variables.json index bd40d2a4b..58b008966 100644 --- a/config/custom-environment-variables.json +++ b/config/custom-environment-variables.json @@ -37,5 +37,4 @@ "cdnInvalidatorServiceURL": "CDN_INVALIDATOR_SERVICE_URL", "searchIndexBucket": "SEARCH_INDEX_BUCKET", "projectsCollection": "PROJECTS_COL_NAME" - } diff --git a/src/repositories/projectsRepository.ts b/src/repositories/projectsRepository.ts index f51912835..0e3e0df0d 100644 --- a/src/repositories/projectsRepository.ts +++ b/src/repositories/projectsRepository.ts @@ -1,15 +1,15 @@ import mongodb, { Db } from 'mongodb'; -import { IConfig } from 'config'; +import c, { IConfig } from 'config'; import { BaseRepository } from './baseRepository'; import { ILogger } from '../services/logger'; //Project information from docs_metadata.projects for parser builds. -const projectsCollectionName = process.env.PROJECTS_COL_NAME || ''; +// const projectsCollectionName = config.get('projectsCollection') || ''; export class ProjectsRepository extends BaseRepository { constructor(db: mongodb.Db, config: IConfig, logger: ILogger) { - super(config, logger, 'ProjectsRepository', db.collection(projectsCollectionName)); + super(config, logger, 'ProjectsRepository', db.collection(config.get('projectsCollection'))); } async getProjectEntry(name: string): Promise | null> { From 7a85e8bf4b013e0424b8c220335d2fa3c5e9ef98 Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Mon, 25 Mar 2024 11:55:41 -0400 Subject: [PATCH 186/254] DOP-4414 testing env vars --- cdk-infra/lib/constructs/api/webhook-env-construct.ts | 4 +++- cdk-infra/lib/constructs/worker/worker-env-construct.ts | 3 ++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/cdk-infra/lib/constructs/api/webhook-env-construct.ts b/cdk-infra/lib/constructs/api/webhook-env-construct.ts index fd8f9cd01..adebf5d8f 100644 --- a/cdk-infra/lib/constructs/api/webhook-env-construct.ts +++ b/cdk-infra/lib/constructs/api/webhook-env-construct.ts @@ -27,6 +27,7 @@ export class WebhookEnvConstruct extends Construct { const dbUsername = StringParameter.valueFromLookup(this, `${ssmPrefix}/atlas/username`); const dbHost = StringParameter.valueFromLookup(this, `${ssmPrefix}/atlas/host`); const jobCollection = StringParameter.valueFromLookup(this, `${ssmPrefix}/atlas/collections/job/queue`); + const projectsCollection = 'projects'; const entitlementCollection = StringParameter.valueFromLookup( this, `${ssmPrefix}/atlas/collections/user/entitlements` @@ -40,9 +41,10 @@ export class WebhookEnvConstruct extends Construct { MONGO_ATLAS_URL: `mongodb+srv://${dbUsername}:${dbPassword}@${dbHost}/admin?retryWrites=true`, DB_NAME: dbName, SNOOTY_DB_NAME: snootyDbName, + METADATA_DB_NAME: 'docs_metadata', REPO_BRANCHES_COL_NAME: repoBranchesCollection, DOCSETS_COL_NAME: docsetsCollection, - PROJECTS_COL_NAME: 'projects', + PROJECTS_COL_NAME: projectsCollection, JOB_QUEUE_COL_NAME: jobCollection, NODE_CONFIG_DIR: './config', JOBS_QUEUE_URL: jobsQueue.queueUrl, diff --git a/cdk-infra/lib/constructs/worker/worker-env-construct.ts b/cdk-infra/lib/constructs/worker/worker-env-construct.ts index e45065886..103019d37 100644 --- a/cdk-infra/lib/constructs/worker/worker-env-construct.ts +++ b/cdk-infra/lib/constructs/worker/worker-env-construct.ts @@ -67,6 +67,7 @@ export class WorkerEnvConstruct extends Construct { const docsetsCollection = StringParameter.valueFromLookup(this, `${ssmPrefix}/atlas/collections/docsets`); const jobCollection = StringParameter.valueFromLookup(this, `${ssmPrefix}/atlas/collections/job/queue`); const gatsbyMarianUrl = StringParameter.valueFromLookup(this, `${ssmPrefix}/frontend/marian_url`); + const projectsCollection = 'projects'; const dbPassword = secureStrings['MONGO_ATLAS_PASSWORD']; this.environment = { @@ -93,7 +94,7 @@ export class WorkerEnvConstruct extends Construct { REPO_BRANCHES_COL_NAME: repoBranchesCollection, DOCSETS_COL_NAME: docsetsCollection, JOB_QUEUE_COL_NAME: jobCollection, - PROJECTS_COL_NAME: 'projects', + PROJECTS_COL_NAME: projectsCollection, CDN_INVALIDATOR_SERVICE_URL: getCdnInvalidatorUrl(env), SEARCH_INDEX_BUCKET: 'docs-search-indexes-test', SEARCH_INDEX_FOLDER: getSearchIndexFolder(env), From aaf71e9520fc46d4e7baa0ae3d9f83f3d7fe6acd Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Mon, 25 Mar 2024 12:03:32 -0400 Subject: [PATCH 187/254] DOP-4414 testing env vars --- src/repositories/projectsRepository.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/repositories/projectsRepository.ts b/src/repositories/projectsRepository.ts index 0e3e0df0d..475a9981d 100644 --- a/src/repositories/projectsRepository.ts +++ b/src/repositories/projectsRepository.ts @@ -9,7 +9,7 @@ import { ILogger } from '../services/logger'; export class ProjectsRepository extends BaseRepository { constructor(db: mongodb.Db, config: IConfig, logger: ILogger) { - super(config, logger, 'ProjectsRepository', db.collection(config.get('projectsCollection'))); + super(config, logger, 'ProjectsRepository', db.collection(process.env.PROJECTS_COL_NAME || '')); } async getProjectEntry(name: string): Promise | null> { From d0071fd1644c059af02f72e26e8ca26527ebde78 Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Mon, 25 Mar 2024 12:27:11 -0400 Subject: [PATCH 188/254] DOP-4414 testing env vars removing from yamls --- infrastructure/ecs-main/ecs_service.yml | 4 ++-- serverless.yml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/infrastructure/ecs-main/ecs_service.yml b/infrastructure/ecs-main/ecs_service.yml index 1bb6a5333..b7841fccd 100644 --- a/infrastructure/ecs-main/ecs_service.yml +++ b/infrastructure/ecs-main/ecs_service.yml @@ -50,8 +50,8 @@ Resources: Value: ${self:custom.repoBranchesCollection} - Name: DOCSETS_COL_NAME Value: ${self:custom.docsetsCollection} - - Name: PROJECTS_COL_NAME - Value: ${self:custom.projectsCollection} + # - Name: PROJECTS_COL_NAME + # Value: ${self:custom.projectsCollection} - Name: NPM_BASE_64_AUTH Value: ${self:custom.npmBase64Auth} - Name: NPM_EMAIL diff --git a/serverless.yml b/serverless.yml index 3299feee3..a6417904e 100644 --- a/serverless.yml +++ b/serverless.yml @@ -111,7 +111,7 @@ webhook-env-core: &webhook-env-core GITHUB_BOT_PASSWORD: ${self:custom.githubBotPW} REPO_BRANCHES_COL_NAME: ${self:custom.repoBranchesCollection} DOCSETS_COL_NAME: ${self:custom.docsetsCollection} - PROJECTS_COL_NAME: ${self:custom.projectsCollection} + # PROJECTS_COL_NAME: ${self:custom.projectsCollection} SLACK_SECRET: ${self:custom.slackSecret} SLACK_TOKEN: ${self:custom.slackAuthToken} SNOOTY_SECRET: ${self:custom.snootySecret} From 2c71ab298596092c75aa1a0fc306383159bec1ab Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Mon, 25 Mar 2024 13:06:36 -0400 Subject: [PATCH 189/254] DOP-4414 testing env vars removing adding back to yamll --- serverless.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/serverless.yml b/serverless.yml index a6417904e..3299feee3 100644 --- a/serverless.yml +++ b/serverless.yml @@ -111,7 +111,7 @@ webhook-env-core: &webhook-env-core GITHUB_BOT_PASSWORD: ${self:custom.githubBotPW} REPO_BRANCHES_COL_NAME: ${self:custom.repoBranchesCollection} DOCSETS_COL_NAME: ${self:custom.docsetsCollection} - # PROJECTS_COL_NAME: ${self:custom.projectsCollection} + PROJECTS_COL_NAME: ${self:custom.projectsCollection} SLACK_SECRET: ${self:custom.slackSecret} SLACK_TOKEN: ${self:custom.slackAuthToken} SNOOTY_SECRET: ${self:custom.snootySecret} From 7a62fc3ed595803382b4910fbeedfd0aa0e4e042 Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Mon, 25 Mar 2024 13:08:09 -0400 Subject: [PATCH 190/254] DOP-4414 testing env vars removing adding back to yamll --- serverless.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/serverless.yml b/serverless.yml index 3299feee3..d7fbd0af1 100644 --- a/serverless.yml +++ b/serverless.yml @@ -57,7 +57,7 @@ custom: entitlementCollection: ${ssm:/env/${self:provider.stage}/docs/worker_pool/atlas/collections/user/entitlements} repoBranchesCollection: ${ssm:/env/${self:provider.stage}/docs/worker_pool/atlas/collections/repo} docsetsCollection: ${ssm:/env/${self:provider.stage}/docs/worker_pool/atlas/collections/docsets} - projectsCollection: ${ssm:/env/${self:provider.stage}/docs/worker_pool/atlas/collections/projects} + # projectsCollection: ${ssm:/env/${self:provider.stage}/docs/worker_pool/atlas/collections/projects} githubSecret: ${ssm:/env/${self:provider.stage}/docs/worker_pool/github/webhook/secret} githubDeletionSecret: ${ssm:/env/${self:provider.stage}/docs/worker_pool/github/webhook/deletionSecret} githubBotPW: ${ssm:/env/${self:provider.stage}/docs/worker_pool/github/bot/password} @@ -111,7 +111,7 @@ webhook-env-core: &webhook-env-core GITHUB_BOT_PASSWORD: ${self:custom.githubBotPW} REPO_BRANCHES_COL_NAME: ${self:custom.repoBranchesCollection} DOCSETS_COL_NAME: ${self:custom.docsetsCollection} - PROJECTS_COL_NAME: ${self:custom.projectsCollection} + # PROJECTS_COL_NAME: ${self:custom.projectsCollection} SLACK_SECRET: ${self:custom.slackSecret} SLACK_TOKEN: ${self:custom.slackAuthToken} SNOOTY_SECRET: ${self:custom.snootySecret} From cde1bdd04a93bbf509d77f5534e7e0961e1652cb Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Mon, 25 Mar 2024 13:29:25 -0400 Subject: [PATCH 191/254] DOP-4414 testing env vars removing from get-env-vars --- src/commands/src/scripts/local-build/utils/get-env-vars.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/commands/src/scripts/local-build/utils/get-env-vars.ts b/src/commands/src/scripts/local-build/utils/get-env-vars.ts index e151ee08e..8c53cd406 100644 --- a/src/commands/src/scripts/local-build/utils/get-env-vars.ts +++ b/src/commands/src/scripts/local-build/utils/get-env-vars.ts @@ -62,7 +62,7 @@ const workerParamPathToEnvName: Record = { '/cdn/client/id': 'CDN_CLIENT_ID', '/cdn/client/secret': 'CDN_CLIENT_SECRET', '/atlas/collections/docsets': 'DOCSETS_COL_NAME', - '/atlas/collections/projects': 'PROJECTS_COL_NAME', + // '/atlas/collections/projects': 'PROJECTS_COL_NAME', '/atlas/dbname': 'DB_NAME', '/atlas/collections/snooty': 'SNOOTY_DB_NAME', '/atlas/username': 'MONGO_ATLAS_USERNAME', From 88def819ed5f064a168d9b210c5ee41dfcfefa14 Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Mon, 25 Mar 2024 13:59:43 -0400 Subject: [PATCH 192/254] DOP-4414 testing env vars removing from get-env-vars --- src/commands/src/scripts/local-build/utils/get-env-vars.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/commands/src/scripts/local-build/utils/get-env-vars.ts b/src/commands/src/scripts/local-build/utils/get-env-vars.ts index 8c53cd406..d3f253bff 100644 --- a/src/commands/src/scripts/local-build/utils/get-env-vars.ts +++ b/src/commands/src/scripts/local-build/utils/get-env-vars.ts @@ -37,7 +37,7 @@ const workerParamStrings = [ '/flag/monorepo_path', '/atlas/collections/repo', '/atlas/collections/docsets', - '/atlas/collections/projects', + // '/atlas/collections/projects', '/atlas/collections/job/queue', '/atlas/collections/user/entitlements', ] as const; From e6a29065c7264309b114344d2037fbf3af5edb12 Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Mon, 25 Mar 2024 14:25:04 -0400 Subject: [PATCH 193/254] DOP-4414 testing env vars removing jsons --- api/config/custom-environment-variables.json | 1 - api/config/default.json | 1 - 2 files changed, 2 deletions(-) diff --git a/api/config/custom-environment-variables.json b/api/config/custom-environment-variables.json index 3937e8174..4199fbce2 100644 --- a/api/config/custom-environment-variables.json +++ b/api/config/custom-environment-variables.json @@ -28,6 +28,5 @@ "cdnClientSecret": "CDN_CLIENT_SECRET", "cdnInvalidatorServiceURL": "CDN_INVALIDATOR_SERVICE_URL", "searchIndexBucket": "SEARCH_INDEX_BUCKET", - "projectsCollection": "PROJECTS_COL_NAME", "searchIndexFolder": "SEARCH_INDEX_FOLDER" } diff --git a/api/config/default.json b/api/config/default.json index 2b9121ae1..16d56ce57 100644 --- a/api/config/default.json +++ b/api/config/default.json @@ -20,7 +20,6 @@ "entitlementCollection": "USER_ENTITLEMENT_COL_NAME", "repoBranchesCollection": "REPO_BRANCHES_COL_NAME", "docsetsCollection": "DOCSETS_COL_NAME", - "projectsCollection": "PROJECTS_COL_NAME", "MONGO_TIMEOUT_S": 15, "JOB_TIMEOUT_S": 900, "RETRY_TIMEOUT_MS": 5000, From 2f6b1a0eeab2e34e3de2023d1bccaad754a418e3 Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Mon, 25 Mar 2024 15:01:20 -0400 Subject: [PATCH 194/254] DOP-4414 ssm putparam --- cdk-infra/lib/constructs/worker/worker-construct.ts | 2 +- src/commands/src/shared/next-gen-deploy.ts | 2 +- src/job/productionJobHandler.ts | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cdk-infra/lib/constructs/worker/worker-construct.ts b/cdk-infra/lib/constructs/worker/worker-construct.ts index 8c00607dc..63f550305 100644 --- a/cdk-infra/lib/constructs/worker/worker-construct.ts +++ b/cdk-infra/lib/constructs/worker/worker-construct.ts @@ -39,7 +39,7 @@ export class WorkerConstruct extends Construct { const taskRoleSsmPolicyStatement = new PolicyStatement({ effect: Effect.ALLOW, - actions: ['ssm:GetParameter'], + actions: ['ssm:GetParameter', 'ssm:PutParameter'], resources: ['*'], }); diff --git a/src/commands/src/shared/next-gen-deploy.ts b/src/commands/src/shared/next-gen-deploy.ts index 5d86baf73..d41cac516 100644 --- a/src/commands/src/shared/next-gen-deploy.ts +++ b/src/commands/src/shared/next-gen-deploy.ts @@ -48,7 +48,7 @@ export async function nextGenDeploy({ console.log( `COMMAND: yes | mut-publish public ${bucket} --prefix=${mutPrefix} --deploy --deployed-url-prefix=${url} --json --all-subdirectories --dry-run` ); - console.log(`${outputText}\n Hosted at ${url}/${mutPrefix}`); + console.log(`${outputText}\n Hosted at ${url}${mutPrefix}`); return { status: CommandExecutorResponseStatus.success, output: outputText, diff --git a/src/job/productionJobHandler.ts b/src/job/productionJobHandler.ts index a7ed1ac28..6190cc018 100644 --- a/src/job/productionJobHandler.ts +++ b/src/job/productionJobHandler.ts @@ -181,7 +181,7 @@ export class ProductionJobHandler extends JobHandler { await this.jobRepository.insertInvalidationRequestStatusUrl(this.currJob._id, 'Invalidation Failed'); } } catch (error) { - await this.logger.save(this.currJob._id, error); + await this.logger.save(this.currJob._id, error.message); } } From ea685a6c6e67be9b207f9e4aec05b790b7bd7def Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Mon, 25 Mar 2024 15:28:27 -0400 Subject: [PATCH 195/254] DOP-4414 removing env vars --- cdk-infra/static/api/config/custom-environment-variables.json | 1 - config/custom-environment-variables.json | 1 - 2 files changed, 2 deletions(-) diff --git a/cdk-infra/static/api/config/custom-environment-variables.json b/cdk-infra/static/api/config/custom-environment-variables.json index ef39f1c7a..8d554d1b5 100644 --- a/cdk-infra/static/api/config/custom-environment-variables.json +++ b/cdk-infra/static/api/config/custom-environment-variables.json @@ -28,6 +28,5 @@ "cdnClientSecret": "CDN_CLIENT_SECRET", "cdnInvalidatorServiceURL": "CDN_INVALIDATOR_SERVICE_URL", "searchIndexBucket": "SEARCH_INDEX_BUCKET", - "projectsCollection": "PROJECTS_COL_NAME", "searchIndexFolder": "SEARCH_INDEX_FOLDER" } diff --git a/config/custom-environment-variables.json b/config/custom-environment-variables.json index 58b008966..b6ce6e7e6 100644 --- a/config/custom-environment-variables.json +++ b/config/custom-environment-variables.json @@ -36,5 +36,4 @@ "cdnClientSecret": "CDN_CLIENT_SECRET", "cdnInvalidatorServiceURL": "CDN_INVALIDATOR_SERVICE_URL", "searchIndexBucket": "SEARCH_INDEX_BUCKET", - "projectsCollection": "PROJECTS_COL_NAME" } From 7d1a06f874b6a961845b188ab4622ab390447f80 Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Mon, 25 Mar 2024 15:30:44 -0400 Subject: [PATCH 196/254] DOP-4414 removing env vars --- config/custom-environment-variables.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/custom-environment-variables.json b/config/custom-environment-variables.json index b6ce6e7e6..6093f03ee 100644 --- a/config/custom-environment-variables.json +++ b/config/custom-environment-variables.json @@ -35,5 +35,5 @@ "cdnClientID": "CDN_CLIENT_ID", "cdnClientSecret": "CDN_CLIENT_SECRET", "cdnInvalidatorServiceURL": "CDN_INVALIDATOR_SERVICE_URL", - "searchIndexBucket": "SEARCH_INDEX_BUCKET", + "searchIndexBucket": "SEARCH_INDEX_BUCKET" } From 8e39c0b10cc169ba69a336bbacdd1af434ec397c Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Mon, 25 Mar 2024 15:51:24 -0400 Subject: [PATCH 197/254] DOP-4414 removing more env vars --- api/controllers/v2/github.ts | 9 +++------ cdk-infra/static/api/config/default.json | 1 - config/default.json | 1 - 3 files changed, 3 insertions(+), 8 deletions(-) diff --git a/api/controllers/v2/github.ts b/api/controllers/v2/github.ts index db4880496..372d136f6 100644 --- a/api/controllers/v2/github.ts +++ b/api/controllers/v2/github.ts @@ -158,17 +158,14 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr return { statusCode: 202, headers: { 'Content-Type': 'text/plain' }, - body: `Build on branch - ${body.workflow_run.head_branch} - is not complete and will not trigger smoke test site deployments`, + body: `Build on branch ${body.workflow_run.head_branch} is not complete and will not trigger smoke test site deployments`, }; if (body.workflow_run.name != 'Deploy Staging ECS') return { statusCode: 202, headers: { 'Content-Type': 'text/plain' }, - body: `Workflow - ${body.workflow_run.name} completed successfully but only Deploy Staging ECS workflow completion will trigger smoke test site deployments`, + body: `Workflow ${body.workflow_run.name} completed successfully but only Deploy Staging ECS workflow completion will trigger smoke test site deployments`, }; // if the build was not building main branch, no need for smoke test sites @@ -244,7 +241,7 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr return { statusCode: 202, headers: { 'Content-Type': 'text/plain' }, - body: 'Smoke Test Jobs Queued with the following Job Ids' + returnVal, + body: 'Smoke Test Jobs Queued with the following Job Ids ' + returnVal, }; }; diff --git a/cdk-infra/static/api/config/default.json b/cdk-infra/static/api/config/default.json index 28d596fe5..3758dea37 100644 --- a/cdk-infra/static/api/config/default.json +++ b/cdk-infra/static/api/config/default.json @@ -20,7 +20,6 @@ "entitlementCollection": "USER_ENTITLEMENT_COL_NAME", "repoBranchesCollection": "REPO_BRANCHES_COL_NAME", "docsetsCollection": "DOCSETS_COL_NAME", - "projectsCollection": "PROJECTS_COL_NAME", "MONGO_TIMEOUT_S": 15, "JOB_TIMEOUT_S": 900, "RETRY_TIMEOUT_MS": 5000, diff --git a/config/default.json b/config/default.json index ae9b5f9a7..8a089b421 100644 --- a/config/default.json +++ b/config/default.json @@ -12,7 +12,6 @@ "githubBotPW": "q1w", "fastlyDochubMap": "devfslydochubmap", "entitlementCollection": "entitlements", - "projectsCollection": "projects", "docsetsCollection": "docsets", "MONGO_TIMEOUT_S": 1500, "JOB_TIMEOUT_S": 900, From 0838c014c30588768a45872d753e94c3f849bf44 Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Mon, 25 Mar 2024 16:10:49 -0400 Subject: [PATCH 198/254] DOP-4414 removing more env vars --- cdk-infra/lib/constructs/api/webhook-env-construct.ts | 3 +-- cdk-infra/lib/constructs/worker/worker-env-construct.ts | 3 +-- infrastructure/ecs-main/ecs_service.yml | 2 -- serverless.yml | 1 - src/commands/src/scripts/local-build/utils/get-env-vars.ts | 1 - 5 files changed, 2 insertions(+), 8 deletions(-) diff --git a/cdk-infra/lib/constructs/api/webhook-env-construct.ts b/cdk-infra/lib/constructs/api/webhook-env-construct.ts index adebf5d8f..0c53cb4a7 100644 --- a/cdk-infra/lib/constructs/api/webhook-env-construct.ts +++ b/cdk-infra/lib/constructs/api/webhook-env-construct.ts @@ -27,7 +27,6 @@ export class WebhookEnvConstruct extends Construct { const dbUsername = StringParameter.valueFromLookup(this, `${ssmPrefix}/atlas/username`); const dbHost = StringParameter.valueFromLookup(this, `${ssmPrefix}/atlas/host`); const jobCollection = StringParameter.valueFromLookup(this, `${ssmPrefix}/atlas/collections/job/queue`); - const projectsCollection = 'projects'; const entitlementCollection = StringParameter.valueFromLookup( this, `${ssmPrefix}/atlas/collections/user/entitlements` @@ -44,7 +43,7 @@ export class WebhookEnvConstruct extends Construct { METADATA_DB_NAME: 'docs_metadata', REPO_BRANCHES_COL_NAME: repoBranchesCollection, DOCSETS_COL_NAME: docsetsCollection, - PROJECTS_COL_NAME: projectsCollection, + PROJECTS_COL_NAME: 'projects', JOB_QUEUE_COL_NAME: jobCollection, NODE_CONFIG_DIR: './config', JOBS_QUEUE_URL: jobsQueue.queueUrl, diff --git a/cdk-infra/lib/constructs/worker/worker-env-construct.ts b/cdk-infra/lib/constructs/worker/worker-env-construct.ts index 103019d37..e45065886 100644 --- a/cdk-infra/lib/constructs/worker/worker-env-construct.ts +++ b/cdk-infra/lib/constructs/worker/worker-env-construct.ts @@ -67,7 +67,6 @@ export class WorkerEnvConstruct extends Construct { const docsetsCollection = StringParameter.valueFromLookup(this, `${ssmPrefix}/atlas/collections/docsets`); const jobCollection = StringParameter.valueFromLookup(this, `${ssmPrefix}/atlas/collections/job/queue`); const gatsbyMarianUrl = StringParameter.valueFromLookup(this, `${ssmPrefix}/frontend/marian_url`); - const projectsCollection = 'projects'; const dbPassword = secureStrings['MONGO_ATLAS_PASSWORD']; this.environment = { @@ -94,7 +93,7 @@ export class WorkerEnvConstruct extends Construct { REPO_BRANCHES_COL_NAME: repoBranchesCollection, DOCSETS_COL_NAME: docsetsCollection, JOB_QUEUE_COL_NAME: jobCollection, - PROJECTS_COL_NAME: projectsCollection, + PROJECTS_COL_NAME: 'projects', CDN_INVALIDATOR_SERVICE_URL: getCdnInvalidatorUrl(env), SEARCH_INDEX_BUCKET: 'docs-search-indexes-test', SEARCH_INDEX_FOLDER: getSearchIndexFolder(env), diff --git a/infrastructure/ecs-main/ecs_service.yml b/infrastructure/ecs-main/ecs_service.yml index b7841fccd..2fdad6e1d 100644 --- a/infrastructure/ecs-main/ecs_service.yml +++ b/infrastructure/ecs-main/ecs_service.yml @@ -50,8 +50,6 @@ Resources: Value: ${self:custom.repoBranchesCollection} - Name: DOCSETS_COL_NAME Value: ${self:custom.docsetsCollection} - # - Name: PROJECTS_COL_NAME - # Value: ${self:custom.projectsCollection} - Name: NPM_BASE_64_AUTH Value: ${self:custom.npmBase64Auth} - Name: NPM_EMAIL diff --git a/serverless.yml b/serverless.yml index d7fbd0af1..bd457b7cc 100644 --- a/serverless.yml +++ b/serverless.yml @@ -111,7 +111,6 @@ webhook-env-core: &webhook-env-core GITHUB_BOT_PASSWORD: ${self:custom.githubBotPW} REPO_BRANCHES_COL_NAME: ${self:custom.repoBranchesCollection} DOCSETS_COL_NAME: ${self:custom.docsetsCollection} - # PROJECTS_COL_NAME: ${self:custom.projectsCollection} SLACK_SECRET: ${self:custom.slackSecret} SLACK_TOKEN: ${self:custom.slackAuthToken} SNOOTY_SECRET: ${self:custom.snootySecret} diff --git a/src/commands/src/scripts/local-build/utils/get-env-vars.ts b/src/commands/src/scripts/local-build/utils/get-env-vars.ts index d3f253bff..e6ff10648 100644 --- a/src/commands/src/scripts/local-build/utils/get-env-vars.ts +++ b/src/commands/src/scripts/local-build/utils/get-env-vars.ts @@ -62,7 +62,6 @@ const workerParamPathToEnvName: Record = { '/cdn/client/id': 'CDN_CLIENT_ID', '/cdn/client/secret': 'CDN_CLIENT_SECRET', '/atlas/collections/docsets': 'DOCSETS_COL_NAME', - // '/atlas/collections/projects': 'PROJECTS_COL_NAME', '/atlas/dbname': 'DB_NAME', '/atlas/collections/snooty': 'SNOOTY_DB_NAME', '/atlas/username': 'MONGO_ATLAS_USERNAME', From 73875bfe65401085a59d99a01f5f28b0680a3851 Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Mon, 25 Mar 2024 17:34:09 -0400 Subject: [PATCH 199/254] DOP-4414 uncomment main branch check --- api/controllers/v2/github.ts | 21 ++++++++++----------- serverless.yml | 1 - 2 files changed, 10 insertions(+), 12 deletions(-) diff --git a/api/controllers/v2/github.ts b/api/controllers/v2/github.ts index 372d136f6..5c8e51f72 100644 --- a/api/controllers/v2/github.ts +++ b/api/controllers/v2/github.ts @@ -169,17 +169,16 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr }; // if the build was not building main branch, no need for smoke test sites - // if (body.workflow_run.head_branch != 'main' || body.repository.fork) { - // console.log('Build was not on master branch in main repo, sites will not deploy as no smoke tests are needed'); - // return { - // statusCode: 202, - // headers: { 'Content-Type': 'text/plain' }, - // body: - // `Build on branch - // ${body.workflow_run.head_branch} - // will not trigger site deployments as it was not on main branch in upstream repo`, - // }; - // } + if (body.workflow_run.head_branch != 'main' || body.repository.fork) { + console.log('Build was not on master branch in main repo, sites will not deploy as no smoke tests are needed'); + return { + statusCode: 202, + headers: { 'Content-Type': 'text/plain' }, + body: `Build on branch + ${body.workflow_run.head_branch} + will not trigger site deployments as it was not on main branch in upstream repo`, + }; + } //automated test builds will always deploy in dotcomstg const env = 'dotcomstg'; diff --git a/serverless.yml b/serverless.yml index bd457b7cc..66a398034 100644 --- a/serverless.yml +++ b/serverless.yml @@ -57,7 +57,6 @@ custom: entitlementCollection: ${ssm:/env/${self:provider.stage}/docs/worker_pool/atlas/collections/user/entitlements} repoBranchesCollection: ${ssm:/env/${self:provider.stage}/docs/worker_pool/atlas/collections/repo} docsetsCollection: ${ssm:/env/${self:provider.stage}/docs/worker_pool/atlas/collections/docsets} - # projectsCollection: ${ssm:/env/${self:provider.stage}/docs/worker_pool/atlas/collections/projects} githubSecret: ${ssm:/env/${self:provider.stage}/docs/worker_pool/github/webhook/secret} githubDeletionSecret: ${ssm:/env/${self:provider.stage}/docs/worker_pool/github/webhook/deletionSecret} githubBotPW: ${ssm:/env/${self:provider.stage}/docs/worker_pool/github/bot/password} From 770ffcce496ff97cf9f0281738d9ddd55502e157 Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Mon, 25 Mar 2024 17:39:21 -0400 Subject: [PATCH 200/254] DOP-4414 cleaning up --- src/commands/src/scripts/local-build/utils/get-env-vars.ts | 1 - src/repositories/projectsRepository.ts | 2 -- 2 files changed, 3 deletions(-) diff --git a/src/commands/src/scripts/local-build/utils/get-env-vars.ts b/src/commands/src/scripts/local-build/utils/get-env-vars.ts index e6ff10648..dcc3aef46 100644 --- a/src/commands/src/scripts/local-build/utils/get-env-vars.ts +++ b/src/commands/src/scripts/local-build/utils/get-env-vars.ts @@ -37,7 +37,6 @@ const workerParamStrings = [ '/flag/monorepo_path', '/atlas/collections/repo', '/atlas/collections/docsets', - // '/atlas/collections/projects', '/atlas/collections/job/queue', '/atlas/collections/user/entitlements', ] as const; diff --git a/src/repositories/projectsRepository.ts b/src/repositories/projectsRepository.ts index 475a9981d..094aa4193 100644 --- a/src/repositories/projectsRepository.ts +++ b/src/repositories/projectsRepository.ts @@ -5,8 +5,6 @@ import { ILogger } from '../services/logger'; //Project information from docs_metadata.projects for parser builds. -// const projectsCollectionName = config.get('projectsCollection') || ''; - export class ProjectsRepository extends BaseRepository { constructor(db: mongodb.Db, config: IConfig, logger: ILogger) { super(config, logger, 'ProjectsRepository', db.collection(process.env.PROJECTS_COL_NAME || '')); From 2da0106bbc0fb02959a3762316d637bd03b03228 Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Mon, 25 Mar 2024 17:54:05 -0400 Subject: [PATCH 201/254] DOP-4414 last cleaning --- api/controllers/v2/github.ts | 4 +--- src/repositories/projectsRepository.ts | 4 ++-- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/api/controllers/v2/github.ts b/api/controllers/v2/github.ts index 5c8e51f72..518543d91 100644 --- a/api/controllers/v2/github.ts +++ b/api/controllers/v2/github.ts @@ -174,9 +174,7 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr return { statusCode: 202, headers: { 'Content-Type': 'text/plain' }, - body: `Build on branch - ${body.workflow_run.head_branch} - will not trigger site deployments as it was not on main branch in upstream repo`, + body: `Build on branch ${body.workflow_run.head_branch} will not trigger site deployments as it was not on main branch in upstream repo`, }; } diff --git a/src/repositories/projectsRepository.ts b/src/repositories/projectsRepository.ts index 094aa4193..08ebb4e19 100644 --- a/src/repositories/projectsRepository.ts +++ b/src/repositories/projectsRepository.ts @@ -1,5 +1,5 @@ -import mongodb, { Db } from 'mongodb'; -import c, { IConfig } from 'config'; +import mongodb from 'mongodb'; +import { IConfig } from 'config'; import { BaseRepository } from './baseRepository'; import { ILogger } from '../services/logger'; From 7fddfaad6e04e655512d09c9fd71f55c199a37da Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Tue, 26 Mar 2024 11:28:26 -0400 Subject: [PATCH 202/254] DOP-4414 removing from preprd --- .github/workflows/deploy-stg-ecs.yml | 1 - .github/workflows/deploy-stg-enhanced-webhooks.yml | 3 +-- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/deploy-stg-ecs.yml b/.github/workflows/deploy-stg-ecs.yml index 1b5c34109..2e0cfb462 100644 --- a/.github/workflows/deploy-stg-ecs.yml +++ b/.github/workflows/deploy-stg-ecs.yml @@ -3,7 +3,6 @@ on: branches: - "main" - "integration" - - "DOP-4414" concurrency: group: environment-stg-${{ github.ref }} cancel-in-progress: true diff --git a/.github/workflows/deploy-stg-enhanced-webhooks.yml b/.github/workflows/deploy-stg-enhanced-webhooks.yml index 40c4347f1..f4d4dd938 100644 --- a/.github/workflows/deploy-stg-enhanced-webhooks.yml +++ b/.github/workflows/deploy-stg-enhanced-webhooks.yml @@ -1,10 +1,9 @@ on: push: - # paths: ["api/v2/**", "cdk-infra/lib/constructs/api/**", "cdk-infra/utils/**"] + paths: ["api/v2/**", "cdk-infra/lib/constructs/api/**", "cdk-infra/utils/**"] branches: - "main" - "integration" - - "DOP-4414" concurrency: group: environment-stg-enhanced-webhooks-${{ github.ref }} From f5f685a87b31d6075e3608b33596e27137c31d44 Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Tue, 26 Mar 2024 16:12:54 -0400 Subject: [PATCH 203/254] DOP-4451 initial comments and push to preprd --- .github/workflows/deploy-stg-ecs.yml | 1 + .../deploy-stg-enhanced-webhooks.yml | 3 +- api/controllers/v2/github.ts | 31 ++++++++++--------- cdk-infra/bin/cdk-infra.ts | 3 ++ cdk-infra/lib/stacks/worker-stack.ts | 6 ++++ 5 files changed, 28 insertions(+), 16 deletions(-) diff --git a/.github/workflows/deploy-stg-ecs.yml b/.github/workflows/deploy-stg-ecs.yml index 2e0cfb462..1b5c34109 100644 --- a/.github/workflows/deploy-stg-ecs.yml +++ b/.github/workflows/deploy-stg-ecs.yml @@ -3,6 +3,7 @@ on: branches: - "main" - "integration" + - "DOP-4414" concurrency: group: environment-stg-${{ github.ref }} cancel-in-progress: true diff --git a/.github/workflows/deploy-stg-enhanced-webhooks.yml b/.github/workflows/deploy-stg-enhanced-webhooks.yml index f4d4dd938..40c4347f1 100644 --- a/.github/workflows/deploy-stg-enhanced-webhooks.yml +++ b/.github/workflows/deploy-stg-enhanced-webhooks.yml @@ -1,9 +1,10 @@ on: push: - paths: ["api/v2/**", "cdk-infra/lib/constructs/api/**", "cdk-infra/utils/**"] + # paths: ["api/v2/**", "cdk-infra/lib/constructs/api/**", "cdk-infra/utils/**"] branches: - "main" - "integration" + - "DOP-4414" concurrency: group: environment-stg-enhanced-webhooks-${{ github.ref }} diff --git a/api/controllers/v2/github.ts b/api/controllers/v2/github.ts index 518543d91..9df881ed8 100644 --- a/api/controllers/v2/github.ts +++ b/api/controllers/v2/github.ts @@ -169,14 +169,14 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr }; // if the build was not building main branch, no need for smoke test sites - if (body.workflow_run.head_branch != 'main' || body.repository.fork) { - console.log('Build was not on master branch in main repo, sites will not deploy as no smoke tests are needed'); - return { - statusCode: 202, - headers: { 'Content-Type': 'text/plain' }, - body: `Build on branch ${body.workflow_run.head_branch} will not trigger site deployments as it was not on main branch in upstream repo`, - }; - } + // if (body.workflow_run.head_branch != 'main' || body.repository.fork) { + // console.log('Build was not on master branch in main repo, sites will not deploy as no smoke tests are needed'); + // return { + // statusCode: 202, + // headers: { 'Content-Type': 'text/plain' }, + // body: `Build on branch ${body.workflow_run.head_branch} will not trigger site deployments as it was not on main branch in upstream repo`, + // }; + // } //automated test builds will always deploy in dotcomstg const env = 'dotcomstg'; @@ -225,9 +225,15 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr ); } - let returnVal; try { - returnVal = await createAndInsertJob(); + const returnVal = await createAndInsertJob(); + // run tasks here + + return { + statusCode: 202, + headers: { 'Content-Type': 'text/plain' }, + body: 'Smoke Test Jobs Queued with the following Job Ids ' + returnVal, + }; } catch (err) { return { statusCode: 500, @@ -235,11 +241,6 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr body: err, }; } - return { - statusCode: 202, - headers: { 'Content-Type': 'text/plain' }, - body: 'Smoke Test Jobs Queued with the following Job Ids ' + returnVal, - }; }; export const TriggerBuild = async (event: APIGatewayEvent): Promise => { diff --git a/cdk-infra/bin/cdk-infra.ts b/cdk-infra/bin/cdk-infra.ts index 5a7737137..2b8e7c23d 100644 --- a/cdk-infra/bin/cdk-infra.ts +++ b/cdk-infra/bin/cdk-infra.ts @@ -39,6 +39,9 @@ async function main() { const queues = new AutoBuilderQueueStack(app, `${stackName}-queues`, { env }); const { clusterName } = new WorkerStack(app, `${stackName}-worker`, { queues, workerSecureStrings, vpc, env }); + + // TODO: Pass the VPC as a prop here like the queues or clusterName + // TODO: Pass the task definition that will be added from the Worker stack new WebhookStack(app, `${stackName}-webhooks`, { queues, clusterName, diff --git a/cdk-infra/lib/stacks/worker-stack.ts b/cdk-infra/lib/stacks/worker-stack.ts index a6c0ea401..79ccddd2b 100644 --- a/cdk-infra/lib/stacks/worker-stack.ts +++ b/cdk-infra/lib/stacks/worker-stack.ts @@ -13,6 +13,8 @@ interface WorkerStackProps extends StackProps { } export class WorkerStack extends Stack { + // TODO: Create the task definition as properties here so + // that they are accessible to the webhook stack public readonly clusterName: string; constructor(scope: Construct, id: string, { queues, workerSecureStrings, vpc, ...props }: WorkerStackProps) { @@ -23,6 +25,8 @@ export class WorkerStack extends Stack { secureStrings: workerSecureStrings, }); + // TODO: retrieve the task definition from this stack as it is required + // for the RunTask in the smoke test build const { clusterName, ecsTaskRole } = new WorkerConstruct(this, 'worker', { vpc, dockerEnvironment: environment, @@ -34,6 +38,8 @@ export class WorkerStack extends Stack { bucket.grantReadWrite(ecsTaskRole); }); + // TODO: Assign the task definition as properties here so + // that they are accessible to the webhook stack this.clusterName = clusterName; } } From 23b57cec7653b030a5d793b6d1ade4920c64a755 Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Tue, 26 Mar 2024 16:26:30 -0400 Subject: [PATCH 204/254] DOP-4451 testing commit prefix --- .github/workflows/deploy-stg-ecs.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/deploy-stg-ecs.yml b/.github/workflows/deploy-stg-ecs.yml index 1b5c34109..2e0cfb462 100644 --- a/.github/workflows/deploy-stg-ecs.yml +++ b/.github/workflows/deploy-stg-ecs.yml @@ -3,7 +3,6 @@ on: branches: - "main" - "integration" - - "DOP-4414" concurrency: group: environment-stg-${{ github.ref }} cancel-in-progress: true From 8f36ebfb00791a243a00a0d2774e53a627431457 Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Tue, 26 Mar 2024 16:27:03 -0400 Subject: [PATCH 205/254] DOP-4451 testing commit prefix --- api/controllers/v2/github.ts | 2 +- src/job/productionJobHandler.ts | 21 ++++++--------------- 2 files changed, 7 insertions(+), 16 deletions(-) diff --git a/api/controllers/v2/github.ts b/api/controllers/v2/github.ts index 9df881ed8..d569ec265 100644 --- a/api/controllers/v2/github.ts +++ b/api/controllers/v2/github.ts @@ -86,7 +86,7 @@ async function createPayload({ if (!githubEvent) { throw new Error(`Non SmokeTest Deploy jobs must have a github Event`); } - action = 'push'; + action = 'automatedTest'; jobType = 'githubPush'; branchName = githubEvent.ref.split('/')[2]; url = githubEvent.repository?.clone_url; diff --git a/src/job/productionJobHandler.ts b/src/job/productionJobHandler.ts index 6190cc018..04aece069 100644 --- a/src/job/productionJobHandler.ts +++ b/src/job/productionJobHandler.ts @@ -141,22 +141,13 @@ export class ProductionJobHandler extends JobHandler { } getPathPrefix(): string { - try { - if (this.currJob.payload.prefix && this.currJob.payload.prefix === '') { - return this.currJob.payload.urlSlug ?? ''; - } - if (this.currJob.payload.urlSlug) { - if (this.currJob.payload.urlSlug === '') { - return this.currJob.payload.prefix; - } else { - return `${this.currJob.payload.prefix}/${this.currJob.payload.urlSlug}`; - } - } - return this.currJob.payload.prefix; - } catch (error) { - this.logger.save(this.currJob._id, error).then(); - throw new InvalidJobError(error.message); + const prefix = this.currJob.payload.urlSlug + ? `${this.currJob.payload.prefix}/${this.currJob.payload.urlSlug}` + : this.currJob.payload.prefix; + if (this.currJob.payload.newHead && this.currJob.payload.action == 'automatedTest') { + return `${prefix}/${this.currJob.payload.newHead}`; } + return prefix; } private async purgePublishedContent(makefileOutput: Array): Promise { From c42fd0c9838d2e7b98ba2fe77e0bc6e92fd86e74 Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Tue, 26 Mar 2024 16:32:06 -0400 Subject: [PATCH 206/254] DOP-4451 checks issues --- .github/workflows/deploy-stg-ecs.yml | 1 + .github/workflows/deploy-stg-enhanced-webhooks.yml | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/deploy-stg-ecs.yml b/.github/workflows/deploy-stg-ecs.yml index 2e0cfb462..ca799a2e3 100644 --- a/.github/workflows/deploy-stg-ecs.yml +++ b/.github/workflows/deploy-stg-ecs.yml @@ -3,6 +3,7 @@ on: branches: - "main" - "integration" + - "DOP-4451" concurrency: group: environment-stg-${{ github.ref }} cancel-in-progress: true diff --git a/.github/workflows/deploy-stg-enhanced-webhooks.yml b/.github/workflows/deploy-stg-enhanced-webhooks.yml index 40c4347f1..f9e423c72 100644 --- a/.github/workflows/deploy-stg-enhanced-webhooks.yml +++ b/.github/workflows/deploy-stg-enhanced-webhooks.yml @@ -1,10 +1,10 @@ on: push: - # paths: ["api/v2/**", "cdk-infra/lib/constructs/api/**", "cdk-infra/utils/**"] + paths: ["api/v2/**", "cdk-infra/lib/constructs/api/**", "cdk-infra/utils/**"] branches: - "main" - "integration" - - "DOP-4414" + - "DOP-4451" concurrency: group: environment-stg-enhanced-webhooks-${{ github.ref }} From 53debf7d8f7d04196b697e2ea0784102b69947dd Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Tue, 26 Mar 2024 16:41:12 -0400 Subject: [PATCH 207/254] DOP-4451 checks issues --- .github/workflows/deploy-stg-ecs.yml | 2 +- .github/workflows/deploy-stg-enhanced-webhooks.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/deploy-stg-ecs.yml b/.github/workflows/deploy-stg-ecs.yml index ca799a2e3..03f08b943 100644 --- a/.github/workflows/deploy-stg-ecs.yml +++ b/.github/workflows/deploy-stg-ecs.yml @@ -3,7 +3,7 @@ on: branches: - "main" - "integration" - - "DOP-4451" + # - "DOP-4451" concurrency: group: environment-stg-${{ github.ref }} cancel-in-progress: true diff --git a/.github/workflows/deploy-stg-enhanced-webhooks.yml b/.github/workflows/deploy-stg-enhanced-webhooks.yml index f9e423c72..345daf718 100644 --- a/.github/workflows/deploy-stg-enhanced-webhooks.yml +++ b/.github/workflows/deploy-stg-enhanced-webhooks.yml @@ -4,7 +4,7 @@ on: branches: - "main" - "integration" - - "DOP-4451" + # - "DOP-4451" concurrency: group: environment-stg-enhanced-webhooks-${{ github.ref }} From e2951483065f117566f64289fcbac4ec5c7e8554 Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Tue, 26 Mar 2024 16:56:34 -0400 Subject: [PATCH 208/254] DOP-4451 updating feature branch --- .github/workflows/deploy-stg-ecs.yml | 2 +- src/job/productionJobHandler.ts | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/deploy-stg-ecs.yml b/.github/workflows/deploy-stg-ecs.yml index 03f08b943..ca799a2e3 100644 --- a/.github/workflows/deploy-stg-ecs.yml +++ b/.github/workflows/deploy-stg-ecs.yml @@ -3,7 +3,7 @@ on: branches: - "main" - "integration" - # - "DOP-4451" + - "DOP-4451" concurrency: group: environment-stg-${{ github.ref }} cancel-in-progress: true diff --git a/src/job/productionJobHandler.ts b/src/job/productionJobHandler.ts index 04aece069..de2ec36a6 100644 --- a/src/job/productionJobHandler.ts +++ b/src/job/productionJobHandler.ts @@ -144,6 +144,10 @@ export class ProductionJobHandler extends JobHandler { const prefix = this.currJob.payload.urlSlug ? `${this.currJob.payload.prefix}/${this.currJob.payload.urlSlug}` : this.currJob.payload.prefix; + this.logger.save( + this.currJob._id, + `current job prefix ${prefix}, newHead ${this.currJob.payload.newHead} & action ${this.currJob.payload.action}` + ); if (this.currJob.payload.newHead && this.currJob.payload.action == 'automatedTest') { return `${prefix}/${this.currJob.payload.newHead}`; } From 045637adf47c8c3a545a041a778a9e2f54469308 Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Tue, 26 Mar 2024 17:13:06 -0400 Subject: [PATCH 209/254] DOP-4451 updating enhanced app --- .github/workflows/deploy-stg-enhanced-webhooks.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/deploy-stg-enhanced-webhooks.yml b/.github/workflows/deploy-stg-enhanced-webhooks.yml index 345daf718..f9e423c72 100644 --- a/.github/workflows/deploy-stg-enhanced-webhooks.yml +++ b/.github/workflows/deploy-stg-enhanced-webhooks.yml @@ -4,7 +4,7 @@ on: branches: - "main" - "integration" - # - "DOP-4451" + - "DOP-4451" concurrency: group: environment-stg-enhanced-webhooks-${{ github.ref }} From 07f0ecb371459f01c4434f7eb513763258f130b2 Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Tue, 26 Mar 2024 17:20:23 -0400 Subject: [PATCH 210/254] DOP-4451 testing path prefix --- src/job/jobHandler.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/job/jobHandler.ts b/src/job/jobHandler.ts index eed59edfb..06e45699d 100644 --- a/src/job/jobHandler.ts +++ b/src/job/jobHandler.ts @@ -149,7 +149,10 @@ export abstract class JobHandler { @throwIfJobInterupted() private async constructPrefix(): Promise { const server_user = this._config.get('GATSBY_PARSER_USER'); - const pathPrefix = await this.getPathPrefix(); + let pathPrefix = this.getPathPrefix(); + if (this.currJob.payload.newHead && this.currJob.payload.action == 'automatedTest') { + pathPrefix = `${pathPrefix}/${this.currJob.payload.newHead}`; + } // TODO: Can empty string check be removed? if (pathPrefix || pathPrefix === '') { this.currJob.payload.pathPrefix = pathPrefix; From 73f09d046043f97650a0e8d558eb9d7d1804bb59 Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Tue, 26 Mar 2024 18:04:32 -0400 Subject: [PATCH 211/254] DOP-4451 update path prefix and log it --- src/job/jobHandler.ts | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/job/jobHandler.ts b/src/job/jobHandler.ts index 06e45699d..42c9ff746 100644 --- a/src/job/jobHandler.ts +++ b/src/job/jobHandler.ts @@ -149,11 +149,12 @@ export abstract class JobHandler { @throwIfJobInterupted() private async constructPrefix(): Promise { const server_user = this._config.get('GATSBY_PARSER_USER'); - let pathPrefix = this.getPathPrefix(); + const prePrefix = this.getPathPrefix(); + let pathPrefix; if (this.currJob.payload.newHead && this.currJob.payload.action == 'automatedTest') { - pathPrefix = `${pathPrefix}/${this.currJob.payload.newHead}`; - } - // TODO: Can empty string check be removed? + pathPrefix = `${prePrefix}/${this.currJob.payload.newHead}`; + } else pathPrefix = prePrefix; + this._logger.save(this.currJob._id, `${pathPrefix}, prePrefix: ${prePrefix}, server user: ${server_user}`); if (pathPrefix || pathPrefix === '') { this.currJob.payload.pathPrefix = pathPrefix; const mutPrefix = pathPrefix.split(`/${server_user}`)[0]; From 697293ed70d4d4d10668e2252321ec1bfa869861 Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Tue, 26 Mar 2024 22:24:42 -0400 Subject: [PATCH 212/254] DOP-4451 messing w prefix --- src/job/jobHandler.ts | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/job/jobHandler.ts b/src/job/jobHandler.ts index 42c9ff746..2aa009181 100644 --- a/src/job/jobHandler.ts +++ b/src/job/jobHandler.ts @@ -149,16 +149,16 @@ export abstract class JobHandler { @throwIfJobInterupted() private async constructPrefix(): Promise { const server_user = this._config.get('GATSBY_PARSER_USER'); - const prePrefix = this.getPathPrefix(); - let pathPrefix; - if (this.currJob.payload.newHead && this.currJob.payload.action == 'automatedTest') { - pathPrefix = `${prePrefix}/${this.currJob.payload.newHead}`; - } else pathPrefix = prePrefix; - this._logger.save(this.currJob._id, `${pathPrefix}, prePrefix: ${prePrefix}, server user: ${server_user}`); + const pathPrefix = this.getPathPrefix(); + // let pathPrefix; + // if (this.currJob.payload.newHead && this.currJob.payload.action == 'automatedTest') { + // pathPrefix = `${prePrefix}/${this.currJob.payload.newHead}`; + // } else pathPrefix = prePrefix; + this._logger.save(this.currJob._id, `${pathPrefix}, prePrefix: ${pathPrefix}, server user: ${server_user}`); if (pathPrefix || pathPrefix === '') { - this.currJob.payload.pathPrefix = pathPrefix; const mutPrefix = pathPrefix.split(`/${server_user}`)[0]; this.currJob.payload.mutPrefix = mutPrefix; + this.currJob.payload.pathPrefix = `${pathPrefix}/${this.currJob.payload.newHead}`; } } From 5cd4e1925773fe5494aaff0c28c10fcdb9ef6bd7 Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Tue, 26 Mar 2024 23:38:28 -0400 Subject: [PATCH 213/254] DOP-4451 messing w prefix more --- src/job/jobHandler.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/job/jobHandler.ts b/src/job/jobHandler.ts index 2aa009181..a5c73f1e5 100644 --- a/src/job/jobHandler.ts +++ b/src/job/jobHandler.ts @@ -154,11 +154,11 @@ export abstract class JobHandler { // if (this.currJob.payload.newHead && this.currJob.payload.action == 'automatedTest') { // pathPrefix = `${prePrefix}/${this.currJob.payload.newHead}`; // } else pathPrefix = prePrefix; - this._logger.save(this.currJob._id, `${pathPrefix}, prePrefix: ${pathPrefix}, server user: ${server_user}`); if (pathPrefix || pathPrefix === '') { const mutPrefix = pathPrefix.split(`/${server_user}`)[0]; - this.currJob.payload.mutPrefix = mutPrefix; - this.currJob.payload.pathPrefix = `${pathPrefix}/${this.currJob.payload.newHead}`; + this.currJob.payload.mutPrefix = `${this.currJob.payload.newHead}`; + this.currJob.payload.pathPrefix = `${this.currJob.payload.newHead}`; + this._logger.save(this.currJob._id, `${mutPrefix}, prePrefix: ${pathPrefix}, server user: ${server_user}`); } } From 5dadded780ab278cef32c6fa0c8cbf67479d22f4 Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Wed, 27 Mar 2024 10:22:53 -0400 Subject: [PATCH 214/254] DOP-4452 spacing nit --- src/job/stagingJobHandler.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/job/stagingJobHandler.ts b/src/job/stagingJobHandler.ts index e543da9fb..fb9c5973a 100644 --- a/src/job/stagingJobHandler.ts +++ b/src/job/stagingJobHandler.ts @@ -49,7 +49,7 @@ export class StagingJobHandler extends JobHandler { prepDeployCommands(): void { this.currJob.deployCommands = [ `cd repos/${getDirectory(this.currJob)}`, - `make next-gen-stage${this.currJob.payload.pathPrefix ? `MUT_PREFIX=${this.currJob.payload.mutPrefix}` : ''}`, + `make next-gen-stage ${this.currJob.payload.pathPrefix ? `MUT_PREFIX=${this.currJob.payload.mutPrefix}` : ''}`, ]; } From 3912f49b541ad1dfdecfbf3b9ba13f531b95467d Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Wed, 27 Mar 2024 15:28:09 -0400 Subject: [PATCH 215/254] DOP-4451 testing mut prefix --- src/job/jobHandler.ts | 7 ++++--- src/job/stagingJobHandler.ts | 3 ++- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/src/job/jobHandler.ts b/src/job/jobHandler.ts index a5c73f1e5..61dab5d88 100644 --- a/src/job/jobHandler.ts +++ b/src/job/jobHandler.ts @@ -149,15 +149,16 @@ export abstract class JobHandler { @throwIfJobInterupted() private async constructPrefix(): Promise { const server_user = this._config.get('GATSBY_PARSER_USER'); - const pathPrefix = this.getPathPrefix(); + // const pathPrefix = this.getPathPrefix(); + const pathPrefix = ''; // let pathPrefix; // if (this.currJob.payload.newHead && this.currJob.payload.action == 'automatedTest') { // pathPrefix = `${prePrefix}/${this.currJob.payload.newHead}`; // } else pathPrefix = prePrefix; if (pathPrefix || pathPrefix === '') { const mutPrefix = pathPrefix.split(`/${server_user}`)[0]; - this.currJob.payload.mutPrefix = `${this.currJob.payload.newHead}`; - this.currJob.payload.pathPrefix = `${this.currJob.payload.newHead}`; + this.currJob.payload.mutPrefix = `slackPrefix`; + this.currJob.payload.pathPrefix = pathPrefix; this._logger.save(this.currJob._id, `${mutPrefix}, prePrefix: ${pathPrefix}, server user: ${server_user}`); } } diff --git a/src/job/stagingJobHandler.ts b/src/job/stagingJobHandler.ts index fb9c5973a..d47c25504 100644 --- a/src/job/stagingJobHandler.ts +++ b/src/job/stagingJobHandler.ts @@ -49,7 +49,8 @@ export class StagingJobHandler extends JobHandler { prepDeployCommands(): void { this.currJob.deployCommands = [ `cd repos/${getDirectory(this.currJob)}`, - `make next-gen-stage ${this.currJob.payload.pathPrefix ? `MUT_PREFIX=${this.currJob.payload.mutPrefix}` : ''}`, + // `make next-gen-stage ${this.currJob.payload.pathPrefix ? ` MUT_PREFIX=${this.currJob.payload.mutPrefix}` : ''}`, + `make next-gen-stage ${true ? ` MUT_PREFIX=${this.currJob.payload.mutPrefix}` : ''}`, ]; } From 90d0acf57380f418117470b83b06d21bb5e6d5ff Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Wed, 27 Mar 2024 16:01:18 -0400 Subject: [PATCH 216/254] DOP-4451 remove uneccesary space" --- src/job/stagingJobHandler.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/job/stagingJobHandler.ts b/src/job/stagingJobHandler.ts index d47c25504..738d692e2 100644 --- a/src/job/stagingJobHandler.ts +++ b/src/job/stagingJobHandler.ts @@ -50,7 +50,7 @@ export class StagingJobHandler extends JobHandler { this.currJob.deployCommands = [ `cd repos/${getDirectory(this.currJob)}`, // `make next-gen-stage ${this.currJob.payload.pathPrefix ? ` MUT_PREFIX=${this.currJob.payload.mutPrefix}` : ''}`, - `make next-gen-stage ${true ? ` MUT_PREFIX=${this.currJob.payload.mutPrefix}` : ''}`, + `make next-gen-stage ${true ? `MUT_PREFIX=${this.currJob.payload.mutPrefix}` : ''}`, ]; } From 49b2f0c444ddcf4eabf68f10667254085b4d3194 Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Wed, 3 Apr 2024 15:09:50 -0400 Subject: [PATCH 217/254] DOP-4451 removing from preprd and enhanced-webhooks --- .github/workflows/deploy-stg-ecs.yml | 1 - .github/workflows/deploy-stg-enhanced-webhooks.yml | 1 - 2 files changed, 2 deletions(-) diff --git a/.github/workflows/deploy-stg-ecs.yml b/.github/workflows/deploy-stg-ecs.yml index ca799a2e3..2e0cfb462 100644 --- a/.github/workflows/deploy-stg-ecs.yml +++ b/.github/workflows/deploy-stg-ecs.yml @@ -3,7 +3,6 @@ on: branches: - "main" - "integration" - - "DOP-4451" concurrency: group: environment-stg-${{ github.ref }} cancel-in-progress: true diff --git a/.github/workflows/deploy-stg-enhanced-webhooks.yml b/.github/workflows/deploy-stg-enhanced-webhooks.yml index f9e423c72..f4d4dd938 100644 --- a/.github/workflows/deploy-stg-enhanced-webhooks.yml +++ b/.github/workflows/deploy-stg-enhanced-webhooks.yml @@ -4,7 +4,6 @@ on: branches: - "main" - "integration" - - "DOP-4451" concurrency: group: environment-stg-enhanced-webhooks-${{ github.ref }} From ce27d064206779c98fd750613948bf869eafd7a7 Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Tue, 9 Apr 2024 15:06:33 -0400 Subject: [PATCH 218/254] DOP-4451 undoing prefix changes --- src/job/jobHandler.ts | 13 ++++--------- src/job/productionJobHandler.ts | 25 +++++++++++++++---------- src/job/stagingJobHandler.ts | 3 +-- 3 files changed, 20 insertions(+), 21 deletions(-) diff --git a/src/job/jobHandler.ts b/src/job/jobHandler.ts index 61dab5d88..eed59edfb 100644 --- a/src/job/jobHandler.ts +++ b/src/job/jobHandler.ts @@ -149,17 +149,12 @@ export abstract class JobHandler { @throwIfJobInterupted() private async constructPrefix(): Promise { const server_user = this._config.get('GATSBY_PARSER_USER'); - // const pathPrefix = this.getPathPrefix(); - const pathPrefix = ''; - // let pathPrefix; - // if (this.currJob.payload.newHead && this.currJob.payload.action == 'automatedTest') { - // pathPrefix = `${prePrefix}/${this.currJob.payload.newHead}`; - // } else pathPrefix = prePrefix; + const pathPrefix = await this.getPathPrefix(); + // TODO: Can empty string check be removed? if (pathPrefix || pathPrefix === '') { - const mutPrefix = pathPrefix.split(`/${server_user}`)[0]; - this.currJob.payload.mutPrefix = `slackPrefix`; this.currJob.payload.pathPrefix = pathPrefix; - this._logger.save(this.currJob._id, `${mutPrefix}, prePrefix: ${pathPrefix}, server user: ${server_user}`); + const mutPrefix = pathPrefix.split(`/${server_user}`)[0]; + this.currJob.payload.mutPrefix = mutPrefix; } } diff --git a/src/job/productionJobHandler.ts b/src/job/productionJobHandler.ts index de2ec36a6..6190cc018 100644 --- a/src/job/productionJobHandler.ts +++ b/src/job/productionJobHandler.ts @@ -141,17 +141,22 @@ export class ProductionJobHandler extends JobHandler { } getPathPrefix(): string { - const prefix = this.currJob.payload.urlSlug - ? `${this.currJob.payload.prefix}/${this.currJob.payload.urlSlug}` - : this.currJob.payload.prefix; - this.logger.save( - this.currJob._id, - `current job prefix ${prefix}, newHead ${this.currJob.payload.newHead} & action ${this.currJob.payload.action}` - ); - if (this.currJob.payload.newHead && this.currJob.payload.action == 'automatedTest') { - return `${prefix}/${this.currJob.payload.newHead}`; + try { + if (this.currJob.payload.prefix && this.currJob.payload.prefix === '') { + return this.currJob.payload.urlSlug ?? ''; + } + if (this.currJob.payload.urlSlug) { + if (this.currJob.payload.urlSlug === '') { + return this.currJob.payload.prefix; + } else { + return `${this.currJob.payload.prefix}/${this.currJob.payload.urlSlug}`; + } + } + return this.currJob.payload.prefix; + } catch (error) { + this.logger.save(this.currJob._id, error).then(); + throw new InvalidJobError(error.message); } - return prefix; } private async purgePublishedContent(makefileOutput: Array): Promise { diff --git a/src/job/stagingJobHandler.ts b/src/job/stagingJobHandler.ts index 738d692e2..e543da9fb 100644 --- a/src/job/stagingJobHandler.ts +++ b/src/job/stagingJobHandler.ts @@ -49,8 +49,7 @@ export class StagingJobHandler extends JobHandler { prepDeployCommands(): void { this.currJob.deployCommands = [ `cd repos/${getDirectory(this.currJob)}`, - // `make next-gen-stage ${this.currJob.payload.pathPrefix ? ` MUT_PREFIX=${this.currJob.payload.mutPrefix}` : ''}`, - `make next-gen-stage ${true ? `MUT_PREFIX=${this.currJob.payload.mutPrefix}` : ''}`, + `make next-gen-stage${this.currJob.payload.pathPrefix ? `MUT_PREFIX=${this.currJob.payload.mutPrefix}` : ''}`, ]; } From b14a005767832beb9bcb4d7d39ea5630d202c566 Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Tue, 9 Apr 2024 18:07:01 -0400 Subject: [PATCH 219/254] DOP-4451 filling out initial TODOs --- cdk-infra/lib/constructs/worker/worker-construct.ts | 3 +++ cdk-infra/lib/stacks/webhook-stack.ts | 6 ++++++ cdk-infra/lib/stacks/worker-stack.ts | 5 ++++- 3 files changed, 13 insertions(+), 1 deletion(-) diff --git a/cdk-infra/lib/constructs/worker/worker-construct.ts b/cdk-infra/lib/constructs/worker/worker-construct.ts index 63f550305..b24e8baa6 100644 --- a/cdk-infra/lib/constructs/worker/worker-construct.ts +++ b/cdk-infra/lib/constructs/worker/worker-construct.ts @@ -6,6 +6,7 @@ import { FargateService, FargateTaskDefinition, LogDrivers, + TaskDefinition, } from 'aws-cdk-lib/aws-ecs'; import { Effect, IRole, PolicyStatement, Role, ServicePrincipal } from 'aws-cdk-lib/aws-iam'; import { LogGroup } from 'aws-cdk-lib/aws-logs'; @@ -23,6 +24,7 @@ interface WorkerConstructProps { export class WorkerConstruct extends Construct { readonly ecsTaskRole: IRole; readonly clusterName: string; + readonly taskDefinition: TaskDefinition; constructor( scope: Construct, @@ -112,6 +114,7 @@ export class WorkerConstruct extends Construct { maxHealthyPercent: 200, }); + this.taskDefinition = taskDefinition; this.clusterName = cluster.clusterName; this.ecsTaskRole = taskRole; } diff --git a/cdk-infra/lib/stacks/webhook-stack.ts b/cdk-infra/lib/stacks/webhook-stack.ts index 41367a41b..89d4b163a 100644 --- a/cdk-infra/lib/stacks/webhook-stack.ts +++ b/cdk-infra/lib/stacks/webhook-stack.ts @@ -1,5 +1,7 @@ import { Stack, StackProps } from 'aws-cdk-lib'; import { Construct } from 'constructs'; +import { Vpc } from 'aws-cdk-lib/aws-ec2'; +import { TaskDefinition } from 'aws-cdk-lib/aws-ecs'; import { AutoBuilderQueues } from './auto-builder-queue-stack'; import { WebhookApiConstruct } from '../constructs/api/webhook-api-construct'; import { WebhookEnvConstruct } from '../constructs/api/webhook-env-construct'; @@ -8,7 +10,11 @@ interface WebhookStackProps extends StackProps { webhookSecureStrings: Record; queues: AutoBuilderQueues; clusterName: string; + vpc: Vpc; + taskDefinition: TaskDefinition; } + +//TODO: use taskDefition and vpc somewhere here export class WebhookStack extends Stack { constructor( scope: Construct, diff --git a/cdk-infra/lib/stacks/worker-stack.ts b/cdk-infra/lib/stacks/worker-stack.ts index 79ccddd2b..1cb605d61 100644 --- a/cdk-infra/lib/stacks/worker-stack.ts +++ b/cdk-infra/lib/stacks/worker-stack.ts @@ -4,6 +4,7 @@ import { WorkerConstruct } from '../constructs/worker/worker-construct'; import { WorkerEnvConstruct } from '../constructs/worker/worker-env-construct'; import { WorkerBucketsConstruct } from '../constructs/worker/buckets-construct'; import { AutoBuilderQueues } from './auto-builder-queue-stack'; +import { TaskDefinition } from 'aws-cdk-lib/aws-ecs'; import { IVpc } from 'aws-cdk-lib/aws-ec2'; interface WorkerStackProps extends StackProps { @@ -16,6 +17,7 @@ export class WorkerStack extends Stack { // TODO: Create the task definition as properties here so // that they are accessible to the webhook stack public readonly clusterName: string; + public readonly taskDefinition: TaskDefinition; constructor(scope: Construct, id: string, { queues, workerSecureStrings, vpc, ...props }: WorkerStackProps) { super(scope, id, props); @@ -27,7 +29,7 @@ export class WorkerStack extends Stack { // TODO: retrieve the task definition from this stack as it is required // for the RunTask in the smoke test build - const { clusterName, ecsTaskRole } = new WorkerConstruct(this, 'worker', { + const { clusterName, ecsTaskRole, taskDefinition } = new WorkerConstruct(this, 'worker', { vpc, dockerEnvironment: environment, ...queues, @@ -40,6 +42,7 @@ export class WorkerStack extends Stack { // TODO: Assign the task definition as properties here so // that they are accessible to the webhook stack + this.taskDefinition = taskDefinition; this.clusterName = clusterName; } } From 37a5184e12e1f4c1564b58e2554dc1777b185a17 Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Tue, 9 Apr 2024 18:30:24 -0400 Subject: [PATCH 220/254] DOP-4451 configured additional workers for each smoketest job --- api/controllers/v2/github.ts | 31 ++++++++++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/api/controllers/v2/github.ts b/api/controllers/v2/github.ts index d569ec265..c35c6dddb 100644 --- a/api/controllers/v2/github.ts +++ b/api/controllers/v2/github.ts @@ -2,7 +2,7 @@ import * as c from 'config'; import * as mongodb from 'mongodb'; import { APIGatewayEvent, APIGatewayProxyResult } from 'aws-lambda'; import { PushEvent, WorkflowRunCompletedEvent } from '@octokit/webhooks-types'; - +import { ECSClient, RunTaskCommand } from '@aws-sdk/client-ecs'; import { JobRepository } from '../../../src/repositories/jobRepository'; import { ConsoleLogger } from '../../../src/services/logger'; import { RepoBranchesRepository } from '../../../src/repositories/repoBranchesRepository'; @@ -181,6 +181,34 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr //automated test builds will always deploy in dotcomstg const env = 'dotcomstg'; + //TODO: I thought we wanted vpc, not subnets? should we be getting the vars from process.env in this way(taken from runcacherebuildjob) + //no overrides needed, correct? + async function runAdditionalECSTasks() { + const { TASK_DEFINITION, CONTAINER_NAME, CLUSTER, SUBNETS } = process.env; + + if (!TASK_DEFINITION) throw new Error('ERROR! process.env.TASK_DEFINITION is not defined'); + if (!CONTAINER_NAME) throw new Error('ERROR! process.env.CONTAINER_NAME is not defined'); + if (!CLUSTER) throw new Error('ERROR! process.env.CLUSTER is not defined'); + if (!SUBNETS) throw new Error('ERROR! process.env.SUBNETS is not defined'); + + const client = new ECSClient({ + region: 'us-east-2', + }); + + const command = new RunTaskCommand({ + taskDefinition: TASK_DEFINITION, + cluster: CLUSTER, + launchType: 'FARGATE', + networkConfiguration: { + awsvpcConfiguration: { + subnets: JSON.parse(SUBNETS), + }, + }, + }); + + await client.send(command); + } + async function createAndInsertJob() { return await Promise.all( SMOKETEST_SITES.map(async (repoName): Promise => { @@ -216,6 +244,7 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr const jobId = await jobRepository.insertJob(job, c.get('jobsQueueUrl')); jobRepository.notify(jobId, c.get('jobUpdatesQueueUrl'), JobStatus.inQueue, 0); consoleLogger.info(job.title, `Created Job ${jobId}`); + // runAdditionalECSTasks(); return jobId; } catch (err) { consoleLogger.error('TriggerBuildError', `${err} Error inserting job for ${repoName}`); From 94174623bd1e7a56d01486616d0451290e2f8462 Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Tue, 9 Apr 2024 18:44:39 -0400 Subject: [PATCH 221/254] DOP-4451 fixed cdk-infra consistency --- cdk-infra/bin/cdk-infra.ts | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/cdk-infra/bin/cdk-infra.ts b/cdk-infra/bin/cdk-infra.ts index 2b8e7c23d..952c89b69 100644 --- a/cdk-infra/bin/cdk-infra.ts +++ b/cdk-infra/bin/cdk-infra.ts @@ -38,7 +38,12 @@ async function main() { const stackName = `auto-builder-stack-${getFeatureName()}`; const queues = new AutoBuilderQueueStack(app, `${stackName}-queues`, { env }); - const { clusterName } = new WorkerStack(app, `${stackName}-worker`, { queues, workerSecureStrings, vpc, env }); + const { clusterName, taskDefinition } = new WorkerStack(app, `${stackName}-worker`, { + queues, + workerSecureStrings, + vpc, + env, + }); // TODO: Pass the VPC as a prop here like the queues or clusterName // TODO: Pass the task definition that will be added from the Worker stack @@ -47,6 +52,8 @@ async function main() { clusterName, webhookSecureStrings, env, + vpc, + taskDefinition, }); new CacheUpdaterStack(app, `${stackName}-cache`, { From 3f511995ea898e758fb65183b696ee1665947a5e Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Wed, 10 Apr 2024 15:54:08 -0400 Subject: [PATCH 222/254] DOP-4451 additional corrections from Brandon session --- api/controllers/v2/github.ts | 5 +---- .../constructs/api/webhook-api-construct.ts | 18 ++++++++++++++++-- cdk-infra/lib/stacks/webhook-stack.ts | 4 +++- 3 files changed, 20 insertions(+), 7 deletions(-) diff --git a/api/controllers/v2/github.ts b/api/controllers/v2/github.ts index c35c6dddb..7351c503e 100644 --- a/api/controllers/v2/github.ts +++ b/api/controllers/v2/github.ts @@ -181,13 +181,10 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr //automated test builds will always deploy in dotcomstg const env = 'dotcomstg'; - //TODO: I thought we wanted vpc, not subnets? should we be getting the vars from process.env in this way(taken from runcacherebuildjob) - //no overrides needed, correct? async function runAdditionalECSTasks() { - const { TASK_DEFINITION, CONTAINER_NAME, CLUSTER, SUBNETS } = process.env; + const { TASK_DEFINITION, CLUSTER, SUBNETS } = process.env; if (!TASK_DEFINITION) throw new Error('ERROR! process.env.TASK_DEFINITION is not defined'); - if (!CONTAINER_NAME) throw new Error('ERROR! process.env.CONTAINER_NAME is not defined'); if (!CLUSTER) throw new Error('ERROR! process.env.CLUSTER is not defined'); if (!SUBNETS) throw new Error('ERROR! process.env.SUBNETS is not defined'); diff --git a/cdk-infra/lib/constructs/api/webhook-api-construct.ts b/cdk-infra/lib/constructs/api/webhook-api-construct.ts index 10862f6b8..1ddb886b5 100644 --- a/cdk-infra/lib/constructs/api/webhook-api-construct.ts +++ b/cdk-infra/lib/constructs/api/webhook-api-construct.ts @@ -7,6 +7,8 @@ import { IQueue } from 'aws-cdk-lib/aws-sqs'; import { Construct } from 'constructs'; import path from 'path'; import { getFeatureName } from '../../../utils/env'; +import { IVpc } from 'aws-cdk-lib/aws-ec2'; +import { TaskDefinition } from 'aws-cdk-lib/aws-ecs'; const HANDLERS_PATH = path.join(__dirname, '/../../../../api/controllers/v2'); @@ -31,10 +33,16 @@ interface WebhookApiConstructProps { jobsQueue: IQueue; jobUpdatesQueue: IQueue; environment: Record; + vpc: IVpc; + taskDefinition: TaskDefinition; } export class WebhookApiConstruct extends Construct { - constructor(scope: Construct, id: string, { jobsQueue, jobUpdatesQueue, environment }: WebhookApiConstructProps) { + constructor( + scope: Construct, + id: string, + { jobsQueue, jobUpdatesQueue, environment, vpc, taskDefinition }: WebhookApiConstructProps + ) { super(scope, id); const timeout = Duration.minutes(2); @@ -79,10 +87,16 @@ export class WebhookApiConstruct extends Construct { runtime, handler: 'triggerSmokeTestAutomatedBuild', bundling, - environment, + environment: { + TASK_DEFINITION: taskDefinition.taskDefinitionArn, + SUBNETS: JSON.stringify(vpc.privateSubnets.map((subnet) => subnet.subnetId)), + ...environment, + }, timeout, }); + taskDefinition.grantRun(githubSmokeTestBuildLambda); + const githubDeleteArtifactsLambda = new NodejsFunction(this, 'githubDeleteArtifactsLambda', { entry: `${HANDLERS_PATH}/github.ts`, runtime, diff --git a/cdk-infra/lib/stacks/webhook-stack.ts b/cdk-infra/lib/stacks/webhook-stack.ts index 89d4b163a..a995c6259 100644 --- a/cdk-infra/lib/stacks/webhook-stack.ts +++ b/cdk-infra/lib/stacks/webhook-stack.ts @@ -19,7 +19,7 @@ export class WebhookStack extends Stack { constructor( scope: Construct, id: string, - { queues, webhookSecureStrings, clusterName, ...props }: WebhookStackProps + { queues, webhookSecureStrings, clusterName, vpc, taskDefinition, ...props }: WebhookStackProps ) { super(scope, id, props); @@ -30,6 +30,8 @@ export class WebhookStack extends Stack { new WebhookApiConstruct(this, 'api', { ...queues, + vpc, + taskDefinition, environment: { ...webhookEnvironment, TASK_DEFINITION_FAMILY: clusterName }, }); } From bb99a62161b073b9b17591b3976826b7fdb74e92 Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Thu, 11 Apr 2024 15:25:31 -0400 Subject: [PATCH 223/254] DOP-4451 push to preprd and runTasks --- .github/workflows/deploy-stg-ecs.yml | 1 + .github/workflows/deploy-stg-enhanced-webhooks.yml | 7 ++++--- .github/workflows/deploy-stg-enhanced-worker.yml | 1 + api/controllers/v2/github.ts | 2 +- 4 files changed, 7 insertions(+), 4 deletions(-) diff --git a/.github/workflows/deploy-stg-ecs.yml b/.github/workflows/deploy-stg-ecs.yml index 2e0cfb462..ca799a2e3 100644 --- a/.github/workflows/deploy-stg-ecs.yml +++ b/.github/workflows/deploy-stg-ecs.yml @@ -3,6 +3,7 @@ on: branches: - "main" - "integration" + - "DOP-4451" concurrency: group: environment-stg-${{ github.ref }} cancel-in-progress: true diff --git a/.github/workflows/deploy-stg-enhanced-webhooks.yml b/.github/workflows/deploy-stg-enhanced-webhooks.yml index 97decd119..a2a82d679 100644 --- a/.github/workflows/deploy-stg-enhanced-webhooks.yml +++ b/.github/workflows/deploy-stg-enhanced-webhooks.yml @@ -1,9 +1,10 @@ on: push: - paths: ['api/**', 'cdk-infra/lib/constructs/api/**', 'cdk-infra/utils/**'] + paths: [api/**', 'cdk-infra/lib/constructs/api/**', 'cdk-infra/utils/**'] branches: - - 'main' - - 'integration' + - "main" + - "integration" + - "DOP-4451" concurrency: group: environment-stg-enhanced-webhooks-${{ github.ref }} diff --git a/.github/workflows/deploy-stg-enhanced-worker.yml b/.github/workflows/deploy-stg-enhanced-worker.yml index 24000195c..150d653db 100644 --- a/.github/workflows/deploy-stg-enhanced-worker.yml +++ b/.github/workflows/deploy-stg-enhanced-worker.yml @@ -4,6 +4,7 @@ on: branches: - 'main' - 'integration' + - "DOP-4451" concurrency: group: environment-stg-enhanced-worker-${{ github.ref }} cancel-in-progress: true diff --git a/api/controllers/v2/github.ts b/api/controllers/v2/github.ts index 917f4df39..2f89867c6 100644 --- a/api/controllers/v2/github.ts +++ b/api/controllers/v2/github.ts @@ -242,7 +242,7 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr const jobId = await jobRepository.insertJob(job, c.get('jobsQueueUrl')); jobRepository.notify(jobId, c.get('jobUpdatesQueueUrl'), JobStatus.inQueue, 0); consoleLogger.info(job.title, `Created Job ${jobId}`); - // runAdditionalECSTasks(); + runAdditionalECSTasks(); return jobId; } catch (err) { consoleLogger.error('TriggerBuildError', `${err} Error inserting job for ${repoName}`); From c4ffa7453aaafd2456171bc711ca191d24a80741 Mon Sep 17 00:00:00 2001 From: branberry Date: Thu, 11 Apr 2024 14:34:25 -0500 Subject: [PATCH 224/254] Fix pipeline --- .github/workflows/update-feature-branch.yml | 51 ++++----------------- 1 file changed, 10 insertions(+), 41 deletions(-) diff --git a/.github/workflows/update-feature-branch.yml b/.github/workflows/update-feature-branch.yml index 1f16579d8..8dd204fe6 100644 --- a/.github/workflows/update-feature-branch.yml +++ b/.github/workflows/update-feature-branch.yml @@ -41,47 +41,6 @@ jobs: cdk-infra/node_modules key: ${{ github.head_ref }} build-webhooks: - needs: prep-build - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@v1 - with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws-region: us-east-2 - - uses: actions/setup-node@v4 - with: - node-version: '18.x' - - uses: actions/cache/restore@v3 - id: cache-restore - with: - path: | - node_modules - cdk-infra/node_modules - key: ${{ github.head_ref }} - - name: Install Dependencies - if: steps.cache-restore.outputs.cache-hit != 'true' - run: | - npm ci - cd cdk-infra/ - npm ci - - uses: dorny/paths-filter@v2 - id: filter - with: - filters: | - webhooks: - - 'api/**' - - 'cdk-infra/lib/constructs/api/**' - - 'cdk-infra/utils/**' - - name: Update Webhook Stack - if: steps.filter.outputs.webhooks == 'true' - run: | - cd cdk-infra/ - npm run deploy:feature:stack -- -c isFeatureBranch=true -c env=stg -c customFeatureName=enhancedApp-stg-${{github.head_ref}} \ - auto-builder-stack-enhancedApp-stg-${{github.head_ref}}-webhooks - build-worker: needs: prep-build runs-on: ubuntu-latest steps: @@ -117,9 +76,19 @@ jobs: - 'cdk-infra/lib/constructs/worker/**' - 'Dockerfile' - 'modules/**' + webhooks: + - 'api/**' + - 'cdk-infra/lib/constructs/api/**' + - 'cdk-infra/utils/**' - name: Update Worker Stack if: steps.filter.outputs.worker == 'true' run: | cd cdk-infra/ npm run deploy:feature:stack -- -c isFeatureBranch=true -c env=stg -c customFeatureName=enhancedApp-stg-${{github.head_ref}} \ auto-builder-stack-enhancedApp-stg-${{github.head_ref}}-worker + - name: Update Webhook Stack + if: steps.filter.outputs.webhooks == 'true' + run: | + cd cdk-infra/ + npm run deploy:feature:stack -- -c isFeatureBranch=true -c env=stg -c customFeatureName=enhancedApp-stg-${{github.head_ref}} \ + auto-builder-stack-enhancedApp-stg-${{github.head_ref}}-webhooks From 9e58f368533774c1c76cda3ad866e11dbe72dff3 Mon Sep 17 00:00:00 2001 From: branberry Date: Thu, 11 Apr 2024 15:26:17 -0500 Subject: [PATCH 225/254] Fix pipeline --- .github/workflows/update-feature-branch.yml | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/.github/workflows/update-feature-branch.yml b/.github/workflows/update-feature-branch.yml index 8dd204fe6..6a8d8230f 100644 --- a/.github/workflows/update-feature-branch.yml +++ b/.github/workflows/update-feature-branch.yml @@ -81,14 +81,6 @@ jobs: - 'cdk-infra/lib/constructs/api/**' - 'cdk-infra/utils/**' - name: Update Worker Stack - if: steps.filter.outputs.worker == 'true' run: | cd cdk-infra/ - npm run deploy:feature:stack -- -c isFeatureBranch=true -c env=stg -c customFeatureName=enhancedApp-stg-${{github.head_ref}} \ - auto-builder-stack-enhancedApp-stg-${{github.head_ref}}-worker - - name: Update Webhook Stack - if: steps.filter.outputs.webhooks == 'true' - run: | - cd cdk-infra/ - npm run deploy:feature:stack -- -c isFeatureBranch=true -c env=stg -c customFeatureName=enhancedApp-stg-${{github.head_ref}} \ - auto-builder-stack-enhancedApp-stg-${{github.head_ref}}-webhooks + npm run deploy:feature -- -c isFeatureBranch=true -c env=stg -c customFeatureName=enhancedApp-stg-${{github.head_ref}} From c5b062754081be4beeaeed6784b3f5e60c5f4347 Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Thu, 11 Apr 2024 17:56:46 -0400 Subject: [PATCH 226/254] DOP-4451 remove main branch check --- api/controllers/v2/github.ts | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/api/controllers/v2/github.ts b/api/controllers/v2/github.ts index 2f89867c6..cf23f70a7 100644 --- a/api/controllers/v2/github.ts +++ b/api/controllers/v2/github.ts @@ -170,14 +170,14 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr // if the build was not building main branch, no need for smoke test sites - if (body.workflow_run.head_branch != 'main' || body.repository.fork) { - console.log('Build was not on master branch in main repo, sites will not deploy as no smoke tests are needed'); - return { - statusCode: 202, - headers: { 'Content-Type': 'text/plain' }, - body: `Build on branch ${body.workflow_run.head_branch} will not trigger site deployments as it was not on main branch in upstream repo`, - }; - } + // if (body.workflow_run.head_branch != 'main' || body.repository.fork) { + // console.log('Build was not on master branch in main repo, sites will not deploy as no smoke tests are needed'); + // return { + // statusCode: 202, + // headers: { 'Content-Type': 'text/plain' }, + // body: `Build on branch ${body.workflow_run.head_branch} will not trigger site deployments as it was not on main branch in upstream repo`, + // }; + // } //automated test builds will always deploy in dotcomstg const env = 'dotcomstg'; From f6ca576c9a55f74cf6d3cf1b058ee1a592882cfc Mon Sep 17 00:00:00 2001 From: branberry Date: Mon, 15 Apr 2024 10:30:11 -0500 Subject: [PATCH 227/254] Update deploy step --- .github/workflows/update-feature-branch.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/update-feature-branch.yml b/.github/workflows/update-feature-branch.yml index 6a8d8230f..0192d8961 100644 --- a/.github/workflows/update-feature-branch.yml +++ b/.github/workflows/update-feature-branch.yml @@ -83,4 +83,7 @@ jobs: - name: Update Worker Stack run: | cd cdk-infra/ - npm run deploy:feature -- -c isFeatureBranch=true -c env=stg -c customFeatureName=enhancedApp-stg-${{github.head_ref}} + npm run deploy:feature:stack -- -c isFeatureBranch=true -c env=stg -c customFeatureName=enhancedApp-stg-${{github.head_ref}} \ + auto-builder-stack-enhancedApp-stg-${{github.head_ref}}-worker + npm run deploy:feature:stack -- -c isFeatureBranch=true -c env=stg -c customFeatureName=enhancedApp-stg-${{github.head_ref}} \ + auto-builder-stack-enhancedApp-stg-${{github.head_ref}}-webhooks From 61d1a970fa4600cd4497523d2cb0663e1921ac3f Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Mon, 15 Apr 2024 15:58:52 -0400 Subject: [PATCH 228/254] DOP-4451 undo changes --- .github/workflows/deploy-stg-ecs.yml | 1 - .github/workflows/deploy-stg-enhanced-webhooks.yml | 2 -- 2 files changed, 3 deletions(-) diff --git a/.github/workflows/deploy-stg-ecs.yml b/.github/workflows/deploy-stg-ecs.yml index ca799a2e3..2e0cfb462 100644 --- a/.github/workflows/deploy-stg-ecs.yml +++ b/.github/workflows/deploy-stg-ecs.yml @@ -3,7 +3,6 @@ on: branches: - "main" - "integration" - - "DOP-4451" concurrency: group: environment-stg-${{ github.ref }} cancel-in-progress: true diff --git a/.github/workflows/deploy-stg-enhanced-webhooks.yml b/.github/workflows/deploy-stg-enhanced-webhooks.yml index a2a82d679..11d0f4e91 100644 --- a/.github/workflows/deploy-stg-enhanced-webhooks.yml +++ b/.github/workflows/deploy-stg-enhanced-webhooks.yml @@ -4,8 +4,6 @@ on: branches: - "main" - "integration" - - "DOP-4451" - concurrency: group: environment-stg-enhanced-webhooks-${{ github.ref }} cancel-in-progress: true From 4630351da3f16c28625abd23360320cd2e4cfab7 Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Mon, 15 Apr 2024 16:28:56 -0400 Subject: [PATCH 229/254] DOP-4451 remove branch from enhanced worker yml --- .github/workflows/deploy-stg-enhanced-worker.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/deploy-stg-enhanced-worker.yml b/.github/workflows/deploy-stg-enhanced-worker.yml index 150d653db..24000195c 100644 --- a/.github/workflows/deploy-stg-enhanced-worker.yml +++ b/.github/workflows/deploy-stg-enhanced-worker.yml @@ -4,7 +4,6 @@ on: branches: - 'main' - 'integration' - - "DOP-4451" concurrency: group: environment-stg-enhanced-worker-${{ github.ref }} cancel-in-progress: true From 1425d16e6b7826b53ec619a26b8d9bad21bb3bb0 Mon Sep 17 00:00:00 2001 From: branberry Date: Mon, 15 Apr 2024 16:12:28 -0500 Subject: [PATCH 230/254] Destroy webhooks before updating --- .github/workflows/update-feature-branch.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/update-feature-branch.yml b/.github/workflows/update-feature-branch.yml index 0192d8961..cf9ff4df6 100644 --- a/.github/workflows/update-feature-branch.yml +++ b/.github/workflows/update-feature-branch.yml @@ -83,6 +83,7 @@ jobs: - name: Update Worker Stack run: | cd cdk-infra/ + npm run cdk destroy -- -c customFeatureName=enhancedApp-stg-${{github.head_ref}} --force auto-builder-stack-enhancedApp-stg-${{github.head_ref}}-webhooks npm run deploy:feature:stack -- -c isFeatureBranch=true -c env=stg -c customFeatureName=enhancedApp-stg-${{github.head_ref}} \ auto-builder-stack-enhancedApp-stg-${{github.head_ref}}-worker npm run deploy:feature:stack -- -c isFeatureBranch=true -c env=stg -c customFeatureName=enhancedApp-stg-${{github.head_ref}} \ From 08932c1f0ad2c4a779a4af2c953d37be35985d1a Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Wed, 1 May 2024 15:09:47 -0400 Subject: [PATCH 231/254] DOP-4451 push to preprd --- .github/workflows/deploy-stg-ecs.yml | 1 + .github/workflows/deploy-stg-enhanced-webhooks.yml | 1 + .github/workflows/deploy-stg-enhanced-worker.yml | 1 + 3 files changed, 3 insertions(+) diff --git a/.github/workflows/deploy-stg-ecs.yml b/.github/workflows/deploy-stg-ecs.yml index 2e0cfb462..ca799a2e3 100644 --- a/.github/workflows/deploy-stg-ecs.yml +++ b/.github/workflows/deploy-stg-ecs.yml @@ -3,6 +3,7 @@ on: branches: - "main" - "integration" + - "DOP-4451" concurrency: group: environment-stg-${{ github.ref }} cancel-in-progress: true diff --git a/.github/workflows/deploy-stg-enhanced-webhooks.yml b/.github/workflows/deploy-stg-enhanced-webhooks.yml index 11d0f4e91..230cefba9 100644 --- a/.github/workflows/deploy-stg-enhanced-webhooks.yml +++ b/.github/workflows/deploy-stg-enhanced-webhooks.yml @@ -4,6 +4,7 @@ on: branches: - "main" - "integration" + - "DOP-4451" concurrency: group: environment-stg-enhanced-webhooks-${{ github.ref }} cancel-in-progress: true diff --git a/.github/workflows/deploy-stg-enhanced-worker.yml b/.github/workflows/deploy-stg-enhanced-worker.yml index 24000195c..150d653db 100644 --- a/.github/workflows/deploy-stg-enhanced-worker.yml +++ b/.github/workflows/deploy-stg-enhanced-worker.yml @@ -4,6 +4,7 @@ on: branches: - 'main' - 'integration' + - "DOP-4451" concurrency: group: environment-stg-enhanced-worker-${{ github.ref }} cancel-in-progress: true From e532d4124202c01483320d55d3703566defa7f60 Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Mon, 6 May 2024 11:33:01 -0400 Subject: [PATCH 232/254] DOP-4451 pushing to preprd --- api/controllers/v2/github.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/controllers/v2/github.ts b/api/controllers/v2/github.ts index cf23f70a7..e8c55b74c 100644 --- a/api/controllers/v2/github.ts +++ b/api/controllers/v2/github.ts @@ -168,7 +168,7 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr body: `Workflow ${body.workflow_run.name} completed successfully but only Deploy Staging ECS workflow completion will trigger smoke test site deployments`, }; - // if the build was not building main branch, no need for smoke test sites + // if the build was not building the main branch, no need for smoke test sites // if (body.workflow_run.head_branch != 'main' || body.repository.fork) { // console.log('Build was not on master branch in main repo, sites will not deploy as no smoke tests are needed'); From 6281f69436dc545158cc2e43bd34d2ebbf73350a Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Mon, 6 May 2024 12:43:35 -0400 Subject: [PATCH 233/254] DOP-4451 comment out paths --- .github/workflows/deploy-stg-enhanced-webhooks.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/deploy-stg-enhanced-webhooks.yml b/.github/workflows/deploy-stg-enhanced-webhooks.yml index 230cefba9..46ebbd0d6 100644 --- a/.github/workflows/deploy-stg-enhanced-webhooks.yml +++ b/.github/workflows/deploy-stg-enhanced-webhooks.yml @@ -1,6 +1,6 @@ on: push: - paths: [api/**', 'cdk-infra/lib/constructs/api/**', 'cdk-infra/utils/**'] + # paths: [api/**', 'cdk-infra/lib/constructs/api/**', 'cdk-infra/utils/**'] branches: - "main" - "integration" From df2894ec73981a0fad7af4b38033583eb60841d1 Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Mon, 6 May 2024 15:12:20 -0400 Subject: [PATCH 234/254] DOP-4451 fixing cluster, logging --- api/controllers/v2/github.ts | 7 ++++--- cdk-infra/lib/constructs/api/webhook-api-construct.ts | 5 +++-- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/api/controllers/v2/github.ts b/api/controllers/v2/github.ts index e8c55b74c..6056a8b41 100644 --- a/api/controllers/v2/github.ts +++ b/api/controllers/v2/github.ts @@ -184,11 +184,14 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr async function runAdditionalECSTasks() { const { TASK_DEFINITION, CLUSTER, SUBNETS } = process.env; + console.log('RUN'); if (!TASK_DEFINITION) throw new Error('ERROR! process.env.TASK_DEFINITION is not defined'); if (!CLUSTER) throw new Error('ERROR! process.env.CLUSTER is not defined'); if (!SUBNETS) throw new Error('ERROR! process.env.SUBNETS is not defined'); + console.log('Env Vars exist for run'); + const client = new ECSClient({ region: 'us-east-2', }); @@ -234,7 +237,6 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr repoOwner, }); - //add logic for getting master branch, latest stable branch const job = await prepGithubPushPayload(body, payload, jobTitle); try { @@ -242,7 +244,6 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr const jobId = await jobRepository.insertJob(job, c.get('jobsQueueUrl')); jobRepository.notify(jobId, c.get('jobUpdatesQueueUrl'), JobStatus.inQueue, 0); consoleLogger.info(job.title, `Created Job ${jobId}`); - runAdditionalECSTasks(); return jobId; } catch (err) { consoleLogger.error('TriggerBuildError', `${err} Error inserting job for ${repoName}`); @@ -254,7 +255,7 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr try { const returnVal = await createAndInsertJob(); - // run tasks here + await runAdditionalECSTasks(); return { statusCode: 202, headers: { 'Content-Type': 'text/plain' }, diff --git a/cdk-infra/lib/constructs/api/webhook-api-construct.ts b/cdk-infra/lib/constructs/api/webhook-api-construct.ts index 1ddb886b5..65186cfe5 100644 --- a/cdk-infra/lib/constructs/api/webhook-api-construct.ts +++ b/cdk-infra/lib/constructs/api/webhook-api-construct.ts @@ -35,13 +35,14 @@ interface WebhookApiConstructProps { environment: Record; vpc: IVpc; taskDefinition: TaskDefinition; + clusterName: string; } export class WebhookApiConstruct extends Construct { constructor( scope: Construct, id: string, - { jobsQueue, jobUpdatesQueue, environment, vpc, taskDefinition }: WebhookApiConstructProps + { jobsQueue, jobUpdatesQueue, environment, vpc, taskDefinition, clusterName }: WebhookApiConstructProps ) { super(scope, id); @@ -90,7 +91,7 @@ export class WebhookApiConstruct extends Construct { environment: { TASK_DEFINITION: taskDefinition.taskDefinitionArn, SUBNETS: JSON.stringify(vpc.privateSubnets.map((subnet) => subnet.subnetId)), - ...environment, + CLUSTER: clusterName, }, timeout, }); From 28c7ed07d181f9fab69b44f58f5999e053a74118 Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Mon, 6 May 2024 15:37:36 -0400 Subject: [PATCH 235/254] DOP-4451 fixing comment out runadditionaltasks --- api/controllers/v2/github.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/controllers/v2/github.ts b/api/controllers/v2/github.ts index 6056a8b41..6ea00a31d 100644 --- a/api/controllers/v2/github.ts +++ b/api/controllers/v2/github.ts @@ -255,7 +255,7 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr try { const returnVal = await createAndInsertJob(); - await runAdditionalECSTasks(); + // await runAdditionalECSTasks(); return { statusCode: 202, headers: { 'Content-Type': 'text/plain' }, From a4f9d47c9a649ebf8ea08f074142828bafeb64cb Mon Sep 17 00:00:00 2001 From: branberry Date: Tue, 7 May 2024 09:23:42 -0500 Subject: [PATCH 236/254] [DOP-4451]: Spread env vars for smoke test --- cdk-infra/lib/constructs/api/webhook-api-construct.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/cdk-infra/lib/constructs/api/webhook-api-construct.ts b/cdk-infra/lib/constructs/api/webhook-api-construct.ts index 65186cfe5..ddfde537a 100644 --- a/cdk-infra/lib/constructs/api/webhook-api-construct.ts +++ b/cdk-infra/lib/constructs/api/webhook-api-construct.ts @@ -89,6 +89,7 @@ export class WebhookApiConstruct extends Construct { handler: 'triggerSmokeTestAutomatedBuild', bundling, environment: { + ...environment, TASK_DEFINITION: taskDefinition.taskDefinitionArn, SUBNETS: JSON.stringify(vpc.privateSubnets.map((subnet) => subnet.subnetId)), CLUSTER: clusterName, From 184ff339079705505fa0513a801256e66228fca4 Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Tue, 7 May 2024 10:25:32 -0400 Subject: [PATCH 237/254] DOP-4451 add environment spread --- cdk-infra/lib/constructs/api/webhook-api-construct.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/cdk-infra/lib/constructs/api/webhook-api-construct.ts b/cdk-infra/lib/constructs/api/webhook-api-construct.ts index ddfde537a..03e64134f 100644 --- a/cdk-infra/lib/constructs/api/webhook-api-construct.ts +++ b/cdk-infra/lib/constructs/api/webhook-api-construct.ts @@ -93,6 +93,7 @@ export class WebhookApiConstruct extends Construct { TASK_DEFINITION: taskDefinition.taskDefinitionArn, SUBNETS: JSON.stringify(vpc.privateSubnets.map((subnet) => subnet.subnetId)), CLUSTER: clusterName, + ...environment, }, timeout, }); From 3625d3313baf145b64667072ddbbaa48033b7653 Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Tue, 7 May 2024 10:53:21 -0400 Subject: [PATCH 238/254] DOP-4451 run tasks --- api/controllers/v2/github.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/controllers/v2/github.ts b/api/controllers/v2/github.ts index 6ea00a31d..6056a8b41 100644 --- a/api/controllers/v2/github.ts +++ b/api/controllers/v2/github.ts @@ -255,7 +255,7 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr try { const returnVal = await createAndInsertJob(); - // await runAdditionalECSTasks(); + await runAdditionalECSTasks(); return { statusCode: 202, headers: { 'Content-Type': 'text/plain' }, From 3083fb1524bb5f1070cf2f74437801277a34c15b Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Tue, 7 May 2024 11:39:39 -0400 Subject: [PATCH 239/254] DOP-4451 update dockerfile to use main branch of snooty --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 65be09e59..cc75bf9e5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -75,7 +75,7 @@ ENV PATH="${WORK_DIRECTORY}/.cargo/bin:${PATH}" RUN curl https://raw.githubusercontent.com/mongodb/docs-worker-pool/meta/makefiles/shared.mk -o shared.mk # install snooty frontend and docs-tools -RUN git clone -b v${SNOOTY_FRONTEND_VERSION} --depth 1 https://github.com/mongodb/snooty.git \ +RUN git clone -b main --depth 1 https://github.com/mongodb/snooty.git \ && cd snooty \ # Need to remove omit dev as the filter functionality for the frontend depends on a dev dependency. && npm ci --legacy-peer-deps \ From 8fd47c936f6a8950d6a73ef43e96415a68e20002 Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Tue, 7 May 2024 11:57:28 -0400 Subject: [PATCH 240/254] DOP-4451 move spin up additional tasks into for loop --- api/controllers/v2/github.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/controllers/v2/github.ts b/api/controllers/v2/github.ts index 6056a8b41..75c631c38 100644 --- a/api/controllers/v2/github.ts +++ b/api/controllers/v2/github.ts @@ -240,6 +240,7 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr const job = await prepGithubPushPayload(body, payload, jobTitle); try { + await runAdditionalECSTasks(); consoleLogger.info(job.title, 'Creating Job'); const jobId = await jobRepository.insertJob(job, c.get('jobsQueueUrl')); jobRepository.notify(jobId, c.get('jobUpdatesQueueUrl'), JobStatus.inQueue, 0); @@ -255,7 +256,6 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr try { const returnVal = await createAndInsertJob(); - await runAdditionalECSTasks(); return { statusCode: 202, headers: { 'Content-Type': 'text/plain' }, From 2be06746680226d2de2405188379d9d3f3f6d7e9 Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Tue, 7 May 2024 12:55:11 -0400 Subject: [PATCH 241/254] DOP-4451 webhookApiConstruct props --- cdk-infra/lib/stacks/webhook-stack.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cdk-infra/lib/stacks/webhook-stack.ts b/cdk-infra/lib/stacks/webhook-stack.ts index a995c6259..6f7ab5e4b 100644 --- a/cdk-infra/lib/stacks/webhook-stack.ts +++ b/cdk-infra/lib/stacks/webhook-stack.ts @@ -5,6 +5,7 @@ import { TaskDefinition } from 'aws-cdk-lib/aws-ecs'; import { AutoBuilderQueues } from './auto-builder-queue-stack'; import { WebhookApiConstruct } from '../constructs/api/webhook-api-construct'; import { WebhookEnvConstruct } from '../constructs/api/webhook-env-construct'; +import cluster from 'cluster'; interface WebhookStackProps extends StackProps { webhookSecureStrings: Record; @@ -31,6 +32,7 @@ export class WebhookStack extends Stack { new WebhookApiConstruct(this, 'api', { ...queues, vpc, + clusterName, taskDefinition, environment: { ...webhookEnvironment, TASK_DEFINITION_FAMILY: clusterName }, }); From ee7b1c9c8f5bc3799e2d4f468108e22feae74600 Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Tue, 7 May 2024 15:29:02 -0400 Subject: [PATCH 242/254] DOP-4451 testing slack stuff and updating props --- api/handlers/slack.ts | 2 ++ cdk-infra/lib/stacks/webhook-stack.ts | 1 - src/repositories/repoBranchesRepository.ts | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/api/handlers/slack.ts b/api/handlers/slack.ts index 83d42c086..1cb5edb6a 100644 --- a/api/handlers/slack.ts +++ b/api/handlers/slack.ts @@ -26,6 +26,8 @@ export async function buildEntitledGroupsList(entitlement: any, repoBranchesRepo const branches = await repoBranchesRepository.getRepoBranches(repoName, directoryPath); const options: any[] = []; + console.log(repoName); + console.log(branches.length); for (const branch of branches) { const buildWithSnooty = branch['buildsWithSnooty']; if (buildWithSnooty) { diff --git a/cdk-infra/lib/stacks/webhook-stack.ts b/cdk-infra/lib/stacks/webhook-stack.ts index 6f7ab5e4b..303e1642d 100644 --- a/cdk-infra/lib/stacks/webhook-stack.ts +++ b/cdk-infra/lib/stacks/webhook-stack.ts @@ -5,7 +5,6 @@ import { TaskDefinition } from 'aws-cdk-lib/aws-ecs'; import { AutoBuilderQueues } from './auto-builder-queue-stack'; import { WebhookApiConstruct } from '../constructs/api/webhook-api-construct'; import { WebhookEnvConstruct } from '../constructs/api/webhook-env-construct'; -import cluster from 'cluster'; interface WebhookStackProps extends StackProps { webhookSecureStrings: Record; diff --git a/src/repositories/repoBranchesRepository.ts b/src/repositories/repoBranchesRepository.ts index 20bce0aeb..34f59d170 100644 --- a/src/repositories/repoBranchesRepository.ts +++ b/src/repositories/repoBranchesRepository.ts @@ -32,7 +32,7 @@ export class RepoBranchesRepository extends BaseRepository { return repo?.['branches'] ?? []; } - async getProdDeployableRepoBranches(): Promise { + async getProdDeployableRepos(): Promise { const reposArray = await this._collection .aggregate([{ $match: { prodDeployable: true, internalOnly: false } }, { $project: { _id: 0, repoName: 1 } }]) .toArray(); From 8a839e20c0dbec2bb6a33f00f33202370b83e72a Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Tue, 7 May 2024 15:41:32 -0400 Subject: [PATCH 243/254] DOP-4451 fix name --- src/services/slack.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/services/slack.ts b/src/services/slack.ts index 229646f1e..5f4898eb3 100644 --- a/src/services/slack.ts +++ b/src/services/slack.ts @@ -72,7 +72,7 @@ export class SlackConnector implements ISlackConnector { } values['deploy_option'] = 'deploy_all'; - values['repo_option'] = await repoBranchesRepository.getProdDeployableRepoBranches(); + values['repo_option'] = await repoBranchesRepository.getProdDeployableRepos(); return values; } From 2576176d7269f951f6be2e116244ecdb0ae5dfed Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Tue, 7 May 2024 16:48:03 -0400 Subject: [PATCH 244/254] DOP-4451 logging --- api/controllers/v1/slack.ts | 2 ++ src/services/slack.ts | 1 + 2 files changed, 3 insertions(+) diff --git a/api/controllers/v1/slack.ts b/api/controllers/v1/slack.ts index 3b2f71864..500abc3b1 100644 --- a/api/controllers/v1/slack.ts +++ b/api/controllers/v1/slack.ts @@ -49,7 +49,9 @@ export const DisplayRepoOptions = async (event: APIGatewayEvent): Promise { const repoOptView = this._getDropDownView(triggerId, repos, isAdmin); + console.log('got dropdown view'); const slackToken = this._config.get('slackAuthToken'); const slackUrl = this._config.get('slackViewOpenUrl'); return await axiosApi.post(slackUrl, repoOptView, { From 3c398d3ce34a0c87e4b49e0177b1c29ee503b2a9 Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Tue, 7 May 2024 17:14:28 -0400 Subject: [PATCH 245/254] DOP-4451 fix logging --- api/controllers/v1/slack.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/controllers/v1/slack.ts b/api/controllers/v1/slack.ts index 500abc3b1..a09eda38f 100644 --- a/api/controllers/v1/slack.ts +++ b/api/controllers/v1/slack.ts @@ -51,7 +51,7 @@ export const DisplayRepoOptions = async (event: APIGatewayEvent): Promise Date: Tue, 7 May 2024 18:07:00 -0400 Subject: [PATCH 246/254] DOP-4451 logging --- api/controllers/v1/slack.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/api/controllers/v1/slack.ts b/api/controllers/v1/slack.ts index a09eda38f..66c504028 100644 --- a/api/controllers/v1/slack.ts +++ b/api/controllers/v1/slack.ts @@ -203,6 +203,7 @@ export const DeployRepo = async (event: any = {}): Promise => { const decoded = decodeURIComponent(event.body).split('=')[1]; const parsed = JSON.parse(decoded); const stateValues = parsed.view.state.values; + console.log(JSON.stringify(stateValues)); //TODO: create an interface for slack view_submission payloads if (parsed.type !== 'view_submission') { From 50d2c9e7e6d6e1df3600eefed9f16d16119fda85 Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Tue, 7 May 2024 18:38:48 -0400 Subject: [PATCH 247/254] DOP-4451 logging --- api/handlers/slack.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/api/handlers/slack.ts b/api/handlers/slack.ts index 1cb5edb6a..634cde712 100644 --- a/api/handlers/slack.ts +++ b/api/handlers/slack.ts @@ -21,10 +21,11 @@ export function prepResponse(statusCode, contentType, body) { //if person is admin, get all prod deployable repos export async function buildEntitledGroupsList(entitlement: any, repoBranchesRepository: RepoBranchesRepository) { const repoOptions: any[] = []; + let count = 0; for (const repo of entitlement.repos) { const [repoOwner, repoName, directoryPath] = repo.split('/'); - const branches = await repoBranchesRepository.getRepoBranches(repoName, directoryPath); + count += branches.length; const options: any[] = []; console.log(repoName); console.log(branches.length); @@ -65,6 +66,7 @@ export async function buildEntitledGroupsList(entitlement: any, repoBranchesRepo }; repoOptions.push(repoOption); } + console.log(count); return repoOptions.sort((repoOne, repoTwo) => repoOne.label.text.localeCompare(repoTwo.label.text)); } From 9b0c30d52fc343873927b1d53e63d73529dbf73d Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Wed, 8 May 2024 13:43:52 -0400 Subject: [PATCH 248/254] DOP-4451 comment out runadditionaltasks --- .github/workflows/deploy-stg-ecs.yml | 2 +- api/controllers/v1/slack.ts | 2 +- api/controllers/v2/github.ts | 2 +- api/handlers/slack.ts | 3 --- 4 files changed, 3 insertions(+), 6 deletions(-) diff --git a/.github/workflows/deploy-stg-ecs.yml b/.github/workflows/deploy-stg-ecs.yml index ca799a2e3..03f08b943 100644 --- a/.github/workflows/deploy-stg-ecs.yml +++ b/.github/workflows/deploy-stg-ecs.yml @@ -3,7 +3,7 @@ on: branches: - "main" - "integration" - - "DOP-4451" + # - "DOP-4451" concurrency: group: environment-stg-${{ github.ref }} cancel-in-progress: true diff --git a/api/controllers/v1/slack.ts b/api/controllers/v1/slack.ts index 66c504028..c4591c002 100644 --- a/api/controllers/v1/slack.ts +++ b/api/controllers/v1/slack.ts @@ -52,7 +52,7 @@ export const DisplayRepoOptions = async (event: APIGatewayEvent): Promise repoOne.label.text.localeCompare(repoTwo.label.text)); } From 79ebea7ecc1f0e6e5a89090d53a9aa8a2f2e1333 Mon Sep 17 00:00:00 2001 From: branberry Date: Wed, 8 May 2024 13:48:43 -0500 Subject: [PATCH 249/254] [DOP-4451]: Update protect tasks --- api/controllers/v2/github.ts | 3 +++ src/enhanced/utils/queue/index.ts | 4 +++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/api/controllers/v2/github.ts b/api/controllers/v2/github.ts index b96e76e22..0d00f47ad 100644 --- a/api/controllers/v2/github.ts +++ b/api/controllers/v2/github.ts @@ -200,6 +200,9 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr taskDefinition: TASK_DEFINITION, cluster: CLUSTER, launchType: 'FARGATE', + overrides: { + containerOverrides: [{ environment: [{ name: 'INDPENDENT_TASK', value: 'true' }] }], + }, networkConfiguration: { awsvpcConfiguration: { subnets: JSON.parse(SUBNETS), diff --git a/src/enhanced/utils/queue/index.ts b/src/enhanced/utils/queue/index.ts index 289ea5306..977d8ea37 100644 --- a/src/enhanced/utils/queue/index.ts +++ b/src/enhanced/utils/queue/index.ts @@ -39,7 +39,9 @@ export async function listenToJobQueue(): Promise { // NOTE: Intentionally not catching here, as this throw should be handled by the method listening to the queue. // We don't want to continue listening to the queue, as there is something wrong with the protect task mechanism. // We can let the task end, as it is unsafe to let an unprotected task process a job. - await protectTask(); + if (process.env.INDEPENDENT_TASK !== 'yes') { + await protectTask(); + } console.log('[listenToJobQueue]: Deleting message...'); From 94208b246518841ec5ad207fd468098963ec62c1 Mon Sep 17 00:00:00 2001 From: branberry Date: Wed, 8 May 2024 13:56:43 -0500 Subject: [PATCH 250/254] [DOP-4451]: Refactor enhanced deploy --- .github/workflows/deploy-stg-enhanced-worker.yml | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/.github/workflows/deploy-stg-enhanced-worker.yml b/.github/workflows/deploy-stg-enhanced-worker.yml index 150d653db..0381f6860 100644 --- a/.github/workflows/deploy-stg-enhanced-worker.yml +++ b/.github/workflows/deploy-stg-enhanced-worker.yml @@ -1,10 +1,10 @@ on: push: - paths: ['src/**', 'cdk-infra/lib/constructs/worker/**', 'Dockerfile', 'modules/**'] + # paths: ['src/**', 'cdk-infra/lib/constructs/worker/**', 'Dockerfile', 'modules/**'] branches: - 'main' - 'integration' - - "DOP-4451" + - 'DOP-4451' concurrency: group: environment-stg-enhanced-worker-${{ github.ref }} cancel-in-progress: true @@ -28,5 +28,8 @@ jobs: npm ci cd cdk-infra/ npm ci - npm run deploy:feature:stack -- -c env=dotcomstg -c customFeatureName=enhancedApp-dotcomstg auto-builder-stack-enhancedApp-dotcomstg-worker - npm run deploy:feature:stack -- -c env=stg -c customFeatureName=enhancedApp-stg auto-builder-stack-enhancedApp-stg-worker + npm run cdk destroy -- -c customFeatureName=enhancedApp-dotcomstg --force auto-builder-stack-enhancedApp-dotcomstg-webhooks + npm run deploy:feature:stack -- -c isFeatureBranch=true -c env=dotcomstg -c customFeatureName=enhancedApp-dotcomstg \ + auto-builder-stack-enhancedApp-dotcomstg-worker + npm run deploy:feature:stack -- -c isFeatureBranch=true -c env=dotcomstg -c customFeatureName=enhancedApp-dotcomstg \ + auto-builder-stack-enhancedApp-dotcomstg-webhooks From f1b61184555a269776c3b54815fb5b26d3d74d9b Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Thu, 9 May 2024 12:06:12 -0400 Subject: [PATCH 251/254] DOP-4451 deploy to preprd with additional slack check --- .github/workflows/deploy-stg-ecs.yml | 2 +- api/handlers/slack.ts | 74 ++++++++++--------- .../constructs/api/webhook-api-construct.ts | 1 - 3 files changed, 39 insertions(+), 38 deletions(-) diff --git a/.github/workflows/deploy-stg-ecs.yml b/.github/workflows/deploy-stg-ecs.yml index 03f08b943..ca799a2e3 100644 --- a/.github/workflows/deploy-stg-ecs.yml +++ b/.github/workflows/deploy-stg-ecs.yml @@ -3,7 +3,7 @@ on: branches: - "main" - "integration" - # - "DOP-4451" + - "DOP-4451" concurrency: group: environment-stg-${{ github.ref }} cancel-in-progress: true diff --git a/api/handlers/slack.ts b/api/handlers/slack.ts index 4c4399d46..42a0dbc99 100644 --- a/api/handlers/slack.ts +++ b/api/handlers/slack.ts @@ -24,45 +24,47 @@ export async function buildEntitledGroupsList(entitlement: any, repoBranchesRepo for (const repo of entitlement.repos) { const [repoOwner, repoName, directoryPath] = repo.split('/'); const branches = await repoBranchesRepository.getRepoBranches(repoName, directoryPath); - const options: any[] = []; - console.log(repoName); - console.log(branches.length); - for (const branch of branches) { - const buildWithSnooty = branch['buildsWithSnooty']; - if (buildWithSnooty) { - const active = branch['active']; - const branchName = `${directoryPath ? `${directoryPath}/` : ''}${branch['gitBranchName']}`; - const repoPath = `${repoOwner}/${repoName}/${branchName}`; - let txt: string; - if (!active) { - txt = `(!inactive) ${repoPath}`; - } else { - txt = repoPath; + if (branches.length) { + const options: any[] = []; + console.log(repoName); + console.log(branches.length); + for (const branch of branches) { + const buildWithSnooty = branch['buildsWithSnooty']; + if (buildWithSnooty) { + const active = branch['active']; + const branchName = `${directoryPath ? `${directoryPath}/` : ''}${branch['gitBranchName']}`; + const repoPath = `${repoOwner}/${repoName}/${branchName}`; + let txt: string; + if (!active) { + txt = `(!inactive) ${repoPath}`; + } else { + txt = repoPath; + } + options.push({ + text: { + type: 'plain_text', + text: txt, + }, + value: repoPath, + }); } - options.push({ - text: { - type: 'plain_text', - text: txt, - }, - value: repoPath, - }); } - } - const repoOption = { - label: { - type: 'plain_text', - text: repoName, - }, - //sort the options by version number - options: options.sort((branchOne, branchTwo) => - branchTwo.text.text - .toString() - .replace(/\d+/g, (n) => +n + 100000) - .localeCompare(branchOne.text.text.toString().replace(/\d+/g, (n) => +n + 100000)) - ), - }; - repoOptions.push(repoOption); + const repoOption = { + label: { + type: 'plain_text', + text: repoName, + }, + //sort the options by version number + options: options.sort((branchOne, branchTwo) => + branchTwo.text.text + .toString() + .replace(/\d+/g, (n) => +n + 100000) + .localeCompare(branchOne.text.text.toString().replace(/\d+/g, (n) => +n + 100000)) + ), + }; + repoOptions.push(repoOption); + } } return repoOptions.sort((repoOne, repoTwo) => repoOne.label.text.localeCompare(repoTwo.label.text)); } diff --git a/cdk-infra/lib/constructs/api/webhook-api-construct.ts b/cdk-infra/lib/constructs/api/webhook-api-construct.ts index 3de771794..c0855349d 100644 --- a/cdk-infra/lib/constructs/api/webhook-api-construct.ts +++ b/cdk-infra/lib/constructs/api/webhook-api-construct.ts @@ -89,7 +89,6 @@ export class WebhookApiConstruct extends Construct { handler: 'triggerSmokeTestAutomatedBuild', bundling, environment: { - ...environment, TASK_DEFINITION: taskDefinition.taskDefinitionArn, SUBNETS: JSON.stringify(vpc.privateSubnets.map((subnet) => subnet.subnetId)), CLUSTER: clusterName, From cc44ac80970e6a3797e7ac0d2a2386b1abc5e98f Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Thu, 9 May 2024 12:39:40 -0400 Subject: [PATCH 252/254] DOP-4451 adding logging --- api/controllers/v1/slack.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/api/controllers/v1/slack.ts b/api/controllers/v1/slack.ts index c4591c002..93d05b3f4 100644 --- a/api/controllers/v1/slack.ts +++ b/api/controllers/v1/slack.ts @@ -203,7 +203,8 @@ export const DeployRepo = async (event: any = {}): Promise => { const decoded = decodeURIComponent(event.body).split('=')[1]; const parsed = JSON.parse(decoded); const stateValues = parsed.view.state.values; - console.log(JSON.stringify(stateValues)); + console.info('testing', 'logging here'); + console.log(JSON.stringify(parsed.view)); //TODO: create an interface for slack view_submission payloads if (parsed.type !== 'view_submission') { From c47bdd6458ee64def207b3ae25996fcd82a2bd6b Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Thu, 9 May 2024 13:42:02 -0400 Subject: [PATCH 253/254] DOP-4451 remove some logs --- api/controllers/v1/slack.ts | 2 -- api/handlers/slack.ts | 2 ++ 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/api/controllers/v1/slack.ts b/api/controllers/v1/slack.ts index 93d05b3f4..f8d9e6c34 100644 --- a/api/controllers/v1/slack.ts +++ b/api/controllers/v1/slack.ts @@ -49,9 +49,7 @@ export const DisplayRepoOptions = async (event: APIGatewayEvent): Promise repoOne.label.text.localeCompare(repoTwo.label.text)); From 884c49204d64496dd94409beca4bb67992ad5cf1 Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Thu, 9 May 2024 13:43:10 -0400 Subject: [PATCH 254/254] DOP-4451 remove some logs --- api/handlers/slack.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/api/handlers/slack.ts b/api/handlers/slack.ts index 6134cc221..1cb90cdd6 100644 --- a/api/handlers/slack.ts +++ b/api/handlers/slack.ts @@ -26,8 +26,6 @@ export async function buildEntitledGroupsList(entitlement: any, repoBranchesRepo const branches = await repoBranchesRepository.getRepoBranches(repoName, directoryPath); if (branches.length) { const options: any[] = []; - console.log(repoName); - console.log(branches.length); for (const branch of branches) { const buildWithSnooty = branch['buildsWithSnooty']; if (buildWithSnooty) {