From 478ebf3a20f90343b30496e5a943a747a8a3abd3 Mon Sep 17 00:00:00 2001 From: Matt Meigs Date: Fri, 17 Nov 2023 10:12:21 -0500 Subject: [PATCH] persistence module --- src/commands/src/shared/next-gen-parse.ts | 25 +++--- src/commands/src/shared/next-gen-stage.ts | 43 +++++++--- src/commands/src/shared/oas-page-build.ts | 19 ++--- src/commands/src/shared/persistence-module.ts | 28 ++++--- src/job/jobHandler.ts | 81 ++++++++++++------- 5 files changed, 117 insertions(+), 79 deletions(-) diff --git a/src/commands/src/shared/next-gen-parse.ts b/src/commands/src/shared/next-gen-parse.ts index 7b1d1bf2c..5391b6492 100644 --- a/src/commands/src/shared/next-gen-parse.ts +++ b/src/commands/src/shared/next-gen-parse.ts @@ -1,22 +1,19 @@ -import { IJobRepoLogger } from '../../../services/logger'; +import path from 'path'; +import { Job } from '../../../entities/job'; +import { getDirectory } from '../../../job/jobHandler'; import { CliCommandResponse, executeCliCommand } from '../helpers'; const RSTSPEC_FLAG = '--rstspec=https://raw.githubusercontent.com/mongodb/snooty-parser/latest/snooty/rstspec.toml'; interface NextGenParseParams { - repoDir: string; - commitHash?: string; - patchId?: string; - logger: IJobRepoLogger; - id: string; + job: Job; + preppedLogger: (message: string) => void; } -export async function nextGenParse({ - repoDir, - patchId, - commitHash, - logger, - id, -}: NextGenParseParams): Promise { +export async function nextGenParse({ job, preppedLogger }: NextGenParseParams): Promise { + const repoDir = path.resolve(process.cwd(), `repos/${getDirectory(job)}`); + const commitHash = job.payload.newHead; + const patchId = job.payload.patch; + const commandArgs = ['build', repoDir, '--output', `${repoDir}/bundle.zip`]; if (patchId && commitHash) { @@ -29,7 +26,7 @@ export async function nextGenParse({ commandArgs.push(RSTSPEC_FLAG); - logger.save(id, `COMMAND for parse: ${commandArgs.join(' ')}`); + preppedLogger(`COMMAND for parse: ${commandArgs.join(' ')}`); return executeCliCommand({ command: 'snooty', args: commandArgs }); } diff --git a/src/commands/src/shared/next-gen-stage.ts b/src/commands/src/shared/next-gen-stage.ts index 92cb338f2..fd3879117 100644 --- a/src/commands/src/shared/next-gen-stage.ts +++ b/src/commands/src/shared/next-gen-stage.ts @@ -1,3 +1,4 @@ +import { Job } from '../../../entities/job'; import { executeCliCommand } from '../helpers'; const DOCS_WORKER_USER = 'docsworker-xlarge'; @@ -10,29 +11,43 @@ interface StageParams { patchId?: string; commitBranch: string; commitHash: string; + job: Job; + preppedLogger: (message: string) => void; } export async function nextGenStage({ - mutPrefix, - projectName, - bucket, - url, - patchId, - commitBranch, - commitHash, -}: StageParams) { - let hostedAtUrl = `${url}/${mutPrefix}/${DOCS_WORKER_USER}/${commitBranch}/`; - let prefix = mutPrefix; + job, + preppedLogger, +}: // mutPrefix, +// projectName, +// bucket, +// url, +// patchId, +// commitBranch, +// commitHash, +StageParams) { + // TODO: replace with a process to get this url?? + const baseUrl = 'https://mongodbcom-cdn.website.staging.corp.mongodb.com'; + // TODO: replace with process to access bucket + const bucket = 'docs-atlas-dotcomstg'; + const { mutPrefix, branchName, patch, project, newHead } = job.payload; + + // TODO: Figure out correct hostedAtUrl + let hostedAtUrl = `${baseUrl}/${mutPrefix}/${DOCS_WORKER_USER}/${branchName}/`; + // TODO: Look further into all possible needs for prefix... + let prefix = mutPrefix || project; const commandArgs = ['public', bucket, '--stage']; - if (patchId && projectName === mutPrefix) { - prefix = `${commitHash}/${patchId}/${mutPrefix}`; - hostedAtUrl = `${url}/${commitHash}/${patchId}/${mutPrefix}/${DOCS_WORKER_USER}/${commitBranch}/`; + if (patch && newHead && project === mutPrefix) { + prefix = `${newHead}/${patch}/${mutPrefix}`; + hostedAtUrl = `${baseUrl}/${newHead}/${patch}/${mutPrefix}/${DOCS_WORKER_USER}/${branchName}/`; } commandArgs.push(`--prefix=${prefix}`); + preppedLogger(`MUT PUBLISH command args: ${commandArgs}`); + const { outputText } = await executeCliCommand({ command: 'mut-publish', args: commandArgs, @@ -42,6 +57,8 @@ export async function nextGenStage({ }); const resultMessage = `${outputText}\n Hosted at ${hostedAtUrl}\n\nHere are the commands: ${commandArgs}`; + preppedLogger(`OUTPUT of mut publish: ${resultMessage}`); + return { resultMessage, commands: commandArgs, diff --git a/src/commands/src/shared/oas-page-build.ts b/src/commands/src/shared/oas-page-build.ts index bed330664..1d86dc244 100644 --- a/src/commands/src/shared/oas-page-build.ts +++ b/src/commands/src/shared/oas-page-build.ts @@ -1,23 +1,24 @@ import { Job } from '../../../entities/job'; -import { IJobRepoLogger } from '../../../services/logger'; import { executeCliCommand, getRepoDir } from '../helpers'; interface OasPageBuildParams { job: Job; - baseUrl: string; - logger: IJobRepoLogger; + preppedLogger: (message: string) => void; } -export async function oasPageBuild({ job, baseUrl, logger }: OasPageBuildParams) { +export async function oasPageBuild({ job, preppedLogger }: OasPageBuildParams) { + // TODO: replace with a process to get this url?? + const baseUrl = 'https://mongodbcom-cdn.website.staging.corp.mongodb.com'; + const siteUrl = job.payload.mutPrefix ? `${baseUrl}/${job.payload.mutPrefix}` : `${baseUrl}/${job.payload.project}/docsworker-xlarge/${job.payload.branchName}`; console.log('siteUrl: ', siteUrl); - logger.save(job._id, `Is there a mutprefix?? : ${job.payload.mutPrefix}`); - logger.save(job._id, `SITE URL: ${siteUrl}`); + preppedLogger(`Is there a mutprefix?? : ${job.payload.mutPrefix}`); + preppedLogger(`SITE URL: ${siteUrl}`); const repoDir = getRepoDir(job.payload.repoName, job.payload.directory); const bundlePath = `${repoDir}/bundle.zip`; - logger.save(job._id, `BUNDLE PATH ? : ${bundlePath}`); + preppedLogger(`BUNDLE PATH ? : ${bundlePath}`); try { const { outputText } = await executeCliCommand({ @@ -37,11 +38,11 @@ export async function oasPageBuild({ job, baseUrl, logger }: OasPageBuildParams) ], }); - logger.save(job._id, `OAS page builder output text? : ${outputText}`); + preppedLogger(`OAS page builder output text? : ${outputText}`); return outputText; } catch (error) { - logger.save(job._id, `Caught error from OAS page builder cli!: ${error}\n\n`); + preppedLogger(`Caught error from OAS page builder cli!: ${error}\n\n`); return ''; } diff --git a/src/commands/src/shared/persistence-module.ts b/src/commands/src/shared/persistence-module.ts index e1b7851e9..a1b21517c 100644 --- a/src/commands/src/shared/persistence-module.ts +++ b/src/commands/src/shared/persistence-module.ts @@ -1,33 +1,37 @@ -import { executeCliCommand } from '../helpers'; +import { Job } from '../../../entities/job'; +import { executeCliCommand, getRepoDir } from '../helpers'; interface PersistenceModuleParams { - bundlePath: string; - jobId?: string; - repoOwner?: string; + job: Job; + preppedLogger: (message: string) => void; } -export async function persistenceModule({ - bundlePath, - jobId, - repoOwner = 'docs-builder-bot', -}: PersistenceModuleParams) { +export async function persistenceModule({ job, preppedLogger }: PersistenceModuleParams) { + const githubUser = job.payload.repoOwner; + const repoDir = getRepoDir(job.payload.repoName, job.payload.directory); + const bundlePath = `${repoDir}/bundle.zip`; + const args = [ `${process.cwd()}/modules/persistence/dist/index.js`, '--unhandled-rejections=strict', '--path', bundlePath, '--githubUser', - repoOwner, + githubUser, ]; - if (jobId) { + if (job._id) { args.push('--jobId'); - args.push(jobId); + args.push(job._id); } + preppedLogger(`persistence args: ${args}`); + const { outputText } = await executeCliCommand({ command: 'node', args, }); + preppedLogger(`output text persistence: ${outputText}`); + return outputText; } diff --git a/src/job/jobHandler.ts b/src/job/jobHandler.ts index b516ba0a0..3e8c3cc62 100644 --- a/src/job/jobHandler.ts +++ b/src/job/jobHandler.ts @@ -14,7 +14,7 @@ import { IJobValidator } from './jobValidator'; import { RepoEntitlementsRepository } from '../repositories/repoEntitlementsRepository'; import { DocsetsRepository } from '../repositories/docsetsRepository'; import { MONOREPO_NAME } from '../monorepo/utils/monorepo-constants'; -import { nextGenHtml, nextGenParse, oasPageBuild } from '../commands'; +import { nextGenHtml, nextGenParse, oasPageBuild, persistenceModule } from '../commands'; import { getRepoDir } from '../commands/src/helpers'; require('fs'); @@ -299,6 +299,16 @@ export abstract class JobHandler { ['oas-page-build']: 'oasPageBuildExe', }; + const commandMap: { + [K: string]: ({ job, preppedLogger }: { job: Job; preppedLogger: (message: string) => void }) => any; + } = { + // ['get-build-dependencies']: 'buildDepsExe', + ['next-gen-parse']: nextGenParse, + ['persistence-module']: persistenceModule, + // ['next-gen-html']: 'htmlExe', + ['oas-page-build']: oasPageBuild, + }; + // get the prerequisite commands which should be all commands up to `rm -f makefile` const endOfPrerequisiteCommands = this.currJob.buildCommands.indexOf('rm -f makefile'); const index = endOfPrerequisiteCommands + 1; @@ -314,40 +324,49 @@ export abstract class JobHandler { // call prerequisite commands await this._commandExecutor.execute(prerequisiteCommands); + // create constants for command inputs + const thisLogger = this._logger; + const thisJob = this.currJob; + const preppedLogger = (message: string) => { + thisLogger.save(thisJob._id, message); + }; + const repoDir = path.resolve(process.cwd(), `repos/${getDirectory(this.currJob)}`); + for (const command of makeCommands) { // works for any make command with the following signature make const key = command.split(' ')[1].trim(); this._logger.save(this.currJob._id, `command: ${command}`); - if (key === 'next-gen-parse') { - this._logger.save(this.currJob._id, `in parse command!!! `); - this._logger.save(this.currJob._id, `repoDir: "repos/${getDirectory(this.currJob)}" `); - this._logger.save(this.currJob._id, `process.cwd!! : ${process.cwd()}`); - const repoDir = path.resolve(process.cwd(), `repos/${getDirectory(this.currJob)}`); - - this._logger.save(this.currJob._id, `repoDir now : ${repoDir}`); - try { - const snootyParseRes = await nextGenParse({ - repoDir, - commitHash: this.currJob.payload.newHead ?? '', - logger: this._logger, - id: this.currJob._id, - }); - this._logger.save( - this.currJob._id, - `nextGenParse response: "${snootyParseRes.outputText}" - or error: "${snootyParseRes.errorText}"\n\n\n` - ); - } catch (err) { - this._logger.save(this.currJob._id, `ERROR`); - this._logger.save(this.currJob._id, `ERROR: ${err}`); - } - } else if (key === 'oas-page-build') { - this._logger.save(this.currJob._id, `IN oas page build`); - const result = await oasPageBuild({ - job: this.currJob, - baseUrl: 'https://mongodbcom-cdn.website.staging.corp.mongodb.com', - logger: this._logger, - }); - this._logger.save(this.currJob._id, `oas page build result... ${result}`); + if (commandMap[key]) { + this._logger.save(this.currJob._id, `running from commandMap: ${key}`); + await commandMap[key]({ job: this.currJob, preppedLogger }); + // } + // if (key === 'next-gen-parse') { + // // this._logger.save(this.currJob._id, `in parse command!!! `); + // // this._logger.save(this.currJob._id, `repoDir: "repos/${getDirectory(this.currJob)}" `); + // // this._logger.save(this.currJob._id, `process.cwd!! : ${process.cwd()}`); + // // const repoDir = path.resolve(process.cwd(), `repos/${getDirectory(this.currJob)}`); + + // this._logger.save(this.currJob._id, `repoDir now : ${repoDir}`); + // try { + // const snootyParseRes = await nextGenParse({ + // job: this.currJob, + // preppedLogger, + // }); + // this._logger.save( + // this.currJob._id, + // `nextGenParse response: "${snootyParseRes.outputText}" - or error: "${snootyParseRes.errorText}"\n\n\n` + // ); + // } catch (err) { + // this._logger.save(this.currJob._id, `ERROR`); + // this._logger.save(this.currJob._id, `ERROR: ${err}`); + // } + // } else if (key === 'oas-page-build') { + // this._logger.save(this.currJob._id, `IN oas page build`); + // const result = await oasPageBuild({ + // job: this.currJob, + // preppedLogger, + // }); + // this._logger.save(this.currJob._id, `oas page build result... ${result}`); } else { if (stages[key]) { const makeCommandsWithBenchmarksResponse = await this.callWithBenchmark(command, stages[key]);