Skip to content

Commit

Permalink
persistence module
Browse files Browse the repository at this point in the history
  • Loading branch information
@mmeigs
mmeigs committed Nov 17, 2023
1 parent 1873ce9 commit 478ebf3
Show file tree
Hide file tree
Showing 5 changed files with 117 additions and 79 deletions.
25 changes: 11 additions & 14 deletions src/commands/src/shared/next-gen-parse.ts
View file
Original file line number Diff line number Diff line change
@@ -1,22 +1,19 @@
import { IJobRepoLogger } from '../../../services/logger';
import path from 'path';
import { Job } from '../../../entities/job';
import { getDirectory } from '../../../job/jobHandler';
import { CliCommandResponse, executeCliCommand } from '../helpers';

const RSTSPEC_FLAG = '--rstspec=https://raw.githubusercontent.com/mongodb/snooty-parser/latest/snooty/rstspec.toml';

interface NextGenParseParams {
repoDir: string;
commitHash?: string;
patchId?: string;
logger: IJobRepoLogger;
id: string;
job: Job;
preppedLogger: (message: string) => void;
}
export async function nextGenParse({
repoDir,
patchId,
commitHash,
logger,
id,
}: NextGenParseParams): Promise<CliCommandResponse> {
export async function nextGenParse({ job, preppedLogger }: NextGenParseParams): Promise<CliCommandResponse> {
const repoDir = path.resolve(process.cwd(), `repos/${getDirectory(job)}`);
const commitHash = job.payload.newHead;
const patchId = job.payload.patch;

const commandArgs = ['build', repoDir, '--output', `${repoDir}/bundle.zip`];

if (patchId && commitHash) {
Expand All @@ -29,7 +26,7 @@ export async function nextGenParse({

commandArgs.push(RSTSPEC_FLAG);

logger.save(id, `COMMAND for parse: ${commandArgs.join(' ')}`);
preppedLogger(`COMMAND for parse: ${commandArgs.join(' ')}`);

return executeCliCommand({ command: 'snooty', args: commandArgs });
}
43 changes: 30 additions & 13 deletions src/commands/src/shared/next-gen-stage.ts
View file
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
import { Job } from '../../../entities/job';
import { executeCliCommand } from '../helpers';

const DOCS_WORKER_USER = 'docsworker-xlarge';
Expand All @@ -10,29 +11,43 @@ interface StageParams {
patchId?: string;
commitBranch: string;
commitHash: string;
job: Job;
preppedLogger: (message: string) => void;
}

export async function nextGenStage({
mutPrefix,
projectName,
bucket,
url,
patchId,
commitBranch,
commitHash,
}: StageParams) {
let hostedAtUrl = `${url}/${mutPrefix}/${DOCS_WORKER_USER}/${commitBranch}/`;
let prefix = mutPrefix;
job,
preppedLogger,
}: // mutPrefix,
// projectName,
// bucket,
// url,
// patchId,
// commitBranch,
// commitHash,
StageParams) {
// TODO: replace with a process to get this url??
const baseUrl = 'https://mongodbcom-cdn.website.staging.corp.mongodb.com';
// TODO: replace with process to access bucket
const bucket = 'docs-atlas-dotcomstg';
const { mutPrefix, branchName, patch, project, newHead } = job.payload;

// TODO: Figure out correct hostedAtUrl
let hostedAtUrl = `${baseUrl}/${mutPrefix}/${DOCS_WORKER_USER}/${branchName}/`;
// TODO: Look further into all possible needs for prefix...
let prefix = mutPrefix || project;

const commandArgs = ['public', bucket, '--stage'];

if (patchId && projectName === mutPrefix) {
prefix = `${commitHash}/${patchId}/${mutPrefix}`;
hostedAtUrl = `${url}/${commitHash}/${patchId}/${mutPrefix}/${DOCS_WORKER_USER}/${commitBranch}/`;
if (patch && newHead && project === mutPrefix) {
prefix = `${newHead}/${patch}/${mutPrefix}`;
hostedAtUrl = `${baseUrl}/${newHead}/${patch}/${mutPrefix}/${DOCS_WORKER_USER}/${branchName}/`;
}

commandArgs.push(`--prefix=${prefix}`);

preppedLogger(`MUT PUBLISH command args: ${commandArgs}`);

const { outputText } = await executeCliCommand({
command: 'mut-publish',
args: commandArgs,
Expand All @@ -42,6 +57,8 @@ export async function nextGenStage({
});

const resultMessage = `${outputText}\n Hosted at ${hostedAtUrl}\n\nHere are the commands: ${commandArgs}`;
preppedLogger(`OUTPUT of mut publish: ${resultMessage}`);

return {
resultMessage,
commands: commandArgs,
Expand Down
19 changes: 10 additions & 9 deletions src/commands/src/shared/oas-page-build.ts
View file
Original file line number Diff line number Diff line change
@@ -1,23 +1,24 @@
import { Job } from '../../../entities/job';
import { IJobRepoLogger } from '../../../services/logger';
import { executeCliCommand, getRepoDir } from '../helpers';

interface OasPageBuildParams {
job: Job;
baseUrl: string;
logger: IJobRepoLogger;
preppedLogger: (message: string) => void;
}

export async function oasPageBuild({ job, baseUrl, logger }: OasPageBuildParams) {
export async function oasPageBuild({ job, preppedLogger }: OasPageBuildParams) {
// TODO: replace with a process to get this url??
const baseUrl = 'https://mongodbcom-cdn.website.staging.corp.mongodb.com';

const siteUrl = job.payload.mutPrefix
? `${baseUrl}/${job.payload.mutPrefix}`
: `${baseUrl}/${job.payload.project}/docsworker-xlarge/${job.payload.branchName}`;
console.log('siteUrl: ', siteUrl);
logger.save(job._id, `Is there a mutprefix?? : ${job.payload.mutPrefix}`);
logger.save(job._id, `SITE URL: ${siteUrl}`);
preppedLogger(`Is there a mutprefix?? : ${job.payload.mutPrefix}`);
preppedLogger(`SITE URL: ${siteUrl}`);
const repoDir = getRepoDir(job.payload.repoName, job.payload.directory);
const bundlePath = `${repoDir}/bundle.zip`;
logger.save(job._id, `BUNDLE PATH ? : ${bundlePath}`);
preppedLogger(`BUNDLE PATH ? : ${bundlePath}`);

try {
const { outputText } = await executeCliCommand({
Expand All @@ -37,11 +38,11 @@ export async function oasPageBuild({ job, baseUrl, logger }: OasPageBuildParams)
],
});

logger.save(job._id, `OAS page builder output text? : ${outputText}`);
preppedLogger(`OAS page builder output text? : ${outputText}`);

return outputText;
} catch (error) {
logger.save(job._id, `Caught error from OAS page builder cli!: ${error}\n\n`);
preppedLogger(`Caught error from OAS page builder cli!: ${error}\n\n`);

return '';
}
Expand Down
28 changes: 16 additions & 12 deletions src/commands/src/shared/persistence-module.ts
View file
Original file line number Diff line number Diff line change
@@ -1,33 +1,37 @@
import { executeCliCommand } from '../helpers';
import { Job } from '../../../entities/job';
import { executeCliCommand, getRepoDir } from '../helpers';

interface PersistenceModuleParams {
bundlePath: string;
jobId?: string;
repoOwner?: string;
job: Job;
preppedLogger: (message: string) => void;
}
export async function persistenceModule({
bundlePath,
jobId,
repoOwner = 'docs-builder-bot',
}: PersistenceModuleParams) {
export async function persistenceModule({ job, preppedLogger }: PersistenceModuleParams) {
const githubUser = job.payload.repoOwner;
const repoDir = getRepoDir(job.payload.repoName, job.payload.directory);
const bundlePath = `${repoDir}/bundle.zip`;

const args = [
`${process.cwd()}/modules/persistence/dist/index.js`,
'--unhandled-rejections=strict',
'--path',
bundlePath,
'--githubUser',
repoOwner,
githubUser,
];

if (jobId) {
if (job._id) {
args.push('--jobId');
args.push(jobId);
args.push(job._id);
}

preppedLogger(`persistence args: ${args}`);

const { outputText } = await executeCliCommand({
command: 'node',
args,
});

preppedLogger(`output text persistence: ${outputText}`);

return outputText;
}
81 changes: 50 additions & 31 deletions src/job/jobHandler.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ import { IJobValidator } from './jobValidator';
import { RepoEntitlementsRepository } from '../repositories/repoEntitlementsRepository';
import { DocsetsRepository } from '../repositories/docsetsRepository';
import { MONOREPO_NAME } from '../monorepo/utils/monorepo-constants';
import { nextGenHtml, nextGenParse, oasPageBuild } from '../commands';
import { nextGenHtml, nextGenParse, oasPageBuild, persistenceModule } from '../commands';
import { getRepoDir } from '../commands/src/helpers';
require('fs');

Expand Down Expand Up @@ -299,6 +299,16 @@ export abstract class JobHandler {
['oas-page-build']: 'oasPageBuildExe',
};

const commandMap: {
[K: string]: ({ job, preppedLogger }: { job: Job; preppedLogger: (message: string) => void }) => any;
} = {
// ['get-build-dependencies']: 'buildDepsExe',
['next-gen-parse']: nextGenParse,
['persistence-module']: persistenceModule,
// ['next-gen-html']: 'htmlExe',
['oas-page-build']: oasPageBuild,
};

// get the prerequisite commands which should be all commands up to `rm -f makefile`
const endOfPrerequisiteCommands = this.currJob.buildCommands.indexOf('rm -f makefile');
const index = endOfPrerequisiteCommands + 1;
Expand All @@ -314,40 +324,49 @@ export abstract class JobHandler {
// call prerequisite commands
await this._commandExecutor.execute(prerequisiteCommands);

// create constants for command inputs
const thisLogger = this._logger;
const thisJob = this.currJob;
const preppedLogger = (message: string) => {
thisLogger.save(thisJob._id, message);
};
const repoDir = path.resolve(process.cwd(), `repos/${getDirectory(this.currJob)}`);

for (const command of makeCommands) {
// works for any make command with the following signature make <make-rule>
const key = command.split(' ')[1].trim();
this._logger.save(this.currJob._id, `command: ${command}`);
if (key === 'next-gen-parse') {
this._logger.save(this.currJob._id, `in parse command!!! `);
this._logger.save(this.currJob._id, `repoDir: "repos/${getDirectory(this.currJob)}" `);
this._logger.save(this.currJob._id, `process.cwd!! : ${process.cwd()}`);
const repoDir = path.resolve(process.cwd(), `repos/${getDirectory(this.currJob)}`);

this._logger.save(this.currJob._id, `repoDir now : ${repoDir}`);
try {
const snootyParseRes = await nextGenParse({
repoDir,
commitHash: this.currJob.payload.newHead ?? '',
logger: this._logger,
id: this.currJob._id,
});
this._logger.save(
this.currJob._id,
`nextGenParse response: "${snootyParseRes.outputText}" - or error: "${snootyParseRes.errorText}"\n\n\n`
);
} catch (err) {
this._logger.save(this.currJob._id, `ERROR`);
this._logger.save(this.currJob._id, `ERROR: ${err}`);
}
} else if (key === 'oas-page-build') {
this._logger.save(this.currJob._id, `IN oas page build`);
const result = await oasPageBuild({
job: this.currJob,
baseUrl: 'https://mongodbcom-cdn.website.staging.corp.mongodb.com',
logger: this._logger,
});
this._logger.save(this.currJob._id, `oas page build result... ${result}`);
if (commandMap[key]) {
this._logger.save(this.currJob._id, `running from commandMap: ${key}`);
await commandMap[key]({ job: this.currJob, preppedLogger });
// }
// if (key === 'next-gen-parse') {
// // this._logger.save(this.currJob._id, `in parse command!!! `);
// // this._logger.save(this.currJob._id, `repoDir: "repos/${getDirectory(this.currJob)}" `);
// // this._logger.save(this.currJob._id, `process.cwd!! : ${process.cwd()}`);
// // const repoDir = path.resolve(process.cwd(), `repos/${getDirectory(this.currJob)}`);

// this._logger.save(this.currJob._id, `repoDir now : ${repoDir}`);
// try {
// const snootyParseRes = await nextGenParse({
// job: this.currJob,
// preppedLogger,
// });
// this._logger.save(
// this.currJob._id,
// `nextGenParse response: "${snootyParseRes.outputText}" - or error: "${snootyParseRes.errorText}"\n\n\n`
// );
// } catch (err) {
// this._logger.save(this.currJob._id, `ERROR`);
// this._logger.save(this.currJob._id, `ERROR: ${err}`);
// }
// } else if (key === 'oas-page-build') {
// this._logger.save(this.currJob._id, `IN oas page build`);
// const result = await oasPageBuild({
// job: this.currJob,
// preppedLogger,
// });
// this._logger.save(this.currJob._id, `oas page build result... ${result}`);
} else {
if (stages[key]) {
const makeCommandsWithBenchmarksResponse = await this.callWithBenchmark(command, stages[key]);
Expand Down

0 comments on commit 478ebf3

Please sign in to comment.