From acbce0131ef655a62d8176dccfb5b4593cfe780f Mon Sep 17 00:00:00 2001 From: Eti Zaguri Date: Sun, 23 Apr 2023 17:25:34 +0100 Subject: [PATCH] CR-17813 (#800) * CR-17813 * WIP * WIP * WIP * WIP * WIP * WIP * WIP * WIP * add tests * add tests files * wip * add documentation * add documentation * wip documentation * wip documentation * wip documentation * address Kim comments * address Daniel comments * address Daniel comments * wip --- docs/content/pipelines/Run Pipeline.md | 38 +++++++-- .../commands/pipeline/pipeline.sdk.spec.js | 82 +++++++++++++++++-- .../cli/commands/pipeline/run.base.js | 12 +-- .../cli/commands/pipeline/run.cmd.js | 11 ++- lib/interface/cli/commands/pipeline/var.json | 12 +++ lib/interface/cli/commands/pipeline/var.yml | 5 ++ .../cli/commands/project/apply.cmd.js | 13 +-- .../cli/commands/project/create.cmd.js | 15 +--- lib/interface/cli/helpers/general.js | 59 +++++++++++-- package.json | 4 +- 10 files changed, 197 insertions(+), 54 deletions(-) create mode 100644 lib/interface/cli/commands/pipeline/var.json create mode 100644 lib/interface/cli/commands/pipeline/var.yml diff --git a/docs/content/pipelines/Run Pipeline.md b/docs/content/pipelines/Run Pipeline.md index eec6cdc8a..8fef78c36 100644 --- a/docs/content/pipelines/Run Pipeline.md +++ b/docs/content/pipelines/Run Pipeline.md @@ -15,22 +15,44 @@ The pipeline will be triggered multiple times according to the array length. #### Variable yaml file with 2 sets of variables ```yaml -- key: value - key2: key1 -- key: value - key2: key2 +- VARIABLE_A: value_a_for_the_first_build + VARIABLE_B: value_b_for_the_first_build +- VARIABLE_A: value_a_for_the_first_build + VARIABLE_B: value_b_for_the_first_build ``` #### Variable json file with 2 sets of variables ```json [ { - "key": "value", - "key2": "key1" + "VARIABLE_A": "value_a_for_the_first_build", + "VARIABLE_B": "value_b_for_the_first_build" }, { - "key": "value", - "key2": "key2" + "VARIABLE_A": "value_a_for_the_first_build", + "VARIABLE_B": "value_b_for_the_first_build" + } +] +``` +### Use encrypted variables in Codefresh build runs; supported from CLI version: 0.82.8 +#### Variable yaml file with single variable set with encrypted variables +```yaml +- key: + val: value + encrypted: true + key2: key1 + +``` + +#### Variable json file single variable set with encrypted variables +```json +[ + { + "key": { + "val": "value", + "encrypted": true + }, + "key2": "key1" } ] ``` diff --git a/lib/interface/cli/commands/pipeline/pipeline.sdk.spec.js b/lib/interface/cli/commands/pipeline/pipeline.sdk.spec.js index 883d72618..5d662281b 100644 --- a/lib/interface/cli/commands/pipeline/pipeline.sdk.spec.js +++ b/lib/interface/cli/commands/pipeline/pipeline.sdk.spec.js @@ -1,3 +1,6 @@ +const yaml = require('js-yaml'); +const request = require('requestretry'); +const fs = require('fs'); const DEFAULTS = require('../../defaults'); const getCmd = require('./get.cmd').toCommand(); const deleteCmd = require('./delete.cmd').toCommand(); @@ -10,18 +13,16 @@ jest.mock('../../helpers/validation'); // eslint-disable-line jest.mock('../../../../../check-version'); jest.mock('../../completion/helpers', () => { // eslint-disable-line return { - authContextWrapper: func => func, + authContextWrapper: (func) => func, }; }); jest.mock('../../../../logic/entities/Pipeline', () => { // eslint-disable-line return { - fromResponse: res => res, + fromResponse: (res) => res, }; }); -const request = require('requestretry'); - const DEFAULT_RESPONSE = request.__defaultResponse(); describe('pipeline', () => { @@ -57,11 +58,11 @@ describe('pipeline', () => { }); it('should return default limit', async () => { - expect(_getLimit(undefined,false)).toEqual(DEFAULTS.GET_LIMIT_RESULTS); + expect(_getLimit(undefined, false)).toEqual(DEFAULTS.GET_LIMIT_RESULTS); }); it('should return `unlimited` value', async () => { - expect(_getLimit(undefined,true)).toEqual(DEFAULTS.GET_ALL_PIPELINES_LIMIT); + expect(_getLimit(undefined, true)).toEqual(DEFAULTS.GET_ALL_PIPELINES_LIMIT); }); }); @@ -84,6 +85,75 @@ describe('pipeline', () => { }); }); + describe('run', () => { + it('should handle running pipeline with encrypted variables', async () => { + const argv = { name: 'some name', + detach: true, + annotation: [], + variable: + [{ + key: 'secret', + value: 'secret', + }, + { + key: 'VAR1', + value: 'VAL1', + }, + ], + encrypted: ['secret'], + }; + const pip = new CfPipeline(argv); + await pip.run(); + expect(pip.executionRequests[0].options.variables).toEqual([ + { + key: 'secret', + value: 'secret', + encrypted: true, + }, + { + key: 'VAR1', + value: 'VAL1', + }, + ]); + await verifyResponsesReturned([DEFAULT_RESPONSE]); // eslint-disable-line + }); + + it('should handle running pipeline with encrypted variables passing inside json file', async () => { + const rawFile = fs.readFileSync('lib/interface/cli/commands/pipeline/var.json', 'utf8'); + + const argv = { name: 'some name', + detach: true, + annotation: [], + 'var-file': JSON.parse(rawFile), + }; + const pip = new CfPipeline(argv); + await pip.run(); + expect(pip.executionRequests[0].options.variables).toEqual( + [{ key: 'help6', value: '85858' }, + { key: 'should_be_encrepted', value: '0000' }, + { encrypted: true, key: 'help7', value: 'test' }], + ); + await verifyResponsesReturned([DEFAULT_RESPONSE]); // eslint-disable-line + }); + + it('should handle running pipeline with encrypted variables passing inside yaml file', async () => { + const rawFile = fs.readFileSync('lib/interface/cli/commands/pipeline/var.yml', 'utf8'); + + const argv = { name: 'some name', + detach: true, + annotation: [], + 'var-file': yaml.safeLoad(rawFile), + }; + const pip = new CfPipeline(argv); + await pip.run(); + expect(pip.executionRequests[0].options.variables).toEqual( + [{ key: 'VAR1', value: 'VAL1' }, + { encrypted: true, key: 'VAR2', value: 'VAL2' }], + ); + await verifyResponsesReturned([DEFAULT_RESPONSE]); // eslint-disable-line + }); + }); + describe('runImpl', () => { it('should handle running pipeline', async () => { const argv = { name: 'some name', detach: true }; diff --git a/lib/interface/cli/commands/pipeline/run.base.js b/lib/interface/cli/commands/pipeline/run.base.js index 8b40ed323..224f30eec 100644 --- a/lib/interface/cli/commands/pipeline/run.base.js +++ b/lib/interface/cli/commands/pipeline/run.base.js @@ -1,9 +1,11 @@ const _ = require('lodash'); const Promise = require('bluebird'); -const { prepareKeyValueFromCLIEnvOption } = require('../../helpers/general'); +const CFError = require('cf-errors'); +const { prepareKeyValueFromCLIEnvOption, + markEncryptedFlagOnRequestedVariables, + prepareKeyValueObjectsFromEnvFileOption } = require('../../helpers/general'); const { validatePipelineYaml } = require('../../helpers/validation'); const { printResult } = require('../root/validate.cmd'); -const CFError = require('cf-errors'); const { sdk } = require('../../../../logic'); class RunBaseCommand { @@ -59,18 +61,18 @@ class RunBaseCommand { if (variablesFromFile) { _.forEach(variablesFromFile, (variables) => { const request = _.cloneDeep(executionRequestTemplate); - request.options.variables = variables; + request.options.variables = prepareKeyValueObjectsFromEnvFileOption(variables); this.executionRequests.push(request); }); } else { - const variables = prepareKeyValueFromCLIEnvOption(this.argv.variable); + const variables = markEncryptedFlagOnRequestedVariables(this.argv.variable, this.argv.encrypted); const request = _.cloneDeep(executionRequestTemplate); request.options.variables = variables; request.options.contexts = contexts; this.executionRequests.push(request); } - const results = await Promise.all(this.executionRequests.map(request => this.runImpl(request))); + const results = await Promise.all(this.executionRequests.map((request) => this.runImpl(request))); const findMaxReducer = (accumulator, currentValue) => (currentValue > accumulator ? currentValue : accumulator); const exitCode = results.reduce(findMaxReducer); await this.postRunRequest(); diff --git a/lib/interface/cli/commands/pipeline/run.cmd.js b/lib/interface/cli/commands/pipeline/run.cmd.js index 7a63157ff..f1bb299ff 100644 --- a/lib/interface/cli/commands/pipeline/run.cmd.js +++ b/lib/interface/cli/commands/pipeline/run.cmd.js @@ -1,6 +1,5 @@ -const debug = require('debug')('codefresh:cli:run:pipeline'); const Command = require('../../Command'); -const { crudFilenameOption } = require('../../helpers/general'); +const { crudFilenameOption, prepareKeyValueObjectsFromCLIEnvOption } = require('../../helpers/general'); const RunLocalCommand = require('./run.local'); const RunExternalCommand = require('./run.cf'); @@ -85,6 +84,13 @@ const run = new Command({ describe: 'Set build variables', default: [], alias: 'v', + coerce: prepareKeyValueObjectsFromCLIEnvOption, + }) + .option('encrypted', { + array: true, + alias: 'e', + describe: 'Variable names to encrypt', + default: [], }) .option('detach', { alias: 'd', @@ -126,6 +132,7 @@ const run = new Command({ .example('codefresh run PIPELINE_ID | PIPELINE_NAME -b=master', 'Defining the source control context using a branch') .example('codefresh run PIPELINE_ID | PIPELINE_NAME -s=52b992e783d2f84dd0123c70ac8623b4f0f938d1', 'Defining the source control context using a commit') .example('codefresh run PIPELINE_ID | PIPELINE_NAME -b=master -v key1=value1 -v key2=value2', 'Setting variables through the command') + .example('codefresh run PIPELINE_ID | PIPELINE_NAME -b=master -v key1=value1 -v key2=value2 -e key1', 'Setting variables through the command with encrypted option') .example('codefresh run PIPELINE_ID | PIPELINE_NAME -b=master --var-file ./var_file.yml', 'Settings variables through a yml file') .example('codefresh run PIPELINE_ID | PIPELINE_NAME -b=master --context context', 'Inject contexts to the pipeline execution') .example('codefresh run PIPELINE_ID | PIPELINE_NAME --skip step1 step2 step3', 'Skip specific steps'); diff --git a/lib/interface/cli/commands/pipeline/var.json b/lib/interface/cli/commands/pipeline/var.json new file mode 100644 index 000000000..bf6ed0f7d --- /dev/null +++ b/lib/interface/cli/commands/pipeline/var.json @@ -0,0 +1,12 @@ +{ + "build1": { + "help6": "85858", + "should_be_encrepted": "0000", + "help7": { + "value": "test", + "encrypted": true + } + } +} + + diff --git a/lib/interface/cli/commands/pipeline/var.yml b/lib/interface/cli/commands/pipeline/var.yml new file mode 100644 index 000000000..db6b028b0 --- /dev/null +++ b/lib/interface/cli/commands/pipeline/var.yml @@ -0,0 +1,5 @@ +build1: + VAR1: 'VAL1' + VAR2: + value: VAL2 + encrypted: true diff --git a/lib/interface/cli/commands/project/apply.cmd.js b/lib/interface/cli/commands/project/apply.cmd.js index 9e574c4f0..082b9106b 100644 --- a/lib/interface/cli/commands/project/apply.cmd.js +++ b/lib/interface/cli/commands/project/apply.cmd.js @@ -4,7 +4,7 @@ const _ = require('lodash'); const { sdk } = require('../../../../logic'); const applyRoot = require('../root/apply.cmd'); -const { prepareKeyValueObjectsFromCLIEnvOption, ignoreHttpError } = require('../../helpers/general'); +const { prepareKeyValueObjectsFromCLIEnvOption, ignoreHttpError, markEncryptedFlagOnRequestedVariables } = require('../../helpers/general'); const command = new Command({ command: 'project [id|name]', @@ -61,14 +61,7 @@ const command = new Command({ encrypted, } = argv; - const variableMap = _.reduce(variables, (acc, v) => _.assign(acc, { [v.key]: v }), {}); - _.forEach(encrypted, (varName) => { - const variable = variableMap[varName]; - if (!variable) { - throw new CFError(`Variable is not provided: "${varName}"`); - } - variable.encrypted = true; - }); + const requestedProjectVariables = markEncryptedFlagOnRequestedVariables(variables, encrypted); let project = await sdk.projects.get({ id }).catch(ignoreHttpError); project = project || await sdk.projects.getByName({ name }).catch(ignoreHttpError); @@ -81,7 +74,7 @@ const command = new Command({ const updatePayload = _.pickBy({ projectName, tags: tags || existingTags, - variables: variables || existingVariables, + variables: requestedProjectVariables || existingVariables, }, _.identity); await sdk.projects.patch({ id: project.id }, updatePayload); diff --git a/lib/interface/cli/commands/project/create.cmd.js b/lib/interface/cli/commands/project/create.cmd.js index eb0371539..306dc2c37 100644 --- a/lib/interface/cli/commands/project/create.cmd.js +++ b/lib/interface/cli/commands/project/create.cmd.js @@ -1,10 +1,8 @@ const Command = require('../../Command'); -const CFError = require('cf-errors'); -const _ = require('lodash'); const { sdk } = require('../../../../logic'); const createRoot = require('../root/create.cmd'); const { checkOrProjectExists } = require('../../helpers/validation'); -const { prepareKeyValueObjectsFromCLIEnvOption } = require('../../helpers/general'); +const { prepareKeyValueObjectsFromCLIEnvOption, markEncryptedFlagOnRequestedVariables } = require('../../helpers/general'); const command = new Command({ command: 'project ', @@ -52,17 +50,10 @@ const command = new Command({ encrypted, } = argv; - const variableMap = _.reduce(variables, (acc, v) => _.assign(acc, { [v.key]: v }), {}); - _.forEach(encrypted, (varName) => { - const variable = variableMap[varName]; - if (!variable) { - throw new CFError(`Variable is not provided: "${varName}"`); - } - variable.encrypted = true; - }); + const requestedProjectVariables = markEncryptedFlagOnRequestedVariables(variables, encrypted); await checkOrProjectExists(projectName); - await sdk.projects.create({ projectName, tags, variables }); + await sdk.projects.create({ projectName, tags, variables: requestedProjectVariables }); console.log(`Project: "${projectName}" created`); }, }); diff --git a/lib/interface/cli/helpers/general.js b/lib/interface/cli/helpers/general.js index 3becbc692..60b797c08 100644 --- a/lib/interface/cli/helpers/general.js +++ b/lib/interface/cli/helpers/general.js @@ -2,9 +2,9 @@ const Promise = require('bluebird'); const _ = require('lodash'); const fs = require('fs'); const yaml = require('js-yaml'); -const defaults = require('../defaults'); const CFError = require('cf-errors'); const path = require('path'); +const defaults = require('../defaults'); const Output = require('../../../output/Output'); const { sdk } = require('../../../logic'); @@ -106,6 +106,39 @@ const prepareKeyValueObjectsFromCLIEnvOption = (environmentVariables) => { return variables; }; +/** + * will return an array of objects { key, value, encrypted } from parsing an array of objects: key=value ,key: {value: "value", encrypted: true } + * @param environmentVariables + * @returns Array of { key, value, encrypted } + */ +const prepareKeyValueObjectsFromEnvFileOption = (environmentVariables) => _.map(environmentVariables, (value, key) => { + if (_.isObject(value)) { + const { value: val, encrypted } = value; + return { key, value: val, encrypted }; + } + return { key, value }; +}); + +/** + * The function takes in a list of requested variables and an array of encrypted variable names, + * and sets the encrypted flag to true on each corresponding variable object in the list. + * @param environmentVariables + * @param encrypted + * @returns Array of { key, value, encrypted } + */ +const markEncryptedFlagOnRequestedVariables = (requestedVariables, encrypted) => { + const variables = _.cloneDeep(requestedVariables); + const variableMap = _.reduce(variables, (acc, v) => _.assign(acc, { [v.key]: v }), {}); + _.forEach(encrypted, (varName) => { + const variable = variableMap[varName]; + if (!variable) { + throw new CFError(`Variable is not provided: "${varName}"`); + } + variable.encrypted = true; + }); + return variables; +}; + const crudFilenameOption = (yargs, options = {}) => { const filenameOption = { alias: options.alias || 'f', @@ -122,13 +155,19 @@ const crudFilenameOption = (yargs, options = {}) => { .coerce(options.name || 'filename', (arg) => { try { const rawFile = fs.readFileSync(path.resolve(process.cwd(), arg), 'utf8'); + let content; if (arg.endsWith('.json')) { - return options.raw ? rawFile : JSON.parse(rawFile); + content = options.raw ? rawFile : JSON.parse(rawFile); + } else if (arg.endsWith('.yml') || arg.endsWith('yaml')) { + content = options.raw ? rawFile : yaml.safeLoad(rawFile); + } else { + throw new CFError('File extension is not recognized'); } - if (arg.endsWith('.yml') || arg.endsWith('yaml')) { - return options.raw ? rawFile : yaml.safeLoad(rawFile); + if (_.isObject(content)) { + return content; } - throw new CFError('File extension is not recognized'); + throw new CFError('Not a valid file.\n' + + 'For more information how to pass a valid file, go to https://codefresh-io.github.io/cli/pipelines/run-pipeline/'); } catch (err) { const error = new CFError({ message: 'Failed to read file', @@ -161,15 +200,15 @@ const selectColumnsOption = (yargs, options = {}) => { }; function pathExists(p) { - return new Promise(resolve => fs.access(p, resolve)) - .then(err => !err); + return new Promise((resolve) => fs.access(p, resolve)) + .then((err) => !err); } const readFile = Promise.promisify(fs.readFile); function watchFile(filename, listener) { fs.watchFile(filename, { interval: 500 }, listener); - const unwatcher = f => () => fs.unwatchFile(f); + const unwatcher = (f) => () => fs.unwatchFile(f); ['exit', 'SIGINT', 'SIGUSR1', 'SIGUSR2', 'uncaughtException', 'SIGTERM'].forEach((eventType) => { process.on(eventType, unwatcher(filename)); }); @@ -177,7 +216,7 @@ function watchFile(filename, listener) { function ignoreHttpError(e) { if (!e.statusCode) { - throw e + throw e; } return undefined; } @@ -232,4 +271,6 @@ module.exports = { detectProxy, keyValueArrayToObject, addProxyVariables, + markEncryptedFlagOnRequestedVariables, + prepareKeyValueObjectsFromEnvFileOption, }; diff --git a/package.json b/package.json index b53a74aef..c0b4ae3da 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "codefresh", - "version": "0.82.7", + "version": "0.82.8", "description": "Codefresh command line utility", "main": "index.js", "preferGlobal": true, @@ -115,4 +115,4 @@ "./test-setup.js" ] } -} \ No newline at end of file +}