From 8d2666b658a20e60e6d2d5111b8eaa9b413f0845 Mon Sep 17 00:00:00 2001 From: Eti Zaguri Date: Tue, 2 May 2023 08:58:11 +0100 Subject: [PATCH] add support for ENCRYPTED Variables in pipeline run cmd --- codefresh-release.yml | 9 +- codefresh.yml | 17 +++- docs/content/pipelines/Run Pipeline.md | 38 ++++++-- .../cli/commands/annotation/create.cmd.js | 2 +- .../commands/pipeline/pipeline.sdk.spec.js | 82 +++++++++++++++-- .../cli/commands/pipeline/run.base.js | 30 +++++-- .../cli/commands/pipeline/run.cmd.js | 7 ++ lib/interface/cli/commands/pipeline/var.json | 12 +++ lib/interface/cli/commands/pipeline/var.yml | 5 ++ .../cli/commands/project/apply.cmd.js | 13 +-- .../cli/commands/project/create.cmd.js | 15 +--- lib/interface/cli/defaults.js | 1 + lib/interface/cli/helpers/general.js | 88 +++++++++++++++++-- package.json | 2 +- 14 files changed, 262 insertions(+), 59 deletions(-) create mode 100644 lib/interface/cli/commands/pipeline/var.json create mode 100644 lib/interface/cli/commands/pipeline/var.yml diff --git a/codefresh-release.yml b/codefresh-release.yml index 0e882c15f..3c3d7d670 100644 --- a/codefresh-release.yml +++ b/codefresh-release.yml @@ -445,14 +445,13 @@ steps: update_documentation: stage: documentation title: "Update documentation http://cli.codefresh.io" - image: docker:18.01 + image: codefresh/build-cli commands: - - "apk update && apk add git nodejs" - - "npm install" + - "yarn" - "echo cleaning previous public dir and recreating worktree" - - "rm -rf public && git worktree prune && git worktree add -B gh-pages public origin/gh-pages" + - "rm -rf public && git worktree prune && git worktree add -B gh-pages public origin/gh-pages" - "echo Building public docs" - - "npm run build-public-docs" + - "yarn run build-public-docs" - "echo Push new docs to gh-pages detached branch" - 'git config --global user.email "auto-ci@codefresh.io" && git config --global user.name "Automated CI"' - 'cd public && git add --all && git commit -m "Publish new documentation for version ${{PACKAGE_VERSION}}" && git push https://${{GITHUB_TOKEN}}@github.com/codefresh-io/cli.git' diff --git a/codefresh.yml b/codefresh.yml index f6875ef16..9f82aeb9b 100644 --- a/codefresh.yml +++ b/codefresh.yml @@ -328,7 +328,7 @@ steps: type: codefresh-run arguments: PIPELINE_ID: 'codefresh-io/cli/release' - DETACH: true + TRIGGER_ID: codefresh-io/cli_1 BRANCH: master VARIABLE: - PACKAGE_VERSION=${{PACKAGE_VERSION}} @@ -342,3 +342,18 @@ steps: - name: create_manifest_list on: - success + + build_documentation: + stage: test + title: "build documentation http://cli.codefresh.io" + image: codefresh/build-cli + commands: + - "echo Building public docs" + - "npm run build-public-docs" + environment: + - HUGO_VERSION=0.32.0 + when: + steps: + - name: install_dependencies + on: + - success diff --git a/docs/content/pipelines/Run Pipeline.md b/docs/content/pipelines/Run Pipeline.md index eec6cdc8a..8fef78c36 100644 --- a/docs/content/pipelines/Run Pipeline.md +++ b/docs/content/pipelines/Run Pipeline.md @@ -15,22 +15,44 @@ The pipeline will be triggered multiple times according to the array length. #### Variable yaml file with 2 sets of variables ```yaml -- key: value - key2: key1 -- key: value - key2: key2 +- VARIABLE_A: value_a_for_the_first_build + VARIABLE_B: value_b_for_the_first_build +- VARIABLE_A: value_a_for_the_first_build + VARIABLE_B: value_b_for_the_first_build ``` #### Variable json file with 2 sets of variables ```json [ { - "key": "value", - "key2": "key1" + "VARIABLE_A": "value_a_for_the_first_build", + "VARIABLE_B": "value_b_for_the_first_build" }, { - "key": "value", - "key2": "key2" + "VARIABLE_A": "value_a_for_the_first_build", + "VARIABLE_B": "value_b_for_the_first_build" + } +] +``` +### Use encrypted variables in Codefresh build runs; supported from CLI version: 0.82.8 +#### Variable yaml file with single variable set with encrypted variables +```yaml +- key: + val: value + encrypted: true + key2: key1 + +``` + +#### Variable json file single variable set with encrypted variables +```json +[ + { + "key": { + "val": "value", + "encrypted": true + }, + "key2": "key1" } ] ``` diff --git a/lib/interface/cli/commands/annotation/create.cmd.js b/lib/interface/cli/commands/annotation/create.cmd.js index 8eadeacc0..7513d2185 100644 --- a/lib/interface/cli/commands/annotation/create.cmd.js +++ b/lib/interface/cli/commands/annotation/create.cmd.js @@ -31,7 +31,7 @@ const command = new Command({ .example('codefresh create annotation image 2dfacdaad466 coverage=75%', 'Annotate entity with a single label') .example('codefresh create annotation image 2dfacdaad466 coverage=75% tests_passed=true', 'Annotate entity with multiple labels') // eslint-disable-next-line max-len - .example('codefresh create annotation image 2dfacdaad466 coverage=75% tests_passed=true --display coverage', 'Annotate entity with multiple labels and display selection'), + .example('codefresh create annotation workflow 643d807b85bbe35931ae2282 ENV=prod tests_passed=true --display ENV', 'Annotate entity with multiple labels and display selection'), handler: async (argv) => { const { entityType, entityId, labels, display } = argv; diff --git a/lib/interface/cli/commands/pipeline/pipeline.sdk.spec.js b/lib/interface/cli/commands/pipeline/pipeline.sdk.spec.js index 883d72618..717508e5e 100644 --- a/lib/interface/cli/commands/pipeline/pipeline.sdk.spec.js +++ b/lib/interface/cli/commands/pipeline/pipeline.sdk.spec.js @@ -1,3 +1,7 @@ +const yaml = require('js-yaml'); +const request = require('requestretry'); +const fs = require('fs'); +const _ = require('lodash'); const DEFAULTS = require('../../defaults'); const getCmd = require('./get.cmd').toCommand(); const deleteCmd = require('./delete.cmd').toCommand(); @@ -10,18 +14,21 @@ jest.mock('../../helpers/validation'); // eslint-disable-line jest.mock('../../../../../check-version'); jest.mock('../../completion/helpers', () => { // eslint-disable-line return { - authContextWrapper: func => func, + authContextWrapper: (func) => func, }; }); +jest.mock('../../helpers/general', () => ({ + ...jest.requireActual('../../helpers/general'), + isCompatibleApiVersion: () => true, +})); + jest.mock('../../../../logic/entities/Pipeline', () => { // eslint-disable-line return { - fromResponse: res => res, + fromResponse: (res) => res, }; }); -const request = require('requestretry'); - const DEFAULT_RESPONSE = request.__defaultResponse(); describe('pipeline', () => { @@ -57,11 +64,11 @@ describe('pipeline', () => { }); it('should return default limit', async () => { - expect(_getLimit(undefined,false)).toEqual(DEFAULTS.GET_LIMIT_RESULTS); + expect(_getLimit(undefined, false)).toEqual(DEFAULTS.GET_LIMIT_RESULTS); }); it('should return `unlimited` value', async () => { - expect(_getLimit(undefined,true)).toEqual(DEFAULTS.GET_ALL_PIPELINES_LIMIT); + expect(_getLimit(undefined, true)).toEqual(DEFAULTS.GET_ALL_PIPELINES_LIMIT); }); }); @@ -84,6 +91,69 @@ describe('pipeline', () => { }); }); + describe('run', () => { + it('should handle running pipeline with encrypted variables', async () => { + const argv = { name: 'some name', + detach: true, + annotation: [], + variable: [ + 'secret=secret', + 'VAR1=VAL1', + ], + encrypted: ['secret'], + }; + const pip = new CfPipeline(argv); + await pip.run(); + expect(pip.executionRequests[0].options.variables).toEqual([ + { + key: 'secret', + value: 'secret', + encrypted: true, + }, + { + key: 'VAR1', + value: 'VAL1', + }, + ]); + await verifyResponsesReturned([DEFAULT_RESPONSE]); // eslint-disable-line + }); + + it('should handle running pipeline with encrypted variables passing inside json file', async () => { + const rawFile = fs.readFileSync('lib/interface/cli/commands/pipeline/var.json', 'utf8'); + + const argv = { name: 'some name', + detach: true, + annotation: [], + 'var-file': JSON.parse(rawFile), + }; + const pip = new CfPipeline(argv); + await pip.run(); + expect(pip.executionRequests[0].options.variables).toEqual( + [{ key: 'help6', value: '85858' }, + { key: 'should_be_encrepted', value: '0000' }, + { encrypted: true, key: 'help7', value: 'test' }], + ); + await verifyResponsesReturned([DEFAULT_RESPONSE]); // eslint-disable-line + }); + + it('should handle running pipeline with encrypted variables passing inside yaml file', async () => { + const rawFile = fs.readFileSync('lib/interface/cli/commands/pipeline/var.yml', 'utf8'); + + const argv = { name: 'some name', + detach: true, + annotation: [], + 'var-file': yaml.safeLoad(rawFile), + }; + const pip = new CfPipeline(argv); + await pip.run(); + expect(pip.executionRequests[0].options.variables).toEqual( + [{ key: 'VAR1', value: 'VAL1' }, + { encrypted: true, key: 'VAR2', value: 'VAL2' }], + ); + await verifyResponsesReturned([DEFAULT_RESPONSE]); // eslint-disable-line + }); + }); + describe('runImpl', () => { it('should handle running pipeline', async () => { const argv = { name: 'some name', detach: true }; diff --git a/lib/interface/cli/commands/pipeline/run.base.js b/lib/interface/cli/commands/pipeline/run.base.js index 8b40ed323..0ab91e896 100644 --- a/lib/interface/cli/commands/pipeline/run.base.js +++ b/lib/interface/cli/commands/pipeline/run.base.js @@ -1,10 +1,16 @@ const _ = require('lodash'); const Promise = require('bluebird'); -const { prepareKeyValueFromCLIEnvOption } = require('../../helpers/general'); +const CFError = require('cf-errors'); +const { prepareKeyValueFromCLIEnvOption, + markEncryptedFlagOnRequestedVariables, + prepareKeyValueObjectsFromEnvFileOption, + prepareKeyValueObjectsFromCLIEnvOption, + isCompatibleApiVersion, +} = require('../../helpers/general'); const { validatePipelineYaml } = require('../../helpers/validation'); const { printResult } = require('../root/validate.cmd'); -const CFError = require('cf-errors'); const { sdk } = require('../../../../logic'); +const defaults = require('../../defaults'); class RunBaseCommand { constructor(argv) { @@ -55,22 +61,34 @@ class RunBaseCommand { packName, }, }; - + const encryptedVarsSupported = await isCompatibleApiVersion({ + supportedVersion: defaults.MIN_API_VERSION_FOR_ENCRYPTED_VARS_SUPPORT_IN_RUN_CMD, + }); if (variablesFromFile) { _.forEach(variablesFromFile, (variables) => { const request = _.cloneDeep(executionRequestTemplate); - request.options.variables = variables; + if (encryptedVarsSupported) { + request.options.variables = prepareKeyValueObjectsFromEnvFileOption(variables); + } else { + request.options.variables = variables; + } this.executionRequests.push(request); }); } else { - const variables = prepareKeyValueFromCLIEnvOption(this.argv.variable); + let variables; + if (encryptedVarsSupported) { + const varsArr = prepareKeyValueObjectsFromCLIEnvOption(this.argv.variable); + variables = markEncryptedFlagOnRequestedVariables(varsArr, this.argv.encrypted); + } else { + variables = prepareKeyValueFromCLIEnvOption(this.argv.variable); + } const request = _.cloneDeep(executionRequestTemplate); request.options.variables = variables; request.options.contexts = contexts; this.executionRequests.push(request); } - const results = await Promise.all(this.executionRequests.map(request => this.runImpl(request))); + const results = await Promise.all(this.executionRequests.map((request) => this.runImpl(request))); const findMaxReducer = (accumulator, currentValue) => (currentValue > accumulator ? currentValue : accumulator); const exitCode = results.reduce(findMaxReducer); await this.postRunRequest(); diff --git a/lib/interface/cli/commands/pipeline/run.cmd.js b/lib/interface/cli/commands/pipeline/run.cmd.js index 7a63157ff..cf024c018 100644 --- a/lib/interface/cli/commands/pipeline/run.cmd.js +++ b/lib/interface/cli/commands/pipeline/run.cmd.js @@ -86,6 +86,12 @@ const run = new Command({ default: [], alias: 'v', }) + .option('encrypted', { + array: true, + alias: 'e', + describe: 'Variable names to encrypt', + default: [], + }) .option('detach', { alias: 'd', describe: 'Run pipeline and print build ID', @@ -126,6 +132,7 @@ const run = new Command({ .example('codefresh run PIPELINE_ID | PIPELINE_NAME -b=master', 'Defining the source control context using a branch') .example('codefresh run PIPELINE_ID | PIPELINE_NAME -s=52b992e783d2f84dd0123c70ac8623b4f0f938d1', 'Defining the source control context using a commit') .example('codefresh run PIPELINE_ID | PIPELINE_NAME -b=master -v key1=value1 -v key2=value2', 'Setting variables through the command') + .example('codefresh run PIPELINE_ID | PIPELINE_NAME -b=master -v key1=value1 -v key2=value2 -e key1', 'Setting variables through the command with encrypted option') .example('codefresh run PIPELINE_ID | PIPELINE_NAME -b=master --var-file ./var_file.yml', 'Settings variables through a yml file') .example('codefresh run PIPELINE_ID | PIPELINE_NAME -b=master --context context', 'Inject contexts to the pipeline execution') .example('codefresh run PIPELINE_ID | PIPELINE_NAME --skip step1 step2 step3', 'Skip specific steps'); diff --git a/lib/interface/cli/commands/pipeline/var.json b/lib/interface/cli/commands/pipeline/var.json new file mode 100644 index 000000000..bf6ed0f7d --- /dev/null +++ b/lib/interface/cli/commands/pipeline/var.json @@ -0,0 +1,12 @@ +{ + "build1": { + "help6": "85858", + "should_be_encrepted": "0000", + "help7": { + "value": "test", + "encrypted": true + } + } +} + + diff --git a/lib/interface/cli/commands/pipeline/var.yml b/lib/interface/cli/commands/pipeline/var.yml new file mode 100644 index 000000000..db6b028b0 --- /dev/null +++ b/lib/interface/cli/commands/pipeline/var.yml @@ -0,0 +1,5 @@ +build1: + VAR1: 'VAL1' + VAR2: + value: VAL2 + encrypted: true diff --git a/lib/interface/cli/commands/project/apply.cmd.js b/lib/interface/cli/commands/project/apply.cmd.js index 9e574c4f0..082b9106b 100644 --- a/lib/interface/cli/commands/project/apply.cmd.js +++ b/lib/interface/cli/commands/project/apply.cmd.js @@ -4,7 +4,7 @@ const _ = require('lodash'); const { sdk } = require('../../../../logic'); const applyRoot = require('../root/apply.cmd'); -const { prepareKeyValueObjectsFromCLIEnvOption, ignoreHttpError } = require('../../helpers/general'); +const { prepareKeyValueObjectsFromCLIEnvOption, ignoreHttpError, markEncryptedFlagOnRequestedVariables } = require('../../helpers/general'); const command = new Command({ command: 'project [id|name]', @@ -61,14 +61,7 @@ const command = new Command({ encrypted, } = argv; - const variableMap = _.reduce(variables, (acc, v) => _.assign(acc, { [v.key]: v }), {}); - _.forEach(encrypted, (varName) => { - const variable = variableMap[varName]; - if (!variable) { - throw new CFError(`Variable is not provided: "${varName}"`); - } - variable.encrypted = true; - }); + const requestedProjectVariables = markEncryptedFlagOnRequestedVariables(variables, encrypted); let project = await sdk.projects.get({ id }).catch(ignoreHttpError); project = project || await sdk.projects.getByName({ name }).catch(ignoreHttpError); @@ -81,7 +74,7 @@ const command = new Command({ const updatePayload = _.pickBy({ projectName, tags: tags || existingTags, - variables: variables || existingVariables, + variables: requestedProjectVariables || existingVariables, }, _.identity); await sdk.projects.patch({ id: project.id }, updatePayload); diff --git a/lib/interface/cli/commands/project/create.cmd.js b/lib/interface/cli/commands/project/create.cmd.js index eb0371539..306dc2c37 100644 --- a/lib/interface/cli/commands/project/create.cmd.js +++ b/lib/interface/cli/commands/project/create.cmd.js @@ -1,10 +1,8 @@ const Command = require('../../Command'); -const CFError = require('cf-errors'); -const _ = require('lodash'); const { sdk } = require('../../../../logic'); const createRoot = require('../root/create.cmd'); const { checkOrProjectExists } = require('../../helpers/validation'); -const { prepareKeyValueObjectsFromCLIEnvOption } = require('../../helpers/general'); +const { prepareKeyValueObjectsFromCLIEnvOption, markEncryptedFlagOnRequestedVariables } = require('../../helpers/general'); const command = new Command({ command: 'project ', @@ -52,17 +50,10 @@ const command = new Command({ encrypted, } = argv; - const variableMap = _.reduce(variables, (acc, v) => _.assign(acc, { [v.key]: v }), {}); - _.forEach(encrypted, (varName) => { - const variable = variableMap[varName]; - if (!variable) { - throw new CFError(`Variable is not provided: "${varName}"`); - } - variable.encrypted = true; - }); + const requestedProjectVariables = markEncryptedFlagOnRequestedVariables(variables, encrypted); await checkOrProjectExists(projectName); - await sdk.projects.create({ projectName, tags, variables }); + await sdk.projects.create({ projectName, tags, variables: requestedProjectVariables }); console.log(`Project: "${projectName}" created`); }, }); diff --git a/lib/interface/cli/defaults.js b/lib/interface/cli/defaults.js index 39801fc77..6e9f1640e 100644 --- a/lib/interface/cli/defaults.js +++ b/lib/interface/cli/defaults.js @@ -13,6 +13,7 @@ const DEFAULTS = { MAX_CONSECUTIVE_ERRORS_LIMIT: 10, CODEFRESH_PATH: path.resolve(homedir(), '.Codefresh'), ENGINE_IMAGE: process.env.ENGINE_IMAGE || 'codefresh/engine:master', + MIN_API_VERSION_FOR_ENCRYPTED_VARS_SUPPORT_IN_RUN_CMD: '21.221.14', }; module.exports = DEFAULTS; diff --git a/lib/interface/cli/helpers/general.js b/lib/interface/cli/helpers/general.js index 3becbc692..82d18c937 100644 --- a/lib/interface/cli/helpers/general.js +++ b/lib/interface/cli/helpers/general.js @@ -1,15 +1,43 @@ +const debug = require('debug')('codefresh:cli:helpers:general'); const Promise = require('bluebird'); const _ = require('lodash'); const fs = require('fs'); const yaml = require('js-yaml'); -const defaults = require('../defaults'); const CFError = require('cf-errors'); const path = require('path'); +const semver = require('semver'); +const request = require('requestretry'); +const defaults = require('../defaults'); const Output = require('../../../output/Output'); const { sdk } = require('../../../logic'); const isDebug = () => (process.env.DEBUG || '').includes(defaults.DEBUG_PATTERN); +async function getApiVersion(token, url) { + const RequestOptions = { + url: `${url}/api/user`, + headers: { + Authorization: token, + }, + json: true, + maxAttempts: 3, // Retry up to 3 times + timeout: 10000, + }; + try { + const res = await request(RequestOptions); + return _.get(res, 'headers.api_version'); + } catch (error) { + debug('Failed to get API version:', error); + return null; + } +} + +async function isCompatibleApiVersion({ supportedVersion }) { + const token = _.get(sdk, 'config.context.token'); + const url = _.get(sdk, 'config.context.url', defaults.URL); + const apiVersion = token ? await getApiVersion(token, url) : null; + return apiVersion && semver.gte(apiVersion, supportedVersion); +} const wrapHandler = (handler, requiresAuthentication) => { return async (argv) => { @@ -106,6 +134,39 @@ const prepareKeyValueObjectsFromCLIEnvOption = (environmentVariables) => { return variables; }; +/** + * will return an array of objects { key, value, encrypted } from parsing an array of objects: key=value ,key: {value: "value", encrypted: true } + * @param environmentVariables + * @returns Array of { key, value, encrypted } + */ +const prepareKeyValueObjectsFromEnvFileOption = (environmentVariables) => _.map(environmentVariables, (value, key) => { + if (_.isObject(value)) { + const { value: val, encrypted } = value; + return { key, value: val, encrypted }; + } + return { key, value }; +}); + +/** + * The function takes in a list of requested variables and an array of encrypted variable names, + * and sets the encrypted flag to true on each corresponding variable object in the list. + * @param environmentVariables + * @param encrypted + * @returns Array of { key, value, encrypted } + */ +const markEncryptedFlagOnRequestedVariables = (requestedVariables, encrypted) => { + const variables = _.cloneDeep(requestedVariables); + const variableMap = _.reduce(variables, (acc, v) => _.assign(acc, { [v.key]: v }), {}); + _.forEach(encrypted, (varName) => { + const variable = variableMap[varName]; + if (!variable) { + throw new CFError(`Variable is not provided: "${varName}"`); + } + variable.encrypted = true; + }); + return variables; +}; + const crudFilenameOption = (yargs, options = {}) => { const filenameOption = { alias: options.alias || 'f', @@ -122,13 +183,19 @@ const crudFilenameOption = (yargs, options = {}) => { .coerce(options.name || 'filename', (arg) => { try { const rawFile = fs.readFileSync(path.resolve(process.cwd(), arg), 'utf8'); + let content; if (arg.endsWith('.json')) { - return options.raw ? rawFile : JSON.parse(rawFile); + content = options.raw ? rawFile : JSON.parse(rawFile); + } else if (arg.endsWith('.yml') || arg.endsWith('yaml')) { + content = options.raw ? rawFile : yaml.safeLoad(rawFile); + } else { + throw new CFError('File extension is not recognized'); } - if (arg.endsWith('.yml') || arg.endsWith('yaml')) { - return options.raw ? rawFile : yaml.safeLoad(rawFile); + if (_.isObject(content)) { + return content; } - throw new CFError('File extension is not recognized'); + throw new CFError('Not a valid file.\n' + + 'For more information how to pass a valid file, go to https://codefresh-io.github.io/cli/pipelines/run-pipeline/'); } catch (err) { const error = new CFError({ message: 'Failed to read file', @@ -161,15 +228,15 @@ const selectColumnsOption = (yargs, options = {}) => { }; function pathExists(p) { - return new Promise(resolve => fs.access(p, resolve)) - .then(err => !err); + return new Promise((resolve) => fs.access(p, resolve)) + .then((err) => !err); } const readFile = Promise.promisify(fs.readFile); function watchFile(filename, listener) { fs.watchFile(filename, { interval: 500 }, listener); - const unwatcher = f => () => fs.unwatchFile(f); + const unwatcher = (f) => () => fs.unwatchFile(f); ['exit', 'SIGINT', 'SIGUSR1', 'SIGUSR2', 'uncaughtException', 'SIGTERM'].forEach((eventType) => { process.on(eventType, unwatcher(filename)); }); @@ -177,7 +244,7 @@ function watchFile(filename, listener) { function ignoreHttpError(e) { if (!e.statusCode) { - throw e + throw e; } return undefined; } @@ -232,4 +299,7 @@ module.exports = { detectProxy, keyValueArrayToObject, addProxyVariables, + markEncryptedFlagOnRequestedVariables, + prepareKeyValueObjectsFromEnvFileOption, + isCompatibleApiVersion, }; diff --git a/package.json b/package.json index 45e63051b..3dc62d6da 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "codefresh", - "version": "0.82.9", + "version": "0.83.1", "description": "Codefresh command line utility", "main": "index.js", "preferGlobal": true,