From 998b44b4b73b9c4ed93d45336fb708bce75c727e Mon Sep 17 00:00:00 2001 From: Roy Razon Date: Mon, 11 Dec 2023 14:58:22 +0200 Subject: [PATCH] add `--skip-volume` flag to `up` command --- packages/cli/src/commands/up.ts | 9 +++++- packages/core/package.json | 1 + packages/core/src/commands/model.ts | 3 ++ packages/core/src/commands/up.ts | 8 +++++ packages/core/src/compose/remote.ts | 47 ++++++++++++++++++++++------- packages/core/src/index.ts | 7 ++++- 6 files changed, 62 insertions(+), 13 deletions(-) diff --git a/packages/cli/src/commands/up.ts b/packages/cli/src/commands/up.ts index 7a778431..10d1ef94 100644 --- a/packages/cli/src/commands/up.ts +++ b/packages/cli/src/commands/up.ts @@ -5,7 +5,7 @@ import { ProfileStore, TunnelOpts, addBaseComposeTunnelAgentService, - commands, ensureMachine, findComposeTunnelAgentUrl, + commands, defaultVolumeSkipList, ensureMachine, findComposeTunnelAgentUrl, findEnvId, findProjectName, getTunnelNamesToServicePorts, jwkThumbprint, profileStore, telemetryEmitter, @@ -82,6 +82,12 @@ export default class Up extends MachineCreationDriverCommand { ...envIdFlags, ...tunnelServerFlags, ...buildFlags, + 'skip-volume': Flags.string({ + description: 'Additional volume glob patterns to skip copying', + multiple: true, + singleValue: true, + default: [], + }), 'skip-unchanged-files': Flags.boolean({ description: 'Detect and skip unchanged files when copying (default: true)', default: true, @@ -181,6 +187,7 @@ export default class Up extends MachineCreationDriverCommand { projectName, expectedServiceUrls, userSpecifiedServices: restArgs, + volumeSkipList: [...defaultVolumeSkipList, ...flags['skip-volume']], debug: flags.debug, userSpecifiedProjectName: flags.project, composeFiles: this.config.composeFiles, diff --git a/packages/core/package.json b/packages/core/package.json index d98f94af..dcb0c8d6 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -19,6 +19,7 @@ "jose": "^4.14.4", "jsonwebtoken": "^9.0.0", "lodash": "^4.17.21", + "minimatch": "^9.0.3", "node-fetch": "2.6.9", "node-forge": "^1.3.1", "open": "^8.4.2", diff --git a/packages/core/src/commands/model.ts b/packages/core/src/commands/model.ts index 16a6eed0..771456b1 100644 --- a/packages/core/src/commands/model.ts +++ b/packages/core/src/commands/model.ts @@ -14,6 +14,7 @@ const composeModel = async ({ tunnelOpts, userSpecifiedProjectName, userSpecifiedServices, + volumeSkipList, scriptInjections, composeFiles, log, @@ -33,6 +34,7 @@ const composeModel = async ({ tunnelOpts: TunnelOpts userSpecifiedProjectName: string | undefined userSpecifiedServices: string[] + volumeSkipList: string[] composeFiles: string[] log: Logger dataDir: string @@ -55,6 +57,7 @@ const composeModel = async ({ debug, userSpecifiedProjectName, userSpecifiedServices, + volumeSkipList, composeFiles, log, cwd, diff --git a/packages/core/src/commands/up.ts b/packages/core/src/commands/up.ts index 274ad377..331b7d48 100644 --- a/packages/core/src/commands/up.ts +++ b/packages/core/src/commands/up.ts @@ -44,6 +44,7 @@ const up = async ({ tunnelOpts, userSpecifiedProjectName, userSpecifiedServices, + volumeSkipList, scriptInjections, composeFiles, log, @@ -67,6 +68,7 @@ const up = async ({ tunnelOpts: TunnelOpts userSpecifiedProjectName: string | undefined userSpecifiedServices: string[] + volumeSkipList: string[] composeFiles: string[] log: Logger dataDir: string @@ -87,6 +89,7 @@ const up = async ({ const { model, filesToCopy, + skippedVolumes, projectLocalDataDir, createCopiedFile, } = await modelCommand({ @@ -98,6 +101,7 @@ const up = async ({ tunnelOpts, userSpecifiedProjectName, userSpecifiedServices, + volumeSkipList, scriptInjections, version, envId, @@ -128,6 +132,10 @@ const up = async ({ })).deployModel } + skippedVolumes.forEach(({ service, source, matchingRule }) => { + log.info(`Not copying volume "${source}" for service "${service}" because it matched skip glob "${matchingRule}"`) + }) + const modelStr = yaml.stringify(composeModel) log.debug('model', modelStr) const composeFilePath = await createCopiedFile(composeModelFilename, modelStr) diff --git a/packages/core/src/compose/remote.ts b/packages/core/src/compose/remote.ts index 084a3112..39f85ba7 100644 --- a/packages/core/src/compose/remote.ts +++ b/packages/core/src/compose/remote.ts @@ -1,10 +1,11 @@ import yaml from 'yaml' import path from 'path' import { mapValues } from 'lodash' +import { MMRegExp, makeRe } from 'minimatch' import { asyncMap, asyncToArray } from 'iter-tools-es' import { COMPOSE_TUNNEL_AGENT_SERVICE_NAME, MachineStatusCommand, ScriptInjection, formatPublicKey } from '@preevy/common' import { MachineConnection } from '../driver' -import { ComposeModel, ComposeSecretOrConfig, composeModelFilename } from './model' +import { ComposeBindVolume, ComposeModel, ComposeSecretOrConfig, composeModelFilename } from './model' import { REMOTE_DIR_BASE, remoteProjectDir } from '../remote-files' import { TunnelOpts } from '../ssh' import { addComposeTunnelAgentService } from '../compose-tunnel-agent-client' @@ -28,23 +29,42 @@ const serviceLinkEnvVars = ( .map(({ name, port, url }) => [`PREEVY_BASE_URI_${name.replace(/[^a-zA-Z0-9_]/g, '_')}_${port}`.toUpperCase(), url]) ) -const volumeSkipList = [ - /^\/var\/log(\/|$)/, - /^\/var\/run(\/|$)/, - /^\/$/, +export const defaultVolumeSkipList: string[] = [ + '/var/log', + '/var/log/**', + '/var/run', + '/var/run/**', + '/', ] const toPosix = (x:string) => x.split(path.sep).join(path.posix.sep) +export type SkippedVolume = { service: string; source: string; matchingRule: string } + const fixModelForRemote = async ( - { skipServices = [], cwd, remoteBaseDir }: { + { skipServices = [], cwd, remoteBaseDir, volumeSkipList = defaultVolumeSkipList }: { skipServices?: string[] cwd: string remoteBaseDir: string + volumeSkipList: string[] }, model: ComposeModel, -): Promise<{ model: Required>; filesToCopy: FileToCopy[] }> => { +): Promise<{ + model: Required> + filesToCopy: FileToCopy[] + skippedVolumes: SkippedVolume[] +}> => { + const volumeSkipRes = volumeSkipList + .map(s => makeRe(path.resolve(cwd, s))) + .map((r, i) => { + if (!r) { + throw new Error(`Invalid glob pattern in volumeSkipList: "${volumeSkipList[i]}"`) + } + return r as MMRegExp + }) + const filesToCopy: FileToCopy[] = [] + const skippedVolumes: SkippedVolume[] = [] const remotePath = (absolutePath: string) => { if (!path.isAbsolute(absolutePath)) { @@ -86,7 +106,9 @@ const fixModelForRemote = async ( if (volume.type !== 'bind') { throw new Error(`Unsupported volume type: ${volume.type} in service ${serviceName}`) } - if (volumeSkipList.some(re => re.test(volume.source))) { + const matchingVolumeSkipIndex = volumeSkipRes.findIndex(re => re.test(volume.source)) + if (matchingVolumeSkipIndex !== -1) { + skippedVolumes.push({ service: serviceName, source: volume.source, matchingRule: volumeSkipList[matchingVolumeSkipIndex] }) return volume } @@ -117,6 +139,7 @@ const fixModelForRemote = async ( networks: model.networks ?? {}, }, filesToCopy, + skippedVolumes, } } @@ -136,6 +159,7 @@ export const remoteComposeModel = async ({ debug, userSpecifiedProjectName, userSpecifiedServices, + volumeSkipList, composeFiles, log, cwd, @@ -147,6 +171,7 @@ export const remoteComposeModel = async ({ debug: boolean userSpecifiedProjectName: string | undefined userSpecifiedServices: string[] + volumeSkipList: string[] composeFiles: string[] log: Logger cwd: string @@ -172,8 +197,8 @@ export const remoteComposeModel = async ({ ? [...userSpecifiedServices].concat(COMPOSE_TUNNEL_AGENT_SERVICE_NAME) : [] - const { model: fixedModel, filesToCopy } = await fixModelForRemote( - { cwd, remoteBaseDir: remoteDir }, + const { model: fixedModel, filesToCopy, skippedVolumes } = await fixModelForRemote( + { cwd, remoteBaseDir: remoteDir, volumeSkipList }, await modelFilter(await composeClientWithInjectedArgs.getModel(services)), ) @@ -215,5 +240,5 @@ export const remoteComposeModel = async ({ filesToCopy.push(sshPrivateKeyFile, knownServerPublicKey) } - return { model, filesToCopy, linkEnvVars } + return { model, filesToCopy, skippedVolumes, linkEnvVars } } diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index c7fac498..bc9e64ea 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -27,7 +27,12 @@ export { } from './profile' export { telemetryEmitter, registerEmitter, wireProcessExit, createTelemetryEmitter, machineId } from './telemetry' export { fsTypeFromUrl, Store, VirtualFS, localFsFromUrl, localFs } from './store' -export { localComposeClient, ComposeModel, resolveComposeFiles, getExposedTcpServicePorts, fetchRemoteUserModel as remoteUserModel, NoComposeFilesError, addScriptInjectionsToServices as addScriptInjectionsToModel } from './compose' +export { + localComposeClient, ComposeModel, resolveComposeFiles, getExposedTcpServicePorts, + fetchRemoteUserModel as remoteUserModel, NoComposeFilesError, + addScriptInjectionsToServices as addScriptInjectionsToModel, + defaultVolumeSkipList, +} from './compose' export { withSpinner } from './spinner' export { findEnvId, findProjectName, findEnvIdByProjectName, validateEnvId, normalize as normalizeEnvId, EnvId } from './env-id' export { sshKeysStore } from './state'