From 9f43e9a21e5fc20c4581a18255775eb10fb5d888 Mon Sep 17 00:00:00 2001 From: Roy Razon Date: Tue, 21 Nov 2023 08:50:16 +0200 Subject: [PATCH] change flags, separate commands --- .../cli-common/src/hooks/init/load-plugins.ts | 1 + packages/cli-common/src/index.ts | 2 +- .../src/lib/common-flags/build-flags.ts | 76 ++++++ .../index.ts} | 28 +- packages/cli/src/commands/logs.ts | 34 ++- packages/cli/src/commands/proxy/connect.ts | 4 +- packages/cli/src/commands/up.ts | 148 ++++++---- packages/cli/src/driver-command.ts | 2 +- packages/cli/tsconfig.json | 2 +- packages/core/src/build.ts | 21 ++ packages/core/src/closable.ts | 17 -- packages/core/src/commands/build.ts | 114 ++++++++ packages/core/src/commands/index.ts | 2 +- packages/core/src/commands/model.ts | 77 ++++++ packages/core/src/commands/up.ts | 151 +++++++++++ packages/core/src/commands/up/index.ts | 256 ------------------ packages/core/src/commands/up/split-build.ts | 13 - .../core/src/compose-tunnel-agent-client.ts | 7 +- packages/core/src/compose/client.ts | 19 +- packages/core/src/compose/model.ts | 99 +------ packages/core/src/compose/remote.ts | 213 ++++++++++++++- .../core/src/compose/script-injection.test.ts | 27 +- packages/core/src/compose/script-injection.ts | 9 +- packages/core/src/compose/service-links.ts | 6 + packages/core/src/docker.ts | 53 +++- packages/core/src/driver/driver.ts | 17 +- packages/core/src/driver/index.ts | 3 +- .../driver/{machine.ts => machine-model.ts} | 0 .../machine-operations.ts} | 29 +- packages/core/src/driver/ssh.ts | 6 +- packages/core/src/env-id.ts | 2 +- packages/core/src/index.ts | 7 +- packages/core/src/nulls.ts | 6 + packages/core/src/remote-files.ts | 16 ++ packages/core/src/ssh/client/forward-out.ts | 5 +- packages/core/src/ssh/client/index.ts | 2 +- packages/core/src/ssh/client/sftp.ts | 2 +- packages/core/src/tunneling/model.ts | 3 +- .../src/driver/client/port-forward.ts | 5 +- packages/driver-kube-pod/src/driver/driver.ts | 11 +- 40 files changed, 928 insertions(+), 567 deletions(-) create mode 100644 packages/cli-common/src/lib/common-flags/build-flags.ts rename packages/cli-common/src/lib/{common-flags.ts => common-flags/index.ts} (58%) create mode 100644 packages/core/src/build.ts delete mode 100644 packages/core/src/closable.ts create mode 100644 packages/core/src/commands/build.ts create mode 100644 packages/core/src/commands/model.ts create mode 100644 packages/core/src/commands/up.ts delete mode 100644 packages/core/src/commands/up/index.ts delete mode 100644 packages/core/src/commands/up/split-build.ts create mode 100644 packages/core/src/compose/service-links.ts rename packages/core/src/driver/{machine.ts => machine-model.ts} (100%) rename packages/core/src/{commands/up/machine.ts => driver/machine-operations.ts} (89%) diff --git a/packages/cli-common/src/hooks/init/load-plugins.ts b/packages/cli-common/src/hooks/init/load-plugins.ts index fab42b2f..80eaacb5 100644 --- a/packages/cli-common/src/hooks/init/load-plugins.ts +++ b/packages/cli-common/src/hooks/init/load-plugins.ts @@ -36,6 +36,7 @@ export const initHook: OclifHook<'init'> = async function hook({ config, id, arg async () => await localComposeClient({ composeFiles, projectName: flags.project, + projectDirectory: process.cwd(), }).getModelOrError(), { text: `Loading compose file${composeFiles.length > 1 ? 's' : ''}: ${composeFiles.join(', ')}`, diff --git a/packages/cli-common/src/index.ts b/packages/cli-common/src/index.ts index 7c09748a..9274350f 100644 --- a/packages/cli-common/src/index.ts +++ b/packages/cli-common/src/index.ts @@ -2,7 +2,7 @@ export * from './lib/plugins/model' export * as text from './lib/text' export { HookName, HookFunc, HooksListeners, Hooks } from './lib/hooks' export { PluginContext, PluginInitContext } from './lib/plugins/context' -export { composeFlags, envIdFlags, tunnelServerFlags, urlFlags } from './lib/common-flags' +export { composeFlags, envIdFlags, tunnelServerFlags, urlFlags, buildFlags, parseBuildFlags, tableFlags } from './lib/common-flags' export { formatFlagsToArgs } from './lib/flags' export { initHook } from './hooks/init/load-plugins' export { default as BaseCommand } from './commands/base-command' diff --git a/packages/cli-common/src/lib/common-flags/build-flags.ts b/packages/cli-common/src/lib/common-flags/build-flags.ts new file mode 100644 index 00000000..0b6cdf7b --- /dev/null +++ b/packages/cli-common/src/lib/common-flags/build-flags.ts @@ -0,0 +1,76 @@ +import { Flags } from '@oclif/core' +import { InferredFlags } from '@oclif/core/lib/interfaces' +import { BuildSpec, parseRegistry } from '@preevy/core' + +const helpGroup = 'BUILD' + +export const buildFlags = { + 'no-build': Flags.boolean({ + description: 'Do not build images', + helpGroup, + allowNo: false, + default: false, + required: false, + }), + registry: Flags.string({ + description: 'Image registry. If this flag is specified, the "build-context" flag defaults to "*local"', + helpGroup, + required: false, + }), + 'registry-single-name': Flags.string({ + description: 'Use single name for image registry, ECR-style. Default: auto-detect from "registry" flag', + helpGroup, + required: false, + dependsOn: ['registry'], + }), + 'no-registry-single-name': Flags.boolean({ + description: 'Disable auto-detection for ECR-style registry single name', + helpGroup, + allowNo: false, + required: false, + exclusive: ['registry-single-name'], + }), + 'no-registry-cache': Flags.boolean({ + description: 'Do not add the registry as a cache source and target', + helpGroup, + required: false, + dependsOn: ['registry'], + }), + load: Flags.boolean({ + description: 'Load build results to the Docker server at the deployment target', + helpGroup, + required: false, + }), + builder: Flags.string({ + description: 'Builder to use', + helpGroup, + required: false, + }), + 'no-cache': Flags.boolean({ + description: 'Do not use cache when building the images', + helpGroup, + allowNo: false, + required: false, + }), +} as const + +export const parseBuildFlags = (flags: Omit, 'json'>): BuildSpec | undefined => { + if (flags['no-build']) { + return undefined + } + + return { + builder: flags.builder, + noCache: flags['no-cache'], + cacheFromRegistry: !flags['no-registry-cache'], + ...flags.registry + ? { + registry: parseRegistry({ + registry: flags.registry, + singleName: flags['no-registry-single-name'] ? false : flags['registry-single-name'], + }), + load: flags.load, + } + : { load: true }, + } +} diff --git a/packages/cli-common/src/lib/common-flags.ts b/packages/cli-common/src/lib/common-flags/index.ts similarity index 58% rename from packages/cli-common/src/lib/common-flags.ts rename to packages/cli-common/src/lib/common-flags/index.ts index 2ffb3784..ba2bf977 100644 --- a/packages/cli-common/src/lib/common-flags.ts +++ b/packages/cli-common/src/lib/common-flags/index.ts @@ -1,16 +1,22 @@ -import { Flags } from '@oclif/core' +import { Flags, ux } from '@oclif/core' +import { mapValues } from 'lodash' +import { EOL } from 'os' + +export * from './build-flags' + +export const tableFlags = mapValues(ux.table.flags(), f => ({ ...f, helpGroup: 'OUTPUT' })) const projectFlag = { project: Flags.string({ char: 'p', - description: 'Project name. Defaults to the Compose project name', + summary: 'Project name. Defaults to the Compose project name', required: false, helpGroup: 'GLOBAL', }), } export const composeFlags = { file: Flags.string({ - description: 'Compose configuration file', + summary: 'Compose configuration file', multiple: true, delimiter: ',', singleValue: true, @@ -20,7 +26,7 @@ export const composeFlags = { helpGroup: 'GLOBAL', }), 'system-compose-file': Flags.string({ - description: 'Add extra Compose configuration file without overriding the defaults', + summary: 'Add extra Compose configuration file without overriding the defaults', multiple: true, delimiter: ',', singleValue: true, @@ -33,7 +39,8 @@ export const composeFlags = { export const envIdFlags = { id: Flags.string({ - description: 'Environment id - affects created URLs. If not specified, will try to detect automatically', + summary: 'Environment id', + description: `Affects created URLs${EOL}If not specified, will detect from the current Git context`, required: false, }), ...projectFlag, @@ -41,30 +48,31 @@ export const envIdFlags = { export const tunnelServerFlags = { 'tunnel-url': Flags.string({ - description: 'Tunnel url, specify ssh://hostname[:port] or ssh+tls://hostname[:port]', + summary: 'Tunnel url, specify ssh://hostname[:port] or ssh+tls://hostname[:port]', char: 't', default: 'ssh+tls://livecycle.run' ?? process.env.PREVIEW_TUNNEL_OVERRIDE, }), 'tls-hostname': Flags.string({ - description: 'Override TLS server name when tunneling via HTTPS', + summary: 'Override TLS server name when tunneling via HTTPS', required: false, }), 'insecure-skip-verify': Flags.boolean({ - description: 'Skip TLS or SSH certificate verification', + summary: 'Skip TLS or SSH certificate verification', default: false, }), } as const export const urlFlags = { 'include-access-credentials': Flags.boolean({ - description: 'Include access credentials for basic auth for each service URL', + summary: 'Include access credentials for basic auth for each service URL', default: false, }), 'show-preevy-service-urls': Flags.boolean({ - description: 'Show URLs for internal Preevy services', + summary: 'Show URLs for internal Preevy services', default: false, }), 'access-credentials-type': Flags.custom<'browser' | 'api'>({ + summary: 'Access credentials type', options: ['api', 'browser'], dependsOn: ['include-access-credentials'], default: 'browser', diff --git a/packages/cli/src/commands/logs.ts b/packages/cli/src/commands/logs.ts index c3f28cb9..e5008a88 100644 --- a/packages/cli/src/commands/logs.ts +++ b/packages/cli/src/commands/logs.ts @@ -2,7 +2,7 @@ import yaml from 'yaml' import { Args, Flags, Interfaces } from '@oclif/core' import { addBaseComposeTunnelAgentService, - localComposeClient, wrapWithDockerSocket, findEnvId, MachineConnection, ComposeModel, remoteUserModel, + localComposeClient, findEnvId, MachineConnection, ComposeModel, remoteUserModel, dockerEnvContext, } from '@preevy/core' import { COMPOSE_TUNNEL_AGENT_SERVICE_NAME } from '@preevy/common' import DriverCommand from '../driver-command' @@ -98,23 +98,21 @@ export default class Logs extends DriverCommand { connection = await this.connect(envId) } - try { - const compose = localComposeClient({ - composeFiles: Buffer.from(yaml.stringify(addBaseComposeTunnelAgentService(userModel))), - projectName: flags.project, - }) + const compose = localComposeClient({ + composeFiles: Buffer.from(yaml.stringify(addBaseComposeTunnelAgentService(userModel))), + projectName: flags.project, + projectDirectory: process.cwd(), + }) - const withDockerSocket = wrapWithDockerSocket({ connection, log }) - await withDockerSocket(() => compose.spawnPromise( - [ - 'logs', - ...serializeDockerComposeLogsFlags(flags), - ...validateServices(restArgs, userModel), - ], - { stdio: 'inherit' }, - )) - } finally { - await connection.close() - } + await using dockerContext = await dockerEnvContext({ connection, log }) + + await compose.spawnPromise( + [ + 'logs', + ...serializeDockerComposeLogsFlags(flags), + ...validateServices(restArgs, userModel), + ], + { stdio: 'inherit', env: dockerContext.env }, + ) } } diff --git a/packages/cli/src/commands/proxy/connect.ts b/packages/cli/src/commands/proxy/connect.ts index 2d3779b5..251875de 100644 --- a/packages/cli/src/commands/proxy/connect.ts +++ b/packages/cli/src/commands/proxy/connect.ts @@ -1,6 +1,6 @@ import { ux, Args, Flags } from '@oclif/core' import { jwkThumbprint, commands, profileStore, withSpinner, SshConnection, machineId, validateEnvId, normalizeEnvId, EnvId } from '@preevy/core' -import { tunnelServerFlags, urlFlags } from '@preevy/cli-common' +import { text, tunnelServerFlags, urlFlags } from '@preevy/cli-common' import { inspect } from 'util' import { formatPublicKey } from '@preevy/common' import { spawn } from 'child_process' @@ -69,7 +69,7 @@ export default class Connect extends ProfileCommand { } else { const deviceId = (await machineId(this.config.dataDir)).substring(0, 2) envId = normalizeEnvId(`${composeProject}-dev-${deviceId}`) - this.logger.info(`Using environment ID ${envId}, based on Docker Compose and local device`) + this.logger.info(`Using environment ID ${text.code(envId)}, based on Docker Compose and local device`) } let client: SshConnection['client'] | undefined let hostKey: Buffer diff --git a/packages/cli/src/commands/up.ts b/packages/cli/src/commands/up.ts index f2d34145..9fa94a73 100644 --- a/packages/cli/src/commands/up.ts +++ b/packages/cli/src/commands/up.ts @@ -1,13 +1,17 @@ -import { Args, Flags, ux, Errors } from '@oclif/core' +import { Args, Flags, ux } from '@oclif/core' import { + ComposeModel, + Logger, + ProfileStore, + TunnelOpts, addBaseComposeTunnelAgentService, - commands, findComposeTunnelAgentUrl, + commands, ensureMachine, findComposeTunnelAgentUrl, findEnvId, findProjectName, getTunnelNamesToServicePorts, jwkThumbprint, profileStore, telemetryEmitter, withSpinner, } from '@preevy/core' -import { tunnelServerFlags } from '@preevy/cli-common' +import { buildFlags, parseBuildFlags, text, tunnelServerFlags } from '@preevy/cli-common' import { inspect } from 'util' import { editUrl, tunnelNameResolver } from '@preevy/common' import MachineCreationDriverCommand from '../machine-creation-driver-command' @@ -15,6 +19,61 @@ import { envIdFlags, urlFlags } from '../common-flags' import { filterUrls, printUrls } from './urls' import { connectToTunnelServerSsh } from '../tunnel-server-client' +const fetchTunnelServerDetails = async ({ + log, + tunnelingKey, + envId, + userModel, + pStore, + tunnelOpts, +}: { + log: Logger + tunnelingKey: string | Buffer + envId: string + userModel: ComposeModel + pStore: ProfileStore + tunnelOpts: TunnelOpts +}) => { + const expectedTunnels = getTunnelNamesToServicePorts( + addBaseComposeTunnelAgentService(userModel), + tunnelNameResolver({ envId }), + ) + + const { hostKey, expectedServiceUrls } = await withSpinner(async spinner => { + spinner.text = 'Connecting...' + + const { hostKey: hk, client: tunnelServerSshClient } = await connectToTunnelServerSsh({ + tunnelingKey, + knownServerPublicKeys: pStore.knownServerPublicKeys, + tunnelOpts, + log, + spinner, + }) + + spinner.text = 'Getting server details...' + + const [{ clientId }, expectedTunnelUrls] = await Promise.all([ + tunnelServerSshClient.execHello(), + tunnelServerSshClient.execTunnelUrl(Object.keys(expectedTunnels)), + ]) + + log.debug('Tunnel server details: %j', { clientId, expectedTunnelUrls }) + + void tunnelServerSshClient.end() + + telemetryEmitter().group({ type: 'profile' }, { proxy_client_id: clientId }) + + const esu = Object.entries(expectedTunnels) + .map(([tunnel, { name, port }]) => ({ name, port, url: expectedTunnelUrls[tunnel] })) + + return { hostKey: hk, expectedServiceUrls: esu } + }, { opPrefix: 'Tunnel server', successText: 'Got tunnel server details' }) + + log.debug('expectedServiceUrls: %j', expectedServiceUrls) + + return { expectedServiceUrls, hostKey } +} + // eslint-disable-next-line no-use-before-define export default class Up extends MachineCreationDriverCommand { static description = 'Bring up a preview environment' @@ -22,18 +81,7 @@ export default class Up extends MachineCreationDriverCommand { static flags = { ...envIdFlags, ...tunnelServerFlags, - 'local-build': Flags.custom({ - description: `Build locally and deploy remotely using an image registry; ${Object.entries(commands.localBuildSpecSchema.shape).map(([k, v]) => `${k}: ${v.description}`).join(', ')}`, - required: false, - parse: async input => { - const pairs = input.split(',') - const result = commands.localBuildSpecSchema.safeParse(Object.fromEntries(pairs.map(pair => pair.split('=')))) - if (!result.success) { - throw new Errors.CLIError(`Invalid local-build arg: ${inspect(result.error)}`) - } - return result.data - }, - })(), + ...buildFlags, 'skip-unchanged-files': Flags.boolean({ description: 'Detect and skip unchanged files when copying (default: true)', default: true, @@ -95,55 +143,45 @@ export default class Up extends MachineCreationDriverCommand { insecureSkipVerify: flags['insecure-skip-verify'], } - const expectedTunnels = getTunnelNamesToServicePorts( - addBaseComposeTunnelAgentService(userModel), - tunnelNameResolver({ envId }), - ) - - const { hostKey, expectedServiceUrls } = await withSpinner(async spinner => { - spinner.text = 'Connecting...' - - const { hostKey: hk, client: tunnelServerSshClient } = await connectToTunnelServerSsh({ - tunnelingKey, - knownServerPublicKeys: pStore.knownServerPublicKeys, - tunnelOpts, - log: this.logger, - spinner, - }) - - spinner.text = 'Getting server details...' - - const [{ clientId }, expectedTunnelUrls] = await Promise.all([ - tunnelServerSshClient.execHello(), - tunnelServerSshClient.execTunnelUrl(Object.keys(expectedTunnels)), - ]) - - this.logger.debug('Tunnel server details: %j', { clientId, expectedTunnelUrls }) + const { expectedServiceUrls, hostKey } = await fetchTunnelServerDetails({ + log: this.logger, + tunnelingKey, + envId, + userModel, + pStore, + tunnelOpts, + }) - void tunnelServerSshClient.end() + const injectWidgetScript = flags['enable-widget'] + ? editUrl(flags['livecycle-widget-url'], { queryParams: { profile: thumbprint, env: envId } }).toString() + : undefined - telemetryEmitter().group({ type: 'profile' }, { proxy_client_id: clientId }) + await using cleanup = new AsyncDisposableStack() - const esu = Object.entries(expectedTunnels) - .map(([tunnel, { name, port }]) => ({ name, port, url: expectedTunnelUrls[tunnel] })) + const { machine, connection, userAndGroup, dockerPlatform } = await ensureMachine({ + log: this.logger, + debug: this.flags.debug, + machineDriver: driver, + machineCreationDriver, + machineDriverName: this.driverName, + envId, + }) - return { hostKey: hk, expectedServiceUrls: esu } - }, { opPrefix: 'Tunnel server', successText: 'Got tunnel server details' }) + const machineStatusCommand = await driver.machineStatusCommand(machine) - this.logger.debug('expectedServiceUrls: %j', expectedServiceUrls) + cleanup.use(connection) - const injectWidgetScript = flags['enable-widget'] - ? editUrl(flags['livecycle-widget-url'], { queryParams: { profile: thumbprint, env: envId } }).toString() - : undefined + const buildSpec = parseBuildFlags(flags) - const { machine } = await commands.up({ + await commands.up({ + connection, + machineStatusCommand, + userAndGroup, + dockerPlatform, projectName, expectedServiceUrls, userSpecifiedServices: restArgs, debug: flags.debug, - machineDriver: driver, - machineDriverName: this.driverName, - machineCreationDriver, userSpecifiedProjectName: flags.project, composeFiles: this.config.composeFiles, envId, @@ -156,10 +194,10 @@ export default class Up extends MachineCreationDriverCommand { cwd: process.cwd(), skipUnchangedFiles: flags['skip-unchanged-files'], version: this.config.version, - localBuildSpec: flags['local-build'], + buildSpec, }) - this.log(`Preview environment ${envId} provisioned at: ${machine.locationDescription}`) + this.log(`Preview environment ${text.code(envId)} provisioned at: ${text.code(machine.locationDescription)}`) const composeTunnelServiceUrl = findComposeTunnelAgentUrl(expectedServiceUrls) const flatTunnels = await withSpinner(() => commands.urls({ diff --git a/packages/cli/src/driver-command.ts b/packages/cli/src/driver-command.ts index 11738789..a96301d9 100644 --- a/packages/cli/src/driver-command.ts +++ b/packages/cli/src/driver-command.ts @@ -78,7 +78,7 @@ abstract class DriverCommand extends ProfileCommand try { return await f(connection) } finally { - await connection.close() + connection[Symbol.dispose]() } } diff --git a/packages/cli/tsconfig.json b/packages/cli/tsconfig.json index 19cbe8e0..76af7590 100644 --- a/packages/cli/tsconfig.json +++ b/packages/cli/tsconfig.json @@ -7,7 +7,7 @@ "rootDir": "src", "strict": true, "target": "es2019", - "lib": ["es2022"], + "lib": ["es2022", "ESNext.Disposable"], "esModuleInterop": true, "sourceMap": true, "baseUrl": ".", diff --git a/packages/core/src/build.ts b/packages/core/src/build.ts new file mode 100644 index 00000000..00deb313 --- /dev/null +++ b/packages/core/src/build.ts @@ -0,0 +1,21 @@ +export type ImageRegistry = { registry: string; singleName?: string } + +export type BuildSpec = ({ registry?: ImageRegistry; load: true } | { registry: ImageRegistry; load: boolean }) & { + cacheFromRegistry?: boolean + noCache?: boolean + builder?: string +} + +const ecrRegex = /^(?[0-9]+\.dkr\.ecr\.[^.]+\.*\.amazonaws\.com)\/(?.+)/ + +export const parseRegistry = ( + { registry, singleName }: { registry: string; singleName: undefined | string | false }, +): ImageRegistry => { + if (singleName === undefined) { + const match = ecrRegex.exec(registry) + if (match) { + return { registry: match.groups?.registry as string, singleName: match.groups?.singleName as string } + } + } + return { registry, singleName: typeof singleName === 'string' ? singleName : undefined } +} diff --git a/packages/core/src/closable.ts b/packages/core/src/closable.ts deleted file mode 100644 index 90c6f5c1..00000000 --- a/packages/core/src/closable.ts +++ /dev/null @@ -1,17 +0,0 @@ -import { isPromise } from 'util/types' - -export type Closable = { close: () => void | Promise } -export const withClosable = async ( - f: (closable: C, ...args: Args) => Return, - closable: C, - ...args: Args -) => { - try { - const result = f(closable, ...args) - return isPromise(result) - ? await result - : result - } finally { - await closable.close() - } -} diff --git a/packages/core/src/commands/build.ts b/packages/core/src/commands/build.ts new file mode 100644 index 00000000..2130b5ad --- /dev/null +++ b/packages/core/src/commands/build.ts @@ -0,0 +1,114 @@ +import fs from 'fs' +import path from 'path' +import yaml from 'yaml' +import { mapValues, pickBy } from 'lodash' +import { spawn } from 'child_process' +import { ComposeModel } from '../compose' +import { Logger } from '../log' +import { BuildSpec, ImageRegistry } from '../build' +import { gitContext } from '../git' +import { randomString } from '../strings' +import { childProcessPromise } from '../child-process' +import { hasProp } from '../nulls' + +type ImageRefFactory = ({ image, tag }: { image: string; tag: string }) => string + +const plainImageRefFactory: ImageRefFactory = ({ image, tag }) => `${image}:${tag}` + +const registryImageRefFactory = ({ registry, singleName }: ImageRegistry): ImageRefFactory => ( + singleName + ? ({ image, tag }) => `${registry}/${singleName}:${image}-${tag}` + : ({ image, tag }) => `${registry}/${image}:${tag}` +) + +const buildCommand = async ({ + log, + composeModel, + projectLocalDataDir, + cwd, + buildSpec, + machineDockerPlatform, + env, +}: { + log: Logger + composeModel: ComposeModel + projectLocalDataDir: string + cwd: string + buildSpec: BuildSpec + machineDockerPlatform: string + env?: Record +}) => { + const tagSuffix = await gitContext(cwd)?.commit({ short: true }) ?? randomString.lowercaseNumeric(8) + + const imageRef = buildSpec.registry + ? registryImageRefFactory(buildSpec.registry) + : plainImageRefFactory + + const imageRefForService = (service: string, tag: string) => imageRef({ + image: `preevy-${composeModel.name}-${service}`, + tag, + }) + + const services = mapValues( + pickBy(composeModel.services ?? {}, hasProp('build')), + ({ build }, serviceName) => { + const latestImage = imageRefForService(serviceName, 'latest') + const thisImage = imageRefForService(serviceName, tagSuffix) + + const cacheFrom = build.cache_from ?? [] + const cacheTo = build.cache_to ?? [] + const tags = build?.tags ?? [] + + if (buildSpec.registry && buildSpec.cacheFromRegistry) { + cacheTo.push(`type=registry,ref=${latestImage},mode=max,oci-mediatypes=true,image-manifest=true`) + cacheFrom.push(latestImage) + cacheFrom.push(thisImage) + } + + tags.push(latestImage) + tags.push(thisImage) + + return { + image: thisImage as string, + build: { + ...build, + tags, + cache_from: cacheFrom, + cache_to: cacheTo, + }, + } + }, + ) + + const buildModel: ComposeModel = { name: composeModel.name, services } + const modelStr = yaml.stringify(buildModel) + log.debug('build model', modelStr) + const modelFilename = path.join(projectLocalDataDir, 'docker-compose.build.yaml') + await fs.promises.writeFile(modelFilename, modelStr, 'utf-8') + + const bakeArgs = [ + 'buildx', 'bake', + '-f', modelFilename, + ...buildSpec.registry ? ['--push'] : [], + ...buildSpec.load ? ['--load'] : [], + ...buildSpec.builder ? [`--builder=${buildSpec.builder}`] : [], + ...buildSpec.noCache ? ['--no-cache'] : [], + `--set=*.platform=${machineDockerPlatform}`, + ] + log.info(`Running: docker ${bakeArgs.join(' ')}`) + await childProcessPromise(spawn('docker', bakeArgs, { stdio: 'inherit', cwd, env })) + + const deployModel: ComposeModel = { + ...composeModel, + services: { + ...mapValues(composeModel.services, (service, serviceName) => ({ + ...service, + image: buildModel.services?.[serviceName]?.image ?? service.image, + })), + }, + } + + return { buildModel, deployModel } +} + +export default buildCommand diff --git a/packages/core/src/commands/index.ts b/packages/core/src/commands/index.ts index 19aa7309..59b95d9c 100644 --- a/packages/core/src/commands/index.ts +++ b/packages/core/src/commands/index.ts @@ -1,4 +1,4 @@ -export { LocalBuildSpec, localBuildSpecSchema, default as up } from './up' +export { default as up } from './up' export { default as ls } from './ls' export { default as shell } from './shell' diff --git a/packages/core/src/commands/model.ts b/packages/core/src/commands/model.ts new file mode 100644 index 00000000..0127b4aa --- /dev/null +++ b/packages/core/src/commands/model.ts @@ -0,0 +1,77 @@ +import { MachineStatusCommand, ScriptInjection } from '@preevy/common' +import path from 'path' +import { rimraf } from 'rimraf' +import { TunnelOpts } from '../ssh' +import { remoteComposeModel } from '../compose' +import { createCopiedFileInDataDir } from '../remote-files' +import { Logger } from '../log' +import { EnvId } from '../env-id' + +const composeModel = async ({ + debug, + machineStatusCommand, + userAndGroup, + tunnelOpts, + userSpecifiedProjectName, + userSpecifiedServices, + scriptInjections, + composeFiles, + log, + dataDir, + allowedSshHostKeys: hostKey, + sshTunnelPrivateKey, + cwd, + version, + envId, + expectedServiceUrls, + projectName, +}: { + debug: boolean + machineStatusCommand?: MachineStatusCommand + userAndGroup: [string, string] + tunnelOpts: TunnelOpts + userSpecifiedProjectName: string | undefined + userSpecifiedServices: string[] + composeFiles: string[] + log: Logger + dataDir: string + scriptInjections?: Record + sshTunnelPrivateKey: string | Buffer + allowedSshHostKeys: Buffer + cwd: string + version: string + envId: EnvId + expectedServiceUrls: { name: string; port: number; url: string }[] + projectName: string +}) => { + const projectLocalDataDir = path.join(dataDir, projectName) + await rimraf(projectLocalDataDir) + + const createCopiedFile = createCopiedFileInDataDir({ projectLocalDataDir }) + + const remoteModel = await remoteComposeModel({ + debug, + userSpecifiedProjectName, + userSpecifiedServices, + composeFiles, + log, + cwd, + expectedServiceUrls, + projectName, + agentSettings: { + allowedSshHostKeys: hostKey, + sshTunnelPrivateKey, + userAndGroup, + createCopiedFile, + envId, + tunnelOpts, + version, + machineStatusCommand, + scriptInjections, + }, + }) + + return { ...remoteModel, projectLocalDataDir, createCopiedFile } +} + +export default composeModel diff --git a/packages/core/src/commands/up.ts b/packages/core/src/commands/up.ts new file mode 100644 index 00000000..ed4c7c9d --- /dev/null +++ b/packages/core/src/commands/up.ts @@ -0,0 +1,151 @@ +import { MachineStatusCommand, ScriptInjection } from '@preevy/common' +import yaml from 'yaml' +import { TunnelOpts } from '../ssh' +import { composeModelFilename, localComposeClient } from '../compose' +import { dockerEnvContext } from '../docker' +import { MachineConnection } from '../driver' +import { remoteProjectDir } from '../remote-files' +import { Logger } from '../log' +import { FileToCopy, uploadWithSpinner } from '../upload-files' +import { EnvId } from '../env-id' +import { BuildSpec } from '../build' +import modelCommand from './model' +import buildCommand from './build' +import { CommandExecuter } from '../command-executer' + +const uploadFiles = async ({ + log, + filesToCopy, + exec, + skipUnchangedFiles, + remoteDir, +}: { + log: Logger + filesToCopy: FileToCopy[] + exec: CommandExecuter + skipUnchangedFiles: boolean + remoteDir: string +}) => { + await exec(`mkdir -p "${remoteDir}"`) + + log.debug('Files to copy', filesToCopy) + + await uploadWithSpinner(exec, remoteDir, filesToCopy, skipUnchangedFiles) +} + +const up = async ({ + debug, + machineStatusCommand, + userAndGroup, + dockerPlatform, + connection, + tunnelOpts, + userSpecifiedProjectName, + userSpecifiedServices, + scriptInjections, + composeFiles, + log, + dataDir, + allowedSshHostKeys, + sshTunnelPrivateKey, + cwd, + skipUnchangedFiles, + version, + envId, + expectedServiceUrls, + projectName, + buildSpec, +}: { + debug: boolean + machineStatusCommand?: MachineStatusCommand + userAndGroup: [string, string] + dockerPlatform: string + connection: Pick + tunnelOpts: TunnelOpts + userSpecifiedProjectName: string | undefined + userSpecifiedServices: string[] + composeFiles: string[] + log: Logger + dataDir: string + scriptInjections?: Record + sshTunnelPrivateKey: string | Buffer + allowedSshHostKeys: Buffer + cwd: string + skipUnchangedFiles: boolean + version: string + envId: EnvId + expectedServiceUrls: { name: string; port: number; url: string }[] + projectName: string + buildSpec?: BuildSpec +}) => { + const remoteDir = remoteProjectDir(projectName) + + const { + model, + filesToCopy, + projectLocalDataDir, + createCopiedFile, + } = await modelCommand({ + debug, + log, + machineStatusCommand, + userAndGroup, + cwd, + tunnelOpts, + userSpecifiedProjectName, + userSpecifiedServices, + scriptInjections, + version, + envId, + allowedSshHostKeys, + composeFiles, + dataDir, + expectedServiceUrls, + projectName, + sshTunnelPrivateKey, + }) + + log.info('build spec: %j', buildSpec ?? 'none') + + let composeModel = model + + if (buildSpec) { + await using dockerContext = await dockerEnvContext({ connection, log }) + + composeModel = (await buildCommand({ + log, + buildSpec, + cwd, + composeModel, + projectLocalDataDir, + machineDockerPlatform: dockerPlatform, + env: dockerContext.env, + })).deployModel + } + + const modelStr = yaml.stringify(composeModel) + log.debug('model', modelStr) + const composeFilePath = await createCopiedFile(composeModelFilename, modelStr) + filesToCopy.push(composeFilePath) + + await uploadFiles({ log, filesToCopy, exec: connection.exec, skipUnchangedFiles, remoteDir }) + + const compose = localComposeClient({ + composeFiles: [composeFilePath.local], + projectDirectory: cwd, + }) + + const composeArgs = [ + ...debug ? ['--verbose'] : [], + 'up', '-d', '--remove-orphans', '--no-build', + ] + + log.info(`Running: docker compose up ${composeArgs.join(' ')}`) + + await using dockerContext = await dockerEnvContext({ connection, log }) + await compose.spawnPromise(composeArgs, { stdio: 'inherit', env: dockerContext.env }) + + return { composeModel, projectLocalDataDir } +} + +export default up diff --git a/packages/core/src/commands/up/index.ts b/packages/core/src/commands/up/index.ts deleted file mode 100644 index a373ec61..00000000 --- a/packages/core/src/commands/up/index.ts +++ /dev/null @@ -1,256 +0,0 @@ -import { COMPOSE_TUNNEL_AGENT_SERVICE_NAME, ScriptInjection, formatPublicKey, readOrUndefined } from '@preevy/common' -import fs from 'fs' -import path from 'path' -import { rimraf } from 'rimraf' -import yaml from 'yaml' -import { spawn } from 'child_process' -import { mapValues, pickBy } from 'lodash' -import { childProcessPromise } from '../../child-process' -import { TunnelOpts } from '../../ssh' -import { composeModelFilename, fixModelForRemote, localComposeClient, addScriptInjectionsToModel, ComposeBuild, ComposeService } from '../../compose' -import { ensureCustomizedMachine } from './machine' -import { wrapWithDockerSocket } from '../../docker' -import { addComposeTunnelAgentService } from '../../compose-tunnel-agent-client' -import { MachineCreationDriver, MachineDriver, MachineBase } from '../../driver' -import { remoteProjectDir } from '../../remote-files' -import { Logger } from '../../log' -import { FileToCopy, uploadWithSpinner } from '../../upload-files' -import { envMetadata } from '../../env-metadata' -import { EnvId } from '../../env-id' -import { LocalBuildSpec } from './split-build' -import { gitContext } from '../../git' -import { randomString } from '../../strings' - -export { localBuildSpecSchema, LocalBuildSpec } from './split-build' - -const createCopiedFileInDataDir = ( - { projectLocalDataDir, filesToCopy } : { - projectLocalDataDir: string - filesToCopy: FileToCopy[] - } -) => async ( - filename: string, - content: string | Buffer -): Promise<{ local: string; remote: string }> => { - const local = path.join(projectLocalDataDir, filename) - const result = { local, remote: filename } - if (await readOrUndefined(local) === Buffer.from(content)) { - return result - } - await fs.promises.mkdir(path.dirname(local), { recursive: true }) - await fs.promises.writeFile(local, content, { flag: 'w' }) - filesToCopy.push(result) - return result -} - -const calcComposeUpArgs = ({ userSpecifiedServices, debug, cwd, build } : { - userSpecifiedServices: string[] - debug: boolean - cwd: string - build: boolean -}) => { - const upServices = userSpecifiedServices.length - ? userSpecifiedServices.concat(COMPOSE_TUNNEL_AGENT_SERVICE_NAME) - : [] - - return [ - ...debug ? ['--verbose'] : [], - '--project-directory', cwd, - 'up', '-d', '--remove-orphans', - ...build ? ['--build'] : [], - ...upServices, - ] -} - -const serviceLinkEnvVars = ( - expectedServiceUrls: { name: string; port: number; url: string }[], -) => Object.fromEntries( - expectedServiceUrls - .map(({ name, port, url }) => [`PREEVY_BASE_URI_${name.replace(/[^a-zA-Z0-9_]/g, '_')}_${port}`.toUpperCase(), url]) -) - -const isEcr = (registry: string) => Boolean(registry.match(/^[0-9]+\.dkr\.ecr\.[^.]+\.*\.amazonaws\.com\/.+/)) - -const registryImageName = ( - { registry, image, tag }: { - registry: string - image: string - tag: string - }, - { ecrFormat }: { ecrFormat?: boolean } = {}, -) => { - const formatForEcr = ecrFormat === undefined ? isEcr(registry) : ecrFormat - return formatForEcr ? `${registry}:${image}-${tag}` : `${registry}/${image}:${tag}` -} - -const up = async ({ - debug, - machineDriver, - machineDriverName, - machineCreationDriver, - tunnelOpts, - userSpecifiedProjectName, - userSpecifiedServices, - scriptInjections, - composeFiles, - log, - dataDir, - allowedSshHostKeys: hostKey, - sshTunnelPrivateKey, - cwd, - skipUnchangedFiles, - version, - envId, - expectedServiceUrls, - projectName, - localBuildSpec, -}: { - debug: boolean - machineDriver: MachineDriver - machineDriverName: string - machineCreationDriver: MachineCreationDriver - tunnelOpts: TunnelOpts - userSpecifiedProjectName: string | undefined - userSpecifiedServices: string[] - composeFiles: string[] - log: Logger - dataDir: string - scriptInjections?: Record - sshTunnelPrivateKey: string | Buffer - allowedSshHostKeys: Buffer - cwd: string - skipUnchangedFiles: boolean - version: string - envId: EnvId - expectedServiceUrls: { name: string; port: number; url: string }[] - projectName: string - localBuildSpec?: LocalBuildSpec -}): Promise<{ machine: MachineBase }> => { - const remoteDir = remoteProjectDir(projectName) - - log.debug(`Using compose files: ${composeFiles.join(', ')}`) - - const composeClientWithInjectedArgs = localComposeClient({ - composeFiles, - env: serviceLinkEnvVars(expectedServiceUrls), - projectName: userSpecifiedProjectName, - }) - - await using cleanup = new AsyncDisposableStack() - - const { machine, connection, userAndGroup, dockerPlatform } = await ensureCustomizedMachine({ - machineDriver, machineCreationDriver, machineDriverName, envId, log, debug, - }) - - cleanup.defer(() => connection.close()) - - const { model: fixedModel, filesToCopy } = await fixModelForRemote( - { cwd, remoteBaseDir: remoteDir }, - await composeClientWithInjectedArgs.getModel() - ) - - const projectLocalDataDir = path.join(dataDir, projectName) - await rimraf(projectLocalDataDir) - - const createCopiedFile = createCopiedFileInDataDir({ projectLocalDataDir, filesToCopy }) - const [sshPrivateKeyFile, knownServerPublicKey] = await Promise.all([ - createCopiedFile('tunnel_client_private_key', sshTunnelPrivateKey), - createCopiedFile('tunnel_server_public_key', formatPublicKey(hostKey)), - ]) - - let remoteModel = addComposeTunnelAgentService({ - envId, - debug, - tunnelOpts, - sshPrivateKeyPath: path.posix.join(remoteDir, sshPrivateKeyFile.remote), - knownServerPublicKeyPath: path.posix.join(remoteDir, knownServerPublicKey.remote), - user: userAndGroup.join(':'), - machineStatusCommand: await machineDriver.machineStatusCommand(machine), - envMetadata: await envMetadata({ envId, version }), - composeModelPath: path.posix.join(remoteDir, composeModelFilename), - privateMode: false, - defaultAccess: 'public', - composeProject: projectName, - }, fixedModel) - - if (scriptInjections) { - remoteModel = addScriptInjectionsToModel( - remoteModel, - serviceName => (serviceName !== COMPOSE_TUNNEL_AGENT_SERVICE_NAME ? scriptInjections : undefined), - ) - } - - if (localBuildSpec) { - const tagSuffix = await gitContext(cwd)?.commit({ short: true }) ?? randomString.lowercaseNumeric(8) - const serviceImage = (service: string, tag: string) => registryImageName( - { - registry: localBuildSpec.registry, - image: `preevy-${envId}-${service}`, - tag, - }, - { ecrFormat: localBuildSpec.ecrFormat }, - ) - - const buildServices = mapValues( - pickBy(remoteModel.services, ({ build }) => build), - ({ build }: { build: ComposeBuild}, service) => { - const latestImage = serviceImage(service, 'latest') - const thisImage = serviceImage(service, tagSuffix) - return ({ - build: Object.assign(build, { - tags: (build.tags ?? []).concat(thisImage, latestImage), - cache_from: (build.cache_from ?? []).concat(latestImage), - cache_to: (build.cache_to ?? []).concat( - ...localBuildSpec.cacheToLatest - ? [`type=registry,ref=${latestImage},image-manifest=true,mode=max`] - : [], - ), - }), - }) - }, - ) - - const buildFilename = path.join(projectLocalDataDir, 'build.yaml') - await fs.promises.writeFile(buildFilename, yaml.stringify({ services: buildServices })) - - await childProcessPromise(spawn('docker', [ - 'buildx', 'bake', - '-f', buildFilename, - `--set=*.platform=${localBuildSpec.platform ?? dockerPlatform}`, - '--push', - ], { - stdio: 'inherit', - })) - - for (const serviceName of Object.keys(buildServices)) { - (remoteModel.services as Record)[serviceName].image = serviceImage(serviceName, tagSuffix) - } - } - - const modelStr = yaml.stringify(remoteModel) - log.debug('model', modelStr) - const composeFilePath = await createCopiedFile(composeModelFilename, modelStr) - - const { exec } = connection - - await exec(`mkdir -p "${remoteDir}"`) - - log.debug('Files to copy', filesToCopy) - - await uploadWithSpinner(exec, remoteDir, filesToCopy, skipUnchangedFiles) - - const compose = localComposeClient({ - composeFiles: [composeFilePath.local], - projectName: userSpecifiedProjectName, - }) - const composeArgs = calcComposeUpArgs({ userSpecifiedServices, debug, cwd, build: !localBuildSpec }) - - const withDockerSocket = wrapWithDockerSocket({ connection, log }) - - log.info(`Running: docker compose up ${composeArgs.join(' ')}`) - await withDockerSocket(() => compose.spawnPromise(composeArgs, { stdio: 'inherit' })) - - return { machine } -} - -export default up diff --git a/packages/core/src/commands/up/split-build.ts b/packages/core/src/commands/up/split-build.ts deleted file mode 100644 index aa70bf31..00000000 --- a/packages/core/src/commands/up/split-build.ts +++ /dev/null @@ -1,13 +0,0 @@ -import z from 'zod' - -const trueValues: unknown[] = [true, 1].flatMap(x => [x, String(x)]) -const booleanStr = () => z.preprocess(v => trueValues.includes(v), z.boolean()) - -export const localBuildSpecSchema = z.object({ - registry: z.string().describe('registry to use'), - platform: z.string().optional().describe('platform to build for (default: driver platform)'), - ecrFormat: booleanStr().optional().describe('use ECR format (default: auto detect)'), - cacheToLatest: booleanStr().default(false).describe('enable cache-to with latest tag (default: false)'), -}) - -export type LocalBuildSpec = z.infer diff --git a/packages/core/src/compose-tunnel-agent-client.ts b/packages/core/src/compose-tunnel-agent-client.ts index 47fb8054..426d0c8c 100644 --- a/packages/core/src/compose-tunnel-agent-client.ts +++ b/packages/core/src/compose-tunnel-agent-client.ts @@ -3,7 +3,7 @@ import fetch from 'node-fetch' import retry from 'p-retry' import util from 'util' import { mapValues, merge } from 'lodash' -import { COMPOSE_TUNNEL_AGENT_PORT, COMPOSE_TUNNEL_AGENT_SERVICE_LABELS, COMPOSE_TUNNEL_AGENT_SERVICE_NAME, MachineStatusCommand, dateReplacer } from '@preevy/common' +import { COMPOSE_TUNNEL_AGENT_PORT, COMPOSE_TUNNEL_AGENT_SERVICE_LABELS, COMPOSE_TUNNEL_AGENT_SERVICE_NAME, MachineStatusCommand, ScriptInjection, dateReplacer } from '@preevy/common' import { ComposeModel, ComposeService, composeModelFilename } from './compose/model' import { TunnelOpts } from './ssh/url' import { Tunnel } from './tunneling' @@ -12,6 +12,7 @@ import { EnvMetadata, driverMetadataFilename } from './env-metadata' import { REMOTE_DIR_BASE } from './remote-files' import { isPacked, pkgSnapshotDir } from './pkg' import { EnvId } from './env-id' +import { addScriptInjectionsToServices } from './compose/script-injection' const COMPOSE_TUNNEL_AGENT_DIR = path.join(path.dirname(require.resolve('@preevy/compose-tunnel-agent')), '..') @@ -62,6 +63,7 @@ export const addComposeTunnelAgentService = ( profileThumbprint, privateMode, defaultAccess, + scriptInjections, }: { tunnelOpts: TunnelOpts sshPrivateKeyPath: string @@ -76,12 +78,13 @@ export const addComposeTunnelAgentService = ( profileThumbprint?: string privateMode: boolean defaultAccess: 'private' | 'public' + scriptInjections?: (serviceName: string, serviceDef: ComposeService) => Record | undefined }, model: ComposeModel, ): ComposeModel => ({ ...model, services: { - ...model.services, + ...scriptInjections ? addScriptInjectionsToServices(model.services, scriptInjections) : model.services, [COMPOSE_TUNNEL_AGENT_SERVICE_NAME]: merge(baseDockerProxyService(), { restart: 'always', diff --git a/packages/core/src/compose/client.ts b/packages/core/src/compose/client.ts index fe89e71b..8706dbac 100644 --- a/packages/core/src/compose/client.ts +++ b/packages/core/src/compose/client.ts @@ -37,11 +37,13 @@ export const getExposedTcpServicePorts = (model: Pick) .map(({ target }) => target), })) -const composeFileArgs = ( - composeFiles: string[] | Buffer, - projectName?: string, -) => [ +const composeFileArgs = ({ composeFiles, projectName, projectDirectory }: { + composeFiles: string[] | Buffer + projectName?: string + projectDirectory?: string +}) => [ ...(projectName ? ['-p', projectName] : []), + ...(projectDirectory ? ['--project-directory', projectDirectory] : []), ...(Buffer.isBuffer(composeFiles) ? ['-f', '-'] : composeFiles.flatMap(file => ['-f', file])), ] @@ -61,11 +63,11 @@ const composeClient = ( throw e }) - const getModel = async () => yaml.parse(await execComposeCommand(['convert'])) as ComposeModel + const getModel = async (services: string[] = []) => yaml.parse(await execComposeCommand(['convert', ...services])) as ComposeModel return { getModel, - getModelOrError: async () => await getModel().catch(e => { + getModelOrError: async (services: string[] = []) => await getModel(services).catch(e => { if (e instanceof DockerIsNotInstalled || (e instanceof ProcessError && (e.code === DOCKER_COMPOSE_NO_CONFIGURATION_FILE_ERROR_CODE))) { return new LoadComposeFileError(e) @@ -85,10 +87,11 @@ export type ComposeClient = ReturnType type ParametersExceptFirst = F extends (arg0: any, ...rest: infer R) => any ? R : never; export const localComposeClient = ( - { composeFiles, projectName, env }: { + { composeFiles, projectName, env, projectDirectory }: { composeFiles: string[] | Buffer projectName?: string env?: NodeJS.ProcessEnv + projectDirectory?: string }, ) => { const insertStdin = (stdio: StdioOptions | undefined) => { @@ -104,7 +107,7 @@ export const localComposeClient = ( return [null, null, null] } - const fileArgs = composeFileArgs(composeFiles, projectName) + const fileArgs = composeFileArgs({ composeFiles, projectName, projectDirectory }) const spawnComposeArgs = (...[args, opts]: ParametersExceptFirst): Parameters => [ 'docker', diff --git a/packages/core/src/compose/model.ts b/packages/core/src/compose/model.ts index 4a021a3b..b9553e46 100644 --- a/packages/core/src/compose/model.ts +++ b/packages/core/src/compose/model.ts @@ -1,9 +1,3 @@ -import { asyncMap, asyncToArray } from 'iter-tools-es' -import { mapValues } from 'lodash' -import path from 'path' -import { asyncMapValues } from '../async' -import { lstatOrUndefined } from '../files' -import { FileToCopy } from '../upload-files' import { PreevyConfig } from '../config' export type ComposeSecretOrConfig = { @@ -34,6 +28,8 @@ export type ComposeBuild = { tags?: string[] cache_from?: string[] cache_to?: string[] + platforms?: string[] + no_cache?: boolean } type ComposePort = { @@ -69,95 +65,4 @@ export type ComposeModel = { 'x-preevy'?: PreevyConfig } -const volumeSkipList = [ - /^\/var\/log(\/|$)/, - /^\/$/, -] - -const toPosix = (x:string) => x.split(path.sep).join(path.posix.sep) - -export const fixModelForRemote = async ( - { skipServices = [], cwd, remoteBaseDir }: { - skipServices?: string[] - cwd: string - remoteBaseDir: string - }, - model: ComposeModel, -): Promise<{ model: Required>; filesToCopy: FileToCopy[] }> => { - const filesToCopy: FileToCopy[] = [] - - const remotePath = (absolutePath: string) => { - if (!path.isAbsolute(absolutePath)) { - throw new Error(`expected absolute path: "${absolutePath}"`) - } - const relativePath = toPosix(path.relative(cwd, absolutePath)) - - return relativePath.startsWith('..') - ? path.posix.join('absolute', absolutePath) - : path.posix.join('relative', relativePath) - } - - const overrideSecretsOrConfigs = ( - c?: Record, - ) => mapValues(c ?? {}, secretOrConfig => { - const remote = remotePath(secretOrConfig.file) - filesToCopy.push({ local: secretOrConfig.file, remote }) - return { ...secretOrConfig, file: path.posix.join(remoteBaseDir, remote) } - }) - - const overrideSecrets = overrideSecretsOrConfigs(model.secrets) - const overrideConfigs = overrideSecretsOrConfigs(model.configs) - - const services = model.services ?? {} - - const overrideServices = await asyncMapValues(services, async (service, serviceName) => { - if (skipServices.includes(serviceName)) { - return service - } - - return ({ - ...service, - - volumes: service.volumes && await asyncToArray(asyncMap(async volume => { - if (volume.type === 'volume') { - return volume - } - - if (volume.type !== 'bind') { - throw new Error(`Unsupported volume type: ${volume.type} in service ${serviceName}`) - } - if (volumeSkipList.some(re => re.test(volume.source))) { - return volume - } - - const remote = remotePath(volume.source) - const stats = await lstatOrUndefined(volume.source) - - if (stats) { - if (!stats.isDirectory() && !stats.isFile() && !stats.isSymbolicLink()) { - return volume - } - - // ignore non-existing files like docker and compose do, - // they will be created as directories in the container - filesToCopy.push({ local: volume.source, remote }) - } - - return { ...volume, source: path.posix.join(remoteBaseDir, remote) } - }, service.volumes)), - }) - }) - - return { - model: { - ...model, - secrets: overrideSecrets, - configs: overrideConfigs, - services: overrideServices, - networks: model.networks ?? {}, - }, - filesToCopy, - } -} - export const composeModelFilename = 'docker-compose.yaml' diff --git a/packages/core/src/compose/remote.ts b/packages/core/src/compose/remote.ts index 12283555..80186cd3 100644 --- a/packages/core/src/compose/remote.ts +++ b/packages/core/src/compose/remote.ts @@ -1,9 +1,216 @@ import yaml from 'yaml' +import path from 'path' +import { mapValues } from 'lodash' +import { asyncMap, asyncToArray } from 'iter-tools-es' +import { COMPOSE_TUNNEL_AGENT_SERVICE_NAME, MachineStatusCommand, ScriptInjection, formatPublicKey } from '@preevy/common' import { MachineConnection } from '../driver' -import { ComposeModel, composeModelFilename } from './model' -import { REMOTE_DIR_BASE } from '../remote-files' +import { ComposeModel, ComposeSecretOrConfig, composeModelFilename } from './model' +import { REMOTE_DIR_BASE, remoteProjectDir } from '../remote-files' +import { TunnelOpts } from '../ssh' +import { addComposeTunnelAgentService } from '../compose-tunnel-agent-client' +import { Logger } from '../log' +import { FileToCopy } from '../upload-files' +import { envMetadata } from '../env-metadata' +import { EnvId } from '../env-id' +import { asyncMapValues } from '../async' +import { lstatOrUndefined } from '../files' +import { localComposeClient } from './client' -export const remoteUserModel = async (connection: MachineConnection) => { +export const fetchRemoteUserModel = async (connection: MachineConnection) => { const userModelStr = (await connection.exec(`cat ${REMOTE_DIR_BASE}/projects/*/${composeModelFilename}`)).stdout return yaml.parse(userModelStr) as ComposeModel } + +const serviceLinkEnvVars = ( + expectedServiceUrls: { name: string; port: number; url: string }[], +) => Object.fromEntries( + expectedServiceUrls + .map(({ name, port, url }) => [`PREEVY_BASE_URI_${name.replace(/[^a-zA-Z0-9_]/g, '_')}_${port}`.toUpperCase(), url]) +) + +const volumeSkipList = [ + /^\/var\/log(\/|$)/, + /^\/$/, +] + +const toPosix = (x:string) => x.split(path.sep).join(path.posix.sep) + +const fixModelForRemote = async ( + { skipServices = [], cwd, remoteBaseDir }: { + skipServices?: string[] + cwd: string + remoteBaseDir: string + }, + model: ComposeModel, +): Promise<{ model: Required>; filesToCopy: FileToCopy[] }> => { + const filesToCopy: FileToCopy[] = [] + + const remotePath = (absolutePath: string) => { + if (!path.isAbsolute(absolutePath)) { + throw new Error(`expected absolute path: "${absolutePath}"`) + } + const relativePath = toPosix(path.relative(cwd, absolutePath)) + + return relativePath.startsWith('..') + ? path.posix.join('absolute', absolutePath) + : path.posix.join('relative', relativePath) + } + + const overrideSecretsOrConfigs = ( + c?: Record, + ) => mapValues(c ?? {}, secretOrConfig => { + const remote = remotePath(secretOrConfig.file) + filesToCopy.push({ local: secretOrConfig.file, remote }) + return { ...secretOrConfig, file: path.posix.join(remoteBaseDir, remote) } + }) + + const overrideSecrets = overrideSecretsOrConfigs(model.secrets) + const overrideConfigs = overrideSecretsOrConfigs(model.configs) + + const services = model.services ?? {} + + const overrideServices = await asyncMapValues(services, async (service, serviceName) => { + if (skipServices.includes(serviceName)) { + return service + } + + return ({ + ...service, + + volumes: service.volumes && await asyncToArray(asyncMap(async volume => { + if (volume.type === 'volume') { + return volume + } + + if (volume.type !== 'bind') { + throw new Error(`Unsupported volume type: ${volume.type} in service ${serviceName}`) + } + if (volumeSkipList.some(re => re.test(volume.source))) { + return volume + } + + const remote = remotePath(volume.source) + const stats = await lstatOrUndefined(volume.source) + + if (stats) { + if (!stats.isDirectory() && !stats.isFile() && !stats.isSymbolicLink()) { + return volume + } + + // ignore non-existing files like docker and compose do, + // they will be created as directories in the container + filesToCopy.push({ local: volume.source, remote }) + } + + return { ...volume, source: path.posix.join(remoteBaseDir, remote) } + }, service.volumes)), + }) + }) + + return { + model: { + ...model, + secrets: overrideSecrets, + configs: overrideConfigs, + services: overrideServices, + networks: model.networks ?? {}, + }, + filesToCopy, + } +} + +type AgentSettings = { + version: string + envId: EnvId + tunnelOpts: TunnelOpts + sshTunnelPrivateKey: string | Buffer + allowedSshHostKeys: Buffer + userAndGroup: [string, string] + machineStatusCommand?: MachineStatusCommand + scriptInjections?: Record + createCopiedFile: (filename: string, content: string | Buffer) => Promise +} + +export const remoteComposeModel = async ({ + debug, + userSpecifiedProjectName, + userSpecifiedServices, + composeFiles, + log, + cwd, + expectedServiceUrls, + projectName, + agentSettings, +}: { + debug: boolean + userSpecifiedProjectName: string | undefined + userSpecifiedServices: string[] + composeFiles: string[] + log: Logger + cwd: string + expectedServiceUrls: { name: string; port: number; url: string }[] + projectName: string + agentSettings?: AgentSettings +}) => { + const remoteDir = remoteProjectDir(projectName) + + log.debug(`Using compose files: ${composeFiles.join(', ')}`) + + const linkEnvVars = serviceLinkEnvVars(expectedServiceUrls) + + const composeClientWithInjectedArgs = localComposeClient({ + composeFiles, + env: linkEnvVars, + projectName: userSpecifiedProjectName, + projectDirectory: cwd, + }) + + const services = userSpecifiedServices.length + ? [...userSpecifiedServices].concat(COMPOSE_TUNNEL_AGENT_SERVICE_NAME) + : [] + + const { model: fixedModel, filesToCopy } = await fixModelForRemote( + { cwd, remoteBaseDir: remoteDir }, + await composeClientWithInjectedArgs.getModel(services) + ) + + let model: ComposeModel = fixedModel + if (agentSettings) { + const { + envId, + machineStatusCommand, + userAndGroup, + scriptInjections, + tunnelOpts, + version, + sshTunnelPrivateKey, + allowedSshHostKeys, + createCopiedFile, + } = agentSettings + + const [sshPrivateKeyFile, knownServerPublicKey] = await Promise.all([ + createCopiedFile('tunnel_client_private_key', sshTunnelPrivateKey), + createCopiedFile('tunnel_server_public_key', formatPublicKey(allowedSshHostKeys)), + ]) + + model = addComposeTunnelAgentService({ + envId, + debug, + tunnelOpts, + sshPrivateKeyPath: path.posix.join(remoteDir, sshPrivateKeyFile.remote), + knownServerPublicKeyPath: path.posix.join(remoteDir, knownServerPublicKey.remote), + user: userAndGroup.join(':'), + machineStatusCommand, + envMetadata: await envMetadata({ envId, version }), + composeModelPath: path.posix.join(remoteDir, composeModelFilename), + privateMode: false, + defaultAccess: 'public', + composeProject: projectName, + scriptInjections: scriptInjections && (() => scriptInjections), + }, fixedModel) + + filesToCopy.push(sshPrivateKeyFile, knownServerPublicKey) + } + + return { model, filesToCopy, linkEnvVars } +} diff --git a/packages/core/src/compose/script-injection.test.ts b/packages/core/src/compose/script-injection.test.ts index 72bd09d4..b5cc7e73 100644 --- a/packages/core/src/compose/script-injection.test.ts +++ b/packages/core/src/compose/script-injection.test.ts @@ -1,24 +1,21 @@ import { describe, expect, jest, beforeEach, it } from '@jest/globals' import { ScriptInjection } from '@preevy/common' import { ComposeModel } from './model' -import { addScriptInjectionsToModel } from './script-injection' +import { addScriptInjectionsToServices } from './script-injection' describe('addScriptInjectionsToModel', () => { - const model: ComposeModel = Object.freeze({ - name: 'my-app', - services: { - frontend1: {}, - frontend2: { - labels: { - other: 'value', - }, + const model: ComposeModel['services'] = Object.freeze({ + frontend1: {}, + frontend2: { + labels: { + other: 'value', }, - frontend3: {}, }, + frontend3: {}, }) let callback: jest.MockedFunction<(name: string) => Record | undefined> - let newModel: ComposeModel + let newModel: ComposeModel['services'] const injection: ScriptInjection = { src: 'https://mydomain.com/myscript.ts', @@ -28,7 +25,7 @@ describe('addScriptInjectionsToModel', () => { beforeEach(() => { callback = jest.fn(name => (['frontend1', 'frontend2'].includes(name) ? ({ test: injection }) : undefined)) - newModel = addScriptInjectionsToModel(model, callback) + newModel = addScriptInjectionsToServices(model, callback) }) it('injects the script for the first two services', () => { @@ -37,12 +34,12 @@ describe('addScriptInjectionsToModel', () => { 'preevy.inject_script.test.async': 'true', 'preevy.inject_script.test.path_regex': '.*', } - expect(newModel.services?.frontend1?.labels).toMatchObject(expectedLabels) - expect(newModel.services?.frontend2?.labels).toMatchObject({ other: 'value', ...expectedLabels }) + expect(newModel?.frontend1?.labels).toMatchObject(expectedLabels) + expect(newModel?.frontend2?.labels).toMatchObject({ other: 'value', ...expectedLabels }) }) it('does not inject the script for the last service', () => { - expect(newModel.services?.frontend3?.labels).toMatchObject({}) + expect(newModel?.frontend3?.labels).toMatchObject({}) }) it('calls the factory correctly', () => { diff --git a/packages/core/src/compose/script-injection.ts b/packages/core/src/compose/script-injection.ts index 0c92f1ee..9de6177d 100644 --- a/packages/core/src/compose/script-injection.ts +++ b/packages/core/src/compose/script-injection.ts @@ -13,10 +13,7 @@ const addScriptInjectionsToService = ( }, }) -export const addScriptInjectionsToModel = ( - model: ComposeModel, +export const addScriptInjectionsToServices = ( + services: ComposeModel['services'], factory: (serviceName: string, serviceDef: ComposeService) => Record | undefined, -): ComposeModel => ({ - ...model, - services: mapValues(model.services ?? {}, (def, name) => addScriptInjectionsToService(def, factory(name, def) ?? {})), -}) +): ComposeModel['services'] => mapValues(services, (def, name) => addScriptInjectionsToService(def, factory(name, def) ?? {})) diff --git a/packages/core/src/compose/service-links.ts b/packages/core/src/compose/service-links.ts new file mode 100644 index 00000000..fb9f9d60 --- /dev/null +++ b/packages/core/src/compose/service-links.ts @@ -0,0 +1,6 @@ +export const serviceLinkEnvVars = ( + expectedServiceUrls: { name: string; port: number; url: string }[], +) => Object.fromEntries( + expectedServiceUrls + .map(({ name, port, url }) => [`PREEVY_BASE_URI_${name.replace(/[^a-zA-Z0-9_]/g, '_')}_${port}`.toUpperCase(), url]) +) diff --git a/packages/core/src/docker.ts b/packages/core/src/docker.ts index d5acb4b6..1848a3d7 100644 --- a/packages/core/src/docker.ts +++ b/packages/core/src/docker.ts @@ -1,10 +1,11 @@ +import { omitBy } from 'lodash' import { Logger } from './log' import { MachineConnection, ForwardSocket } from './driver' import { withSpinner } from './spinner' -export type FuncWrapper = ( - f: () => Promise, -) => Promise +// export type FuncWrapper = ( +// f: (...args: Args) => Promise, +// ) => Promise const dockerHost = (s: string | ForwardSocket['address']) => ( typeof s === 'string' @@ -12,15 +13,14 @@ const dockerHost = (s: string | ForwardSocket['address']) => ( : `tcp://${s.host}:${s.port}` ) -export const wrapWithDockerSocket = ( - { connection, log }: { - connection: MachineConnection +export const dockerEnvContext = async ( + { connection, log, env = process.env }: { + connection: Pick log: Logger + env?: Record }, -): FuncWrapper => async ( - f: () => Promise, -): Promise => { - const { address, close } = await withSpinner( +): Promise }> => { + const { address, [Symbol.asyncDispose]: dispose } = await withSpinner( () => connection.dockerSocket(), { text: 'Connecting to remote docker socket...', successText: 'Connected to remote docker socket' }, ) @@ -32,7 +32,34 @@ export const wrapWithDockerSocket = ( delete process.env[k] }) - process.env.DOCKER_HOST = dockerHost(address) - - return await f().finally(close) + return { + env: { + ...omitBy(env, (_, k) => k.startsWith('DOCKER_')), + DOCKER_HOST: dockerHost(address), + }, + [Symbol.asyncDispose]: dispose, + } } + +// export const wrapWithDockerSocket = ( +// { connection, log }: { +// connection: Pick +// log: Logger +// }, +// ) => async ( +// f: (env: Record) => Promise, +// ): Promise => { +// const { address, close } = await withSpinner( +// () => connection.dockerSocket(), +// { text: 'Connecting to remote docker socket...', successText: 'Connected to remote docker socket' }, +// ) + +// log.debug(`Local socket: ${JSON.stringify(address)}`) + +// Object.keys(process.env).filter(k => k !== 'DOCKER_HOST' && k.startsWith('DOCKER_')).forEach(k => { +// log.warn(`deleting conflicting env var ${k}`) +// delete process.env[k] +// }) + +// return await f({ DOCKER_HOST: dockerHost(address) }).finally(close) +// } diff --git a/packages/core/src/driver/driver.ts b/packages/core/src/driver/driver.ts index fd07fe51..7f469a1a 100644 --- a/packages/core/src/driver/driver.ts +++ b/packages/core/src/driver/driver.ts @@ -1,31 +1,20 @@ -import { AddressInfo } from 'net' import { MachineStatusCommand } from '@preevy/common' import { PartialStdioOptions } from '../child-process' import { CommandExecuter } from '../command-executer' import { Profile } from '../profile' -import { MachineBase, PartialMachine, Resource, SpecDiffItem } from './machine' +import { MachineBase, PartialMachine, Resource, SpecDiffItem } from './machine-model' import { Store } from '../store' import { Logger } from '../log' -import { EnvMachineMetadata } from '../env-metadata' -export type ForwardOutStreamLocal = { - localSocket: string | AddressInfo - close: () => Promise -} - -export type ForwardSocket = { +export type ForwardSocket = AsyncDisposable & { address: { host: string; port: number } - close: () => Promise } -export type MachineConnection = { +export type MachineConnection = Disposable & { exec: CommandExecuter dockerSocket: () => Promise - close: () => Promise } -export type MachineMetadata = Omit - export type MachineDriver< Machine extends MachineBase = MachineBase, ResourceType extends string = string diff --git a/packages/core/src/driver/index.ts b/packages/core/src/driver/index.ts index 5d7b8817..6a9373c0 100644 --- a/packages/core/src/driver/index.ts +++ b/packages/core/src/driver/index.ts @@ -6,7 +6,8 @@ export { Resource, MachineResource, machineResourceType, -} from './machine' +} from './machine-model' +export * from './machine-operations' export { SshMachine, sshDriver, getStoredKey, getStoredKeyOrUndefined } from './ssh' export { machineStatusNodeExporterCommand } from './machine-status-node-exporter' export { diff --git a/packages/core/src/driver/machine.ts b/packages/core/src/driver/machine-model.ts similarity index 100% rename from packages/core/src/driver/machine.ts rename to packages/core/src/driver/machine-model.ts diff --git a/packages/core/src/commands/up/machine.ts b/packages/core/src/driver/machine-operations.ts similarity index 89% rename from packages/core/src/commands/up/machine.ts rename to packages/core/src/driver/machine-operations.ts index e7d9d6b9..26e4f75c 100644 --- a/packages/core/src/commands/up/machine.ts +++ b/packages/core/src/driver/machine-operations.ts @@ -1,20 +1,21 @@ import { EOL } from 'os' import retry from 'p-retry' import { dateReplacer } from '@preevy/common' -import { withSpinner } from '../../spinner' -import { MachineCreationDriver, SpecDiffItem, MachineDriver, MachineConnection, MachineBase, isPartialMachine, machineResourceType } from '../../driver' -import { telemetryEmitter } from '../../telemetry' -import { Logger } from '../../log' -import { scriptExecuter } from '../../remote-script-executer' -import { EnvMetadata, driverMetadataFilename } from '../../env-metadata' -import { REMOTE_DIR_BASE } from '../../remote-files' +import { withSpinner } from '../spinner' +import { telemetryEmitter } from '../telemetry' +import { Logger } from '../log' +import { scriptExecuter } from '../remote-script-executer' +import { EnvMetadata, driverMetadataFilename } from '../env-metadata' +import { REMOTE_DIR_BASE } from '../remote-files' +import { MachineBase, SpecDiffItem, isPartialMachine, machineResourceType } from './machine-model' +import { MachineConnection, MachineCreationDriver, MachineDriver } from './driver' const machineDiffText = (diff: SpecDiffItem[]) => diff .map(({ name, old, new: n }) => `* ${name}: ${old} -> ${n}`).join(EOL) type Origin = 'existing' | 'new-from-snapshot' | 'new-from-scratch' -const ensureMachine = async ({ +const ensureBareMachine = async ({ machineDriver, machineCreationDriver, envId, @@ -98,13 +99,13 @@ const writeMetadata = async ( }) } -const getUserAndGroup = async (connection: Pick) => ( +export const getUserAndGroup = async (connection: Pick) => ( await connection.exec('echo "$(id -u):$(stat -c %g /var/run/docker.sock)"') ).stdout .trim() .split(':') as [string, string] -const getDockerPlatform = async (connection: Pick) => { +export const getDockerPlatform = async (connection: Pick) => { const arch = (await connection.exec('docker info -f "{{.Architecture}}"')).stdout.trim() return arch === 'aarch64' ? 'linux/arm64' : 'linux/amd64' } @@ -147,7 +148,7 @@ const customizeNewMachine = ({ retries: 5, onFailedAttempt: async err => { log.debug(`Failed to execute docker run hello-world: ${err}`) - await connection.close() + connection[Symbol.dispose]() connection = await machineDriver.connect(machine, { log, debug }) }, } @@ -169,7 +170,7 @@ const customizeNewMachine = ({ return { connection, userAndGroup, machine, dockerPlatform } } -export const ensureCustomizedMachine = async ({ +export const ensureMachine = async ({ machineDriver, machineCreationDriver, machineDriverName, @@ -189,7 +190,7 @@ export const ensureCustomizedMachine = async ({ userAndGroup: [string, string] dockerPlatform: string }> => { - const { machine, connection: connectionPromise, origin } = await ensureMachine( + const { machine, connection: connectionPromise, origin } = await ensureBareMachine( { machineDriver, machineCreationDriver, envId, log, debug }, ) @@ -221,7 +222,7 @@ export const ensureCustomizedMachine = async ({ return { machine, connection, userAndGroup, dockerPlatform } } catch (e) { - await connection.close() + connection[Symbol.dispose]() throw e } }, { opPrefix: 'Configuring machine', successText: 'Machine configured' }) diff --git a/packages/core/src/driver/ssh.ts b/packages/core/src/driver/ssh.ts index c33d9941..6d258386 100644 --- a/packages/core/src/driver/ssh.ts +++ b/packages/core/src/driver/ssh.ts @@ -9,7 +9,7 @@ import retry, { Options as RetryOptions } from 'p-retry' import { Store } from '../store' import { SshKeyPair, connectSshClient } from '../ssh' import { MachineConnection, MachineDriver } from './driver' -import { MachineBase } from './machine' +import { MachineBase } from './machine-model' import { sshKeysStore } from '../state' import { Logger } from '../log' @@ -57,13 +57,13 @@ export const sshDriver = ( ) return { - close: async () => connection.close(), + [Symbol.dispose]: () => connection[Symbol.dispose](), exec: connection.exec, dockerSocket: async () => { const host = '0.0.0.0' const forward = await connection.forwardOutStreamLocal({ port: 0, host }, '/var/run/docker.sock') return { - close: forward.close, + [Symbol.asyncDispose]: forward[Symbol.asyncDispose], address: { host, port: (forward.localSocket as AddressInfo).port }, } }, diff --git a/packages/core/src/env-id.ts b/packages/core/src/env-id.ts index d9aba845..4f87561d 100644 --- a/packages/core/src/env-id.ts +++ b/packages/core/src/env-id.ts @@ -1,6 +1,6 @@ import { detectCiProvider } from './ci-providers' import { gitContext } from './git' -import { ComposeModel } from './compose' +import { ComposeModel } from './compose/model' import { Logger } from './log' export type EnvId = string & { diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index e320b2b4..7233bb2a 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -18,11 +18,12 @@ export { getStoredKeyOrUndefined as getStoredSshKeyOrUndefined, ForwardSocket, machineStatusNodeExporterCommand, + ensureMachine, } from './driver' export { profileStore, Profile, ProfileStore, link, Org } from './profile' export { telemetryEmitter, registerEmitter, wireProcessExit, createTelemetryEmitter, machineId } from './telemetry' export { fsTypeFromUrl, Store, VirtualFS, localFsFromUrl, localFs } from './store' -export { localComposeClient, ComposeModel, resolveComposeFiles, getExposedTcpServicePorts, remoteUserModel, NoComposeFilesError, addScriptInjectionsToModel } from './compose' +export { localComposeClient, ComposeModel, resolveComposeFiles, getExposedTcpServicePorts, fetchRemoteUserModel as remoteUserModel, NoComposeFilesError, addScriptInjectionsToServices as addScriptInjectionsToModel } from './compose' export { withSpinner } from './spinner' export { findEnvId, findProjectName, findEnvIdByProjectName, validateEnvId, normalize as normalizeEnvId, EnvId } from './env-id' export { sshKeysStore } from './state' @@ -45,7 +46,8 @@ export { findComposeTunnelAgentUrl, } from './compose-tunnel-agent-client' export * as commands from './commands' -export { wrapWithDockerSocket } from './docker' +export { BuildSpec, ImageRegistry, parseRegistry } from './build' +export { dockerEnvContext } from './docker' export { FlatTunnel, flattenTunnels, @@ -57,7 +59,6 @@ export { } from './tunneling' export { TunnelOpts } from './ssh' export { Spinner } from './spinner' -export { withClosable } from './closable' export { generateBasicAuthCredentials as getUserCredentials, jwtGenerator, jwkThumbprint, jwkThumbprintUri, parseKey } from './credentials' export { ciProviders, detectCiProvider } from './ci-providers' export { paginationIterator } from './pagination' diff --git a/packages/core/src/nulls.ts b/packages/core/src/nulls.ts index 3b666c98..cb395809 100644 --- a/packages/core/src/nulls.ts +++ b/packages/core/src/nulls.ts @@ -44,3 +44,9 @@ export function extractDefined( ? defined.then(obj => obj[prop] as unknown as NonNullable) : (o as T)[prop] as NonNullable } + +export const hasProp = < + K extends string | symbol | number +>(prop: K) => < + T extends { [k in K]?: unknown } +>(obj: T): obj is T & { [k in K]-?: NonNullable } => Boolean(obj[prop]) diff --git a/packages/core/src/remote-files.ts b/packages/core/src/remote-files.ts index 2de1ad3c..4dca68da 100644 --- a/packages/core/src/remote-files.ts +++ b/packages/core/src/remote-files.ts @@ -1,5 +1,21 @@ import path from 'path' +import fs from 'fs' export const REMOTE_DIR_BASE = '/var/lib/preevy' export const remoteProjectDir = (projectName: string) => path.posix.join(REMOTE_DIR_BASE, 'projects', projectName) + +export const createCopiedFileInDataDir = ( + { projectLocalDataDir } : { + projectLocalDataDir: string + } +) => async ( + filename: string, + content: string | Buffer +): Promise<{ local: string; remote: string }> => { + const local = path.join(projectLocalDataDir, filename) + const result = { local, remote: filename } + await fs.promises.mkdir(path.dirname(local), { recursive: true }) + await fs.promises.writeFile(local, content, { flag: 'w' }) + return result +} diff --git a/packages/core/src/ssh/client/forward-out.ts b/packages/core/src/ssh/client/forward-out.ts index 89be2541..da0b003f 100644 --- a/packages/core/src/ssh/client/forward-out.ts +++ b/packages/core/src/ssh/client/forward-out.ts @@ -3,9 +3,8 @@ import net, { AddressInfo, ListenOptions } from 'net' import ssh2 from 'ssh2' import { Logger } from '../../log' -export type ForwardOutStreamLocal = { +export type ForwardOutStreamLocal = AsyncDisposable & { localSocket: string | AddressInfo - close: () => Promise } export const forwardOutStreamLocal = ({ ssh, log, listenAddress, remoteSocket, onClose }: { @@ -50,7 +49,7 @@ export const forwardOutStreamLocal = ({ ssh, log, listenAddress, remoteSocket, o reject(new Error(message)) return } - resolve({ localSocket: address, close: async () => { socketServer.close() } }) + resolve({ localSocket: address, [Symbol.asyncDispose]: async () => { socketServer.close() } }) }) .on('error', (err: unknown) => { log.error('socketServer error', err) diff --git a/packages/core/src/ssh/client/index.ts b/packages/core/src/ssh/client/index.ts index f13efb57..73d0ddd8 100644 --- a/packages/core/src/ssh/client/index.ts +++ b/packages/core/src/ssh/client/index.ts @@ -35,7 +35,7 @@ export const connectSshClient = async ( listenAddress: string | number | ListenOptions, remoteSocket: string, ) => forwardOutStreamLocal({ ssh, log, listenAddress, remoteSocket }), - close: () => { ssh.end() }, + [Symbol.dispose]: () => { ssh.end() }, } return self diff --git a/packages/core/src/ssh/client/sftp.ts b/packages/core/src/ssh/client/sftp.ts index 38f06ec5..fdc6bac9 100644 --- a/packages/core/src/ssh/client/sftp.ts +++ b/packages/core/src/ssh/client/sftp.ts @@ -111,7 +111,7 @@ export const sftpClient = ( files.map(f => self.putFile(f, options)), ).then(() => undefined), - close: () => sftp.end(), + [Symbol.dispose]: () => sftp.end(), } return self diff --git a/packages/core/src/tunneling/model.ts b/packages/core/src/tunneling/model.ts index 0ffe2001..1575f772 100644 --- a/packages/core/src/tunneling/model.ts +++ b/packages/core/src/tunneling/model.ts @@ -1,6 +1,7 @@ import { TunnelNameResolver } from '@preevy/common' import { generateSshKeyPair } from '../ssh/keypair' -import { ComposeModel, getExposedTcpServicePorts } from '../compose' +import { ComposeModel } from '../compose/model' +import { getExposedTcpServicePorts } from '../compose/client' type port = string type url = string diff --git a/packages/driver-kube-pod/src/driver/client/port-forward.ts b/packages/driver-kube-pod/src/driver/client/port-forward.ts index 020e5417..9e37cb72 100644 --- a/packages/driver-kube-pod/src/driver/client/port-forward.ts +++ b/packages/driver-kube-pod/src/driver/client/port-forward.ts @@ -3,9 +3,8 @@ import * as k8s from '@kubernetes/client-node' import { promisify } from 'util' import { Logger } from '@preevy/core' -type ForwardSocket = { +type ForwardSocket = AsyncDisposable & { localSocket: string | AddressInfo - close: () => Promise } type Closable = { close: () => void } @@ -46,7 +45,7 @@ const portForward = ( server.listen(listenAddress, () => { resolve({ localSocket: server.address() as string | AddressInfo, - close: () => { + [Symbol.asyncDispose]: () => { sockets.forEach(ws => ws.close()) return closeServer() }, diff --git a/packages/driver-kube-pod/src/driver/driver.ts b/packages/driver-kube-pod/src/driver/driver.ts index a43383e1..0cbeabc3 100644 --- a/packages/driver-kube-pod/src/driver/driver.ts +++ b/packages/driver-kube-pod/src/driver/driver.ts @@ -37,7 +37,7 @@ export const machineConnection = async ( log.debug(`Found pod "${pod.metadata?.name}"`) return ({ - close: async () => undefined, + [Symbol.dispose]: () => undefined, exec: async (command, opts) => { const { code, output } = await client.exec({ @@ -55,10 +55,15 @@ export const machineConnection = async ( dockerSocket: async () => { const host = '0.0.0.0' - const { localSocket, close } = await client.portForward(deployment, 2375, { host, port: 0 }) + + const { + localSocket, + [Symbol.asyncDispose]: dispose, + } = await client.portForward(deployment, 2375, { host, port: 0 }) + return { address: { host, port: (localSocket as AddressInfo).port }, - close, + [Symbol.asyncDispose]: dispose, } }, })