From 37871e5dc4ca1b342e5a7dbf31cc3ef0fe13297e Mon Sep 17 00:00:00 2001 From: Roy Razon Date: Sun, 12 May 2024 02:51:28 +0300 Subject: [PATCH] wait for tunnels to be ready - CTA: add state prop to `/tunnels` API endpoint. checks that all compose services have been started by docker, and that all potential tunnels for these services have been started by the CTA. - CLI: check new prop in `up` and `urls` commands. can be disabled with the `--no-wait` flag Compatible with compose services which have no ports, e.g, one-off services such as DB init --- .../cli-common/src/lib/common-flags/index.ts | 5 ++ packages/cli/package.json | 2 +- packages/cli/src/commands/up.ts | 10 ++-- packages/cli/src/commands/urls.ts | 13 ++++- packages/common/index.ts | 1 + .../common/src/compose-tunnel-agent/index.ts | 10 ++++ packages/compose-tunnel-agent/index.ts | 30 +++++++++-- packages/compose-tunnel-agent/package.json | 3 +- .../src/api-server/env.ts | 11 ++-- .../src/api-server/index.test.ts | 3 +- .../src/api-server/index.ts | 7 +-- .../compose-tunnel-agent/src/tunnels-state.ts | 51 +++++++++++++++++++ packages/core/src/commands/urls.ts | 3 ++ .../core/src/compose-tunnel-agent-client.ts | 21 ++++++-- 14 files changed, 143 insertions(+), 27 deletions(-) create mode 100644 packages/compose-tunnel-agent/src/tunnels-state.ts diff --git a/packages/cli-common/src/lib/common-flags/index.ts b/packages/cli-common/src/lib/common-flags/index.ts index 967a1f27..4699947c 100644 --- a/packages/cli-common/src/lib/common-flags/index.ts +++ b/packages/cli-common/src/lib/common-flags/index.ts @@ -94,4 +94,9 @@ export const urlFlags = { summary: 'Timeout for fetching URLs request in milliseconds', default: 2500, }), + wait: Flags.boolean({ + description: 'Wait for all tunnels to be ready', + default: true, + allowNo: true, + }), } as const diff --git a/packages/cli/package.json b/packages/cli/package.json index 1a444316..7f257d42 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -123,4 +123,4 @@ "preview" ], "types": "dist/index.d.ts" -} +} \ No newline at end of file diff --git a/packages/cli/src/commands/up.ts b/packages/cli/src/commands/up.ts index 90337c4e..108b91ec 100644 --- a/packages/cli/src/commands/up.ts +++ b/packages/cli/src/commands/up.ts @@ -16,7 +16,7 @@ import { inspect } from 'util' import { editUrl, tunnelNameResolver } from '@preevy/common' import MachineCreationDriverCommand from '../machine-creation-driver-command.js' import { envIdFlags, urlFlags } from '../common-flags.js' -import { filterUrls, printUrls, writeUrlsToFile } from './urls.js' +import { filterUrls, printUrls, urlsRetryOpts, writeUrlsToFile } from './urls.js' import { connectToTunnelServerSsh } from '../tunnel-server-client.js' const fetchTunnelServerDetails = async ({ @@ -207,13 +207,9 @@ export default class Up extends MachineCreationDriverCommand { tunnelingKey, includeAccessCredentials: flags['include-access-credentials'] && (flags['access-credentials-type'] as 'api' | 'browser'), showPreevyService: flags['show-preevy-service-urls'], - retryOpts: { - minTimeout: 1000, - maxTimeout: 2000, - retries: 10, - onFailedAttempt: e => { this.logger.debug(`Failed to query tunnels: ${inspect(e)}`) }, - }, + retryOpts: urlsRetryOpts(this.logger), fetchTimeout: flags['fetch-urls-timeout'], + waitForAllTunnels: flags.wait, }), { text: 'Getting tunnel URLs...' }) const urls = await filterUrls({ diff --git a/packages/cli/src/commands/urls.ts b/packages/cli/src/commands/urls.ts index 973a5464..dc965a06 100644 --- a/packages/cli/src/commands/urls.ts +++ b/packages/cli/src/commands/urls.ts @@ -5,6 +5,7 @@ import { FlatTunnel, Logger, TunnelOpts, addBaseComposeTunnelAgentService, comma import { HooksListeners, PluginContext, parseTunnelServerFlags, tableFlags, text, tunnelServerFlags } from '@preevy/cli-common' import { asyncReduce } from 'iter-tools-es' import { tunnelNameResolver } from '@preevy/common' +import { inspect } from 'util' import { connectToTunnelServerSsh } from '../tunnel-server-client.js' import ProfileCommand from '../profile-command.js' import { envIdFlags, urlFlags } from '../common-flags.js' @@ -49,6 +50,15 @@ export const filterUrls = ({ flatTunnels, context, filters }: { filters, ) +export const urlsRetryOpts = (log: Logger) => ({ + minTimeout: 1000, + maxTimeout: 2000, + retries: 10, + onFailedAttempt: (e: unknown) => { log.debug(`Failed to query tunnels: ${inspect(e)}`) }, +} as const) + +export const noWaitUrlsRetryOpts = { retries: 2 } as const + // eslint-disable-next-line no-use-before-define export default class Urls extends ProfileCommand { static description = 'Show urls for an existing environment' @@ -128,8 +138,9 @@ export default class Urls extends ProfileCommand { tunnelingKey, includeAccessCredentials: flags['include-access-credentials'] && (flags['access-credentials-type'] as 'api' | 'browser'), showPreevyService: flags['show-preevy-service-urls'], - retryOpts: { retries: 2 }, + retryOpts: flags.wait ? urlsRetryOpts(this.logger) : noWaitUrlsRetryOpts, fetchTimeout: flags['fetch-urls-timeout'], + waitForAllTunnels: flags.wait, }) const urls = await filterUrls({ diff --git a/packages/common/index.ts b/packages/common/index.ts index 1be4f7fd..50ff7fb5 100644 --- a/packages/common/index.ts +++ b/packages/common/index.ts @@ -30,6 +30,7 @@ export { COMPOSE_TUNNEL_AGENT_PORT, COMPOSE_TUNNEL_AGENT_SERVICE_LABELS, COMPOSE_TUNNEL_AGENT_SERVICE_NAME, + ComposeTunnelAgentState, } from './src/compose-tunnel-agent/index.js' export { MachineStatusCommand, DockerMachineStatusCommandRecipe } from './src/machine-status-command.js' export { ProcessOutputBuffers, orderedOutput, OrderedOutput } from './src/process-output-buffers.js' diff --git a/packages/common/src/compose-tunnel-agent/index.ts b/packages/common/src/compose-tunnel-agent/index.ts index 96f039b8..a07edf5d 100644 --- a/packages/common/src/compose-tunnel-agent/index.ts +++ b/packages/common/src/compose-tunnel-agent/index.ts @@ -3,3 +3,13 @@ export { ScriptInjection, parseScriptInjectionLabels, scriptInjectionsToLabels } export const COMPOSE_TUNNEL_AGENT_SERVICE_NAME = 'preevy_proxy' export const COMPOSE_TUNNEL_AGENT_PORT = 3000 + +export type ComposeTunnelAgentState = { + state: 'unknown' + reason: string +} | { + state: 'pending' + pendingServices: string[] +} | { + state: 'stable' +} diff --git a/packages/compose-tunnel-agent/index.ts b/packages/compose-tunnel-agent/index.ts index 7e737c29..3c0af4f0 100644 --- a/packages/compose-tunnel-agent/index.ts +++ b/packages/compose-tunnel-agent/index.ts @@ -4,6 +4,7 @@ import Docker from 'dockerode' import { rimraf } from 'rimraf' import { pino } from 'pino' import pinoPrettyModule from 'pino-pretty' +import yaml from 'yaml' import { requiredEnv, formatPublicKey, @@ -20,11 +21,13 @@ import { runMachineStatusCommand } from './src/machine-status.js' import { envMetadata } from './src/metadata.js' import { readAllFiles } from './src/files.js' import { eventsClient as dockerEventsClient, filteredClient as dockerFilteredClient } from './src/docker/index.js' +import { tunnelsStateCalculator } from './src/tunnels-state.js' const PinoPretty = pinoPrettyModule.default const homeDir = process.env.HOME || '/root' const dockerSocket = '/var/run/docker.sock' +const COMPOSE_FILE_PATH = '/preevy/docker-compose.yaml' const targetComposeProject = process.env.COMPOSE_PROJECT const defaultAccess = process.env.DEFAULT_ACCESS_LEVEL === 'private' ? 'private' : 'public' @@ -106,22 +109,39 @@ const main = async () => { }) sshLog.info('ssh client connected to %j', sshUrl) - let currentTunnels = dockerClient.getRunningServices().then(services => sshClient.updateTunnels(services)) + let currentState = dockerClient.getRunningServices().then(async runningServices => ({ + runningServices, + sshTunnels: await sshClient.updateTunnels(runningServices), + })) void dockerClient.startListening({ - onChange: async services => { - currentTunnels = sshClient.updateTunnels(services) + onChange: runningServices => { + currentState = (async () => ({ + runningServices, + sshTunnels: await sshClient.updateTunnels(runningServices), + }))() }, }) + const calcTunnelsState = tunnelsStateCalculator({ + composeProject: targetComposeProject, + composeModelReader: async () => yaml.parse(await fs.promises.readFile(COMPOSE_FILE_PATH, { encoding: 'utf8' })), + }) + const app = await createApp({ log: log.child({ name: 'api' }), - currentSshState: async () => (await currentTunnels), + tunnels: async () => { + const { sshTunnels, runningServices } = await currentState + return { + ...sshTunnels, + state: await calcTunnelsState(runningServices), + } + }, machineStatus: machineStatusCommand ? async () => await runMachineStatusCommand({ log, docker })(machineStatusCommand) : undefined, envMetadata: await envMetadata({ env: process.env, log }), - composeModelPath: '/preevy/docker-compose.yaml', + composeModelPath: COMPOSE_FILE_PATH, docker, dockerFilter: dockerFilteredClient({ docker, composeProject: targetComposeProject }), }) diff --git a/packages/compose-tunnel-agent/package.json b/packages/compose-tunnel-agent/package.json index b2b69ab6..236f6589 100644 --- a/packages/compose-tunnel-agent/package.json +++ b/packages/compose-tunnel-agent/package.json @@ -31,6 +31,7 @@ "rimraf": "^5.0.5", "ssh2": "^1.12.0", "ws": "^8.13.0", + "yaml": "^2.3.2", "zod": "^3.21.4" }, "devDependencies": { @@ -65,4 +66,4 @@ "prepare": "cd ../.. && husky install", "test": "node --no-warnings=ExperimentalWarning --experimental-vm-modules ../../node_modules/.bin/jest" } -} +} \ No newline at end of file diff --git a/packages/compose-tunnel-agent/src/api-server/env.ts b/packages/compose-tunnel-agent/src/api-server/env.ts index b283004f..24825a1a 100644 --- a/packages/compose-tunnel-agent/src/api-server/env.ts +++ b/packages/compose-tunnel-agent/src/api-server/env.ts @@ -1,14 +1,19 @@ import fs from 'node:fs' import { FastifyPluginAsync } from 'fastify' +import { ComposeTunnelAgentState } from '@preevy/common' import { SshState } from '../ssh/index.js' export const env: FastifyPluginAsync<{ - currentSshState: () => Promise + tunnels: () => Promise machineStatus?: () => Promise<{ data: Buffer; contentType: string }> envMetadata?: Record composeModelPath: string -}> = async (app, { currentSshState, machineStatus, envMetadata, composeModelPath }) => { - app.get('/tunnels', async () => await currentSshState()) +}> = async (app, { tunnels, machineStatus, envMetadata, composeModelPath }) => { + app.get('/tunnels', async ({ log }) => { + const response = await tunnels() + log.debug('tunnels response: %j', response) + return response + }) if (machineStatus) { app.get('/machine-status', async (_req, res) => { diff --git a/packages/compose-tunnel-agent/src/api-server/index.test.ts b/packages/compose-tunnel-agent/src/api-server/index.test.ts index 0aedfba4..3eaff73f 100644 --- a/packages/compose-tunnel-agent/src/api-server/index.test.ts +++ b/packages/compose-tunnel-agent/src/api-server/index.test.ts @@ -8,6 +8,7 @@ import { inspect, promisify } from 'node:util' import waitForExpect from 'wait-for-expect' import WebSocket from 'ws' import stripAnsi from 'strip-ansi' +import { ComposeTunnelAgentState } from '@preevy/common' import { createApp } from './index.js' import { filteredClient } from '../docker/index.js' import { SshState } from '../ssh/index.js' @@ -71,7 +72,7 @@ const setupApiServer = () => { docker, dockerFilter: filteredClient({ docker, composeProject: TEST_COMPOSE_PROJECT }), composeModelPath: '', - currentSshState: () => Promise.resolve({} as unknown as SshState), + tunnels: () => Promise.resolve({} as unknown as SshState & { state: ComposeTunnelAgentState }), }) await app.listen({ port: 0 }) const { port } = app.server.address() as AddressInfo diff --git a/packages/compose-tunnel-agent/src/api-server/index.ts b/packages/compose-tunnel-agent/src/api-server/index.ts index 1d674928..5894b606 100644 --- a/packages/compose-tunnel-agent/src/api-server/index.ts +++ b/packages/compose-tunnel-agent/src/api-server/index.ts @@ -3,6 +3,7 @@ import Dockerode from 'dockerode' import fastify from 'fastify' import cors from '@fastify/cors' import { validatorCompiler, serializerCompiler, ZodTypeProvider } from 'fastify-type-provider-zod' +import { ComposeTunnelAgentState } from '@preevy/common' import { SshState } from '../ssh/index.js' import { DockerFilterClient } from '../docker/index.js' import { containers } from './containers/index.js' @@ -10,7 +11,7 @@ import { env } from './env.js' export const createApp = async ({ log, - currentSshState, + tunnels, machineStatus, envMetadata, composeModelPath, @@ -18,7 +19,7 @@ export const createApp = async ({ docker, }: { log: Logger - currentSshState: () => Promise + tunnels: () => Promise machineStatus?: () => Promise<{ data: Buffer; contentType: string }> envMetadata?: Record composeModelPath: string @@ -39,7 +40,7 @@ export const createApp = async ({ app.get('/healthz', { logLevel: 'warn' }, async () => 'OK') - await app.register(env, { composeModelPath, currentSshState, envMetadata, machineStatus }) + await app.register(env, { composeModelPath, tunnels, envMetadata, machineStatus }) await app.register(containers, { docker, dockerFilter, prefix: '/containers' }) return app diff --git a/packages/compose-tunnel-agent/src/tunnels-state.ts b/packages/compose-tunnel-agent/src/tunnels-state.ts new file mode 100644 index 00000000..ede16b62 --- /dev/null +++ b/packages/compose-tunnel-agent/src/tunnels-state.ts @@ -0,0 +1,51 @@ +import { inspect } from 'util' +import { ComposeTunnelAgentState } from '@preevy/common' +import { RunningService } from './docker/index.js' + +const findPendingComposeServiceTunnels = ({ + composeProject, + composeModel, + runningServices, +}: { + composeProject: string + composeModel: { services: Record } + runningServices: Pick[] +}) => { + const composeServiceNames = Object.keys(composeModel.services) + + const runningServiceNames = new Set( + runningServices + .filter(({ project }) => project === composeProject) + .map(({ name }) => name) + ) + + return composeServiceNames.filter(service => !runningServiceNames.has(service)) +} + +export const tunnelsStateCalculator = ({ + composeProject, + composeModelReader, +}: { + composeProject?: string + composeModelReader: () => Promise<{ services: Record }> +}) => async ( + runningServices: Pick[] +): Promise => { + if (!composeProject) { + return { state: 'unknown', reason: 'COMPOSE_PROJECT not set' } + } + + let composeModel: { services: Record } + try { + composeModel = await composeModelReader() + } catch (e) { + return { state: 'unknown', reason: `Could not read compose file: ${inspect(e)}` } + } + + const pendingServices = findPendingComposeServiceTunnels({ composeProject, composeModel, runningServices }) + if (pendingServices.length) { + return { state: 'pending', pendingServices } + } + + return { state: 'stable' } +} diff --git a/packages/core/src/commands/urls.ts b/packages/core/src/commands/urls.ts index a7dff859..3b6060d1 100644 --- a/packages/core/src/commands/urls.ts +++ b/packages/core/src/commands/urls.ts @@ -26,6 +26,7 @@ export const urls = async ({ showPreevyService, composeTunnelServiceUrl, fetchTimeout, + waitForAllTunnels, }: { serviceAndPort?: { service: string; port?: number } tunnelingKey: string | Buffer @@ -34,6 +35,7 @@ export const urls = async ({ showPreevyService: boolean composeTunnelServiceUrl: string fetchTimeout: number + waitForAllTunnels?: boolean }) => { const credentials = await generateBasicAuthCredentials(jwtGenerator(tunnelingKey)) @@ -43,6 +45,7 @@ export const urls = async ({ credentials, includeAccessCredentials, fetchTimeout, + waitForAllTunnels, }) return flattenTunnels(tunnels).filter(tunnelFilter({ serviceAndPort, showPreevyService })) diff --git a/packages/core/src/compose-tunnel-agent-client.ts b/packages/core/src/compose-tunnel-agent-client.ts index 31f55ea0..3a8a0168 100644 --- a/packages/core/src/compose-tunnel-agent-client.ts +++ b/packages/core/src/compose-tunnel-agent-client.ts @@ -1,9 +1,9 @@ import path from 'path' import retry from 'p-retry' -import util from 'util' +import { inspect } from 'util' import { createRequire } from 'module' import { mapValues, merge } from 'lodash-es' -import { COMPOSE_TUNNEL_AGENT_PORT, COMPOSE_TUNNEL_AGENT_SERVICE_LABELS, COMPOSE_TUNNEL_AGENT_SERVICE_NAME, MachineStatusCommand, ScriptInjection, dateReplacer } from '@preevy/common' +import { COMPOSE_TUNNEL_AGENT_PORT, COMPOSE_TUNNEL_AGENT_SERVICE_LABELS, COMPOSE_TUNNEL_AGENT_SERVICE_NAME, ComposeTunnelAgentState, MachineStatusCommand, ScriptInjection, dateReplacer } from '@preevy/common' import { ComposeModel, ComposeService, composeModelFilename } from './compose/model.js' import { TunnelOpts } from './ssh/url.js' import { Tunnel } from './tunneling/index.js' @@ -150,7 +150,7 @@ export const findComposeTunnelAgentUrl = ( )?.url if (!serviceUrl) { - throw new Error(`Cannot find compose tunnel agent API service URL ${COMPOSE_TUNNEL_AGENT_SERVICE_NAME}:${COMPOSE_TUNNEL_AGENT_PORT} in: ${util.inspect(serviceUrls)}`) + throw new Error(`Cannot find compose tunnel agent API service URL ${COMPOSE_TUNNEL_AGENT_SERVICE_NAME}:${COMPOSE_TUNNEL_AGENT_PORT} in: ${inspect(serviceUrls)}`) } return serviceUrl @@ -182,16 +182,27 @@ const fetchFromComposeTunnelAgent = async ({ return r }, retryOpts) +type TunnelsResponse = { + tunnels: Tunnel[] + state: ComposeTunnelAgentState +} + export const queryTunnels = async ({ includeAccessCredentials, + waitForAllTunnels, ...fetchOpts }: ComposeTunnelAgentFetchOpts & { includeAccessCredentials: false | 'browser' | 'api' + waitForAllTunnels?: boolean }) => { const r = await fetchFromComposeTunnelAgent({ ...fetchOpts, pathAndQuery: 'tunnels' }) - const { tunnels } = await (r.json() as Promise<{ tunnels: Tunnel[] }>) + const response = await (r.json() as Promise) + + if (waitForAllTunnels && response.state.state !== 'stable') { + throw new AgentFetchError(`Not all configured tunnels are ready yet: ${inspect(response, { depth: null })}`) + } - return tunnels + return response.tunnels .map(tunnel => ({ ...tunnel, ports: mapValues(