From 46a4a63b30c7948e9e6b314780d3bbdab0c29780 Mon Sep 17 00:00:00 2001 From: Roy Razon Date: Thu, 9 May 2024 13:41:17 +0300 Subject: [PATCH] add env metadata command (#481) * local dev: fix gitignore and vscode settings * add env metadata command --- .gitignore | 2 + .vscode/settings.json | 10 +- packages/cli-common/src/index.ts | 3 +- .../cli-common/src/lib/common-flags/index.ts | 17 +-- .../lib/common-flags/tunnel-server-flags.ts | 24 +++ packages/cli/src/commands/env/metadata.ts | 144 ++++++++++++++++++ packages/cli/src/commands/proxy/connect.ts | 8 +- packages/cli/src/commands/up.ts | 8 +- packages/cli/src/commands/urls.ts | 8 +- packages/cli/src/driver-command.ts | 14 +- .../core/src/compose-tunnel-agent-client.ts | 52 ++++--- .../core/src/driver/machine-operations.ts | 17 ++- packages/core/src/index.ts | 4 + 13 files changed, 244 insertions(+), 67 deletions(-) create mode 100644 packages/cli-common/src/lib/common-flags/tunnel-server-flags.ts create mode 100644 packages/cli/src/commands/env/metadata.ts diff --git a/.gitignore b/.gitignore index 45a22c69..54af6e98 100644 --- a/.gitignore +++ b/.gitignore @@ -28,3 +28,5 @@ oclif.manifest.json packages/*/dist packages/*/tsconfig.tsbuildinfo + +.history diff --git a/.vscode/settings.json b/.vscode/settings.json index 2305faed..004b2723 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -15,7 +15,10 @@ "eslint.packageManager": "yarn", "eslint.lintTask.enable": true, "[typescript]": { - "editor.defaultFormatter": "dbaeumer.vscode-eslint" + "editor.defaultFormatter": "dbaeumer.vscode-eslint", + "editor.codeActionsOnSave": { + "source.organizeImports": "never", + } }, "[javascript]": { "editor.defaultFormatter": "dbaeumer.vscode-eslint" @@ -29,5 +32,8 @@ "editor.defaultFormatter": "dbaeumer.vscode-eslint", "[json]": { "editor.defaultFormatter": "vscode.json-language-features" - } + }, + "files.exclude": { + ".history/**": true, + }, } diff --git a/packages/cli-common/src/index.ts b/packages/cli-common/src/index.ts index aa6b7ce7..10cd7550 100644 --- a/packages/cli-common/src/index.ts +++ b/packages/cli-common/src/index.ts @@ -4,7 +4,8 @@ export { HookName, HookFunc, HooksListeners, Hooks } from './lib/hooks.js' export { PluginContext, PluginInitContext } from './lib/plugins/context.js' export { errorToJson } from './lib/errors.js' export { - composeFlags, pluginFlags, envIdFlags, tunnelServerFlags, urlFlags, buildFlags, tableFlags, parseBuildFlags, + composeFlags, pluginFlags, envIdFlags, tunnelServerFlags, parseTunnelServerFlags, + urlFlags, buildFlags, tableFlags, parseBuildFlags, } from './lib/common-flags/index.js' export { formatFlagsToArgs, parseFlags, ParsedFlags } from './lib/flags.js' export { initHook } from './hooks/init/load-plugins.js' diff --git a/packages/cli-common/src/lib/common-flags/index.ts b/packages/cli-common/src/lib/common-flags/index.ts index c80ff233..967a1f27 100644 --- a/packages/cli-common/src/lib/common-flags/index.ts +++ b/packages/cli-common/src/lib/common-flags/index.ts @@ -4,6 +4,7 @@ import { EOL } from 'os' import { DEFAULT_PLUGINS } from '../plugins/default-plugins.js' export * from './build-flags.js' +export * from './tunnel-server-flags.js' export const tableFlags = mapValues(ux.table.flags(), f => ({ ...f, helpGroup: 'OUTPUT' })) as ReturnType @@ -69,22 +70,6 @@ export const envIdFlags = { ...projectFlag, } as const -export const tunnelServerFlags = { - 'tunnel-url': Flags.string({ - summary: 'Tunnel url, specify ssh://hostname[:port] or ssh+tls://hostname[:port]', - char: 't', - default: 'ssh+tls://livecycle.run' ?? process.env.PREVIEW_TUNNEL_OVERRIDE, - }), - 'tls-hostname': Flags.string({ - summary: 'Override TLS server name when tunneling via HTTPS', - required: false, - }), - 'insecure-skip-verify': Flags.boolean({ - summary: 'Skip TLS or SSH certificate verification', - default: false, - }), -} as const - export const urlFlags = { 'include-access-credentials': Flags.boolean({ summary: 'Include access credentials for basic auth for each service URL', diff --git a/packages/cli-common/src/lib/common-flags/tunnel-server-flags.ts b/packages/cli-common/src/lib/common-flags/tunnel-server-flags.ts new file mode 100644 index 00000000..f31f759c --- /dev/null +++ b/packages/cli-common/src/lib/common-flags/tunnel-server-flags.ts @@ -0,0 +1,24 @@ +import { Flags } from '@oclif/core' +import { InferredFlags } from '@oclif/core/lib/interfaces' + +export const tunnelServerFlags = { + 'tunnel-url': Flags.string({ + summary: 'Tunnel url, specify ssh://hostname[:port] or ssh+tls://hostname[:port]', + char: 't', + default: 'ssh+tls://livecycle.run' ?? process.env.PREVIEW_TUNNEL_OVERRIDE, + }), + 'tls-hostname': Flags.string({ + summary: 'Override TLS server name when tunneling via HTTPS', + required: false, + }), + 'insecure-skip-verify': Flags.boolean({ + summary: 'Skip TLS or SSH certificate verification', + default: false, + }), +} as const + +export const parseTunnelServerFlags = (flags: Omit, 'json'>) => ({ + url: flags['tunnel-url'], + tlsServerName: flags['tls-hostname'], + insecureSkipVerify: flags['insecure-skip-verify'], +}) diff --git a/packages/cli/src/commands/env/metadata.ts b/packages/cli/src/commands/env/metadata.ts new file mode 100644 index 00000000..c6d455d9 --- /dev/null +++ b/packages/cli/src/commands/env/metadata.ts @@ -0,0 +1,144 @@ +import { Flags, ux } from '@oclif/core' +import { envIdFlags, parseTunnelServerFlags, text, tunnelServerFlags } from '@preevy/cli-common' +import { TunnelOpts, addBaseComposeTunnelAgentService, findComposeTunnelAgentUrl, findEnvId, getTunnelNamesToServicePorts, getUserCredentials, jwtGenerator, profileStore, queryEnvMetadata, readMetadata } from '@preevy/core' +import { tunnelNameResolver } from '@preevy/common' +import { inspect } from 'util' +import DriverCommand from '../../driver-command.js' +import { connectToTunnelServerSsh } from '../../tunnel-server-client.js' + +type MetadataSource = 'agent' | 'driver' +type UnknownMetadata = Record + +// eslint-disable-next-line no-use-before-define +export default class EnvMetadataCommand extends DriverCommand { + static description = 'Show metadata for a preview environment' + static enableJsonFlag = true + + static flags = { + ...envIdFlags, + ...tunnelServerFlags, + source: Flags.custom<'driver' | 'agent'>({ + summary: 'Show metadata from the driver, the agent, or the driver if the agent is not available', + default: ['agent', 'driver'], + multiple: true, + delimiter: ',', + multipleNonGreedy: true, + })(), + 'fetch-timeout': Flags.integer({ + default: 2500, + summary: 'Timeout for fetching metadata from the agent in milliseconds', + }), + } as const + + async getComposeTunnelAgentUrl( + envId: string, + tunnelOpts: TunnelOpts, + tunnelingKey: string | Buffer, + ) { + const expectedTunnels = getTunnelNamesToServicePorts( + addBaseComposeTunnelAgentService({ name: '' }), + tunnelNameResolver({ envId }), + ) + + const { client: tunnelServerSshClient } = await connectToTunnelServerSsh({ + tunnelOpts, + profileStore: profileStore(this.store), + tunnelingKey, + log: this.logger, + }) + + try { + const expectedTunnelUrls = await tunnelServerSshClient.execTunnelUrl(Object.keys(expectedTunnels)) + + const expectedServiceUrls = Object.entries(expectedTunnels) + .map(([tunnel, { name, port }]) => ({ name, port, url: expectedTunnelUrls[tunnel] })) + + return findComposeTunnelAgentUrl(expectedServiceUrls) + } finally { + void tunnelServerSshClient.end() + } + } + + #envId: string | undefined + async envId() { + if (!this.#envId) { + const { flags } = this + this.#envId = await findEnvId({ + userSpecifiedEnvId: flags.id, + userSpecifiedProjectName: flags.project, + userModel: () => this.ensureUserModel(), + log: this.logger, + }) + } + return this.#envId + } + + async getMetadataFromDriver() { + return await this.withConnection(await this.envId(), readMetadata) + } + + async getMetadataFromAgent() { + const pStore = profileStore(this.store).ref + const tunnelingKey = await pStore.tunnelingKey() + const composeTunnelServiceUrl = await this.getComposeTunnelAgentUrl( + await this.envId(), + parseTunnelServerFlags(this.flags), + tunnelingKey, + ) + const credentials = await getUserCredentials(jwtGenerator(tunnelingKey)) + // eslint-disable-next-line @typescript-eslint/return-await + return await queryEnvMetadata({ + composeTunnelServiceUrl, + credentials, + fetchTimeout: this.flags['fetch-timeout'], + retryOpts: { retries: 2 }, + }) + } + + metadataFactories: Record Promise> = { + driver: this.getMetadataFromDriver.bind(this), + agent: this.getMetadataFromAgent.bind(this), + } + + async getMetatdata() { + const { flags: { source: sources } } = this + const errors: { source: MetadataSource; error: unknown }[] = [] + for (const source of sources) { + try { + this.logger.debug(`Fetching metadata from ${source}`) + return { + // eslint-disable-next-line no-await-in-loop + metadata: await this.metadataFactories[source](), + errors, + source, + } + } catch (err) { + errors.push({ source, error: err }) + } + } + + return { errors } + } + + async run(): Promise { + const { metadata, source: metadataSource, errors } = await this.getMetatdata() + + if (!metadata) { + throw new Error(`Could not get metadata: ${inspect(errors)}`) + } + + if (errors.length) { + for (const { source: errorSource, error } of errors) { + this.logger.warn(`Error fetching metadata from ${errorSource}: ${error}`) + } + } + + if (this.jsonEnabled()) { + return { ...metadata, _source: metadataSource } + } + + this.logger.info(`Metadata from ${text.code(metadataSource)}`) + this.logger.info(inspect(metadata, { depth: null, colors: text.supportsColor !== false })) + return undefined + } +} diff --git a/packages/cli/src/commands/proxy/connect.ts b/packages/cli/src/commands/proxy/connect.ts index a49476b5..b406a14d 100644 --- a/packages/cli/src/commands/proxy/connect.ts +++ b/packages/cli/src/commands/proxy/connect.ts @@ -1,6 +1,6 @@ import { Args, Flags } from '@oclif/core' import { jwkThumbprint, commands, profileStore, withSpinner, SshConnection, machineId, validateEnvId, normalizeEnvId, EnvId } from '@preevy/core' -import { tableFlags, text, tunnelServerFlags, urlFlags } from '@preevy/cli-common' +import { parseTunnelServerFlags, tableFlags, text, tunnelServerFlags, urlFlags } from '@preevy/cli-common' import { inspect } from 'util' import { formatPublicKey } from '@preevy/common' import { spawn } from 'child_process' @@ -58,11 +58,7 @@ export default class Connect extends ProfileCommand { const pStoreRef = pStore.ref const tunnelingKey = await pStoreRef.tunnelingKey() - const tunnelOpts = { - url: flags['tunnel-url'], - tlsServerName: flags['tls-hostname'], - insecureSkipVerify: flags['insecure-skip-verify'], - } + const tunnelOpts = parseTunnelServerFlags(flags) const composeProject = args['compose-project'] let envId:EnvId if (flags.id) { diff --git a/packages/cli/src/commands/up.ts b/packages/cli/src/commands/up.ts index 14706994..90337c4e 100644 --- a/packages/cli/src/commands/up.ts +++ b/packages/cli/src/commands/up.ts @@ -11,7 +11,7 @@ import { telemetryEmitter, withSpinner, } from '@preevy/core' -import { buildFlags, parseBuildFlags, tableFlags, text, tunnelServerFlags } from '@preevy/cli-common' +import { buildFlags, parseBuildFlags, parseTunnelServerFlags, tableFlags, text, tunnelServerFlags } from '@preevy/cli-common' import { inspect } from 'util' import { editUrl, tunnelNameResolver } from '@preevy/common' import MachineCreationDriverCommand from '../machine-creation-driver-command.js' @@ -143,11 +143,7 @@ export default class Up extends MachineCreationDriverCommand { ) const thumbprint = await jwkThumbprint(tunnelingKey) - const tunnelOpts = { - url: flags['tunnel-url'], - tlsServerName: flags['tls-hostname'], - insecureSkipVerify: flags['insecure-skip-verify'], - } + const tunnelOpts = parseTunnelServerFlags(flags) const { expectedServiceUrls, hostKey } = await fetchTunnelServerDetails({ log: this.logger, diff --git a/packages/cli/src/commands/urls.ts b/packages/cli/src/commands/urls.ts index 06e3abc0..973a5464 100644 --- a/packages/cli/src/commands/urls.ts +++ b/packages/cli/src/commands/urls.ts @@ -2,7 +2,7 @@ import fs from 'fs' import yaml from 'yaml' import { Args, ux, Interfaces } from '@oclif/core' import { FlatTunnel, Logger, TunnelOpts, addBaseComposeTunnelAgentService, commands, findComposeTunnelAgentUrl, findEnvId, getTunnelNamesToServicePorts, profileStore } from '@preevy/core' -import { HooksListeners, PluginContext, tableFlags, text, tunnelServerFlags } from '@preevy/cli-common' +import { HooksListeners, PluginContext, parseTunnelServerFlags, tableFlags, text, tunnelServerFlags } from '@preevy/cli-common' import { asyncReduce } from 'iter-tools-es' import { tunnelNameResolver } from '@preevy/common' import { connectToTunnelServerSsh } from '../tunnel-server-client.js' @@ -110,11 +110,7 @@ export default class Urls extends ProfileCommand { log, }) - const tunnelOpts = { - url: flags['tunnel-url'], - tlsServerName: flags['tls-hostname'], - insecureSkipVerify: flags['insecure-skip-verify'], - } + const tunnelOpts = parseTunnelServerFlags(flags) const pStore = profileStore(this.store).ref diff --git a/packages/cli/src/driver-command.ts b/packages/cli/src/driver-command.ts index 28bedcda..a5e76aa0 100644 --- a/packages/cli/src/driver-command.ts +++ b/packages/cli/src/driver-command.ts @@ -24,15 +24,17 @@ abstract class DriverCommand extends ProfileCommand protected flags!: Flags protected args!: Args - public async init(): Promise { - await super.init() - this.#driverName = this.flags.driver ?? this.preevyConfig?.driver as DriverName ?? this.profile.driver as DriverName - } - #driverName: DriverName | undefined get driverName() : DriverName { if (!this.#driverName) { - throw new Error("Driver wasn't specified") + const driverName = this.flags.driver + ?? this.preevyConfig?.driver as DriverName + ?? this.profile.driver as DriverName + + if (!driverName) { + throw new Error("Driver wasn't specified") + } + this.#driverName = driverName } return this.#driverName } diff --git a/packages/core/src/compose-tunnel-agent-client.ts b/packages/core/src/compose-tunnel-agent-client.ts index a39e3fd2..31f55ea0 100644 --- a/packages/core/src/compose-tunnel-agent-client.ts +++ b/packages/core/src/compose-tunnel-agent-client.ts @@ -156,29 +156,40 @@ export const findComposeTunnelAgentUrl = ( return serviceUrl } -export const queryTunnels = async ({ +export type ComposeTunnelAgentFetchOpts = { + composeTunnelServiceUrl: string + credentials: { user: string; password: string } + retryOpts?: retry.Options + fetchTimeout: number +} + +export class AgentFetchError extends Error {} + +const fetchFromComposeTunnelAgent = async ({ retryOpts = { retries: 0 }, composeTunnelServiceUrl, credentials, - includeAccessCredentials, fetchTimeout, -}: { - composeTunnelServiceUrl: string - credentials: { user: string; password: string } - retryOpts?: retry.Options + pathAndQuery, +}: ComposeTunnelAgentFetchOpts & { pathAndQuery: string }) => await retry(async () => { + const r = await fetch( + `${composeTunnelServiceUrl}/${pathAndQuery}`, + { signal: AbortSignal.timeout(fetchTimeout), headers: { Authorization: `Bearer ${credentials.password}` } } + ).catch(e => { throw new AgentFetchError(`Failed to connect to preevy agent at ${composeTunnelServiceUrl}: ${e}`, { cause: e }) }) + if (!r.ok) { + throw new AgentFetchError(`Failed to connect to preevy agent at ${composeTunnelServiceUrl}: ${r.status}: ${r.statusText}`) + } + return r +}, retryOpts) + +export const queryTunnels = async ({ + includeAccessCredentials, + ...fetchOpts +}: ComposeTunnelAgentFetchOpts & { includeAccessCredentials: false | 'browser' | 'api' - fetchTimeout: number }) => { - const { tunnels } = await retry(async () => { - const r = await fetch( - `${composeTunnelServiceUrl}/tunnels`, - { signal: AbortSignal.timeout(fetchTimeout), headers: { Authorization: `Bearer ${credentials.password}` } } - ).catch(e => { throw new Error(`Failed to connect to docker proxy at ${composeTunnelServiceUrl}: ${e}`, { cause: e }) }) - if (!r.ok) { - throw new Error(`Failed to connect to docker proxy at ${composeTunnelServiceUrl}: ${r.status}: ${r.statusText}`) - } - return await (r.json() as Promise<{ tunnels: Tunnel[] }>) - }, retryOpts) + const r = await fetchFromComposeTunnelAgent({ ...fetchOpts, pathAndQuery: 'tunnels' }) + const { tunnels } = await (r.json() as Promise<{ tunnels: Tunnel[] }>) return tunnels .map(tunnel => ({ @@ -186,8 +197,13 @@ export const queryTunnels = async ({ ports: mapValues( tunnel.ports, includeAccessCredentials - ? withBasicAuthCredentials(credentials, includeAccessCredentials) + ? withBasicAuthCredentials(fetchOpts.credentials, includeAccessCredentials) : x => x, ), })) } + +export const queryEnvMetadata = async (fetchOpts: ComposeTunnelAgentFetchOpts): Promise => { + const r = await fetchFromComposeTunnelAgent({ ...fetchOpts, pathAndQuery: 'env-metadata' }) + return await r.json() as EnvMetadata +} diff --git a/packages/core/src/driver/machine-operations.ts b/packages/core/src/driver/machine-operations.ts index b890e551..99042e94 100644 --- a/packages/core/src/driver/machine-operations.ts +++ b/packages/core/src/driver/machine-operations.ts @@ -1,14 +1,14 @@ +import { dateReplacer } from '@preevy/common' import { EOL } from 'os' import retry from 'p-retry' -import { dateReplacer } from '@preevy/common' -import { withSpinner } from '../spinner.js' -import { telemetryEmitter } from '../telemetry/index.js' -import { Logger } from '../log.js' -import { scriptExecuter } from '../remote-script-executer.js' import { EnvMetadata, driverMetadataFilename } from '../env-metadata.js' +import { Logger } from '../log.js' import { REMOTE_DIR_BASE } from '../remote-files.js' -import { MachineBase, SpecDiffItem, isPartialMachine, machineResourceType } from './machine-model.js' +import { scriptExecuter } from '../remote-script-executer.js' +import { withSpinner } from '../spinner.js' +import { telemetryEmitter } from '../telemetry/index.js' import { MachineConnection, MachineCreationDriver, MachineDriver } from './driver.js' +import { MachineBase, SpecDiffItem, isPartialMachine, machineResourceType } from './machine-model.js' const machineDiffText = (diff: SpecDiffItem[]) => diff .map(({ name, old, new: n }) => `* ${name}: ${old} -> ${n}`).join(EOL) @@ -102,6 +102,11 @@ const writeMetadata = async ( }) } +export const readMetadata = async (connection: MachineConnection): Promise> => { + const { stdout } = await connection.exec(`cat "${REMOTE_DIR_BASE}/${driverMetadataFilename}"`) + return JSON.parse(stdout) +} + export const getUserAndGroup = async (connection: Pick) => ( await connection.exec('echo "$(id -u):$(stat -c %g /var/run/docker.sock)"') ).stdout diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index 71cac76e..2f2d8691 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -19,6 +19,7 @@ export { ForwardSocket, machineStatusNodeExporterCommand, ensureMachine, + readMetadata, } from './driver/index.js' export { profileStore, Profile, ProfileStore, ProfileStoreRef, ProfileStoreTransaction, ProfileEditor, @@ -54,7 +55,9 @@ export { export { addBaseComposeTunnelAgentService, queryTunnels, + queryEnvMetadata, findComposeTunnelAgentUrl, + AgentFetchError, } from './compose-tunnel-agent-client.js' export * as commands from './commands/index.js' export { BuildSpec, ImageRegistry, parseRegistry } from './build/index.js' @@ -79,3 +82,4 @@ export { pSeries } from './p-series.js' export { gitContext, GitContext } from './git.js' export * as config from './config.js' export { login, getTokensFromLocalFs as getLivecycleTokensFromLocalFs, TokenExpiredError } from './login.js' +export { EnvMetadata, EnvMachineMetadata } from './env-metadata.js'