diff --git a/x-pack/plugins/security_solution/scripts/endpoint/agent_downloader_cli/agent_downloader.test.ts b/x-pack/plugins/security_solution/scripts/endpoint/agent_downloader_cli/agent_downloader.test.ts new file mode 100644 index 0000000000000..a39ad186e62b6 --- /dev/null +++ b/x-pack/plugins/security_solution/scripts/endpoint/agent_downloader_cli/agent_downloader.test.ts @@ -0,0 +1,203 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { getAgentDownloadUrl, getAgentFileName } from '../common/fleet_services'; +import { downloadAndStoreAgent } from '../common/agent_downloads_service'; +import type { ToolingLog } from '@kbn/tooling-log'; +import { agentDownloaderRunner } from './agent_downloader'; +import type { RunContext } from '@kbn/dev-cli-runner'; + +jest.mock('../common/fleet_services'); +jest.mock('../common/agent_downloads_service'); + +describe('agentDownloaderRunner', () => { + let log: ToolingLog; + + beforeEach(() => { + log = { + info: jest.fn(), + error: jest.fn(), + } as unknown as ToolingLog; + + jest.clearAllMocks(); + }); + + const version = '8.15.0'; + let closestMatch = false; + const url = 'http://example.com/agent.tar.gz'; + const fileName = 'elastic-agent-8.15.0.tar.gz'; + + it('downloads and stores the specified version', async () => { + (getAgentDownloadUrl as jest.Mock).mockResolvedValue({ url }); + (getAgentFileName as jest.Mock).mockReturnValue('elastic-agent-8.15.0'); + (downloadAndStoreAgent as jest.Mock).mockResolvedValue(undefined); + + await agentDownloaderRunner({ + flags: { version, closestMatch }, + log, + } as unknown as RunContext); + + expect(getAgentDownloadUrl).toHaveBeenCalledWith(version, closestMatch, log); + expect(getAgentFileName).toHaveBeenCalledWith(version); + expect(downloadAndStoreAgent).toHaveBeenCalledWith(url, fileName); + expect(log.info).toHaveBeenCalledWith('Successfully downloaded and stored version 8.15.0'); + }); + + it('logs an error if the download fails', async () => { + (getAgentDownloadUrl as jest.Mock).mockResolvedValue({ url }); + (getAgentFileName as jest.Mock).mockReturnValue('elastic-agent-8.15.0'); + (downloadAndStoreAgent as jest.Mock).mockRejectedValue(new Error('Download failed')); + + await agentDownloaderRunner({ + flags: { version, closestMatch }, + log, + } as unknown as RunContext); + + expect(getAgentDownloadUrl).toHaveBeenCalledWith(version, closestMatch, log); + expect(getAgentFileName).toHaveBeenCalledWith(version); + expect(downloadAndStoreAgent).toHaveBeenCalledWith(url, fileName); + expect(log.error).toHaveBeenCalledWith( + 'Failed to download or store version 8.15.0: Download failed' + ); + }); + + it('downloads and stores the previous patch version if the specified version fails', async () => { + const fallbackVersion = '8.15.0'; + const fallbackFileName = 'elastic-agent-8.15.0.tar.gz'; + + (getAgentDownloadUrl as jest.Mock) + .mockResolvedValueOnce({ url }) + .mockResolvedValueOnce({ url }); + (getAgentFileName as jest.Mock) + .mockReturnValueOnce('elastic-agent-8.15.1') + .mockReturnValueOnce('elastic-agent-8.15.0'); + (downloadAndStoreAgent as jest.Mock) + .mockRejectedValueOnce(new Error('Download failed')) + .mockResolvedValueOnce(undefined); + + await agentDownloaderRunner({ + flags: { version: '8.15.1', closestMatch }, + log, + } as unknown as RunContext); + + expect(getAgentDownloadUrl).toHaveBeenCalledWith('8.15.1', closestMatch, log); + expect(getAgentDownloadUrl).toHaveBeenCalledWith(fallbackVersion, closestMatch, log); + expect(getAgentFileName).toHaveBeenCalledWith('8.15.1'); + expect(getAgentFileName).toHaveBeenCalledWith(fallbackVersion); + expect(downloadAndStoreAgent).toHaveBeenCalledWith(url, 'elastic-agent-8.15.1.tar.gz'); + expect(downloadAndStoreAgent).toHaveBeenCalledWith(url, fallbackFileName); + expect(log.error).toHaveBeenCalledWith( + 'Failed to download or store version 8.15.1: Download failed' + ); + expect(log.info).toHaveBeenCalledWith('Successfully downloaded and stored version 8.15.0'); + }); + + it('logs an error if all downloads fail', async () => { + (getAgentDownloadUrl as jest.Mock).mockResolvedValue({ url }); + (getAgentFileName as jest.Mock) + .mockReturnValueOnce('elastic-agent-8.15.1') + .mockReturnValueOnce('elastic-agent-8.15.0'); + (downloadAndStoreAgent as jest.Mock) + .mockRejectedValueOnce(new Error('Download failed')) + .mockRejectedValueOnce(new Error('Download failed')); + + await agentDownloaderRunner({ + flags: { version: '8.15.1', closestMatch }, + log, + } as unknown as RunContext); + + expect(getAgentDownloadUrl).toHaveBeenCalledWith('8.15.1', closestMatch, log); + expect(getAgentDownloadUrl).toHaveBeenCalledWith('8.15.0', closestMatch, log); + expect(getAgentFileName).toHaveBeenCalledWith('8.15.1'); + expect(getAgentFileName).toHaveBeenCalledWith('8.15.0'); + expect(downloadAndStoreAgent).toHaveBeenCalledWith(url, 'elastic-agent-8.15.1.tar.gz'); + expect(downloadAndStoreAgent).toHaveBeenCalledWith(url, 'elastic-agent-8.15.0.tar.gz'); + expect(log.error).toHaveBeenCalledWith( + 'Failed to download or store version 8.15.1: Download failed' + ); + expect(log.error).toHaveBeenCalledWith( + 'Failed to download or store version 8.15.0: Download failed' + ); + }); + + it('does not attempt fallback when patch version is 0', async () => { + (getAgentDownloadUrl as jest.Mock).mockResolvedValue({ url }); + (getAgentFileName as jest.Mock).mockReturnValue('elastic-agent-8.15.0'); + (downloadAndStoreAgent as jest.Mock).mockResolvedValue(undefined); + + await agentDownloaderRunner({ + flags: { version: '8.15.0', closestMatch }, + log, + } as unknown as RunContext); + + expect(getAgentDownloadUrl).toHaveBeenCalledTimes(1); // Only one call for 8.15.0 + expect(getAgentFileName).toHaveBeenCalledTimes(1); + expect(downloadAndStoreAgent).toHaveBeenCalledWith(url, fileName); + expect(log.info).toHaveBeenCalledWith('Successfully downloaded and stored version 8.15.0'); + }); + + it('logs an error for an invalid version format', async () => { + const invalidVersion = '7.x.x'; + + await expect( + agentDownloaderRunner({ + flags: { version: invalidVersion, closestMatch }, + log, + } as unknown as RunContext) + ).rejects.toThrow('Invalid version format'); + }); + + it('passes the closestMatch flag correctly', async () => { + closestMatch = true; + + (getAgentDownloadUrl as jest.Mock).mockResolvedValue({ url }); + (getAgentFileName as jest.Mock).mockReturnValue('elastic-agent-8.15.0'); + (downloadAndStoreAgent as jest.Mock).mockResolvedValue(undefined); + + await agentDownloaderRunner({ + flags: { version, closestMatch }, + log, + } as unknown as RunContext); + + expect(getAgentDownloadUrl).toHaveBeenCalledWith(version, closestMatch, log); + }); + + it('throws an error when version is not provided', async () => { + await expect( + agentDownloaderRunner({ + flags: { closestMatch }, + log, + } as unknown as RunContext) + ).rejects.toThrow('version argument is required'); + }); + + it('logs the correct messages when both version and fallback version are processed', async () => { + const primaryVersion = '8.15.1'; + + (getAgentDownloadUrl as jest.Mock) + .mockResolvedValueOnce({ url }) + .mockResolvedValueOnce({ url }); + + (getAgentFileName as jest.Mock) + .mockReturnValueOnce('elastic-agent-8.15.1') + .mockReturnValueOnce('elastic-agent-8.15.0'); + + (downloadAndStoreAgent as jest.Mock) + .mockRejectedValueOnce(new Error('Download failed')) // Fail on primary + .mockResolvedValueOnce(undefined); // Success on fallback + + await agentDownloaderRunner({ + flags: { version: primaryVersion, closestMatch }, + log, + } as unknown as RunContext); + + expect(log.error).toHaveBeenCalledWith( + 'Failed to download or store version 8.15.1: Download failed' + ); + expect(log.info).toHaveBeenCalledWith('Successfully downloaded and stored version 8.15.0'); + }); +}); diff --git a/x-pack/plugins/security_solution/scripts/endpoint/agent_downloader_cli/agent_downloader.ts b/x-pack/plugins/security_solution/scripts/endpoint/agent_downloader_cli/agent_downloader.ts index ab1da6a3f208f..8366c77575e70 100644 --- a/x-pack/plugins/security_solution/scripts/endpoint/agent_downloader_cli/agent_downloader.ts +++ b/x-pack/plugins/security_solution/scripts/endpoint/agent_downloader_cli/agent_downloader.ts @@ -8,24 +8,72 @@ import { ok } from 'assert'; import type { RunFn } from '@kbn/dev-cli-runner'; import type { ToolingLog } from '@kbn/tooling-log'; +import semver from 'semver'; import { getAgentDownloadUrl, getAgentFileName } from '../common/fleet_services'; import { downloadAndStoreAgent } from '../common/agent_downloads_service'; +// Decrement the patch version by 1 and preserve pre-release tag (if any) +const decrementPatchVersion = (version: string): string | null => { + const parsedVersion = semver.parse(version); + if (!parsedVersion) { + return null; + } + const newPatchVersion = parsedVersion.patch - 1; + // Create a new version string with the decremented patch - removing any possible pre-release tag + const newVersion = `${parsedVersion.major}.${parsedVersion.minor}.${newPatchVersion}`; + return semver.valid(newVersion) ? newVersion : null; +}; + +// Generate a list of versions to attempt downloading, including a fallback to the previous patch (GA) +const getVersionsToDownload = (version: string): string[] => { + const parsedVersion = semver.parse(version); + if (!parsedVersion) return []; + // If patch version is 0, return only the current version. + if (parsedVersion.patch === 0) { + return [version]; + } + + const decrementedVersion = decrementPatchVersion(version); + return decrementedVersion ? [version, decrementedVersion] : [version]; +}; + +// Download and store the Elastic Agent for the specified version(s) const downloadAndStoreElasticAgent = async ( version: string, closestMatch: boolean, log: ToolingLog -) => { - const downloadUrlResponse = await getAgentDownloadUrl(version, closestMatch, log); - const fileNameNoExtension = getAgentFileName(version); - const agentFile = `${fileNameNoExtension}.tar.gz`; - await downloadAndStoreAgent(downloadUrlResponse.url, agentFile); +): Promise => { + const versionsToDownload = getVersionsToDownload(version); + + // Although we have a list of versions to try downloading, we only need to download one, and will return as soon as it succeeds. + for (const versionToDownload of versionsToDownload) { + try { + const { url } = await getAgentDownloadUrl(versionToDownload, closestMatch, log); + const fileName = `${getAgentFileName(versionToDownload)}.tar.gz`; + + await downloadAndStoreAgent(url, fileName); + log.info(`Successfully downloaded and stored version ${versionToDownload}`); + return; // Exit once successful + } catch (error) { + log.error(`Failed to download or store version ${versionToDownload}: ${error.message}`); + } + } + + log.error(`Failed to download agent for any available version: ${versionsToDownload.join(', ')}`); }; export const agentDownloaderRunner: RunFn = async (cliContext) => { - ok(cliContext.flags.version, 'version argument is required'); + const { version } = cliContext.flags; + + ok(version, 'version argument is required'); + + // Validate version format + if (!semver.valid(version as string)) { + throw new Error('Invalid version format'); + } + await downloadAndStoreElasticAgent( - cliContext.flags.version as string, + version as string, cliContext.flags.closestMatch as boolean, cliContext.log ); diff --git a/x-pack/plugins/security_solution/scripts/endpoint/common/agent_downloads_service.test.ts b/x-pack/plugins/security_solution/scripts/endpoint/common/agent_downloads_service.test.ts new file mode 100644 index 0000000000000..0a7a9d3104798 --- /dev/null +++ b/x-pack/plugins/security_solution/scripts/endpoint/common/agent_downloads_service.test.ts @@ -0,0 +1,77 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +// Adjust path if needed + +import { downloadAndStoreAgent, isAgentDownloadFromDiskAvailable } from './agent_downloads_service'; +import fs from 'fs'; +import nodeFetch from 'node-fetch'; +import { finished } from 'stream/promises'; + +jest.mock('fs'); +jest.mock('node-fetch'); +jest.mock('stream/promises', () => ({ + finished: jest.fn(), +})); +jest.mock('../../../common/endpoint/data_loaders/utils', () => ({ + createToolingLogger: jest.fn(() => ({ + debug: jest.fn(), + info: jest.fn(), + error: jest.fn(), + })), +})); + +describe('AgentDownloadStorage', () => { + const url = 'http://example.com/agent.tar.gz'; + const fileName = 'elastic-agent-7.10.0.tar.gz'; + beforeEach(() => { + jest.clearAllMocks(); // Ensure no previous test state affects the current one + }); + + it('downloads and stores the agent if not cached', async () => { + (fs.existsSync as unknown as jest.Mock).mockReturnValue(false); + (fs.createWriteStream as unknown as jest.Mock).mockReturnValue({ + on: jest.fn(), + end: jest.fn(), + }); + (nodeFetch as unknown as jest.Mock).mockResolvedValue({ body: { pipe: jest.fn() } }); + (finished as unknown as jest.Mock).mockResolvedValue(undefined); + + const result = await downloadAndStoreAgent(url, fileName); + + expect(result).toEqual({ + url, + filename: fileName, + directory: expect.any(String), + fullFilePath: expect.stringContaining(fileName), // Dynamically match the file path + }); + }); + + it('reuses cached agent if available', async () => { + (fs.existsSync as unknown as jest.Mock).mockReturnValue(true); + + const result = await downloadAndStoreAgent(url, fileName); + + expect(result).toEqual({ + url, + filename: fileName, + directory: expect.any(String), + fullFilePath: expect.stringContaining(fileName), // Dynamically match the path + }); + }); + + it('checks if agent download is available from disk', () => { + (fs.existsSync as unknown as jest.Mock).mockReturnValue(true); + + const result = isAgentDownloadFromDiskAvailable(fileName); + + expect(result).toEqual({ + filename: fileName, + directory: expect.any(String), + fullFilePath: expect.stringContaining(fileName), // Dynamically match the path + }); + }); +}); diff --git a/x-pack/plugins/security_solution/scripts/endpoint/common/agent_downloads_service.ts b/x-pack/plugins/security_solution/scripts/endpoint/common/agent_downloads_service.ts index 488e1b10160e8..4c963332ad0c2 100644 --- a/x-pack/plugins/security_solution/scripts/endpoint/common/agent_downloads_service.ts +++ b/x-pack/plugins/security_solution/scripts/endpoint/common/agent_downloads_service.ts @@ -5,6 +5,7 @@ * 2.0. */ +import pRetry from 'p-retry'; import { mkdir, readdir, stat, unlink } from 'fs/promises'; import { join } from 'path'; import fs from 'fs'; @@ -24,7 +25,7 @@ export interface DownloadedAgentInfo { interface AgentDownloadStorageSettings { /** - * Last time a cleanup was ran. Date in ISO format + * Last time a cleanup was performed. Date in ISO format */ lastCleanup: string; @@ -47,7 +48,7 @@ class AgentDownloadStorage extends SettingsStorage constructor() { super('agent_download_storage_settings.json', { defaultSettings: { - maxFileAge: 1.728e8, // 2 days + maxFileAge: 1.728e8, // 2 days in milliseconds lastCleanup: new Date().toISOString(), }, }); @@ -55,20 +56,25 @@ class AgentDownloadStorage extends SettingsStorage this.downloadsDirFullPath = this.buildPath(this.downloadsDirName); } + /** + * Ensures the download directory exists on disk + */ protected async ensureExists(): Promise { await super.ensureExists(); if (!this.downloadsFolderExists) { await mkdir(this.downloadsDirFullPath, { recursive: true }); - this.log.debug(`Created directory [this.downloadsDirFullPath] for cached agent downloads`); + this.log.debug(`Created directory [${this.downloadsDirFullPath}] for cached agent downloads`); this.downloadsFolderExists = true; } } + /** + * Gets the file paths for a given download URL and optional file name. + */ public getPathsForUrl(agentDownloadUrl: string, agentFileName?: string): DownloadedAgentInfo { - const filename = agentFileName - ? agentFileName - : agentDownloadUrl.replace(/^https?:\/\//gi, '').replace(/\//g, '#'); + const filename = + agentFileName || agentDownloadUrl.replace(/^https?:\/\//gi, '').replace(/\//g, '#'); const directory = this.downloadsDirFullPath; const fullFilePath = this.buildPath(join(this.downloadsDirName, filename)); @@ -79,59 +85,67 @@ class AgentDownloadStorage extends SettingsStorage }; } + /** + * Downloads the agent and stores it locally. Reuses existing downloads if available. + */ public async downloadAndStore( agentDownloadUrl: string, agentFileName?: string ): Promise { - this.log.debug(`Downloading and storing: ${agentDownloadUrl}`); - - // TODO: should we add "retry" attempts to file downloads? + this.log.debug(`Starting download: ${agentDownloadUrl}`); await this.ensureExists(); - const newDownloadInfo = this.getPathsForUrl(agentDownloadUrl, agentFileName); - // If download is already present on disk, then just return that info. No need to re-download it + // Return cached version if the file already exists if (fs.existsSync(newDownloadInfo.fullFilePath)) { this.log.debug(`Download already cached at [${newDownloadInfo.fullFilePath}]`); return newDownloadInfo; } try { - const outputStream = fs.createWriteStream(newDownloadInfo.fullFilePath); - - await handleProcessInterruptions( - async () => { - const { body } = await nodeFetch(agentDownloadUrl); - await finished(body.pipe(outputStream)); + await pRetry( + async (attempt) => { + this.log.info( + `Attempt ${attempt} - Downloading agent from [${agentDownloadUrl}] to [${newDownloadInfo.fullFilePath}]` + ); + const outputStream = fs.createWriteStream(newDownloadInfo.fullFilePath); + + await handleProcessInterruptions( + async () => { + const { body } = await nodeFetch(agentDownloadUrl); + await finished(body.pipe(outputStream)); + }, + () => fs.unlinkSync(newDownloadInfo.fullFilePath) // Clean up on interruption + ); + this.log.info(`Successfully downloaded agent to [${newDownloadInfo.fullFilePath}]`); }, - () => { - fs.unlinkSync(newDownloadInfo.fullFilePath); + { + retries: 2, // 2 retries = 3 total attempts (1 initial + 2 retries) + onFailedAttempt: (error) => { + this.log.error(`Download attempt ${error.attemptNumber} failed: ${error.message}`); + // Cleanup failed download + return unlink(newDownloadInfo.fullFilePath); + }, } ); - } catch (e) { - // Try to clean up download case it failed halfway through - await unlink(newDownloadInfo.fullFilePath); - - throw e; + } catch (error) { + throw new Error(`Download failed after multiple attempts: ${error.message}`); } await this.cleanupDownloads(); - return newDownloadInfo; } public async cleanupDownloads(): Promise<{ deleted: string[] }> { - this.log.debug(`Performing cleanup of cached Agent downlaods`); + this.log.debug('Performing cleanup of cached Agent downloads'); const settings = await this.get(); - const maxAgeDate = new Date(); + const maxAgeDate = new Date(Date.now() - settings.maxFileAge); const response: { deleted: string[] } = { deleted: [] }; - maxAgeDate.setMilliseconds(settings.maxFileAge * -1); // `* -1` to set time back - - // If cleanup already happen within the file age, then nothing to do. Exit. if (settings.lastCleanup > maxAgeDate.toISOString()) { + this.log.debug('Skipping cleanup, as it was performed recently.'); return response; } @@ -140,41 +154,48 @@ class AgentDownloadStorage extends SettingsStorage lastCleanup: new Date().toISOString(), }); - const deleteFilePromises: Array> = []; - const allFiles = await readdir(this.downloadsDirFullPath); - - for (const fileName of allFiles) { - const filePath = join(this.downloadsDirFullPath, fileName); - const fileStats = await stat(filePath); + try { + const allFiles = await readdir(this.downloadsDirFullPath); + const deleteFilePromises = allFiles.map(async (fileName) => { + const filePath = join(this.downloadsDirFullPath, fileName); + const fileStats = await stat(filePath); + if (fileStats.isFile() && fileStats.birthtime < maxAgeDate) { + try { + await unlink(filePath); + response.deleted.push(filePath); + } catch (err) { + this.log.error(`Failed to delete file [${filePath}]: ${err.message}`); + } + } + }); - if (fileStats.isFile() && fileStats.birthtime < maxAgeDate) { - deleteFilePromises.push(unlink(filePath)); - response.deleted.push(filePath); - } + await Promise.allSettled(deleteFilePromises); + this.log.debug(`Deleted ${response.deleted.length} file(s)`); + return response; + } catch (err) { + this.log.error(`Error during cleanup: ${err.message}`); + return response; } - - await Promise.allSettled(deleteFilePromises); - - this.log.debug(`Deleted [${response.deleted.length}] file(s)`); - this.log.verbose(`files deleted:\n`, response.deleted.join('\n')); - - return response; } + /** + * Checks if a specific agent download is available locally. + */ public isAgentDownloadFromDiskAvailable(filename: string): DownloadedAgentInfo | undefined { - if (fs.existsSync(join(this.downloadsDirFullPath, filename))) { + const filePath = join(this.downloadsDirFullPath, filename); + if (fs.existsSync(filePath)) { return { filename, /** The local directory where downloads are stored */ directory: this.downloadsDirFullPath, /** The full local file path and name */ - fullFilePath: join(this.downloadsDirFullPath, filename), + fullFilePath: filePath, }; } } } -const agentDownloadsClient = new AgentDownloadStorage(); +export const agentDownloadsClient = new AgentDownloadStorage(); export interface DownloadAndStoreAgentResponse extends DownloadedAgentInfo { url: string; @@ -203,12 +224,15 @@ export const downloadAndStoreAgent = async ( }; /** - * Cleans up the old agent downloads on disk. + * Cleans up old agent downloads on disk. */ export const cleanupDownloads = async (): ReturnType => { return agentDownloadsClient.cleanupDownloads(); }; +/** + * Checks if a specific agent download is available from disk. + */ export const isAgentDownloadFromDiskAvailable = ( fileName: string ): DownloadedAgentInfo | undefined => {