From 55ef74df7b1509beb6447398ebdc0769f55e0e91 Mon Sep 17 00:00:00 2001 From: Thom Heymann <190132+thomheymann@users.noreply.github.com> Date: Thu, 14 Nov 2024 14:33:03 +0000 Subject: [PATCH] [8.x] [Observability Onboarding] Display next steps (#197179) (#200139) # Backport This will backport the following commits from `main` to `8.x`: - [[Observability Onboarding] Display next steps (#197179)](https://github.com/elastic/kibana/pull/197179) ### Questions ? Please refer to the [Backport tool documentation](https://github.com/sqren/backport) --- .../services/epm/package_service.mock.ts | 2 +- .../server/services/epm/package_service.ts | 13 ++-- .../public/assets/auto_detect.sh | 35 ++++++--- .../public/assets/integrations.conf | 26 +++---- .../server/routes/flow/route.ts | 72 ++++++------------- .../server/routes/types.ts | 2 +- 6 files changed, 67 insertions(+), 83 deletions(-) diff --git a/x-pack/plugins/fleet/server/services/epm/package_service.mock.ts b/x-pack/plugins/fleet/server/services/epm/package_service.mock.ts index 39d0451687de5..eeaa80b0c9449 100644 --- a/x-pack/plugins/fleet/server/services/epm/package_service.mock.ts +++ b/x-pack/plugins/fleet/server/services/epm/package_service.mock.ts @@ -16,7 +16,7 @@ const createClientMock = (): jest.Mocked => ({ installCustomIntegration: jest.fn(), fetchFindLatestPackage: jest.fn(), readBundledPackage: jest.fn(), - getAgentPolicyInputs: jest.fn(), + getAgentPolicyConfigYAML: jest.fn(), getPackage: jest.fn(), getPackageFieldsMetadata: jest.fn(), getPackages: jest.fn(), diff --git a/x-pack/plugins/fleet/server/services/epm/package_service.ts b/x-pack/plugins/fleet/server/services/epm/package_service.ts index 7882af4b72138..a097db584b460 100644 --- a/x-pack/plugins/fleet/server/services/epm/package_service.ts +++ b/x-pack/plugins/fleet/server/services/epm/package_service.ts @@ -28,7 +28,6 @@ import type { InstallablePackage, Installation, RegistryPackage, - TemplateAgentPolicyInput, } from '../../types'; import type { FleetAuthzRouteConfig } from '../security/types'; @@ -120,12 +119,12 @@ export interface PackageClient { prerelease?: false; }): Promise; - getAgentPolicyInputs( + getAgentPolicyConfigYAML( pkgName: string, pkgVersion?: string, prerelease?: false, ignoreUnverified?: boolean - ): Promise; + ): Promise; reinstallEsAssets( packageInfo: InstallablePackage, @@ -288,7 +287,7 @@ class PackageClientImpl implements PackageClient { return generatePackageInfoFromArchiveBuffer(archiveBuffer, 'application/zip'); } - public async getAgentPolicyInputs( + public async getAgentPolicyConfigYAML( pkgName: string, pkgVersion?: string, prerelease?: false, @@ -302,16 +301,14 @@ class PackageClientImpl implements PackageClient { pkgVersion = pkg.version; } - const { inputs } = await getTemplateInputs( + return getTemplateInputs( this.internalSoClient, pkgName, pkgVersion, - 'json', + 'yml', prerelease, ignoreUnverified ); - - return inputs; } public async getPackage( diff --git a/x-pack/plugins/observability_solution/observability_onboarding/public/assets/auto_detect.sh b/x-pack/plugins/observability_solution/observability_onboarding/public/assets/auto_detect.sh index 252e021b8a276..06f6a90f82a45 100755 --- a/x-pack/plugins/observability_solution/observability_onboarding/public/assets/auto_detect.sh +++ b/x-pack/plugins/observability_solution/observability_onboarding/public/assets/auto_detect.sh @@ -105,6 +105,7 @@ elastic_agent_config_path="/opt/Elastic/Agent/elastic-agent.yml" elastic_agent_tmp_config_path="/tmp/elastic-agent-config.tar" integration_names=() integration_titles=() +config_files_with_password=() OS="$(uname)" ARCH="$(uname -m)" @@ -155,7 +156,7 @@ download_elastic_agent() { curl -L -O "$download_url" --silent --fail if [ "$?" -eq 0 ]; then - printf "\e[1;32m✓\e[0m %s\n" "Elastic Agent downloaded to $(pwd)/$elastic_agent_artifact_name.tar.gz" + printf "\e[32;1m✓\e[0m %s\n" "Elastic Agent downloaded to $(pwd)/$elastic_agent_artifact_name.tar.gz" update_step_progress "ea-download" "complete" else update_step_progress "ea-download" "danger" "Failed to download Elastic Agent, see script output for error." @@ -167,7 +168,7 @@ extract_elastic_agent() { tar -xzf "${elastic_agent_artifact_name}.tar.gz" if [ "$?" -eq 0 ]; then - printf "\e[1;32m✓\e[0m %s\n" "Archive extracted" + printf "\e[32;1m✓\e[0m %s\n" "Archive extracted" update_step_progress "ea-extract" "complete" else update_step_progress "ea-extract" "danger" "Failed to extract Elastic Agent, see script output for error." @@ -179,7 +180,7 @@ install_elastic_agent() { "./${elastic_agent_artifact_name}/elastic-agent" install -f -n >/dev/null if [ "$?" -eq 0 ]; then - printf "\e[1;32m✓\e[0m %s\n" "Elastic Agent installed to $(dirname "$elastic_agent_config_path")" + printf "\e[32;1m✓\e[0m %s\n" "Elastic Agent installed to $(dirname "$elastic_agent_config_path")" update_step_progress "ea-install" "complete" else update_step_progress "ea-install" "danger" "Failed to install Elastic Agent, see script output for error." @@ -224,7 +225,7 @@ ensure_elastic_agent_healthy() { backup_elastic_agent_config() { if [ -f "$elastic_agent_config_path" ]; then - echo -e "\nExisting config found at $elastic_agent_config_path" + printf "\n%s \e[36m%s\e[0m\n" "Existing config found at" "$elastic_agent_config_path" printf "\n\e[1;36m?\e[0m \e[1m%s\e[0m \e[2m%s\e[0m" "Create backup and continue installation?" "[Y/n] (default: Yes): " read confirmation_reply @@ -241,7 +242,7 @@ backup_elastic_agent_config() { fi if [ "$?" -eq 0 ]; then - printf "\n\e[1;32m✓\e[0m %s\n" "Backup saved to $backup_path" + printf "\n\e[32;1m✓\e[0m %s \e[36m%s\e[0m\n" "Backup saved to" "$backup_path" else update_step_progress "ea-config" "warning" "Failed to backup existing configuration" fail "Failed to backup existing config - Try manually creating a backup or delete your existing config before re-running this script" @@ -286,7 +287,7 @@ install_integrations() { --output "$elastic_agent_tmp_config_path" if [ "$?" -eq 0 ]; then - printf "\n\e[1;32m✓\e[0m %s\n" "Integrations installed" + printf "\n\e[32;1m✓\e[0m %s\n" "Integrations installed" else update_step_progress "ea-config" "warning" "Failed to install integrations" fail "Failed to install integrations" @@ -305,10 +306,15 @@ apply_elastic_agent_config() { # Replace placeholder with the Ingest API key sed -i='' "s/\${API_KEY}/$decoded_ingest_api_key/" "$elastic_agent_config_path" if [ "$?" -eq 0 ]; then - printf "\e[1;32m✓\e[0m %s\n" "Config written to:" - tar --list --file "$elastic_agent_tmp_config_path" | grep '\.yml$' | while read -r file; do - echo " - $(dirname "$elastic_agent_config_path")/$file" - done + printf "\e[32;1m✓\e[0m %s\n" "Config files written to:" + while IFS= read -r file; do + local path="$(dirname "$elastic_agent_config_path")/$file" + printf " \e[36m%s\e[0m\n" "$path" + grep '' "$path" >/dev/null + if [ "$?" -eq 0 ]; then + config_files_with_password+=("$path") + fi + done < <(tar --list --file "$elastic_agent_tmp_config_path" | grep '\.yml$') update_step_progress "ea-config" "complete" else @@ -610,4 +616,11 @@ printf "\n\e[1m%s\e[0m\n" "Waiting for healthy status..." wait_for_elastic_agent_status ensure_elastic_agent_healthy -printf "\n\e[32m%s\e[0m\n" "🎉 Elastic Agent is configured and running. You can now go back to Kibana and check for incoming logs." +printf "\n\e[32m%s\e[0m\n" "🎉 Elastic Agent is configured and running!" + +printf "\n\e[1m%s\e[0m\n" "Next steps:" +printf "\n• %s\n" "Go back to Kibana and check for incoming data" +for path in "${config_files_with_password[@]}"; do + printf "\n• %s:\n \e[36m%s\e[0m\n" "Collect $(known_integration_title "$(basename "${path%.yml}")") metrics by adding your username and password to" "$path" +done +printf "\n• %s:\n \e[36;4m%s\e[0m\n" "For information on other standalone integration setups, visit" "https://www.elastic.co/guide/en/fleet/current/elastic-agent-configuration.html" diff --git a/x-pack/plugins/observability_solution/observability_onboarding/public/assets/integrations.conf b/x-pack/plugins/observability_solution/observability_onboarding/public/assets/integrations.conf index e9afdc3cdfb2e..48ad18035edc0 100644 --- a/x-pack/plugins/observability_solution/observability_onboarding/public/assets/integrations.conf +++ b/x-pack/plugins/observability_solution/observability_onboarding/public/assets/integrations.conf @@ -1,14 +1,14 @@ [system] -title=System Logs And Metrics +title=System [nginx] -title=Nginx Logs +title=Nginx patterns= /var/log/nginx/access.log* /var/log/nginx/error.log* [apache] -title=Apache Logs +title=Apache patterns= /var/log/apache2/access.log* /var/log/apache2/other_vhosts_access.log* @@ -17,13 +17,13 @@ patterns= /var/log/httpd/error_log* [docker] -title=Docker Container Logs +title=Docker patterns= /var/lib/docker/containers/*/*-json.log /var/run/docker.sock [mysql] -title=MySQL Logs +title=MySQL patterns= /var/log/mysql/*error.log* /var/log/mysqld.log* @@ -31,7 +31,7 @@ patterns= /var/lib/mysql/*-slow.log* [postgresql] -title=PostgreSQL Logs +title=PostgreSQL patterns= /var/log/postgresql/postgresql-*-*.log* /var/log/postgresql/postgresql-*-*.csv* @@ -39,26 +39,26 @@ patterns= /etc/postgresql/*/main/postgresql.conf [redis] -title=Redis Logs +title=Redis patterns= /var/log/redis/redis-server.log* /etc/redis/redis.conf [haproxy] -title=HAProxy Logs +title=HAProxy patterns= /var/log/haproxy.log /etc/haproxy/haproxy.cfg [rabbitmq] -title=RabbitMQ Logs +title=RabbitMQ patterns= /var/log/rabbitmq/rabbit@*.log /etc/rabbitmq/rabbitmq.conf /etc/rabbitmq/rabbitmq.config [kafka] -title=Kafka Logs +title=Kafka patterns= /var/log/kafka/server.log /etc/kafka/server.properties @@ -68,19 +68,19 @@ patterns= /*/logs/kafka-*.log* [mongodb] -title=MongoDB Logs +title=MongoDB patterns= /var/log/mongodb/mongod.log [apache_tomcat] -title=Apache Tomcat Logs +title=Apache Tomcat patterns= /opt/tomcat/logs/localhost_access_log.*.txt /opt/tomcat/logs/catalina.*.log /opt/tomcat/logs/localhost.*.log [prometheus] -title=Prometheus Server overview +title=Prometheus patterns= /var/log/prometheus/prometheus.log /etc/prometheus/prometheus.yml \ No newline at end of file diff --git a/x-pack/plugins/observability_solution/observability_onboarding/server/routes/flow/route.ts b/x-pack/plugins/observability_solution/observability_onboarding/server/routes/flow/route.ts index 610d8c624c7c3..f93e4f71f108b 100644 --- a/x-pack/plugins/observability_solution/observability_onboarding/server/routes/flow/route.ts +++ b/x-pack/plugins/observability_solution/observability_onboarding/server/routes/flow/route.ts @@ -263,8 +263,8 @@ const createFlowRoute = createObservabilityOnboardingServerRoute({ * * The request format is TSV (tab-separated values) to simplify parsing in bash. * - * The response format is either a YAML file or a tar archive containing the Elastic Agent - * configuration, depending on the `Accept` header. + * The response format is a tar archive containing the Elastic Agent configuration, depending on the + * `Accept` header. * * Errors during installation are ignore unless all integrations fail to install. When that happens * a 500 Internal Server Error is returned with the first error message. @@ -348,7 +348,7 @@ const integrationsInstallRoute = createObservabilityOnboardingServerRoute({ } return acc; }, []); - // Errors during installation are ignore unless all integrations fail to install. When that happens + // Errors during installation are ignored unless all integrations fail to install. When that happens // a 500 Internal Server Error is returned with the first error message. if (!installedIntegrations.length) { throw (settledResults[0] as PromiseRejectedResult).reason; @@ -383,20 +383,11 @@ const integrationsInstallRoute = createObservabilityOnboardingServerRoute({ ? [plugins.cloud?.setup?.elasticsearchUrl] : await getFallbackESUrl(services.esLegacyConfigService); - if (request.headers.accept === 'application/x-tar') { - return response.ok({ - headers: { - 'content-type': 'application/x-tar', - }, - body: generateAgentConfigTar({ elasticsearchUrl, installedIntegrations }), - }); - } - return response.ok({ headers: { - 'content-type': 'application/yaml', + 'content-type': 'application/x-tar', }, - body: generateAgentConfigYAML({ elasticsearchUrl, installedIntegrations }), + body: generateAgentConfigTar({ elasticsearchUrl, installedIntegrations }), }); }, }); @@ -430,7 +421,7 @@ async function ensureInstalledIntegrations( if (installSource === 'registry') { const installation = await packageClient.ensureInstalledPackage({ pkgName }); const pkg = installation.package; - const inputs = await packageClient.getAgentPolicyInputs(pkg.name, pkg.version); + const config = await packageClient.getAgentPolicyConfigYAML(pkg.name, pkg.version); const { packageInfo } = await packageClient.getPackage(pkg.name, pkg.version); return { @@ -438,7 +429,7 @@ async function ensureInstalledIntegrations( pkgName: pkg.name, pkgVersion: pkg.version, title: packageInfo.title, - inputs: inputs.filter((input) => input.type !== 'httpjson'), + config, dataStreams: packageInfo.data_streams?.map(({ type, dataset }) => ({ type, dataset })) ?? [], kibanaAssets: pkg.installed_kibana, @@ -455,19 +446,21 @@ async function ensureInstalledIntegrations( pkgName, pkgVersion: '1.0.0', // Custom integrations are always installed as version `1.0.0` title: pkgName, - inputs: [ - { - id: `filestream-${pkgName}`, - type: 'filestream', - streams: [ - { - id: `filestream-${pkgName}`, - data_stream: dataStream, - paths: integration.logFilePaths, - }, - ], - }, - ], + config: safeDump({ + inputs: [ + { + id: `filestream-${pkgName}`, + type: 'filestream', + streams: [ + { + id: `filestream-${pkgName}`, + data_stream: dataStream, + paths: integration.logFilePaths, + }, + ], + }, + ], + }), dataStreams: [dataStream], kibanaAssets: [], }; @@ -544,25 +537,6 @@ function parseIntegrationsTSV(tsv: string) { ); } -const generateAgentConfigYAML = ({ - elasticsearchUrl, - installedIntegrations, -}: { - elasticsearchUrl: string[]; - installedIntegrations: InstalledIntegration[]; -}) => { - return safeDump({ - outputs: { - default: { - type: 'elasticsearch', - hosts: elasticsearchUrl, - api_key: '${API_KEY}', // Placeholder to be replaced by bash script with the actual API key - }, - }, - inputs: installedIntegrations.map(({ inputs }) => inputs).flat(), - }); -}; - function parseRegistryIntegrationMetadata( pkgName: string, parameter: string @@ -614,7 +588,7 @@ const generateAgentConfigTar = ({ path: `inputs.d/${integration.pkgName}.yml`, mode: 0o644, mtime: now, - data: safeDump({ inputs: integration.inputs }), + data: integration.config, })), ]); }; diff --git a/x-pack/plugins/observability_solution/observability_onboarding/server/routes/types.ts b/x-pack/plugins/observability_solution/observability_onboarding/server/routes/types.ts index a2a2128a5c698..4b35272eaa330 100644 --- a/x-pack/plugins/observability_solution/observability_onboarding/server/routes/types.ts +++ b/x-pack/plugins/observability_solution/observability_onboarding/server/routes/types.ts @@ -58,7 +58,7 @@ export const IntegrationRT = t.intersection([ pkgName: t.string, pkgVersion: t.string, title: t.string, - inputs: t.array(t.unknown), + config: t.string, dataStreams: t.array( t.type({ type: t.string,