[8.x] [Observability Onboarding] Display next steps (#197179) (#200139)

# Backport

This will backport the following commits from `main` to `8.x`:
- [[Observability Onboarding] Display next steps
(#197179)](#197179)

<!--- Backport version: 8.9.8 -->

### Questions ?
Please refer to the [Backport tool
documentation](https://github.com/sqren/backport)

<!--BACKPORT [{"author":{"name":"Thom
Heymann","email":"[email protected]"},"sourceCommit":{"committedDate":"2024-10-24T14:43:01Z","message":"[Observability
Onboarding] Display next steps (#197179)\n\n## Summary\r\n\r\nDisplay
next steps after the auto-detect script completes.\r\n\r\n##
Screenshot\r\n\r\n<img width=\"905\" alt=\"Screenshot 2024-10-22 at 09
30
51\"\r\nsrc=\"https://github.com/user-attachments/assets/85108ea2-510a-46f7-8b79-135be91b586f\">","sha":"48959e769cb29b02e8c49d68fb2c7f9f8c3418d0","branchLabelMapping":{"^v9.0.0$":"main","^v8.17.0$":"8.x","^v(\\d+).(\\d+).\\d+$":"$1.$2"}},"sourcePullRequest":{"labels":["release_note:skip","backport
missing","Team:Fleet","v9.0.0","backport:prev-minor","ci:project-deploy-observability"],"number":197179,"url":"https://github.com/elastic/kibana/pull/197179","mergeCommit":{"message":"[Observability
Onboarding] Display next steps (#197179)\n\n## Summary\r\n\r\nDisplay
next steps after the auto-detect script completes.\r\n\r\n##
Screenshot\r\n\r\n<img width=\"905\" alt=\"Screenshot 2024-10-22 at 09
30
51\"\r\nsrc=\"https://github.com/user-attachments/assets/85108ea2-510a-46f7-8b79-135be91b586f\">","sha":"48959e769cb29b02e8c49d68fb2c7f9f8c3418d0"}},"sourceBranch":"main","suggestedTargetBranches":[],"targetPullRequestStates":[{"branch":"main","label":"v9.0.0","labelRegex":"^v9.0.0$","isSourceBranch":true,"state":"MERGED","url":"https://github.com/elastic/kibana/pull/197179","number":197179,"mergeCommit":{"message":"[Observability
Onboarding] Display next steps (#197179)\n\n## Summary\r\n\r\nDisplay
next steps after the auto-detect script completes.\r\n\r\n##
Screenshot\r\n\r\n<img width=\"905\" alt=\"Screenshot 2024-10-22 at 09
30
51\"\r\nsrc=\"https://github.com/user-attachments/assets/85108ea2-510a-46f7-8b79-135be91b586f\">","sha":"48959e769cb29b02e8c49d68fb2c7f9f8c3418d0"}},{"url":"https://github.com/elastic/kibana/pull/198267","number":198267,"branch":"8.x","state":"OPEN"}]}]
BACKPORT-->

x-pack/plugins/fleet/server/services/epm/package_service.mock.ts

@@ -16,7 +16,7 @@ const createClientMock = (): jest.Mocked<PackageClient> => ({
   installCustomIntegration: jest.fn(),
   fetchFindLatestPackage: jest.fn(),
   readBundledPackage: jest.fn(),
-  getAgentPolicyInputs: jest.fn(),
+  getAgentPolicyConfigYAML: jest.fn(),
   getPackage: jest.fn(),
   getPackageFieldsMetadata: jest.fn(),
   getPackages: jest.fn(),

x-pack/plugins/fleet/server/services/epm/package_service.ts

@@ -28,7 +28,6 @@ import type {
   InstallablePackage,
   Installation,
   RegistryPackage,
-  TemplateAgentPolicyInput,
 } from '../../types';
 
 import type { FleetAuthzRouteConfig } from '../security/types';
@@ -120,12 +119,12 @@ export interface PackageClient {
     prerelease?: false;
   }): Promise<PackageList>;
 
-  getAgentPolicyInputs(
+  getAgentPolicyConfigYAML(
     pkgName: string,
     pkgVersion?: string,
     prerelease?: false,
     ignoreUnverified?: boolean
-  ): Promise<TemplateAgentPolicyInput[]>;
+  ): Promise<string>;
 
   reinstallEsAssets(
     packageInfo: InstallablePackage,
@@ -288,7 +287,7 @@ class PackageClientImpl implements PackageClient {
     return generatePackageInfoFromArchiveBuffer(archiveBuffer, 'application/zip');
   }
 
-  public async getAgentPolicyInputs(
+  public async getAgentPolicyConfigYAML(
     pkgName: string,
     pkgVersion?: string,
     prerelease?: false,
@@ -302,16 +301,14 @@ class PackageClientImpl implements PackageClient {
       pkgVersion = pkg.version;
     }
 
-    const { inputs } = await getTemplateInputs(
+    return getTemplateInputs(
       this.internalSoClient,
       pkgName,
       pkgVersion,
-      'json',
+      'yml',
       prerelease,
       ignoreUnverified
     );
-
-    return inputs;
   }
 
   public async getPackage(

x-pack/plugins/observability_solution/observability_onboarding/public/assets/auto_detect.sh

@@ -105,6 +105,7 @@ elastic_agent_config_path="/opt/Elastic/Agent/elastic-agent.yml"
 elastic_agent_tmp_config_path="/tmp/elastic-agent-config.tar"
 integration_names=()
 integration_titles=()
+config_files_with_password=()
 
 OS="$(uname)"
 ARCH="$(uname -m)"
@@ -155,7 +156,7 @@ download_elastic_agent() {
   curl -L -O "$download_url" --silent --fail
 
   if [ "$?" -eq 0 ]; then
-    printf "\e[1;32m✓\e[0m %s\n" "Elastic Agent downloaded to $(pwd)/$elastic_agent_artifact_name.tar.gz"
+    printf "\e[32;1m✓\e[0m %s\n" "Elastic Agent downloaded to $(pwd)/$elastic_agent_artifact_name.tar.gz"
     update_step_progress "ea-download" "complete"
   else
     update_step_progress "ea-download" "danger" "Failed to download Elastic Agent, see script output for error."
@@ -167,7 +168,7 @@ extract_elastic_agent() {
   tar -xzf "${elastic_agent_artifact_name}.tar.gz"
 
   if [ "$?" -eq 0 ]; then
-    printf "\e[1;32m✓\e[0m %s\n" "Archive extracted"
+    printf "\e[32;1m✓\e[0m %s\n" "Archive extracted"
     update_step_progress "ea-extract" "complete"
   else
     update_step_progress "ea-extract" "danger" "Failed to extract Elastic Agent, see script output for error."
@@ -179,7 +180,7 @@ install_elastic_agent() {
   "./${elastic_agent_artifact_name}/elastic-agent" install -f -n >/dev/null
 
   if [ "$?" -eq 0 ]; then
-    printf "\e[1;32m✓\e[0m %s\n" "Elastic Agent installed to $(dirname "$elastic_agent_config_path")"
+    printf "\e[32;1m✓\e[0m %s\n" "Elastic Agent installed to $(dirname "$elastic_agent_config_path")"
     update_step_progress "ea-install" "complete"
   else
     update_step_progress "ea-install" "danger" "Failed to install Elastic Agent, see script output for error."
@@ -224,7 +225,7 @@ ensure_elastic_agent_healthy() {
 
 backup_elastic_agent_config() {
   if [ -f "$elastic_agent_config_path" ]; then
-    echo -e "\nExisting config found at $elastic_agent_config_path"
+    printf "\n%s \e[36m%s\e[0m\n" "Existing config found at" "$elastic_agent_config_path"
 
     printf "\n\e[1;36m?\e[0m \e[1m%s\e[0m \e[2m%s\e[0m" "Create backup and continue installation?" "[Y/n] (default: Yes): "
     read confirmation_reply
@@ -241,7 +242,7 @@ backup_elastic_agent_config() {
       fi
 
       if [ "$?" -eq 0 ]; then
-        printf "\n\e[1;32m✓\e[0m %s\n" "Backup saved to $backup_path"
+        printf "\n\e[32;1m✓\e[0m %s \e[36m%s\e[0m\n" "Backup saved to" "$backup_path"
       else
         update_step_progress "ea-config" "warning" "Failed to backup existing configuration"
         fail "Failed to backup existing config - Try manually creating a backup or delete your existing config before re-running this script"
@@ -286,7 +287,7 @@ install_integrations() {
     --output "$elastic_agent_tmp_config_path"
 
   if [ "$?" -eq 0 ]; then
-    printf "\n\e[1;32m✓\e[0m %s\n" "Integrations installed"
+    printf "\n\e[32;1m✓\e[0m %s\n" "Integrations installed"
   else
     update_step_progress "ea-config" "warning" "Failed to install integrations"
     fail "Failed to install integrations"
@@ -305,10 +306,15 @@ apply_elastic_agent_config() {
     # Replace placeholder with the Ingest API key
     sed -i='' "s/\${API_KEY}/$decoded_ingest_api_key/" "$elastic_agent_config_path"
   if [ "$?" -eq 0 ]; then
-    printf "\e[1;32m✓\e[0m %s\n" "Config written to:"
-    tar --list --file "$elastic_agent_tmp_config_path" | grep '\.yml$' | while read -r file; do
-      echo "  - $(dirname "$elastic_agent_config_path")/$file"
-    done
+    printf "\e[32;1m✓\e[0m %s\n" "Config files written to:"
+    while IFS= read -r file; do
+      local path="$(dirname "$elastic_agent_config_path")/$file"
+      printf "  \e[36m%s\e[0m\n" "$path"
+      grep '<PASSWORD>' "$path" >/dev/null
+      if [ "$?" -eq 0 ]; then
+        config_files_with_password+=("$path")
+      fi
+    done < <(tar --list --file "$elastic_agent_tmp_config_path" | grep '\.yml$')
 
     update_step_progress "ea-config" "complete"
   else
@@ -610,4 +616,11 @@ printf "\n\e[1m%s\e[0m\n" "Waiting for healthy status..."
 wait_for_elastic_agent_status
 ensure_elastic_agent_healthy
 
-printf "\n\e[32m%s\e[0m\n" "🎉 Elastic Agent is configured and running. You can now go back to Kibana and check for incoming logs."
+printf "\n\e[32m%s\e[0m\n" "🎉 Elastic Agent is configured and running!"
+
+printf "\n\e[1m%s\e[0m\n" "Next steps:"
+printf "\n• %s\n" "Go back to Kibana and check for incoming data"
+for path in "${config_files_with_password[@]}"; do
+  printf "\n• %s:\n  \e[36m%s\e[0m\n" "Collect $(known_integration_title "$(basename "${path%.yml}")") metrics by adding your username and password to" "$path"
+done
+printf "\n• %s:\n  \e[36;4m%s\e[0m\n" "For information on other standalone integration setups, visit" "https://www.elastic.co/guide/en/fleet/current/elastic-agent-configuration.html"

x-pack/plugins/observability_solution/observability_onboarding/public/assets/integrations.conf

@@ -1,14 +1,14 @@
 [system]
-title=System Logs And Metrics
+title=System
 
 [nginx]
-title=Nginx Logs
+title=Nginx
 patterns=
     /var/log/nginx/access.log*
     /var/log/nginx/error.log*
 
 [apache]
-title=Apache Logs
+title=Apache
 patterns=
     /var/log/apache2/access.log*
     /var/log/apache2/other_vhosts_access.log*
@@ -17,48 +17,48 @@ patterns=
     /var/log/httpd/error_log*
 
 [docker]
-title=Docker Container Logs
+title=Docker
 patterns=
     /var/lib/docker/containers/*/*-json.log
     /var/run/docker.sock
 
 [mysql]
-title=MySQL Logs
+title=MySQL
 patterns=
     /var/log/mysql/*error.log*
     /var/log/mysqld.log*
     /var/log/mysql/*-slow.log*
     /var/lib/mysql/*-slow.log*
 
 [postgresql]
-title=PostgreSQL Logs
+title=PostgreSQL
 patterns=
     /var/log/postgresql/postgresql-*-*.log*
     /var/log/postgresql/postgresql-*-*.csv*
     /*/postgresql-logs/*.log
     /etc/postgresql/*/main/postgresql.conf
 
 [redis]
-title=Redis Logs
+title=Redis
 patterns=
     /var/log/redis/redis-server.log*
     /etc/redis/redis.conf
 
 [haproxy]
-title=HAProxy Logs
+title=HAProxy
 patterns=
     /var/log/haproxy.log
     /etc/haproxy/haproxy.cfg
 
 [rabbitmq]
-title=RabbitMQ Logs
+title=RabbitMQ
 patterns=
     /var/log/rabbitmq/rabbit@*.log
     /etc/rabbitmq/rabbitmq.conf
     /etc/rabbitmq/rabbitmq.config
 
 [kafka]
-title=Kafka Logs
+title=Kafka
 patterns=
     /var/log/kafka/server.log
     /etc/kafka/server.properties
@@ -68,19 +68,19 @@ patterns=
     /*/logs/kafka-*.log*
 
 [mongodb]
-title=MongoDB Logs
+title=MongoDB
 patterns=
     /var/log/mongodb/mongod.log
 
 [apache_tomcat]
-title=Apache Tomcat Logs
+title=Apache Tomcat
 patterns=
     /opt/tomcat/logs/localhost_access_log.*.txt
     /opt/tomcat/logs/catalina.*.log
     /opt/tomcat/logs/localhost.*.log
 
 [prometheus]
-title=Prometheus Server overview
+title=Prometheus
 patterns=
     /var/log/prometheus/prometheus.log
     /etc/prometheus/prometheus.yml

x-pack/plugins/observability_solution/observability_onboarding/server/routes/flow/route.ts

@@ -263,8 +263,8 @@ const createFlowRoute = createObservabilityOnboardingServerRoute({
  *
  * The request format is TSV (tab-separated values) to simplify parsing in bash.
  *
- * The response format is either a YAML file or a tar archive containing the Elastic Agent
- * configuration, depending on the `Accept` header.
+ * The response format is a tar archive containing the Elastic Agent configuration, depending on the
+ * `Accept` header.
  *
  * Errors during installation are ignore unless all integrations fail to install. When that happens
  * a 500 Internal Server Error is returned with the first error message.
@@ -348,7 +348,7 @@ const integrationsInstallRoute = createObservabilityOnboardingServerRoute({
         }
         return acc;
       }, []);
-      // Errors during installation are ignore unless all integrations fail to install. When that happens
+      // Errors during installation are ignored unless all integrations fail to install. When that happens
       // a 500 Internal Server Error is returned with the first error message.
       if (!installedIntegrations.length) {
         throw (settledResults[0] as PromiseRejectedResult).reason;
@@ -383,20 +383,11 @@ const integrationsInstallRoute = createObservabilityOnboardingServerRoute({
       ? [plugins.cloud?.setup?.elasticsearchUrl]
       : await getFallbackESUrl(services.esLegacyConfigService);
 
-    if (request.headers.accept === 'application/x-tar') {
-      return response.ok({
-        headers: {
-          'content-type': 'application/x-tar',
-        },
-        body: generateAgentConfigTar({ elasticsearchUrl, installedIntegrations }),
-      });
-    }
-
     return response.ok({
       headers: {
-        'content-type': 'application/yaml',
+        'content-type': 'application/x-tar',
       },
-      body: generateAgentConfigYAML({ elasticsearchUrl, installedIntegrations }),
+      body: generateAgentConfigTar({ elasticsearchUrl, installedIntegrations }),
     });
   },
 });
@@ -430,15 +421,15 @@ async function ensureInstalledIntegrations(
       if (installSource === 'registry') {
         const installation = await packageClient.ensureInstalledPackage({ pkgName });
         const pkg = installation.package;
-        const inputs = await packageClient.getAgentPolicyInputs(pkg.name, pkg.version);
+        const config = await packageClient.getAgentPolicyConfigYAML(pkg.name, pkg.version);
         const { packageInfo } = await packageClient.getPackage(pkg.name, pkg.version);
 
         return {
           installSource,
           pkgName: pkg.name,
           pkgVersion: pkg.version,
           title: packageInfo.title,
-          inputs: inputs.filter((input) => input.type !== 'httpjson'),
+          config,
           dataStreams:
             packageInfo.data_streams?.map(({ type, dataset }) => ({ type, dataset })) ?? [],
           kibanaAssets: pkg.installed_kibana,
@@ -455,19 +446,21 @@ async function ensureInstalledIntegrations(
         pkgName,
         pkgVersion: '1.0.0', // Custom integrations are always installed as version `1.0.0`
         title: pkgName,
-        inputs: [
-          {
-            id: `filestream-${pkgName}`,
-            type: 'filestream',
-            streams: [
-              {
-                id: `filestream-${pkgName}`,
-                data_stream: dataStream,
-                paths: integration.logFilePaths,
-              },
-            ],
-          },
-        ],
+        config: safeDump({
+          inputs: [
+            {
+              id: `filestream-${pkgName}`,
+              type: 'filestream',
+              streams: [
+                {
+                  id: `filestream-${pkgName}`,
+                  data_stream: dataStream,
+                  paths: integration.logFilePaths,
+                },
+              ],
+            },
+          ],
+        }),
         dataStreams: [dataStream],
         kibanaAssets: [],
       };
@@ -544,25 +537,6 @@ function parseIntegrationsTSV(tsv: string) {
   );
 }
 
-const generateAgentConfigYAML = ({
-  elasticsearchUrl,
-  installedIntegrations,
-}: {
-  elasticsearchUrl: string[];
-  installedIntegrations: InstalledIntegration[];
-}) => {
-  return safeDump({
-    outputs: {
-      default: {
-        type: 'elasticsearch',
-        hosts: elasticsearchUrl,
-        api_key: '${API_KEY}', // Placeholder to be replaced by bash script with the actual API key
-      },
-    },
-    inputs: installedIntegrations.map(({ inputs }) => inputs).flat(),
-  });
-};
-
 function parseRegistryIntegrationMetadata(
   pkgName: string,
   parameter: string
@@ -614,7 +588,7 @@ const generateAgentConfigTar = ({
       path: `inputs.d/${integration.pkgName}.yml`,
       mode: 0o644,
       mtime: now,
-      data: safeDump({ inputs: integration.inputs }),
+      data: integration.config,
     })),
   ]);
 };

x-pack/plugins/observability_solution/observability_onboarding/server/routes/types.ts

@@ -58,7 +58,7 @@ export const IntegrationRT = t.intersection([
     pkgName: t.string,
     pkgVersion: t.string,
     title: t.string,
-    inputs: t.array(t.unknown),
+    config: t.string,
     dataStreams: t.array(
       t.type({
         type: t.string,