Refactor configuration of ClusterLogForwarder (#149)

* Update from template Template version: main (8840f87) * Refactor configuration of ClusterLogForwarder With this commit configuring the ClusterLogForwarder resource will be done with a new parameter `openshift4_logging.clusterLogForwarder`, which will follow the upstream convention / spec. The old parameter `openshift4_logging.clusterLogForwarding` is deprecated, but existing legacy config will be respected. --------- Co-authored-by: commodore <commodore@43197c8cbe55>
appuio · Aug 13, 2024 · 55866f5 · 55866f5
1 parent 07ce1a1
commit 55866f5
Show file tree

Hide file tree

Showing 30 changed files with 1,031 additions and 294 deletions.
diff --git a/.cruft.json b/.cruft.json
@@ -7,7 +7,7 @@
       "name": "OpenShift4 Logging",
       "slug": "openshift4-logging",
       "parameter_key": "openshift4_logging",
-      "test_cases": "defaults master elasticsearch multilineerr forwardingonly",
+      "test_cases": "defaults master elasticsearch multilineerr forwardingonly legacy",
       "add_lib": "n",
       "add_pp": "n",
       "add_golden": "y",

diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml
@@ -38,6 +38,7 @@ jobs:
           - elasticsearch
           - multilineerr
           - forwardingonly
+          - legacy
     defaults:
       run:
         working-directory: ${{ env.COMPONENT_NAME }}
@@ -57,6 +58,7 @@ jobs:
           - elasticsearch
           - multilineerr
           - forwardingonly
+          - legacy
     defaults:
       run:
         working-directory: ${{ env.COMPONENT_NAME }}

diff --git a/Makefile.vars.mk b/Makefile.vars.mk
@@ -57,4 +57,4 @@ KUBENT_IMAGE    ?= ghcr.io/doitintl/kube-no-trouble:latest
 KUBENT_DOCKER   ?= $(DOCKER_CMD) $(DOCKER_ARGS) $(root_volume) --entrypoint=/app/kubent $(KUBENT_IMAGE)
 
 instance ?= defaults
-test_instances = tests/defaults.yml tests/master.yml tests/elasticsearch.yml tests/multilineerr.yml tests/forwardingonly.yml
+test_instances = tests/defaults.yml tests/master.yml tests/elasticsearch.yml tests/multilineerr.yml tests/forwardingonly.yml tests/legacy.yml
diff --git a/class/defaults.yml b/class/defaults.yml
@@ -71,20 +71,7 @@ parameters:
               memory: 128Mi
 
     clusterLogging: {}
-
-    clusterLogForwarding:
-      enabled: false
-      forwarders: {}
-      namespace_groups: {}
-      application_logs: {}
-      audit_logs:
-        enabled: false
-      infrastructure_logs:
-        enabled: true
-      json:
-        enabled: false
-        typekey: 'kubernetes.labels.logFormat'
-        typename: 'nologformat'
+    clusterLogForwarder: {}
 
     operatorResources:
       clusterLogging:

diff --git a/component/config_forwarding.libsonnet b/component/config_forwarding.libsonnet
@@ -1,3 +1,4 @@
+local com = import 'lib/commodore.libjsonnet';
 local kap = import 'lib/kapitan.libjsonnet';
 local lib = import 'lib/openshift4-logging.libsonnet';
 
@@ -8,104 +9,121 @@ local deployLokistack = params.components.lokistack.enabled;
 local deployElasticsearch = params.components.elasticsearch.enabled;
 local forwardingOnly = !deployLokistack && !deployElasticsearch;
 
+// Make sure the default output is added to the pipelines `outputRefs`,
+// if the logging stack is not disabled.
 local pipelineOutputRefs(pipeline) =
   local default = if forwardingOnly then [] else [ 'default' ];
   std.get(pipeline, 'forwarders', []) + default;
 
+// -----------------------------------------------------------------------------
+//   Legacy Rendering
+// -----------------------------------------------------------------------------
+
+local legacyConfig = std.get(params, 'clusterLogForwarding', {});
+local hasLegacyConfig = if std.length(legacyConfig) > 0 then std.trace(
+  'Parameter `clusterLogForwarding` is deprecated. Please update your config to use `clusterLogForwarder`',
+  true
+) else false;
+
 // Apply default config for application logs.
-local patchAppLogDefaults = {
-  local outputRefs = pipelineOutputRefs(params.clusterLogForwarding.application_logs),
-  local enablePipeline = std.length(outputRefs) > 0,
+local patchLegacyAppLogDefaults = {
+  local pipeline = std.get(legacyConfig, 'application_logs', { enabled: true }),
+  local pipelineOutputs = pipelineOutputRefs(pipeline),
+  local pipelineEnabled = std.length(pipelineOutputs) > 0,
 
-  pipelines: {
-    [if enablePipeline then 'application-logs']: {
+  [if hasLegacyConfig then 'pipelines']: {
+    [if pipelineEnabled then 'application-logs']: {
       inputRefs: [ 'application' ],
-      outputRefs: outputRefs,
+      outputRefs: pipelineOutputs,
     },
   },
 };
 
 // Apply default config for infra logs.
-local patchInfraLogDefaults = {
-  local outputRefs = pipelineOutputRefs(params.clusterLogForwarding.infrastructure_logs),
-  local enablePipeline = params.clusterLogForwarding.infrastructure_logs.enabled && std.length(outputRefs) > 0,
+local patchLegacyInfraLogDefaults = {
+  local pipeline = { enabled: true } + std.get(legacyConfig, 'infrastructure_logs', {}),
+  local pipelineOutputs = pipelineOutputRefs(pipeline),
+  local pipelineEnabled = pipeline.enabled && std.length(pipelineOutputs) > 0,
 
-  pipelines: {
-    [if enablePipeline then 'infrastructure-logs']: {
+  [if hasLegacyConfig then 'pipelines']: {
+    [if pipelineEnabled then 'infrastructure-logs']: {
       inputRefs: [ 'infrastructure' ],
-      outputRefs: outputRefs,
+      outputRefs: pipelineOutputs,
     },
   },
 };
 
 // Apply default config for audit logs.
-local patchAuditLogDefaults = {
-  local outputRefs = pipelineOutputRefs(params.clusterLogForwarding.audit_logs),
-  local enablePipeline = params.clusterLogForwarding.audit_logs.enabled && std.length(outputRefs) > 0,
+local patchLegacyAuditLogDefaults = {
+  local pipeline = std.get(legacyConfig, 'audit_logs', { enabled: false }),
+  local pipelineOutputs = pipelineOutputRefs(pipeline),
+  local pipelineEnabled = pipeline.enabled && std.length(pipelineOutputs) > 0,
 
-  pipelines: {
-    [if enablePipeline then 'audit-logs']: {
+  [if hasLegacyConfig then 'pipelines']: {
+    [if pipelineEnabled then 'audit-logs']: {
       inputRefs: [ 'audit' ],
-      outputRefs: outputRefs,
+      outputRefs: pipelineOutputs,
     },
   },
 };
 
 // Enable json parsing for default pipelines if configured.
-local patchJsonLogging = {
-  local enableAppLogs = std.get(params.clusterLogForwarding.application_logs, 'json', false),
-  local enableInfraLogs = std.get(params.clusterLogForwarding.infrastructure_logs, 'json', false),
+local legacyEnableJson = std.get(std.get(legacyConfig, 'json', {}), 'enabled', false);
+local patchLegacyJsonLogging = {
+  local enableAppLogs = std.get(std.get(legacyConfig, 'application_logs', {}), 'json', false),
+  local enableInfraLogs = std.get(std.get(legacyConfig, 'infrastructure_logs', {}), 'json', false),
 
-  pipelines: {
+  [if hasLegacyConfig then 'pipelines']: {
     [if enableAppLogs then 'application-logs']: { parse: 'json' },
     [if enableInfraLogs then 'infrastructure-logs']: { parse: 'json' },
   },
-  [if deployElasticsearch && params.clusterLogForwarding.json.enabled then 'outputDefaults']: {
+  [if deployElasticsearch && legacyEnableJson then 'outputDefaults']: {
     elasticsearch: {
-      structuredTypeKey: params.clusterLogForwarding.json.typekey,
-      structuredTypeName: params.clusterLogForwarding.json.typename,
+      structuredTypeKey: std.get(legacyConfig.json, 'typekey', 'kubernetes.labels.logFormat'),
+      structuredTypeName: std.get(legacyConfig.json, 'typename', 'nologformat'),
     },
   },
 };
 
 // Enable detectMultilineErrors for default pipelines if configured.
-local patchMultilineErrors = {
-  local enableAppLogs = std.get(params.clusterLogForwarding.application_logs, 'detectMultilineErrors', false),
-  local enableInfraLogs = std.get(params.clusterLogForwarding.infrastructure_logs, 'detectMultilineErrors', false),
+local patchLegacyMultilineErrors = {
+  local enableAppLogs = std.get(std.get(legacyConfig, 'application_logs', {}), 'detectMultilineErrors', false),
+  local enableInfraLogs = std.get(std.get(legacyConfig, 'infrastructure_logs', {}), 'detectMultilineErrors', false),
 
-  pipelines: {
+  [if hasLegacyConfig then 'pipelines']: {
     [if enableAppLogs then 'application-logs']: { detectMultilineErrors: true },
     [if enableInfraLogs then 'infrastructure-logs']: { detectMultilineErrors: true },
   },
 };
 
 // --- patch deprecated `clusterLogForwarding.namespace` config
 local namespaceGroups = (
-  if std.objectHas(params.clusterLogForwarding, 'namespaces') then
+  if std.objectHas(legacyConfig, 'namespaces') then
     {
       [ns]: {
         namespaces: [ ns ],
-        forwarders: [ params.clusterLogForwarding.namespaces[ns].forwarder ],
+        forwarders: [ legacyConfig.namespaces[ns].forwarder ],
       }
-      for ns in std.objectFields(params.clusterLogForwarding.namespaces)
+      for ns in std.objectFields(legacyConfig.namespaces)
     } else {}
-) + params.clusterLogForwarding.namespace_groups;
+) + std.get(legacyConfig, 'namespace_groups', {});
 // --- patch end
 
 // Add inputs entry for every namespace_group defined in `clusterLogForwarding.namespace_groups`.
-local patchCustomInputs = {
+local patchLegacyCustomInputs = {
   [if std.length(namespaceGroups) > 0 then 'inputs']: {
     [group]: {
       application: {
         namespaces: namespaceGroups[group].namespaces,
       },
     }
     for group in std.objectFields(namespaceGroups)
+    if hasLegacyConfig
   },
 };
 
 // Add pipelines entry for every namespace_group defined in `clusterLogForwarding.namespace_groups`.
-local patchCustomPipelines = {
+local patchLegacyCustomPipelines = {
   [if std.length(namespaceGroups) > 0 then 'pipelines']: {
     local enableJson = std.get(namespaceGroups[group], 'json', false),
     local enableMultilineError = std.get(namespaceGroups[group], 'detectMultilineErrors', false),
@@ -117,39 +135,68 @@ local patchCustomPipelines = {
       [if enableMultilineError then 'detectMultilineErrors']: true,
     }
     for group in std.objectFields(namespaceGroups)
+    if hasLegacyConfig
   },
 };
 
 // Add outputs entry for every forwarder defined in `clusterLogForwarding.forwarders`.
-local patchCustomOutputs = {
-  [if std.length(params.clusterLogForwarding.forwarders) > 0 then 'outputs']: {
-    [name]: params.clusterLogForwarding.forwarders[name]
-    for name in std.objectFields(params.clusterLogForwarding.forwarders)
+local patchLegacyCustomOutputs = {
+  [if std.length(std.get(legacyConfig, 'forwarders', {})) > 0 then 'outputs']: {
+    [name]: legacyConfig.forwarders[name]
+    for name in std.objectFields(legacyConfig.forwarders)
+    if hasLegacyConfig
+  },
+};
+
+// -----------------------------------------------------------------------------
+//   End Legacy Rendering
+// -----------------------------------------------------------------------------
+
+// Add defaults to pipelines config
+local patchPipelineDefaults = {
+  local appsPipeline = std.get(std.get(params.clusterLogForwarder, 'pipelines', {}), 'application-logs', {}),
+  local infraPipeline = std.get(std.get(params.clusterLogForwarder, 'pipelines', {}), 'infrastructure-logs', {}),
+  local auditPipeline = std.get(std.get(params.clusterLogForwarder, 'pipelines', {}), 'audit-logs', {}),
+
+  pipelines: {
+    'application-logs': {
+      inputRefs: [ 'application' ],
+      outputRefs: pipelineOutputRefs(appsPipeline),
+    },
+    'infrastructure-logs': {
+      inputRefs: [ 'infrastructure' ],
+      outputRefs: pipelineOutputRefs(infraPipeline),
+    },
+    [if std.length(auditPipeline) > 0 then 'audit-logs']: {
+      inputRefs: [ 'audit' ],
+    },
   },
 };
 
-// ClusterLogForwarderSpecs:
+// clusterLogForwarderSpec:
 // Consecutively apply patches to result of previous apply.
 local clusterLogForwarderSpec = std.foldl(
   // we use std.mergePatch here, because this way we don't need
   // to make each patch object mergeable by suffixing all keys with a +.
   function(manifest, patch) std.mergePatch(manifest, patch),
   [
-    patchAppLogDefaults,
-    patchInfraLogDefaults,
-    patchAuditLogDefaults,
-    patchJsonLogging,
-    patchMultilineErrors,
-    patchCustomInputs,
-    patchCustomOutputs,
-    patchCustomPipelines,
+    patchPipelineDefaults,
+    // Apply legacy patches / defaults
+    patchLegacyAppLogDefaults,
+    patchLegacyInfraLogDefaults,
+    patchLegacyAuditLogDefaults,
+    patchLegacyJsonLogging,
+    patchLegacyMultilineErrors,
+    patchLegacyCustomInputs,
+    patchLegacyCustomOutputs,
+    patchLegacyCustomPipelines,
   ],
   {
     inputs: {},
     outputs: {},
     pipelines: {},
-  }
-);
+  },
+) + com.makeMergeable(params.clusterLogForwarder);
 
 // ClusterLogForwarder:
 // Create definitive ClusterLogForwarder resource from specs.
@@ -176,8 +223,10 @@ local clusterLogForwarder = lib.ClusterLogForwarder(params.namespace, 'instance'
   },
 };
 
+local enableLogForwarder = std.length(params.clusterLogForwarder) > 0 || std.get(legacyConfig, 'enabled', false);
+
 // Define outputs below
-if params.clusterLogForwarding.enabled then
+if enableLogForwarder then
   {
     '31_cluster_logforwarding': clusterLogForwarder,
   }

diff --git a/docs/modules/ROOT/pages/how-tos/upgrade-v3.x-v4.x.adoc b/docs/modules/ROOT/pages/how-tos/upgrade-v3.x-v4.x.adoc
@@ -0,0 +1,6 @@
+= Upgrade from v3.x to v4.x
+
+The parameter `clusterLogForwarding` is deprecated.
+The component is backwards compatible, but moving the parameters to `clusterLogForwarder` is highly encouraged.
+
+See xref:references/parameters.adoc#_examples[Examples] for reference.