From 9a8a65e455570a68aca7e44bf0a52179c3898023 Mon Sep 17 00:00:00 2001 From: Stephan Feurer Date: Sun, 11 Aug 2024 14:59:20 +0200 Subject: [PATCH] Support multi-log-forwarder Supports enabling namespaced ClusterLogForwarder resources. Enabling the feature requires redeploying the Cluster Logging Operator. --- class/defaults.yml | 3 + component/config_forwarding.libsonnet | 111 ++++++++++++++---- component/main.jsonnet | 2 +- component/utils.libsonnet | 7 ++ .../pages/how-tos/enable-multi-forwarder.adoc | 70 +++++++++++ .../ROOT/pages/references/parameters.adoc | 43 +++++++ docs/modules/ROOT/partials/nav.adoc | 1 + .../openshift4-logging/10_operator_group.yaml | 4 +- .../32_namespace_logforwarding.yaml | 48 ++++++++ .../32_namespace_rolebinding.yaml | 33 ++++++ .../32_namespace_serviceaccount.yaml | 17 +++ tests/master.yml | 33 ++++++ 12 files changed, 348 insertions(+), 24 deletions(-) create mode 100644 docs/modules/ROOT/pages/how-tos/enable-multi-forwarder.adoc create mode 100644 tests/golden/master/openshift4-logging/openshift4-logging/32_namespace_logforwarding.yaml create mode 100644 tests/golden/master/openshift4-logging/openshift4-logging/32_namespace_rolebinding.yaml create mode 100644 tests/golden/master/openshift4-logging/openshift4-logging/32_namespace_serviceaccount.yaml diff --git a/class/defaults.yml b/class/defaults.yml index 50cbc84..e7f48e6 100644 --- a/class/defaults.yml +++ b/class/defaults.yml @@ -73,6 +73,9 @@ parameters: clusterLogging: {} clusterLogForwarder: {} + namespaceLogForwarderEnabled: false + namespaceLogForwarder: {} + operatorResources: clusterLogging: requests: diff --git a/component/config_forwarding.libsonnet b/component/config_forwarding.libsonnet index 201d968..ca9a14e 100644 --- a/component/config_forwarding.libsonnet +++ b/component/config_forwarding.libsonnet @@ -1,6 +1,8 @@ local com = import 'lib/commodore.libjsonnet'; local kap = import 'lib/kapitan.libjsonnet'; +local kube = import 'lib/kube.libjsonnet'; local lib = import 'lib/openshift4-logging.libsonnet'; +local utils = import 'utils.libsonnet'; local inv = kap.inventory(); local params = inv.parameters.openshift4_logging; @@ -198,37 +200,106 @@ local clusterLogForwarderSpec = std.foldl( }, ) + com.makeMergeable(params.clusterLogForwarder); +// Unfold objects into array for ClusterLogForwarder resource. +local unfoldSpecs(specs) = { + // Unfold objects into array. + [if std.length(specs.inputs) > 0 then 'inputs']: [ + { name: name } + specs.inputs[name] + for name in std.objectFields(specs.inputs) + ], + [if std.length(specs.outputs) > 0 then 'outputs']: [ + { name: name } + specs.outputs[name] + for name in std.objectFields(specs.outputs) + ], + [if std.length(specs.pipelines) > 0 then 'pipelines']: [ + { name: name } + specs.pipelines[name] + for name in std.objectFields(specs.pipelines) + ], +} + { + // Import remaining specs as is. + [key]: specs[key] + for key in std.objectFields(specs) + if !std.member([ 'inputs', 'outputs', 'pipelines' ], key) +}; + // ClusterLogForwarder: // Create definitive ClusterLogForwarder resource from specs. local clusterLogForwarder = lib.ClusterLogForwarder(params.namespace, 'instance') { - spec: { - // Unfold objects into array. - [if std.length(clusterLogForwarderSpec.inputs) > 0 then 'inputs']: [ - { name: name } + clusterLogForwarderSpec.inputs[name] - for name in std.objectFields(clusterLogForwarderSpec.inputs) - ], - [if std.length(clusterLogForwarderSpec.outputs) > 0 then 'outputs']: [ - { name: name } + clusterLogForwarderSpec.outputs[name] - for name in std.objectFields(clusterLogForwarderSpec.outputs) - ], - [if std.length(clusterLogForwarderSpec.pipelines) > 0 then 'pipelines']: [ - { name: name } + clusterLogForwarderSpec.pipelines[name] - for name in std.objectFields(clusterLogForwarderSpec.pipelines) - ], - } + { - // Import remaining specs as is. - [key]: clusterLogForwarderSpec[key] - for key in std.objectFields(clusterLogForwarderSpec) - if !std.member([ 'inputs', 'outputs', 'pipelines' ], key) - }, + spec: unfoldSpecs(clusterLogForwarderSpec), }; +// namespaceLogForwarderIgnoreKeys +// List of keys to ignore in namespaceLogForwarder +local namespaceLogForwarderIgnoreKeys = [ + 'instance', + 'openshift-logging/instance', +]; +// namespaceLogForwarder: +// Create namespaced LogForwarder resource from specs. +local namespaceLogForwarder = [ + local specs = { inputs: {}, outputs: {}, pipelines: {} } + com.makeMergeable(params.namespaceLogForwarder[forwarder]); + local name = utils.namespacedName(forwarder).name; + local namespace = utils.namespacedName(forwarder).namespace; + local serviceAccount = std.get(specs, 'serviceAccountName', utils.namespacedName(forwarder).name); + + lib.ClusterLogForwarder(namespace, name) { + spec: { serviceAccountName: serviceAccount } + com.makeMergeable(unfoldSpecs(specs)), + } + for forwarder in std.objectFields(params.namespaceLogForwarder) + if !std.member(namespaceLogForwarderIgnoreKeys, forwarder) +]; + +// namespaceServiceAccount: +// Create ServiceAccount for namespaced LogForwarder specs. +local namespaceServiceAccount = [ + local specs = params.namespaceLogForwarder[forwarder]; + local namespace = utils.namespacedName(forwarder).namespace; + local serviceAccount = std.get(specs, 'serviceAccountName', utils.namespacedName(forwarder).name); + + kube.ServiceAccount(serviceAccount) { + metadata+: { + namespace: namespace, + }, + } + for forwarder in std.objectFields(params.namespaceLogForwarder) + if !std.member(namespaceLogForwarderIgnoreKeys, forwarder) +]; + +// namespaceRoleBinding: +// Create RoleBinding for namespaced LogForwarder. +local namespaceRoleBinding = [ + local specs = params.namespaceLogForwarder[forwarder]; + local namespace = utils.namespacedName(forwarder).namespace; + local serviceAccount = std.get(specs, 'serviceAccountName', utils.namespacedName(forwarder).name); + + kube.RoleBinding(serviceAccount) { + metadata+: { + namespace: namespace, + }, + roleRef: { + apiGroup: 'rbac.authorization.k8s.io', + kind: 'ClusterRole', + name: 'collect-application-logs', + }, + subjects: [ { + kind: 'ServiceAccount', + name: serviceAccount, + namespace: namespace, + } ], + } + for forwarder in std.objectFields(params.namespaceLogForwarder) + if !std.member(namespaceLogForwarderIgnoreKeys, forwarder) +]; + local enableLogForwarder = std.length(params.clusterLogForwarder) > 0 || std.get(legacyConfig, 'enabled', false); // Define outputs below if enableLogForwarder then { '31_cluster_logforwarding': clusterLogForwarder, + [if std.length(params.namespaceLogForwarder) > 1 then '32_namespace_logforwarding']: namespaceLogForwarder, + [if std.length(params.namespaceLogForwarder) > 1 then '32_namespace_serviceaccount']: namespaceServiceAccount, + [if std.length(params.namespaceLogForwarder) > 1 then '32_namespace_rolebinding']: namespaceRoleBinding, } else std.trace( diff --git a/component/main.jsonnet b/component/main.jsonnet index 54ee15c..bed6382 100644 --- a/component/main.jsonnet +++ b/component/main.jsonnet @@ -30,7 +30,7 @@ local operatorGroup = operatorlib.OperatorGroup('cluster-logging') { namespace: params.namespace, }, spec: { - targetNamespaces: [ + [if !params.namespaceLogForwarderEnabled then 'targetNamespaces']: [ params.namespace, ], }, diff --git a/component/utils.libsonnet b/component/utils.libsonnet index 3475786..2a6ccad 100644 --- a/component/utils.libsonnet +++ b/component/utils.libsonnet @@ -20,7 +20,14 @@ local isVersion59 = else if std.parseInt(major) == 5 && std.parseInt(minor) >= 9 then true else false; +local namespacedName(name) = { + local namespaced = std.splitLimit(name, '/', 1), + namespace: if std.length(namespaced) > 1 then namespaced[0] else params.namespace, + name: if std.length(namespaced) > 1 then namespaced[1] else namespaced[0], +}; + { isVersion58: isVersion58, isVersion59: isVersion59, + namespacedName: namespacedName, } diff --git a/docs/modules/ROOT/pages/how-tos/enable-multi-forwarder.adoc b/docs/modules/ROOT/pages/how-tos/enable-multi-forwarder.adoc new file mode 100644 index 0000000..d3ee23e --- /dev/null +++ b/docs/modules/ROOT/pages/how-tos/enable-multi-forwarder.adoc @@ -0,0 +1,70 @@ += Enable Multi LogForwarder + +Red Hat OpenShift Logging Operator only watches the `openshift-logging` namespace. +If you want the Red Hat OpenShift Logging Operator to watch all namespaces on your cluster, you must redeploy the Operator. +You can complete the following procedure to redeploy the Operator without deleting your logging components. + + +== Disable ArgoCD sync + +Disable ArgoCD sync of component-openshift4-logging: +[source,bash] +---- +kubectl --as=cluster-admin -n syn patch apps root --type=json \ + -p '[{"op":"replace", "path":"/spec/syncPolicy", "value": {}}]' +kubectl --as=cluster-admin -n syn patch apps openshift4-logging --type=json \ + -p '[{"op":"replace", "path":"/spec/syncPolicy", "value": {}}]' +---- + +== Remove Cluster Logging Opeartor Group + +1. Remove Subscription: ++ +[source,bash] +---- +kubectl --as=cluster-admin -n openshift-logging delete sub cluster-logging +---- + +1. Remove OperatorGroup: ++ +[source,bash] +---- +kubectl --as=cluster-admin -n openshift-logging delete og cluster-logging +---- + +1. Remove ClusterServiceVersion: ++ +[source,bash] +---- +kubectl --as=cluster-admin -n openshift-logging delete csv -l operators.coreos.com/cluster-logging.openshift-logging= +---- + +== Enable namespaced LogForwarding + +1. Enable the following parameter in the tenant repo: ++ +[source,bash] +---- +parameters: + openshift4_logging: + namespaceLogForwarder: + enabled: true +---- + +1. Compile and push catalog + + +== Enable ArgoCD sync + +NOTE: Make sure ArgoCD is refreshed before enabling the sync again. + +Enable ArgoCD sync of component-openshift4-logging: +[source,bash] +---- +kubectl --as=cluster-admin -n syn patch apps root --type=json \ + -p '[{ + "op":"replace", + "path":"/spec/syncPolicy", + "value": {"automated": {"prune": true, "selfHeal": true}} + }]' +---- diff --git a/docs/modules/ROOT/pages/references/parameters.adoc b/docs/modules/ROOT/pages/references/parameters.adoc index ff62c25..568d1ee 100644 --- a/docs/modules/ROOT/pages/references/parameters.adoc +++ b/docs/modules/ROOT/pages/references/parameters.adoc @@ -368,6 +368,26 @@ See the https://docs.openshift.com/container-platform/latest/observability/loggi IMPORTANT: `clusterLogForwarding` is deprecated, please use `clusterLogForwarder` +== `namespaceLogForwarderEnabled` + +[horizontal] +type:: bool +default:: false + +NOTE: Enabling namespaced log forwarding requires redeploying the logging operator. See xref:how-tos/enable-multi-forwarder.adoc[How-To] for instructions. + + +== `namespaceLogForwarder` + +[horizontal] +type:: dictionary +default:: {} + +A dictionary holding the `.spec` for namespaced log forwarding. + +See in examples below for configuration. + + == Examples [source,yaml] @@ -381,6 +401,29 @@ clusterLogging: nodeCount: 5 ---- +=== Use namespaced ClusterLogForwarder + +Example creates a `ClusterLogForwarder`, `ServiceAccount` and `RoleBinding` in namespace `my-namespace`. + +[source,yaml] +---- +namespaceLogForwarderEnabled: true +namespaceLogForwarder: + my-namespace/my-forwarder: + outputs: + splunk-forwarder: + secret: + name: splunk-forwarder + type: fluentdForward + url: tls://splunk-forwarder:24224 + pipelines: + application-logs: + inputRefs: + - application + outputRefs: + - splunk-forwarder +---- + === Forward logs for all application logs to third-party [source,yaml] diff --git a/docs/modules/ROOT/partials/nav.adoc b/docs/modules/ROOT/partials/nav.adoc index f527278..36f5603 100644 --- a/docs/modules/ROOT/partials/nav.adoc +++ b/docs/modules/ROOT/partials/nav.adoc @@ -9,6 +9,7 @@ * xref:how-tos/upgrade-v2.x-v3.x.adoc[Upgrade from v2.x to v3.x] * xref:how-tos/upgrade-v3.x-v4.x.adoc[Upgrade from v3.x to v4.x] * xref:how-tos/switch-to-lokistack.adoc[Switch to Lokistack] +* xref:how-tos/enable-multi-forwarder.adoc[Enable Multi LogForwarder] .Alert runbooks * xref:runbooks/SYN_ElasticsearchExpectNodeToReachDiskWatermark.adoc[SYN_ElasticsearchExpectNodeToReachDiskWatermark] diff --git a/tests/golden/master/openshift4-logging/openshift4-logging/10_operator_group.yaml b/tests/golden/master/openshift4-logging/openshift4-logging/10_operator_group.yaml index ff11675..b72498d 100644 --- a/tests/golden/master/openshift4-logging/openshift4-logging/10_operator_group.yaml +++ b/tests/golden/master/openshift4-logging/openshift4-logging/10_operator_group.yaml @@ -6,6 +6,4 @@ metadata: name: cluster-logging name: cluster-logging namespace: openshift-logging -spec: - targetNamespaces: - - openshift-logging +spec: {} diff --git a/tests/golden/master/openshift4-logging/openshift4-logging/32_namespace_logforwarding.yaml b/tests/golden/master/openshift4-logging/openshift4-logging/32_namespace_logforwarding.yaml new file mode 100644 index 0000000..1c2d4e6 --- /dev/null +++ b/tests/golden/master/openshift4-logging/openshift4-logging/32_namespace_logforwarding.yaml @@ -0,0 +1,48 @@ +apiVersion: logging.openshift.io/v1 +kind: ClusterLogForwarder +metadata: + annotations: {} + labels: + name: bar + name: bar + namespace: foo +spec: + inputs: + - application: + namespaces: + - app-one + - app-two + name: my-apps + outputs: + - name: custom-forwarder + type: syslog + pipelines: + - inputRefs: + - my-apps + name: my-apps + outputRefs: + - custom-forwarder + serviceAccountName: ueli +--- +apiVersion: logging.openshift.io/v1 +kind: ClusterLogForwarder +metadata: + annotations: {} + labels: + name: hands + name: hands + namespace: jazz +spec: + outputs: + - name: splunk-forwarder + secret: + name: splunk-forwarder + type: fluentdForward + url: tls://splunk-forwarder:24224 + pipelines: + - inputRefs: + - application + name: application-logs + outputRefs: + - splunk-forwarder + serviceAccountName: hands diff --git a/tests/golden/master/openshift4-logging/openshift4-logging/32_namespace_rolebinding.yaml b/tests/golden/master/openshift4-logging/openshift4-logging/32_namespace_rolebinding.yaml new file mode 100644 index 0000000..14a605e --- /dev/null +++ b/tests/golden/master/openshift4-logging/openshift4-logging/32_namespace_rolebinding.yaml @@ -0,0 +1,33 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + annotations: {} + labels: + name: ueli + name: ueli + namespace: foo +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: collect-application-logs +subjects: + - kind: ServiceAccount + name: ueli + namespace: foo +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + annotations: {} + labels: + name: hands + name: hands + namespace: jazz +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: collect-application-logs +subjects: + - kind: ServiceAccount + name: hands + namespace: jazz diff --git a/tests/golden/master/openshift4-logging/openshift4-logging/32_namespace_serviceaccount.yaml b/tests/golden/master/openshift4-logging/openshift4-logging/32_namespace_serviceaccount.yaml new file mode 100644 index 0000000..fc3c944 --- /dev/null +++ b/tests/golden/master/openshift4-logging/openshift4-logging/32_namespace_serviceaccount.yaml @@ -0,0 +1,17 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + annotations: {} + labels: + name: ueli + name: ueli + namespace: foo +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + annotations: {} + labels: + name: hands + name: hands + namespace: jazz diff --git a/tests/master.yml b/tests/master.yml index 2d0cb51..b46f792 100644 --- a/tests/master.yml +++ b/tests/master.yml @@ -49,3 +49,36 @@ parameters: audit-logs: outputRefs: - custom-forwarder + + namespaceLogForwarderEnabled: true + namespaceLogForwarder: + jazz/hands: + outputs: + splunk-forwarder: + secret: + name: splunk-forwarder + type: fluentdForward + url: tls://splunk-forwarder:24224 + pipelines: + application-logs: + inputRefs: + - application + outputRefs: + - splunk-forwarder + foo/bar: + serviceAccountName: ueli + inputs: + my-apps: + application: + namespaces: + - app-one + - app-two + outputs: + custom-forwarder: + type: syslog + pipelines: + my-apps: + inputRefs: + - my-apps + outputRefs: + - custom-forwarder