feat: add support for logs_body_key parameter on Opentelemetry output…

… plugin (#1410)

Signed-off-by: yilmazo <[email protected]>

apis/fluentbit/v1alpha2/plugins/output/open_telemetry_types.go

@@ -39,7 +39,9 @@ type OpenTelemetry struct {
 	AddLabel map[string]string `json:"addLabel,omitempty"`
 	// If true, remaining unmatched keys are added as attributes.
 	LogsBodyKeyAttributes *bool `json:"logsBodyKeyAttributes,omitempty"`
-	*plugins.TLS          `json:"tls,omitempty"`
+	// The log body key to look up in the log events body/message. Sets the Body field of the opentelemtry logs data model.
+	LogsBodyKey  string `json:"logsBodyKey,omitempty"`
+	*plugins.TLS `json:"tls,omitempty"`
 	// Include fluentbit networking options for this output-plugin
 	*plugins.Networking `json:"networking,omitempty"`
 }
@@ -96,6 +98,9 @@ func (o *OpenTelemetry) Params(sl plugins.SecretLoader) (*params.KVs, error) {
 	if o.LogsBodyKeyAttributes != nil {
 		kvs.Insert("logs_body_key_attributes", fmt.Sprint(*o.LogsBodyKeyAttributes))
 	}
+	if o.LogsBodyKey != "" {
+		kvs.Insert("logs_body_key", o.LogsBodyKey)
+	}
 	if o.TLS != nil {
 		tls, err := o.TLS.Params(sl)
 		if err != nil {

apis/fluentbit/v1alpha2/plugins/output/open_telemetry_types_test.go

@@ -0,0 +1,64 @@
+package output
+
+import (
+	"github.com/fluent/fluent-operator/v3/apis/fluentbit/v1alpha2/plugins"
+	"github.com/fluent/fluent-operator/v3/apis/fluentbit/v1alpha2/plugins/params"
+	. "github.com/onsi/gomega"
+	v1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"sigs.k8s.io/controller-runtime/pkg/client/fake"
+	"testing"
+)
+
+func TestOpenTelemetry_Params(t *testing.T) {
+	g := NewGomegaWithT(t)
+	fcb := fake.ClientBuilder{}
+	fc := fcb.WithObjects(&v1.Secret{
+		ObjectMeta: metav1.ObjectMeta{Namespace: "test_namespace", Name: "http_secret"},
+		Data: map[string][]byte{
+			"http_user":   []byte("expected_http_user"),
+			"http_passwd": []byte("expected_http_passwd"),
+		},
+	}).Build()
+
+	sl := plugins.NewSecretLoader(fc, "test_namespace")
+	ot := OpenTelemetry{
+		Host:                  "otlp-collector.example.com",
+		Port:                  ptrAny(int32(443)),
+		HTTPUser:              &plugins.Secret{ValueFrom: plugins.ValueSource{SecretKeyRef: v1.SecretKeySelector{LocalObjectReference: v1.LocalObjectReference{Name: "http_secret"}, Key: "http_user"}}},
+		HTTPPasswd:            &plugins.Secret{ValueFrom: plugins.ValueSource{SecretKeyRef: v1.SecretKeySelector{LocalObjectReference: v1.LocalObjectReference{Name: "http_secret"}, Key: "http_passwd"}}},
+		Proxy:                 "expected_proxy",
+		MetricsUri:            "expected_metrics_uri",
+		LogsUri:               "expected_logs_uri",
+		TracesUri:             "expected_traces_uri",
+		Header:                map[string]string{"custom_header_key": "custom_header_val"},
+		LogResponsePayload:    ptrBool(true),
+		AddLabel:              map[string]string{"add_label_key": "add_label_val"},
+		LogsBodyKeyAttributes: ptrBool(true),
+		LogsBodyKey:           "expected_logs_body_key",
+		TLS:                   &plugins.TLS{Verify: ptrBool(false)},
+		Networking:            &plugins.Networking{SourceAddress: ptrAny("expected_source_address")},
+	}
+
+	expected := params.NewKVs()
+	expected.Insert("host", "otlp-collector.example.com")
+	expected.Insert("port", "443")
+	expected.Insert("http_user", "expected_http_user")
+	expected.Insert("http_passwd", "expected_http_passwd")
+	expected.Insert("proxy", "expected_proxy")
+	expected.Insert("metrics_uri", "expected_metrics_uri")
+	expected.Insert("logs_uri", "expected_logs_uri")
+	expected.Insert("traces_uri", "expected_traces_uri")
+	expected.Insert("header", " custom_header_key    custom_header_val")
+	expected.Insert("log_response_payload", "true")
+	expected.Insert("add_label", " add_label_key    add_label_val")
+	expected.Insert("logs_body_key_attributes", "true")
+	expected.Insert("logs_body_key", "expected_logs_body_key")
+	expected.Insert("tls", "On")
+	expected.Insert("tls.verify", "false")
+	expected.Insert("net.source_address", "expected_source_address")
+
+	kvs, err := ot.Params(sl)
+	g.Expect(err).NotTo(HaveOccurred())
+	g.Expect(kvs).To(Equal(expected))
+}

charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_clusteroutputs.yaml

@@ -2930,6 +2930,10 @@ spec:
                   logResponsePayload:
                     description: Log the response payload within the Fluent Bit log.
                     type: boolean
+                  logsBodyKey:
+                    description: The log body key to look up in the log events body/message.
+                      Sets the Body field of the opentelemtry logs data model.
+                    type: string
                   logsBodyKeyAttributes:
                     description: If true, remaining unmatched keys are added as attributes.
                     type: boolean

charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_outputs.yaml

@@ -2930,6 +2930,10 @@ spec:
                   logResponsePayload:
                     description: Log the response payload within the Fluent Bit log.
                     type: boolean
+                  logsBodyKey:
+                    description: The log body key to look up in the log events body/message.
+                      Sets the Body field of the opentelemtry logs data model.
+                    type: string
                   logsBodyKeyAttributes:
                     description: If true, remaining unmatched keys are added as attributes.
                     type: boolean

config/crd/bases/fluentbit.fluent.io_clusteroutputs.yaml

@@ -2930,6 +2930,10 @@ spec:
                   logResponsePayload:
                     description: Log the response payload within the Fluent Bit log.
                     type: boolean
+                  logsBodyKey:
+                    description: The log body key to look up in the log events body/message.
+                      Sets the Body field of the opentelemtry logs data model.
+                    type: string
                   logsBodyKeyAttributes:
                     description: If true, remaining unmatched keys are added as attributes.
                     type: boolean

config/crd/bases/fluentbit.fluent.io_outputs.yaml

@@ -2930,6 +2930,10 @@ spec:
                   logResponsePayload:
                     description: Log the response payload within the Fluent Bit log.
                     type: boolean
+                  logsBodyKey:
+                    description: The log body key to look up in the log events body/message.
+                      Sets the Body field of the opentelemtry logs data model.
+                    type: string
                   logsBodyKeyAttributes:
                     description: If true, remaining unmatched keys are added as attributes.
                     type: boolean

docs/plugins/fluentbit/output/open_telemetry.md

@@ -17,5 +17,6 @@ The OpenTelemetry plugin allows you to take logs, metrics, and traces from Fluen
 | logResponsePayload | Log the response payload within the Fluent Bit log. | *bool |
 | addLabel | This allows you to add custom labels to all metrics exposed through the OpenTelemetry exporter. You may have multiple of these fields. | map[string]string |
 | logsBodyKeyAttributes | If true, remaining unmatched keys are added as attributes. | *bool |
+| logsBodyKey | The log body key to look up in the log events body/message. Sets the Body field of the opentelemtry logs data model. | string |
 | tls |  | *[plugins.TLS](../tls.md) |
 | networking | Include fluentbit networking options for this output-plugin | *plugins.Networking |

manifests/setup/fluent-operator-crd.yaml

@@ -6966,6 +6966,10 @@ spec:
                   logResponsePayload:
                     description: Log the response payload within the Fluent Bit log.
                     type: boolean
+                  logsBodyKey:
+                    description: The log body key to look up in the log events body/message.
+                      Sets the Body field of the opentelemtry logs data model.
+                    type: string
                   logsBodyKeyAttributes:
                     description: If true, remaining unmatched keys are added as attributes.
                     type: boolean
@@ -35656,6 +35660,10 @@ spec:
                   logResponsePayload:
                     description: Log the response payload within the Fluent Bit log.
                     type: boolean
+                  logsBodyKey:
+                    description: The log body key to look up in the log events body/message.
+                      Sets the Body field of the opentelemtry logs data model.
+                    type: string
                   logsBodyKeyAttributes:
                     description: If true, remaining unmatched keys are added as attributes.
                     type: boolean

manifests/setup/setup.yaml

@@ -6966,6 +6966,10 @@ spec:
                   logResponsePayload:
                     description: Log the response payload within the Fluent Bit log.
                     type: boolean
+                  logsBodyKey:
+                    description: The log body key to look up in the log events body/message.
+                      Sets the Body field of the opentelemtry logs data model.
+                    type: string
                   logsBodyKeyAttributes:
                     description: If true, remaining unmatched keys are added as attributes.
                     type: boolean
@@ -35656,6 +35660,10 @@ spec:
                   logResponsePayload:
                     description: Log the response payload within the Fluent Bit log.
                     type: boolean
+                  logsBodyKey:
+                    description: The log body key to look up in the log events body/message.
+                      Sets the Body field of the opentelemtry logs data model.
+                    type: string
                   logsBodyKeyAttributes:
                     description: If true, remaining unmatched keys are added as attributes.
                     type: boolean