From 4cc792e31101ea1dc77be8377bd9cb10f40738b3 Mon Sep 17 00:00:00 2001 From: zzzk1 Date: Sat, 28 Dec 2024 00:22:47 +0800 Subject: [PATCH 01/10] [exporter] Remove jaeger dbmodel dependency --- exporter/logzioexporter/dbmodel.go | 90 +++++++++ exporter/logzioexporter/from_domain.go | 126 ++++++++++++ exporter/logzioexporter/logziospan.go | 35 ++-- exporter/logzioexporter/to_domain.go | 255 +++++++++++++++++++++++++ 4 files changed, 488 insertions(+), 18 deletions(-) create mode 100644 exporter/logzioexporter/dbmodel.go create mode 100644 exporter/logzioexporter/from_domain.go create mode 100644 exporter/logzioexporter/to_domain.go diff --git a/exporter/logzioexporter/dbmodel.go b/exporter/logzioexporter/dbmodel.go new file mode 100644 index 000000000000..194e7a6373a0 --- /dev/null +++ b/exporter/logzioexporter/dbmodel.go @@ -0,0 +1,90 @@ +// Copyright (c) 2019 The Jaeger Authors. +// Copyright (c) 2018 Uber Technologies, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package logzioexporter + +// ReferenceType is the reference type of one span to another +type ReferenceType string + +// TraceID is the shared trace ID of all spans in the trace. +type TraceID string + +// SpanID is the id of a span +type SpanID string + +// ValueType is the type of a value stored in KeyValue struct. +type ValueType string + +const ( + // ChildOf means a span is the child of another span + ChildOf ReferenceType = "CHILD_OF" + // FollowsFrom means a span follows from another span + FollowsFrom ReferenceType = "FOLLOWS_FROM" + + // StringType indicates a string value stored in KeyValue + StringType ValueType = "string" + // BoolType indicates a Boolean value stored in KeyValue + BoolType ValueType = "bool" + // Int64Type indicates a 64bit signed integer value stored in KeyValue + Int64Type ValueType = "int64" + // Float64Type indicates a 64bit float value stored in KeyValue + Float64Type ValueType = "float64" + // BinaryType indicates an arbitrary byte array stored in KeyValue + BinaryType ValueType = "binary" +) + +// Span is ES database representation of the domain span. +type Span struct { + TraceID TraceID `json:"traceID"` + SpanID SpanID `json:"spanID"` + ParentSpanID SpanID `json:"parentSpanID,omitempty"` // deprecated + Flags uint32 `json:"flags,omitempty"` + OperationName string `json:"operationName"` + References []Reference `json:"references"` + StartTime uint64 `json:"startTime"` // microseconds since Unix epoch + // ElasticSearch does not support a UNIX Epoch timestamp in microseconds, + // so Jaeger maps StartTime to a 'long' type. This extra StartTimeMillis field + // works around this issue, enabling timerange queries. + StartTimeMillis uint64 `json:"startTimeMillis"` + Duration uint64 `json:"duration"` // microseconds + Tags []KeyValue `json:"tags"` + // Alternative representation of tags for better kibana support + Tag map[string]any `json:"tag,omitempty"` + Logs []Log `json:"logs"` + Process Process `json:"process,omitempty"` +} + +// Reference is a reference from one span to another +type Reference struct { + RefType ReferenceType `json:"refType"` + TraceID TraceID `json:"traceID"` + SpanID SpanID `json:"spanID"` +} + +// Process is the process emitting a set of spans +type Process struct { + ServiceName string `json:"serviceName"` + Tags []KeyValue `json:"tags"` + // Alternative representation of tags for better kibana support + Tag map[string]any `json:"tag,omitempty"` +} + +// Log is a log emitted in a span +type Log struct { + Timestamp uint64 `json:"timestamp"` + Fields []KeyValue `json:"fields"` +} + +// KeyValue is a key-value pair with typed value. +type KeyValue struct { + Key string `json:"key"` + Type ValueType `json:"type,omitempty"` + Value any `json:"value"` +} + +// Service is the JSON struct for service:operation documents in ElasticSearch +type Service struct { + ServiceName string `json:"serviceName"` + OperationName string `json:"operationName"` +} diff --git a/exporter/logzioexporter/from_domain.go b/exporter/logzioexporter/from_domain.go new file mode 100644 index 000000000000..0b9c7329e0cb --- /dev/null +++ b/exporter/logzioexporter/from_domain.go @@ -0,0 +1,126 @@ +// Copyright (c) 2019 The Jaeger Authors. +// Copyright (c) 2018 Uber Technologies, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package logzioexporter + +import ( + "strings" + + "github.com/jaegertracing/jaeger/model" +) + +// NewFromDomain creates FromDomain used to convert model span to db span +func NewFromDomain(allTagsAsObject bool, tagKeysAsFields []string, tagDotReplacement string) FromDomain { + tags := map[string]bool{} + for _, k := range tagKeysAsFields { + tags[k] = true + } + return FromDomain{allTagsAsFields: allTagsAsObject, tagKeysAsFields: tags, tagDotReplacement: tagDotReplacement} +} + +// FromDomain is used to convert model span to db span +type FromDomain struct { + allTagsAsFields bool + tagKeysAsFields map[string]bool + tagDotReplacement string +} + +// FromDomainEmbedProcess converts model.Span into json.Span format. +// This format includes a ParentSpanID and an embedded Process. +func (fd FromDomain) FromDomainEmbedProcess(span *model.Span) *Span { + return fd.convertSpanEmbedProcess(span) +} + +func (fd FromDomain) convertSpanInternal(span *model.Span) Span { + tags, tagsMap := fd.convertKeyValuesString(span.Tags) + return Span{ + TraceID: TraceID(span.TraceID.String()), + SpanID: SpanID(span.SpanID.String()), + Flags: uint32(span.Flags), + OperationName: span.OperationName, + StartTime: model.TimeAsEpochMicroseconds(span.StartTime), + StartTimeMillis: model.TimeAsEpochMicroseconds(span.StartTime) / 1000, + Duration: model.DurationAsMicroseconds(span.Duration), + Tags: tags, + Tag: tagsMap, + Logs: fd.convertLogs(span.Logs), + } +} + +func (fd FromDomain) convertSpanEmbedProcess(span *model.Span) *Span { + s := fd.convertSpanInternal(span) + s.Process = fd.convertProcess(span.Process) + s.References = fd.convertReferences(span) + return &s +} + +func (fd FromDomain) convertReferences(span *model.Span) []Reference { + out := make([]Reference, 0, len(span.References)) + for _, ref := range span.References { + out = append(out, Reference{ + RefType: fd.convertRefType(ref.RefType), + TraceID: TraceID(ref.TraceID.String()), + SpanID: SpanID(ref.SpanID.String()), + }) + } + return out +} + +func (FromDomain) convertRefType(refType model.SpanRefType) ReferenceType { + if refType == model.FollowsFrom { + return FollowsFrom + } + return ChildOf +} + +func (fd FromDomain) convertKeyValuesString(keyValues model.KeyValues) ([]KeyValue, map[string]any) { + var tagsMap map[string]any + var kvs []KeyValue + for _, kv := range keyValues { + if kv.GetVType() != model.BinaryType && (fd.allTagsAsFields || fd.tagKeysAsFields[kv.Key]) { + if tagsMap == nil { + tagsMap = map[string]any{} + } + tagsMap[strings.ReplaceAll(kv.Key, ".", fd.tagDotReplacement)] = kv.Value() + } else { + kvs = append(kvs, convertKeyValue(kv)) + } + } + if kvs == nil { + kvs = make([]KeyValue, 0) + } + return kvs, tagsMap +} + +func (FromDomain) convertLogs(logs []model.Log) []Log { + out := make([]Log, len(logs)) + for i, log := range logs { + var kvs []KeyValue + for _, kv := range log.Fields { + kvs = append(kvs, convertKeyValue(kv)) + } + out[i] = Log{ + Timestamp: model.TimeAsEpochMicroseconds(log.Timestamp), + Fields: kvs, + } + } + return out +} + +func (fd FromDomain) convertProcess(process *model.Process) Process { + tags, tagsMap := fd.convertKeyValuesString(process.Tags) + return Process{ + ServiceName: process.ServiceName, + Tags: tags, + Tag: tagsMap, + } +} + +func convertKeyValue(kv model.KeyValue) KeyValue { + return KeyValue{ + Key: kv.Key, + Type: ValueType(strings.ToLower(kv.VType.String())), + Value: kv.AsString(), + } +} diff --git a/exporter/logzioexporter/logziospan.go b/exporter/logzioexporter/logziospan.go index 7bc485fd1409..227f505e10b3 100644 --- a/exporter/logzioexporter/logziospan.go +++ b/exporter/logzioexporter/logziospan.go @@ -7,7 +7,6 @@ import ( "encoding/json" "github.com/jaegertracing/jaeger/model" - "github.com/jaegertracing/jaeger/plugin/storage/es/spanstore/dbmodel" ) const ( @@ -18,20 +17,20 @@ const ( // logzioSpan is same as esSpan with a few different json field names and an addition on type field. type logzioSpan struct { - TraceID dbmodel.TraceID `json:"traceID"` - OperationName string `json:"operationName,omitempty"` - SpanID dbmodel.SpanID `json:"spanID"` - References []dbmodel.Reference `json:"references"` - Flags uint32 `json:"flags,omitempty"` - StartTime uint64 `json:"startTime"` - StartTimeMillis uint64 `json:"startTimeMillis"` - Timestamp uint64 `json:"@timestamp"` - Duration uint64 `json:"duration"` - Tags []dbmodel.KeyValue `json:"JaegerTags,omitempty"` - Tag map[string]any `json:"JaegerTag,omitempty"` - Logs []dbmodel.Log `json:"logs"` - Process dbmodel.Process `json:"process,omitempty"` - Type string `json:"type"` + TraceID TraceID `json:"traceID"` + OperationName string `json:"operationName,omitempty"` + SpanID SpanID `json:"spanID"` + References []Reference `json:"references"` + Flags uint32 `json:"flags,omitempty"` + StartTime uint64 `json:"startTime"` + StartTimeMillis uint64 `json:"startTimeMillis"` + Timestamp uint64 `json:"@timestamp"` + Duration uint64 `json:"duration"` + Tags []KeyValue `json:"JaegerTags,omitempty"` + Tag map[string]any `json:"JaegerTag,omitempty"` + Logs []Log `json:"logs"` + Process Process `json:"process,omitempty"` + Type string `json:"type"` } func getTagsValues(tags []model.KeyValue) []string { @@ -45,7 +44,7 @@ func getTagsValues(tags []model.KeyValue) []string { // transformToLogzioSpanBytes receives a Jaeger span, converts it to logzio span and returns it as a byte array. // The main differences between Jaeger span and logzio span are arrays which are represented as maps func transformToLogzioSpanBytes(span *model.Span) ([]byte, error) { - spanConverter := dbmodel.NewFromDomain(true, getTagsValues(span.Tags), tagDotReplacementCharacter) + spanConverter := NewFromDomain(true, getTagsValues(span.Tags), tagDotReplacementCharacter) jsonSpan := spanConverter.FromDomainEmbedProcess(span) newSpan := logzioSpan{ TraceID: jsonSpan.TraceID, @@ -67,8 +66,8 @@ func transformToLogzioSpanBytes(span *model.Span) ([]byte, error) { } // transformToDbModelSpan coverts logz.io span to ElasticSearch span -func (span *logzioSpan) transformToDbModelSpan() *dbmodel.Span { - return &dbmodel.Span{ +func (span *logzioSpan) transformToDbModelSpan() *Span { + return &Span{ OperationName: span.OperationName, Process: span.Process, Tags: span.Tags, diff --git a/exporter/logzioexporter/to_domain.go b/exporter/logzioexporter/to_domain.go new file mode 100644 index 000000000000..92aa72682a4c --- /dev/null +++ b/exporter/logzioexporter/to_domain.go @@ -0,0 +1,255 @@ +// Copyright (c) 2019 The Jaeger Authors. +// Copyright (c) 2018 Uber Technologies, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package logzioexporter + +import ( + "encoding/hex" + "encoding/json" + "fmt" + "strconv" + "strings" + + "github.com/jaegertracing/jaeger/model" +) + +// NewToDomain creates ToDomain +func NewToDomain(tagDotReplacement string) ToDomain { + return ToDomain{tagDotReplacement: tagDotReplacement} +} + +// ToDomain is used to convert Span to model.Span +type ToDomain struct { + tagDotReplacement string +} + +// ReplaceDot replaces dot with dotReplacement +func (td ToDomain) ReplaceDot(k string) string { + return strings.ReplaceAll(k, ".", td.tagDotReplacement) +} + +// ReplaceDotReplacement replaces dotReplacement with dot +func (td ToDomain) ReplaceDotReplacement(k string) string { + return strings.ReplaceAll(k, td.tagDotReplacement, ".") +} + +// SpanToDomain converts db span into model Span +func (td ToDomain) SpanToDomain(dbSpan *Span) (*model.Span, error) { + tags, err := td.convertKeyValues(dbSpan.Tags) + if err != nil { + return nil, err + } + logs, err := td.convertLogs(dbSpan.Logs) + if err != nil { + return nil, err + } + refs, err := td.convertRefs(dbSpan.References) + if err != nil { + return nil, err + } + process, err := td.convertProcess(dbSpan.Process) + if err != nil { + return nil, err + } + traceID, err := model.TraceIDFromString(string(dbSpan.TraceID)) + if err != nil { + return nil, err + } + + spanIDInt, err := model.SpanIDFromString(string(dbSpan.SpanID)) + if err != nil { + return nil, err + } + + if dbSpan.ParentSpanID != "" { + parentSpanID, err := model.SpanIDFromString(string(dbSpan.ParentSpanID)) + if err != nil { + return nil, err + } + refs = model.MaybeAddParentSpanID(traceID, parentSpanID, refs) + } + + fieldTags, err := td.convertTagFields(dbSpan.Tag) + if err != nil { + return nil, err + } + tags = append(tags, fieldTags...) + + span := &model.Span{ + TraceID: traceID, + SpanID: model.NewSpanID(uint64(spanIDInt)), + OperationName: dbSpan.OperationName, + References: refs, + Flags: model.Flags(uint32(dbSpan.Flags)), + StartTime: model.EpochMicrosecondsAsTime(dbSpan.StartTime), + Duration: model.MicrosecondsAsDuration(dbSpan.Duration), + Tags: tags, + Logs: logs, + Process: process, + } + return span, nil +} + +func (ToDomain) convertRefs(refs []Reference) ([]model.SpanRef, error) { + retMe := make([]model.SpanRef, len(refs)) + for i, r := range refs { + // There are some inconsistencies with ReferenceTypes, hence the hacky fix. + var refType model.SpanRefType + switch r.RefType { + case ChildOf: + refType = model.ChildOf + case FollowsFrom: + refType = model.FollowsFrom + default: + return nil, fmt.Errorf("not a valid SpanRefType string %s", string(r.RefType)) + } + + traceID, err := model.TraceIDFromString(string(r.TraceID)) + if err != nil { + return nil, err + } + + spanID, err := strconv.ParseUint(string(r.SpanID), 16, 64) + if err != nil { + return nil, err + } + + retMe[i] = model.SpanRef{ + RefType: refType, + TraceID: traceID, + SpanID: model.NewSpanID(spanID), + } + } + return retMe, nil +} + +func (td ToDomain) convertKeyValues(tags []KeyValue) ([]model.KeyValue, error) { + retMe := make([]model.KeyValue, len(tags)) + for i := range tags { + kv, err := td.convertKeyValue(&tags[i]) + if err != nil { + return nil, err + } + retMe[i] = kv + } + return retMe, nil +} + +func (td ToDomain) convertTagFields(tagsMap map[string]any) ([]model.KeyValue, error) { + kvs := make([]model.KeyValue, len(tagsMap)) + i := 0 + for k, v := range tagsMap { + tag, err := td.convertTagField(k, v) + if err != nil { + return nil, err + } + kvs[i] = tag + i++ + } + return kvs, nil +} + +func (td ToDomain) convertTagField(k string, v any) (model.KeyValue, error) { + dKey := td.ReplaceDotReplacement(k) + switch val := v.(type) { + case int64: + return model.Int64(dKey, val), nil + case float64: + return model.Float64(dKey, val), nil + case bool: + return model.Bool(dKey, val), nil + case string: + return model.String(dKey, val), nil + // the binary is never returned, ES returns it as string with base64 encoding + case []byte: + return model.Binary(dKey, val), nil + // in spans are decoded using json.UseNumber() to preserve the type + // however note that float(1) will be parsed as int as ES does not store decimal point + case json.Number: + n, err := val.Int64() + if err == nil { + return model.Int64(dKey, n), nil + } + f, err := val.Float64() + if err == nil { + return model.Float64(dKey, f), nil + } + return model.String("", ""), fmt.Errorf("invalid tag type in %+v: %w", v, err) + default: + return model.String("", ""), fmt.Errorf("invalid tag type in %+v", v) + } +} + +// convertKeyValue expects the Value field to be string, because it only works +// as a reverse transformation after FromDomain() for ElasticSearch model. +func (ToDomain) convertKeyValue(tag *KeyValue) (model.KeyValue, error) { + if tag.Value == nil { + return model.KeyValue{}, fmt.Errorf("invalid nil Value in %v", tag) + } + tagValue, ok := tag.Value.(string) + if !ok { + return model.KeyValue{}, fmt.Errorf("non-string Value of type %t in %v", tag.Value, tag) + } + switch tag.Type { + case StringType: + return model.String(tag.Key, tagValue), nil + case BoolType: + value, err := strconv.ParseBool(tagValue) + if err != nil { + return model.KeyValue{}, err + } + return model.Bool(tag.Key, value), nil + case Int64Type: + value, err := strconv.ParseInt(tagValue, 10, 64) + if err != nil { + return model.KeyValue{}, err + } + return model.Int64(tag.Key, value), nil + case Float64Type: + value, err := strconv.ParseFloat(tagValue, 64) + if err != nil { + return model.KeyValue{}, err + } + return model.Float64(tag.Key, value), nil + case BinaryType: + value, err := hex.DecodeString(tagValue) + if err != nil { + return model.KeyValue{}, err + } + return model.Binary(tag.Key, value), nil + } + return model.KeyValue{}, fmt.Errorf("not a valid ValueType string %s", string(tag.Type)) +} + +func (td ToDomain) convertLogs(logs []Log) ([]model.Log, error) { + retMe := make([]model.Log, len(logs)) + for i, l := range logs { + fields, err := td.convertKeyValues(l.Fields) + if err != nil { + return nil, err + } + retMe[i] = model.Log{ + Timestamp: model.EpochMicrosecondsAsTime(l.Timestamp), + Fields: fields, + } + } + return retMe, nil +} + +func (td ToDomain) convertProcess(process Process) (*model.Process, error) { + tags, err := td.convertKeyValues(process.Tags) + if err != nil { + return nil, err + } + fieldTags, err := td.convertTagFields(process.Tag) + if err != nil { + return nil, err + } + tags = append(tags, fieldTags...) + + return &model.Process{ + Tags: tags, + ServiceName: process.ServiceName, + }, nil +} From 43399e66cd16316be5145992e7acabd98d9e779c Mon Sep 17 00:00:00 2001 From: zzzk1 Date: Sun, 29 Dec 2024 02:03:21 +0800 Subject: [PATCH 02/10] add unit test --- exporter/logzioexporter/dbmodel.go | 90 ------- exporter/logzioexporter/exporter_test.go | 2 +- exporter/logzioexporter/from_domain.go | 8 +- exporter/logzioexporter/from_domain_test.go | 139 +++++++++++ exporter/logzioexporter/logziospan.go | 99 +++++++- exporter/logzioexporter/logziospan_test.go | 2 +- exporter/logzioexporter/testdata/es.json | 90 +++++++ exporter/logzioexporter/to_domain.go | 255 -------------------- 8 files changed, 327 insertions(+), 358 deletions(-) delete mode 100644 exporter/logzioexporter/dbmodel.go create mode 100644 exporter/logzioexporter/from_domain_test.go create mode 100644 exporter/logzioexporter/testdata/es.json delete mode 100644 exporter/logzioexporter/to_domain.go diff --git a/exporter/logzioexporter/dbmodel.go b/exporter/logzioexporter/dbmodel.go deleted file mode 100644 index 194e7a6373a0..000000000000 --- a/exporter/logzioexporter/dbmodel.go +++ /dev/null @@ -1,90 +0,0 @@ -// Copyright (c) 2019 The Jaeger Authors. -// Copyright (c) 2018 Uber Technologies, Inc. -// SPDX-License-Identifier: Apache-2.0 - -package logzioexporter - -// ReferenceType is the reference type of one span to another -type ReferenceType string - -// TraceID is the shared trace ID of all spans in the trace. -type TraceID string - -// SpanID is the id of a span -type SpanID string - -// ValueType is the type of a value stored in KeyValue struct. -type ValueType string - -const ( - // ChildOf means a span is the child of another span - ChildOf ReferenceType = "CHILD_OF" - // FollowsFrom means a span follows from another span - FollowsFrom ReferenceType = "FOLLOWS_FROM" - - // StringType indicates a string value stored in KeyValue - StringType ValueType = "string" - // BoolType indicates a Boolean value stored in KeyValue - BoolType ValueType = "bool" - // Int64Type indicates a 64bit signed integer value stored in KeyValue - Int64Type ValueType = "int64" - // Float64Type indicates a 64bit float value stored in KeyValue - Float64Type ValueType = "float64" - // BinaryType indicates an arbitrary byte array stored in KeyValue - BinaryType ValueType = "binary" -) - -// Span is ES database representation of the domain span. -type Span struct { - TraceID TraceID `json:"traceID"` - SpanID SpanID `json:"spanID"` - ParentSpanID SpanID `json:"parentSpanID,omitempty"` // deprecated - Flags uint32 `json:"flags,omitempty"` - OperationName string `json:"operationName"` - References []Reference `json:"references"` - StartTime uint64 `json:"startTime"` // microseconds since Unix epoch - // ElasticSearch does not support a UNIX Epoch timestamp in microseconds, - // so Jaeger maps StartTime to a 'long' type. This extra StartTimeMillis field - // works around this issue, enabling timerange queries. - StartTimeMillis uint64 `json:"startTimeMillis"` - Duration uint64 `json:"duration"` // microseconds - Tags []KeyValue `json:"tags"` - // Alternative representation of tags for better kibana support - Tag map[string]any `json:"tag,omitempty"` - Logs []Log `json:"logs"` - Process Process `json:"process,omitempty"` -} - -// Reference is a reference from one span to another -type Reference struct { - RefType ReferenceType `json:"refType"` - TraceID TraceID `json:"traceID"` - SpanID SpanID `json:"spanID"` -} - -// Process is the process emitting a set of spans -type Process struct { - ServiceName string `json:"serviceName"` - Tags []KeyValue `json:"tags"` - // Alternative representation of tags for better kibana support - Tag map[string]any `json:"tag,omitempty"` -} - -// Log is a log emitted in a span -type Log struct { - Timestamp uint64 `json:"timestamp"` - Fields []KeyValue `json:"fields"` -} - -// KeyValue is a key-value pair with typed value. -type KeyValue struct { - Key string `json:"key"` - Type ValueType `json:"type,omitempty"` - Value any `json:"value"` -} - -// Service is the JSON struct for service:operation documents in ElasticSearch -type Service struct { - ServiceName string `json:"serviceName"` - OperationName string `json:"operationName"` -} diff --git a/exporter/logzioexporter/exporter_test.go b/exporter/logzioexporter/exporter_test.go index 2943c50a9413..a9628951fa62 100644 --- a/exporter/logzioexporter/exporter_test.go +++ b/exporter/logzioexporter/exporter_test.go @@ -255,7 +255,7 @@ func TestPushTraceData(tester *testing.T) { res.Attributes().PutStr(conventions.AttributeHostName, testHost) err := testTracesExporter(tester, td, &cfg) require.NoError(tester, err) - var newSpan logzioSpan + var newSpan LogzioSpan decoded, _ := gUnzipData(recordedRequests) requests := strings.Split(string(decoded), "\n") assert.NoError(tester, json.Unmarshal([]byte(requests[0]), &newSpan)) diff --git a/exporter/logzioexporter/from_domain.go b/exporter/logzioexporter/from_domain.go index 0b9c7329e0cb..a62752c2e7a6 100644 --- a/exporter/logzioexporter/from_domain.go +++ b/exporter/logzioexporter/from_domain.go @@ -28,13 +28,13 @@ type FromDomain struct { // FromDomainEmbedProcess converts model.Span into json.Span format. // This format includes a ParentSpanID and an embedded Process. -func (fd FromDomain) FromDomainEmbedProcess(span *model.Span) *Span { +func (fd FromDomain) FromDomainEmbedProcess(span *model.Span) *LogzioSpan { return fd.convertSpanEmbedProcess(span) } -func (fd FromDomain) convertSpanInternal(span *model.Span) Span { +func (fd FromDomain) convertSpanInternal(span *model.Span) LogzioSpan { tags, tagsMap := fd.convertKeyValuesString(span.Tags) - return Span{ + return LogzioSpan{ TraceID: TraceID(span.TraceID.String()), SpanID: SpanID(span.SpanID.String()), Flags: uint32(span.Flags), @@ -48,7 +48,7 @@ func (fd FromDomain) convertSpanInternal(span *model.Span) Span { } } -func (fd FromDomain) convertSpanEmbedProcess(span *model.Span) *Span { +func (fd FromDomain) convertSpanEmbedProcess(span *model.Span) *LogzioSpan { s := fd.convertSpanInternal(span) s.Process = fd.convertProcess(span.Process) s.References = fd.convertReferences(span) diff --git a/exporter/logzioexporter/from_domain_test.go b/exporter/logzioexporter/from_domain_test.go new file mode 100644 index 000000000000..24aae34c0700 --- /dev/null +++ b/exporter/logzioexporter/from_domain_test.go @@ -0,0 +1,139 @@ +// Copyright (c) 2019 The Jaeger Authors. +// Copyright (c) 2018 Uber Technologies, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package logzioexporter + +import ( + "bytes" + "encoding/hex" + "encoding/json" + "fmt" + "os" + "testing" + + "github.com/gogo/protobuf/jsonpb" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/jaegertracing/jaeger/model" +) + +func TestFromDomainEmbedProcess(t *testing.T) { + domainStr, jsonStr := loadFixtures(t) + + var span model.Span + require.NoError(t, jsonpb.Unmarshal(bytes.NewReader(domainStr), &span)) + converter := NewFromDomain(false, nil, ":") + embeddedSpan := converter.FromDomainEmbedProcess(&span) + + var expectedSpan LogzioSpan + require.NoError(t, json.Unmarshal(jsonStr, &expectedSpan)) + + testJSONEncoding(t, jsonStr, embeddedSpan.transformToDbModelSpan()) +} + +// Loads and returns domain model and JSON model fixtures with given number i. +func loadFixtures(t *testing.T) ([]byte, []byte) { + in := fmt.Sprintf("./testdata/span.json") + inStr, err := os.ReadFile(in) + require.NoError(t, err) + out := fmt.Sprintf("./testdata/es.json") + outStr, err := os.ReadFile(out) + require.NoError(t, err) + return inStr, outStr +} + +func testJSONEncoding(t *testing.T, expectedStr []byte, object any) { + buf := &bytes.Buffer{} + enc := json.NewEncoder(buf) + enc.SetIndent("", " ") + + outFile := fmt.Sprintf("./testdata/es.json") + require.NoError(t, enc.Encode(object)) + + if !assert.Equal(t, string(expectedStr), buf.String()) { + err := os.WriteFile(outFile+"-actual.json", buf.Bytes(), 0o644) + require.NoError(t, err) + } +} + +func TestEmptyTags(t *testing.T) { + tags := make([]model.KeyValue, 0) + span := model.Span{Tags: tags, Process: &model.Process{Tags: tags}} + converter := NewFromDomain(false, nil, ":") + dbSpan := converter.FromDomainEmbedProcess(&span) + assert.Empty(t, dbSpan.Tags) + assert.Empty(t, dbSpan.Tag) +} + +func TestTagMap(t *testing.T) { + tags := []model.KeyValue{ + model.String("foo", "foo"), + model.Bool("a", true), + model.Int64("b.b", 1), + } + span := model.Span{Tags: tags, Process: &model.Process{Tags: tags}} + converter := NewFromDomain(false, []string{"a", "b.b", "b*"}, ":") + dbSpan := converter.FromDomainEmbedProcess(&span) + + assert.Len(t, dbSpan.Tags, 1) + assert.Equal(t, "foo", dbSpan.Tags[0].Key) + assert.Len(t, dbSpan.Process.Tags, 1) + assert.Equal(t, "foo", dbSpan.Process.Tags[0].Key) + + tagsMap := map[string]any{} + tagsMap["a"] = true + tagsMap["b:b"] = int64(1) + assert.Equal(t, tagsMap, dbSpan.Tag) + assert.Equal(t, tagsMap, dbSpan.Process.Tag) +} + +func TestConvertKeyValueValue(t *testing.T) { + longString := `Bender Bending Rodrigues Bender Bending Rodrigues Bender Bending Rodrigues Bender Bending Rodrigues + Bender Bending Rodrigues Bender Bending Rodrigues Bender Bending Rodrigues Bender Bending Rodrigues Bender Bending Rodrigues + Bender Bending Rodrigues Bender Bending Rodrigues Bender Bending Rodrigues Bender Bending Rodrigues Bender Bending Rodrigues + Bender Bending Rodrigues Bender Bending Rodrigues Bender Bending Rodrigues Bender Bending Rodrigues Bender Bending Rodrigues + Bender Bending Rodrigues Bender Bending Rodrigues Bender Bending Rodrigues Bender Bending Rodrigues Bender Bending Rodrigues ` + key := "key" + tests := []struct { + kv model.KeyValue + expected KeyValue + }{ + { + kv: model.Bool(key, true), + expected: KeyValue{Key: key, Value: "true", Type: "bool"}, + }, + { + kv: model.Bool(key, false), + expected: KeyValue{Key: key, Value: "false", Type: "bool"}, + }, + { + kv: model.Int64(key, int64(1499)), + expected: KeyValue{Key: key, Value: "1499", Type: "int64"}, + }, + { + kv: model.Float64(key, float64(15.66)), + expected: KeyValue{Key: key, Value: "15.66", Type: "float64"}, + }, + { + kv: model.String(key, longString), + expected: KeyValue{Key: key, Value: longString, Type: "string"}, + }, + { + kv: model.Binary(key, []byte(longString)), + expected: KeyValue{Key: key, Value: hex.EncodeToString([]byte(longString)), Type: "binary"}, + }, + { + kv: model.KeyValue{VType: 1500, Key: key}, + expected: KeyValue{Key: key, Value: "unknown type 1500", Type: "1500"}, + }, + } + + for _, test := range tests { + t.Run(fmt.Sprintf("%s:%s", test.expected.Type, test.expected.Key), func(t *testing.T) { + actual := convertKeyValue(test.kv) + assert.Equal(t, test.expected, actual) + }) + } +} diff --git a/exporter/logzioexporter/logziospan.go b/exporter/logzioexporter/logziospan.go index 227f505e10b3..dc08e0cc797d 100644 --- a/exporter/logzioexporter/logziospan.go +++ b/exporter/logzioexporter/logziospan.go @@ -15,11 +15,75 @@ const ( tagDotReplacementCharacter = "@" ) -// logzioSpan is same as esSpan with a few different json field names and an addition on type field. -type logzioSpan struct { +// ReferenceType is the reference type of one span to another +type ReferenceType string + +// TraceID is the shared trace ID of all spans in the trace. +type TraceID string + +// SpanID is the id of a span +type SpanID string + +// ValueType is the type of a value stored in KeyValue struct. +type ValueType string + +const ( + // ChildOf means a span is the child of another span + ChildOf ReferenceType = "CHILD_OF" + // FollowsFrom means a span follows from another span + FollowsFrom ReferenceType = "FOLLOWS_FROM" + + // StringType indicates a string value stored in KeyValue + StringType ValueType = "string" + // BoolType indicates a Boolean value stored in KeyValue + BoolType ValueType = "bool" + // Int64Type indicates a 64bit signed integer value stored in KeyValue + Int64Type ValueType = "int64" + // Float64Type indicates a 64bit float value stored in KeyValue + Float64Type ValueType = "float64" + // BinaryType indicates an arbitrary byte array stored in KeyValue + BinaryType ValueType = "binary" +) + +// Reference is a reference from one span to another +type Reference struct { + RefType ReferenceType `json:"refType"` + TraceID TraceID `json:"traceID"` + SpanID SpanID `json:"spanID"` +} + +// Process is the process emitting a set of spans +type Process struct { + ServiceName string `json:"serviceName"` + Tags []KeyValue `json:"tags"` + // Alternative representation of tags for better kibana support + Tag map[string]any `json:"tag,omitempty"` +} + +// Log is a log emitted in a span +type Log struct { + Timestamp uint64 `json:"timestamp"` + Fields []KeyValue `json:"fields"` +} + +// KeyValue is a key-value pair with typed value. +type KeyValue struct { + Key string `json:"key"` + Type ValueType `json:"type,omitempty"` + Value any `json:"value"` +} + +// Service is the JSON struct for service:operation documents in ElasticSearch +type Service struct { + ServiceName string `json:"serviceName"` + OperationName string `json:"operationName"` +} + +// LogzioSpan is same as ESSpan with a few different json field names and an addition on type field. +type LogzioSpan struct { TraceID TraceID `json:"traceID"` - OperationName string `json:"operationName,omitempty"` SpanID SpanID `json:"spanID"` + OperationName string `json:"operationName,omitempty"` References []Reference `json:"references"` Flags uint32 `json:"flags,omitempty"` StartTime uint64 `json:"startTime"` @@ -33,6 +97,27 @@ type logzioSpan struct { Type string `json:"type"` } +// only for testing EsSpan is ES database representation of the domain span. +type EsSpan struct { + TraceID TraceID `json:"traceID"` + SpanID SpanID `json:"spanID"` + ParentSpanID SpanID `json:"parentSpanID,omitempty"` // deprecated + Flags uint32 `json:"flags,omitempty"` + OperationName string `json:"operationName"` + References []Reference `json:"references"` + StartTime uint64 `json:"startTime"` // microseconds since Unix epoch + // ElasticSearch does not support a UNIX Epoch timestamp in microseconds, + // so Jaeger maps StartTime to a 'long' type. This extra StartTimeMillis field + // works around this issue, enabling timerange queries. + StartTimeMillis uint64 `json:"startTimeMillis"` + Duration uint64 `json:"duration"` // microseconds + Tags []KeyValue `json:"tags"` + // Alternative representation of tags for better kibana support + Tag map[string]any `json:"tag,omitempty"` + Logs []Log `json:"logs"` + Process Process `json:"process,omitempty"` +} + func getTagsValues(tags []model.KeyValue) []string { values := make([]string, len(tags)) for i := range tags { @@ -46,7 +131,7 @@ func getTagsValues(tags []model.KeyValue) []string { func transformToLogzioSpanBytes(span *model.Span) ([]byte, error) { spanConverter := NewFromDomain(true, getTagsValues(span.Tags), tagDotReplacementCharacter) jsonSpan := spanConverter.FromDomainEmbedProcess(span) - newSpan := logzioSpan{ + newSpan := LogzioSpan{ TraceID: jsonSpan.TraceID, OperationName: jsonSpan.OperationName, SpanID: jsonSpan.SpanID, @@ -65,9 +150,9 @@ func transformToLogzioSpanBytes(span *model.Span) ([]byte, error) { return json.Marshal(newSpan) } -// transformToDbModelSpan coverts logz.io span to ElasticSearch span -func (span *logzioSpan) transformToDbModelSpan() *Span { - return &Span{ +// only for testing transformToDbModelSpan coverts logz.io span to ElasticSearch span +func (span *LogzioSpan) transformToDbModelSpan() *EsSpan { + return &EsSpan{ OperationName: span.OperationName, Process: span.Process, Tags: span.Tags, diff --git a/exporter/logzioexporter/logziospan_test.go b/exporter/logzioexporter/logziospan_test.go index faea671f4b40..cd5620742459 100644 --- a/exporter/logzioexporter/logziospan_test.go +++ b/exporter/logzioexporter/logziospan_test.go @@ -42,7 +42,7 @@ func TestTransformToDbModelSpan(tester *testing.T) { } newSpan, err := transformToLogzioSpanBytes(&span) require.NoError(tester, err) - var testLogzioSpan logzioSpan + var testLogzioSpan LogzioSpan err = json.Unmarshal(newSpan, &testLogzioSpan) require.NoError(tester, err) dbModelSpan := testLogzioSpan.transformToDbModelSpan() diff --git a/exporter/logzioexporter/testdata/es.json b/exporter/logzioexporter/testdata/es.json new file mode 100644 index 000000000000..8b30c120d9cd --- /dev/null +++ b/exporter/logzioexporter/testdata/es.json @@ -0,0 +1,90 @@ +{ + "traceID": "0000000000000001", + "spanID": "0000000000000002", + "flags": 1, + "operationName": "test-general-conversion", + "references": [ + { + "refType": "CHILD_OF", + "traceID": "0000000000000001", + "spanID": "0000000000000003" + }, + { + "refType": "FOLLOWS_FROM", + "traceID": "0000000000000001", + "spanID": "0000000000000004" + }, + { + "refType": "CHILD_OF", + "traceID": "00000000000000ff", + "spanID": "00000000000000ff" + } + ], + "startTime": 1485467191639875, + "startTimeMillis": 1485467191639, + "duration": 5, + "tags": [ + { + "key": "peer.service", + "type": "string", + "value": "service-y" + }, + { + "key": "peer.ipv4", + "type": "int64", + "value": "23456" + }, + { + "key": "error", + "type": "bool", + "value": "true" + }, + { + "key": "temperature", + "type": "float64", + "value": "72.5" + }, + { + "key": "blob", + "type": "binary", + "value": "00003039" + } + ], + "logs": [ + { + "timestamp": 1485467191639875, + "fields": [ + { + "key": "event", + "type": "int64", + "value": "123415" + } + ] + }, + { + "timestamp": 1485467191639875, + "fields": [ + { + "key": "x", + "type": "string", + "value": "y" + } + ] + } + ], + "process": { + "serviceName": "service-x", + "tags": [ + { + "key": "peer.ipv4", + "type": "int64", + "value": "23456" + }, + { + "key": "error", + "type": "bool", + "value": "true" + } + ] + } +} diff --git a/exporter/logzioexporter/to_domain.go b/exporter/logzioexporter/to_domain.go deleted file mode 100644 index 92aa72682a4c..000000000000 --- a/exporter/logzioexporter/to_domain.go +++ /dev/null @@ -1,255 +0,0 @@ -// Copyright (c) 2019 The Jaeger Authors. -// Copyright (c) 2018 Uber Technologies, Inc. -// SPDX-License-Identifier: Apache-2.0 - -package logzioexporter - -import ( - "encoding/hex" - "encoding/json" - "fmt" - "strconv" - "strings" - - "github.com/jaegertracing/jaeger/model" -) - -// NewToDomain creates ToDomain -func NewToDomain(tagDotReplacement string) ToDomain { - return ToDomain{tagDotReplacement: tagDotReplacement} -} - -// ToDomain is used to convert Span to model.Span -type ToDomain struct { - tagDotReplacement string -} - -// ReplaceDot replaces dot with dotReplacement -func (td ToDomain) ReplaceDot(k string) string { - return strings.ReplaceAll(k, ".", td.tagDotReplacement) -} - -// ReplaceDotReplacement replaces dotReplacement with dot -func (td ToDomain) ReplaceDotReplacement(k string) string { - return strings.ReplaceAll(k, td.tagDotReplacement, ".") -} - -// SpanToDomain converts db span into model Span -func (td ToDomain) SpanToDomain(dbSpan *Span) (*model.Span, error) { - tags, err := td.convertKeyValues(dbSpan.Tags) - if err != nil { - return nil, err - } - logs, err := td.convertLogs(dbSpan.Logs) - if err != nil { - return nil, err - } - refs, err := td.convertRefs(dbSpan.References) - if err != nil { - return nil, err - } - process, err := td.convertProcess(dbSpan.Process) - if err != nil { - return nil, err - } - traceID, err := model.TraceIDFromString(string(dbSpan.TraceID)) - if err != nil { - return nil, err - } - - spanIDInt, err := model.SpanIDFromString(string(dbSpan.SpanID)) - if err != nil { - return nil, err - } - - if dbSpan.ParentSpanID != "" { - parentSpanID, err := model.SpanIDFromString(string(dbSpan.ParentSpanID)) - if err != nil { - return nil, err - } - refs = model.MaybeAddParentSpanID(traceID, parentSpanID, refs) - } - - fieldTags, err := td.convertTagFields(dbSpan.Tag) - if err != nil { - return nil, err - } - tags = append(tags, fieldTags...) - - span := &model.Span{ - TraceID: traceID, - SpanID: model.NewSpanID(uint64(spanIDInt)), - OperationName: dbSpan.OperationName, - References: refs, - Flags: model.Flags(uint32(dbSpan.Flags)), - StartTime: model.EpochMicrosecondsAsTime(dbSpan.StartTime), - Duration: model.MicrosecondsAsDuration(dbSpan.Duration), - Tags: tags, - Logs: logs, - Process: process, - } - return span, nil -} - -func (ToDomain) convertRefs(refs []Reference) ([]model.SpanRef, error) { - retMe := make([]model.SpanRef, len(refs)) - for i, r := range refs { - // There are some inconsistencies with ReferenceTypes, hence the hacky fix. - var refType model.SpanRefType - switch r.RefType { - case ChildOf: - refType = model.ChildOf - case FollowsFrom: - refType = model.FollowsFrom - default: - return nil, fmt.Errorf("not a valid SpanRefType string %s", string(r.RefType)) - } - - traceID, err := model.TraceIDFromString(string(r.TraceID)) - if err != nil { - return nil, err - } - - spanID, err := strconv.ParseUint(string(r.SpanID), 16, 64) - if err != nil { - return nil, err - } - - retMe[i] = model.SpanRef{ - RefType: refType, - TraceID: traceID, - SpanID: model.NewSpanID(spanID), - } - } - return retMe, nil -} - -func (td ToDomain) convertKeyValues(tags []KeyValue) ([]model.KeyValue, error) { - retMe := make([]model.KeyValue, len(tags)) - for i := range tags { - kv, err := td.convertKeyValue(&tags[i]) - if err != nil { - return nil, err - } - retMe[i] = kv - } - return retMe, nil -} - -func (td ToDomain) convertTagFields(tagsMap map[string]any) ([]model.KeyValue, error) { - kvs := make([]model.KeyValue, len(tagsMap)) - i := 0 - for k, v := range tagsMap { - tag, err := td.convertTagField(k, v) - if err != nil { - return nil, err - } - kvs[i] = tag - i++ - } - return kvs, nil -} - -func (td ToDomain) convertTagField(k string, v any) (model.KeyValue, error) { - dKey := td.ReplaceDotReplacement(k) - switch val := v.(type) { - case int64: - return model.Int64(dKey, val), nil - case float64: - return model.Float64(dKey, val), nil - case bool: - return model.Bool(dKey, val), nil - case string: - return model.String(dKey, val), nil - // the binary is never returned, ES returns it as string with base64 encoding - case []byte: - return model.Binary(dKey, val), nil - // in spans are decoded using json.UseNumber() to preserve the type - // however note that float(1) will be parsed as int as ES does not store decimal point - case json.Number: - n, err := val.Int64() - if err == nil { - return model.Int64(dKey, n), nil - } - f, err := val.Float64() - if err == nil { - return model.Float64(dKey, f), nil - } - return model.String("", ""), fmt.Errorf("invalid tag type in %+v: %w", v, err) - default: - return model.String("", ""), fmt.Errorf("invalid tag type in %+v", v) - } -} - -// convertKeyValue expects the Value field to be string, because it only works -// as a reverse transformation after FromDomain() for ElasticSearch model. -func (ToDomain) convertKeyValue(tag *KeyValue) (model.KeyValue, error) { - if tag.Value == nil { - return model.KeyValue{}, fmt.Errorf("invalid nil Value in %v", tag) - } - tagValue, ok := tag.Value.(string) - if !ok { - return model.KeyValue{}, fmt.Errorf("non-string Value of type %t in %v", tag.Value, tag) - } - switch tag.Type { - case StringType: - return model.String(tag.Key, tagValue), nil - case BoolType: - value, err := strconv.ParseBool(tagValue) - if err != nil { - return model.KeyValue{}, err - } - return model.Bool(tag.Key, value), nil - case Int64Type: - value, err := strconv.ParseInt(tagValue, 10, 64) - if err != nil { - return model.KeyValue{}, err - } - return model.Int64(tag.Key, value), nil - case Float64Type: - value, err := strconv.ParseFloat(tagValue, 64) - if err != nil { - return model.KeyValue{}, err - } - return model.Float64(tag.Key, value), nil - case BinaryType: - value, err := hex.DecodeString(tagValue) - if err != nil { - return model.KeyValue{}, err - } - return model.Binary(tag.Key, value), nil - } - return model.KeyValue{}, fmt.Errorf("not a valid ValueType string %s", string(tag.Type)) -} - -func (td ToDomain) convertLogs(logs []Log) ([]model.Log, error) { - retMe := make([]model.Log, len(logs)) - for i, l := range logs { - fields, err := td.convertKeyValues(l.Fields) - if err != nil { - return nil, err - } - retMe[i] = model.Log{ - Timestamp: model.EpochMicrosecondsAsTime(l.Timestamp), - Fields: fields, - } - } - return retMe, nil -} - -func (td ToDomain) convertProcess(process Process) (*model.Process, error) { - tags, err := td.convertKeyValues(process.Tags) - if err != nil { - return nil, err - } - fieldTags, err := td.convertTagFields(process.Tag) - if err != nil { - return nil, err - } - tags = append(tags, fieldTags...) - - return &model.Process{ - Tags: tags, - ServiceName: process.ServiceName, - }, nil -} From df47ed9484035da76f7b7d9fb6955fc934db78aa Mon Sep 17 00:00:00 2001 From: zzzk1 Date: Sun, 29 Dec 2024 19:16:30 +0800 Subject: [PATCH 03/10] add changelog --- .../remove-jaeger-dbmodel-dependency.yaml | 27 ++++ exporter/logzioexporter/exporter_test.go | 2 +- exporter/logzioexporter/from_domain.go | 115 +++++++++++++++--- exporter/logzioexporter/from_domain_test.go | 14 +-- exporter/logzioexporter/logziospan.go | 99 ++------------- exporter/logzioexporter/logziospan_test.go | 2 +- 6 files changed, 143 insertions(+), 116 deletions(-) create mode 100644 .chloggen/remove-jaeger-dbmodel-dependency.yaml diff --git a/.chloggen/remove-jaeger-dbmodel-dependency.yaml b/.chloggen/remove-jaeger-dbmodel-dependency.yaml new file mode 100644 index 000000000000..28738af5dbe5 --- /dev/null +++ b/.chloggen/remove-jaeger-dbmodel-dependency.yaml @@ -0,0 +1,27 @@ +# Use this changelog template to create an entry for release notes. + +# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix' +change_type: enhancement + +# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver) +component: logzioexporter + +# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`). +note: Remove jaeger dbmodel dependency. + +# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists. +issues: [36972] + +# (Optional) One or more lines of additional information to render under the primary note. +# These lines will be padded with 2 spaces and then inserted directly into the document. +# Use pipe (|) for multiline entries. +subtext: + +# If your change doesn't affect end users or the exported elements of any package, +# you should instead start your pull request title with [chore] or use the "Skip Changelog" label. +# Optional: The change log or logs in which this entry should be included. +# e.g. '[user]' or '[user, api]' +# Include 'user' if the change is relevant to end users. +# Include 'api' if there is a change to a library API. +# Default: '[user]' +change_logs: [user] diff --git a/exporter/logzioexporter/exporter_test.go b/exporter/logzioexporter/exporter_test.go index a9628951fa62..2943c50a9413 100644 --- a/exporter/logzioexporter/exporter_test.go +++ b/exporter/logzioexporter/exporter_test.go @@ -255,7 +255,7 @@ func TestPushTraceData(tester *testing.T) { res.Attributes().PutStr(conventions.AttributeHostName, testHost) err := testTracesExporter(tester, td, &cfg) require.NoError(tester, err) - var newSpan LogzioSpan + var newSpan logzioSpan decoded, _ := gUnzipData(recordedRequests) requests := strings.Split(string(decoded), "\n") assert.NoError(tester, json.Unmarshal([]byte(requests[0]), &newSpan)) diff --git a/exporter/logzioexporter/from_domain.go b/exporter/logzioexporter/from_domain.go index a62752c2e7a6..5eb88a8b4216 100644 --- a/exporter/logzioexporter/from_domain.go +++ b/exporter/logzioexporter/from_domain.go @@ -10,31 +10,116 @@ import ( "github.com/jaegertracing/jaeger/model" ) -// NewFromDomain creates FromDomain used to convert model span to db span -func NewFromDomain(allTagsAsObject bool, tagKeysAsFields []string, tagDotReplacement string) FromDomain { +// ReferenceType is the reference type of one span to another +type ReferenceType string + +// TraceID is the shared trace ID of all spans in the trace. +type TraceID string + +// SpanID is the id of a span +type SpanID string + +// ValueType is the type of a value stored in KeyValue struct. +type ValueType string + +const ( + // ChildOf means a span is the child of another span + ChildOf ReferenceType = "CHILD_OF" + // FollowsFrom means a span follows from another span + FollowsFrom ReferenceType = "FOLLOWS_FROM" + + // StringType indicates a string value stored in KeyValue + StringType ValueType = "string" + // BoolType indicates a Boolean value stored in KeyValue + BoolType ValueType = "bool" + // Int64Type indicates a 64bit signed integer value stored in KeyValue + Int64Type ValueType = "int64" + // Float64Type indicates a 64bit float value stored in KeyValue + Float64Type ValueType = "float64" + // BinaryType indicates an arbitrary byte array stored in KeyValue + BinaryType ValueType = "binary" +) + +// Reference is a reference from one span to another +type Reference struct { + RefType ReferenceType `json:"refType"` + TraceID TraceID `json:"traceID"` + SpanID SpanID `json:"spanID"` +} + +// Process is the process emitting a set of spans +type Process struct { + ServiceName string `json:"serviceName"` + Tags []KeyValue `json:"tags"` + // Alternative representation of tags for better kibana support + Tag map[string]any `json:"tag,omitempty"` +} + +// Log is a log emitted in a span +type Log struct { + Timestamp uint64 `json:"timestamp"` + Fields []KeyValue `json:"fields"` +} + +// KeyValue is a key-value pair with typed value. +type KeyValue struct { + Key string `json:"key"` + Type ValueType `json:"type,omitempty"` + Value any `json:"value"` +} + +// Service is the JSON struct for service:operation documents in ElasticSearch +type Service struct { + ServiceName string `json:"serviceName"` + OperationName string `json:"operationName"` +} + +// only for testing Span is ES database representation of the domain span. +type Span struct { + TraceID TraceID `json:"traceID"` + SpanID SpanID `json:"spanID"` + ParentSpanID SpanID `json:"parentSpanID,omitempty"` // deprecated + Flags uint32 `json:"flags,omitempty"` + OperationName string `json:"operationName"` + References []Reference `json:"references"` + StartTime uint64 `json:"startTime"` // microseconds since Unix epoch + // ElasticSearch does not support a UNIX Epoch timestamp in microseconds, + // so Jaeger maps StartTime to a 'long' type. This extra StartTimeMillis field + // works around this issue, enabling timerange queries. + StartTimeMillis uint64 `json:"startTimeMillis"` + Duration uint64 `json:"duration"` // microseconds + Tags []KeyValue `json:"tags"` + // Alternative representation of tags for better kibana support + Tag map[string]any `json:"tag,omitempty"` + Logs []Log `json:"logs"` + Process Process `json:"process,omitempty"` +} + +// newFromDomain creates fromDomain used to convert model span to db span +func newFromDomain(allTagsAsObject bool, tagKeysAsFields []string, tagDotReplacement string) fromDomain { tags := map[string]bool{} for _, k := range tagKeysAsFields { tags[k] = true } - return FromDomain{allTagsAsFields: allTagsAsObject, tagKeysAsFields: tags, tagDotReplacement: tagDotReplacement} + return fromDomain{allTagsAsFields: allTagsAsObject, tagKeysAsFields: tags, tagDotReplacement: tagDotReplacement} } -// FromDomain is used to convert model span to db span -type FromDomain struct { +// fromDomain is used to convert model span to db span +type fromDomain struct { allTagsAsFields bool tagKeysAsFields map[string]bool tagDotReplacement string } -// FromDomainEmbedProcess converts model.Span into json.Span format. +// fromDomainEmbedProcess converts model.Span into json.Span format. // This format includes a ParentSpanID and an embedded Process. -func (fd FromDomain) FromDomainEmbedProcess(span *model.Span) *LogzioSpan { +func (fd fromDomain) fromDomainEmbedProcess(span *model.Span) *logzioSpan { return fd.convertSpanEmbedProcess(span) } -func (fd FromDomain) convertSpanInternal(span *model.Span) LogzioSpan { +func (fd fromDomain) convertSpanInternal(span *model.Span) logzioSpan { tags, tagsMap := fd.convertKeyValuesString(span.Tags) - return LogzioSpan{ + return logzioSpan{ TraceID: TraceID(span.TraceID.String()), SpanID: SpanID(span.SpanID.String()), Flags: uint32(span.Flags), @@ -48,14 +133,14 @@ func (fd FromDomain) convertSpanInternal(span *model.Span) LogzioSpan { } } -func (fd FromDomain) convertSpanEmbedProcess(span *model.Span) *LogzioSpan { +func (fd fromDomain) convertSpanEmbedProcess(span *model.Span) *logzioSpan { s := fd.convertSpanInternal(span) s.Process = fd.convertProcess(span.Process) s.References = fd.convertReferences(span) return &s } -func (fd FromDomain) convertReferences(span *model.Span) []Reference { +func (fd fromDomain) convertReferences(span *model.Span) []Reference { out := make([]Reference, 0, len(span.References)) for _, ref := range span.References { out = append(out, Reference{ @@ -67,14 +152,14 @@ func (fd FromDomain) convertReferences(span *model.Span) []Reference { return out } -func (FromDomain) convertRefType(refType model.SpanRefType) ReferenceType { +func (fromDomain) convertRefType(refType model.SpanRefType) ReferenceType { if refType == model.FollowsFrom { return FollowsFrom } return ChildOf } -func (fd FromDomain) convertKeyValuesString(keyValues model.KeyValues) ([]KeyValue, map[string]any) { +func (fd fromDomain) convertKeyValuesString(keyValues model.KeyValues) ([]KeyValue, map[string]any) { var tagsMap map[string]any var kvs []KeyValue for _, kv := range keyValues { @@ -93,7 +178,7 @@ func (fd FromDomain) convertKeyValuesString(keyValues model.KeyValues) ([]KeyVal return kvs, tagsMap } -func (FromDomain) convertLogs(logs []model.Log) []Log { +func (fromDomain) convertLogs(logs []model.Log) []Log { out := make([]Log, len(logs)) for i, log := range logs { var kvs []KeyValue @@ -108,7 +193,7 @@ func (FromDomain) convertLogs(logs []model.Log) []Log { return out } -func (fd FromDomain) convertProcess(process *model.Process) Process { +func (fd fromDomain) convertProcess(process *model.Process) Process { tags, tagsMap := fd.convertKeyValuesString(process.Tags) return Process{ ServiceName: process.ServiceName, diff --git a/exporter/logzioexporter/from_domain_test.go b/exporter/logzioexporter/from_domain_test.go index 24aae34c0700..f7586bf2c782 100644 --- a/exporter/logzioexporter/from_domain_test.go +++ b/exporter/logzioexporter/from_domain_test.go @@ -24,10 +24,10 @@ func TestFromDomainEmbedProcess(t *testing.T) { var span model.Span require.NoError(t, jsonpb.Unmarshal(bytes.NewReader(domainStr), &span)) - converter := NewFromDomain(false, nil, ":") - embeddedSpan := converter.FromDomainEmbedProcess(&span) + converter := newFromDomain(false, nil, ":") + embeddedSpan := converter.fromDomainEmbedProcess(&span) - var expectedSpan LogzioSpan + var expectedSpan logzioSpan require.NoError(t, json.Unmarshal(jsonStr, &expectedSpan)) testJSONEncoding(t, jsonStr, embeddedSpan.transformToDbModelSpan()) @@ -61,8 +61,8 @@ func testJSONEncoding(t *testing.T, expectedStr []byte, object any) { func TestEmptyTags(t *testing.T) { tags := make([]model.KeyValue, 0) span := model.Span{Tags: tags, Process: &model.Process{Tags: tags}} - converter := NewFromDomain(false, nil, ":") - dbSpan := converter.FromDomainEmbedProcess(&span) + converter := newFromDomain(false, nil, ":") + dbSpan := converter.fromDomainEmbedProcess(&span) assert.Empty(t, dbSpan.Tags) assert.Empty(t, dbSpan.Tag) } @@ -74,8 +74,8 @@ func TestTagMap(t *testing.T) { model.Int64("b.b", 1), } span := model.Span{Tags: tags, Process: &model.Process{Tags: tags}} - converter := NewFromDomain(false, []string{"a", "b.b", "b*"}, ":") - dbSpan := converter.FromDomainEmbedProcess(&span) + converter := newFromDomain(false, []string{"a", "b.b", "b*"}, ":") + dbSpan := converter.fromDomainEmbedProcess(&span) assert.Len(t, dbSpan.Tags, 1) assert.Equal(t, "foo", dbSpan.Tags[0].Key) diff --git a/exporter/logzioexporter/logziospan.go b/exporter/logzioexporter/logziospan.go index dc08e0cc797d..63a7ba772125 100644 --- a/exporter/logzioexporter/logziospan.go +++ b/exporter/logzioexporter/logziospan.go @@ -15,72 +15,8 @@ const ( tagDotReplacementCharacter = "@" ) -// ReferenceType is the reference type of one span to another -type ReferenceType string - -// TraceID is the shared trace ID of all spans in the trace. -type TraceID string - -// SpanID is the id of a span -type SpanID string - -// ValueType is the type of a value stored in KeyValue struct. -type ValueType string - -const ( - // ChildOf means a span is the child of another span - ChildOf ReferenceType = "CHILD_OF" - // FollowsFrom means a span follows from another span - FollowsFrom ReferenceType = "FOLLOWS_FROM" - - // StringType indicates a string value stored in KeyValue - StringType ValueType = "string" - // BoolType indicates a Boolean value stored in KeyValue - BoolType ValueType = "bool" - // Int64Type indicates a 64bit signed integer value stored in KeyValue - Int64Type ValueType = "int64" - // Float64Type indicates a 64bit float value stored in KeyValue - Float64Type ValueType = "float64" - // BinaryType indicates an arbitrary byte array stored in KeyValue - BinaryType ValueType = "binary" -) - -// Reference is a reference from one span to another -type Reference struct { - RefType ReferenceType `json:"refType"` - TraceID TraceID `json:"traceID"` - SpanID SpanID `json:"spanID"` -} - -// Process is the process emitting a set of spans -type Process struct { - ServiceName string `json:"serviceName"` - Tags []KeyValue `json:"tags"` - // Alternative representation of tags for better kibana support - Tag map[string]any `json:"tag,omitempty"` -} - -// Log is a log emitted in a span -type Log struct { - Timestamp uint64 `json:"timestamp"` - Fields []KeyValue `json:"fields"` -} - -// KeyValue is a key-value pair with typed value. -type KeyValue struct { - Key string `json:"key"` - Type ValueType `json:"type,omitempty"` - Value any `json:"value"` -} - -// Service is the JSON struct for service:operation documents in ElasticSearch -type Service struct { - ServiceName string `json:"serviceName"` - OperationName string `json:"operationName"` -} - -// LogzioSpan is same as ESSpan with a few different json field names and an addition on type field. -type LogzioSpan struct { +// logzioSpan is same as ESSpan with a few different json field names and an addition on type field. +type logzioSpan struct { TraceID TraceID `json:"traceID"` SpanID SpanID `json:"spanID"` OperationName string `json:"operationName,omitempty"` @@ -97,27 +33,6 @@ type LogzioSpan struct { Type string `json:"type"` } -// only for testing EsSpan is ES database representation of the domain span. -type EsSpan struct { - TraceID TraceID `json:"traceID"` - SpanID SpanID `json:"spanID"` - ParentSpanID SpanID `json:"parentSpanID,omitempty"` // deprecated - Flags uint32 `json:"flags,omitempty"` - OperationName string `json:"operationName"` - References []Reference `json:"references"` - StartTime uint64 `json:"startTime"` // microseconds since Unix epoch - // ElasticSearch does not support a UNIX Epoch timestamp in microseconds, - // so Jaeger maps StartTime to a 'long' type. This extra StartTimeMillis field - // works around this issue, enabling timerange queries. - StartTimeMillis uint64 `json:"startTimeMillis"` - Duration uint64 `json:"duration"` // microseconds - Tags []KeyValue `json:"tags"` - // Alternative representation of tags for better kibana support - Tag map[string]any `json:"tag,omitempty"` - Logs []Log `json:"logs"` - Process Process `json:"process,omitempty"` -} - func getTagsValues(tags []model.KeyValue) []string { values := make([]string, len(tags)) for i := range tags { @@ -129,9 +44,9 @@ func getTagsValues(tags []model.KeyValue) []string { // transformToLogzioSpanBytes receives a Jaeger span, converts it to logzio span and returns it as a byte array. // The main differences between Jaeger span and logzio span are arrays which are represented as maps func transformToLogzioSpanBytes(span *model.Span) ([]byte, error) { - spanConverter := NewFromDomain(true, getTagsValues(span.Tags), tagDotReplacementCharacter) - jsonSpan := spanConverter.FromDomainEmbedProcess(span) - newSpan := LogzioSpan{ + spanConverter := newFromDomain(true, getTagsValues(span.Tags), tagDotReplacementCharacter) + jsonSpan := spanConverter.fromDomainEmbedProcess(span) + newSpan := logzioSpan{ TraceID: jsonSpan.TraceID, OperationName: jsonSpan.OperationName, SpanID: jsonSpan.SpanID, @@ -151,8 +66,8 @@ func transformToLogzioSpanBytes(span *model.Span) ([]byte, error) { } // only for testing transformToDbModelSpan coverts logz.io span to ElasticSearch span -func (span *LogzioSpan) transformToDbModelSpan() *EsSpan { - return &EsSpan{ +func (span *logzioSpan) transformToDbModelSpan() *Span { + return &Span{ OperationName: span.OperationName, Process: span.Process, Tags: span.Tags, diff --git a/exporter/logzioexporter/logziospan_test.go b/exporter/logzioexporter/logziospan_test.go index cd5620742459..faea671f4b40 100644 --- a/exporter/logzioexporter/logziospan_test.go +++ b/exporter/logzioexporter/logziospan_test.go @@ -42,7 +42,7 @@ func TestTransformToDbModelSpan(tester *testing.T) { } newSpan, err := transformToLogzioSpanBytes(&span) require.NoError(tester, err) - var testLogzioSpan LogzioSpan + var testLogzioSpan logzioSpan err = json.Unmarshal(newSpan, &testLogzioSpan) require.NoError(tester, err) dbModelSpan := testLogzioSpan.transformToDbModelSpan() From 9f2b0f53ea45882cb8bc8b713bf7afb3e6251c8f Mon Sep 17 00:00:00 2001 From: zzzk1 Date: Sun, 29 Dec 2024 19:36:17 +0800 Subject: [PATCH 04/10] public -> private --- exporter/logzioexporter/from_domain.go | 90 ++++++++++----------- exporter/logzioexporter/from_domain_test.go | 16 ++-- exporter/logzioexporter/logziospan.go | 38 ++++----- 3 files changed, 72 insertions(+), 72 deletions(-) diff --git a/exporter/logzioexporter/from_domain.go b/exporter/logzioexporter/from_domain.go index 5eb88a8b4216..05483e165a85 100644 --- a/exporter/logzioexporter/from_domain.go +++ b/exporter/logzioexporter/from_domain.go @@ -19,7 +19,7 @@ type TraceID string // SpanID is the id of a span type SpanID string -// ValueType is the type of a value stored in KeyValue struct. +// ValueType is the type of a value stored in keyValue struct. type ValueType string const ( @@ -28,71 +28,71 @@ const ( // FollowsFrom means a span follows from another span FollowsFrom ReferenceType = "FOLLOWS_FROM" - // StringType indicates a string value stored in KeyValue + // StringType indicates a string value stored in keyValue StringType ValueType = "string" - // BoolType indicates a Boolean value stored in KeyValue + // BoolType indicates a Boolean value stored in keyValue BoolType ValueType = "bool" - // Int64Type indicates a 64bit signed integer value stored in KeyValue + // Int64Type indicates a 64bit signed integer value stored in keyValue Int64Type ValueType = "int64" - // Float64Type indicates a 64bit float value stored in KeyValue + // Float64Type indicates a 64bit float value stored in keyValue Float64Type ValueType = "float64" - // BinaryType indicates an arbitrary byte array stored in KeyValue + // BinaryType indicates an arbitrary byte array stored in keyValue BinaryType ValueType = "binary" ) -// Reference is a reference from one span to another -type Reference struct { +// reference is a reference from one span to another +type reference struct { RefType ReferenceType `json:"refType"` TraceID TraceID `json:"traceID"` SpanID SpanID `json:"spanID"` } -// Process is the process emitting a set of spans -type Process struct { +// process is the process emitting a set of spans +type process struct { ServiceName string `json:"serviceName"` - Tags []KeyValue `json:"tags"` + Tags []keyValue `json:"tags"` // Alternative representation of tags for better kibana support Tag map[string]any `json:"tag,omitempty"` } -// Log is a log emitted in a span -type Log struct { +// log is a log emitted in a span +type log struct { Timestamp uint64 `json:"timestamp"` - Fields []KeyValue `json:"fields"` + Fields []keyValue `json:"fields"` } -// KeyValue is a key-value pair with typed value. -type KeyValue struct { +// keyValue is a key-value pair with typed value. +type keyValue struct { Key string `json:"key"` Type ValueType `json:"type,omitempty"` Value any `json:"value"` } -// Service is the JSON struct for service:operation documents in ElasticSearch -type Service struct { +// service is the JSON struct for service:operation documents in ElasticSearch +type service struct { ServiceName string `json:"serviceName"` OperationName string `json:"operationName"` } -// only for testing Span is ES database representation of the domain span. -type Span struct { +// only for testing span is ES database representation of the domain span. +type span struct { TraceID TraceID `json:"traceID"` SpanID SpanID `json:"spanID"` ParentSpanID SpanID `json:"parentSpanID,omitempty"` // deprecated Flags uint32 `json:"flags,omitempty"` OperationName string `json:"operationName"` - References []Reference `json:"references"` + References []reference `json:"references"` StartTime uint64 `json:"startTime"` // microseconds since Unix epoch // ElasticSearch does not support a UNIX Epoch timestamp in microseconds, // so Jaeger maps StartTime to a 'long' type. This extra StartTimeMillis field // works around this issue, enabling timerange queries. StartTimeMillis uint64 `json:"startTimeMillis"` Duration uint64 `json:"duration"` // microseconds - Tags []KeyValue `json:"tags"` + Tags []keyValue `json:"tags"` // Alternative representation of tags for better kibana support Tag map[string]any `json:"tag,omitempty"` - Logs []Log `json:"logs"` - Process Process `json:"process,omitempty"` + Logs []log `json:"logs"` + Process process `json:"process,omitempty"` } // newFromDomain creates fromDomain used to convert model span to db span @@ -111,8 +111,8 @@ type fromDomain struct { tagDotReplacement string } -// fromDomainEmbedProcess converts model.Span into json.Span format. -// This format includes a ParentSpanID and an embedded Process. +// fromDomainEmbedProcess converts model.span into json.span format. +// This format includes a ParentSpanID and an embedded process. func (fd fromDomain) fromDomainEmbedProcess(span *model.Span) *logzioSpan { return fd.convertSpanEmbedProcess(span) } @@ -140,10 +140,10 @@ func (fd fromDomain) convertSpanEmbedProcess(span *model.Span) *logzioSpan { return &s } -func (fd fromDomain) convertReferences(span *model.Span) []Reference { - out := make([]Reference, 0, len(span.References)) +func (fd fromDomain) convertReferences(span *model.Span) []reference { + out := make([]reference, 0, len(span.References)) for _, ref := range span.References { - out = append(out, Reference{ + out = append(out, reference{ RefType: fd.convertRefType(ref.RefType), TraceID: TraceID(ref.TraceID.String()), SpanID: SpanID(ref.SpanID.String()), @@ -159,9 +159,9 @@ func (fromDomain) convertRefType(refType model.SpanRefType) ReferenceType { return ChildOf } -func (fd fromDomain) convertKeyValuesString(keyValues model.KeyValues) ([]KeyValue, map[string]any) { +func (fd fromDomain) convertKeyValuesString(keyValues model.KeyValues) ([]keyValue, map[string]any) { var tagsMap map[string]any - var kvs []KeyValue + var kvs []keyValue for _, kv := range keyValues { if kv.GetVType() != model.BinaryType && (fd.allTagsAsFields || fd.tagKeysAsFields[kv.Key]) { if tagsMap == nil { @@ -173,37 +173,37 @@ func (fd fromDomain) convertKeyValuesString(keyValues model.KeyValues) ([]KeyVal } } if kvs == nil { - kvs = make([]KeyValue, 0) + kvs = make([]keyValue, 0) } return kvs, tagsMap } -func (fromDomain) convertLogs(logs []model.Log) []Log { - out := make([]Log, len(logs)) - for i, log := range logs { - var kvs []KeyValue - for _, kv := range log.Fields { +func (fromDomain) convertLogs(logs []model.Log) []log { + out := make([]log, len(logs)) + for i, l := range logs { + var kvs []keyValue + for _, kv := range l.Fields { kvs = append(kvs, convertKeyValue(kv)) } - out[i] = Log{ - Timestamp: model.TimeAsEpochMicroseconds(log.Timestamp), + out[i] = log{ + Timestamp: model.TimeAsEpochMicroseconds(l.Timestamp), Fields: kvs, } } return out } -func (fd fromDomain) convertProcess(process *model.Process) Process { - tags, tagsMap := fd.convertKeyValuesString(process.Tags) - return Process{ - ServiceName: process.ServiceName, +func (fd fromDomain) convertProcess(p *model.Process) process { + tags, tagsMap := fd.convertKeyValuesString(p.Tags) + return process{ + ServiceName: p.ServiceName, Tags: tags, Tag: tagsMap, } } -func convertKeyValue(kv model.KeyValue) KeyValue { - return KeyValue{ +func convertKeyValue(kv model.KeyValue) keyValue { + return keyValue{ Key: kv.Key, Type: ValueType(strings.ToLower(kv.VType.String())), Value: kv.AsString(), diff --git a/exporter/logzioexporter/from_domain_test.go b/exporter/logzioexporter/from_domain_test.go index f7586bf2c782..40510fb565fb 100644 --- a/exporter/logzioexporter/from_domain_test.go +++ b/exporter/logzioexporter/from_domain_test.go @@ -98,35 +98,35 @@ func TestConvertKeyValueValue(t *testing.T) { key := "key" tests := []struct { kv model.KeyValue - expected KeyValue + expected keyValue }{ { kv: model.Bool(key, true), - expected: KeyValue{Key: key, Value: "true", Type: "bool"}, + expected: keyValue{Key: key, Value: "true", Type: "bool"}, }, { kv: model.Bool(key, false), - expected: KeyValue{Key: key, Value: "false", Type: "bool"}, + expected: keyValue{Key: key, Value: "false", Type: "bool"}, }, { kv: model.Int64(key, int64(1499)), - expected: KeyValue{Key: key, Value: "1499", Type: "int64"}, + expected: keyValue{Key: key, Value: "1499", Type: "int64"}, }, { kv: model.Float64(key, float64(15.66)), - expected: KeyValue{Key: key, Value: "15.66", Type: "float64"}, + expected: keyValue{Key: key, Value: "15.66", Type: "float64"}, }, { kv: model.String(key, longString), - expected: KeyValue{Key: key, Value: longString, Type: "string"}, + expected: keyValue{Key: key, Value: longString, Type: "string"}, }, { kv: model.Binary(key, []byte(longString)), - expected: KeyValue{Key: key, Value: hex.EncodeToString([]byte(longString)), Type: "binary"}, + expected: keyValue{Key: key, Value: hex.EncodeToString([]byte(longString)), Type: "binary"}, }, { kv: model.KeyValue{VType: 1500, Key: key}, - expected: KeyValue{Key: key, Value: "unknown type 1500", Type: "1500"}, + expected: keyValue{Key: key, Value: "unknown type 1500", Type: "1500"}, }, } diff --git a/exporter/logzioexporter/logziospan.go b/exporter/logzioexporter/logziospan.go index 63a7ba772125..e3dd75cfc4e6 100644 --- a/exporter/logzioexporter/logziospan.go +++ b/exporter/logzioexporter/logziospan.go @@ -15,21 +15,21 @@ const ( tagDotReplacementCharacter = "@" ) -// logzioSpan is same as ESSpan with a few different json field names and an addition on type field. +// logzioSpan is same as esSpan with a few different json field names and an addition on type field. type logzioSpan struct { TraceID TraceID `json:"traceID"` SpanID SpanID `json:"spanID"` OperationName string `json:"operationName,omitempty"` - References []Reference `json:"references"` + References []reference `json:"references"` Flags uint32 `json:"flags,omitempty"` StartTime uint64 `json:"startTime"` StartTimeMillis uint64 `json:"startTimeMillis"` Timestamp uint64 `json:"@timestamp"` Duration uint64 `json:"duration"` - Tags []KeyValue `json:"JaegerTags,omitempty"` + Tags []keyValue `json:"JaegerTags,omitempty"` Tag map[string]any `json:"JaegerTag,omitempty"` - Logs []Log `json:"logs"` - Process Process `json:"process,omitempty"` + Logs []log `json:"logs"` + Process process `json:"process,omitempty"` Type string `json:"type"` } @@ -66,19 +66,19 @@ func transformToLogzioSpanBytes(span *model.Span) ([]byte, error) { } // only for testing transformToDbModelSpan coverts logz.io span to ElasticSearch span -func (span *logzioSpan) transformToDbModelSpan() *Span { - return &Span{ - OperationName: span.OperationName, - Process: span.Process, - Tags: span.Tags, - Tag: span.Tag, - References: span.References, - Logs: span.Logs, - Duration: span.Duration, - StartTimeMillis: span.StartTimeMillis, - StartTime: span.StartTime, - Flags: span.Flags, - SpanID: span.SpanID, - TraceID: span.TraceID, +func (logziospan *logzioSpan) transformToDbModelSpan() *span { + return &span{ + OperationName: logziospan.OperationName, + Process: logziospan.Process, + Tags: logziospan.Tags, + Tag: logziospan.Tag, + References: logziospan.References, + Logs: logziospan.Logs, + Duration: logziospan.Duration, + StartTimeMillis: logziospan.StartTimeMillis, + StartTime: logziospan.StartTime, + Flags: logziospan.Flags, + SpanID: logziospan.SpanID, + TraceID: logziospan.TraceID, } } From f161913eb77a47daaa725331e69baa719826fd15 Mon Sep 17 00:00:00 2001 From: zzzk1 Date: Sun, 29 Dec 2024 23:32:34 +0800 Subject: [PATCH 05/10] rename `log` struct --- exporter/logzioexporter/from_domain.go | 14 +++++++------- exporter/logzioexporter/logziospan.go | 2 +- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/exporter/logzioexporter/from_domain.go b/exporter/logzioexporter/from_domain.go index 05483e165a85..c5f5bae125d1 100644 --- a/exporter/logzioexporter/from_domain.go +++ b/exporter/logzioexporter/from_domain.go @@ -2,7 +2,7 @@ // Copyright (c) 2018 Uber Technologies, Inc. // SPDX-License-Identifier: Apache-2.0 -package logzioexporter +package logzioexporter // import "github.com/open-telemetry/opentelemetry-collector-contrib/exporter/logzioexporter" import ( "strings" @@ -55,8 +55,8 @@ type process struct { Tag map[string]any `json:"tag,omitempty"` } -// log is a log emitted in a span -type log struct { +// spanLog is a log emitted in a span +type spanLog struct { Timestamp uint64 `json:"timestamp"` Fields []keyValue `json:"fields"` } @@ -91,7 +91,7 @@ type span struct { Tags []keyValue `json:"tags"` // Alternative representation of tags for better kibana support Tag map[string]any `json:"tag,omitempty"` - Logs []log `json:"logs"` + Logs []spanLog `json:"logs"` Process process `json:"process,omitempty"` } @@ -178,14 +178,14 @@ func (fd fromDomain) convertKeyValuesString(keyValues model.KeyValues) ([]keyVal return kvs, tagsMap } -func (fromDomain) convertLogs(logs []model.Log) []log { - out := make([]log, len(logs)) +func (fromDomain) convertLogs(logs []model.Log) []spanLog { + out := make([]spanLog, len(logs)) for i, l := range logs { var kvs []keyValue for _, kv := range l.Fields { kvs = append(kvs, convertKeyValue(kv)) } - out[i] = log{ + out[i] = spanLog{ Timestamp: model.TimeAsEpochMicroseconds(l.Timestamp), Fields: kvs, } diff --git a/exporter/logzioexporter/logziospan.go b/exporter/logzioexporter/logziospan.go index e3dd75cfc4e6..78c87d290505 100644 --- a/exporter/logzioexporter/logziospan.go +++ b/exporter/logzioexporter/logziospan.go @@ -28,7 +28,7 @@ type logzioSpan struct { Duration uint64 `json:"duration"` Tags []keyValue `json:"JaegerTags,omitempty"` Tag map[string]any `json:"JaegerTag,omitempty"` - Logs []log `json:"logs"` + Logs []spanLog `json:"logs"` Process process `json:"process,omitempty"` Type string `json:"type"` } From 1a939f1cae4139e849d3275d9987589f0fe33c7a Mon Sep 17 00:00:00 2001 From: zzzk1 Date: Mon, 30 Dec 2024 21:09:35 +0800 Subject: [PATCH 06/10] remove to dbmodel package --- exporter/logzioexporter/from_domain.go | 139 ++++-------------- exporter/logzioexporter/from_domain_test.go | 18 ++- .../logzioexporter/internal/dbmodel/model.go | 90 ++++++++++++ exporter/logzioexporter/logziospan.go | 60 ++++---- 4 files changed, 159 insertions(+), 148 deletions(-) create mode 100644 exporter/logzioexporter/internal/dbmodel/model.go diff --git a/exporter/logzioexporter/from_domain.go b/exporter/logzioexporter/from_domain.go index c5f5bae125d1..8f5955cb049c 100644 --- a/exporter/logzioexporter/from_domain.go +++ b/exporter/logzioexporter/from_domain.go @@ -8,93 +8,10 @@ import ( "strings" "github.com/jaegertracing/jaeger/model" -) - -// ReferenceType is the reference type of one span to another -type ReferenceType string - -// TraceID is the shared trace ID of all spans in the trace. -type TraceID string - -// SpanID is the id of a span -type SpanID string - -// ValueType is the type of a value stored in keyValue struct. -type ValueType string -const ( - // ChildOf means a span is the child of another span - ChildOf ReferenceType = "CHILD_OF" - // FollowsFrom means a span follows from another span - FollowsFrom ReferenceType = "FOLLOWS_FROM" - - // StringType indicates a string value stored in keyValue - StringType ValueType = "string" - // BoolType indicates a Boolean value stored in keyValue - BoolType ValueType = "bool" - // Int64Type indicates a 64bit signed integer value stored in keyValue - Int64Type ValueType = "int64" - // Float64Type indicates a 64bit float value stored in keyValue - Float64Type ValueType = "float64" - // BinaryType indicates an arbitrary byte array stored in keyValue - BinaryType ValueType = "binary" + "github.com/open-telemetry/opentelemetry-collector-contrib/exporter/logzioexporter/internal/dbmodel" ) -// reference is a reference from one span to another -type reference struct { - RefType ReferenceType `json:"refType"` - TraceID TraceID `json:"traceID"` - SpanID SpanID `json:"spanID"` -} - -// process is the process emitting a set of spans -type process struct { - ServiceName string `json:"serviceName"` - Tags []keyValue `json:"tags"` - // Alternative representation of tags for better kibana support - Tag map[string]any `json:"tag,omitempty"` -} - -// spanLog is a log emitted in a span -type spanLog struct { - Timestamp uint64 `json:"timestamp"` - Fields []keyValue `json:"fields"` -} - -// keyValue is a key-value pair with typed value. -type keyValue struct { - Key string `json:"key"` - Type ValueType `json:"type,omitempty"` - Value any `json:"value"` -} - -// service is the JSON struct for service:operation documents in ElasticSearch -type service struct { - ServiceName string `json:"serviceName"` - OperationName string `json:"operationName"` -} - -// only for testing span is ES database representation of the domain span. -type span struct { - TraceID TraceID `json:"traceID"` - SpanID SpanID `json:"spanID"` - ParentSpanID SpanID `json:"parentSpanID,omitempty"` // deprecated - Flags uint32 `json:"flags,omitempty"` - OperationName string `json:"operationName"` - References []reference `json:"references"` - StartTime uint64 `json:"startTime"` // microseconds since Unix epoch - // ElasticSearch does not support a UNIX Epoch timestamp in microseconds, - // so Jaeger maps StartTime to a 'long' type. This extra StartTimeMillis field - // works around this issue, enabling timerange queries. - StartTimeMillis uint64 `json:"startTimeMillis"` - Duration uint64 `json:"duration"` // microseconds - Tags []keyValue `json:"tags"` - // Alternative representation of tags for better kibana support - Tag map[string]any `json:"tag,omitempty"` - Logs []spanLog `json:"logs"` - Process process `json:"process,omitempty"` -} - // newFromDomain creates fromDomain used to convert model span to db span func newFromDomain(allTagsAsObject bool, tagKeysAsFields []string, tagDotReplacement string) fromDomain { tags := map[string]bool{} @@ -120,8 +37,8 @@ func (fd fromDomain) fromDomainEmbedProcess(span *model.Span) *logzioSpan { func (fd fromDomain) convertSpanInternal(span *model.Span) logzioSpan { tags, tagsMap := fd.convertKeyValuesString(span.Tags) return logzioSpan{ - TraceID: TraceID(span.TraceID.String()), - SpanID: SpanID(span.SpanID.String()), + TraceID: dbmodel.TraceID(span.TraceID.String()), + SpanID: dbmodel.SpanID(span.SpanID.String()), Flags: uint32(span.Flags), OperationName: span.OperationName, StartTime: model.TimeAsEpochMicroseconds(span.StartTime), @@ -140,28 +57,28 @@ func (fd fromDomain) convertSpanEmbedProcess(span *model.Span) *logzioSpan { return &s } -func (fd fromDomain) convertReferences(span *model.Span) []reference { - out := make([]reference, 0, len(span.References)) +func (fd fromDomain) convertReferences(span *model.Span) []dbmodel.Reference { + out := make([]dbmodel.Reference, 0, len(span.References)) for _, ref := range span.References { - out = append(out, reference{ + out = append(out, dbmodel.Reference{ RefType: fd.convertRefType(ref.RefType), - TraceID: TraceID(ref.TraceID.String()), - SpanID: SpanID(ref.SpanID.String()), + TraceID: dbmodel.TraceID(ref.TraceID.String()), + SpanID: dbmodel.SpanID(ref.SpanID.String()), }) } return out } -func (fromDomain) convertRefType(refType model.SpanRefType) ReferenceType { +func (fromDomain) convertRefType(refType model.SpanRefType) dbmodel.ReferenceType { if refType == model.FollowsFrom { - return FollowsFrom + return dbmodel.FollowsFrom } - return ChildOf + return dbmodel.ChildOf } -func (fd fromDomain) convertKeyValuesString(keyValues model.KeyValues) ([]keyValue, map[string]any) { +func (fd fromDomain) convertKeyValuesString(keyValues model.KeyValues) ([]dbmodel.KeyValue, map[string]any) { var tagsMap map[string]any - var kvs []keyValue + var kvs []dbmodel.KeyValue for _, kv := range keyValues { if kv.GetVType() != model.BinaryType && (fd.allTagsAsFields || fd.tagKeysAsFields[kv.Key]) { if tagsMap == nil { @@ -173,39 +90,39 @@ func (fd fromDomain) convertKeyValuesString(keyValues model.KeyValues) ([]keyVal } } if kvs == nil { - kvs = make([]keyValue, 0) + kvs = make([]dbmodel.KeyValue, 0) } return kvs, tagsMap } -func (fromDomain) convertLogs(logs []model.Log) []spanLog { - out := make([]spanLog, len(logs)) - for i, l := range logs { - var kvs []keyValue - for _, kv := range l.Fields { +func (fromDomain) convertLogs(logs []model.Log) []dbmodel.Log { + out := make([]dbmodel.Log, len(logs)) + for i, log := range logs { + var kvs []dbmodel.KeyValue + for _, kv := range log.Fields { kvs = append(kvs, convertKeyValue(kv)) } - out[i] = spanLog{ - Timestamp: model.TimeAsEpochMicroseconds(l.Timestamp), + out[i] = dbmodel.Log{ + Timestamp: model.TimeAsEpochMicroseconds(log.Timestamp), Fields: kvs, } } return out } -func (fd fromDomain) convertProcess(p *model.Process) process { - tags, tagsMap := fd.convertKeyValuesString(p.Tags) - return process{ - ServiceName: p.ServiceName, +func (fd fromDomain) convertProcess(process *model.Process) dbmodel.Process { + tags, tagsMap := fd.convertKeyValuesString(process.Tags) + return dbmodel.Process{ + ServiceName: process.ServiceName, Tags: tags, Tag: tagsMap, } } -func convertKeyValue(kv model.KeyValue) keyValue { - return keyValue{ +func convertKeyValue(kv model.KeyValue) dbmodel.KeyValue { + return dbmodel.KeyValue{ Key: kv.Key, - Type: ValueType(strings.ToLower(kv.VType.String())), + Type: dbmodel.ValueType(strings.ToLower(kv.VType.String())), Value: kv.AsString(), } } diff --git a/exporter/logzioexporter/from_domain_test.go b/exporter/logzioexporter/from_domain_test.go index 40510fb565fb..d0b41e358471 100644 --- a/exporter/logzioexporter/from_domain_test.go +++ b/exporter/logzioexporter/from_domain_test.go @@ -17,6 +17,8 @@ import ( "github.com/stretchr/testify/require" "github.com/jaegertracing/jaeger/model" + + "github.com/open-telemetry/opentelemetry-collector-contrib/exporter/logzioexporter/internal/dbmodel" ) func TestFromDomainEmbedProcess(t *testing.T) { @@ -98,35 +100,35 @@ func TestConvertKeyValueValue(t *testing.T) { key := "key" tests := []struct { kv model.KeyValue - expected keyValue + expected dbmodel.KeyValue }{ { kv: model.Bool(key, true), - expected: keyValue{Key: key, Value: "true", Type: "bool"}, + expected: dbmodel.KeyValue{Key: key, Value: "true", Type: "bool"}, }, { kv: model.Bool(key, false), - expected: keyValue{Key: key, Value: "false", Type: "bool"}, + expected: dbmodel.KeyValue{Key: key, Value: "false", Type: "bool"}, }, { kv: model.Int64(key, int64(1499)), - expected: keyValue{Key: key, Value: "1499", Type: "int64"}, + expected: dbmodel.KeyValue{Key: key, Value: "1499", Type: "int64"}, }, { kv: model.Float64(key, float64(15.66)), - expected: keyValue{Key: key, Value: "15.66", Type: "float64"}, + expected: dbmodel.KeyValue{Key: key, Value: "15.66", Type: "float64"}, }, { kv: model.String(key, longString), - expected: keyValue{Key: key, Value: longString, Type: "string"}, + expected: dbmodel.KeyValue{Key: key, Value: longString, Type: "string"}, }, { kv: model.Binary(key, []byte(longString)), - expected: keyValue{Key: key, Value: hex.EncodeToString([]byte(longString)), Type: "binary"}, + expected: dbmodel.KeyValue{Key: key, Value: hex.EncodeToString([]byte(longString)), Type: "binary"}, }, { kv: model.KeyValue{VType: 1500, Key: key}, - expected: keyValue{Key: key, Value: "unknown type 1500", Type: "1500"}, + expected: dbmodel.KeyValue{Key: key, Value: "unknown type 1500", Type: "1500"}, }, } diff --git a/exporter/logzioexporter/internal/dbmodel/model.go b/exporter/logzioexporter/internal/dbmodel/model.go new file mode 100644 index 000000000000..d849c4885733 --- /dev/null +++ b/exporter/logzioexporter/internal/dbmodel/model.go @@ -0,0 +1,90 @@ +// Copyright (c) 2019 The Jaeger Authors. +// Copyright (c) 2018 Uber Technologies, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package dbmodel + +// ReferenceType is the reference type of one span to another +type ReferenceType string + +// TraceID is the shared trace ID of all spans in the trace. +type TraceID string + +// SpanID is the id of a span +type SpanID string + +// ValueType is the type of a value stored in KeyValue struct. +type ValueType string + +const ( + // ChildOf means a span is the child of another span + ChildOf ReferenceType = "CHILD_OF" + // FollowsFrom means a span follows from another span + FollowsFrom ReferenceType = "FOLLOWS_FROM" + + // StringType indicates a string value stored in KeyValue + StringType ValueType = "string" + // BoolType indicates a Boolean value stored in KeyValue + BoolType ValueType = "bool" + // Int64Type indicates a 64bit signed integer value stored in KeyValue + Int64Type ValueType = "int64" + // Float64Type indicates a 64bit float value stored in KeyValue + Float64Type ValueType = "float64" + // BinaryType indicates an arbitrary byte array stored in KeyValue + BinaryType ValueType = "binary" +) + +// Span is ES database representation of the domain span. +type Span struct { + TraceID TraceID `json:"traceID"` + SpanID SpanID `json:"spanID"` + ParentSpanID SpanID `json:"parentSpanID,omitempty"` // deprecated + Flags uint32 `json:"flags,omitempty"` + OperationName string `json:"operationName"` + References []Reference `json:"references"` + StartTime uint64 `json:"startTime"` // microseconds since Unix epoch + // ElasticSearch does not support a UNIX Epoch timestamp in microseconds, + // so Jaeger maps StartTime to a 'long' type. This extra StartTimeMillis field + // works around this issue, enabling timerange queries. + StartTimeMillis uint64 `json:"startTimeMillis"` + Duration uint64 `json:"duration"` // microseconds + Tags []KeyValue `json:"tags"` + // Alternative representation of tags for better kibana support + Tag map[string]any `json:"tag,omitempty"` + Logs []Log `json:"logs"` + Process Process `json:"process,omitempty"` +} + +// Reference is a reference from one span to another +type Reference struct { + RefType ReferenceType `json:"refType"` + TraceID TraceID `json:"traceID"` + SpanID SpanID `json:"spanID"` +} + +// Process is the process emitting a set of spans +type Process struct { + ServiceName string `json:"serviceName"` + Tags []KeyValue `json:"tags"` + // Alternative representation of tags for better kibana support + Tag map[string]any `json:"tag,omitempty"` +} + +// Log is a log emitted in a span +type Log struct { + Timestamp uint64 `json:"timestamp"` + Fields []KeyValue `json:"fields"` +} + +// KeyValue is a key-value pair with typed value. +type KeyValue struct { + Key string `json:"key"` + Type ValueType `json:"type,omitempty"` + Value any `json:"value"` +} + +// Service is the JSON struct for service:operation documents in ElasticSearch +type Service struct { + ServiceName string `json:"serviceName"` + OperationName string `json:"operationName"` +} diff --git a/exporter/logzioexporter/logziospan.go b/exporter/logzioexporter/logziospan.go index 78c87d290505..2c2512c2811e 100644 --- a/exporter/logzioexporter/logziospan.go +++ b/exporter/logzioexporter/logziospan.go @@ -7,6 +7,8 @@ import ( "encoding/json" "github.com/jaegertracing/jaeger/model" + + "github.com/open-telemetry/opentelemetry-collector-contrib/exporter/logzioexporter/internal/dbmodel" ) const ( @@ -17,20 +19,20 @@ const ( // logzioSpan is same as esSpan with a few different json field names and an addition on type field. type logzioSpan struct { - TraceID TraceID `json:"traceID"` - SpanID SpanID `json:"spanID"` - OperationName string `json:"operationName,omitempty"` - References []reference `json:"references"` - Flags uint32 `json:"flags,omitempty"` - StartTime uint64 `json:"startTime"` - StartTimeMillis uint64 `json:"startTimeMillis"` - Timestamp uint64 `json:"@timestamp"` - Duration uint64 `json:"duration"` - Tags []keyValue `json:"JaegerTags,omitempty"` - Tag map[string]any `json:"JaegerTag,omitempty"` - Logs []spanLog `json:"logs"` - Process process `json:"process,omitempty"` - Type string `json:"type"` + TraceID dbmodel.TraceID `json:"traceID"` + OperationName string `json:"operationName,omitempty"` + SpanID dbmodel.SpanID `json:"spanID"` + References []dbmodel.Reference `json:"references"` + Flags uint32 `json:"flags,omitempty"` + StartTime uint64 `json:"startTime"` + StartTimeMillis uint64 `json:"startTimeMillis"` + Timestamp uint64 `json:"@timestamp"` + Duration uint64 `json:"duration"` + Tags []dbmodel.KeyValue `json:"JaegerTags,omitempty"` + Tag map[string]any `json:"JaegerTag,omitempty"` + Logs []dbmodel.Log `json:"logs"` + Process dbmodel.Process `json:"process,omitempty"` + Type string `json:"type"` } func getTagsValues(tags []model.KeyValue) []string { @@ -65,20 +67,20 @@ func transformToLogzioSpanBytes(span *model.Span) ([]byte, error) { return json.Marshal(newSpan) } -// only for testing transformToDbModelSpan coverts logz.io span to ElasticSearch span -func (logziospan *logzioSpan) transformToDbModelSpan() *span { - return &span{ - OperationName: logziospan.OperationName, - Process: logziospan.Process, - Tags: logziospan.Tags, - Tag: logziospan.Tag, - References: logziospan.References, - Logs: logziospan.Logs, - Duration: logziospan.Duration, - StartTimeMillis: logziospan.StartTimeMillis, - StartTime: logziospan.StartTime, - Flags: logziospan.Flags, - SpanID: logziospan.SpanID, - TraceID: logziospan.TraceID, +// transformToDbModelSpan coverts logz.io span to ElasticSearch span +func (span *logzioSpan) transformToDbModelSpan() *dbmodel.Span { + return &dbmodel.Span{ + OperationName: span.OperationName, + Process: span.Process, + Tags: span.Tags, + Tag: span.Tag, + References: span.References, + Logs: span.Logs, + Duration: span.Duration, + StartTimeMillis: span.StartTimeMillis, + StartTime: span.StartTime, + Flags: span.Flags, + SpanID: span.SpanID, + TraceID: span.TraceID, } } From f0dacaa079cc5bb6ebd38eb8e210c3066f51057a Mon Sep 17 00:00:00 2001 From: zzzk1 Date: Mon, 30 Dec 2024 22:16:36 +0800 Subject: [PATCH 07/10] improve test --- exporter/logzioexporter/from_domain_test.go | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/exporter/logzioexporter/from_domain_test.go b/exporter/logzioexporter/from_domain_test.go index d0b41e358471..b513d3fb6214 100644 --- a/exporter/logzioexporter/from_domain_test.go +++ b/exporter/logzioexporter/from_domain_test.go @@ -22,7 +22,7 @@ import ( ) func TestFromDomainEmbedProcess(t *testing.T) { - domainStr, jsonStr := loadFixtures(t) + domainStr, jsonStr := loadModel(t) var span model.Span require.NoError(t, jsonpb.Unmarshal(bytes.NewReader(domainStr), &span)) @@ -35,8 +35,8 @@ func TestFromDomainEmbedProcess(t *testing.T) { testJSONEncoding(t, jsonStr, embeddedSpan.transformToDbModelSpan()) } -// Loads and returns domain model and JSON model fixtures with given number i. -func loadFixtures(t *testing.T) ([]byte, []byte) { +// Loads and returns domain model and JSON model. +func loadModel(t *testing.T) ([]byte, []byte) { in := fmt.Sprintf("./testdata/span.json") inStr, err := os.ReadFile(in) require.NoError(t, err) @@ -50,12 +50,9 @@ func testJSONEncoding(t *testing.T, expectedStr []byte, object any) { buf := &bytes.Buffer{} enc := json.NewEncoder(buf) enc.SetIndent("", " ") - - outFile := fmt.Sprintf("./testdata/es.json") require.NoError(t, enc.Encode(object)) - if !assert.Equal(t, string(expectedStr), buf.String()) { - err := os.WriteFile(outFile+"-actual.json", buf.Bytes(), 0o644) + err := os.WriteFile("model-actual.json", buf.Bytes(), 0o644) require.NoError(t, err) } } From 43e10a509761cf80df04b8ee5784b9b68115b299 Mon Sep 17 00:00:00 2001 From: zzzk1 Date: Tue, 31 Dec 2024 23:27:21 +0800 Subject: [PATCH 08/10] remove to internal --- exporter/logzioexporter/exporter.go | 4 +- exporter/logzioexporter/exporter_test.go | 4 +- .../{ => internal/dbmodel}/from_domain.go | 56 ++++++------ .../dbmodel}/from_domain_test.go | 28 +++--- .../internal/dbmodel/logziospan.go | 66 ++++++++++++++ .../internal/dbmodel/logziospan_test.go | 33 +++++++ .../logzioexporter/internal/dbmodel/model.go | 21 ----- exporter/logzioexporter/logziospan.go | 86 ------------------- exporter/logzioexporter/logziospan_test.go | 50 ----------- .../testdata/{es.json => logziospan.json} | 10 ++- 10 files changed, 151 insertions(+), 207 deletions(-) rename exporter/logzioexporter/{ => internal/dbmodel}/from_domain.go (64%) rename exporter/logzioexporter/{ => internal/dbmodel}/from_domain_test.go (80%) create mode 100644 exporter/logzioexporter/internal/dbmodel/logziospan.go create mode 100644 exporter/logzioexporter/internal/dbmodel/logziospan_test.go delete mode 100644 exporter/logzioexporter/logziospan.go delete mode 100644 exporter/logzioexporter/logziospan_test.go rename exporter/logzioexporter/testdata/{es.json => logziospan.json} (96%) diff --git a/exporter/logzioexporter/exporter.go b/exporter/logzioexporter/exporter.go index a4cc01780ec9..30d9932abd07 100644 --- a/exporter/logzioexporter/exporter.go +++ b/exporter/logzioexporter/exporter.go @@ -15,6 +15,8 @@ import ( "strconv" "time" + "github.com/open-telemetry/opentelemetry-collector-contrib/exporter/logzioexporter/internal/dbmodel" + "github.com/hashicorp/go-hclog" "github.com/jaegertracing/jaeger/model" "github.com/jaegertracing/jaeger/pkg/cache" @@ -189,7 +191,7 @@ func (exporter *logzioExporter) pushTraceData(ctx context.Context, traces ptrace span.Process = batch.Process span.Tags = exporter.dropEmptyTags(span.Tags) span.Process.Tags = exporter.dropEmptyTags(span.Process.Tags) - logzioSpan, transformErr := transformToLogzioSpanBytes(span) + logzioSpan, transformErr := dbmodel.TransformToLogzioSpanBytes(span) if transformErr != nil { return transformErr } diff --git a/exporter/logzioexporter/exporter_test.go b/exporter/logzioexporter/exporter_test.go index 2943c50a9413..0dbd06dee86d 100644 --- a/exporter/logzioexporter/exporter_test.go +++ b/exporter/logzioexporter/exporter_test.go @@ -16,6 +16,8 @@ import ( "testing" "time" + "github.com/open-telemetry/opentelemetry-collector-contrib/exporter/logzioexporter/internal/dbmodel" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.opentelemetry.io/collector/component/componenttest" @@ -255,7 +257,7 @@ func TestPushTraceData(tester *testing.T) { res.Attributes().PutStr(conventions.AttributeHostName, testHost) err := testTracesExporter(tester, td, &cfg) require.NoError(tester, err) - var newSpan logzioSpan + var newSpan dbmodel.LogzioSpan decoded, _ := gUnzipData(recordedRequests) requests := strings.Split(string(decoded), "\n") assert.NoError(tester, json.Unmarshal([]byte(requests[0]), &newSpan)) diff --git a/exporter/logzioexporter/from_domain.go b/exporter/logzioexporter/internal/dbmodel/from_domain.go similarity index 64% rename from exporter/logzioexporter/from_domain.go rename to exporter/logzioexporter/internal/dbmodel/from_domain.go index 8f5955cb049c..1f02075f618e 100644 --- a/exporter/logzioexporter/from_domain.go +++ b/exporter/logzioexporter/internal/dbmodel/from_domain.go @@ -2,14 +2,12 @@ // Copyright (c) 2018 Uber Technologies, Inc. // SPDX-License-Identifier: Apache-2.0 -package logzioexporter // import "github.com/open-telemetry/opentelemetry-collector-contrib/exporter/logzioexporter" +package dbmodel // import "github.com/open-telemetry/opentelemetry-collector-contrib/exporter/logzioexporter" import ( "strings" "github.com/jaegertracing/jaeger/model" - - "github.com/open-telemetry/opentelemetry-collector-contrib/exporter/logzioexporter/internal/dbmodel" ) // newFromDomain creates fromDomain used to convert model span to db span @@ -30,15 +28,15 @@ type fromDomain struct { // fromDomainEmbedProcess converts model.span into json.span format. // This format includes a ParentSpanID and an embedded process. -func (fd fromDomain) fromDomainEmbedProcess(span *model.Span) *logzioSpan { +func (fd fromDomain) fromDomainEmbedProcess(span *model.Span) *LogzioSpan { return fd.convertSpanEmbedProcess(span) } -func (fd fromDomain) convertSpanInternal(span *model.Span) logzioSpan { +func (fd fromDomain) convertSpanInternal(span *model.Span) LogzioSpan { tags, tagsMap := fd.convertKeyValuesString(span.Tags) - return logzioSpan{ - TraceID: dbmodel.TraceID(span.TraceID.String()), - SpanID: dbmodel.SpanID(span.SpanID.String()), + return LogzioSpan{ + TraceID: TraceID(span.TraceID.String()), + SpanID: SpanID(span.SpanID.String()), Flags: uint32(span.Flags), OperationName: span.OperationName, StartTime: model.TimeAsEpochMicroseconds(span.StartTime), @@ -50,35 +48,35 @@ func (fd fromDomain) convertSpanInternal(span *model.Span) logzioSpan { } } -func (fd fromDomain) convertSpanEmbedProcess(span *model.Span) *logzioSpan { +func (fd fromDomain) convertSpanEmbedProcess(span *model.Span) *LogzioSpan { s := fd.convertSpanInternal(span) s.Process = fd.convertProcess(span.Process) s.References = fd.convertReferences(span) return &s } -func (fd fromDomain) convertReferences(span *model.Span) []dbmodel.Reference { - out := make([]dbmodel.Reference, 0, len(span.References)) +func (fd fromDomain) convertReferences(span *model.Span) []Reference { + out := make([]Reference, 0, len(span.References)) for _, ref := range span.References { - out = append(out, dbmodel.Reference{ + out = append(out, Reference{ RefType: fd.convertRefType(ref.RefType), - TraceID: dbmodel.TraceID(ref.TraceID.String()), - SpanID: dbmodel.SpanID(ref.SpanID.String()), + TraceID: TraceID(ref.TraceID.String()), + SpanID: SpanID(ref.SpanID.String()), }) } return out } -func (fromDomain) convertRefType(refType model.SpanRefType) dbmodel.ReferenceType { +func (fromDomain) convertRefType(refType model.SpanRefType) ReferenceType { if refType == model.FollowsFrom { - return dbmodel.FollowsFrom + return FollowsFrom } - return dbmodel.ChildOf + return ChildOf } -func (fd fromDomain) convertKeyValuesString(keyValues model.KeyValues) ([]dbmodel.KeyValue, map[string]any) { +func (fd fromDomain) convertKeyValuesString(keyValues model.KeyValues) ([]KeyValue, map[string]any) { var tagsMap map[string]any - var kvs []dbmodel.KeyValue + var kvs []KeyValue for _, kv := range keyValues { if kv.GetVType() != model.BinaryType && (fd.allTagsAsFields || fd.tagKeysAsFields[kv.Key]) { if tagsMap == nil { @@ -90,19 +88,19 @@ func (fd fromDomain) convertKeyValuesString(keyValues model.KeyValues) ([]dbmode } } if kvs == nil { - kvs = make([]dbmodel.KeyValue, 0) + kvs = make([]KeyValue, 0) } return kvs, tagsMap } -func (fromDomain) convertLogs(logs []model.Log) []dbmodel.Log { - out := make([]dbmodel.Log, len(logs)) +func (fromDomain) convertLogs(logs []model.Log) []Log { + out := make([]Log, len(logs)) for i, log := range logs { - var kvs []dbmodel.KeyValue + var kvs []KeyValue for _, kv := range log.Fields { kvs = append(kvs, convertKeyValue(kv)) } - out[i] = dbmodel.Log{ + out[i] = Log{ Timestamp: model.TimeAsEpochMicroseconds(log.Timestamp), Fields: kvs, } @@ -110,19 +108,19 @@ func (fromDomain) convertLogs(logs []model.Log) []dbmodel.Log { return out } -func (fd fromDomain) convertProcess(process *model.Process) dbmodel.Process { +func (fd fromDomain) convertProcess(process *model.Process) Process { tags, tagsMap := fd.convertKeyValuesString(process.Tags) - return dbmodel.Process{ + return Process{ ServiceName: process.ServiceName, Tags: tags, Tag: tagsMap, } } -func convertKeyValue(kv model.KeyValue) dbmodel.KeyValue { - return dbmodel.KeyValue{ +func convertKeyValue(kv model.KeyValue) KeyValue { + return KeyValue{ Key: kv.Key, - Type: dbmodel.ValueType(strings.ToLower(kv.VType.String())), + Type: ValueType(strings.ToLower(kv.VType.String())), Value: kv.AsString(), } } diff --git a/exporter/logzioexporter/from_domain_test.go b/exporter/logzioexporter/internal/dbmodel/from_domain_test.go similarity index 80% rename from exporter/logzioexporter/from_domain_test.go rename to exporter/logzioexporter/internal/dbmodel/from_domain_test.go index b513d3fb6214..05e6a8e61957 100644 --- a/exporter/logzioexporter/from_domain_test.go +++ b/exporter/logzioexporter/internal/dbmodel/from_domain_test.go @@ -2,7 +2,7 @@ // Copyright (c) 2018 Uber Technologies, Inc. // SPDX-License-Identifier: Apache-2.0 -package logzioexporter +package dbmodel import ( "bytes" @@ -17,8 +17,6 @@ import ( "github.com/stretchr/testify/require" "github.com/jaegertracing/jaeger/model" - - "github.com/open-telemetry/opentelemetry-collector-contrib/exporter/logzioexporter/internal/dbmodel" ) func TestFromDomainEmbedProcess(t *testing.T) { @@ -29,18 +27,18 @@ func TestFromDomainEmbedProcess(t *testing.T) { converter := newFromDomain(false, nil, ":") embeddedSpan := converter.fromDomainEmbedProcess(&span) - var expectedSpan logzioSpan + var expectedSpan LogzioSpan require.NoError(t, json.Unmarshal(jsonStr, &expectedSpan)) - testJSONEncoding(t, jsonStr, embeddedSpan.transformToDbModelSpan()) + testJSONEncoding(t, jsonStr, embeddedSpan) } // Loads and returns domain model and JSON model. func loadModel(t *testing.T) ([]byte, []byte) { - in := fmt.Sprintf("./testdata/span.json") + in := fmt.Sprintf("../../testdata/span.json") inStr, err := os.ReadFile(in) require.NoError(t, err) - out := fmt.Sprintf("./testdata/es.json") + out := fmt.Sprintf("../../testdata/logziospan.json") outStr, err := os.ReadFile(out) require.NoError(t, err) return inStr, outStr @@ -97,35 +95,35 @@ func TestConvertKeyValueValue(t *testing.T) { key := "key" tests := []struct { kv model.KeyValue - expected dbmodel.KeyValue + expected KeyValue }{ { kv: model.Bool(key, true), - expected: dbmodel.KeyValue{Key: key, Value: "true", Type: "bool"}, + expected: KeyValue{Key: key, Value: "true", Type: "bool"}, }, { kv: model.Bool(key, false), - expected: dbmodel.KeyValue{Key: key, Value: "false", Type: "bool"}, + expected: KeyValue{Key: key, Value: "false", Type: "bool"}, }, { kv: model.Int64(key, int64(1499)), - expected: dbmodel.KeyValue{Key: key, Value: "1499", Type: "int64"}, + expected: KeyValue{Key: key, Value: "1499", Type: "int64"}, }, { kv: model.Float64(key, float64(15.66)), - expected: dbmodel.KeyValue{Key: key, Value: "15.66", Type: "float64"}, + expected: KeyValue{Key: key, Value: "15.66", Type: "float64"}, }, { kv: model.String(key, longString), - expected: dbmodel.KeyValue{Key: key, Value: longString, Type: "string"}, + expected: KeyValue{Key: key, Value: longString, Type: "string"}, }, { kv: model.Binary(key, []byte(longString)), - expected: dbmodel.KeyValue{Key: key, Value: hex.EncodeToString([]byte(longString)), Type: "binary"}, + expected: KeyValue{Key: key, Value: hex.EncodeToString([]byte(longString)), Type: "binary"}, }, { kv: model.KeyValue{VType: 1500, Key: key}, - expected: dbmodel.KeyValue{Key: key, Value: "unknown type 1500", Type: "1500"}, + expected: KeyValue{Key: key, Value: "unknown type 1500", Type: "1500"}, }, } diff --git a/exporter/logzioexporter/internal/dbmodel/logziospan.go b/exporter/logzioexporter/internal/dbmodel/logziospan.go new file mode 100644 index 000000000000..35d12efc150b --- /dev/null +++ b/exporter/logzioexporter/internal/dbmodel/logziospan.go @@ -0,0 +1,66 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package dbmodel // import "github.com/open-telemetry/opentelemetry-collector-contrib/exporter/logzioexporter" + +import ( + "encoding/json" + + "github.com/jaegertracing/jaeger/model" +) + +const ( + spanLogType = "jaegerSpan" + // TagDotReplacementCharacter state which character should replace the dot in es + tagDotReplacementCharacter = "@" +) + +// LogzioSpan is same as esSpan with a few different json field names and an addition on type field. +type LogzioSpan struct { + TraceID TraceID `json:"traceID"` + OperationName string `json:"operationName,omitempty"` + SpanID SpanID `json:"spanID"` + References []Reference `json:"references"` + Flags uint32 `json:"flags,omitempty"` + StartTime uint64 `json:"startTime"` + StartTimeMillis uint64 `json:"startTimeMillis"` + Timestamp uint64 `json:"@timestamp"` + Duration uint64 `json:"duration"` + Tags []KeyValue `json:"JaegerTags,omitempty"` + Tag map[string]any `json:"JaegerTag,omitempty"` + Logs []Log `json:"logs"` + Process Process `json:"process,omitempty"` + Type string `json:"type"` +} + +func getTagsValues(tags []model.KeyValue) []string { + values := make([]string, len(tags)) + for i := range tags { + values[i] = tags[i].VStr + } + return values +} + +// transformToLogzioSpanBytes receives a Jaeger span, converts it to logzio span and returns it as a byte array. +// The main differences between Jaeger span and logzio span are arrays which are represented as maps +func TransformToLogzioSpanBytes(span *model.Span) ([]byte, error) { + spanConverter := newFromDomain(true, getTagsValues(span.Tags), tagDotReplacementCharacter) + jsonSpan := spanConverter.fromDomainEmbedProcess(span) + newSpan := LogzioSpan{ + TraceID: jsonSpan.TraceID, + OperationName: jsonSpan.OperationName, + SpanID: jsonSpan.SpanID, + References: jsonSpan.References, + Flags: jsonSpan.Flags, + StartTime: jsonSpan.StartTime, + StartTimeMillis: jsonSpan.StartTimeMillis, + Timestamp: jsonSpan.StartTimeMillis, + Duration: jsonSpan.Duration, + Tags: jsonSpan.Tags, + Tag: jsonSpan.Tag, + Process: jsonSpan.Process, + Logs: jsonSpan.Logs, + Type: spanLogType, + } + return json.Marshal(newSpan) +} diff --git a/exporter/logzioexporter/internal/dbmodel/logziospan_test.go b/exporter/logzioexporter/internal/dbmodel/logziospan_test.go new file mode 100644 index 000000000000..fec3b034160b --- /dev/null +++ b/exporter/logzioexporter/internal/dbmodel/logziospan_test.go @@ -0,0 +1,33 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package dbmodel + +import ( + "encoding/json" + "fmt" + "os" + "testing" + + "github.com/jaegertracing/jaeger/model" + "github.com/stretchr/testify/require" +) + +func TestTransformToLogzioSpanBytes(tester *testing.T) { + inStr, err := os.ReadFile("../../testdata/span.json") + require.NoError(tester, err, "error opening sample span file") + + var span model.Span + err = json.Unmarshal(inStr, &span) + if err != nil { + fmt.Println("json.Unmarshal") + } + newSpan, err := TransformToLogzioSpanBytes(&span) + require.NoError(tester, err) + m := make(map[string]any) + err = json.Unmarshal(newSpan, &m) + require.NoError(tester, err) + if _, ok := m["JaegerTag"]; !ok { + tester.Error("error converting span to logzioSpan, JaegerTag is not found") + } +} diff --git a/exporter/logzioexporter/internal/dbmodel/model.go b/exporter/logzioexporter/internal/dbmodel/model.go index d849c4885733..f71b9984b9ff 100644 --- a/exporter/logzioexporter/internal/dbmodel/model.go +++ b/exporter/logzioexporter/internal/dbmodel/model.go @@ -34,27 +34,6 @@ const ( BinaryType ValueType = "binary" ) -// Span is ES database representation of the domain span. -type Span struct { - TraceID TraceID `json:"traceID"` - SpanID SpanID `json:"spanID"` - ParentSpanID SpanID `json:"parentSpanID,omitempty"` // deprecated - Flags uint32 `json:"flags,omitempty"` - OperationName string `json:"operationName"` - References []Reference `json:"references"` - StartTime uint64 `json:"startTime"` // microseconds since Unix epoch - // ElasticSearch does not support a UNIX Epoch timestamp in microseconds, - // so Jaeger maps StartTime to a 'long' type. This extra StartTimeMillis field - // works around this issue, enabling timerange queries. - StartTimeMillis uint64 `json:"startTimeMillis"` - Duration uint64 `json:"duration"` // microseconds - Tags []KeyValue `json:"tags"` - // Alternative representation of tags for better kibana support - Tag map[string]any `json:"tag,omitempty"` - Logs []Log `json:"logs"` - Process Process `json:"process,omitempty"` -} - // Reference is a reference from one span to another type Reference struct { RefType ReferenceType `json:"refType"` diff --git a/exporter/logzioexporter/logziospan.go b/exporter/logzioexporter/logziospan.go deleted file mode 100644 index 2c2512c2811e..000000000000 --- a/exporter/logzioexporter/logziospan.go +++ /dev/null @@ -1,86 +0,0 @@ -// Copyright The OpenTelemetry Authors -// SPDX-License-Identifier: Apache-2.0 - -package logzioexporter // import "github.com/open-telemetry/opentelemetry-collector-contrib/exporter/logzioexporter" - -import ( - "encoding/json" - - "github.com/jaegertracing/jaeger/model" - - "github.com/open-telemetry/opentelemetry-collector-contrib/exporter/logzioexporter/internal/dbmodel" -) - -const ( - spanLogType = "jaegerSpan" - // TagDotReplacementCharacter state which character should replace the dot in es - tagDotReplacementCharacter = "@" -) - -// logzioSpan is same as esSpan with a few different json field names and an addition on type field. -type logzioSpan struct { - TraceID dbmodel.TraceID `json:"traceID"` - OperationName string `json:"operationName,omitempty"` - SpanID dbmodel.SpanID `json:"spanID"` - References []dbmodel.Reference `json:"references"` - Flags uint32 `json:"flags,omitempty"` - StartTime uint64 `json:"startTime"` - StartTimeMillis uint64 `json:"startTimeMillis"` - Timestamp uint64 `json:"@timestamp"` - Duration uint64 `json:"duration"` - Tags []dbmodel.KeyValue `json:"JaegerTags,omitempty"` - Tag map[string]any `json:"JaegerTag,omitempty"` - Logs []dbmodel.Log `json:"logs"` - Process dbmodel.Process `json:"process,omitempty"` - Type string `json:"type"` -} - -func getTagsValues(tags []model.KeyValue) []string { - values := make([]string, len(tags)) - for i := range tags { - values[i] = tags[i].VStr - } - return values -} - -// transformToLogzioSpanBytes receives a Jaeger span, converts it to logzio span and returns it as a byte array. -// The main differences between Jaeger span and logzio span are arrays which are represented as maps -func transformToLogzioSpanBytes(span *model.Span) ([]byte, error) { - spanConverter := newFromDomain(true, getTagsValues(span.Tags), tagDotReplacementCharacter) - jsonSpan := spanConverter.fromDomainEmbedProcess(span) - newSpan := logzioSpan{ - TraceID: jsonSpan.TraceID, - OperationName: jsonSpan.OperationName, - SpanID: jsonSpan.SpanID, - References: jsonSpan.References, - Flags: jsonSpan.Flags, - StartTime: jsonSpan.StartTime, - StartTimeMillis: jsonSpan.StartTimeMillis, - Timestamp: jsonSpan.StartTimeMillis, - Duration: jsonSpan.Duration, - Tags: jsonSpan.Tags, - Tag: jsonSpan.Tag, - Process: jsonSpan.Process, - Logs: jsonSpan.Logs, - Type: spanLogType, - } - return json.Marshal(newSpan) -} - -// transformToDbModelSpan coverts logz.io span to ElasticSearch span -func (span *logzioSpan) transformToDbModelSpan() *dbmodel.Span { - return &dbmodel.Span{ - OperationName: span.OperationName, - Process: span.Process, - Tags: span.Tags, - Tag: span.Tag, - References: span.References, - Logs: span.Logs, - Duration: span.Duration, - StartTimeMillis: span.StartTimeMillis, - StartTime: span.StartTime, - Flags: span.Flags, - SpanID: span.SpanID, - TraceID: span.TraceID, - } -} diff --git a/exporter/logzioexporter/logziospan_test.go b/exporter/logzioexporter/logziospan_test.go deleted file mode 100644 index faea671f4b40..000000000000 --- a/exporter/logzioexporter/logziospan_test.go +++ /dev/null @@ -1,50 +0,0 @@ -// Copyright The OpenTelemetry Authors -// SPDX-License-Identifier: Apache-2.0 - -package logzioexporter - -import ( - "encoding/json" - "fmt" - "os" - "testing" - - "github.com/jaegertracing/jaeger/model" - "github.com/stretchr/testify/require" -) - -func TestTransformToLogzioSpanBytes(tester *testing.T) { - inStr, err := os.ReadFile("./testdata/span.json") - require.NoError(tester, err, "error opening sample span file") - - var span model.Span - err = json.Unmarshal(inStr, &span) - if err != nil { - fmt.Println("json.Unmarshal") - } - newSpan, err := transformToLogzioSpanBytes(&span) - require.NoError(tester, err) - m := make(map[string]any) - err = json.Unmarshal(newSpan, &m) - require.NoError(tester, err) - if _, ok := m["JaegerTag"]; !ok { - tester.Error("error converting span to logzioSpan, JaegerTag is not found") - } -} - -func TestTransformToDbModelSpan(tester *testing.T) { - inStr, err := os.ReadFile("./testdata/span.json") - require.NoError(tester, err, "error opening sample span file") - var span model.Span - err = json.Unmarshal(inStr, &span) - if err != nil { - fmt.Println("json.Unmarshal") - } - newSpan, err := transformToLogzioSpanBytes(&span) - require.NoError(tester, err) - var testLogzioSpan logzioSpan - err = json.Unmarshal(newSpan, &testLogzioSpan) - require.NoError(tester, err) - dbModelSpan := testLogzioSpan.transformToDbModelSpan() - require.Len(tester, dbModelSpan.References, 3, "Error converting logzio span to dbmodel span") -} diff --git a/exporter/logzioexporter/testdata/es.json b/exporter/logzioexporter/testdata/logziospan.json similarity index 96% rename from exporter/logzioexporter/testdata/es.json rename to exporter/logzioexporter/testdata/logziospan.json index 8b30c120d9cd..a13250d2ce9b 100644 --- a/exporter/logzioexporter/testdata/es.json +++ b/exporter/logzioexporter/testdata/logziospan.json @@ -1,8 +1,7 @@ { "traceID": "0000000000000001", - "spanID": "0000000000000002", - "flags": 1, "operationName": "test-general-conversion", + "spanID": "0000000000000002", "references": [ { "refType": "CHILD_OF", @@ -20,10 +19,12 @@ "spanID": "00000000000000ff" } ], + "flags": 1, "startTime": 1485467191639875, "startTimeMillis": 1485467191639, + "@timestamp": 0, "duration": 5, - "tags": [ + "JaegerTags": [ { "key": "peer.service", "type": "string", @@ -86,5 +87,6 @@ "value": "true" } ] - } + }, + "type": "" } From 930e342fb7ebc91510b5833bbfe97c078a68b96b Mon Sep 17 00:00:00 2001 From: zzzk1 Date: Thu, 2 Jan 2025 21:35:04 +0800 Subject: [PATCH 09/10] remove back 0.0 --- exporter/logzioexporter/exporter.go | 4 +- exporter/logzioexporter/exporter_test.go | 4 +- .../{internal/dbmodel => }/from_domain.go | 57 ++++++++++--------- .../dbmodel => }/from_domain_test.go | 32 +++++------ .../logzioexporter/internal/dbmodel/model.go | 3 +- .../{internal/dbmodel => }/logziospan.go | 40 ++++++------- .../{internal/dbmodel => }/logziospan_test.go | 6 +- 7 files changed, 74 insertions(+), 72 deletions(-) rename exporter/logzioexporter/{internal/dbmodel => }/from_domain.go (63%) rename exporter/logzioexporter/{internal/dbmodel => }/from_domain_test.go (78%) rename exporter/logzioexporter/{internal/dbmodel => }/logziospan.go (54%) rename exporter/logzioexporter/{internal/dbmodel => }/logziospan_test.go (84%) diff --git a/exporter/logzioexporter/exporter.go b/exporter/logzioexporter/exporter.go index 30d9932abd07..a4cc01780ec9 100644 --- a/exporter/logzioexporter/exporter.go +++ b/exporter/logzioexporter/exporter.go @@ -15,8 +15,6 @@ import ( "strconv" "time" - "github.com/open-telemetry/opentelemetry-collector-contrib/exporter/logzioexporter/internal/dbmodel" - "github.com/hashicorp/go-hclog" "github.com/jaegertracing/jaeger/model" "github.com/jaegertracing/jaeger/pkg/cache" @@ -191,7 +189,7 @@ func (exporter *logzioExporter) pushTraceData(ctx context.Context, traces ptrace span.Process = batch.Process span.Tags = exporter.dropEmptyTags(span.Tags) span.Process.Tags = exporter.dropEmptyTags(span.Process.Tags) - logzioSpan, transformErr := dbmodel.TransformToLogzioSpanBytes(span) + logzioSpan, transformErr := transformToLogzioSpanBytes(span) if transformErr != nil { return transformErr } diff --git a/exporter/logzioexporter/exporter_test.go b/exporter/logzioexporter/exporter_test.go index 0dbd06dee86d..2943c50a9413 100644 --- a/exporter/logzioexporter/exporter_test.go +++ b/exporter/logzioexporter/exporter_test.go @@ -16,8 +16,6 @@ import ( "testing" "time" - "github.com/open-telemetry/opentelemetry-collector-contrib/exporter/logzioexporter/internal/dbmodel" - "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.opentelemetry.io/collector/component/componenttest" @@ -257,7 +255,7 @@ func TestPushTraceData(tester *testing.T) { res.Attributes().PutStr(conventions.AttributeHostName, testHost) err := testTracesExporter(tester, td, &cfg) require.NoError(tester, err) - var newSpan dbmodel.LogzioSpan + var newSpan logzioSpan decoded, _ := gUnzipData(recordedRequests) requests := strings.Split(string(decoded), "\n") assert.NoError(tester, json.Unmarshal([]byte(requests[0]), &newSpan)) diff --git a/exporter/logzioexporter/internal/dbmodel/from_domain.go b/exporter/logzioexporter/from_domain.go similarity index 63% rename from exporter/logzioexporter/internal/dbmodel/from_domain.go rename to exporter/logzioexporter/from_domain.go index 1f02075f618e..5eee5652677d 100644 --- a/exporter/logzioexporter/internal/dbmodel/from_domain.go +++ b/exporter/logzioexporter/from_domain.go @@ -1,13 +1,16 @@ +// Copyright The OpenTelemetry Authors // Copyright (c) 2019 The Jaeger Authors. // Copyright (c) 2018 Uber Technologies, Inc. // SPDX-License-Identifier: Apache-2.0 -package dbmodel // import "github.com/open-telemetry/opentelemetry-collector-contrib/exporter/logzioexporter" +package logzioexporter // import "github.com/open-telemetry/opentelemetry-collector-contrib/exporter/logzioexporter" import ( "strings" "github.com/jaegertracing/jaeger/model" + + "github.com/open-telemetry/opentelemetry-collector-contrib/exporter/logzioexporter/internal/dbmodel" ) // newFromDomain creates fromDomain used to convert model span to db span @@ -28,15 +31,15 @@ type fromDomain struct { // fromDomainEmbedProcess converts model.span into json.span format. // This format includes a ParentSpanID and an embedded process. -func (fd fromDomain) fromDomainEmbedProcess(span *model.Span) *LogzioSpan { +func (fd fromDomain) fromDomainEmbedProcess(span *model.Span) *logzioSpan { return fd.convertSpanEmbedProcess(span) } -func (fd fromDomain) convertSpanInternal(span *model.Span) LogzioSpan { +func (fd fromDomain) convertSpanInternal(span *model.Span) logzioSpan { tags, tagsMap := fd.convertKeyValuesString(span.Tags) - return LogzioSpan{ - TraceID: TraceID(span.TraceID.String()), - SpanID: SpanID(span.SpanID.String()), + return logzioSpan{ + TraceID: dbmodel.TraceID(span.TraceID.String()), + SpanID: dbmodel.SpanID(span.SpanID.String()), Flags: uint32(span.Flags), OperationName: span.OperationName, StartTime: model.TimeAsEpochMicroseconds(span.StartTime), @@ -48,35 +51,35 @@ func (fd fromDomain) convertSpanInternal(span *model.Span) LogzioSpan { } } -func (fd fromDomain) convertSpanEmbedProcess(span *model.Span) *LogzioSpan { +func (fd fromDomain) convertSpanEmbedProcess(span *model.Span) *logzioSpan { s := fd.convertSpanInternal(span) s.Process = fd.convertProcess(span.Process) s.References = fd.convertReferences(span) return &s } -func (fd fromDomain) convertReferences(span *model.Span) []Reference { - out := make([]Reference, 0, len(span.References)) +func (fd fromDomain) convertReferences(span *model.Span) []dbmodel.Reference { + out := make([]dbmodel.Reference, 0, len(span.References)) for _, ref := range span.References { - out = append(out, Reference{ + out = append(out, dbmodel.Reference{ RefType: fd.convertRefType(ref.RefType), - TraceID: TraceID(ref.TraceID.String()), - SpanID: SpanID(ref.SpanID.String()), + TraceID: dbmodel.TraceID(ref.TraceID.String()), + SpanID: dbmodel.SpanID(ref.SpanID.String()), }) } return out } -func (fromDomain) convertRefType(refType model.SpanRefType) ReferenceType { +func (fromDomain) convertRefType(refType model.SpanRefType) dbmodel.ReferenceType { if refType == model.FollowsFrom { - return FollowsFrom + return dbmodel.FollowsFrom } - return ChildOf + return dbmodel.ChildOf } -func (fd fromDomain) convertKeyValuesString(keyValues model.KeyValues) ([]KeyValue, map[string]any) { +func (fd fromDomain) convertKeyValuesString(keyValues model.KeyValues) ([]dbmodel.KeyValue, map[string]any) { var tagsMap map[string]any - var kvs []KeyValue + var kvs []dbmodel.KeyValue for _, kv := range keyValues { if kv.GetVType() != model.BinaryType && (fd.allTagsAsFields || fd.tagKeysAsFields[kv.Key]) { if tagsMap == nil { @@ -88,19 +91,19 @@ func (fd fromDomain) convertKeyValuesString(keyValues model.KeyValues) ([]KeyVal } } if kvs == nil { - kvs = make([]KeyValue, 0) + kvs = make([]dbmodel.KeyValue, 0) } return kvs, tagsMap } -func (fromDomain) convertLogs(logs []model.Log) []Log { - out := make([]Log, len(logs)) +func (fromDomain) convertLogs(logs []model.Log) []dbmodel.Log { + out := make([]dbmodel.Log, len(logs)) for i, log := range logs { - var kvs []KeyValue + var kvs []dbmodel.KeyValue for _, kv := range log.Fields { kvs = append(kvs, convertKeyValue(kv)) } - out[i] = Log{ + out[i] = dbmodel.Log{ Timestamp: model.TimeAsEpochMicroseconds(log.Timestamp), Fields: kvs, } @@ -108,19 +111,19 @@ func (fromDomain) convertLogs(logs []model.Log) []Log { return out } -func (fd fromDomain) convertProcess(process *model.Process) Process { +func (fd fromDomain) convertProcess(process *model.Process) dbmodel.Process { tags, tagsMap := fd.convertKeyValuesString(process.Tags) - return Process{ + return dbmodel.Process{ ServiceName: process.ServiceName, Tags: tags, Tag: tagsMap, } } -func convertKeyValue(kv model.KeyValue) KeyValue { - return KeyValue{ +func convertKeyValue(kv model.KeyValue) dbmodel.KeyValue { + return dbmodel.KeyValue{ Key: kv.Key, - Type: ValueType(strings.ToLower(kv.VType.String())), + Type: dbmodel.ValueType(strings.ToLower(kv.VType.String())), Value: kv.AsString(), } } diff --git a/exporter/logzioexporter/internal/dbmodel/from_domain_test.go b/exporter/logzioexporter/from_domain_test.go similarity index 78% rename from exporter/logzioexporter/internal/dbmodel/from_domain_test.go rename to exporter/logzioexporter/from_domain_test.go index 05e6a8e61957..b345d3d961ee 100644 --- a/exporter/logzioexporter/internal/dbmodel/from_domain_test.go +++ b/exporter/logzioexporter/from_domain_test.go @@ -1,8 +1,9 @@ +// Copyright The OpenTelemetry Authors // Copyright (c) 2019 The Jaeger Authors. // Copyright (c) 2018 Uber Technologies, Inc. // SPDX-License-Identifier: Apache-2.0 -package dbmodel +package logzioexporter import ( "bytes" @@ -13,10 +14,11 @@ import ( "testing" "github.com/gogo/protobuf/jsonpb" + "github.com/jaegertracing/jaeger/model" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "github.com/jaegertracing/jaeger/model" + "github.com/open-telemetry/opentelemetry-collector-contrib/exporter/logzioexporter/internal/dbmodel" ) func TestFromDomainEmbedProcess(t *testing.T) { @@ -27,7 +29,7 @@ func TestFromDomainEmbedProcess(t *testing.T) { converter := newFromDomain(false, nil, ":") embeddedSpan := converter.fromDomainEmbedProcess(&span) - var expectedSpan LogzioSpan + var expectedSpan logzioSpan require.NoError(t, json.Unmarshal(jsonStr, &expectedSpan)) testJSONEncoding(t, jsonStr, embeddedSpan) @@ -35,11 +37,9 @@ func TestFromDomainEmbedProcess(t *testing.T) { // Loads and returns domain model and JSON model. func loadModel(t *testing.T) ([]byte, []byte) { - in := fmt.Sprintf("../../testdata/span.json") - inStr, err := os.ReadFile(in) + inStr, err := os.ReadFile("./testdata/span.json") require.NoError(t, err) - out := fmt.Sprintf("../../testdata/logziospan.json") - outStr, err := os.ReadFile(out) + outStr, err := os.ReadFile("./testdata/logziospan.json") require.NoError(t, err) return inStr, outStr } @@ -50,7 +50,7 @@ func testJSONEncoding(t *testing.T, expectedStr []byte, object any) { enc.SetIndent("", " ") require.NoError(t, enc.Encode(object)) if !assert.Equal(t, string(expectedStr), buf.String()) { - err := os.WriteFile("model-actual.json", buf.Bytes(), 0o644) + err := os.WriteFile("model-actual.json", buf.Bytes(), 0o600) require.NoError(t, err) } } @@ -95,35 +95,35 @@ func TestConvertKeyValueValue(t *testing.T) { key := "key" tests := []struct { kv model.KeyValue - expected KeyValue + expected dbmodel.KeyValue }{ { kv: model.Bool(key, true), - expected: KeyValue{Key: key, Value: "true", Type: "bool"}, + expected: dbmodel.KeyValue{Key: key, Value: "true", Type: "bool"}, }, { kv: model.Bool(key, false), - expected: KeyValue{Key: key, Value: "false", Type: "bool"}, + expected: dbmodel.KeyValue{Key: key, Value: "false", Type: "bool"}, }, { kv: model.Int64(key, int64(1499)), - expected: KeyValue{Key: key, Value: "1499", Type: "int64"}, + expected: dbmodel.KeyValue{Key: key, Value: "1499", Type: "int64"}, }, { kv: model.Float64(key, float64(15.66)), - expected: KeyValue{Key: key, Value: "15.66", Type: "float64"}, + expected: dbmodel.KeyValue{Key: key, Value: "15.66", Type: "float64"}, }, { kv: model.String(key, longString), - expected: KeyValue{Key: key, Value: longString, Type: "string"}, + expected: dbmodel.KeyValue{Key: key, Value: longString, Type: "string"}, }, { kv: model.Binary(key, []byte(longString)), - expected: KeyValue{Key: key, Value: hex.EncodeToString([]byte(longString)), Type: "binary"}, + expected: dbmodel.KeyValue{Key: key, Value: hex.EncodeToString([]byte(longString)), Type: "binary"}, }, { kv: model.KeyValue{VType: 1500, Key: key}, - expected: KeyValue{Key: key, Value: "unknown type 1500", Type: "1500"}, + expected: dbmodel.KeyValue{Key: key, Value: "unknown type 1500", Type: "1500"}, }, } diff --git a/exporter/logzioexporter/internal/dbmodel/model.go b/exporter/logzioexporter/internal/dbmodel/model.go index f71b9984b9ff..af42f10eecb9 100644 --- a/exporter/logzioexporter/internal/dbmodel/model.go +++ b/exporter/logzioexporter/internal/dbmodel/model.go @@ -1,8 +1,9 @@ +// Copyright The OpenTelemetry Authors // Copyright (c) 2019 The Jaeger Authors. // Copyright (c) 2018 Uber Technologies, Inc. // SPDX-License-Identifier: Apache-2.0 -package dbmodel +package dbmodel // import "github.com/open-telemetry/opentelemetry-collector-contrib/exporter/logzioexporter/internal/dbmodel" // ReferenceType is the reference type of one span to another type ReferenceType string diff --git a/exporter/logzioexporter/internal/dbmodel/logziospan.go b/exporter/logzioexporter/logziospan.go similarity index 54% rename from exporter/logzioexporter/internal/dbmodel/logziospan.go rename to exporter/logzioexporter/logziospan.go index 35d12efc150b..edeef69b0ea9 100644 --- a/exporter/logzioexporter/internal/dbmodel/logziospan.go +++ b/exporter/logzioexporter/logziospan.go @@ -1,12 +1,14 @@ // Copyright The OpenTelemetry Authors // SPDX-License-Identifier: Apache-2.0 -package dbmodel // import "github.com/open-telemetry/opentelemetry-collector-contrib/exporter/logzioexporter" +package logzioexporter // import "github.com/open-telemetry/opentelemetry-collector-contrib/exporter/logzioexporter" import ( "encoding/json" "github.com/jaegertracing/jaeger/model" + + "github.com/open-telemetry/opentelemetry-collector-contrib/exporter/logzioexporter/internal/dbmodel" ) const ( @@ -15,22 +17,22 @@ const ( tagDotReplacementCharacter = "@" ) -// LogzioSpan is same as esSpan with a few different json field names and an addition on type field. -type LogzioSpan struct { - TraceID TraceID `json:"traceID"` - OperationName string `json:"operationName,omitempty"` - SpanID SpanID `json:"spanID"` - References []Reference `json:"references"` - Flags uint32 `json:"flags,omitempty"` - StartTime uint64 `json:"startTime"` - StartTimeMillis uint64 `json:"startTimeMillis"` - Timestamp uint64 `json:"@timestamp"` - Duration uint64 `json:"duration"` - Tags []KeyValue `json:"JaegerTags,omitempty"` - Tag map[string]any `json:"JaegerTag,omitempty"` - Logs []Log `json:"logs"` - Process Process `json:"process,omitempty"` - Type string `json:"type"` +// logzioSpan is same as esSpan with a few different json field names and an addition on type field. +type logzioSpan struct { + TraceID dbmodel.TraceID `json:"traceID"` + OperationName string `json:"operationName,omitempty"` + SpanID dbmodel.SpanID `json:"spanID"` + References []dbmodel.Reference `json:"references"` + Flags uint32 `json:"flags,omitempty"` + StartTime uint64 `json:"startTime"` + StartTimeMillis uint64 `json:"startTimeMillis"` + Timestamp uint64 `json:"@timestamp"` + Duration uint64 `json:"duration"` + Tags []dbmodel.KeyValue `json:"JaegerTags,omitempty"` + Tag map[string]any `json:"JaegerTag,omitempty"` + Logs []dbmodel.Log `json:"logs"` + Process dbmodel.Process `json:"process,omitempty"` + Type string `json:"type"` } func getTagsValues(tags []model.KeyValue) []string { @@ -43,10 +45,10 @@ func getTagsValues(tags []model.KeyValue) []string { // transformToLogzioSpanBytes receives a Jaeger span, converts it to logzio span and returns it as a byte array. // The main differences between Jaeger span and logzio span are arrays which are represented as maps -func TransformToLogzioSpanBytes(span *model.Span) ([]byte, error) { +func transformToLogzioSpanBytes(span *model.Span) ([]byte, error) { spanConverter := newFromDomain(true, getTagsValues(span.Tags), tagDotReplacementCharacter) jsonSpan := spanConverter.fromDomainEmbedProcess(span) - newSpan := LogzioSpan{ + newSpan := logzioSpan{ TraceID: jsonSpan.TraceID, OperationName: jsonSpan.OperationName, SpanID: jsonSpan.SpanID, diff --git a/exporter/logzioexporter/internal/dbmodel/logziospan_test.go b/exporter/logzioexporter/logziospan_test.go similarity index 84% rename from exporter/logzioexporter/internal/dbmodel/logziospan_test.go rename to exporter/logzioexporter/logziospan_test.go index fec3b034160b..af4b768e1b2a 100644 --- a/exporter/logzioexporter/internal/dbmodel/logziospan_test.go +++ b/exporter/logzioexporter/logziospan_test.go @@ -1,7 +1,7 @@ // Copyright The OpenTelemetry Authors // SPDX-License-Identifier: Apache-2.0 -package dbmodel +package logzioexporter import ( "encoding/json" @@ -14,7 +14,7 @@ import ( ) func TestTransformToLogzioSpanBytes(tester *testing.T) { - inStr, err := os.ReadFile("../../testdata/span.json") + inStr, err := os.ReadFile("./testdata/span.json") require.NoError(tester, err, "error opening sample span file") var span model.Span @@ -22,7 +22,7 @@ func TestTransformToLogzioSpanBytes(tester *testing.T) { if err != nil { fmt.Println("json.Unmarshal") } - newSpan, err := TransformToLogzioSpanBytes(&span) + newSpan, err := transformToLogzioSpanBytes(&span) require.NoError(tester, err) m := make(map[string]any) err = json.Unmarshal(newSpan, &m) From 8c2f11faed6d28978de5d5c70af0bab929cdc80e Mon Sep 17 00:00:00 2001 From: zzzk1 Date: Sat, 4 Jan 2025 12:14:30 +0800 Subject: [PATCH 10/10] gotidy --- exporter/logzioexporter/go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/exporter/logzioexporter/go.mod b/exporter/logzioexporter/go.mod index e5830c48ce41..7476427aa019 100644 --- a/exporter/logzioexporter/go.mod +++ b/exporter/logzioexporter/go.mod @@ -5,6 +5,7 @@ go 1.22.7 toolchain go1.22.8 require ( + github.com/gogo/protobuf v1.3.2 github.com/hashicorp/go-hclog v1.6.3 github.com/jaegertracing/jaeger v1.64.0 github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/jaeger v0.116.0 @@ -38,7 +39,6 @@ require ( github.com/go-logr/logr v1.4.2 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-viper/mapstructure/v2 v2.2.1 // indirect - github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/snappy v0.0.4 // indirect github.com/google/uuid v1.6.0 // indirect github.com/hashicorp/go-version v1.7.0 // indirect