diff --git a/CHANGELOG.md b/CHANGELOG.md index 03ec34c..f473f5c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -115,6 +115,7 @@ - Update materialized_version for profile functions metrics ([#522](https://github.com/getsentry/vroom/pull/522)) - Support writing functions metrics we extract from chunks into the functions dataset ([#524](https://github.com/getsentry/vroom/pull/524)) - Keep top N samples in flamegraph. ([#526](https://github.com/getsentry/vroom/pull/526)) +- Add utility to merge a list of android chunks and generate a speedscope result ([#531](https://github.com/getsentry/vroom/pull/531)) - Remove unused legacy flamegraph code path. ([#533](https://github.com/getsentry/vroom/pull/533)) - Remove generic metrics ingestion ([#534](https://github.com/getsentry/vroom/pull/534)) diff --git a/internal/chunk/android_utils.go b/internal/chunk/android_utils.go new file mode 100644 index 0000000..56f6748 --- /dev/null +++ b/internal/chunk/android_utils.go @@ -0,0 +1,181 @@ +package chunk + +import ( + "encoding/json" + "sort" + "time" + + "github.com/getsentry/vroom/internal/measurements" + "github.com/getsentry/vroom/internal/profile" + "github.com/getsentry/vroom/internal/speedscope" +) + +var member void + +type void struct{} + +func SpeedscopeFromAndroidChunks(chunks []AndroidChunk, startTS, endTS uint64) (speedscope.Output, error) { + if len(chunks) == 0 { + return speedscope.Output{}, nil + } + maxTsNS := uint64(0) + threadSet := make(map[uint64]void) + // fingerprint to method ID + methodToID := make(map[uint32]uint64) + sort.Slice(chunks, func(i, j int) bool { + return chunks[i].EndTimestamp() <= chunks[j].StartTimestamp() + }) + + mergedMeasurement := make(map[string]measurements.MeasurementV2) + + chunk := chunks[0] + firstChunkStartTimestampNS := uint64(chunk.StartTimestamp() * 1e9) + // Initially, adjustedChunkStartTimestampNS will just be the + // chunk timestamp. If the chunk starts before the allowed + // time range though, we only keep events that fall within + // the range, set the startTimestamp to the start value of + // the allowed range and adjust the relative ts of each + // events. + adjustedChunkStartTimestampNS := firstChunkStartTimestampNS + buildTimestamp := chunk.Profile.TimestampGetter() + // clean up the events in the first chunk + events := make([]profile.AndroidEvent, 0, len(chunk.Profile.Events)) + methods := make([]profile.AndroidMethod, 0, len(chunk.Profile.Methods)) + // updates methods ID + tmpMethodsID := make(map[uint64]uint64) + for i, method := range chunk.Profile.Methods { + id := uint64(i + 1) + tmpMethodsID[method.ID] = id + method.ID = id + methodToID[method.Frame().Fingerprint()] = id + methods = append(methods, method) + } + delta := int64(0) + if firstChunkStartTimestampNS < startTS { + delta = -int64(startTS - firstChunkStartTimestampNS) + adjustedChunkStartTimestampNS = startTS + } + addTimeDelta := chunk.Profile.AddTimeDelta(delta) + for _, event := range chunk.Profile.Events { + ts := buildTimestamp(event.Time) + firstChunkStartTimestampNS + if ts < startTS || ts > endTS { + // we filter out events out of range + continue + } + // If the event falls within allowed range, but the first chunk + // begins before the start range (delta != 0), adjust the relative ts + // of each event by subtracting the delta. + if delta != 0 { + err := addTimeDelta(&event) + if err != nil { + return speedscope.Output{}, err + } + // update ts + ts = buildTimestamp(event.Time) + adjustedChunkStartTimestampNS + } + event.MethodID = tmpMethodsID[event.MethodID] + events = append(events, event) + maxTsNS = max(maxTsNS, ts) + } + for _, thread := range chunk.Profile.Threads { + threadSet[thread.ID] = member + } + if len(chunk.Measurements) > 0 { + err := json.Unmarshal(chunk.Measurements, &mergedMeasurement) + if err != nil { + return speedscope.Output{}, err + } + } + + // If chunk started before the allowed time range + // update the chunk timestamp (firstChunkStartTimestampNS) + // since later on, other chunks will use this to compute + // the right offset (relative ts in nanoseconds). + if delta != 0 { + firstChunkStartTimestampNS = adjustedChunkStartTimestampNS + } + + for i := 1; i < len(chunks); i++ { + c := chunks[i] + chunkStartTimestampNs := uint64(c.StartTimestamp() * 1e9) + buildTimestamp := c.Profile.TimestampGetter() + // Delta between the current chunk timestamp and the very first one. + // This will be needed to correctly offset the events relative ts, + // which need to be relative not to the start of this chunk, but to + // the start of the very first one. + delta := chunkStartTimestampNs - firstChunkStartTimestampNS + addTimeDelta := c.Profile.AddTimeDelta(int64(delta)) + // updates methods ID + tmpMethodsID = make(map[uint64]uint64) + for _, method := range c.Profile.Methods { + fingerprint := method.Frame().Fingerprint() + if id, ok := methodToID[fingerprint]; !ok { + newID := uint64(len(methodToID) + 1) + methodToID[fingerprint] = newID + tmpMethodsID[method.ID] = newID + method.ID = newID + methods = append(methods, method) + } else { + tmpMethodsID[method.ID] = id + } + } + + // filter events + for _, event := range c.Profile.Events { + ts := buildTimestamp(event.Time) + chunkStartTimestampNs + if ts < startTS || ts > endTS { + continue + } + event.MethodID = tmpMethodsID[event.MethodID] + // Before adding the event, update its relative timestamp + // which, in this case, should not be relative to the current + // chunk timestamp, but rather relative to the very 1st one. + err := addTimeDelta(&event) + if err != nil { + return speedscope.Output{}, err + } + ts = buildTimestamp(event.Time) + firstChunkStartTimestampNS + events = append(events, event) + maxTsNS = max(maxTsNS, ts) + } + // Update threads. + for _, thread := range c.Profile.Threads { + if _, ok := threadSet[thread.ID]; !ok { + chunk.Profile.Threads = append(c.Profile.Threads, thread) + threadSet[thread.ID] = member + } + } + // In case we have measurements, merge them too. + if len(c.Measurements) > 0 { + var chunkMeasurements map[string]measurements.MeasurementV2 + err := json.Unmarshal(c.Measurements, &chunkMeasurements) + if err != nil { + return speedscope.Output{}, err + } + for k, measurement := range chunkMeasurements { + if el, ok := mergedMeasurement[k]; ok { + el.Values = append(el.Values, measurement.Values...) + mergedMeasurement[k] = el + } else { + mergedMeasurement[k] = measurement + } + } + } + } + chunk.Profile.Events = events + chunk.Profile.Methods = methods + chunk.DurationNS = maxTsNS + + s, err := chunk.Profile.Speedscope() + if err != nil { + return speedscope.Output{}, err + } + s.DurationNS = chunk.DurationNS + s.Metadata.Timestamp = time.Unix(0, int64(firstChunkStartTimestampNS)).UTC() + + if len(mergedMeasurement) > 0 { + s.MeasurementsV2 = mergedMeasurement + } + + return s, nil +} diff --git a/internal/chunk/android_utils_test.go b/internal/chunk/android_utils_test.go new file mode 100644 index 0000000..b517249 --- /dev/null +++ b/internal/chunk/android_utils_test.go @@ -0,0 +1,351 @@ +package chunk + +import ( + "testing" + "time" + + "github.com/getsentry/vroom/internal/profile" + "github.com/getsentry/vroom/internal/speedscope" + "github.com/getsentry/vroom/internal/testutil" +) + +var androidChunk1 = AndroidChunk{ + Timestamp: 0.0, + DurationNS: 1500, + Profile: profile.Android{ + Clock: "Dual", + Events: []profile.AndroidEvent{ + { + Action: "Enter", + ThreadID: 1, + MethodID: 1, + Time: profile.EventTime{ + Monotonic: profile.EventMonotonic{ + Wall: profile.Duration{ + Secs: 0, + Nanos: 1000, + }, + }, + }, + }, + { + Action: "Enter", + ThreadID: 1, + MethodID: 2, + Time: profile.EventTime{ + Monotonic: profile.EventMonotonic{ + Wall: profile.Duration{ + Secs: 0, + Nanos: 1500, + }, + }, + }, + }, + { + Action: "Enter", + ThreadID: 1, + MethodID: 3, + Time: profile.EventTime{ + Monotonic: profile.EventMonotonic{ + Wall: profile.Duration{ + Secs: 0, + Nanos: 2000, + }, + }, + }, + }, + { + Action: "Exit", + ThreadID: 1, + MethodID: 3, + Time: profile.EventTime{ + Monotonic: profile.EventMonotonic{ + Wall: profile.Duration{ + Secs: 0, + Nanos: 2500, + }, + }, + }, + }, + }, + Methods: []profile.AndroidMethod{ + { + ClassName: "class1", + ID: 1, + Name: "method1", + Signature: "()", + }, + { + ClassName: "class2", + ID: 2, + Name: "method2", + Signature: "()", + }, + { + ClassName: "class3", + ID: 3, + Name: "method3", + Signature: "()", + }, + }, + StartTime: 0, + Threads: []profile.AndroidThread{ + { + ID: 1, + Name: "main", + }, + }, + }, +} + +var androidChunk2 = AndroidChunk{ + Timestamp: 2.5e-6, + DurationNS: 2000, + Profile: profile.Android{ + Clock: "Dual", + Events: []profile.AndroidEvent{ + { + Action: "Enter", + ThreadID: 1, + MethodID: 1, + Time: profile.EventTime{ + Monotonic: profile.EventMonotonic{ + Wall: profile.Duration{ + Secs: 0, + Nanos: 500, + }, + }, + }, + }, + { + Action: "Exit", + ThreadID: 1, + MethodID: 1, + Time: profile.EventTime{ + Monotonic: profile.EventMonotonic{ + Wall: profile.Duration{ + Secs: 0, + Nanos: 1000, + }, + }, + }, + }, + { + Action: "Exit", + ThreadID: 1, + MethodID: 3, + Time: profile.EventTime{ + Monotonic: profile.EventMonotonic{ + Wall: profile.Duration{ + Secs: 0, + Nanos: 1500, + }, + }, + }, + }, + { + Action: "Exit", + ThreadID: 1, + MethodID: 2, + Time: profile.EventTime{ + Monotonic: profile.EventMonotonic{ + Wall: profile.Duration{ + Secs: 0, + Nanos: 2000, + }, + }, + }, + }, + }, + Methods: []profile.AndroidMethod{ + { + ClassName: "class4", + ID: 1, + Name: "method4", + Signature: "()", + }, + { + ClassName: "class1", + ID: 2, + Name: "method1", + Signature: "()", + }, + { + ClassName: "class2", + ID: 3, + Name: "method2", + Signature: "()", + }, + }, + StartTime: 0, + Threads: []profile.AndroidThread{ + { + ID: 1, + Name: "main", + }, + }, + }, +} + +func TestSpeedscopeFromAndroidChunks(t *testing.T) { + tests := []struct { + name string + have []AndroidChunk + want speedscope.Output + start uint64 + end uint64 + }{ + { + name: "All chunks included in the time range", + have: []AndroidChunk{androidChunk1, androidChunk2}, + want: speedscope.Output{ + AndroidClock: "Dual", + DurationNS: 4500, + Profiles: []any{ + &speedscope.EventedProfile{ + EndValue: 4500, + Events: []speedscope.Event{ + { + Type: "O", + Frame: 0, + At: 1000, + }, + { + Type: "O", + Frame: 1, + At: 1500, + }, + { + Type: "O", + Frame: 2, + At: 2000, + }, + { + Type: "C", + Frame: 2, + At: 2500, + }, + { + Type: "O", + Frame: 3, + At: 3000, + }, + { + Type: "C", + Frame: 3, + At: 3500, + }, + { + Type: "C", + Frame: 1, + At: 4000, + }, + { + Type: "C", + Frame: 0, + At: 4500, + }, + }, + Name: "main", + StartValue: 1000, + ThreadID: 1, + Type: "evented", + Unit: "nanoseconds", + }, + }, + Shared: speedscope.SharedData{ + Frames: []speedscope.Frame{ + {Image: "class1", IsApplication: true, Name: "class1.method1()"}, + {Image: "class2", IsApplication: true, Name: "class2.method2()"}, + {Image: "class3", IsApplication: true, Name: "class3.method3()"}, + {Image: "class4", IsApplication: true, Name: "class4.method4()"}, + }, + }, + Metadata: speedscope.ProfileMetadata{ + ProfileView: speedscope.ProfileView{ + Timestamp: time.Unix(0, 0).UTC(), + }, + }, + }, + start: 0, + end: 6000, + }, + { + name: "First chunk begins before allowed range (overlap )", + have: []AndroidChunk{androidChunk1, androidChunk2}, + want: speedscope.Output{ + AndroidClock: "Dual", + DurationNS: 4500, + Profiles: []any{ + &speedscope.EventedProfile{ + EndValue: 3000, + Events: []speedscope.Event{ + { + Type: "O", + Frame: 1, + At: 0, + }, + { + Type: "O", + Frame: 2, + At: 500, + }, + { + Type: "C", + Frame: 2, + At: 1000, + }, + { + Type: "O", + Frame: 3, + At: 1500, + }, + { + Type: "C", + Frame: 3, + At: 2000, + }, + { + Type: "C", + Frame: 1, + At: 2500, + }, + }, + Name: "main", + StartValue: 0, + ThreadID: 1, + Type: "evented", + Unit: "nanoseconds", + }, + }, + Shared: speedscope.SharedData{ + Frames: []speedscope.Frame{ + {Image: "class1", IsApplication: true, Name: "class1.method1()"}, + {Image: "class2", IsApplication: true, Name: "class2.method2()"}, + {Image: "class3", IsApplication: true, Name: "class3.method3()"}, + {Image: "class4", IsApplication: true, Name: "class4.method4()"}, + }, + }, + Metadata: speedscope.ProfileMetadata{ + ProfileView: speedscope.ProfileView{ + Timestamp: time.Unix(0, 1500).UTC(), + }, + }, + }, + start: 1500, + end: 6000, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + s, err := SpeedscopeFromAndroidChunks(test.have, test.start, test.end) + if err != nil { + t.Fatal(err) + } + if diff := testutil.Diff(s, test.want); diff != "" { + t.Fatalf("Result mismatch: got - want +\n%s", diff) + } + }) + } +} diff --git a/internal/profile/android.go b/internal/profile/android.go index a7a8656..5fcfe81 100644 --- a/internal/profile/android.go +++ b/internal/profile/android.go @@ -1,6 +1,7 @@ package profile import ( + "errors" "fmt" "hash/fnv" "math" @@ -244,6 +245,62 @@ func (p *Android) FixSamplesTime() { } } +func (p *Android) AddTimeDelta(deltaNS int64) func(*AndroidEvent) error { + var addDeltaTimestamp func(e *AndroidEvent) error + timestampBuilder := p.TimestampGetter() + switch p.Clock { + case GlobalClock: + addDeltaTimestamp = func(e *AndroidEvent) error { + ts := timestampBuilder(e.Time) + ts, err := getTsFromDelta(ts, deltaNS) + if err != nil { + return err + } + secs := (ts / 1e9) + nanos := (ts % 1e9) + e.Time.Global.Secs = secs + e.Time.Global.Nanos = nanos + return nil + } + case CPUClock: + addDeltaTimestamp = func(e *AndroidEvent) error { + ts := timestampBuilder(e.Time) + ts, err := getTsFromDelta(ts, deltaNS) + if err != nil { + return err + } + secs := (ts / 1e9) + nanos := (ts % 1e9) + e.Time.Monotonic.CPU.Secs = secs + e.Time.Monotonic.CPU.Nanos = nanos + return nil + } + default: + addDeltaTimestamp = func(e *AndroidEvent) error { + ts := timestampBuilder(e.Time) + ts, err := getTsFromDelta(ts, deltaNS) + if err != nil { + return err + } + secs := (ts / 1e9) + nanos := (ts % 1e9) + e.Time.Monotonic.Wall.Secs = secs + e.Time.Monotonic.Wall.Nanos = nanos + return nil + } + } + return addDeltaTimestamp +} + +func getTsFromDelta(ts uint64, deltaNS int64) (uint64, error) { + if deltaNS < 0 && uint64(-deltaNS) <= ts { + return ts - uint64(-deltaNS), nil + } else if deltaNS >= 0 { + return ts + uint64(deltaNS), nil + } + return 0, errors.New("error: cannot subtract a delta bigger than the timestamp itself") +} + // CallTrees generates call trees for a given profile. func (p Android) CallTrees() map[uint64][]*nodetree.Node { return p.CallTreesWithMaxDepth(MaxStackDepth) diff --git a/internal/profile/android_test.go b/internal/profile/android_test.go index 4db9f78..0c8b8f8 100644 --- a/internal/profile/android_test.go +++ b/internal/profile/android_test.go @@ -880,3 +880,99 @@ func TestFixSamplesTime(t *testing.T) { }) } } + +func TestAddTimeDelta(t *testing.T) { + tests := []struct { + name string + delta int64 + trace Android + want AndroidEvent + }{ + { + name: "Delta increase seconds", + delta: 50, + trace: Android{ + Clock: "Dual", + Events: []AndroidEvent{ + { + Action: "Enter", + ThreadID: 1, + MethodID: 1, + Time: EventTime{ + Monotonic: EventMonotonic{ + Wall: Duration{ + Secs: 1, + Nanos: 1e9, + }, + }, + }, + }, + }, + StartTime: 0, + }, + want: AndroidEvent{ + Action: "Enter", + ThreadID: 1, + MethodID: 1, + Time: EventTime{ + Monotonic: EventMonotonic{ + Wall: Duration{ + Secs: 2, + Nanos: 50, + }, + }, + }, + }, + }, + { + name: "Delta decrease nanos", + delta: -50, + trace: Android{ + Clock: "Dual", + Events: []AndroidEvent{ + { + Action: "Enter", + ThreadID: 1, + MethodID: 1, + Time: EventTime{ + Monotonic: EventMonotonic{ + Wall: Duration{ + Secs: 1, + Nanos: 100, + }, + }, + }, + }, + }, + StartTime: 0, + }, + want: AndroidEvent{ + Action: "Enter", + ThreadID: 1, + MethodID: 1, + Time: EventTime{ + Monotonic: EventMonotonic{ + Wall: Duration{ + Secs: 1, + Nanos: 50, + }, + }, + }, + }, + }, + } // end tests + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + addTimeDelta := test.trace.AddTimeDelta(test.delta) + event := test.trace.Events[0] + err := addTimeDelta(&event) + if err != nil { + t.Fatal(err) + } + if diff := testutil.Diff(event, test.want); diff != "" { + t.Fatalf("Result mismatch: got - want +\n%s", diff) + } + }) + } +} diff --git a/internal/speedscope/speedscope.go b/internal/speedscope/speedscope.go index eb62ade..5db4dee 100644 --- a/internal/speedscope/speedscope.go +++ b/internal/speedscope/speedscope.go @@ -88,20 +88,21 @@ type ( ValueUnit string Output struct { - ActiveProfileIndex int `json:"activeProfileIndex"` - AndroidClock string `json:"androidClock,omitempty"` - DurationNS uint64 `json:"durationNS,omitempty"` - Images []debugmeta.Image `json:"images,omitempty"` - Measurements map[string]measurements.Measurement `json:"measurements,omitempty"` - Metadata ProfileMetadata `json:"metadata"` - Platform platform.Platform `json:"platform"` - ProfileID string `json:"profileID,omitempty"` - Profiles []interface{} `json:"profiles"` - ProjectID uint64 `json:"projectID"` - Shared SharedData `json:"shared"` - TransactionName string `json:"transactionName"` - Version string `json:"version,omitempty"` - Metrics *[]utils.FunctionMetrics `json:"metrics"` + ActiveProfileIndex int `json:"activeProfileIndex"` + AndroidClock string `json:"androidClock,omitempty"` + DurationNS uint64 `json:"durationNS,omitempty"` + Images []debugmeta.Image `json:"images,omitempty"` + Measurements map[string]measurements.Measurement `json:"measurements,omitempty"` + MeasurementsV2 map[string]measurements.MeasurementV2 `json:"measurements_v2,omitempty"` + Metadata ProfileMetadata `json:"metadata"` + Platform platform.Platform `json:"platform"` + ProfileID string `json:"profileID,omitempty"` + Profiles []interface{} `json:"profiles"` + ProjectID uint64 `json:"projectID"` + Shared SharedData `json:"shared"` + TransactionName string `json:"transactionName"` + Version string `json:"version,omitempty"` + Metrics *[]utils.FunctionMetrics `json:"metrics"` } ProfileMetadata struct {