diff --git a/analytics/client_test.go b/analytics/client_test.go index 931800c..584524a 100644 --- a/analytics/client_test.go +++ b/analytics/client_test.go @@ -8,7 +8,8 @@ import ( "testing" "time" - "github.com/bitrise-io/go-utils/v2/analytics/mocks" + "github.com/bitrise-io/go-utils/v2/mocks" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" ) diff --git a/analytics/track_test.go b/analytics/track_test.go index 5c8b86d..fc1f2f6 100644 --- a/analytics/track_test.go +++ b/analytics/track_test.go @@ -6,7 +6,8 @@ import ( "testing" "time" - "github.com/bitrise-io/go-utils/v2/analytics/mocks" + "github.com/bitrise-io/go-utils/v2/mocks" + "github.com/stretchr/testify/mock" ) diff --git a/analytics/mocks/Client.go b/mocks/Client.go similarity index 100% rename from analytics/mocks/Client.go rename to mocks/Client.go diff --git a/analytics/mocks/Logger.go b/mocks/Logger.go similarity index 100% rename from analytics/mocks/Logger.go rename to mocks/Logger.go diff --git a/redactwriter/range.go b/redactwriter/range.go new file mode 100644 index 0000000..71b9e81 --- /dev/null +++ b/redactwriter/range.go @@ -0,0 +1,41 @@ +package redactwriter + +import ( + "bytes" + "sort" +) + +type matchRange struct{ first, last int } + +// allRanges returns every indexes of instance of pattern in b, or nil if pattern is not present in b. +func allRanges(b, pattern []byte) (ranges []matchRange) { + i := 0 + for { + sub := b[i:] + idx := bytes.Index(sub, pattern) + if idx == -1 { + return + } + + ranges = append(ranges, matchRange{first: idx + i, last: idx + i + len(pattern)}) + + i += idx + 1 + if i > len(b)-1 { + return + } + } +} + +// mergeAllRanges merges every overlapping ranges in r. +func mergeAllRanges(r []matchRange) []matchRange { + sort.Slice(r, func(i, j int) bool { return r[i].first < r[j].first }) + for i := 0; i < len(r)-1; i++ { + for i+1 < len(r) && r[i+1].first <= r[i].last { + if r[i+1].last > r[i].last { + r[i].last = r[i+1].last + } + r = append(r[:i+1], r[i+2:]...) + } + } + return r +} diff --git a/redactwriter/range_test.go b/redactwriter/range_test.go new file mode 100644 index 0000000..f82b7df --- /dev/null +++ b/redactwriter/range_test.go @@ -0,0 +1,77 @@ +package redactwriter + +import ( + "reflect" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestAllRanges(t *testing.T) { + { + ranges := allRanges([]byte("test"), []byte("t")) + require.Equal(t, []matchRange{{first: 0, last: 1}, {first: 3, last: 4}}, ranges) + } + + { + ranges := allRanges([]byte("test rangetest"), []byte("test")) + require.Equal(t, []matchRange{{first: 0, last: 4}, {first: 10, last: 14}}, ranges) + } + + { + ranges := allRanges([]byte("\n"), []byte("\n")) + require.Equal(t, []matchRange{{first: 0, last: 1}}, ranges) + } + + { + ranges := allRanges([]byte("test\n"), []byte("\n")) + require.Equal(t, []matchRange{{first: 4, last: 5}}, ranges) + } + + { + ranges := allRanges([]byte("\n\ntest\n"), []byte("\n")) + require.Equal(t, []matchRange{{first: 0, last: 1}, {first: 1, last: 2}, {first: 6, last: 7}}, ranges) + } + + { + ranges := allRanges([]byte("\n\ntest\n"), []byte("test\n")) + require.Equal(t, []matchRange{{first: 2, last: 7}}, ranges) + } +} + +func TestMergeAllRanges(t *testing.T) { + var testCases = []struct { + name string + ranges []matchRange + want []matchRange + }{ + { + name: "merges overlapping ranges", + ranges: []matchRange{{0, 2}, {1, 3}}, + want: []matchRange{{0, 3}}, + }, + { + name: "does not merge distinct ranges", + ranges: []matchRange{{0, 2}, {3, 5}}, + want: []matchRange{{0, 2}, {3, 5}}, + }, + { + name: "returns the wider range", + ranges: []matchRange{{0, 2}, {1, 2}}, + want: []matchRange{{0, 2}}, + }, + { + name: "complex test", + ranges: []matchRange{{11, 15}, {0, 2}, {11, 13}, {2, 4}, {6, 9}, {5, 10}}, + want: []matchRange{{0, 4}, {5, 10}, {11, 15}}, + }, + } + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + if got := mergeAllRanges(tc.ranges); !reflect.DeepEqual(got, tc.want) { + t.Errorf("got %v, want %v", got, tc.want) + } + }) + } + +} diff --git a/redactwriter/redactwriter.go b/redactwriter/redactwriter.go new file mode 100644 index 0000000..a140e3a --- /dev/null +++ b/redactwriter/redactwriter.go @@ -0,0 +1,369 @@ +package redactwriter + +import ( + "bytes" + "io" + "sort" + "strings" + "sync" + "time" + + "github.com/bitrise-io/go-utils/v2/log" +) + +// RedactStr ... +const RedactStr = "[REDACTED]" + +var newLine = []byte("\n") + +// Writer ... +type Writer struct { + writer io.Writer + secrets [][][]byte + + chunk []byte + store [][]byte + mux sync.Mutex + timer *time.Timer + logger log.Logger +} + +// New ... +func New(secrets []string, target io.Writer, logger log.Logger) *Writer { + extendedSecrets := secrets + // adding transformed secrets with escaped newline characters to ensure that these are also obscured if found in logs + for _, secret := range secrets { + if strings.Contains(secret, "\n") { + extendedSecrets = append(extendedSecrets, strings.ReplaceAll(secret, "\n", `\n`)) + } + } + + return &Writer{ + writer: target, + secrets: secretsByteList(extendedSecrets), + logger: logger, + } +} + +// Write implements io.Writer interface. +// Splits p into lines, the lines are matched against the secrets, +// this determines which lines can be redacted and passed to the next writer (target). +// There might be lines that need to be buffered since they partially match a secret. +// We do not know the last Write call, so Close needs to be called to flush the buffer. +func (w *Writer) Write(p []byte) (int, error) { + if len(p) == 0 { + return 0, nil + } + + w.mux.Lock() + defer w.mux.Unlock() + + // previous bytes may not end with newline + data := append(w.chunk, p...) + + lastLines, chunk := splitAfterNewline(data) + w.chunk = chunk + if len(chunk) > 0 { + // we have remaining bytes, do not swallow them + if w.timer != nil { + w.timer.Stop() + w.timer.C = nil + } + w.timer = time.AfterFunc(100*time.Millisecond, func() { + if _, err := w.flush(); err != nil { + w.logger.Errorf("Failed to print last lines: %s", err) + } + }) + } + + if len(lastLines) == 0 { + // it is necessary to return the count of incoming bytes + return len(p), nil + } + + for _, line := range lastLines { + lines := append(w.store, line) + matchMap, partialMatchIndexes := w.matchSecrets(lines) + + var linesToPrint [][]byte + linesToPrint, w.store = w.matchLines(lines, partialMatchIndexes) + if linesToPrint == nil { + continue + } + + redactedLines := w.redact(linesToPrint, matchMap) + redactedBytes := bytes.Join(redactedLines, nil) + if c, err := w.writer.Write(redactedBytes); err != nil { + return c, err + } + } + + // it is necessary to return the count of incoming bytes + // to let the exec.Command work properly + return len(p), nil +} + +// Close implements io.Writer interface +func (w *Writer) Close() error { + _, err := w.flush() + return err +} + +// flush writes the remaining bytes. +func (w *Writer) flush() (int, error) { + defer func() { + w.mux.Unlock() + }() + w.mux.Lock() + + if len(w.chunk) > 0 { + // lines are containing newline, chunk may not + chunk := w.chunk + w.chunk = nil + w.store = append(w.store, chunk) + } + + // we only need to care about the full matches in the remaining lines + // (no more lines were come, why care about the partial matches?) + matchMap, _ := w.matchSecrets(w.store) + redactedLines := w.redact(w.store, matchMap) + w.store = nil + + return w.writer.Write(bytes.Join(redactedLines, nil)) +} + +// matchSecrets collects which secrets matches from which line indexes +// and which secrets matches partially from which line indexes. +// matchMap: matching line chunk's first line indexes by secret index +// partialMatchIndexes: line indexes from which secrets matching but not fully contained in lines +func (w *Writer) matchSecrets(lines [][]byte) (matchMap map[int][]int, partialMatchIndexes map[int]bool) { + matchMap = make(map[int][]int) + partialMatchIndexes = make(map[int]bool) + + for secretIdx, secret := range w.secrets { + secretLine := secret[0] // every match should begin from the secret first line + var lineIndexes []int // the indexes of lines which contains the secret's first line + + for i, line := range lines { + if bytes.Contains(line, secretLine) { + lineIndexes = append(lineIndexes, i) + } + } + + if len(lineIndexes) == 0 { + // this secret can not be found in the lines + continue + } + + for _, lineIdx := range lineIndexes { + if len(secret) == 1 { + // the single line secret found in the lines + indexes := matchMap[secretIdx] + matchMap[secretIdx] = append(indexes, lineIdx) + continue + } + + // lineIdx. line matches to a multi line secret's first line + // if lines has more line, every subsequent line must match to the secret's subsequent lines + partialMatch := true + match := false + + for i := lineIdx + 1; i < len(lines); i++ { + secretLineIdx := i - lineIdx + + secretLine = secret[secretLineIdx] + line := lines[i] + + if !bytes.Contains(line, secretLine) { + partialMatch = false + break + } + + if secretLineIdx == len(secret)-1 { + // multi line secret found in the lines + match = true + break + } + } + + if match { + // multi line secret found in the lines + indexes := matchMap[secretIdx] + matchMap[secretIdx] = append(indexes, lineIdx) + continue + } + + if partialMatch { + // this secret partially can be found in the lines + partialMatchIndexes[lineIdx] = true + } + } + } + + return +} + +// linesToKeepRange returns the first line index needs to be observed +// since they contain partially matching secrets. +func (w *Writer) linesToKeepRange(partialMatchIndexes map[int]bool) int { + first := -1 + + for lineIdx := range partialMatchIndexes { + if first == -1 { + first = lineIdx + continue + } + + if first > lineIdx { + first = lineIdx + } + } + + return first +} + +// matchLines return which lines can be printed and which should be kept for further observing. +func (w *Writer) matchLines(lines [][]byte, partialMatchIndexes map[int]bool) ([][]byte, [][]byte) { + first := w.linesToKeepRange(partialMatchIndexes) + switch first { + case -1: + // no lines need to be kept + return lines, nil + case 0: + // partial match is always longer then the lines + return nil, lines + default: + return lines[:first], lines[first:] + } +} + +// secretLinesToRedact returns which secret lines should be redacted +func (w *Writer) secretLinesToRedact(lineIdxToRedact int, matchMap map[int][]int) [][]byte { + // which line is which secrets first matching line + secretIdxsByLine := make(map[int][]int) + for secretIdx, lineIndexes := range matchMap { + for _, lineIdx := range lineIndexes { + secretIdxsByLine[lineIdx] = append(secretIdxsByLine[lineIdx], secretIdx) + } + } + + var secretChunks [][]byte + for firstMatchingLineIdx, secretIndexes := range secretIdxsByLine { + if lineIdxToRedact < firstMatchingLineIdx { + continue + } + + for _, secretIdx := range secretIndexes { + secret := w.secrets[secretIdx] + + if lineIdxToRedact > firstMatchingLineIdx+len(secret)-1 { + continue + } + + secretLineIdx := lineIdxToRedact - firstMatchingLineIdx + secretChunks = append(secretChunks, secret[secretLineIdx]) + } + } + + sort.Slice(secretChunks, func(i, j int) bool { return len(secretChunks[i]) < len(secretChunks[j]) }) + return secretChunks +} + +// redact hides the given ranges in the given line. +func redact(line []byte, ranges []matchRange) []byte { + var offset int // the offset of ranges generated by replacing x bytes by the RedactStr + for _, r := range ranges { + length := r.last - r.first + first := r.first + offset + last := first + length + + toRedact := line[first:last] + redactStr := RedactStr + if bytes.HasSuffix(toRedact, []byte("\n")) { + // if string to redact ends with newline redact message should also + redactStr += "\n" + } + + newLine := append([]byte{}, line[:first]...) + newLine = append(newLine, redactStr...) + newLine = append(newLine, line[last:]...) + + offset += len(redactStr) - length + + line = newLine + } + return line +} + +// redact hides the given secrets in the given lines. +func (w *Writer) redact(lines [][]byte, matchMap map[int][]int) [][]byte { + secretIdxsByLine := map[int][]int{} + for secretIdx, lineIndexes := range matchMap { + for _, lineIdx := range lineIndexes { + secretIdxsByLine[lineIdx] = append(secretIdxsByLine[lineIdx], secretIdx) + } + } + + for i, line := range lines { + linesToRedact := w.secretLinesToRedact(i, matchMap) + if linesToRedact == nil { + continue + } + + var ranges []matchRange + for _, lineToRedact := range linesToRedact { + ranges = append(ranges, allRanges(line, lineToRedact)...) + } + + lines[i] = redact(line, mergeAllRanges(ranges)) + } + + return lines +} + +// secretsByteList returns the list of secret byte lines. +func secretsByteList(secrets []string) [][][]byte { + var s [][][]byte + for _, secret := range secrets { + lines, lastLine := splitAfterNewline([]byte(secret)) + if lines == nil && lastLine == nil { + continue + } + + var secretLines [][]byte + if lines != nil { + secretLines = append(secretLines, lines...) + } + if lastLine != nil { + secretLines = append(secretLines, lastLine) + } + s = append(s, secretLines) + } + return s +} + +// splitAfterNewline splits p after "\n", the split is assigned to lines +// if last line has no "\n" it is assigned to the chunk. +// If p is nil both lines and chunk is set to nil. +func splitAfterNewline(p []byte) ([][]byte, []byte) { + chunk := p + var lines [][]byte + + for len(chunk) > 0 { + idx := bytes.Index(chunk, newLine) + if idx == -1 { + return lines, chunk + } + + lines = append(lines, chunk[:idx+1]) + + if idx == len(chunk)-1 { + chunk = nil + break + } + + chunk = chunk[idx+1:] + } + + return lines, chunk +} diff --git a/redactwriter/redactwriter_test.go b/redactwriter/redactwriter_test.go new file mode 100644 index 0000000..32825ba --- /dev/null +++ b/redactwriter/redactwriter_test.go @@ -0,0 +1,560 @@ +//go:build !race + +package redactwriter + +import ( + "bytes" + "fmt" + "runtime" + "testing" + + "github.com/bitrise-io/go-utils/v2/mocks" + + "github.com/bitrise-io/go-utils/v2/log" + "github.com/stretchr/testify/require" +) + +func TestSecretsByteList(t *testing.T) { + { + secrets := []string{"secret value"} + byteList := secretsByteList(secrets) + require.Equal(t, [][][]byte{ + [][]byte{ + []byte("secret value"), + }, + }, byteList) + } + + { + secrets := []string{"secret value1", "secret value2"} + byteList := secretsByteList(secrets) + require.Equal(t, [][][]byte{ + [][]byte{ + []byte("secret value1"), + }, + [][]byte{ + []byte("secret value2"), + }, + }, byteList) + } + + { + secrets := []string{"multi\nline\nsecret"} + byteList := secretsByteList(secrets) + require.Equal(t, [][][]byte{ + [][]byte{ + []byte("multi\n"), + []byte("line\n"), + []byte("secret"), + }, + }, byteList) + } + + { + secrets := []string{"ending\nwith\nnewline\n"} + byteList := secretsByteList(secrets) + require.Equal(t, [][][]byte{ + [][]byte{ + []byte("ending\n"), + []byte("with\n"), + []byte("newline\n"), + }, + }, byteList) + } + + { + secrets := []string{"\nstarting\nwith\nnewline"} + byteList := secretsByteList(secrets) + require.Equal(t, [][][]byte{ + [][]byte{ + []byte("\n"), + []byte("starting\n"), + []byte("with\n"), + []byte("newline"), + }, + }, byteList) + } + + { + secrets := []string{"newlines\nin\n\nthe\n\n\nmiddle"} + byteList := secretsByteList(secrets) + require.Equal(t, [][][]byte{ + [][]byte{ + []byte("newlines\n"), + []byte("in\n"), + []byte("\n"), + []byte("the\n"), + []byte("\n"), + []byte("\n"), + []byte("middle"), + }, + }, byteList) + } +} + +func TestWrite(t *testing.T) { + t.Log("trivial test") + { + var buff bytes.Buffer + var mockLogger log.Logger = new(mocks.Logger) + out := New([]string{"abc", "a\nb\nc"}, &buff, mockLogger) + log := []byte("test with\nnew line\nand single line secret:abc\nand multiline secret:a\nb\nc") + wc, err := out.Write(log) + require.NoError(t, err) + require.Equal(t, len(log), wc) + + err = out.Close() + require.NoError(t, err) + require.Equal(t, "test with\nnew line\nand single line secret:[REDACTED]\nand multiline secret:[REDACTED]\n[REDACTED]\n[REDACTED]", buff.String()) + } + + t.Log("chunk without newline") + { + var buff bytes.Buffer + var mockLogger log.Logger = new(mocks.Logger) + out := New([]string{"ab", "a\nb"}, &buff, mockLogger) + log := []byte("test without newline, secret:ab") + wc, err := out.Write(log) + require.NoError(t, err) + require.Equal(t, len(log), wc) + + err = out.Close() + require.NoError(t, err) + require.Equal(t, "test without newline, secret:[REDACTED]", buff.String()) + } + + t.Log("multiple secret in the same line") + { + var buff bytes.Buffer + var mockLogger log.Logger = new(mocks.Logger) + out := New([]string{"x1", "x\n2"}, &buff, mockLogger) + log := []byte("multiple secrets like: x1 and x\n2 and some extra text") + wc, err := out.Write(log) + require.NoError(t, err) + require.Equal(t, len(log), wc) + + err = out.Close() + require.NoError(t, err) + require.Equal(t, "multiple secrets like: [REDACTED] and [REDACTED]\n[REDACTED] and some extra text", buff.String()) + } + + maxRun := 150000 + t.Log("multiple secret in the same line with multiple goroutine ") + { + cherr := make(chan error, maxRun) + chStr := make(chan string, maxRun) + + var buff bytes.Buffer + var mockLogger log.Logger = new(mocks.Logger) + out := New([]string{"x1", "x\n2"}, &buff, mockLogger) + log := []byte("multiple secrets like: x1 and x\n2 and some extra text") + for i := 0; i < maxRun; i++ { + go func(buff bytes.Buffer, out *Writer, log []byte) { + runtime.Gosched() + buff.Reset() + + wc, err := out.Write(log) + require.NoError(t, err) + require.Equal(t, len(log), wc) + + err = out.Close() + + cherr <- err + chStr <- buff.String() + }(buff, out, log) + } + + errCounter := 0 + for err := range cherr { + require.NoError(t, err) + + errCounter++ + if errCounter == maxRun { + close(cherr) + } + } + + strCounter := 0 + for str := range chStr { + fmt.Println(str) + + strCounter++ + if strCounter == maxRun { + close(chStr) + } + } + } +} + +func TestSecrets(t *testing.T) { + secrets := []string{ + "a\nb\nc", + "b", + "c\nb", + "x\nc\nb\nd", + "f", + } + + var buff bytes.Buffer + var mockLogger log.Logger = new(mocks.Logger) + out := New(secrets, &buff, mockLogger) + require.Equal(t, [][][]byte{ + [][]byte{[]byte("a\n"), []byte("b\n"), []byte("c")}, + [][]byte{[]byte("b")}, + [][]byte{[]byte("c\n"), []byte("b")}, + [][]byte{[]byte("x\n"), []byte("c\n"), []byte("b\n"), []byte("d")}, + [][]byte{[]byte("f")}, + [][]byte{[]byte(`a\nb\nc`)}, + [][]byte{[]byte(`c\nb`)}, + [][]byte{[]byte(`x\nc\nb\nd`)}, + }, out.secrets) +} + +func TestMatchSecrets(t *testing.T) { + secrets := []string{ + "a\nb\nc", + "b", + "c\nb", + "x\nc\nb\nd", + "f", + } + lines := [][]byte{ + []byte("x\n"), + []byte("a\n"), + []byte("a\n"), + []byte("b\n"), + []byte("c\n"), + []byte("x\n"), + []byte("c\n"), + []byte("b\n")} + + var buff bytes.Buffer + var mockLogger log.Logger = new(mocks.Logger) + out := New(secrets, &buff, mockLogger) + + matchMap, partialMatchMap := out.matchSecrets(lines) + require.Equal(t, map[int][]int{ + 0: []int{2}, + 1: []int{3, 7}, + 2: []int{6}, + }, matchMap) + require.Equal(t, map[int]bool{5: true}, partialMatchMap) +} + +func TestLinesToKeepRange(t *testing.T) { + t.Log() + secrets := []string{ + "a\nb\nc", + "b", + "c\nb", + "x\nc\nb\nd", + } + // lines := [][]byte{ + // []byte("x\n"), + // []byte("a\n"), + // []byte("a\n"), + // []byte("b\n"), + // []byte("c\n"), + // []byte("x\n"), 5.line + // []byte("c\n"), + // []byte("b\n")} + + var buff bytes.Buffer + var mockLogger log.Logger = new(mocks.Logger) + out := New(secrets, &buff, mockLogger) + + partialMatchMap := map[int]bool{6: true, 2: true, 5: true, 7: true} + first := out.linesToKeepRange(partialMatchMap) // minimal index in the partialMatchMap + require.Equal(t, 2, first) +} + +func TestMatchLine(t *testing.T) { + secrets := []string{ + "a\nb\nc", + "b", + "c\nb", + "x\nc\nb\nd", + "f", + } + lines := [][]byte{ + []byte("x\n"), // 0. + []byte("a\n"), + []byte("a\n"), // 2. + []byte("b\n"), + []byte("c\n"), // 4. + []byte("x\n"), + []byte("c\n"), // 6. + []byte("b\n")} + + var buff bytes.Buffer + var mockLogger log.Logger = new(mocks.Logger) + out := New(secrets, &buff, mockLogger) + + _, partialMatchMap := out.matchSecrets(lines) + print, remaining := out.matchLines(lines, partialMatchMap) + require.Equal(t, [][]byte{ + []byte("x\n"), + []byte("a\n"), + []byte("a\n"), + []byte("b\n"), + []byte("c\n"), + }, print) + require.Equal(t, [][]byte{ + []byte("x\n"), + []byte("c\n"), + []byte("b\n"), + }, remaining) +} + +func TestSecretLinesToRedact(t *testing.T) { + secrets := []string{ + "a\nb\nc", + "b", + } + lines := [][]byte{ + []byte("x\n"), + []byte("a\n"), + []byte("b\n"), + []byte("c\n"), + []byte("b\n"), + } + + var buff bytes.Buffer + var mockLogger log.Logger = new(mocks.Logger) + out := New(secrets, &buff, mockLogger) + + matchMap, _ := out.matchSecrets(lines) + require.Equal(t, map[int][]int{ + 0: []int{1}, + 1: []int{2, 4}, + }, matchMap) + + secretLines := out.secretLinesToRedact(0, matchMap) + require.Equal(t, ([][]byte)(nil), secretLines, fmt.Sprintf("%s\n", secretLines)) + + secretLines = out.secretLinesToRedact(1, matchMap) + require.Equal(t, [][]byte{[]byte("a\n")}, secretLines, fmt.Sprintf("%s\n", secretLines)) + + secretLines = out.secretLinesToRedact(2, matchMap) + require.Equal(t, [][]byte{[]byte("b"), []byte("b\n")}, secretLines, fmt.Sprintf("%s\n", secretLines)) + + secretLines = out.secretLinesToRedact(3, matchMap) + require.Equal(t, [][]byte{[]byte("c")}, secretLines, fmt.Sprintf("%s\n", secretLines)) + + secretLines = out.secretLinesToRedact(4, matchMap) + require.Equal(t, [][]byte{[]byte("b")}, secretLines, fmt.Sprintf("%s\n", secretLines)) +} + +func TestRedactLine(t *testing.T) { + t.Log("redacts the middle of the line") + { + line := []byte("asdfabcasdf") + ranges := []matchRange{ // asdfabcasdf + {first: 4, last: 7}, // ****abc**** + } + redacted := redact(line, ranges) + require.Equal(t, []byte("asdf[REDACTED]asdf"), redacted, string(redacted)) + } + + t.Log("redacts the begining of the line") + { + line := []byte("asdfabcasdf") + ranges := []matchRange{ // asdfabcasdf + {first: 0, last: 5}, // asdfa****** + } + redacted := redact(line, ranges) + require.Equal(t, []byte("[REDACTED]bcasdf"), redacted, string(redacted)) + } + + t.Log("redacts the end of the line") + { + line := []byte("asdfabcasdf") + ranges := []matchRange{ // asdfabcasdf + {first: 9, last: 11}, // *********df + } + redacted := redact(line, ranges) + require.Equal(t, []byte("asdfabcas[REDACTED]"), redacted, string(redacted)) + } + + t.Log("redacts multiple secrets") + { + line := []byte("asdfabcasdf") + ranges := []matchRange{ // asdfabcasdf + {first: 4, last: 7}, // ****abc**** + {first: 8, last: 10}, // ********sd* + } + redacted := redact(line, ranges) + require.Equal(t, []byte("asdf[REDACTED]a[REDACTED]f"), redacted, string(redacted)) + } + + t.Log("redacts the whole line") + { + line := []byte("asdfabcasdf") + ranges := []matchRange{ // asdfabcasdf + {first: 0, last: 4}, // asdf******* + {first: 7, last: 11}, // *******asdf + {first: 3, last: 9}, // ***fabcas** + } + ranges = mergeAllRanges(ranges) + redacted := redact(line, ranges) + require.Equal(t, []byte("[REDACTED]"), redacted, string(redacted)) + } +} + +func TestRedact(t *testing.T) { + secrets := []string{ + "a\nb\nc", + "b", + } + lines := [][]byte{ + []byte("x\n"), + []byte("a\n"), + []byte("a\n"), + []byte("b\n"), + []byte("c\n"), + } + + var buff bytes.Buffer + var mockLogger log.Logger = new(mocks.Logger) + out := New(secrets, &buff, mockLogger) + + matchMap := map[int][]int{0: []int{2}, 1: []int{3}} + redacted := out.redact(lines, matchMap) + require.Equal(t, [][]byte{ + []byte("x\n"), + []byte("a\n"), + []byte(RedactStr + "\n"), + []byte(RedactStr + "\n"), + []byte(RedactStr + "\n"), + }, redacted) + + { + secrets := []string{ + "106\n105", + "99", + } + lines := [][]byte{ + []byte("106\n"), + []byte("105\n"), + []byte("104\n"), + []byte("103\n"), + []byte("102\n"), + []byte("101\n"), + []byte("100\n"), + []byte("99\n")} + + var buff bytes.Buffer + var mockLogger log.Logger = new(mocks.Logger) + out := New(secrets, &buff, mockLogger) + + matchMap := map[int][]int{ + 0: []int{0}, + 1: []int{7}, + } + redacted := out.redact(lines, matchMap) + require.Equal(t, [][]byte{ + []byte(RedactStr + "\n"), + []byte(RedactStr + "\n"), + []byte("104" + "\n"), + []byte("103" + "\n"), + []byte("102" + "\n"), + []byte("101" + "\n"), + []byte("100" + "\n"), + []byte(RedactStr + "\n"), + }, redacted, fmt.Sprintf("%s", redacted)) + } +} + +func TestSplitAfterNewline(t *testing.T) { + t.Log("bytes") + { + require.Equal(t, []byte{}, []byte("")) + } + + t.Log("empty test") + { + b := []byte{} + lines, chunk := splitAfterNewline(b) + require.Equal(t, [][]byte(nil), lines) + require.Equal(t, []byte{}, chunk) + } + + t.Log("empty test - empty string bytes") + { + b := []byte("") + lines, chunk := splitAfterNewline(b) + require.Equal(t, [][]byte(nil), lines) + require.Equal(t, []byte{}, chunk) + } + + t.Log("newline test") + { + b := []byte("\n") + lines, chunk := splitAfterNewline(b) + require.Equal(t, [][]byte{[]byte("\n")}, lines) + require.Equal(t, []byte(nil), chunk) + } + + t.Log("multi line test") + { + b := []byte(`line 1 +line 2 +line 3 +`) + lines, chunk := splitAfterNewline(b) + require.Equal(t, 3, len(lines)) + require.Equal(t, []byte("line 1\n"), lines[0]) + require.Equal(t, []byte("line 2\n"), lines[1]) + require.Equal(t, []byte("line 3\n"), lines[2]) + require.Equal(t, []byte(nil), chunk) + } + + t.Log("multi line test - newlines") + { + b := []byte(` + +line 1 + +line 2 +`) + + lines, chunk := splitAfterNewline(b) + require.Equal(t, 5, len(lines)) + require.Equal(t, []byte("\n"), lines[0]) + require.Equal(t, []byte("\n"), lines[1]) + require.Equal(t, []byte("line 1\n"), lines[2]) + require.Equal(t, []byte("\n"), lines[3]) + require.Equal(t, []byte("line 2\n"), lines[4]) + require.Equal(t, []byte(nil), chunk) + } + + t.Log("chunk test") + { + b := []byte("line 1") + lines, chunk := splitAfterNewline(b) + require.Equal(t, [][]byte(nil), lines) + require.Equal(t, []byte("line 1"), chunk) + } + + t.Log("chunk test") + { + b := []byte(`line 1 +line 2`) + + lines, chunk := splitAfterNewline(b) + require.Equal(t, 1, len(lines)) + require.Equal(t, []byte("line 1\n"), lines[0]) + require.Equal(t, []byte("line 2"), chunk) + } + t.Log("chunk test") + { + b := []byte("test\n\ntest\n") + + lines, chunk := splitAfterNewline(b) + require.Equal(t, 3, len(lines)) + require.Equal(t, []byte("test\n"), lines[0]) + require.Equal(t, []byte("\n"), lines[1]) + require.Equal(t, []byte("test\n"), lines[2]) + require.Equal(t, []byte(nil), chunk) + } +}