-
Notifications
You must be signed in to change notification settings - Fork 39
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
* Update dependencies * Add new input * Add file redacting * Add secret sharing * Add e2e test * Add failure warning to the input * Simplify writer handling * Simpler mock exclusion * Remove env var expansion * Skip redaction when possible * Update fileredactor/fileredactor.go Co-authored-by: lpusok <[email protected]> * Update errors --------- Co-authored-by: lpusok <[email protected]>
- Loading branch information
Showing
33 changed files
with
1,637 additions
and
76 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,64 @@ | ||
| package fileredactor | ||
|
|
||
| import ( | ||
| "fmt" | ||
| "strings" | ||
|
|
||
| "github.com/bitrise-io/go-utils/v2/pathutil" | ||
| ) | ||
|
|
||
| // FilePathProcessor is an interface for an entity which accepts file paths separated by the | ||
| // newline (`\n`) character and returns a slice of absolute paths. | ||
| type FilePathProcessor interface { | ||
| ProcessFilePaths(string) ([]string, error) | ||
| } | ||
|
|
||
| type filePathProcessor struct { | ||
| pathModifier pathutil.PathModifier | ||
| pathChecker pathutil.PathChecker | ||
| } | ||
|
|
||
| // NewFilePathProcessor returns a structure which implements the FilePathProcessor interface. | ||
| // The implementation includes handling filepaths defined as environment variables, relative file paths, | ||
| // and absolute file paths. | ||
| // The implementation also includes making sure the filepath exists and is not a directory. | ||
| func NewFilePathProcessor(modifier pathutil.PathModifier, checker pathutil.PathChecker) FilePathProcessor { | ||
| return filePathProcessor{ | ||
| pathModifier: modifier, | ||
| pathChecker: checker, | ||
| } | ||
| } | ||
|
|
||
| func (f filePathProcessor) ProcessFilePaths(filePaths string) ([]string, error) { | ||
| filePaths = strings.TrimSpace(filePaths) | ||
| if filePaths == "" { | ||
| return nil, nil | ||
| } | ||
|
|
||
| var processedFilePaths []string | ||
|
|
||
| list := strings.Split(filePaths, "\n") | ||
| for _, item := range list { | ||
| item = strings.TrimSpace(item) | ||
| if item == "" { | ||
| continue | ||
| } | ||
|
|
||
| path, err := f.pathModifier.AbsPath(item) | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
|
|
||
| isDir, err := f.pathChecker.IsDirExists(path) | ||
| if err != nil { | ||
| return nil, fmt.Errorf("failed to check if path (%s) is a directory: %w", path, err) | ||
| } | ||
| if isDir { | ||
| return nil, fmt.Errorf("path (%s) is a directory and cannot be redacted, please make sure to only provide filepaths as inputs", path) | ||
| } | ||
|
|
||
| processedFilePaths = append(processedFilePaths, path) | ||
| } | ||
|
|
||
| return processedFilePaths, nil | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,67 @@ | ||
| package fileredactor | ||
|
|
||
| import ( | ||
| "reflect" | ||
| "testing" | ||
|
|
||
| "github.com/bitrise-steplib/steps-deploy-to-bitrise-io/mocks" | ||
| "github.com/stretchr/testify/assert" | ||
| "github.com/stretchr/testify/mock" | ||
| ) | ||
|
|
||
| func Test_ProcessFilePaths(t *testing.T) { | ||
| deployDirPath := "/some/absolute/path/deploy_dir" | ||
| tests := []struct { | ||
| name string | ||
| input string | ||
| output []string | ||
| outputErr string | ||
| envs map[string]string | ||
| }{ | ||
| { | ||
| name: "Empty input", | ||
| input: " ", | ||
| output: nil, | ||
| outputErr: "", | ||
| envs: nil, | ||
| }, | ||
| { | ||
| name: "Expand relative path", | ||
| input: ` | ||
| /some/absolute/path/to/file.txt | ||
| file_in_deploy_dir.txt | ||
| `, | ||
| output: []string{ | ||
| "/some/absolute/path/to/file.txt", | ||
| "/some/absolute/path/deploy_dir/file_in_deploy_dir.txt", | ||
| }, | ||
| outputErr: "", | ||
| envs: nil, | ||
| }, | ||
| } | ||
|
|
||
| for _, tt := range tests { | ||
| t.Run(tt.name, func(t *testing.T) { | ||
| mockModifier := new(mocks.PathModifier) | ||
| mockModifier.On("AbsPath", "a_file.txt").Return(deployDirPath+"/a_file.txt", nil) | ||
| mockModifier.On("AbsPath", "/some/absolute/path/to/file.txt").Return("/some/absolute/path/to/file.txt", nil) | ||
| mockModifier.On("AbsPath", "file_in_deploy_dir.txt").Return(deployDirPath+"/file_in_deploy_dir.txt", nil) | ||
|
|
||
| mockChecker := new(mocks.PathChecker) | ||
| mockChecker.On("IsDirExists", mock.Anything).Return(false, nil) | ||
|
|
||
| pathProcessor := NewFilePathProcessor(mockModifier, mockChecker) | ||
| result, err := pathProcessor.ProcessFilePaths(tt.input) | ||
|
|
||
| if err != nil && tt.outputErr != "" { | ||
| assert.EqualError(t, err, tt.outputErr) | ||
| } else if err != nil { | ||
| t.Errorf("%s got = %v, want %v", t.Name(), err, tt.outputErr) | ||
| } | ||
|
|
||
| if !reflect.DeepEqual(result, tt.output) { | ||
| t.Errorf("%s got = %v, want %v", t.Name(), result, tt.output) | ||
| } | ||
| }) | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,81 @@ | ||
| package fileredactor | ||
|
|
||
| import ( | ||
| "fmt" | ||
| "io" | ||
| "os" | ||
|
|
||
| "github.com/bitrise-io/go-utils/v2/fileutil" | ||
| "github.com/bitrise-io/go-utils/v2/log" | ||
| "github.com/bitrise-io/go-utils/v2/redactwriter" | ||
| ) | ||
|
|
||
| const bufferSize = 64 * 1024 | ||
|
|
||
| // FileRedactor is an interface for a structure which, given a slice of file paths and another slice of secrets can | ||
| // process the specified files to redact secrets from them. | ||
| type FileRedactor interface { | ||
| RedactFiles([]string, []string) error | ||
| } | ||
|
|
||
| type fileRedactor struct { | ||
| fileManager fileutil.FileManager | ||
| } | ||
|
|
||
| // NewFileRedactor returns a structure that implements the FileRedactor interface | ||
| func NewFileRedactor(manager fileutil.FileManager) FileRedactor { | ||
| return fileRedactor{ | ||
| fileManager: manager, | ||
| } | ||
| } | ||
|
|
||
| func (f fileRedactor) RedactFiles(filePaths []string, secrets []string) error { | ||
| logger := log.NewLogger() | ||
| for _, path := range filePaths { | ||
| if err := f.redactFile(path, secrets, logger); err != nil { | ||
| return fmt.Errorf("failed to redact file (%s): %w", path, err) | ||
| } | ||
| } | ||
|
|
||
| return nil | ||
| } | ||
|
|
||
| func (f fileRedactor) redactFile(path string, secrets []string, logger log.Logger) error { | ||
| source, err := f.fileManager.Open(path) | ||
| if err != nil { | ||
| return fmt.Errorf("failed to open file for redaction (%s): %w", path, err) | ||
| } | ||
| defer func() { | ||
| if err := source.Close(); err != nil { | ||
| logger.Warnf("Failed to close file: %s", err) | ||
| } | ||
| }() | ||
|
|
||
| newPath := path + ".redacted" | ||
| destination, err := os.Create(newPath) | ||
| if err != nil { | ||
| return fmt.Errorf("failed to create temporary file for redaction: %w", err) | ||
| } | ||
| defer func() { | ||
| if err := destination.Close(); err != nil { | ||
| logger.Warnf("Failed to close file: %s", err) | ||
| } | ||
| }() | ||
|
|
||
| redactWriter := redactwriter.New(secrets, destination, logger) | ||
| if _, err := io.Copy(redactWriter, source); err != nil { | ||
| return fmt.Errorf("failed to redact secrets: %w", err) | ||
| } | ||
|
|
||
| if err := redactWriter.Close(); err != nil { | ||
| return fmt.Errorf("failed to close redact writer: %w", err) | ||
| } | ||
|
|
||
| //rename new file to old file name | ||
| err = os.Rename(newPath, path) | ||
| if err != nil { | ||
| return fmt.Errorf("failed to overwrite old file (%s) with redacted file: %w", path, err) | ||
| } | ||
|
|
||
| return nil | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,37 @@ | ||
| package fileredactor | ||
|
|
||
| import ( | ||
| "os" | ||
| "path" | ||
| "testing" | ||
|
|
||
| "github.com/bitrise-io/go-utils/v2/fileutil" | ||
| "github.com/stretchr/testify/assert" | ||
| "github.com/stretchr/testify/require" | ||
| ) | ||
|
|
||
| func Test_RedactFiles(t *testing.T) { | ||
| secrets := []string{ | ||
| "SUPER_SECRET_WORD", | ||
| "ANOTHER_SECRET_WORD", | ||
| } | ||
| filePath := path.Join(t.TempDir(), "step-output.txt") | ||
| content, err := os.ReadFile("testdata/before_redaction.txt") | ||
| require.NoError(t, err) | ||
|
|
||
| fileManager := fileutil.NewFileManager() | ||
| err = fileManager.WriteBytes(filePath, content) | ||
| require.NoError(t, err) | ||
|
|
||
| fileRedactor := NewFileRedactor(fileutil.NewFileManager()) | ||
| err = fileRedactor.RedactFiles([]string{filePath}, secrets) | ||
| require.NoError(t, err) | ||
|
|
||
| got, err := os.ReadFile(filePath) | ||
| require.NoError(t, err) | ||
|
|
||
| want, err := os.ReadFile("testdata/after_redaction.txt") | ||
| require.NoError(t, err) | ||
|
|
||
| assert.Equal(t, want, got) | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,8 @@ | ||
| [REDACTED] | ||
|
|
||
| Normal output from any of our steps. | ||
| More output. | ||
|
|
||
| [REDACTED] | ||
|
|
||
| The end. |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,8 @@ | ||
| SUPER_SECRET_WORD | ||
|
|
||
| Normal output from any of our steps. | ||
| More output. | ||
|
|
||
| ANOTHER_SECRET_WORD | ||
|
|
||
| The end. |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.