From a699eb807dc025859defaf3d5caed0e3f14adfd1 Mon Sep 17 00:00:00 2001 From: Szabolcs Toth <54896607+tothszabi@users.noreply.github.com> Date: Thu, 14 Sep 2023 14:26:33 +0100 Subject: [PATCH] Ci 1502 (#184) * Update dependencies * Add new input * Add file redacting * Add secret sharing * Add e2e test * Add failure warning to the input * Simplify writer handling * Simpler mock exclusion * Remove env var expansion * Skip redaction when possible * Update fileredactor/fileredactor.go Co-authored-by: lpusok <7979773+lpusok@users.noreply.github.com> * Update errors --------- Co-authored-by: lpusok <7979773+lpusok@users.noreply.github.com> --- README.md | 1 + bitrise.yml | 49 ++- fileredactor/filepathprocessor.go | 64 +++ fileredactor/filepathprocessor_test.go | 67 ++++ fileredactor/fileredactor.go | 81 ++++ fileredactor/fileredactor_test.go | 37 ++ fileredactor/testdata/after_redaction.txt | 8 + fileredactor/testdata/before_redaction.txt | 8 + go.mod | 11 +- go.sum | 55 ++- main.go | 57 ++- mocks/PathChecker.go | 72 ++++ mocks/PathModifier.go | 48 +++ mocks/Repository.go | 26 +- step.yml | 17 + .../bitrise-io/bitrise/models/models.go | 22 +- .../bitrise/models/models_methods.go | 5 +- .../bitrise-io/envman/envman/configs.go | 4 +- .../bitrise-io/envman/envman/util.go | 3 + .../envman/models/models_methods.go | 25 +- .../go-steputils/v2/secretkeys/secretkeys.go | 39 ++ .../go-utils/v2/exitcode/exitcode.go | 15 + .../go-utils/v2/exitcode/exitcode_string.go | 24 ++ .../go-utils/v2/fileutil/fileutil.go | 91 +++++ .../v2/log/colorstring/colorstring.go | 111 ++++++ .../bitrise-io/go-utils/v2/log/log.go | 139 +++++++ .../bitrise-io/go-utils/v2/log/severity.go | 35 ++ .../go-utils/v2/pathutil/pathutil.go | 127 ++++++ .../go-utils/v2/redactwriter/range.go | 41 ++ .../go-utils/v2/redactwriter/redactwriter.go | 369 ++++++++++++++++++ .../bitrise-io/stepman/models/models.go | 36 +- .../stepman/models/models_methods.go | 5 + vendor/modules.txt | 21 +- 33 files changed, 1637 insertions(+), 76 deletions(-) create mode 100644 fileredactor/filepathprocessor.go create mode 100644 fileredactor/filepathprocessor_test.go create mode 100644 fileredactor/fileredactor.go create mode 100644 fileredactor/fileredactor_test.go create mode 100644 fileredactor/testdata/after_redaction.txt create mode 100644 fileredactor/testdata/before_redaction.txt create mode 100644 mocks/PathChecker.go create mode 100644 mocks/PathModifier.go create mode 100644 vendor/github.com/bitrise-io/go-steputils/v2/secretkeys/secretkeys.go create mode 100644 vendor/github.com/bitrise-io/go-utils/v2/exitcode/exitcode.go create mode 100644 vendor/github.com/bitrise-io/go-utils/v2/exitcode/exitcode_string.go create mode 100644 vendor/github.com/bitrise-io/go-utils/v2/fileutil/fileutil.go create mode 100644 vendor/github.com/bitrise-io/go-utils/v2/log/colorstring/colorstring.go create mode 100644 vendor/github.com/bitrise-io/go-utils/v2/log/log.go create mode 100644 vendor/github.com/bitrise-io/go-utils/v2/log/severity.go create mode 100644 vendor/github.com/bitrise-io/go-utils/v2/pathutil/pathutil.go create mode 100644 vendor/github.com/bitrise-io/go-utils/v2/redactwriter/range.go create mode 100644 vendor/github.com/bitrise-io/go-utils/v2/redactwriter/redactwriter.go diff --git a/README.md b/README.md index 9c6732a0..b57c2b4b 100644 --- a/README.md +++ b/README.md @@ -137,6 +137,7 @@ This metadata will be saved with the individual files and restored by the [Pull | `addon_api_token` | The token required to authenticate with the API. | sensitive | `$ADDON_VDTESTING_API_TOKEN` | | `public_install_page_url_map_format` | Provide a language template description using [Golang templates](https://golang.org/pkg/text/template) so that the **Deploy to Bitrise.io** Step can build the required custom output. | required | `{{range $index, $element := .}}{{if $index}}\|{{end}}{{$element.File}}=>{{$element.URL}}{{end}}` | | `permanent_download_url_map_format` | Provide a language template description using [Golang templates](https://golang.org/pkg/text/template) so that the **Deploy to Bitrise.io** Step can build the required custom output for the permanent download URL. | required | `{{range $index, $element := .}}{{if $index}}\|{{end}}{{$element.File}}=>{{$element.URL}}{{end}}` | +| `files_to_redact` | A newline (`\n`) separated list of file paths to redact secrets from before the step deploys them. | | | | `debug_mode` | The Step will print more verbose logs if enabled. | required | `false` | diff --git a/bitrise.yml b/bitrise.yml index 037816c8..108c088d 100644 --- a/bitrise.yml +++ b/bitrise.yml @@ -17,7 +17,11 @@ app: workflows: test: steps: - - go-list: {} + - go-list: + inputs: + - exclude: |- + */vendor/* + */mocks - golint: {} - errcheck: {} - go-test: {} @@ -28,6 +32,7 @@ workflows: - _download_sample_artifacts - test after_run: + - _file_redaction - _zero_length_file_upload - _test_results_deployment steps: @@ -246,6 +251,48 @@ workflows: - deploy_path: 0_length_file.txt - debug_mode: "true" + _file_redaction: + steps: + - script: + is_always_run: true + inputs: + - content: |- + #!/bin/bash + set -ex + # Create empty temporary file + temp_file_path=$(mktemp) + # Add some content + echo "this is not a secret\nthis is a secret\nagain not a secret\n" > $temp_file_path + + # Export the file and the secret as env vars + envman add --key FILE_TO_REDACT --value "${temp_file_path}" + envman add --key SUPER_SECRET --value "this is a secret" --sensitive + - path::./: + title: File redaction test + run_if: true + inputs: + - build_url: $BITRISE_BUILD_URL + - build_api_token: $BITRISE_BUILD_API_TOKEN + - deploy_path: "" + - files_to_redact: $FILE_TO_REDACT + - debug_mode: "true" + - script: + is_always_run: true + inputs: + - content: |- + #!/bin/bash + set -ex + actual_content=$(<$FILE_TO_REDACT) + expected_content="this is not a secret\n[REDACTED]\nagain not a secret\n" + + # Fail if the content of the redacted file does not meat our expected value + if [[ "$actual_content" != "$expected_content" ]]; then + echo "$FILE_TO_REDACT was not redacted" + echo "Actual content: "$actual_content"" + echo "Expected content: "$expected_content"" + exit 1 + fi + generate_readme: steps: - git::https://github.com/bitrise-steplib/steps-readme-generator.git@main: diff --git a/fileredactor/filepathprocessor.go b/fileredactor/filepathprocessor.go new file mode 100644 index 00000000..113d395e --- /dev/null +++ b/fileredactor/filepathprocessor.go @@ -0,0 +1,64 @@ +package fileredactor + +import ( + "fmt" + "strings" + + "github.com/bitrise-io/go-utils/v2/pathutil" +) + +// FilePathProcessor is an interface for an entity which accepts file paths separated by the +// newline (`\n`) character and returns a slice of absolute paths. +type FilePathProcessor interface { + ProcessFilePaths(string) ([]string, error) +} + +type filePathProcessor struct { + pathModifier pathutil.PathModifier + pathChecker pathutil.PathChecker +} + +// NewFilePathProcessor returns a structure which implements the FilePathProcessor interface. +// The implementation includes handling filepaths defined as environment variables, relative file paths, +// and absolute file paths. +// The implementation also includes making sure the filepath exists and is not a directory. +func NewFilePathProcessor(modifier pathutil.PathModifier, checker pathutil.PathChecker) FilePathProcessor { + return filePathProcessor{ + pathModifier: modifier, + pathChecker: checker, + } +} + +func (f filePathProcessor) ProcessFilePaths(filePaths string) ([]string, error) { + filePaths = strings.TrimSpace(filePaths) + if filePaths == "" { + return nil, nil + } + + var processedFilePaths []string + + list := strings.Split(filePaths, "\n") + for _, item := range list { + item = strings.TrimSpace(item) + if item == "" { + continue + } + + path, err := f.pathModifier.AbsPath(item) + if err != nil { + return nil, err + } + + isDir, err := f.pathChecker.IsDirExists(path) + if err != nil { + return nil, fmt.Errorf("failed to check if path (%s) is a directory: %w", path, err) + } + if isDir { + return nil, fmt.Errorf("path (%s) is a directory and cannot be redacted, please make sure to only provide filepaths as inputs", path) + } + + processedFilePaths = append(processedFilePaths, path) + } + + return processedFilePaths, nil +} diff --git a/fileredactor/filepathprocessor_test.go b/fileredactor/filepathprocessor_test.go new file mode 100644 index 00000000..0d7616be --- /dev/null +++ b/fileredactor/filepathprocessor_test.go @@ -0,0 +1,67 @@ +package fileredactor + +import ( + "reflect" + "testing" + + "github.com/bitrise-steplib/steps-deploy-to-bitrise-io/mocks" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" +) + +func Test_ProcessFilePaths(t *testing.T) { + deployDirPath := "/some/absolute/path/deploy_dir" + tests := []struct { + name string + input string + output []string + outputErr string + envs map[string]string + }{ + { + name: "Empty input", + input: " ", + output: nil, + outputErr: "", + envs: nil, + }, + { + name: "Expand relative path", + input: ` +/some/absolute/path/to/file.txt +file_in_deploy_dir.txt +`, + output: []string{ + "/some/absolute/path/to/file.txt", + "/some/absolute/path/deploy_dir/file_in_deploy_dir.txt", + }, + outputErr: "", + envs: nil, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + mockModifier := new(mocks.PathModifier) + mockModifier.On("AbsPath", "a_file.txt").Return(deployDirPath+"/a_file.txt", nil) + mockModifier.On("AbsPath", "/some/absolute/path/to/file.txt").Return("/some/absolute/path/to/file.txt", nil) + mockModifier.On("AbsPath", "file_in_deploy_dir.txt").Return(deployDirPath+"/file_in_deploy_dir.txt", nil) + + mockChecker := new(mocks.PathChecker) + mockChecker.On("IsDirExists", mock.Anything).Return(false, nil) + + pathProcessor := NewFilePathProcessor(mockModifier, mockChecker) + result, err := pathProcessor.ProcessFilePaths(tt.input) + + if err != nil && tt.outputErr != "" { + assert.EqualError(t, err, tt.outputErr) + } else if err != nil { + t.Errorf("%s got = %v, want %v", t.Name(), err, tt.outputErr) + } + + if !reflect.DeepEqual(result, tt.output) { + t.Errorf("%s got = %v, want %v", t.Name(), result, tt.output) + } + }) + } +} diff --git a/fileredactor/fileredactor.go b/fileredactor/fileredactor.go new file mode 100644 index 00000000..b932d2cb --- /dev/null +++ b/fileredactor/fileredactor.go @@ -0,0 +1,81 @@ +package fileredactor + +import ( + "fmt" + "io" + "os" + + "github.com/bitrise-io/go-utils/v2/fileutil" + "github.com/bitrise-io/go-utils/v2/log" + "github.com/bitrise-io/go-utils/v2/redactwriter" +) + +const bufferSize = 64 * 1024 + +// FileRedactor is an interface for a structure which, given a slice of file paths and another slice of secrets can +// process the specified files to redact secrets from them. +type FileRedactor interface { + RedactFiles([]string, []string) error +} + +type fileRedactor struct { + fileManager fileutil.FileManager +} + +// NewFileRedactor returns a structure that implements the FileRedactor interface +func NewFileRedactor(manager fileutil.FileManager) FileRedactor { + return fileRedactor{ + fileManager: manager, + } +} + +func (f fileRedactor) RedactFiles(filePaths []string, secrets []string) error { + logger := log.NewLogger() + for _, path := range filePaths { + if err := f.redactFile(path, secrets, logger); err != nil { + return fmt.Errorf("failed to redact file (%s): %w", path, err) + } + } + + return nil +} + +func (f fileRedactor) redactFile(path string, secrets []string, logger log.Logger) error { + source, err := f.fileManager.Open(path) + if err != nil { + return fmt.Errorf("failed to open file for redaction (%s): %w", path, err) + } + defer func() { + if err := source.Close(); err != nil { + logger.Warnf("Failed to close file: %s", err) + } + }() + + newPath := path + ".redacted" + destination, err := os.Create(newPath) + if err != nil { + return fmt.Errorf("failed to create temporary file for redaction: %w", err) + } + defer func() { + if err := destination.Close(); err != nil { + logger.Warnf("Failed to close file: %s", err) + } + }() + + redactWriter := redactwriter.New(secrets, destination, logger) + if _, err := io.Copy(redactWriter, source); err != nil { + return fmt.Errorf("failed to redact secrets: %w", err) + } + + if err := redactWriter.Close(); err != nil { + return fmt.Errorf("failed to close redact writer: %w", err) + } + + //rename new file to old file name + err = os.Rename(newPath, path) + if err != nil { + return fmt.Errorf("failed to overwrite old file (%s) with redacted file: %w", path, err) + } + + return nil +} diff --git a/fileredactor/fileredactor_test.go b/fileredactor/fileredactor_test.go new file mode 100644 index 00000000..6c7cd8cb --- /dev/null +++ b/fileredactor/fileredactor_test.go @@ -0,0 +1,37 @@ +package fileredactor + +import ( + "os" + "path" + "testing" + + "github.com/bitrise-io/go-utils/v2/fileutil" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func Test_RedactFiles(t *testing.T) { + secrets := []string{ + "SUPER_SECRET_WORD", + "ANOTHER_SECRET_WORD", + } + filePath := path.Join(t.TempDir(), "step-output.txt") + content, err := os.ReadFile("testdata/before_redaction.txt") + require.NoError(t, err) + + fileManager := fileutil.NewFileManager() + err = fileManager.WriteBytes(filePath, content) + require.NoError(t, err) + + fileRedactor := NewFileRedactor(fileutil.NewFileManager()) + err = fileRedactor.RedactFiles([]string{filePath}, secrets) + require.NoError(t, err) + + got, err := os.ReadFile(filePath) + require.NoError(t, err) + + want, err := os.ReadFile("testdata/after_redaction.txt") + require.NoError(t, err) + + assert.Equal(t, want, got) +} diff --git a/fileredactor/testdata/after_redaction.txt b/fileredactor/testdata/after_redaction.txt new file mode 100644 index 00000000..6fa1d010 --- /dev/null +++ b/fileredactor/testdata/after_redaction.txt @@ -0,0 +1,8 @@ +[REDACTED] + +Normal output from any of our steps. +More output. + +[REDACTED] + +The end. diff --git a/fileredactor/testdata/before_redaction.txt b/fileredactor/testdata/before_redaction.txt new file mode 100644 index 00000000..ce4e29f3 --- /dev/null +++ b/fileredactor/testdata/before_redaction.txt @@ -0,0 +1,8 @@ +SUPER_SECRET_WORD + +Normal output from any of our steps. +More output. + +ANOTHER_SECRET_WORD + +The end. diff --git a/go.mod b/go.mod index 6a376631..b1ffe155 100644 --- a/go.mod +++ b/go.mod @@ -4,12 +4,13 @@ go 1.20 require ( github.com/avast/apkparser v0.0.0-20210301101811-6256c76f738e - github.com/bitrise-io/bitrise v0.0.0-20210519130014-380842fb41c1 - github.com/bitrise-io/envman v0.0.0-20210517135508-b2b4fe89eac5 + github.com/bitrise-io/bitrise v0.0.0-20220808135808-3483087dd853 + github.com/bitrise-io/envman v0.0.0-20220401145857-d11e00a5dc55 github.com/bitrise-io/go-android v0.0.0-20210517091621-72f0eb7a5197 github.com/bitrise-io/go-steputils v1.0.5 + github.com/bitrise-io/go-steputils/v2 v2.0.0-alpha.20 github.com/bitrise-io/go-utils v1.0.9 - github.com/bitrise-io/go-utils/v2 v2.0.0-alpha.15 + github.com/bitrise-io/go-utils/v2 v2.0.0-alpha.19 github.com/bitrise-io/go-xcode v1.0.16 github.com/google/go-cmp v0.5.9 github.com/gorilla/mux v1.8.0 @@ -20,8 +21,8 @@ require ( require ( github.com/bitrise-io/go-pkcs12 v0.0.0-20230815095624-feb898696e02 // indirect - github.com/bitrise-io/goinp v0.0.0-20210504152833-8559b0680ab1 // indirect - github.com/bitrise-io/stepman v0.0.0-20210517135458-203f7a48d37a // indirect + github.com/bitrise-io/goinp v0.0.0-20211005113137-305e91b481f4 // indirect + github.com/bitrise-io/stepman v0.0.0-20220808095634-6e12d2726f30 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/fullsailor/pkcs7 v0.0.0-20190404230743-d7302db945fa // indirect github.com/hashicorp/go-cleanhttp v0.5.2 // indirect diff --git a/go.sum b/go.sum index eda65b67..f4ae3360 100644 --- a/go.sum +++ b/go.sum @@ -11,12 +11,12 @@ github.com/avast/apkverifier v0.0.0-20200217135742-aa28c80b82ae/go.mod h1:SV58cy github.com/avast/apkverifier v0.0.0-20200416105355-97c5338f32f0/go.mod h1:HskRSJJJbP3poUkDRAyRAdDVSsh5J1mz8cRc2/B4kbc= github.com/avast/apkverifier v0.0.0-20210219091843-33631264c352/go.mod h1:uhY/I/3Vh3V6ZFgLm/EFX/j5//MdoXpvcULTtzRW3YA= github.com/avast/apkverifier v0.0.0-20210301101718-290c8f7fccf7/go.mod h1:APQFx11UQTdbLKlZVJQFddZcJZxoHl6NnJfHN7foLD8= -github.com/bitrise-io/bitrise v0.0.0-20210519130014-380842fb41c1 h1:PERqeGZZkMrGYSSN/Stk+YkhErgdk37aVqc5UgnIEoo= -github.com/bitrise-io/bitrise v0.0.0-20210519130014-380842fb41c1/go.mod h1:Jqf2PLwOKL1bYdbQIhVkNd55YyYKJBXyY0EtlRg5uw4= +github.com/bitrise-io/bitrise v0.0.0-20220808135808-3483087dd853 h1:VW9r3GyaDWUrSkrsvkEJKOVigCjhkld1k9XIoma5u7Q= +github.com/bitrise-io/bitrise v0.0.0-20220808135808-3483087dd853/go.mod h1:IHzkBy9xZQL5nyNzA3ryS7YrTYVEqoQLQvbyyKhRc94= github.com/bitrise-io/colorstring v0.0.0-20180614154802-a8cd70115192/go.mod h1:CIHVcxZUvsG99XUJV6JlR7okNsMMGY81jMvPC20W+O0= -github.com/bitrise-io/envman v0.0.0-20200512105748-919e33f391ee/go.mod h1:m8pTp1o3Sw9uzDxb1WRm5IBRnMau2iOvPMSnRCAhQNI= -github.com/bitrise-io/envman v0.0.0-20210517135508-b2b4fe89eac5 h1:3jVXjsejV+1EW9Sp7ef+sMinD0J3HFsPgh4Du6ctqZc= -github.com/bitrise-io/envman v0.0.0-20210517135508-b2b4fe89eac5/go.mod h1:m8pTp1o3Sw9uzDxb1WRm5IBRnMau2iOvPMSnRCAhQNI= +github.com/bitrise-io/envman v0.0.0-20211026063720-03283f9c3f32/go.mod h1:L4WQyg88d87Z4dxNwrYEa0Cwd9/W0gSfXsibw30r8Vw= +github.com/bitrise-io/envman v0.0.0-20220401145857-d11e00a5dc55 h1:uU2j30ndTLUdtkAw/RicxczXlOiSh2Ng/if9D/1xtp8= +github.com/bitrise-io/envman v0.0.0-20220401145857-d11e00a5dc55/go.mod h1:L4WQyg88d87Z4dxNwrYEa0Cwd9/W0gSfXsibw30r8Vw= github.com/bitrise-io/go-android v0.0.0-20210517091621-72f0eb7a5197 h1:rdMOrZ6f5T8evFF8o+lBj9p5dPiwm+t7yCwHBGD3qZI= github.com/bitrise-io/go-android v0.0.0-20210517091621-72f0eb7a5197/go.mod h1:vcNcrAM6B5gDdgDZ9n51FTLT/kTHmiWatjCkJcstCNY= github.com/bitrise-io/go-pkcs12 v0.0.0-20230815095624-feb898696e02 h1:DoXD85rP+di4sJplai0Fyvvt0HBK7umrqVHTGBnkaaQ= @@ -24,31 +24,36 @@ github.com/bitrise-io/go-pkcs12 v0.0.0-20230815095624-feb898696e02/go.mod h1:R3y github.com/bitrise-io/go-steputils v0.0.0-20210514150206-5b6261447e77/go.mod h1:H0iZjgsAR5NA6pnlD/zKB6AbxEsskq55pwJ9klVmP8w= github.com/bitrise-io/go-steputils v1.0.5 h1:OBH7CPXeqIWFWJw6BOUMQnUb8guspwKr2RhYBhM9tfc= github.com/bitrise-io/go-steputils v1.0.5/go.mod h1:YIUaQnIAyK4pCvQG0hYHVkSzKNT9uL2FWmkFNW4mfNI= -github.com/bitrise-io/go-utils v0.0.0-20210505091801-98b7dc39ee61/go.mod h1:nhdaDQFvaMny1CugVV6KjK92/q97ENo0RuKSW5I4fbA= +github.com/bitrise-io/go-steputils/v2 v2.0.0-alpha.20 h1:ac+HUrl4H+2qM/0FmP6mOJQvoxSPpO68er3tKNJoM6E= +github.com/bitrise-io/go-steputils/v2 v2.0.0-alpha.20/go.mod h1:YeLAqkNuns7tiZNAlGJ8kOWvSEZy3SOKAQCv2CeWk5A= +github.com/bitrise-io/go-utils v0.0.0-20200224122728-e212188d99b4/go.mod h1:tTEsKvbz1LbzuN/KpVFHXnLtcAPdEgIdM41s0lL407s= github.com/bitrise-io/go-utils v0.0.0-20210505121718-07411d72e36e/go.mod h1:nhdaDQFvaMny1CugVV6KjK92/q97ENo0RuKSW5I4fbA= github.com/bitrise-io/go-utils v0.0.0-20210507100250-37de47dfa6ce/go.mod h1:15EZZf02noI5nWFqXMZEoyb1CyqYRXTMz5Fyu4CWFzI= github.com/bitrise-io/go-utils v1.0.1/go.mod h1:ZY1DI+fEpZuFpO9szgDeICM4QbqoWVt0RSY3tRI1heY= +github.com/bitrise-io/go-utils v1.0.2/go.mod h1:ZY1DI+fEpZuFpO9szgDeICM4QbqoWVt0RSY3tRI1heY= github.com/bitrise-io/go-utils v1.0.9 h1:wy7FewUpseNSTZr41BbGH0csfFqzptFt4zy2pOAEOg0= github.com/bitrise-io/go-utils v1.0.9/go.mod h1:ZY1DI+fEpZuFpO9szgDeICM4QbqoWVt0RSY3tRI1heY= -github.com/bitrise-io/go-utils/v2 v2.0.0-alpha.15 h1:ERQb+OOa+eKMWb+HByyPd5ugz6TeaJPnQ3xHgyaR7hA= -github.com/bitrise-io/go-utils/v2 v2.0.0-alpha.15/go.mod h1:Laih4ji980SQkRgdnMCH0g4u2GZI/5nnbqmYT9UfKFQ= +github.com/bitrise-io/go-utils/v2 v2.0.0-alpha.10/go.mod h1:Ta/ards3Ih/3Q6X8tBtcj6zTHcNf1hRSXv1E8lPgIYk= +github.com/bitrise-io/go-utils/v2 v2.0.0-alpha.19 h1:55as5Iv0N4btuRP3YwRzN+BCMtKO210MnJ8mpxmeI7o= +github.com/bitrise-io/go-utils/v2 v2.0.0-alpha.19/go.mod h1:Laih4ji980SQkRgdnMCH0g4u2GZI/5nnbqmYT9UfKFQ= github.com/bitrise-io/go-xcode v1.0.16 h1:G1IItfD2dvPNm7MLIWXFQHNPcafMVnw83M1lqCUH5L4= github.com/bitrise-io/go-xcode v1.0.16/go.mod h1:9OwsvrhZ4A2JxHVoEY7CPcABAKA+OE7FQqFfBfvbFuY= -github.com/bitrise-io/goinp v0.0.0-20210504152833-8559b0680ab1 h1:iQZl/dKHp14xol7ye4e9FaSrw8o27TF5/nv3aUDfdD0= github.com/bitrise-io/goinp v0.0.0-20210504152833-8559b0680ab1/go.mod h1:iRbd8zAXLeNy+0gic0eqNCxXvDGe8ZEY/uYX2CCeAoo= -github.com/bitrise-io/gows v0.0.0-20210505125306-dd92ff463938/go.mod h1:3Cp9ceJ8wHl1Av6oEE2ff1iWaYLliQuD+oaNdyM0NWQ= -github.com/bitrise-io/stepman v0.0.0-20210505110307-5c2296bcc558/go.mod h1:WLh58JYBgbD1Z/yyw1AkFz/90F6oBL0HS/luBpUW9dI= -github.com/bitrise-io/stepman v0.0.0-20210517135458-203f7a48d37a h1:qsEFYqhVeo0mLrdl7nLTNbCz0AZ+8nOxQD+LPFw2Oq4= -github.com/bitrise-io/stepman v0.0.0-20210517135458-203f7a48d37a/go.mod h1:WLh58JYBgbD1Z/yyw1AkFz/90F6oBL0HS/luBpUW9dI= +github.com/bitrise-io/goinp v0.0.0-20211005113137-305e91b481f4 h1:ytUxnO7iSGHlNpbdjhDUefEM5WRy1kD2ElGfBA7r1PE= +github.com/bitrise-io/goinp v0.0.0-20211005113137-305e91b481f4/go.mod h1:iRbd8zAXLeNy+0gic0eqNCxXvDGe8ZEY/uYX2CCeAoo= +github.com/bitrise-io/gows v0.0.0-20211005113107-14f65e686b88/go.mod h1:3Cp9ceJ8wHl1Av6oEE2ff1iWaYLliQuD+oaNdyM0NWQ= +github.com/bitrise-io/stepman v0.0.0-20220808095634-6e12d2726f30 h1:6Ra9CW9GMBRpT1FfRabM77breQTEZ9AzD/WQiW330RM= +github.com/bitrise-io/stepman v0.0.0-20220808095634-6e12d2726f30/go.mod h1:25vk5IaQiOpXLMcjyJjY6RmZe6JEOqMe8TJoTvKgyuw= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= -github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= +github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/fullsailor/pkcs7 v0.0.0-20190404230743-d7302db945fa h1:RDBNVkRviHZtvDvId8XSGPu3rmpmSe+wKRcEWNgsfWU= github.com/fullsailor/pkcs7 v0.0.0-20190404230743-d7302db945fa/go.mod h1:KnogPXtdwXqoenmZCw6S+25EAm2MkxbG0deNDu4cbSA= +github.com/gofrs/uuid v4.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= @@ -63,15 +68,19 @@ github.com/hashicorp/go-retryablehttp v0.7.0/go.mod h1:vAew36LZh98gCBJNLH42IQ1ER github.com/hashicorp/go-retryablehttp v0.7.1 h1:sUiuQAnLlbvmExtFQs72iFW/HXeUn8Z1aJLQ4LJJbTQ= github.com/hashicorp/go-retryablehttp v0.7.1/go.mod h1:vAew36LZh98gCBJNLH42IQ1ER/9wtLZZ8meHqQvEYWY= github.com/hashicorp/go-version v1.3.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= +github.com/hashicorp/go-version v1.4.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/hashicorp/go-version v1.6.0 h1:feTTfFNnjP967rlCxM/I9g701jU+RN74YKx2mOkIeek= github.com/hashicorp/go-version v1.6.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= +github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/klauspost/compress v1.11.0/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/compress v1.11.7/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/compress v1.11.8/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/compress v1.12.2 h1:2KCfW3I9M7nSc5wOqXAlW2v2U6v+w6cbjvbfp+OykW8= github.com/klauspost/compress v1.12.2/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= +github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/mitchellh/mapstructure v1.4.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= @@ -81,6 +90,7 @@ github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQD github.com/ryanuber/go-glob v1.0.0 h1:iQh3xXAumdQ+4Ufa5b25cRpC5TYKlno6hsv6Cb3pkBk= github.com/ryanuber/go-glob v1.0.0/go.mod h1:807d1WSdnB0XRJzKNil9Om6lcp/3a0v4qIHxIXzX/Yc= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= +github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -96,19 +106,28 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/urfave/cli v1.21.0/go.mod h1:lxDj6qX9Q6lWQxIrbrT0nwecwUtRnhVZAJjJZrVUZZQ= github.com/urfave/cli v1.22.5/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/whilp/git-urls v1.0.0/go.mod h1:J16SAmobsqc3Qcy98brfl5f5+e0clUvg1krgwk/qCfE= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200220183623-bac4c82f6975/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210503195802-e9a32991a82e/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8= golang.org/x/crypto v0.0.0-20211202192323-5770296d904e/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.0.0-20220315160706-3147a52a75dd/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.12.0 h1:tFM/ta59kqch6LlvYnPa0yx5a83cL2nHflFhYKvv9Yk= golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98yw= +golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200219091948-cb0a6d8edb6c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -116,6 +135,8 @@ golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210503173754-0981d6026fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211205182925-97ca703d548d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220712014510-0a85c31ab51e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0 h1:eG7RXZHdqOJ1i+0lgLgCpSXAp6M3LYlAo6osgSi0xOM= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= @@ -127,6 +148,10 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200220224806-8a925fa4c0df/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/main.go b/main.go index bf5155da..dff69853 100644 --- a/main.go +++ b/main.go @@ -8,18 +8,21 @@ import ( "path/filepath" "strings" - "github.com/bitrise-io/go-utils/v2/env" - "github.com/bitrise-io/bitrise/models" - "github.com/bitrise-io/envman/envman" "github.com/bitrise-io/go-steputils/stepconf" "github.com/bitrise-io/go-steputils/tools" + "github.com/bitrise-io/go-steputils/v2/secretkeys" "github.com/bitrise-io/go-utils/log" "github.com/bitrise-io/go-utils/pathutil" + "github.com/bitrise-io/go-utils/v2/env" "github.com/bitrise-io/go-utils/v2/errorutil" + "github.com/bitrise-io/go-utils/v2/exitcode" + "github.com/bitrise-io/go-utils/v2/fileutil" + pathutil2 "github.com/bitrise-io/go-utils/v2/pathutil" "github.com/bitrise-io/go-utils/ziputil" "github.com/bitrise-steplib/steps-deploy-to-bitrise-io/deployment" + "github.com/bitrise-steplib/steps-deploy-to-bitrise-io/fileredactor" "github.com/bitrise-steplib/steps-deploy-to-bitrise-io/test" "github.com/bitrise-steplib/steps-deploy-to-bitrise-io/uploaders" ) @@ -44,6 +47,7 @@ type Config struct { AppSlug string `env:"BITRISE_APP_SLUG,required"` AddonAPIBaseURL string `env:"addon_api_base_url,required"` AddonAPIToken string `env:"addon_api_token"` + FilesToRedact string `env:"files_to_redact"` DebugMode bool `env:"debug_mode,opt[true,false]"` BundletoolVersion string `env:"bundletool_version,required"` } @@ -85,6 +89,36 @@ func main() { fail("Failed to create tmp dir, error: %s", err) } + fmt.Println() + log.Infof("Collecting files to redact...") + + pathModifier := pathutil2.NewPathModifier() + pathChecker := pathutil2.NewPathChecker() + pathProcessor := fileredactor.NewFilePathProcessor(pathModifier, pathChecker) + filePaths, err := pathProcessor.ProcessFilePaths(config.FilesToRedact) + if err != nil { + log.Errorf(errorutil.FormattedError(fmt.Errorf("failed to collect file paths to redact: %w", err))) + os.Exit(int(exitcode.Failure)) + } + + if len(filePaths) > 0 { + log.Printf("List of files to redact:") + for _, path := range filePaths { + log.Printf("- %s", path) + } + + fileManager := fileutil.NewFileManager() + redactor := fileredactor.NewFileRedactor(fileManager) + secrets := loadSecrets() + err = redactor.RedactFiles(filePaths, secrets) + if err != nil { + log.Errorf(errorutil.FormattedError(fmt.Errorf("failed to redact files: %w", err))) + os.Exit(int(exitcode.Failure)) + } + } else { + log.Printf("No files to redact...") + } + fmt.Println() log.Infof("Collecting files to deploy...") @@ -105,7 +139,8 @@ func main() { if strings.TrimSpace(config.PipelineIntermediateFiles) != "" { zipComparator := deployment.NewZipComparator(deployment.DefaultReadZipFunction) - collector := deployment.NewCollector(zipComparator, deployment.DefaultIsDirFunction, ziputil.ZipDir, env.NewRepository(), tmpDir) + repository := env.NewRepository() + collector := deployment.NewCollector(zipComparator, deployment.DefaultIsDirFunction, ziputil.ZipDir, repository, tmpDir) deployableItems, err = collector.AddIntermediateFiles(deployableItems, config.PipelineIntermediateFiles) if err != nil { fail("%s", err) @@ -139,6 +174,20 @@ func main() { } } +func loadSecrets() []string { + envRepository := env.NewRepository() + keys := secretkeys.NewManager().Load(envRepository) + + var values []string + for _, key := range keys { + value := envRepository.Get(key) + if value != "" { + values = append(values, value) + } + } + return values +} + func exportInstallPages(artifactURLCollection ArtifactURLCollection, config Config) error { if len(artifactURLCollection.PublicInstallPageURLs) > 0 { pages := mapURLsToInstallPages(artifactURLCollection.PublicInstallPageURLs) diff --git a/mocks/PathChecker.go b/mocks/PathChecker.go new file mode 100644 index 00000000..2d8ea057 --- /dev/null +++ b/mocks/PathChecker.go @@ -0,0 +1,72 @@ +// Code generated by mockery v2.33.2. DO NOT EDIT. + +package mocks + +import mock "github.com/stretchr/testify/mock" + +// PathChecker is an autogenerated mock type for the PathChecker type +type PathChecker struct { + mock.Mock +} + +// IsDirExists provides a mock function with given fields: pth +func (_m *PathChecker) IsDirExists(pth string) (bool, error) { + ret := _m.Called(pth) + + var r0 bool + var r1 error + if rf, ok := ret.Get(0).(func(string) (bool, error)); ok { + return rf(pth) + } + if rf, ok := ret.Get(0).(func(string) bool); ok { + r0 = rf(pth) + } else { + r0 = ret.Get(0).(bool) + } + + if rf, ok := ret.Get(1).(func(string) error); ok { + r1 = rf(pth) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// IsPathExists provides a mock function with given fields: pth +func (_m *PathChecker) IsPathExists(pth string) (bool, error) { + ret := _m.Called(pth) + + var r0 bool + var r1 error + if rf, ok := ret.Get(0).(func(string) (bool, error)); ok { + return rf(pth) + } + if rf, ok := ret.Get(0).(func(string) bool); ok { + r0 = rf(pth) + } else { + r0 = ret.Get(0).(bool) + } + + if rf, ok := ret.Get(1).(func(string) error); ok { + r1 = rf(pth) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// NewPathChecker creates a new instance of PathChecker. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewPathChecker(t interface { + mock.TestingT + Cleanup(func()) +}) *PathChecker { + mock := &PathChecker{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/mocks/PathModifier.go b/mocks/PathModifier.go new file mode 100644 index 00000000..fd640faf --- /dev/null +++ b/mocks/PathModifier.go @@ -0,0 +1,48 @@ +// Code generated by mockery v2.33.2. DO NOT EDIT. + +package mocks + +import mock "github.com/stretchr/testify/mock" + +// PathModifier is an autogenerated mock type for the PathModifier type +type PathModifier struct { + mock.Mock +} + +// AbsPath provides a mock function with given fields: pth +func (_m *PathModifier) AbsPath(pth string) (string, error) { + ret := _m.Called(pth) + + var r0 string + var r1 error + if rf, ok := ret.Get(0).(func(string) (string, error)); ok { + return rf(pth) + } + if rf, ok := ret.Get(0).(func(string) string); ok { + r0 = rf(pth) + } else { + r0 = ret.Get(0).(string) + } + + if rf, ok := ret.Get(1).(func(string) error); ok { + r1 = rf(pth) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// NewPathModifier creates a new instance of PathModifier. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewPathModifier(t interface { + mock.TestingT + Cleanup(func()) +}) *PathModifier { + mock := &PathModifier{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/mocks/Repository.go b/mocks/Repository.go index 5603bf59..7de27af5 100644 --- a/mocks/Repository.go +++ b/mocks/Repository.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.10.4. DO NOT EDIT. +// Code generated by mockery v2.33.2. DO NOT EDIT. package mocks @@ -17,10 +17,7 @@ func (_m *Repository) Get(key string) string { if rf, ok := ret.Get(0).(func(string) string); ok { r0 = rf(key) } else { - r0, ok = ret.Get(0).(string) - if !ok { - - } + r0 = ret.Get(0).(string) } return r0 @@ -35,10 +32,7 @@ func (_m *Repository) List() []string { r0 = rf() } else { if ret.Get(0) != nil { - r0, ok = ret.Get(0).([]string) - if !ok { - - } + r0 = ret.Get(0).([]string) } } @@ -72,3 +66,17 @@ func (_m *Repository) Unset(key string) error { return r0 } + +// NewRepository creates a new instance of Repository. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewRepository(t interface { + mock.TestingT + Cleanup(func()) +}) *Repository { + mock := &Repository{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/step.yml b/step.yml index 27393fae..2aec3957 100644 --- a/step.yml +++ b/step.yml @@ -274,6 +274,23 @@ inputs: so that the **Deploy to Bitrise.io** Step can build the required custom output for the permanent download URL. is_required: true is_expand: false + - files_to_redact: + opts: + title: Files to redact + description: |- + A newline (`\n`) separated list of file paths to redact secrets from before the step deploys them. + summary: |- + A newline (`\n`) separated list of file paths to redact secrets from before the step deploys them. + + The file path can be specified with environment variables or direct paths: + ``` + $XCODE_BUILD_RAW_RESULT_TEXT_PATH + ./path/to/build_log.xml + ``` + + The step will fail (and potentially also fail the build) if there is any error during the redaction process. + This will prevent secrets in files accidentally leaking out of builds. + is_required: false - debug_mode: "false" opts: category: Debugging diff --git a/vendor/github.com/bitrise-io/bitrise/models/models.go b/vendor/github.com/bitrise-io/bitrise/models/models.go index b222e2f0..47a3a52f 100644 --- a/vendor/github.com/bitrise-io/bitrise/models/models.go +++ b/vendor/github.com/bitrise-io/bitrise/models/models.go @@ -8,19 +8,19 @@ import ( ) const ( - // StepRunStatusCodeSuccess ... - StepRunStatusCodeSuccess = 0 - // StepRunStatusCodeFailed ... - StepRunStatusCodeFailed = 1 - // StepRunStatusCodeFailedSkippable ... - StepRunStatusCodeFailedSkippable = 2 - // StepRunStatusCodeSkipped ... - StepRunStatusCodeSkipped = 3 - // StepRunStatusCodeSkippedWithRunIf ... - StepRunStatusCodeSkippedWithRunIf = 4 + StepRunStatusCodeSuccess = 0 + StepRunStatusCodeFailed = 1 + StepRunStatusCodeFailedSkippable = 2 + StepRunStatusCodeSkipped = 3 + StepRunStatusCodeSkippedWithRunIf = 4 + StepRunStatusCodePreparationFailed = 5 + // StepRunStatusAbortedTimeout is used when a step times out due to a custom timeout + StepRunStatusAbortedTimeout = 7 + // StepRunStatusAbortedNoOutputTimeout is used when a step times out due to no output received (hang) + StepRunStatusAbortedNoOutputTimeout = 8 // Version ... - Version = "11" + Version = "12" ) // StepListItemModel ... diff --git a/vendor/github.com/bitrise-io/bitrise/models/models_methods.go b/vendor/github.com/bitrise-io/bitrise/models/models_methods.go index c04ccc1e..69330a5e 100644 --- a/vendor/github.com/bitrise-io/bitrise/models/models_methods.go +++ b/vendor/github.com/bitrise-io/bitrise/models/models_methods.go @@ -944,7 +944,7 @@ func MergeStepWith(step, otherStep stepmanModels.StepModel) (stepmanModels.StepM step.Toolkit = new(stepmanModels.StepToolkitModel) *step.Toolkit = *otherStep.Toolkit } - if otherStep.Deps != nil && (len(otherStep.Deps.Brew) > 0 || len(otherStep.Deps.AptGet) > 0 || len(otherStep.Deps.CheckOnly) > 0) { + if otherStep.Deps != nil && (len(otherStep.Deps.Brew) > 0 || len(otherStep.Deps.AptGet) > 0) { step.Deps = otherStep.Deps } if otherStep.IsRequiresAdminUser != nil { @@ -963,6 +963,9 @@ func MergeStepWith(step, otherStep stepmanModels.StepModel) (stepmanModels.StepM if otherStep.Timeout != nil { step.Timeout = pointers.NewIntPtr(*otherStep.Timeout) } + if otherStep.NoOutputTimeout != nil { + step.NoOutputTimeout = pointers.NewIntPtr(*otherStep.NoOutputTimeout) + } for _, input := range step.Inputs { key, _, err := input.GetKeyValuePair() diff --git a/vendor/github.com/bitrise-io/envman/envman/configs.go b/vendor/github.com/bitrise-io/envman/envman/configs.go index 8c09c803..87f8a8c4 100644 --- a/vendor/github.com/bitrise-io/envman/envman/configs.go +++ b/vendor/github.com/bitrise-io/envman/envman/configs.go @@ -11,8 +11,8 @@ import ( const ( envmanConfigFileName = "configs.json" - defaultEnvBytesLimitInKB = 20 - defaultEnvListBytesLimitInKB = 100 + defaultEnvBytesLimitInKB = 256 + defaultEnvListBytesLimitInKB = 256 ) // ConfigsModel ... diff --git a/vendor/github.com/bitrise-io/envman/envman/util.go b/vendor/github.com/bitrise-io/envman/envman/util.go index f3e356a3..e99463ad 100644 --- a/vendor/github.com/bitrise-io/envman/envman/util.go +++ b/vendor/github.com/bitrise-io/envman/envman/util.go @@ -98,6 +98,9 @@ func removeDefaults(env *models.EnvironmentItemModel) error { if opts.Description != nil && *opts.Description == "" { opts.Description = nil } + if opts.Category != nil && *opts.Category == "" { + opts.Category = nil + } if opts.Summary != nil && *opts.Summary == "" { opts.Summary = nil } diff --git a/vendor/github.com/bitrise-io/envman/models/models_methods.go b/vendor/github.com/bitrise-io/envman/models/models_methods.go index 000d3210..833b59cf 100644 --- a/vendor/github.com/bitrise-io/envman/models/models_methods.go +++ b/vendor/github.com/bitrise-io/envman/models/models_methods.go @@ -23,7 +23,7 @@ const ( // DefaultSkipIfEmpty ... DefaultSkipIfEmpty = false - //DefaultIsRequired ... + // DefaultIsRequired ... DefaultIsRequired = false // DefaultIsDontChangeValue ... DefaultIsDontChangeValue = false @@ -42,11 +42,11 @@ func NewEnvJSONList(jsonStr string) (EnvsJSONListModel, error) { return list, nil } -// GetKeyValuePair ... -func (env EnvironmentItemModel) GetKeyValuePair() (string, string, error) { +// GetKeyValuePairWithType ... +func (env EnvironmentItemModel) GetKeyValuePairWithType() (string, interface{}, error) { // Collect keys and values - keys := []string{} - values := []interface{}{} + var keys []string + var values []interface{} for key, value := range env { keys = append(keys, key) @@ -84,6 +84,16 @@ func (env EnvironmentItemModel) GetKeyValuePair() (string, string, error) { return "", "", fmt.Errorf("more than 1 environment key specified: %v", keys) } + return key, value, nil +} + +// GetKeyValuePair ... +func (env EnvironmentItemModel) GetKeyValuePair() (string, string, error) { + key, value, err := env.GetKeyValuePairWithType() + if err != nil { + return "", "", err + } + // Cast env value to string valueStr := "" @@ -312,13 +322,10 @@ func (env *EnvironmentItemModel) FillMissingDefaults() error { // Validate ... func (env EnvironmentItemModel) Validate() error { - key, _, err := env.GetKeyValuePair() + _, _, err := env.GetKeyValuePair() if err != nil { return err } - if key == "" { - return errors.New("no environment key found") - } _, err = env.GetOptions() if err != nil { return err diff --git a/vendor/github.com/bitrise-io/go-steputils/v2/secretkeys/secretkeys.go b/vendor/github.com/bitrise-io/go-steputils/v2/secretkeys/secretkeys.go new file mode 100644 index 00000000..aa70812d --- /dev/null +++ b/vendor/github.com/bitrise-io/go-steputils/v2/secretkeys/secretkeys.go @@ -0,0 +1,39 @@ +package secretkeys + +import ( + "strings" + + "github.com/bitrise-io/go-utils/v2/env" +) + +const ( + // EnvKey ... + EnvKey = "BITRISE_SECRET_ENV_KEY_LIST" + separator = "," +) + +// Manager ... +type Manager interface { + Load(envRepository env.Repository) []string + Format(keys []string) string +} + +type manager struct { +} + +// NewManager ... +func NewManager() Manager { + return manager{} +} + +// Load ... +func (manager) Load(envRepository env.Repository) []string { + value := envRepository.Get(EnvKey) + keys := strings.Split(value, separator) + return keys +} + +// Format ... +func (manager) Format(keys []string) string { + return strings.Join(keys, separator) +} diff --git a/vendor/github.com/bitrise-io/go-utils/v2/exitcode/exitcode.go b/vendor/github.com/bitrise-io/go-utils/v2/exitcode/exitcode.go new file mode 100644 index 00000000..e2cc00c3 --- /dev/null +++ b/vendor/github.com/bitrise-io/go-utils/v2/exitcode/exitcode.go @@ -0,0 +1,15 @@ +//go:generate stringer -type=ExitCode + +package exitcode + +// ExitCode is a simple integer type that represents an exit code. +// It can be used to provide a more semantically meaningful exit code than a simple integer. +type ExitCode int + +const ( + // Success indicates that the program exited successfully. + Success ExitCode = 0 + + // Failure indicates that the program exited unsuccessfully. + Failure ExitCode = 1 +) diff --git a/vendor/github.com/bitrise-io/go-utils/v2/exitcode/exitcode_string.go b/vendor/github.com/bitrise-io/go-utils/v2/exitcode/exitcode_string.go new file mode 100644 index 00000000..063d6f24 --- /dev/null +++ b/vendor/github.com/bitrise-io/go-utils/v2/exitcode/exitcode_string.go @@ -0,0 +1,24 @@ +// Code generated by "stringer -type=ExitCode"; DO NOT EDIT. + +package exitcode + +import "strconv" + +func _() { + // An "invalid array index" compiler error signifies that the constant values have changed. + // Re-run the stringer command to generate them again. + var x [1]struct{} + _ = x[Success-0] + _ = x[Failure-1] +} + +const _ExitCode_name = "SuccessFailure" + +var _ExitCode_index = [...]uint8{0, 7, 14} + +func (i ExitCode) String() string { + if i < 0 || i >= ExitCode(len(_ExitCode_index)-1) { + return "ExitCode(" + strconv.FormatInt(int64(i), 10) + ")" + } + return _ExitCode_name[_ExitCode_index[i]:_ExitCode_index[i+1]] +} diff --git a/vendor/github.com/bitrise-io/go-utils/v2/fileutil/fileutil.go b/vendor/github.com/bitrise-io/go-utils/v2/fileutil/fileutil.go new file mode 100644 index 00000000..b2b9ba86 --- /dev/null +++ b/vendor/github.com/bitrise-io/go-utils/v2/fileutil/fileutil.go @@ -0,0 +1,91 @@ +package fileutil + +import ( + "errors" + "io" + "io/fs" + "os" + "path/filepath" +) + +// FileManager ... +type FileManager interface { + Open(path string) (*os.File, error) + OpenReaderIfExists(path string) (io.Reader, error) + ReadDirEntryNames(path string) ([]string, error) + Remove(path string) error + RemoveAll(path string) error + Write(path string, value string, perm os.FileMode) error + WriteBytes(path string, value []byte) error +} + +type fileManager struct { +} + +// NewFileManager ... +func NewFileManager() FileManager { + return fileManager{} +} + +// ReadDirEntryNames reads the named directory using os.ReadDir and returns the dir entries' names. +func (fileManager) ReadDirEntryNames(path string) ([]string, error) { + entries, err := os.ReadDir(path) + if err != nil { + return nil, err + } + var names []string + for _, entry := range entries { + names = append(names, entry.Name()) + } + return names, nil +} + +// Open ... +func (fileManager) Open(path string) (*os.File, error) { + return os.Open(path) +} + +// OpenReaderIfExists opens the named file using os.Open and returns an io.Reader. +// An ErrNotExist error is absorbed and the returned io.Reader will be nil, +// other errors from os.Open are returned as is. +func (fileManager) OpenReaderIfExists(path string) (io.Reader, error) { + file, err := os.Open(path) + if errors.Is(err, fs.ErrNotExist) { + return nil, nil + } + if err != nil { + return nil, err + } + return file, nil +} + +// Remove ... +func (fileManager) Remove(path string) error { + return os.Remove(path) +} + +// RemoveAll ... +func (fileManager) RemoveAll(path string) error { + return os.RemoveAll(path) +} + +// Write ... +func (f fileManager) Write(path string, value string, mode os.FileMode) error { + if err := f.ensureSavePath(path); err != nil { + return err + } + if err := os.WriteFile(path, []byte(value), mode); err != nil { + return err + } + return os.Chmod(path, mode) +} + +func (fileManager) ensureSavePath(savePath string) error { + dirPath := filepath.Dir(savePath) + return os.MkdirAll(dirPath, 0700) +} + +// WriteBytes ... +func (f fileManager) WriteBytes(path string, value []byte) error { + return os.WriteFile(path, value, 0600) +} diff --git a/vendor/github.com/bitrise-io/go-utils/v2/log/colorstring/colorstring.go b/vendor/github.com/bitrise-io/go-utils/v2/log/colorstring/colorstring.go new file mode 100644 index 00000000..5401b91d --- /dev/null +++ b/vendor/github.com/bitrise-io/go-utils/v2/log/colorstring/colorstring.go @@ -0,0 +1,111 @@ +package colorstring + +// ANSI color escape sequences + +import ( + "fmt" +) + +// Color ... +type Color string + +const ( + blackColor Color = "\x1b[30;1m" + redColor Color = "\x1b[31;1m" + greenColor Color = "\x1b[32;1m" + yellowColor Color = "\x1b[33;1m" + blueColor Color = "\x1b[34;1m" + magentaColor Color = "\x1b[35;1m" + cyanColor Color = "\x1b[36;1m" + resetColor Color = "\x1b[0m" +) + +// ColorFunc ... +type ColorFunc func(a ...interface{}) string + +func addColor(color Color, msg string) string { + return string(color) + msg + string(resetColor) +} + +// NoColor ... +func NoColor(a ...interface{}) string { + return fmt.Sprint(a...) +} + +// Black ... +func Black(a ...interface{}) string { + return addColor(blackColor, fmt.Sprint(a...)) +} + +// Red ... +func Red(a ...interface{}) string { + return addColor(redColor, fmt.Sprint(a...)) +} + +// Green ... +func Green(a ...interface{}) string { + return addColor(greenColor, fmt.Sprint(a...)) +} + +// Yellow ... +func Yellow(a ...interface{}) string { + return addColor(yellowColor, fmt.Sprint(a...)) +} + +// Blue ... +func Blue(a ...interface{}) string { + return addColor(blueColor, fmt.Sprint(a...)) +} + +// Magenta ... +func Magenta(a ...interface{}) string { + return addColor(magentaColor, fmt.Sprint(a...)) +} + +// Cyan ... +func Cyan(a ...interface{}) string { + return addColor(cyanColor, fmt.Sprint(a...)) +} + +// ColorfFunc ... +type ColorfFunc func(format string, a ...interface{}) string + +// NoColorf ... +func NoColorf(format string, a ...interface{}) string { + return NoColor(fmt.Sprintf(format, a...)) +} + +// Blackf ... +func Blackf(format string, a ...interface{}) string { + return Black(fmt.Sprintf(format, a...)) +} + +// Redf ... +func Redf(format string, a ...interface{}) string { + return Red(fmt.Sprintf(format, a...)) +} + +// Greenf ... +func Greenf(format string, a ...interface{}) string { + return Green(fmt.Sprintf(format, a...)) +} + +// Yellowf ... +func Yellowf(format string, a ...interface{}) string { + return Yellow(fmt.Sprintf(format, a...)) +} + +// Bluef ... +func Bluef(format string, a ...interface{}) string { + return Blue(fmt.Sprintf(format, a...)) +} + +// Magentaf ... +func Magentaf(format string, a ...interface{}) string { + return Magenta(fmt.Sprintf(format, a...)) +} + +// Cyanf ... +func Cyanf(format string, a ...interface{}) string { + return Cyan(fmt.Sprintf(format, a...)) +} diff --git a/vendor/github.com/bitrise-io/go-utils/v2/log/log.go b/vendor/github.com/bitrise-io/go-utils/v2/log/log.go new file mode 100644 index 00000000..0c77db95 --- /dev/null +++ b/vendor/github.com/bitrise-io/go-utils/v2/log/log.go @@ -0,0 +1,139 @@ +package log + +import ( + "fmt" + "io" + "os" + "time" +) + +// Logger ... +type Logger interface { + Infof(format string, v ...interface{}) + Warnf(format string, v ...interface{}) + Printf(format string, v ...interface{}) + Donef(format string, v ...interface{}) + Debugf(format string, v ...interface{}) + Errorf(format string, v ...interface{}) + TInfof(format string, v ...interface{}) + TWarnf(format string, v ...interface{}) + TPrintf(format string, v ...interface{}) + TDonef(format string, v ...interface{}) + TDebugf(format string, v ...interface{}) + TErrorf(format string, v ...interface{}) + Println() + EnableDebugLog(enable bool) +} + +const defaultTimeStampLayout = "15:04:05" + +type logger struct { + enableDebugLog bool + timestampLayout string + stdout io.Writer +} + +// NewLogger ... +func NewLogger() Logger { + return &logger{enableDebugLog: false, timestampLayout: defaultTimeStampLayout, stdout: os.Stdout} +} + +// EnableDebugLog ... +func (l *logger) EnableDebugLog(enable bool) { + l.enableDebugLog = enable +} + +// Infof ... +func (l *logger) Infof(format string, v ...interface{}) { + l.printf(infoSeverity, false, format, v...) +} + +// Warnf ... +func (l *logger) Warnf(format string, v ...interface{}) { + l.printf(warnSeverity, false, format, v...) +} + +// Printf ... +func (l *logger) Printf(format string, v ...interface{}) { + l.printf(normalSeverity, false, format, v...) +} + +// Donef ... +func (l *logger) Donef(format string, v ...interface{}) { + l.printf(doneSeverity, false, format, v...) +} + +// Debugf ... +func (l *logger) Debugf(format string, v ...interface{}) { + if l.enableDebugLog { + l.printf(debugSeverity, false, format, v...) + } +} + +// Errorf ... +func (l *logger) Errorf(format string, v ...interface{}) { + l.printf(errorSeverity, false, format, v...) +} + +// TInfof ... +func (l *logger) TInfof(format string, v ...interface{}) { + l.printf(infoSeverity, true, format, v...) +} + +// TWarnf ... +func (l *logger) TWarnf(format string, v ...interface{}) { + l.printf(warnSeverity, true, format, v...) +} + +// TPrintf ... +func (l *logger) TPrintf(format string, v ...interface{}) { + l.printf(normalSeverity, true, format, v...) +} + +// TDonef ... +func (l *logger) TDonef(format string, v ...interface{}) { + l.printf(doneSeverity, true, format, v...) +} + +// TDebugf ... +func (l *logger) TDebugf(format string, v ...interface{}) { + if l.enableDebugLog { + l.printf(debugSeverity, true, format, v...) + } +} + +// TErrorf ... +func (l *logger) TErrorf(format string, v ...interface{}) { + l.printf(errorSeverity, true, format, v...) +} + +// Println ... +func (l *logger) Println() { + fmt.Println() +} + +func (l *logger) timestampField() string { + currentTime := time.Now() + return fmt.Sprintf("[%s]", currentTime.Format(l.timestampLayout)) +} + +func (l *logger) prefixCurrentTime(message string) string { + return fmt.Sprintf("%s %s", l.timestampField(), message) +} + +func (l *logger) createLogMsg(severity Severity, withTime bool, format string, v ...interface{}) string { + colorFunc := severityColorFuncMap[severity] + message := colorFunc(format, v...) + if withTime { + message = l.prefixCurrentTime(message) + } + + return message +} + +func (l *logger) printf(severity Severity, withTime bool, format string, v ...interface{}) { + message := l.createLogMsg(severity, withTime, format, v...) + if _, err := fmt.Fprintln(l.stdout, message); err != nil { + fmt.Printf("failed to print message: %s, error: %s\n", message, err) + } +} diff --git a/vendor/github.com/bitrise-io/go-utils/v2/log/severity.go b/vendor/github.com/bitrise-io/go-utils/v2/log/severity.go new file mode 100644 index 00000000..02a14316 --- /dev/null +++ b/vendor/github.com/bitrise-io/go-utils/v2/log/severity.go @@ -0,0 +1,35 @@ +package log + +import "github.com/bitrise-io/go-utils/v2/log/colorstring" + +// Severity ... +type Severity uint8 + +const ( + errorSeverity Severity = iota + warnSeverity + normalSeverity + infoSeverity + doneSeverity + debugSeverity +) + +type severityColorFunc colorstring.ColorfFunc + +var ( + doneSeverityColorFunc severityColorFunc = colorstring.Greenf + infoSeverityColorFunc severityColorFunc = colorstring.Bluef + normalSeverityColorFunc severityColorFunc = colorstring.NoColorf + debugSeverityColorFunc severityColorFunc = colorstring.Magentaf + warnSeverityColorFunc severityColorFunc = colorstring.Yellowf + errorSeverityColorFunc severityColorFunc = colorstring.Redf +) + +var severityColorFuncMap = map[Severity]severityColorFunc{ + doneSeverity: doneSeverityColorFunc, + infoSeverity: infoSeverityColorFunc, + normalSeverity: normalSeverityColorFunc, + debugSeverity: debugSeverityColorFunc, + warnSeverity: warnSeverityColorFunc, + errorSeverity: errorSeverityColorFunc, +} diff --git a/vendor/github.com/bitrise-io/go-utils/v2/pathutil/pathutil.go b/vendor/github.com/bitrise-io/go-utils/v2/pathutil/pathutil.go new file mode 100644 index 00000000..f173693d --- /dev/null +++ b/vendor/github.com/bitrise-io/go-utils/v2/pathutil/pathutil.go @@ -0,0 +1,127 @@ +package pathutil + +import ( + "errors" + "os" + "os/user" + "path/filepath" + "strings" +) + +// PathProvider ... +type PathProvider interface { + CreateTempDir(prefix string) (string, error) +} + +type pathProvider struct{} + +// NewPathProvider ... +func NewPathProvider() PathProvider { + return pathProvider{} +} + +// CreateTempDir creates a temp dir, and returns its path. +// If prefix is provided it'll be used as the tmp dir's name prefix. +// Normalized: it's guaranteed that the path won't end with '/'. +func (pathProvider) CreateTempDir(prefix string) (dir string, err error) { + dir, err = os.MkdirTemp("", prefix) + dir = strings.TrimSuffix(dir, "/") + + return +} + +// PathChecker ... +type PathChecker interface { + IsPathExists(pth string) (bool, error) + IsDirExists(pth string) (bool, error) +} + +type pathChecker struct{} + +// NewPathChecker ... +func NewPathChecker() PathChecker { + return pathChecker{} +} + +// IsPathExists ... +func (c pathChecker) IsPathExists(pth string) (bool, error) { + _, isExists, err := c.genericIsPathExists(pth) + return isExists, err +} + +// IsDirExists ... +func (c pathChecker) IsDirExists(pth string) (bool, error) { + info, isExists, err := c.genericIsPathExists(pth) + return isExists && info.IsDir(), err +} + +func (pathChecker) genericIsPathExists(pth string) (os.FileInfo, bool, error) { + if pth == "" { + return nil, false, errors.New("no path provided") + } + + fileInf, err := os.Lstat(pth) + if err == nil { + return fileInf, true, nil + } + + if os.IsNotExist(err) { + return nil, false, nil + } + + return fileInf, false, err +} + +// PathModifier ... +type PathModifier interface { + AbsPath(pth string) (string, error) +} + +type pathModifier struct{} + +// NewPathModifier ... +func NewPathModifier() PathModifier { + return pathModifier{} +} + +// AbsPath expands ENV vars and the ~ character then calls Go's Abs +func (p pathModifier) AbsPath(pth string) (string, error) { + if pth == "" { + return "", errors.New("No Path provided") + } + + pth, err := p.expandTilde(pth) + if err != nil { + return "", err + } + + return filepath.Abs(os.ExpandEnv(pth)) +} + +func (pathModifier) expandTilde(pth string) (string, error) { + if pth == "" { + return "", errors.New("No Path provided") + } + + if strings.HasPrefix(pth, "~") { + pth = strings.TrimPrefix(pth, "~") + + if len(pth) == 0 || strings.HasPrefix(pth, "/") { + return os.ExpandEnv("$HOME" + pth), nil + } + + splitPth := strings.Split(pth, "/") + username := splitPth[0] + + usr, err := user.Lookup(username) + if err != nil { + return "", err + } + + pathInUsrHome := strings.Join(splitPth[1:], "/") + + return filepath.Join(usr.HomeDir, pathInUsrHome), nil + } + + return pth, nil +} diff --git a/vendor/github.com/bitrise-io/go-utils/v2/redactwriter/range.go b/vendor/github.com/bitrise-io/go-utils/v2/redactwriter/range.go new file mode 100644 index 00000000..71b9e815 --- /dev/null +++ b/vendor/github.com/bitrise-io/go-utils/v2/redactwriter/range.go @@ -0,0 +1,41 @@ +package redactwriter + +import ( + "bytes" + "sort" +) + +type matchRange struct{ first, last int } + +// allRanges returns every indexes of instance of pattern in b, or nil if pattern is not present in b. +func allRanges(b, pattern []byte) (ranges []matchRange) { + i := 0 + for { + sub := b[i:] + idx := bytes.Index(sub, pattern) + if idx == -1 { + return + } + + ranges = append(ranges, matchRange{first: idx + i, last: idx + i + len(pattern)}) + + i += idx + 1 + if i > len(b)-1 { + return + } + } +} + +// mergeAllRanges merges every overlapping ranges in r. +func mergeAllRanges(r []matchRange) []matchRange { + sort.Slice(r, func(i, j int) bool { return r[i].first < r[j].first }) + for i := 0; i < len(r)-1; i++ { + for i+1 < len(r) && r[i+1].first <= r[i].last { + if r[i+1].last > r[i].last { + r[i].last = r[i+1].last + } + r = append(r[:i+1], r[i+2:]...) + } + } + return r +} diff --git a/vendor/github.com/bitrise-io/go-utils/v2/redactwriter/redactwriter.go b/vendor/github.com/bitrise-io/go-utils/v2/redactwriter/redactwriter.go new file mode 100644 index 00000000..a140e3a3 --- /dev/null +++ b/vendor/github.com/bitrise-io/go-utils/v2/redactwriter/redactwriter.go @@ -0,0 +1,369 @@ +package redactwriter + +import ( + "bytes" + "io" + "sort" + "strings" + "sync" + "time" + + "github.com/bitrise-io/go-utils/v2/log" +) + +// RedactStr ... +const RedactStr = "[REDACTED]" + +var newLine = []byte("\n") + +// Writer ... +type Writer struct { + writer io.Writer + secrets [][][]byte + + chunk []byte + store [][]byte + mux sync.Mutex + timer *time.Timer + logger log.Logger +} + +// New ... +func New(secrets []string, target io.Writer, logger log.Logger) *Writer { + extendedSecrets := secrets + // adding transformed secrets with escaped newline characters to ensure that these are also obscured if found in logs + for _, secret := range secrets { + if strings.Contains(secret, "\n") { + extendedSecrets = append(extendedSecrets, strings.ReplaceAll(secret, "\n", `\n`)) + } + } + + return &Writer{ + writer: target, + secrets: secretsByteList(extendedSecrets), + logger: logger, + } +} + +// Write implements io.Writer interface. +// Splits p into lines, the lines are matched against the secrets, +// this determines which lines can be redacted and passed to the next writer (target). +// There might be lines that need to be buffered since they partially match a secret. +// We do not know the last Write call, so Close needs to be called to flush the buffer. +func (w *Writer) Write(p []byte) (int, error) { + if len(p) == 0 { + return 0, nil + } + + w.mux.Lock() + defer w.mux.Unlock() + + // previous bytes may not end with newline + data := append(w.chunk, p...) + + lastLines, chunk := splitAfterNewline(data) + w.chunk = chunk + if len(chunk) > 0 { + // we have remaining bytes, do not swallow them + if w.timer != nil { + w.timer.Stop() + w.timer.C = nil + } + w.timer = time.AfterFunc(100*time.Millisecond, func() { + if _, err := w.flush(); err != nil { + w.logger.Errorf("Failed to print last lines: %s", err) + } + }) + } + + if len(lastLines) == 0 { + // it is necessary to return the count of incoming bytes + return len(p), nil + } + + for _, line := range lastLines { + lines := append(w.store, line) + matchMap, partialMatchIndexes := w.matchSecrets(lines) + + var linesToPrint [][]byte + linesToPrint, w.store = w.matchLines(lines, partialMatchIndexes) + if linesToPrint == nil { + continue + } + + redactedLines := w.redact(linesToPrint, matchMap) + redactedBytes := bytes.Join(redactedLines, nil) + if c, err := w.writer.Write(redactedBytes); err != nil { + return c, err + } + } + + // it is necessary to return the count of incoming bytes + // to let the exec.Command work properly + return len(p), nil +} + +// Close implements io.Writer interface +func (w *Writer) Close() error { + _, err := w.flush() + return err +} + +// flush writes the remaining bytes. +func (w *Writer) flush() (int, error) { + defer func() { + w.mux.Unlock() + }() + w.mux.Lock() + + if len(w.chunk) > 0 { + // lines are containing newline, chunk may not + chunk := w.chunk + w.chunk = nil + w.store = append(w.store, chunk) + } + + // we only need to care about the full matches in the remaining lines + // (no more lines were come, why care about the partial matches?) + matchMap, _ := w.matchSecrets(w.store) + redactedLines := w.redact(w.store, matchMap) + w.store = nil + + return w.writer.Write(bytes.Join(redactedLines, nil)) +} + +// matchSecrets collects which secrets matches from which line indexes +// and which secrets matches partially from which line indexes. +// matchMap: matching line chunk's first line indexes by secret index +// partialMatchIndexes: line indexes from which secrets matching but not fully contained in lines +func (w *Writer) matchSecrets(lines [][]byte) (matchMap map[int][]int, partialMatchIndexes map[int]bool) { + matchMap = make(map[int][]int) + partialMatchIndexes = make(map[int]bool) + + for secretIdx, secret := range w.secrets { + secretLine := secret[0] // every match should begin from the secret first line + var lineIndexes []int // the indexes of lines which contains the secret's first line + + for i, line := range lines { + if bytes.Contains(line, secretLine) { + lineIndexes = append(lineIndexes, i) + } + } + + if len(lineIndexes) == 0 { + // this secret can not be found in the lines + continue + } + + for _, lineIdx := range lineIndexes { + if len(secret) == 1 { + // the single line secret found in the lines + indexes := matchMap[secretIdx] + matchMap[secretIdx] = append(indexes, lineIdx) + continue + } + + // lineIdx. line matches to a multi line secret's first line + // if lines has more line, every subsequent line must match to the secret's subsequent lines + partialMatch := true + match := false + + for i := lineIdx + 1; i < len(lines); i++ { + secretLineIdx := i - lineIdx + + secretLine = secret[secretLineIdx] + line := lines[i] + + if !bytes.Contains(line, secretLine) { + partialMatch = false + break + } + + if secretLineIdx == len(secret)-1 { + // multi line secret found in the lines + match = true + break + } + } + + if match { + // multi line secret found in the lines + indexes := matchMap[secretIdx] + matchMap[secretIdx] = append(indexes, lineIdx) + continue + } + + if partialMatch { + // this secret partially can be found in the lines + partialMatchIndexes[lineIdx] = true + } + } + } + + return +} + +// linesToKeepRange returns the first line index needs to be observed +// since they contain partially matching secrets. +func (w *Writer) linesToKeepRange(partialMatchIndexes map[int]bool) int { + first := -1 + + for lineIdx := range partialMatchIndexes { + if first == -1 { + first = lineIdx + continue + } + + if first > lineIdx { + first = lineIdx + } + } + + return first +} + +// matchLines return which lines can be printed and which should be kept for further observing. +func (w *Writer) matchLines(lines [][]byte, partialMatchIndexes map[int]bool) ([][]byte, [][]byte) { + first := w.linesToKeepRange(partialMatchIndexes) + switch first { + case -1: + // no lines need to be kept + return lines, nil + case 0: + // partial match is always longer then the lines + return nil, lines + default: + return lines[:first], lines[first:] + } +} + +// secretLinesToRedact returns which secret lines should be redacted +func (w *Writer) secretLinesToRedact(lineIdxToRedact int, matchMap map[int][]int) [][]byte { + // which line is which secrets first matching line + secretIdxsByLine := make(map[int][]int) + for secretIdx, lineIndexes := range matchMap { + for _, lineIdx := range lineIndexes { + secretIdxsByLine[lineIdx] = append(secretIdxsByLine[lineIdx], secretIdx) + } + } + + var secretChunks [][]byte + for firstMatchingLineIdx, secretIndexes := range secretIdxsByLine { + if lineIdxToRedact < firstMatchingLineIdx { + continue + } + + for _, secretIdx := range secretIndexes { + secret := w.secrets[secretIdx] + + if lineIdxToRedact > firstMatchingLineIdx+len(secret)-1 { + continue + } + + secretLineIdx := lineIdxToRedact - firstMatchingLineIdx + secretChunks = append(secretChunks, secret[secretLineIdx]) + } + } + + sort.Slice(secretChunks, func(i, j int) bool { return len(secretChunks[i]) < len(secretChunks[j]) }) + return secretChunks +} + +// redact hides the given ranges in the given line. +func redact(line []byte, ranges []matchRange) []byte { + var offset int // the offset of ranges generated by replacing x bytes by the RedactStr + for _, r := range ranges { + length := r.last - r.first + first := r.first + offset + last := first + length + + toRedact := line[first:last] + redactStr := RedactStr + if bytes.HasSuffix(toRedact, []byte("\n")) { + // if string to redact ends with newline redact message should also + redactStr += "\n" + } + + newLine := append([]byte{}, line[:first]...) + newLine = append(newLine, redactStr...) + newLine = append(newLine, line[last:]...) + + offset += len(redactStr) - length + + line = newLine + } + return line +} + +// redact hides the given secrets in the given lines. +func (w *Writer) redact(lines [][]byte, matchMap map[int][]int) [][]byte { + secretIdxsByLine := map[int][]int{} + for secretIdx, lineIndexes := range matchMap { + for _, lineIdx := range lineIndexes { + secretIdxsByLine[lineIdx] = append(secretIdxsByLine[lineIdx], secretIdx) + } + } + + for i, line := range lines { + linesToRedact := w.secretLinesToRedact(i, matchMap) + if linesToRedact == nil { + continue + } + + var ranges []matchRange + for _, lineToRedact := range linesToRedact { + ranges = append(ranges, allRanges(line, lineToRedact)...) + } + + lines[i] = redact(line, mergeAllRanges(ranges)) + } + + return lines +} + +// secretsByteList returns the list of secret byte lines. +func secretsByteList(secrets []string) [][][]byte { + var s [][][]byte + for _, secret := range secrets { + lines, lastLine := splitAfterNewline([]byte(secret)) + if lines == nil && lastLine == nil { + continue + } + + var secretLines [][]byte + if lines != nil { + secretLines = append(secretLines, lines...) + } + if lastLine != nil { + secretLines = append(secretLines, lastLine) + } + s = append(s, secretLines) + } + return s +} + +// splitAfterNewline splits p after "\n", the split is assigned to lines +// if last line has no "\n" it is assigned to the chunk. +// If p is nil both lines and chunk is set to nil. +func splitAfterNewline(p []byte) ([][]byte, []byte) { + chunk := p + var lines [][]byte + + for len(chunk) > 0 { + idx := bytes.Index(chunk, newLine) + if idx == -1 { + return lines, chunk + } + + lines = append(lines, chunk[:idx+1]) + + if idx == len(chunk)-1 { + chunk = nil + break + } + + chunk = chunk[idx+1:] + } + + return lines, chunk +} diff --git a/vendor/github.com/bitrise-io/stepman/models/models.go b/vendor/github.com/bitrise-io/stepman/models/models.go index 2ba805ff..2b502f1d 100644 --- a/vendor/github.com/bitrise-io/stepman/models/models.go +++ b/vendor/github.com/bitrise-io/stepman/models/models.go @@ -40,16 +40,10 @@ type AptGetDepModel struct { BinName string `json:"bin_name,omitempty" yaml:"bin_name,omitempty"` } -// CheckOnlyDepModel ... -type CheckOnlyDepModel struct { - Name string `json:"name,omitempty" yaml:"name,omitempty"` -} - // DepsModel ... type DepsModel struct { - Brew []BrewDepModel `json:"brew,omitempty" yaml:"brew,omitempty"` - AptGet []AptGetDepModel `json:"apt_get,omitempty" yaml:"apt_get,omitempty"` - CheckOnly []CheckOnlyDepModel `json:"check_only,omitempty" yaml:"check_only,omitempty"` + Brew []BrewDepModel `json:"brew,omitempty" yaml:"brew,omitempty"` + AptGet []AptGetDepModel `json:"apt_get,omitempty" yaml:"apt_get,omitempty"` } // BashStepToolkitModel ... @@ -63,10 +57,23 @@ type GoStepToolkitModel struct { PackageName string `json:"package_name" yaml:"package_name"` } +// SwiftStepToolkitModel ... +type SwiftStepToolkitModel struct { + BinaryLocation string `json:"binary_location,omitempty" yaml:"binary_location,omitempty"` + ExecutableName string `json:"executable_name,omitempty" yaml:"executable_name,omitempty"` +} + +// KotlinStepToolkitModel ... +type KotlinStepToolkitModel struct { + ExecutableName string `json:"executable_name,omitempty" yaml:"executable_name,omitempty"` +} + // StepToolkitModel ... type StepToolkitModel struct { - Bash *BashStepToolkitModel `json:"bash,omitempty" yaml:"bash,omitempty"` - Go *GoStepToolkitModel `json:"go,omitempty" yaml:"go,omitempty"` + Bash *BashStepToolkitModel `json:"bash,omitempty" yaml:"bash,omitempty"` + Go *GoStepToolkitModel `json:"go,omitempty" yaml:"go,omitempty"` + Swift *SwiftStepToolkitModel `json:"swift,omitempty" yaml:"swift,omitempty"` + Kotlin *KotlinStepToolkitModel `json:"kotlin,omitempty" yaml:"kotlin,omitempty"` } // StepModel ... @@ -98,9 +105,12 @@ type StepModel struct { // steps will run which are marked with IsAlwaysRun. IsSkippable *bool `json:"is_skippable,omitempty" yaml:"is_skippable,omitempty"` // RunIf : only run the step if the template example evaluates to true - RunIf *string `json:"run_if,omitempty" yaml:"run_if,omitempty"` - Timeout *int `json:"timeout,omitempty" yaml:"timeout,omitempty"` - Meta map[string]interface{} `json:"meta,omitempty" yaml:"meta,omitempty"` + RunIf *string `json:"run_if,omitempty" yaml:"run_if,omitempty"` + Timeout *int `json:"timeout,omitempty" yaml:"timeout,omitempty"` + // The timeout (in seconds) until a Step with no output (stdout/stderr) is aborted + // 0 means timeout is disabled. + NoOutputTimeout *int `json:"no_output_timeout,omitempty" yaml:"no_output_timeout,omitempty"` + Meta map[string]interface{} `json:"meta,omitempty" yaml:"meta,omitempty"` // Inputs []envmanModels.EnvironmentItemModel `json:"inputs,omitempty" yaml:"inputs,omitempty"` Outputs []envmanModels.EnvironmentItemModel `json:"outputs,omitempty" yaml:"outputs,omitempty"` diff --git a/vendor/github.com/bitrise-io/stepman/models/models_methods.go b/vendor/github.com/bitrise-io/stepman/models/models_methods.go index 87928d86..c7ef9c80 100644 --- a/vendor/github.com/bitrise-io/stepman/models/models_methods.go +++ b/vendor/github.com/bitrise-io/stepman/models/models_methods.go @@ -177,6 +177,10 @@ func (step *StepModel) AuditBeforeShare() error { return errors.New("Invalid step: timeout less then 0") } + if step.NoOutputTimeout != nil && *step.NoOutputTimeout < 0 { + return errors.New("Invalid step: 'no_output_timeout' is less then 0") + } + return step.ValidateInputAndOutputEnvs(true) } @@ -230,6 +234,7 @@ func (step *StepModel) FillMissingDefaults() error { if step.Timeout == nil { step.Timeout = pointers.NewIntPtr(DefaultTimeout) } + // NoOutputTimeout is left as is, so we can tell when it is nil (unset) vs set to 0 (disabled). for _, input := range step.Inputs { err := input.FillMissingDefaults() diff --git a/vendor/modules.txt b/vendor/modules.txt index c363392e..f4754a6c 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -1,11 +1,11 @@ # github.com/avast/apkparser v0.0.0-20210301101811-6256c76f738e ## explicit; go 1.10 github.com/avast/apkparser -# github.com/bitrise-io/bitrise v0.0.0-20210519130014-380842fb41c1 +# github.com/bitrise-io/bitrise v0.0.0-20220808135808-3483087dd853 ## explicit; go 1.16 github.com/bitrise-io/bitrise/models -# github.com/bitrise-io/envman v0.0.0-20210517135508-b2b4fe89eac5 -## explicit +# github.com/bitrise-io/envman v0.0.0-20220401145857-d11e00a5dc55 +## explicit; go 1.16 github.com/bitrise-io/envman/envman github.com/bitrise-io/envman/models # github.com/bitrise-io/go-android v0.0.0-20210517091621-72f0eb7a5197 @@ -19,6 +19,9 @@ github.com/bitrise-io/go-pkcs12/internal/rc2 ## explicit; go 1.15 github.com/bitrise-io/go-steputils/stepconf github.com/bitrise-io/go-steputils/tools +# github.com/bitrise-io/go-steputils/v2 v2.0.0-alpha.20 +## explicit; go 1.17 +github.com/bitrise-io/go-steputils/v2/secretkeys # github.com/bitrise-io/go-utils v1.0.9 ## explicit; go 1.13 github.com/bitrise-io/go-utils/colorstring @@ -36,10 +39,16 @@ github.com/bitrise-io/go-utils/retry github.com/bitrise-io/go-utils/sliceutil github.com/bitrise-io/go-utils/urlutil github.com/bitrise-io/go-utils/ziputil -# github.com/bitrise-io/go-utils/v2 v2.0.0-alpha.15 +# github.com/bitrise-io/go-utils/v2 v2.0.0-alpha.19 ## explicit; go 1.17 github.com/bitrise-io/go-utils/v2/env github.com/bitrise-io/go-utils/v2/errorutil +github.com/bitrise-io/go-utils/v2/exitcode +github.com/bitrise-io/go-utils/v2/fileutil +github.com/bitrise-io/go-utils/v2/log +github.com/bitrise-io/go-utils/v2/log/colorstring +github.com/bitrise-io/go-utils/v2/pathutil +github.com/bitrise-io/go-utils/v2/redactwriter # github.com/bitrise-io/go-xcode v1.0.16 ## explicit; go 1.20 github.com/bitrise-io/go-xcode/certificateutil @@ -51,10 +60,10 @@ github.com/bitrise-io/go-xcode/profileutil github.com/bitrise-io/go-xcode/utility github.com/bitrise-io/go-xcode/xcarchive github.com/bitrise-io/go-xcode/xcodeproject/serialized -# github.com/bitrise-io/goinp v0.0.0-20210504152833-8559b0680ab1 +# github.com/bitrise-io/goinp v0.0.0-20211005113137-305e91b481f4 ## explicit; go 1.16 github.com/bitrise-io/goinp/goinp -# github.com/bitrise-io/stepman v0.0.0-20210517135458-203f7a48d37a +# github.com/bitrise-io/stepman v0.0.0-20220808095634-6e12d2726f30 ## explicit; go 1.16 github.com/bitrise-io/stepman/models # github.com/davecgh/go-spew v1.1.1