diff --git a/e2e/bitrise.yml b/e2e/bitrise.yml index 8c2f69f..68816db 100644 --- a/e2e/bitrise.yml +++ b/e2e/bitrise.yml @@ -84,14 +84,14 @@ workflows: set -e json_response=$(curl --fail -X POST https://auth.services.bitrise.io/auth/realms/bitrise-services/protocol/openid-connect/token -k \ - --data "client_id=abcs-steps" \ + --data "client_id=bitrise-steps" \ --data "client_secret=$CACHE_API_CLIENT_SECRET" \ --data "grant_type=urn:ietf:params:oauth:grant-type:uma-ticket" \ - --data "claim_token=eyJhcHBfaWQiOlsiNDhmYThmYmVlNjk4NjIyYyJdLCAiYWNjZXNzX2dyYW50ZWQiOlsidHJ1ZSJdfQ==" \ + --data "claim_token=eyJhcHBfaWQiOlsiY2FjaGUtc3RlcHMtdGVzdHMiXSwgIm9yZ19pZCI6WyJ0ZXN0LW9yZy1pZCJdLCAiYWJjc19hY2Nlc3NfZ3JhbnRlZCI6WyJ0cnVlIl19" \ --data "claim_token_format=urn:ietf:params:oauth:token-type:jwt" \ - --data "audience=advanced-build-cache-service") + --data "audience=bitrise-services") auth_token=$(echo $json_response | jq -r .access_token) - envman add --key BITRISEIO_ABCS_API_URL --value $BITRISEIO_CACHE_SERVICE_URL --sensitive - envman add --key BITRISEIO_ABCS_ACCESS_TOKEN --value $auth_token --sensitive + envman add --key BITRISEIO_ABCS_API_URL --value $BITRISEIO_CACHE_SERVICE_URL + envman add --key BITRISEIO_BITRISE_SERVICES_ACCESS_TOKEN --value $auth_token --sensitive diff --git a/go.mod b/go.mod index c74fedc..0501b60 100644 --- a/go.mod +++ b/go.mod @@ -3,8 +3,8 @@ module github.com/bitrise-steplib/steps-restore-gradle-cache go 1.17 require ( - github.com/bitrise-io/go-steputils/v2 v2.0.0-alpha.12 - github.com/bitrise-io/go-utils/v2 v2.0.0-alpha.12 + github.com/bitrise-io/go-steputils/v2 v2.0.0-alpha.15 + github.com/bitrise-io/go-utils/v2 v2.0.0-alpha.13 ) require ( diff --git a/go.sum b/go.sum index 355883d..bca1842 100644 --- a/go.sum +++ b/go.sum @@ -1,11 +1,9 @@ -github.com/bitrise-io/go-steputils v1.0.1 h1:lwPl2W1njfANrBoTCkuqOOYbTha263ZFqoWQH0fwhaY= -github.com/bitrise-io/go-steputils v1.0.1/go.mod h1:YIUaQnIAyK4pCvQG0hYHVkSzKNT9uL2FWmkFNW4mfNI= -github.com/bitrise-io/go-steputils/v2 v2.0.0-alpha.12 h1:lQ/ayzGLU5f69CJCIvHt9gmhrbcwcwbNw5v1mWmXVX0= -github.com/bitrise-io/go-steputils/v2 v2.0.0-alpha.12/go.mod h1:ub3KVohX1vc1f2alPeVg7EoXAifm7p+SD326c2OUKdo= +github.com/bitrise-io/go-steputils/v2 v2.0.0-alpha.15 h1:wG037NV+pS8cEwtalE5K58bmKLyUkU0+4m4IuXjTzmo= +github.com/bitrise-io/go-steputils/v2 v2.0.0-alpha.15/go.mod h1:M09BbxYoh6B7KJnXk/yvtuU5nZPh7RQBJGKc1dp+0hQ= github.com/bitrise-io/go-utils v1.0.1 h1:e7mepVBkVN1DXRPESNXb0djEw6bxB6B93p/Q74zzcvk= github.com/bitrise-io/go-utils v1.0.1/go.mod h1:ZY1DI+fEpZuFpO9szgDeICM4QbqoWVt0RSY3tRI1heY= -github.com/bitrise-io/go-utils/v2 v2.0.0-alpha.12 h1:FxzmUw3B3eQuxPxBFN/oORunIdxSBqZgJZmRb8CAiTc= -github.com/bitrise-io/go-utils/v2 v2.0.0-alpha.12/go.mod h1:gZWtM7PLn1VOroa4gN1La/24aRVc0jg5R701jTsPaO8= +github.com/bitrise-io/go-utils/v2 v2.0.0-alpha.13 h1:QtAAfm/FpMDv/PnDxgzylVbbSx21pyl7+5T/ToJnWAQ= +github.com/bitrise-io/go-utils/v2 v2.0.0-alpha.13/go.mod h1:gZWtM7PLn1VOroa4gN1La/24aRVc0jg5R701jTsPaO8= github.com/bmatcuk/doublestar/v4 v4.2.0 h1:Qu+u9wR3Vd89LnlLMHvnZ5coJMWKQamqdz9/p5GNthA= github.com/bmatcuk/doublestar/v4 v4.2.0/go.mod h1:xBQ8jztBU6kakFMg+8WGxn0c6z1fTSPVIjEY1Wr7jzc= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -48,7 +46,6 @@ golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= diff --git a/main.go b/main.go index 713f713..370edd1 100644 --- a/main.go +++ b/main.go @@ -4,6 +4,7 @@ import ( "os" "github.com/bitrise-io/go-steputils/v2/stepconf" + "github.com/bitrise-io/go-utils/v2/command" "github.com/bitrise-io/go-utils/v2/env" "github.com/bitrise-io/go-utils/v2/log" "github.com/bitrise-steplib/steps-restore-gradle-cache/step" @@ -17,7 +18,8 @@ func run() int { logger := log.NewLogger() envRepo := env.NewRepository() inputParser := stepconf.NewInputParser(envRepo) - cacheStep := step.New(logger, inputParser, envRepo) + cmdFactory := command.NewFactory(envRepo) + cacheStep := step.New(logger, inputParser, envRepo, cmdFactory) exitCode := 0 diff --git a/step/step.go b/step/step.go index f4c7fc2..fcf0d30 100644 --- a/step/step.go +++ b/step/step.go @@ -6,6 +6,7 @@ import ( "github.com/bitrise-io/go-steputils/v2/cache" "github.com/bitrise-io/go-steputils/v2/stepconf" + "github.com/bitrise-io/go-utils/v2/command" "github.com/bitrise-io/go-utils/v2/env" "github.com/bitrise-io/go-utils/v2/log" ) @@ -32,17 +33,20 @@ type RestoreCacheStep struct { logger log.Logger inputParser stepconf.InputParser envRepo env.Repository + cmdFactory command.Factory } func New( logger log.Logger, inputParser stepconf.InputParser, envRepo env.Repository, + cmdFactory command.Factory, ) RestoreCacheStep { return RestoreCacheStep{ logger: logger, inputParser: inputParser, envRepo: envRepo, + cmdFactory: cmdFactory, } } @@ -59,8 +63,8 @@ func (step RestoreCacheStep) Run() error { step.logger.EnableDebugLog(input.Verbose) - saver := cache.NewRestorer(step.envRepo, step.logger) - return saver.Restore(cache.RestoreCacheInput{ + restorer := cache.NewRestorer(step.envRepo, step.logger, step.cmdFactory) + return restorer.Restore(cache.RestoreCacheInput{ StepId: stepId, Verbose: input.Verbose, Keys: keys, diff --git a/vendor/github.com/bitrise-io/go-steputils/v2/cache/common.go b/vendor/github.com/bitrise-io/go-steputils/v2/cache/common.go new file mode 100644 index 0000000..398306d --- /dev/null +++ b/vendor/github.com/bitrise-io/go-steputils/v2/cache/common.go @@ -0,0 +1,28 @@ +package cache + +import ( + "crypto/sha256" + "encoding/hex" + "io" + "os" +) + +// We need this prefix because there could be multiple restore steps in one workflow with multiple cache keys +const cacheHitEnvVarPrefix = "BITRISE_CACHE_HIT__" + +func checksumOfFile(path string) (string, error) { + hash := sha256.New() + + file, err := os.Open(path) + if err != nil { + return "", err + } + defer file.Close() //nolint:errcheck + + _, err = io.Copy(hash, file) + if err != nil { + return "", err + } + + return hex.EncodeToString(hash.Sum(nil)), nil +} diff --git a/vendor/github.com/bitrise-io/go-steputils/v2/cache/keytemplate/checksum.go b/vendor/github.com/bitrise-io/go-steputils/v2/cache/keytemplate/checksum.go index 8f41cb7..ddf57ba 100644 --- a/vendor/github.com/bitrise-io/go-steputils/v2/cache/keytemplate/checksum.go +++ b/vendor/github.com/bitrise-io/go-steputils/v2/cache/keytemplate/checksum.go @@ -3,12 +3,13 @@ package keytemplate import ( "crypto/sha256" "encoding/hex" + "io" "os" "path/filepath" "sort" "strings" - "github.com/bitrise-io/go-utils/pathutil" + "github.com/bitrise-io/go-utils/v2/pathutil" "github.com/bmatcuk/doublestar/v4" ) @@ -83,11 +84,17 @@ func (m Model) evaluateGlobPatterns(paths []string) []string { func checksumOfFile(path string) ([]byte, error) { hash := sha256.New() - b, err := os.ReadFile(path) + file, err := os.Open(path) if err != nil { return nil, err } - hash.Write(b) + defer file.Close() //nolint:errcheck + + _, err = io.Copy(hash, file) + if err != nil { + return nil, err + } + return hash.Sum(nil), nil } diff --git a/vendor/github.com/bitrise-io/go-steputils/v2/cache/network/api.go b/vendor/github.com/bitrise-io/go-steputils/v2/cache/network/api.go index be48174..1782f77 100644 --- a/vendor/github.com/bitrise-io/go-steputils/v2/cache/network/api.go +++ b/vendor/github.com/bitrise-io/go-steputils/v2/cache/network/api.go @@ -32,7 +32,8 @@ type prepareUploadResponse struct { } type restoreResponse struct { - URL string `json:"url"` + URL string `json:"url"` + MatchedKey string `json:"matched_cache_key"` } type apiClient struct { @@ -146,22 +147,22 @@ func (c apiClient) acknowledgeUpload(uploadID string) error { return nil } -func (c apiClient) restore(cacheKeys []string) (string, error) { +func (c apiClient) restore(cacheKeys []string) (restoreResponse, error) { keysInQuery, err := validateKeys(cacheKeys) if err != nil { - return "", err + return restoreResponse{}, err } apiURL := fmt.Sprintf("%s/restore?cache_keys=%s", c.baseURL, keysInQuery) req, err := retryablehttp.NewRequest(http.MethodGet, apiURL, nil) if err != nil { - return "", err + return restoreResponse{}, err } req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", c.accessToken)) resp, err := c.httpClient.Do(req) if err != nil { - return "", err + return restoreResponse{}, err } defer func(body io.ReadCloser) { err := body.Close() @@ -171,19 +172,19 @@ func (c apiClient) restore(cacheKeys []string) (string, error) { }(resp.Body) if resp.StatusCode == http.StatusNotFound { - return "", ErrCacheNotFound + return restoreResponse{}, ErrCacheNotFound } if resp.StatusCode != http.StatusOK { - return "", unwrapError(resp) + return restoreResponse{}, unwrapError(resp) } var response restoreResponse err = json.NewDecoder(resp.Body).Decode(&response) if err != nil { - return "", err + return restoreResponse{}, err } - return response.URL, nil + return response, nil } func (c apiClient) downloadArchive(url string) (io.ReadCloser, error) { diff --git a/vendor/github.com/bitrise-io/go-steputils/v2/cache/network/download.go b/vendor/github.com/bitrise-io/go-steputils/v2/cache/network/download.go index 56b6151..1908094 100644 --- a/vendor/github.com/bitrise-io/go-steputils/v2/cache/network/download.go +++ b/vendor/github.com/bitrise-io/go-steputils/v2/cache/network/download.go @@ -23,31 +23,31 @@ var ErrCacheNotFound = errors.New("no cache archive found for the provided keys" // Download archive from the cache API based on the provided keys in params. // If there is no match for any of the keys, the error is ErrCacheNotFound. -func Download(params DownloadParams, logger log.Logger) error { +func Download(params DownloadParams, logger log.Logger) (matchedKey string, err error) { if params.APIBaseURL == "" { - return fmt.Errorf("API base URL is empty") + return "", fmt.Errorf("API base URL is empty") } if params.Token == "" { - return fmt.Errorf("API token is empty") + return "", fmt.Errorf("API token is empty") } if len(params.CacheKeys) == 0 { - return fmt.Errorf("cache key list is empty") + return "", fmt.Errorf("cache key list is empty") } client := newAPIClient(retryhttp.NewClient(logger), params.APIBaseURL, params.Token) logger.Debugf("Get download URL") - url, err := client.restore(params.CacheKeys) + restoreResponse, err := client.restore(params.CacheKeys) if err != nil { - return fmt.Errorf("failed to get download URL: %w", err) + return "", fmt.Errorf("failed to get download URL: %w", err) } logger.Debugf("Download archive") file, err := os.Create(params.DownloadPath) if err != nil { - return fmt.Errorf("can't open download location: %w", err) + return "", fmt.Errorf("can't open download location: %w", err) } defer func(file *os.File) { err := file.Close() @@ -56,9 +56,9 @@ func Download(params DownloadParams, logger log.Logger) error { } }(file) - respBody, err := client.downloadArchive(url) + respBody, err := client.downloadArchive(restoreResponse.URL) if err != nil { - return fmt.Errorf("failed to download archive: %w", err) + return "", fmt.Errorf("failed to download archive: %w", err) } defer func(respBody io.ReadCloser) { err := respBody.Close() @@ -68,8 +68,8 @@ func Download(params DownloadParams, logger log.Logger) error { }(respBody) _, err = io.Copy(file, respBody) if err != nil { - return fmt.Errorf("failed to save archive to disk: %w", err) + return "", fmt.Errorf("failed to save archive to disk: %w", err) } - return nil + return restoreResponse.MatchedKey, nil } diff --git a/vendor/github.com/bitrise-io/go-steputils/v2/cache/restore.go b/vendor/github.com/bitrise-io/go-steputils/v2/cache/restore.go index 97b48b2..e5860c8 100644 --- a/vendor/github.com/bitrise-io/go-steputils/v2/cache/restore.go +++ b/vendor/github.com/bitrise-io/go-steputils/v2/cache/restore.go @@ -10,7 +10,9 @@ import ( "github.com/bitrise-io/go-steputils/v2/cache/compression" "github.com/bitrise-io/go-steputils/v2/cache/keytemplate" "github.com/bitrise-io/go-steputils/v2/cache/network" + "github.com/bitrise-io/go-steputils/v2/export" "github.com/bitrise-io/go-steputils/v2/stepconf" + "github.com/bitrise-io/go-utils/v2/command" "github.com/bitrise-io/go-utils/v2/env" "github.com/bitrise-io/go-utils/v2/log" "github.com/docker/go-units" @@ -37,13 +39,19 @@ type restoreCacheConfig struct { } type restorer struct { - envRepo env.Repository - logger log.Logger + envRepo env.Repository + logger log.Logger + cmdFactory command.Factory +} + +type downloadResult struct { + filePath string + matchedKey string } // NewRestorer ... -func NewRestorer(envRepo env.Repository, logger log.Logger) *restorer { - return &restorer{envRepo: envRepo, logger: logger} +func NewRestorer(envRepo env.Repository, logger log.Logger, cmdFactory command.Factory) *restorer { + return &restorer{envRepo: envRepo, logger: logger, cmdFactory: cmdFactory} } // Restore ... @@ -54,19 +62,27 @@ func (r *restorer) Restore(input RestoreCacheInput) error { } tracker := newStepTracker(input.StepId, r.envRepo, r.logger) + defer tracker.wait() r.logger.Println() r.logger.Infof("Downloading archive...") downloadStartTime := time.Now() - archivePath, err := r.download(config) + result, err := r.download(config) if err != nil { if errors.Is(err, network.ErrCacheNotFound) { r.logger.Donef("No cache entry found for the provided key") + tracker.logRestoreResult(false, "", config.Keys) return nil } return fmt.Errorf("download failed: %w", err) } - fileInfo, err := os.Stat(archivePath) + if result.matchedKey == config.Keys[0] { + r.logger.Printf("Exact hit for first key") + } else { + r.logger.Printf("Cache hit for key: %s", result.matchedKey) + } + + fileInfo, err := os.Stat(result.filePath) if err != nil { return err } @@ -78,14 +94,19 @@ func (r *restorer) Restore(input RestoreCacheInput) error { r.logger.Println() r.logger.Infof("Restoring archive...") extractionStartTime := time.Now() - if err := compression.Decompress(archivePath, r.logger, r.envRepo); err != nil { + if err := compression.Decompress(result.filePath, r.logger, r.envRepo); err != nil { return fmt.Errorf("failed to decompress cache archive: %w", err) } extractionTime := time.Since(extractionStartTime).Round(time.Second) r.logger.Donef("Restored archive in %s", extractionTime) tracker.logArchiveExtracted(extractionTime, len(config.Keys)) - tracker.wait() + err = r.exposeCacheHit(result) + if err != nil { + return err + } + + tracker.logRestoreResult(true, result.matchedKey, config.Keys) return nil } @@ -94,9 +115,9 @@ func (r *restorer) createConfig(input RestoreCacheInput) (restoreCacheConfig, er if apiBaseURL == "" { return restoreCacheConfig{}, fmt.Errorf("the secret 'BITRISEIO_ABCS_API_URL' is not defined") } - apiAccessToken := r.envRepo.Get("BITRISEIO_ABCS_ACCESS_TOKEN") + apiAccessToken := r.envRepo.Get("BITRISEIO_BITRISE_SERVICES_ACCESS_TOKEN") if apiAccessToken == "" { - return restoreCacheConfig{}, fmt.Errorf("the secret 'BITRISEIO_ABCS_ACCESS_TOKEN' is not defined") + return restoreCacheConfig{}, fmt.Errorf("the secret 'BITRISEIO_BITRISE_SERVICES_ACCESS_TOKEN' is not defined") } keys, err := r.evaluateKeys(input.Keys) @@ -134,10 +155,10 @@ func (r *restorer) evaluateKeys(keys []string) ([]string, error) { return evaluatedKeys, nil } -func (r *restorer) download(config restoreCacheConfig) (string, error) { +func (r *restorer) download(config restoreCacheConfig) (downloadResult, error) { dir, err := os.MkdirTemp("", "restore-cache") if err != nil { - return "", err + return downloadResult{}, err } name := fmt.Sprintf("cache-%s.tzst", time.Now().UTC().Format("20060102-150405")) downloadPath := filepath.Join(dir, name) @@ -148,12 +169,35 @@ func (r *restorer) download(config restoreCacheConfig) (string, error) { CacheKeys: config.Keys, DownloadPath: downloadPath, } - err = network.Download(params, r.logger) + matchedKey, err := network.Download(params, r.logger) if err != nil { - return "", err + return downloadResult{}, err } r.logger.Debugf("Archive downloaded to %s", downloadPath) - return downloadPath, nil + return downloadResult{filePath: downloadPath, matchedKey: matchedKey}, nil +} + +func (r *restorer) exposeCacheHit(result downloadResult) error { + if result.filePath == "" || result.matchedKey == "" { + return nil + } + + checksum, err := checksumOfFile(result.filePath) + if err != nil { + return err + } + + r.logger.Debugf("Exposing cache hit info:") + r.logger.Debugf("Matched key: %s", result.matchedKey) + r.logger.Debugf("Archive checksum: %s", checksum) + + envKey := cacheHitEnvVarPrefix + result.matchedKey + exporter := export.NewExporter(r.cmdFactory) + err = exporter.ExportOutput(envKey, checksum) + if err != nil { + return err + } + return r.envRepo.Set(envKey, checksum) } diff --git a/vendor/github.com/bitrise-io/go-steputils/v2/cache/save.go b/vendor/github.com/bitrise-io/go-steputils/v2/cache/save.go index e5bd137..53b4498 100644 --- a/vendor/github.com/bitrise-io/go-steputils/v2/cache/save.go +++ b/vendor/github.com/bitrise-io/go-steputils/v2/cache/save.go @@ -25,6 +25,12 @@ type SaveCacheInput struct { Verbose bool Key string Paths []string + // IsKeyUnique indicates that the cache key is enough for knowing the cache archive is different from + // another cache archive. + // This can be set to true if the cache key contains a checksum that changes when any of the cached files change. + // Example of such key: my-cache-key-{{ checksum "package-lock.json" }} + // Example where this is not true: my-cache-key-{{ .OS }}-{{ .Arch }} + IsKeyUnique bool } // Saver ... @@ -73,6 +79,20 @@ func (s *saver) Save(input SaveCacheInput) error { } tracker := newStepTracker(input.StepId, s.envRepo, s.logger) + defer tracker.wait() + + canSkipSave, reason := s.canSkipSave(input.Key, config.Key, input.IsKeyUnique) + tracker.logSkipSaveResult(canSkipSave, reason) + s.logger.Println() + if canSkipSave { + s.logger.Donef("Cache save can be skipped, reason: %s", reason.description()) + return nil + } else { + s.logger.Infof("Can't skip saving the cache, reason: %s", reason.description()) + if reason == reasonNoRestoreThisKey { + s.logOtherHits() + } + } s.logger.Println() s.logger.Infof("Creating archive...") @@ -92,6 +112,20 @@ func (s *saver) Save(input SaveCacheInput) error { s.logger.Printf("Archive size: %s", units.HumanSizeWithPrecision(float64(fileInfo.Size()), 3)) s.logger.Debugf("Archive path: %s", archivePath) + archiveChecksum, err := checksumOfFile(archivePath) + if err != nil { + s.logger.Warnf(err.Error()) + // fail silently and continue + } + canSkipUpload, reason := s.canSkipUpload(config.Key, archiveChecksum) + tracker.logSkipUploadResult(canSkipUpload, reason) + s.logger.Println() + if canSkipUpload { + s.logger.Donef("Cache upload can be skipped, reason: %s", reason.description()) + return nil + } + s.logger.Infof("Can't skip uploading the cache, reason: %s", reason.description()) + s.logger.Println() s.logger.Infof("Uploading archive...") uploadStartTime := time.Now() @@ -102,7 +136,6 @@ func (s *saver) Save(input SaveCacheInput) error { uploadTime := time.Since(uploadStartTime).Round(time.Second) s.logger.Donef("Archive uploaded in %s", uploadTime) tracker.logArchiveUploaded(uploadTime, fileInfo, len(config.Paths)) - tracker.wait() return nil } @@ -129,9 +162,9 @@ func (s *saver) createConfig(input SaveCacheInput) (saveCacheConfig, error) { if apiBaseURL == "" { return saveCacheConfig{}, fmt.Errorf("the secret 'BITRISEIO_ABCS_API_URL' is not defined") } - apiAccessToken := s.envRepo.Get("BITRISEIO_ABCS_ACCESS_TOKEN") + apiAccessToken := s.envRepo.Get("BITRISEIO_BITRISE_SERVICES_ACCESS_TOKEN") if apiAccessToken == "" { - return saveCacheConfig{}, fmt.Errorf("the secret 'BITRISEIO_ABCS_ACCESS_TOKEN' is not defined") + return saveCacheConfig{}, fmt.Errorf("the secret 'BITRISEIO_BITRISE_SERVICES_ACCESS_TOKEN' is not defined") } return saveCacheConfig{ diff --git a/vendor/github.com/bitrise-io/go-steputils/v2/cache/save_skip.go b/vendor/github.com/bitrise-io/go-steputils/v2/cache/save_skip.go new file mode 100644 index 0000000..8300040 --- /dev/null +++ b/vendor/github.com/bitrise-io/go-steputils/v2/cache/save_skip.go @@ -0,0 +1,130 @@ +package cache + +import ( + "strings" +) + +type skipReason int + +const ( + reasonKeyNotDynamic skipReason = iota + reasonNoRestore + reasonRestoreSameUniqueKey + reasonRestoreSameKeyNotUnique + reasonNoRestoreThisKey + reasonNewArchiveChecksumMatch + reasonNewArchiveChecksumMismatch +) + +func (r skipReason) String() string { + switch r { + case reasonKeyNotDynamic: + return "key_not_dynamic" + case reasonNoRestore: + return "no_restore_found" + case reasonRestoreSameUniqueKey: + return "restore_same_unique_key" + case reasonRestoreSameKeyNotUnique: + return "restore_same_key_not_unique" + case reasonNoRestoreThisKey: + return "no_restore_with_this_key" + case reasonNewArchiveChecksumMatch: + return "new_archive_checksum_match" + case reasonNewArchiveChecksumMismatch: + return "new_archive_checksum_mismatch" + default: + return "unknown" + } +} + +func (r skipReason) description() string { + switch r { + case reasonKeyNotDynamic: + return "key is not dynamic; the expectation is that the same key is used for saving different cache contents over and over" + case reasonNoRestore: + return "no cache was restored in the workflow, creating a new cache entry" + case reasonRestoreSameUniqueKey: + return "a cache with the same key was restored in the workflow, new cache would have the same content" + case reasonRestoreSameKeyNotUnique: + return "a cache with the same key was restored in the workflow, but contents might have changed since then" + case reasonNoRestoreThisKey: + return "there was no cache restore in the workflow with this key, but was for other(s)" + case reasonNewArchiveChecksumMatch: + return "new cache archive is the same as the restored one" + case reasonNewArchiveChecksumMismatch: + return "new cache archive doesn't match the restored one" + default: + return "unrecognized skipReason" + } +} + +func (s *saver) canSkipSave(keyTemplate, evaluatedKey string, isKeyUnique bool) (bool, skipReason) { + if keyTemplate == evaluatedKey { + return false, reasonKeyNotDynamic + } + + cacheHits := s.getCacheHits() + if len(cacheHits) == 0 { + return false, reasonNoRestore + } + + if _, ok := cacheHits[evaluatedKey]; ok { + if isKeyUnique { + return true, reasonRestoreSameUniqueKey + } else { + return false, reasonRestoreSameKeyNotUnique + } + } + + return false, reasonNoRestoreThisKey +} + +func (s *saver) canSkipUpload(newCacheKey, newCacheChecksum string) (bool, skipReason) { + cacheHits := s.getCacheHits() + + if len(cacheHits) == 0 { + return false, reasonNoRestore + } + + checksumForNewKey, ok := cacheHits[newCacheKey] + if !ok { + return false, reasonNoRestoreThisKey + } + if checksumForNewKey == newCacheChecksum { + return true, reasonNewArchiveChecksumMatch + } + + return false, reasonNewArchiveChecksumMismatch +} + +// Returns cache hit information exposed by previous restore cache steps. +// The returned map's key is the restored cache key, and the value is the checksum of the cache archive +func (s *saver) getCacheHits() map[string]string { + cacheHits := map[string]string{} + for _, e := range s.envRepo.List() { + envParts := strings.SplitN(e, "=", 2) + if len(envParts) < 2 { + continue + } + envKey := envParts[0] + envValue := envParts[1] + + if strings.HasPrefix(envKey, cacheHitEnvVarPrefix) { + cacheKey := strings.TrimPrefix(envKey, cacheHitEnvVarPrefix) + cacheHits[cacheKey] = envValue + } + } + return cacheHits +} + +func (s *saver) logOtherHits() { + otherKeys := []string{} + for k := range s.getCacheHits() { + otherKeys = append(otherKeys, k) + } + if len(otherKeys) == 0 { + return + } + s.logger.Printf("Other restored cache keys:") + s.logger.Printf(strings.Join(otherKeys, "\n")) +} diff --git a/vendor/github.com/bitrise-io/go-steputils/v2/cache/tracker.go b/vendor/github.com/bitrise-io/go-steputils/v2/cache/tracker.go index 1235a20..83504aa 100644 --- a/vendor/github.com/bitrise-io/go-steputils/v2/cache/tracker.go +++ b/vendor/github.com/bitrise-io/go-steputils/v2/cache/tracker.go @@ -62,6 +62,37 @@ func (t *stepTracker) logArchiveExtracted(extractionTime time.Duration, keyCount t.tracker.Enqueue("step_restore_cache_archive_extracted", properties) } +func (t *stepTracker) logRestoreResult(isMatch bool, matchedKey string, evaluatedKeys []string) { + if len(evaluatedKeys) == 0 { + return + } + + properties := analytics.Properties{ + "is_match": isMatch, + "is_first_key_matched": matchedKey == evaluatedKeys[0], + "key_count": len(evaluatedKeys), + } + t.tracker.Enqueue("step_restore_cache_result", properties) +} + +func (t *stepTracker) logSkipSaveResult(isSaveSkipped bool, reason skipReason) { + + properties := analytics.Properties{ + "is_save_skipped": isSaveSkipped, + "reason": reason.String(), + } + t.tracker.Enqueue("step_save_cache_save_skipped", properties) +} + +func (t *stepTracker) logSkipUploadResult(isUploadSkipped bool, reason skipReason) { + + properties := analytics.Properties{ + "is_upload_skipped": isUploadSkipped, + "reason": reason.String(), + } + t.tracker.Enqueue("step_save_cache_upload_skipped", properties) +} + func (t *stepTracker) wait() { t.tracker.Wait() } diff --git a/vendor/github.com/bitrise-io/go-steputils/v2/export/export.go b/vendor/github.com/bitrise-io/go-steputils/v2/export/export.go new file mode 100644 index 0000000..81d295a --- /dev/null +++ b/vendor/github.com/bitrise-io/go-steputils/v2/export/export.go @@ -0,0 +1,166 @@ +package export + +import ( + "fmt" + "io" + "log" + "os" + "path/filepath" + + "github.com/bitrise-io/go-utils/v2/command" + "github.com/bitrise-io/go-utils/v2/pathutil" + "github.com/bitrise-io/go-utils/ziputil" +) + +const ( + filesType = "files" + foldersType = "folders" + mixedFileAndFolderType = "mixed" +) + +// Exporter ... +type Exporter struct { + cmdFactory command.Factory +} + +// NewExporter ... +func NewExporter(cmdFactory command.Factory) Exporter { + return Exporter{cmdFactory: cmdFactory} +} + +// ExportOutput is used for exposing values for other steps. +// Regular env vars are isolated between steps, so instead of calling `os.Setenv()`, use this to explicitly expose +// a value for subsequent steps. +func (e *Exporter) ExportOutput(key, value string) error { + cmd := e.cmdFactory.Create("envman", []string{"add", "--key", key, "--value", value}, nil) + out, err := cmd.RunAndReturnTrimmedCombinedOutput() + if err != nil { + return fmt.Errorf("exporting output with envman failed: %s, output: %s", err, out) + } + return nil +} + +// ExportOutputFile is a convenience method for copying sourcePath to destinationPath and then exporting the +// absolute destination path with ExportOutput() +func (e *Exporter) ExportOutputFile(key, sourcePath, destinationPath string) error { + pathModifier := pathutil.NewPathModifier() + absSourcePath, err := pathModifier.AbsPath(sourcePath) + if err != nil { + return err + } + absDestinationPath, err := pathModifier.AbsPath(destinationPath) + if err != nil { + return err + } + + if absSourcePath != absDestinationPath { + if err = copyFile(absSourcePath, absDestinationPath); err != nil { + return err + } + } + + return e.ExportOutput(key, absDestinationPath) +} + +// ExportOutputFilesZip is a convenience method for creating a ZIP archive from sourcePaths at zipPath and then +// exporting the absolute path of the ZIP with ExportOutput() +func (e *Exporter) ExportOutputFilesZip(key string, sourcePaths []string, zipPath string) error { + tempZipPath, err := zipFilePath() + if err != nil { + return err + } + + // We have separate zip functions for files and folders and that is the main reason we cannot have mixed + // paths (files and also folders) in the input. It has to be either folders or files. Everything + // else leads to an error. + inputType, err := getInputType(sourcePaths) + if err != nil { + return err + } + switch inputType { + case filesType: + err = ziputil.ZipFiles(sourcePaths, tempZipPath) + case foldersType: + err = ziputil.ZipDirs(sourcePaths, tempZipPath) + case mixedFileAndFolderType: + return fmt.Errorf("source path list (%s) contains a mix of files and folders", sourcePaths) + default: + return fmt.Errorf("source path list (%s) is empty", sourcePaths) + } + + if err != nil { + return err + } + + return e.ExportOutputFile(key, tempZipPath, zipPath) +} + +func zipFilePath() (string, error) { + tmpDir, err := pathutil.NewPathProvider().CreateTempDir("__export_tmp_dir__") + if err != nil { + return "", err + } + + return filepath.Join(tmpDir, "temp-zip-file.zip"), nil +} + +func getInputType(sourcePths []string) (string, error) { + var folderCount, fileCount int + pathChecker := pathutil.NewPathChecker() + + for _, path := range sourcePths { + exist, err := pathChecker.IsDirExists(path) + if err != nil { + return "", err + } + + if exist { + folderCount++ + continue + } + + exist, err = pathChecker.IsPathExists(path) + if err != nil { + return "", err + } + + if exist { + fileCount++ + } + } + + if fileCount == len(sourcePths) { + return filesType, nil + } else if folderCount == len(sourcePths) { + return foldersType, nil + } else if 0 < folderCount && 0 < fileCount { + return mixedFileAndFolderType, nil + } + + return "", nil +} + +func copyFile(source, destination string) error { + in, err := os.Open(source) + if err != nil { + return err + } + defer in.Close() //nolint:errcheck + + out, err := os.Create(destination) + if err != nil { + return err + } + defer func(out *os.File) { + err := out.Close() + if err != nil { + log.Fatalf(err.Error()) + } + }(out) + + _, err = io.Copy(out, in) + if err != nil { + return err + } + return nil +} diff --git a/vendor/github.com/bitrise-io/go-steputils/v2/stepconf/stepconf.go b/vendor/github.com/bitrise-io/go-steputils/v2/stepconf/stepconf.go index 1f0acf6..8fb7e90 100644 --- a/vendor/github.com/bitrise-io/go-steputils/v2/stepconf/stepconf.go +++ b/vendor/github.com/bitrise-io/go-steputils/v2/stepconf/stepconf.go @@ -9,7 +9,6 @@ import ( "strconv" "strings" - "github.com/bitrise-io/go-utils/parseutil" "github.com/bitrise-io/go-utils/v2/env" ) @@ -86,11 +85,11 @@ func setField(field reflect.Value, value, constraint string) error { field = field.Elem() } - switch field.Kind() { + switch field.Kind() { //nolint:exhaustive case reflect.String: field.SetString(value) case reflect.Bool: - b, err := parseutil.ParseBool(value) + b, err := parseBool(value) if err != nil { return errors.New("can't convert to bool") } @@ -448,3 +447,19 @@ func contains(s, opt string) bool { } return false } + +func parseBool(userInputStr string) (bool, error) { + if userInputStr == "" { + return false, errors.New("no string to parse") + } + userInputStr = strings.TrimSpace(userInputStr) + + lowercased := strings.ToLower(userInputStr) + if lowercased == "yes" || lowercased == "y" { + return true, nil + } + if lowercased == "no" || lowercased == "n" { + return false, nil + } + return strconv.ParseBool(lowercased) +} diff --git a/vendor/github.com/bitrise-io/go-steputils/v2/stepconf/strings.go b/vendor/github.com/bitrise-io/go-steputils/v2/stepconf/strings.go index b631189..af4742c 100644 --- a/vendor/github.com/bitrise-io/go-steputils/v2/stepconf/strings.go +++ b/vendor/github.com/bitrise-io/go-steputils/v2/stepconf/strings.go @@ -4,7 +4,7 @@ import ( "fmt" "reflect" - "github.com/bitrise-io/go-utils/colorstring" + "github.com/bitrise-io/go-utils/v2/log/colorstring" "golang.org/x/text/cases" "golang.org/x/text/language" ) diff --git a/vendor/github.com/bitrise-io/go-utils/colorstring/colorstring.go b/vendor/github.com/bitrise-io/go-utils/colorstring/colorstring.go deleted file mode 100644 index 5f31fa9..0000000 --- a/vendor/github.com/bitrise-io/go-utils/colorstring/colorstring.go +++ /dev/null @@ -1,110 +0,0 @@ -package colorstring - -import ( - "fmt" -) - -// Color ... -// ANSI color escape sequences -type Color string - -const ( - blackColor Color = "\x1b[30;1m" - redColor Color = "\x1b[31;1m" - greenColor Color = "\x1b[32;1m" - yellowColor Color = "\x1b[33;1m" - blueColor Color = "\x1b[34;1m" - magentaColor Color = "\x1b[35;1m" - cyanColor Color = "\x1b[36;1m" - resetColor Color = "\x1b[0m" -) - -// ColorFunc ... -type ColorFunc func(a ...interface{}) string - -func addColor(color Color, msg string) string { - return string(color) + msg + string(resetColor) -} - -// NoColor ... -func NoColor(a ...interface{}) string { - return fmt.Sprint(a...) -} - -// Black ... -func Black(a ...interface{}) string { - return addColor(blackColor, fmt.Sprint(a...)) -} - -// Red ... -func Red(a ...interface{}) string { - return addColor(redColor, fmt.Sprint(a...)) -} - -// Green ... -func Green(a ...interface{}) string { - return addColor(greenColor, fmt.Sprint(a...)) -} - -// Yellow ... -func Yellow(a ...interface{}) string { - return addColor(yellowColor, fmt.Sprint(a...)) -} - -// Blue ... -func Blue(a ...interface{}) string { - return addColor(blueColor, fmt.Sprint(a...)) -} - -// Magenta ... -func Magenta(a ...interface{}) string { - return addColor(magentaColor, fmt.Sprint(a...)) -} - -// Cyan ... -func Cyan(a ...interface{}) string { - return addColor(cyanColor, fmt.Sprint(a...)) -} - -// ColorfFunc ... -type ColorfFunc func(format string, a ...interface{}) string - -// NoColorf ... -func NoColorf(format string, a ...interface{}) string { - return NoColor(fmt.Sprintf(format, a...)) -} - -// Blackf ... -func Blackf(format string, a ...interface{}) string { - return Black(fmt.Sprintf(format, a...)) -} - -// Redf ... -func Redf(format string, a ...interface{}) string { - return Red(fmt.Sprintf(format, a...)) -} - -// Greenf ... -func Greenf(format string, a ...interface{}) string { - return Green(fmt.Sprintf(format, a...)) -} - -// Yellowf ... -func Yellowf(format string, a ...interface{}) string { - return Yellow(fmt.Sprintf(format, a...)) -} - -// Bluef ... -func Bluef(format string, a ...interface{}) string { - return Blue(fmt.Sprintf(format, a...)) -} - -// Magentaf ... -func Magentaf(format string, a ...interface{}) string { - return Magenta(fmt.Sprintf(format, a...)) -} - -// Cyanf ... -func Cyanf(format string, a ...interface{}) string { - return Cyan(fmt.Sprintf(format, a...)) -} diff --git a/vendor/github.com/bitrise-io/go-utils/command/command.go b/vendor/github.com/bitrise-io/go-utils/command/command.go new file mode 100644 index 0000000..c068490 --- /dev/null +++ b/vendor/github.com/bitrise-io/go-utils/command/command.go @@ -0,0 +1,252 @@ +package command + +import ( + "errors" + "io" + "os" + "os/exec" + "strconv" + "strings" +) + +// ---------- + +// Model ... +type Model struct { + cmd *exec.Cmd +} + +// New ... +func New(name string, args ...string) *Model { + return &Model{ + cmd: exec.Command(name, args...), + } +} + +// NewWithStandardOuts - same as NewCommand, but sets the command's +// stdout and stderr to the standard (OS) out (os.Stdout) and err (os.Stderr) +func NewWithStandardOuts(name string, args ...string) *Model { + return New(name, args...).SetStdout(os.Stdout).SetStderr(os.Stderr) +} + +// NewWithParams ... +func NewWithParams(params ...string) (*Model, error) { + if len(params) == 0 { + return nil, errors.New("no command provided") + } else if len(params) == 1 { + return New(params[0]), nil + } + + return New(params[0], params[1:]...), nil +} + +// NewFromSlice ... +func NewFromSlice(slice []string) (*Model, error) { + return NewWithParams(slice...) +} + +// NewWithCmd ... +func NewWithCmd(cmd *exec.Cmd) *Model { + return &Model{ + cmd: cmd, + } +} + +// GetCmd ... +func (m *Model) GetCmd() *exec.Cmd { + return m.cmd +} + +// SetDir ... +func (m *Model) SetDir(dir string) *Model { + m.cmd.Dir = dir + return m +} + +// SetEnvs ... +func (m *Model) SetEnvs(envs ...string) *Model { + m.cmd.Env = envs + return m +} + +// AppendEnvs - appends the envs to the current os.Environ() +// Calling this multiple times will NOT appens the envs one by one, +// only the last "envs" set will be appended to os.Environ()! +func (m *Model) AppendEnvs(envs ...string) *Model { + return m.SetEnvs(append(os.Environ(), envs...)...) +} + +// SetStdin ... +func (m *Model) SetStdin(in io.Reader) *Model { + m.cmd.Stdin = in + return m +} + +// SetStdout ... +func (m *Model) SetStdout(out io.Writer) *Model { + m.cmd.Stdout = out + return m +} + +// SetStderr ... +func (m *Model) SetStderr(err io.Writer) *Model { + m.cmd.Stderr = err + return m +} + +// Run ... +func (m Model) Run() error { + return m.cmd.Run() +} + +// RunAndReturnExitCode ... +func (m Model) RunAndReturnExitCode() (int, error) { + return RunCmdAndReturnExitCode(m.cmd) +} + +// RunAndReturnTrimmedOutput ... +func (m Model) RunAndReturnTrimmedOutput() (string, error) { + return RunCmdAndReturnTrimmedOutput(m.cmd) +} + +// RunAndReturnTrimmedCombinedOutput ... +func (m Model) RunAndReturnTrimmedCombinedOutput() (string, error) { + return RunCmdAndReturnTrimmedCombinedOutput(m.cmd) +} + +// PrintableCommandArgs ... +func (m Model) PrintableCommandArgs() string { + return PrintableCommandArgs(false, m.cmd.Args) +} + +// ---------- + +// PrintableCommandArgs ... +func PrintableCommandArgs(isQuoteFirst bool, fullCommandArgs []string) string { + cmdArgsDecorated := []string{} + for idx, anArg := range fullCommandArgs { + quotedArg := strconv.Quote(anArg) + if idx == 0 && !isQuoteFirst { + quotedArg = anArg + } + cmdArgsDecorated = append(cmdArgsDecorated, quotedArg) + } + + return strings.Join(cmdArgsDecorated, " ") +} + +// RunCmdAndReturnExitCode ... +func RunCmdAndReturnExitCode(cmd *exec.Cmd) (exitCode int, err error) { + err = cmd.Run() + exitCode = cmd.ProcessState.ExitCode() + return +} + +// RunCmdAndReturnTrimmedOutput ... +func RunCmdAndReturnTrimmedOutput(cmd *exec.Cmd) (string, error) { + outBytes, err := cmd.Output() + outStr := string(outBytes) + return strings.TrimSpace(outStr), err +} + +// RunCmdAndReturnTrimmedCombinedOutput ... +func RunCmdAndReturnTrimmedCombinedOutput(cmd *exec.Cmd) (string, error) { + outBytes, err := cmd.CombinedOutput() + outStr := string(outBytes) + return strings.TrimSpace(outStr), err +} + +// RunCommandWithReaderAndWriters ... +func RunCommandWithReaderAndWriters(inReader io.Reader, outWriter, errWriter io.Writer, name string, args ...string) error { + cmd := exec.Command(name, args...) + cmd.Stdin = inReader + cmd.Stdout = outWriter + cmd.Stderr = errWriter + return cmd.Run() +} + +// RunCommandWithWriters ... +func RunCommandWithWriters(outWriter, errWriter io.Writer, name string, args ...string) error { + cmd := exec.Command(name, args...) + cmd.Stdout = outWriter + cmd.Stderr = errWriter + return cmd.Run() +} + +// RunCommandInDirWithEnvsAndReturnExitCode ... +func RunCommandInDirWithEnvsAndReturnExitCode(envs []string, dir, name string, args ...string) (int, error) { + cmd := exec.Command(name, args...) + cmd.Stdin = os.Stdin + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + if dir != "" { + cmd.Dir = dir + } + if len(envs) > 0 { + cmd.Env = envs + } + + return RunCmdAndReturnExitCode(cmd) +} + +// RunCommandInDirAndReturnExitCode ... +func RunCommandInDirAndReturnExitCode(dir, name string, args ...string) (int, error) { + return RunCommandInDirWithEnvsAndReturnExitCode([]string{}, dir, name, args...) +} + +// RunCommandWithEnvsAndReturnExitCode ... +func RunCommandWithEnvsAndReturnExitCode(envs []string, name string, args ...string) (int, error) { + return RunCommandInDirWithEnvsAndReturnExitCode(envs, "", name, args...) +} + +// RunCommandInDir ... +func RunCommandInDir(dir, name string, args ...string) error { + cmd := exec.Command(name, args...) + cmd.Stdin = os.Stdin + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + if dir != "" { + cmd.Dir = dir + } + return cmd.Run() +} + +// RunCommand ... +func RunCommand(name string, args ...string) error { + return RunCommandInDir("", name, args...) +} + +// RunCommandAndReturnStdout .. +func RunCommandAndReturnStdout(name string, args ...string) (string, error) { + cmd := exec.Command(name, args...) + return RunCmdAndReturnTrimmedOutput(cmd) +} + +// RunCommandInDirAndReturnCombinedStdoutAndStderr ... +func RunCommandInDirAndReturnCombinedStdoutAndStderr(dir, name string, args ...string) (string, error) { + cmd := exec.Command(name, args...) + if dir != "" { + cmd.Dir = dir + } + return RunCmdAndReturnTrimmedCombinedOutput(cmd) +} + +// RunCommandAndReturnCombinedStdoutAndStderr .. +func RunCommandAndReturnCombinedStdoutAndStderr(name string, args ...string) (string, error) { + return RunCommandInDirAndReturnCombinedStdoutAndStderr("", name, args...) +} + +// RunBashCommand ... +func RunBashCommand(cmdStr string) error { + return RunCommand("bash", "-c", cmdStr) +} + +// RunBashCommandLines ... +func RunBashCommandLines(cmdLines []string) error { + for _, aLine := range cmdLines { + if err := RunCommand("bash", "-c", aLine); err != nil { + return err + } + } + return nil +} diff --git a/vendor/github.com/bitrise-io/go-utils/command/file.go b/vendor/github.com/bitrise-io/go-utils/command/file.go new file mode 100644 index 0000000..6b22172 --- /dev/null +++ b/vendor/github.com/bitrise-io/go-utils/command/file.go @@ -0,0 +1,69 @@ +package command + +import ( + "errors" + "os" + "strings" + + "github.com/bitrise-io/go-utils/pathutil" +) + +// CopyFile ... +func CopyFile(src, dst string) error { + // replace with a pure Go implementation? + // Golang proposal was: https://go-review.googlesource.com/#/c/1591/5/src/io/ioutil/ioutil.go + isDir, err := pathutil.IsDirExists(src) + if err != nil { + return err + } + if isDir { + return errors.New("Source is a directory: " + src) + } + args := []string{src, dst} + return RunCommand("rsync", args...) +} + +// CopyDir ... +func CopyDir(src, dst string, isOnlyContent bool) error { + if isOnlyContent && !strings.HasSuffix(src, "/") { + src = src + "/" + } + args := []string{"-ar", src, dst} + return RunCommand("rsync", args...) +} + +// RemoveDir ... +// Deprecated: use RemoveAll instead. +func RemoveDir(dirPth string) error { + if exist, err := pathutil.IsPathExists(dirPth); err != nil { + return err + } else if exist { + if err := os.RemoveAll(dirPth); err != nil { + return err + } + } + return nil +} + +// RemoveFile ... +// Deprecated: use RemoveAll instead. +func RemoveFile(pth string) error { + if exist, err := pathutil.IsPathExists(pth); err != nil { + return err + } else if exist { + if err := os.Remove(pth); err != nil { + return err + } + } + return nil +} + +// RemoveAll removes recursively every file on the given paths. +func RemoveAll(pths ...string) error { + for _, pth := range pths { + if err := os.RemoveAll(pth); err != nil { + return err + } + } + return nil +} diff --git a/vendor/github.com/bitrise-io/go-utils/command/zip.go b/vendor/github.com/bitrise-io/go-utils/command/zip.go new file mode 100644 index 0000000..b3e899c --- /dev/null +++ b/vendor/github.com/bitrise-io/go-utils/command/zip.go @@ -0,0 +1,115 @@ +package command + +import ( + "archive/zip" + "errors" + "io" + "log" + "net/http" + "os" + "path/filepath" + + "github.com/bitrise-io/go-utils/pathutil" +) + +// UnZIP ... +func UnZIP(src, dest string) error { + r, err := zip.OpenReader(src) + if err != nil { + return err + } + defer func() { + if err := r.Close(); err != nil { + log.Fatal(err) + } + }() + + if err := os.MkdirAll(dest, 0755); err != nil { + return err + } + + // Closure to address file descriptors issue with all the deferred .Close() methods + extractAndWriteFile := func(f *zip.File) error { + rc, err := f.Open() + if err != nil { + return err + } + defer func() { + if err := rc.Close(); err != nil { + log.Fatal(err) + } + }() + + path := filepath.Join(dest, f.Name) + + if f.FileInfo().IsDir() { + if err := os.MkdirAll(path, f.Mode()); err != nil { + return err + } + } else { + f, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, f.Mode()) + if err != nil { + return err + } + defer func() { + if err := f.Close(); err != nil { + log.Fatal(err) + } + }() + + if _, err = io.Copy(f, rc); err != nil { + return err + } + } + return nil + } + + for _, f := range r.File { + if err := extractAndWriteFile(f); err != nil { + return err + } + } + return nil +} + +// DownloadAndUnZIP ... +func DownloadAndUnZIP(url, pth string) error { + tmpDir, err := pathutil.NormalizedOSTempDirPath("") + if err != nil { + return err + } + srcFilePath := tmpDir + "/target.zip" + srcFile, err := os.Create(srcFilePath) + if err != nil { + return err + } + defer func() { + if err := srcFile.Close(); err != nil { + log.Fatal("Failed to close srcFile:", err) + } + if err := os.Remove(srcFilePath); err != nil { + log.Fatal("Failed to remove srcFile:", err) + } + }() + + response, err := http.Get(url) + if err != nil { + return err + } + defer func() { + if err := response.Body.Close(); err != nil { + log.Fatal("Failed to close response body:", err) + } + }() + + if response.StatusCode != http.StatusOK { + errorMsg := "Failed to download target from: " + url + return errors.New(errorMsg) + } + + if _, err := io.Copy(srcFile, response.Body); err != nil { + return err + } + + return UnZIP(srcFilePath, pth) +} diff --git a/vendor/github.com/bitrise-io/go-utils/parseutil/parseutil.go b/vendor/github.com/bitrise-io/go-utils/parseutil/parseutil.go deleted file mode 100644 index 08cec36..0000000 --- a/vendor/github.com/bitrise-io/go-utils/parseutil/parseutil.go +++ /dev/null @@ -1,95 +0,0 @@ -package parseutil - -import ( - "errors" - "fmt" - "strconv" - "strings" - - "github.com/bitrise-io/go-utils/pointers" -) - -// ParseBool ... -func ParseBool(userInputStr string) (bool, error) { - if userInputStr == "" { - return false, errors.New("No string to parse") - } - userInputStr = strings.TrimSpace(userInputStr) - - lowercased := strings.ToLower(userInputStr) - if lowercased == "yes" || lowercased == "y" { - return true, nil - } - if lowercased == "no" || lowercased == "n" { - return false, nil - } - return strconv.ParseBool(lowercased) -} - -// CastToString ... -func CastToString(value interface{}) string { - casted, ok := value.(string) - - if !ok { - castedStr := fmt.Sprintf("%v", value) - casted = castedStr - } - - return casted -} - -// CastToStringPtr ... -func CastToStringPtr(value interface{}) *string { - castedValue := CastToString(value) - return pointers.NewStringPtr(castedValue) -} - -// CastToBool ... -func CastToBool(value interface{}) (bool, bool) { - casted, ok := value.(bool) - - if !ok { - castedStr := CastToString(value) - - castedBool, err := ParseBool(castedStr) - if err != nil { - return false, false - } - - casted = castedBool - } - - return casted, true -} - -// CastToBoolPtr ... -func CastToBoolPtr(value interface{}) (*bool, bool) { - castedValue, ok := CastToBool(value) - if !ok { - return nil, false - } - return pointers.NewBoolPtr(castedValue), true -} - -// CastToMapStringInterface ... -func CastToMapStringInterface(value interface{}) (map[string]interface{}, bool) { - castedValue, ok := value.(map[interface{}]interface{}) - desiredMap := map[string]interface{}{} - for key, value := range castedValue { - keyStr, ok := key.(string) - if !ok { - return map[string]interface{}{}, false - } - desiredMap[keyStr] = value - } - return desiredMap, ok -} - -// CastToMapStringInterfacePtr ... -func CastToMapStringInterfacePtr(value interface{}) (*map[string]interface{}, bool) { - casted, ok := CastToMapStringInterface(value) - if !ok { - return nil, false - } - return pointers.NewMapStringInterfacePtr(casted), true -} diff --git a/vendor/github.com/bitrise-io/go-utils/pointers/pointers.go b/vendor/github.com/bitrise-io/go-utils/pointers/pointers.go deleted file mode 100644 index e26647d..0000000 --- a/vendor/github.com/bitrise-io/go-utils/pointers/pointers.go +++ /dev/null @@ -1,98 +0,0 @@ -package pointers - -import "time" - -// NewBoolPtr ... -func NewBoolPtr(val bool) *bool { - ptrValue := new(bool) - *ptrValue = val - return ptrValue -} - -// NewStringPtr ... -func NewStringPtr(val string) *string { - ptrValue := new(string) - *ptrValue = val - return ptrValue -} - -// NewTimePtr ... -func NewTimePtr(val time.Time) *time.Time { - ptrValue := new(time.Time) - *ptrValue = val - return ptrValue -} - -// NewIntPtr ... -func NewIntPtr(val int) *int { - ptrValue := new(int) - *ptrValue = val - return ptrValue -} - -// NewInt64Ptr ... -func NewInt64Ptr(val int64) *int64 { - ptrValue := new(int64) - *ptrValue = val - return ptrValue -} - -// NewMapStringInterfacePtr ... -func NewMapStringInterfacePtr(val map[string]interface{}) *map[string]interface{} { - ptrValue := new(map[string]interface{}) - *ptrValue = map[string]interface{}{} - for key, value := range val { - (*ptrValue)[key] = value - } - return ptrValue -} - -// ------------------------------------------------------ -// --- Safe Getters - -// Bool ... -func Bool(val *bool) bool { - return BoolWithDefault(val, false) -} - -// BoolWithDefault ... -func BoolWithDefault(val *bool, defaultValue bool) bool { - if val == nil { - return defaultValue - } - return *val -} - -// String ... -func String(val *string) string { - return StringWithDefault(val, "") -} - -// StringWithDefault ... -func StringWithDefault(val *string, defaultValue string) string { - if val == nil { - return defaultValue - } - return *val -} - -// TimeWithDefault ... -func TimeWithDefault(val *time.Time, defaultValue time.Time) time.Time { - if val == nil { - return defaultValue - } - return *val -} - -// Int ... -func Int(val *int) int { - return IntWithDefault(val, 0) -} - -// IntWithDefault ... -func IntWithDefault(val *int, defaultValue int) int { - if val == nil { - return defaultValue - } - return *val -} diff --git a/vendor/github.com/bitrise-io/go-utils/v2/pathutil/pathutil.go b/vendor/github.com/bitrise-io/go-utils/v2/pathutil/pathutil.go index aaa00df..78d6aa6 100644 --- a/vendor/github.com/bitrise-io/go-utils/v2/pathutil/pathutil.go +++ b/vendor/github.com/bitrise-io/go-utils/v2/pathutil/pathutil.go @@ -34,6 +34,7 @@ func (pathProvider) CreateTempDir(prefix string) (dir string, err error) { // PathChecker ... type PathChecker interface { IsPathExists(pth string) (bool, error) + IsDirExists(pth string) (bool, error) } type pathChecker struct{} @@ -49,9 +50,15 @@ func (c pathChecker) IsPathExists(pth string) (bool, error) { return isExists, err } +// IsDirExists ... +func (c pathChecker) IsDirExists(pth string) (bool, error) { + info, isExists, err := c.genericIsPathExists(pth) + return isExists && info.IsDir(), err +} + func (pathChecker) genericIsPathExists(pth string) (os.FileInfo, bool, error) { if pth == "" { - return nil, false, errors.New("No path provided") + return nil, false, errors.New("no path provided") } fileInf, err := os.Lstat(pth) diff --git a/vendor/github.com/bitrise-io/go-utils/ziputil/ziputil.go b/vendor/github.com/bitrise-io/go-utils/ziputil/ziputil.go new file mode 100644 index 0000000..e96ae01 --- /dev/null +++ b/vendor/github.com/bitrise-io/go-utils/ziputil/ziputil.go @@ -0,0 +1,113 @@ +package ziputil + +import ( + "fmt" + "log" + "os" + "path/filepath" + + "github.com/bitrise-io/go-utils/command" + "github.com/bitrise-io/go-utils/pathutil" +) + +// ZipDir ... +func ZipDir(sourceDirPth, destinationZipPth string, isContentOnly bool) error { + if exist, err := pathutil.IsDirExists(sourceDirPth); err != nil { + return err + } else if !exist { + return fmt.Errorf("dir (%s) not exist", sourceDirPth) + } + + workDir := filepath.Dir(sourceDirPth) + zipTarget := filepath.Base(sourceDirPth) + + if isContentOnly { + workDir = sourceDirPth + zipTarget = "." + } + + return internalZipDir(destinationZipPth, zipTarget, workDir) + +} + +// ZipDirs ... +func ZipDirs(sourceDirPths []string, destinationZipPth string) error { + for _, path := range sourceDirPths { + if exist, err := pathutil.IsDirExists(path); err != nil { + return err + } else if !exist { + return fmt.Errorf("directory (%s) not exist", path) + } + } + + tempDir, err := pathutil.NormalizedOSTempDirPath("zip") + if err != nil { + return err + } + + defer func() { + if err = os.RemoveAll(tempDir); err != nil { + log.Fatal(err) + } + }() + + for _, path := range sourceDirPths { + err := command.CopyDir(path, tempDir, false) + if err != nil { + return err + } + } + + return internalZipDir(destinationZipPth, ".", tempDir) +} + +func internalZipDir(destinationZipPth, zipTarget, workDir string) error { + // -r - Travel the directory structure recursively + // -T - Test the integrity of the new zip file + // -y - Store symbolic links as such in the zip archive, instead of compressing and storing the file referred to by the link + cmd := command.New("/usr/bin/zip", "-rTy", destinationZipPth, zipTarget) + cmd.SetDir(workDir) + if out, err := cmd.RunAndReturnTrimmedCombinedOutput(); err != nil { + return fmt.Errorf("command: (%s) failed, output: %s, error: %s", cmd.PrintableCommandArgs(), out, err) + } + + return nil +} + +// ZipFile ... +func ZipFile(sourceFilePth, destinationZipPth string) error { + return ZipFiles([]string{sourceFilePth}, destinationZipPth) +} + +// ZipFiles ... +func ZipFiles(sourceFilePths []string, destinationZipPth string) error { + for _, path := range sourceFilePths { + if exist, err := pathutil.IsPathExists(path); err != nil { + return err + } else if !exist { + return fmt.Errorf("file (%s) not exist", path) + } + } + + // -T - Test the integrity of the new zip file + // -y - Store symbolic links as such in the zip archive, instead of compressing and storing the file referred to by the link + // -j - Do not recreate the directory structure inside the zip. Kind of equivalent of copying all the files in one folder and zipping it. + parameters := []string{"-Tyj", destinationZipPth} + parameters = append(parameters, sourceFilePths...) + cmd := command.New("/usr/bin/zip", parameters...) + if out, err := cmd.RunAndReturnTrimmedCombinedOutput(); err != nil { + return fmt.Errorf("command: (%s) failed, output: %s, error: %s", cmd.PrintableCommandArgs(), out, err) + } + + return nil +} + +// UnZip ... +func UnZip(zip, intoDir string) error { + cmd := command.New("/usr/bin/unzip", zip, "-d", intoDir) + if out, err := cmd.RunAndReturnTrimmedCombinedOutput(); err != nil { + return fmt.Errorf("command: (%s) failed, output: %s, error: %s", cmd.PrintableCommandArgs(), out, err) + } + + return nil +} diff --git a/vendor/modules.txt b/vendor/modules.txt index 0dc5007..f774153 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -1,17 +1,17 @@ -# github.com/bitrise-io/go-steputils/v2 v2.0.0-alpha.12 -## explicit; go 1.16 +# github.com/bitrise-io/go-steputils/v2 v2.0.0-alpha.15 +## explicit; go 1.17 github.com/bitrise-io/go-steputils/v2/cache github.com/bitrise-io/go-steputils/v2/cache/compression github.com/bitrise-io/go-steputils/v2/cache/keytemplate github.com/bitrise-io/go-steputils/v2/cache/network +github.com/bitrise-io/go-steputils/v2/export github.com/bitrise-io/go-steputils/v2/stepconf # github.com/bitrise-io/go-utils v1.0.1 ## explicit; go 1.13 -github.com/bitrise-io/go-utils/colorstring -github.com/bitrise-io/go-utils/parseutil +github.com/bitrise-io/go-utils/command github.com/bitrise-io/go-utils/pathutil -github.com/bitrise-io/go-utils/pointers -# github.com/bitrise-io/go-utils/v2 v2.0.0-alpha.12 +github.com/bitrise-io/go-utils/ziputil +# github.com/bitrise-io/go-utils/v2 v2.0.0-alpha.13 ## explicit; go 1.17 github.com/bitrise-io/go-utils/v2/analytics github.com/bitrise-io/go-utils/v2/command