Tool 692 caching (#53)

* go mod init, typo fixes, vendor dir removed * clean and refactor cache funcs * fastlane cache * enable verbose log for integration tests; cacheing wire in * vendore deps * flutter sample app and cache output test * added cert and profile installer step * dep-update -> vendor-update as dependency management migrated to go modules; cocoapodsDeps and carthageDeps code duplication removed; debug logs added; redundant alias removed; cache collide fixed
bitrise-steplib · Sep 5, 2019 · 410fc35 · 410fc35
1 parent 4fcb9f5
commit 410fc35
Show file tree

Hide file tree

Showing 69 changed files with 17,566 additions and 148 deletions.
diff --git a/Gopkg.lock b/Gopkg.lock
diff --git a/Gopkg.toml b/Gopkg.toml
diff --git a/bitrise.yml b/bitrise.yml
@@ -1,16 +1,13 @@
 format_version: 7
 default_step_lib_source: https://github.com/bitrise-io/bitrise-steplib.git
 
-app:
-  envs:
-  - SAMPLE_APP_URL: "https://github.com/bitrise-samples/sample-apps-fastlane-test.git"
-  - BITRISE_PROJECT_PATH: ./sample-apps-fastlane-test.xcodeproj
-  - BITRISE_SCHEME: sample-apps-fastlane-test
-
 workflows:
   # ----------------------------------------------------------------
   # --- workflow to test this step
   ci:
+    envs:
+    - SAMPLE_APP_URL: "https://github.com/bitrise-io/sample-apps-flutter-veggieseasons.git"
+    - BRANCH: fastlane
     before-run:
     - audit-this-step
     steps:
@@ -29,13 +26,34 @@ workflows:
         - is_create_path: true
     - script:
         inputs:
-        - content: git clone $SAMPLE_APP_URL .
+        - content: |-
+            set -ex
+            git clone $SAMPLE_APP_URL -b $BRANCH .
+    - certificate-and-profile-installer:
     - path::./:
         inputs:
-        - lane: tests
+        - lane: build
         - work_dir: ./
+        - verbose_log: "yes"
+    - script:
+        inputs:
+        - content: |-
+            set -ex
+            if [[ -z "$BITRISE_CACHE_EXCLUDE_PATHS" ]] ; then
+                echo "BITRISE_CACHE_EXCLUDE_PATHS is empty"
+                exit 1
+            fi
+
+            if [[ -z "$BITRISE_CACHE_INCLUDE_PATHS" ]] ; then
+                echo "BITRISE_CACHE_INCLUDE_PATHS is empty"
+                exit 1
+            fi
 
   fastlane-session-test:
+    envs:
+    - SAMPLE_APP_URL: "https://github.com/bitrise-samples/sample-apps-fastlane-test.git"
+    - BITRISE_PROJECT_PATH: ./sample-apps-fastlane-test.xcodeproj
+    - BITRISE_SCHEME: sample-apps-fastlane-test
     steps:
     - script:
         inputs:
@@ -57,23 +75,22 @@ workflows:
         inputs:
         - lane: release
         - work_dir: ./
+        - verbose_log: "yes"
 
   # ----------------------------------------------------------------
   # --- Utility workflows
-  dep-update:
-    title: Dep update
+  vendor-update:
+    title: Vendor update
     description: |
-      Used for updating bitrise dependencies with dep
+      Used for updating the vendored dependencies
     steps:
     - script:
-        title: Dependency update
+        title: Vendor update
         inputs:
         - content: |-
             #!/bin/bash
             set -ex
-            go get -u -v github.com/golang/dep/cmd/dep
-            dep ensure -v
-            dep ensure -v -update
+            go mod vendor
 
   # ----------------------------------------------------------------
   # --- workflows to Share this step into a Step Library

diff --git a/cache.go b/cache.go
@@ -0,0 +1,92 @@
+package main
+
+import (
+	"fmt"
+	"path/filepath"
+	"strings"
+
+	"github.com/bitrise-io/bitrise-init/scanners/android"
+	"github.com/bitrise-io/bitrise-init/utility"
+	"github.com/bitrise-io/go-utils/log"
+	"github.com/bitrise-io/go-utils/pathutil"
+	androidCache "github.com/bitrise-steplib/bitrise-step-android-unit-test/cache"
+)
+
+type depsFunc func(dir string) ([]string, []string, error)
+
+var depsFuncs = []depsFunc{
+	cocoapodsDeps,
+	carthageDeps,
+	androidDeps,
+}
+
+func iosDeps(dir string, buildDirName, lockFileName string) ([]string, []string, error) {
+	files, err := utility.ListPathInDirSortedByComponents(dir, false)
+	if err != nil {
+		return nil, nil, fmt.Errorf("failed to search for files in (%s), error: %s", dir, err)
+	}
+
+	locks, err := utility.FilterPaths(files, utility.BaseFilter(lockFileName, true))
+	if err != nil {
+		return nil, nil, err
+	}
+
+	buildDirToLockFile := map[string]string{}
+	for _, lock := range locks {
+		buildDir := filepath.Join(filepath.Dir(lock), buildDirName)
+		exist, err := pathutil.IsPathExists(buildDir)
+		if err != nil {
+			return nil, nil, err
+		}
+
+		if exist {
+			buildDirToLockFile[buildDir] = lock
+		}
+	}
+
+	if len(buildDirToLockFile) > 1 {
+		var locks []string
+		for _, lock := range buildDirToLockFile {
+			locks = append(locks, lock)
+		}
+		log.Debugf("Multiple %s found: %s", lockFileName, strings.Join(locks, ", "))
+	}
+
+	var include []string
+	for buildDir, lockFile := range buildDirToLockFile {
+		include = append(include, fmt.Sprintf("%s -> %s", buildDir, lockFile))
+	}
+
+	return include, nil, nil
+}
+
+func cocoapodsDeps(dir string) ([]string, []string, error) {
+	return iosDeps(dir, "Pods", "Podfile.lock")
+}
+
+func carthageDeps(dir string) ([]string, []string, error) {
+	return iosDeps(dir, "Carthage", "Cartfile.resolved")
+}
+
+func androidDeps(dir string) ([]string, []string, error) {
+	scanner := android.NewScanner()
+	detected, err := scanner.DetectPlatform(dir)
+	if err != nil {
+		return nil, nil, err
+	}
+	log.Debugf("android platform detected: %v", detected)
+
+	var include []string
+	var exclude []string
+	for _, dir := range scanner.ProjectRoots {
+		i, e, err := androidCache.Deps(dir, androidCache.LevelDeps)
+		if err != nil {
+			return nil, nil, err
+		}
+
+		include = append(include, i...)
+		exclude = append(exclude, e...)
+	}
+
+	return include, exclude, err
+}
diff --git a/go.mod b/go.mod
@@ -0,0 +1,25 @@
+module github.com/bitrise-io/steps-fastlane
+
+go 1.12
+
+require (
+	github.com/beevik/etree v1.1.0 // indirect
+	github.com/bitrise-io/bitrise v0.0.0-20190829132137-518e23eca82d // indirect
+	github.com/bitrise-io/bitrise-init v0.0.0-20190813135927-7ff42ba329cb
+	github.com/bitrise-io/envman v0.0.0-20190813133714-27a300a1ed43 // indirect
+	github.com/bitrise-io/go-steputils v0.0.0-20190806143347-f540824d77df
+	github.com/bitrise-io/go-utils v0.0.0-20190814105345-ad86bf775472
+	github.com/bitrise-io/go-xcode v0.0.0-20190718082554-07ccf582b6b1 // indirect
+	github.com/bitrise-io/goinp v0.0.0-20190611131639-bd18a8681e27 // indirect
+	github.com/bitrise-io/stepman v0.0.0-20190813144014-10564a4888a6 // indirect
+	github.com/bitrise-io/xcode-project v0.0.0-20190829132102-743450f7d0f0
+	github.com/bitrise-steplib/bitrise-step-android-unit-test v0.0.0-20190902203028-ff8e682d8645
+	github.com/bitrise-steplib/steps-deploy-to-itunesconnect-deliver v0.0.0-20190815132021-09bfb54640c4
+	github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51
+	github.com/pkg/errors v0.8.1 // indirect
+	github.com/ryanuber/go-glob v1.0.0 // indirect
+	github.com/stretchr/testify v1.4.0 // indirect
+	golang.org/x/crypto v0.0.0-20190829043050-9756ffdc2472 // indirect
+	golang.org/x/text v0.3.2 // indirect
+	howett.net/plist v0.0.0-20181124034731-591f970eefbb // indirect
+)
diff --git a/go.sum b/go.sum
@@ -0,0 +1,61 @@
+github.com/beevik/etree v1.1.0 h1:T0xke/WvNtMoCqgzPhkX2r4rjY3GDZFi+FjpRZY2Jbs=
+github.com/beevik/etree v1.1.0/go.mod h1:r8Aw8JqVegEf0w2fDnATrX9VpkMcyFeM0FhwO62wh+A=
+github.com/bitrise-io/bitrise v0.0.0-20190829132137-518e23eca82d h1:yx3kfOeDr7bfsGE7r3Jsz5UVMjQKuy5YQIyT+criGAg=
+github.com/bitrise-io/bitrise v0.0.0-20190829132137-518e23eca82d/go.mod h1:+8ruIdZ7mWWfrd/Of38jmN50x6Y80H7SMmwf78HhBno=
+github.com/bitrise-io/bitrise-init v0.0.0-20190813135927-7ff42ba329cb h1:e1cLjVTYW1sGCiTQ9uEJqBqb00mq5HoYg0dX7q56A5s=
+github.com/bitrise-io/bitrise-init v0.0.0-20190813135927-7ff42ba329cb/go.mod h1:6tVeJwc47w4nYwexFkYLRwQ2gUnfL568ySfMTLBsONo=
+github.com/bitrise-io/envman v0.0.0-20190813133714-27a300a1ed43 h1:BRhP0pS7NPK4iFKlQszhbHoAwD6m88GFys2w+bgwiN8=
+github.com/bitrise-io/envman v0.0.0-20190813133714-27a300a1ed43/go.mod h1:m8pTp1o3Sw9uzDxb1WRm5IBRnMau2iOvPMSnRCAhQNI=
+github.com/bitrise-io/go-steputils v0.0.0-20190806143347-f540824d77df h1:UaDw6nAsI2jlGjNqhdMUbk0xhJJ+iQZ1buEXHRKrtU8=
+github.com/bitrise-io/go-steputils v0.0.0-20190806143347-f540824d77df/go.mod h1:GXgBV3Frd3qcnsg+NryQTyx1CHjZHr/2w7Bx4WAcB4o=
+github.com/bitrise-io/go-utils v0.0.0-20190814105345-ad86bf775472 h1:jM4pIBkBUOucinQ2IdlfbpIZxJqgS/k1TZZrENFej8A=
+github.com/bitrise-io/go-utils v0.0.0-20190814105345-ad86bf775472/go.mod h1:Pp48eQd8BSNEzWWTBxdFUd/ufS5b5Bgkmt9D5NedjWg=
+github.com/bitrise-io/go-xcode v0.0.0-20190718082554-07ccf582b6b1 h1:Om1A4iSAyR4G8sEi1VYRfJrEHbmskSpeSeK0+uAVqqc=
+github.com/bitrise-io/go-xcode v0.0.0-20190718082554-07ccf582b6b1/go.mod h1:EFvdJln2CESNPSz2+ChAGOtfHNN/l56csk1nuRnz8nI=
+github.com/bitrise-io/goinp v0.0.0-20190611131639-bd18a8681e27 h1:NuGIfwKcZvdR1RT4sQ32kE8h35w9XQjQrrCJPJqS4ek=
+github.com/bitrise-io/goinp v0.0.0-20190611131639-bd18a8681e27/go.mod h1:G0M1sK06a1l0KAg4rQei7q5dDKC/JrZoaXFqak91osU=
+github.com/bitrise-io/stepman v0.0.0-20190813144014-10564a4888a6 h1:/GnB2kEaO/6KSMfGpmntvohfacADpWFcf8iCPLsZqh4=
+github.com/bitrise-io/stepman v0.0.0-20190813144014-10564a4888a6/go.mod h1:hGCjd8leP411yt5QkQi+VBNWGxIZ4H02LNIVeKBeMUk=
+github.com/bitrise-io/xcode-project v0.0.0-20190829132102-743450f7d0f0 h1:Ft6zMZVUwyzJPyqgDMROm6tHaLyWabs27jr8psYHKyU=
+github.com/bitrise-io/xcode-project v0.0.0-20190829132102-743450f7d0f0/go.mod h1:L7ue7ix2GIAf0pQovKmWub7l956nsHU68kk5fU4KczQ=
+github.com/bitrise-steplib/bitrise-step-android-unit-test v0.0.0-20190830131210-9d5c1d4e8630 h1:cT8ax87pLOCCE+OaEhT2VXQiMFc1V44xfjENA/jF6r0=
+github.com/bitrise-steplib/bitrise-step-android-unit-test v0.0.0-20190830131210-9d5c1d4e8630/go.mod h1:0yqqJw+MqwsfHKq4pL90IoSYskLF91oDCyyZYnIehWA=
+github.com/bitrise-steplib/bitrise-step-android-unit-test v0.0.0-20190902203028-ff8e682d8645 h1:9molXzIAxnKStwV78lt7MSgUQwxIWl4+r9/oYTQA7no=
+github.com/bitrise-steplib/bitrise-step-android-unit-test v0.0.0-20190902203028-ff8e682d8645/go.mod h1:0yqqJw+MqwsfHKq4pL90IoSYskLF91oDCyyZYnIehWA=
+github.com/bitrise-steplib/steps-deploy-to-itunesconnect-deliver v0.0.0-20190815132021-09bfb54640c4 h1:rzEqvBYa+WJ44mAsiD4DAW4zX6vLjdwAg3nvJSqkJc4=
+github.com/bitrise-steplib/steps-deploy-to-itunesconnect-deliver v0.0.0-20190815132021-09bfb54640c4/go.mod h1:mm5WIhfyTnf89pNO+4ylSsUAbEh8JGSpV3F7V0XWB4U=
+github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
+github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNUXsshfwJMBgNA0RU6/i7WVaAegv3PtuIHPMs=
+github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8=
+github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
+github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
+github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/ryanuber/go-glob v1.0.0 h1:iQh3xXAumdQ+4Ufa5b25cRpC5TYKlno6hsv6Cb3pkBk=
+github.com/ryanuber/go-glob v1.0.0/go.mod h1:807d1WSdnB0XRJzKNil9Om6lcp/3a0v4qIHxIXzX/Yc=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
+github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20190829043050-9756ffdc2472 h1:Gv7RPwsi3eZ2Fgewe3CBsuOebPwO27PoXzRpJPsvSSM=
+golang.org/x/crypto v0.0.0-20190829043050-9756ffdc2472/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190412213103-97732733099d h1:+R4KGOnez64A81RvjARKc4UT5/tI9ujCIVX+P5KiHuI=
+golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
+golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
+gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+howett.net/plist v0.0.0-20181124034731-591f970eefbb h1:jhnBjNi9UFpfpl8YZhA9CrOqpnJdvzuiHsl/dnxl11M=
+howett.net/plist v0.0.0-20181124034731-591f970eefbb/go.mod h1:vMygbs4qMhSZSc4lCUl2OEE+rDiIIJAIdR4m7MiMcm0=