diff --git a/.travis.yml b/.travis.yml index 28d4f263b..20446450b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,5 +1,7 @@ language: go go: - '1.10' +install: +- curl -fSL --retry 5 https://github.com/gobuffalo/packr/releases/download/v1.13.1/packr_1.13.1_linux_amd64.tar.gz | sudo tar zx -C /usr/bin/ script: - make diff --git a/Gopkg.lock b/Gopkg.lock index 581d11e88..49148d146 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -2,48 +2,63 @@ [[projects]] + digest = "1:55388fd080150b9a072912f97b1f5891eb0b50df43401f8b75fb4273d3fec9fc" name = "github.com/Masterminds/semver" packages = ["."] + pruneopts = "UT" revision = "c7af12943936e8c39859482e61f0574c2fd7fc75" version = "v1.4.2" [[projects]] + digest = "1:46a054a232ea2b7f0a35398d682b433d26ba9975fce9197b1784824402059f5b" name = "github.com/Masterminds/sprig" packages = ["."] + pruneopts = "UT" revision = "6b2a58267f6a8b1dc8e2eb5519b984008fa85e8c" version = "v2.15.0" [[projects]] + digest = "1:dc2ba13235a4c8f80a40599575a3f10acb4736cb372f1a68b88333c06564471d" name = "github.com/agext/levenshtein" packages = ["."] + pruneopts = "UT" revision = "5f10fee965225ac1eecdc234c09daf5cd9e7f7b6" version = "v1.2.1" [[projects]] + digest = "1:8f5416c7f59da8600725ae1ff00a99af1da8b04c211ae6f3c8f8bcab0164f650" name = "github.com/aokoli/goutils" packages = ["."] + pruneopts = "UT" revision = "3391d3790d23d03408670993e957e8f408993c34" version = "v1.0.1" [[projects]] branch = "master" + digest = "1:ef98942770803ae37c4787ad6bf70e401c99834dfba5e3034b97ba7c24e66c10" name = "github.com/apparentlymart/go-cidr" packages = ["cidr"] + pruneopts = "UT" revision = "2bd8b58cf4275aeb086ade613de226773e29e853" [[projects]] branch = "master" + digest = "1:2bf7da77216264bb61bd8cb5f3380a03ab509e8a826fe359008d4a6ba0789b13" name = "github.com/apparentlymart/go-textseg" packages = ["textseg"] + pruneopts = "UT" revision = "b836f5c4d331d1945a2fead7188db25432d73b69" [[projects]] branch = "master" + digest = "1:9fd3a6ab34bb103ba228eefd044d3f9aa476237ea95a46d12e8cccd3abf3fea2" name = "github.com/armon/go-radix" packages = ["."] + pruneopts = "UT" revision = "1fca145dffbcaa8fe914309b1ec0cfc67500fe61" [[projects]] + digest = "1:2c0d4aa114bd7ecf9cd014765c0107422c743a51fc80e6ae31b2fc3370a9d51d" name = "github.com/aws/aws-sdk-go" packages = [ "aws", @@ -75,121 +90,157 @@ "private/protocol/restxml", "private/protocol/xml/xmlutil", "service/s3", - "service/sts" + "service/sts", ] + pruneopts = "UT" revision = "5e98666858b97db54a61044bb1150d3b81436860" version = "v1.14.16" [[projects]] branch = "master" + digest = "1:37011b20a70e205b93ebea5287e1afa5618db54bf3998c36ff5a8e4b146a170a" name = "github.com/bgentry/go-netrc" packages = ["netrc"] + pruneopts = "UT" revision = "9fd32a8b3d3d3f9d43c341bfe098430e07609480" [[projects]] + digest = "1:1343a2963481a305ca4d051e84bc2abd16b601ee22ed324f8d605de1adb291b0" name = "github.com/bgentry/speakeasy" packages = ["."] + pruneopts = "UT" revision = "4aabc24848ce5fd31929f7d1e4ea74d3709c14cd" version = "v0.1.0" [[projects]] + digest = "1:705c40022f5c03bf96ffeb6477858d88565064485a513abcd0f11a0911546cb6" name = "github.com/blang/semver" packages = ["."] + pruneopts = "UT" revision = "2ee87856327ba09384cabd113bc6b5d174e9ec0f" version = "v3.5.1" [[projects]] + digest = "1:a2c1d0e43bd3baaa071d1b9ed72c27d78169b2b269f71c105ac4ba34b1be4a39" name = "github.com/davecgh/go-spew" packages = ["spew"] + pruneopts = "UT" revision = "346938d642f2ec3594ed81d874461961cd0faa76" version = "v1.1.0" [[projects]] + digest = "1:865079840386857c809b72ce300be7580cb50d3d3129ce11bf9aa6ca2bc1934a" name = "github.com/fatih/color" packages = ["."] + pruneopts = "UT" revision = "5b77d2a35fb0ede96d138fc9a99f5c9b6aef11b4" version = "v1.7.0" [[projects]] + digest = "1:fb46255681497314debedde38b64be32a75bae50bad107586c22f1662bf2d352" name = "github.com/go-ini/ini" packages = ["."] + pruneopts = "UT" revision = "06f5f3d67269ccec1fe5fe4134ba6e982984f7f5" version = "v1.37.0" [[projects]] + digest = "1:e1ff887e232b2d8f4f7c7db15a5fac7be418025afc4dda53c59c765dbb5aa6b4" name = "github.com/go-playground/locales" packages = [ ".", - "currency" + "currency", ] + pruneopts = "UT" revision = "f63010822830b6fe52288ee52d5a1151088ce039" version = "v0.12.1" [[projects]] + digest = "1:e022cf244bcac1b6ef933f1a2e0adcf6a6dfd7b872d8d41e4d4179bb09a87cbc" name = "github.com/go-playground/universal-translator" packages = ["."] + pruneopts = "UT" revision = "b32fa301c9fe55953584134cb6853a13c87ec0a1" version = "v0.16.0" [[projects]] + digest = "1:234bd3c0b701cdb49dbda54821eecf2fc76431618751ee69e95248e358b26fdb" name = "github.com/gobuffalo/packr" packages = ["."] + pruneopts = "UT" revision = "bd47f2894846e32edcf9aa37290fef76c327883f" version = "v1.11.1" [[projects]] + digest = "1:8f8811f9be822914c3a25c6a071e93beb4c805d7b026cbf298bc577bc1cc945b" name = "github.com/google/uuid" packages = ["."] + pruneopts = "UT" revision = "064e2069ce9c359c118179501254f67d7d37ba24" version = "0.2" [[projects]] branch = "master" + digest = "1:07671f8997086ed115824d1974507d2b147d1e0463675ea5dbf3be89b1c2c563" name = "github.com/hashicorp/errwrap" packages = ["."] + pruneopts = "UT" revision = "7554cd9344cec97297fa6649b055a8c98c2a1e55" [[projects]] branch = "master" + digest = "1:77cb3be9b21ba7f1a4701e870c84ea8b66e7d74c7c8951c58155fdadae9414ec" name = "github.com/hashicorp/go-cleanhttp" packages = ["."] + pruneopts = "UT" revision = "d5fe4b57a186c716b0e00b8c301cbd9b4182694d" [[projects]] branch = "master" + digest = "1:49be3bd056300f2156364200a3dd871cdc548adf09242ea6d4353d512574834c" name = "github.com/hashicorp/go-getter" packages = [ ".", - "helper/url" + "helper/url", ] + pruneopts = "UT" revision = "3f60ec5cfbb2a39731571b9ddae54b303bb0a969" [[projects]] branch = "master" + digest = "1:e5048c5da80697be2fcdecc944e29d2999e01fd7f48b643168443209779f3463" name = "github.com/hashicorp/go-multierror" packages = ["."] + pruneopts = "UT" revision = "b7773ae218740a7be65057fc60b366a49b538a44" [[projects]] branch = "master" + digest = "1:0fd42c2104e1b30aaee9f4f538fadef25290f5196dd1973166bcf51bb71f66f4" name = "github.com/hashicorp/go-safetemp" packages = ["."] + pruneopts = "UT" revision = "b1a1dbde6fdc11e3ae79efd9039009e22d4ae240" [[projects]] branch = "master" + digest = "1:354978aad16c56c27f57e5b152224806d87902e4935da3b03e18263d82ae77aa" name = "github.com/hashicorp/go-uuid" packages = ["."] + pruneopts = "UT" revision = "27454136f0364f2d44b1276c552d69105cf8c498" [[projects]] branch = "master" + digest = "1:e12b92b8bb20af6e299e9829534cfe790857702a988d3f0443e772c9d82a4fd2" name = "github.com/hashicorp/go-version" packages = ["."] + pruneopts = "UT" revision = "23480c0665776210b5fbbac6eaaee40e3e6a96b7" [[projects]] branch = "master" + digest = "1:a361611b8c8c75a1091f00027767f7779b29cb37c456a71b8f2604c88057ab40" name = "github.com/hashicorp/hcl" packages = [ ".", @@ -201,34 +252,40 @@ "hcl/token", "json/parser", "json/scanner", - "json/token" + "json/token", ] + pruneopts = "UT" revision = "ef8a98b0bbce4a65b5aa4c368430a80ddc533168" [[projects]] branch = "master" + digest = "1:f87c2cc4dd38efb53201c8484544c8b3ab15fa96062896bc71b9288d7ddf0583" name = "github.com/hashicorp/hcl2" packages = [ "gohcl", "hcl", "hcl/hclsyntax", "hcl/json", - "hclparse" + "hclparse", ] + pruneopts = "UT" revision = "36446359d27574bf110611001414da561731b62d" [[projects]] branch = "master" + digest = "1:a0fcb763bbae723b790db78143ac713c2a96c7542a4e1a2628ba3a9aedd13ae9" name = "github.com/hashicorp/hil" packages = [ ".", "ast", "parser", - "scanner" + "scanner", ] + pruneopts = "UT" revision = "fa9f258a92500514cc8e9c67020487709df92432" [[projects]] + digest = "1:7557123c3cbddc88549375ea944cf3698447818c63e03ad8d530327099b56a47" name = "github.com/hashicorp/terraform" packages = [ "config", @@ -237,175 +294,227 @@ "httpclient", "plugin/discovery", "tfdiags", - "version" + "version", ] + pruneopts = "UT" revision = "41e50bd32a8825a84535e353c3674af8ce799161" version = "v0.11.7" [[projects]] branch = "master" + digest = "1:0778dc7fce1b4669a8bfa7ae506ec1f595b6ab0f8989c1c0d22a8ca1144e9972" name = "github.com/howeyc/gopass" packages = ["."] + pruneopts = "UT" revision = "bf9dde6d0d2c004a008c27aaee91170c786f6db8" [[projects]] + digest = "1:6b1e7618931d0ed9a17fec1c1d84271907aa61172c50b8272327f6cd4fafb650" name = "github.com/huandu/xstrings" packages = ["."] + pruneopts = "UT" revision = "2bf18b218c51864a87384c06996e40ff9dcff8e1" version = "v1.0.0" [[projects]] + digest = "1:3e260afa138eab6492b531a3b3d10ab4cb70512d423faa78b8949dec76e66a21" name = "github.com/imdario/mergo" packages = ["."] + pruneopts = "UT" revision = "9316a62528ac99aaecb4e47eadd6dc8aa6533d58" version = "v0.3.5" [[projects]] + digest = "1:870d441fe217b8e689d7949fef6e43efbc787e50f200cb1e70dbca9204a1d6be" name = "github.com/inconshreveable/mousetrap" packages = ["."] + pruneopts = "UT" revision = "76626ae9c91c4f2a10f34cad8ce83ea42c93bb75" version = "v1.0" [[projects]] + digest = "1:e22af8c7518e1eab6f2eab2b7d7558927f816262586cd6ed9f349c97a6c285c4" name = "github.com/jmespath/go-jmespath" packages = ["."] + pruneopts = "UT" revision = "0b12d6b5" [[projects]] + digest = "1:c658e84ad3916da105a761660dcaeb01e63416c8ec7bc62256a9b411a05fcd67" name = "github.com/mattn/go-colorable" packages = ["."] + pruneopts = "UT" revision = "167de6bfdfba052fa6b2d3664c8f5272e23c9072" version = "v0.0.9" [[projects]] + digest = "1:d4d17353dbd05cb52a2a52b7fe1771883b682806f68db442b436294926bbfafb" name = "github.com/mattn/go-isatty" packages = ["."] + pruneopts = "UT" revision = "0360b2af4f38e8d38c7fce2a9f4e702702d73a39" version = "v0.0.3" [[projects]] branch = "master" + digest = "1:0fbbd0f1fa0d71a357594b0f40b25458eccbe1809d95843e81129b93e7a7b1cb" name = "github.com/mitchellh/cli" packages = ["."] + pruneopts = "UT" revision = "c48282d14eba4b0817ddef3f832ff8d13851aefd" [[projects]] branch = "master" + digest = "1:dfb4eb6168a4e7f16e9fda5b9164013a0f5a8eb06bb5ca3a1980964a7beedde4" name = "github.com/mitchellh/copystructure" packages = ["."] + pruneopts = "UT" revision = "d23ffcb85de31694d6ccaa23ccb4a03e55c1303f" [[projects]] branch = "master" + digest = "1:8eb17c2ec4df79193ae65b621cd1c0c4697db3bc317fe6afdc76d7f2746abd05" name = "github.com/mitchellh/go-homedir" packages = ["."] + pruneopts = "UT" revision = "3864e76763d94a6df2f9960b16a20a33da9f9a66" [[projects]] branch = "master" + digest = "1:cae1afe858922bd10e9573b87130f730a6e4183a00eba79920d6656629468bfa" name = "github.com/mitchellh/go-testing-interface" packages = ["."] + pruneopts = "UT" revision = "a61a99592b77c9ba629d254a693acffaeb4b7e28" [[projects]] branch = "master" + digest = "1:e68cd472b96cdf7c9f6971ac41bcc1d4d3b23d67c2a31d2399446e295bc88ae9" name = "github.com/mitchellh/go-wordwrap" packages = ["."] + pruneopts = "UT" revision = "ad45545899c7b13c020ea92b2072220eefad42b8" [[projects]] branch = "master" + digest = "1:544635141f5fb084637823f438f3563be046a9730c256d2e7760dbb741cd88b5" name = "github.com/mitchellh/hashstructure" packages = ["."] + pruneopts = "UT" revision = "2bca23e0e452137f789efbc8610126fd8b94f73b" [[projects]] branch = "master" + digest = "1:e730597b38a4d56e2361e0b6236cb800e52c73cace2ff91396f4ff35792ddfa7" name = "github.com/mitchellh/mapstructure" packages = ["."] + pruneopts = "UT" revision = "bb74f1db0675b241733089d5a1faa5dd8b0ef57b" [[projects]] branch = "master" + digest = "1:012bcbda750df8b57e302656a0820833eaa98009a7546b22620283c65996743b" name = "github.com/mitchellh/reflectwalk" packages = ["."] + pruneopts = "UT" revision = "63d60e9d0dbc60cf9164e6510889b0db6683d98c" [[projects]] branch = "master" + digest = "1:3891cc78541df6e4596b3e73978eb062d32967084069270ec881b959be5155ae" name = "github.com/pkg/errors" packages = ["."] + pruneopts = "UT" revision = "816c9085562cd7ee03e7f8188a1cfd942858cded" [[projects]] + digest = "1:0028cb19b2e4c3112225cd871870f2d9cf49b9b4276531f03438a88e94be86fe" name = "github.com/pmezard/go-difflib" packages = ["difflib"] + pruneopts = "UT" revision = "792786c7400a136282c1664665ae0a8db921c6c2" version = "v1.0.0" [[projects]] + digest = "1:4d76324ccff5546792b4274c5fe1639bc5983c533c85d1b4be2168d57daabdf0" name = "github.com/posener/complete" packages = [ ".", "cmd", "cmd/install", - "match" + "match", ] + pruneopts = "UT" revision = "98eb9847f27ba2008d380a32c98be474dea55bdf" version = "v1.1.1" [[projects]] branch = "master" + digest = "1:4618466244d5f6dbca839b6c0151f76bc58b592d81c83fe909ea9a070269f324" name = "github.com/segmentio/go-prompt" packages = ["."] + pruneopts = "UT" revision = "f0d19b6901ade831d5a3204edc0d6a7d6457fbb2" [[projects]] + digest = "1:9e9193aa51197513b3abcb108970d831fbcf40ef96aa845c4f03276e1fa316d2" name = "github.com/sirupsen/logrus" packages = ["."] + pruneopts = "UT" revision = "c155da19408a8799da419ed3eeb0cb5db0ad5dbc" version = "v1.0.5" [[projects]] + digest = "1:bd1ae00087d17c5a748660b8e89e1043e1e5479d0fea743352cda2f8dd8c4f84" name = "github.com/spf13/afero" packages = [ ".", - "mem" + "mem", ] + pruneopts = "UT" revision = "787d034dfe70e44075ccc060d346146ef53270ad" version = "v1.1.1" [[projects]] + digest = "1:645cabccbb4fa8aab25a956cbcbdf6a6845ca736b2c64e197ca7cbb9d210b939" name = "github.com/spf13/cobra" packages = ["."] + pruneopts = "UT" revision = "ef82de70bb3f60c65fb8eebacbb2d122ef517385" version = "v0.0.3" [[projects]] + digest = "1:9424f440bba8f7508b69414634aef3b2b3a877e522d8a4624692412805407bb7" name = "github.com/spf13/pflag" packages = ["."] + pruneopts = "UT" revision = "583c0c0531f06d5278b7d917446061adc344b5cd" version = "v1.0.1" [[projects]] + digest = "1:18752d0b95816a1b777505a97f71c7467a8445b8ffb55631a7bf779f6ba4fa83" name = "github.com/stretchr/testify" packages = ["assert"] + pruneopts = "UT" revision = "f35b8ab0b5a2cef36673838d662e249dd9c94686" version = "v1.2.2" [[projects]] + digest = "1:4aeb3860275fa1fd60cccfb5a6ef85da438bf17402e1e84412ade4d4b55066a0" name = "github.com/ulikunitz/xz" packages = [ ".", "internal/hash", "internal/xlog", - "lzma" + "lzma", ] + pruneopts = "UT" revision = "0c6b41e72360850ca4f98dc341fd999726ea007f" version = "v0.5.4" [[projects]] branch = "master" + digest = "1:34c512c2e2033bc473dd486f162e1e9ae36522d86a45d53702e60b448f10ba57" name = "github.com/zclconf/go-cty" packages = [ "cty", @@ -414,12 +523,14 @@ "cty/function/stdlib", "cty/gocty", "cty/json", - "cty/set" + "cty/set", ] + pruneopts = "UT" revision = "c96d660229f9ad0a16eb5869819ea51e1ed49896" [[projects]] branch = "master" + digest = "1:784b0bdb6f351578fef5ad5fbc5efa55c26ff4cd305a277aff50ffef27ab4817" name = "golang.org/x/crypto" packages = [ "bcrypt", @@ -433,29 +544,35 @@ "openpgp/s2k", "pbkdf2", "scrypt", - "ssh/terminal" + "ssh/terminal", ] + pruneopts = "UT" revision = "a49355c7e3f8fe157a85be2f77e6e269a0f89602" [[projects]] branch = "master" + digest = "1:0516820422512741f45f4e72d4ade116c1d1bf3bb2d846001d42f6787ec00f22" name = "golang.org/x/net" packages = [ "html", - "html/atom" + "html/atom", ] + pruneopts = "UT" revision = "4cb1c02c05b0e749b0365f61ae859a8e0cfceed9" [[projects]] branch = "master" + digest = "1:50e49f00c462e4531c6987ab12ab81a9a9f76bc0c3235c6e9cf9b75c2b5ff638" name = "golang.org/x/sys" packages = [ "unix", - "windows" + "windows", ] + pruneopts = "UT" revision = "7138fd3d9dc8335c567ca206f4333fb75eb05d56" [[projects]] + digest = "1:8029e9743749d4be5bc9f7d42ea1659471767860f0cdc34d37c3111bd308a295" name = "golang.org/x/text" packages = [ "internal/gen", @@ -463,20 +580,39 @@ "internal/ucd", "transform", "unicode/cldr", - "unicode/norm" + "unicode/norm", ] + pruneopts = "UT" revision = "f21a4dfb5e38f5895301dc265a8def02365cc3d0" version = "v0.3.0" [[projects]] + digest = "1:a351fd4cb95e313ecb345a521f276e7a00913d1f2e84c61543bc4fa1785ba7f3" name = "gopkg.in/go-playground/validator.v9" packages = ["."] + pruneopts = "UT" revision = "ab2a8a99087e827c9af87ed6777ba897348fb178" version = "v9.20.2" [solve-meta] analyzer-name = "dep" analyzer-version = 1 - inputs-digest = "b91cbccdce08e0bab280b733b64c9d167735ee775c95e29d071175a24e08eb2b" + input-imports = [ + "github.com/Masterminds/sprig", + "github.com/blang/semver", + "github.com/gobuffalo/packr", + "github.com/hashicorp/go-getter", + "github.com/hashicorp/go-multierror", + "github.com/hashicorp/hcl/hcl/printer", + "github.com/hashicorp/terraform/config", + "github.com/mitchellh/go-homedir", + "github.com/pkg/errors", + "github.com/segmentio/go-prompt", + "github.com/sirupsen/logrus", + "github.com/spf13/afero", + "github.com/spf13/cobra", + "github.com/stretchr/testify/assert", + "gopkg.in/go-playground/validator.v9", + ] solver-name = "gps-cdcl" solver-version = 1 diff --git a/Makefile b/Makefile index 5a045cf2e..567c48b0a 100644 --- a/Makefile +++ b/Makefile @@ -18,16 +18,16 @@ packr: ## run the packr tool to generate our static files release: packr goreleaser release --rm-dist -build: ## build the binary +build: packr ## build the binary go build ${LDFLAGS} . coverage: ## run the go coverage tool, reading file coverage.out go tool cover -html=coverage.out -test: ## run the tests +test: packr ## run the tests go test -cover ./... -install: ## install the fogg binary in $GOPATH/bin +install: packr ## install the fogg binary in $GOPATH/bin go install ${LDFLAGS} . help: ## display help for this makefile diff --git a/apply/apply.go b/apply/apply.go index d631e378a..f89c23d96 100644 --- a/apply/apply.go +++ b/apply/apply.go @@ -13,6 +13,7 @@ import ( "github.com/chanzuckerberg/fogg/config" "github.com/chanzuckerberg/fogg/plan" + "github.com/chanzuckerberg/fogg/plugins" "github.com/chanzuckerberg/fogg/templates" "github.com/chanzuckerberg/fogg/util" "github.com/gobuffalo/packr" @@ -52,16 +53,37 @@ func Apply(fs afero.Fs, conf *config.Config, tmp *templates.T) error { } e = applyModules(fs, p.Modules, &tmp.Module) - return errors.Wrap(e, "unable to apply modules") } func applyRepo(fs afero.Fs, p *plan.Plan, repoTemplates *packr.Box) error { - e := applyTree(repoTemplates, fs, "", p) + e := applyTree(fs, repoTemplates, "", p) if e != nil { return e } - return nil + return applyPlugins(fs, p) +} + +func applyPlugins(fs afero.Fs, p *plan.Plan) (err error) { + apply := func(name string, plugin *plugins.CustomPlugin) error { + log.Infof("Applying plugin %s", name) + return errors.Wrapf(plugin.Install(fs, name), "Error applying plugin %s", name) + } + + for pluginName, plugin := range p.Plugins.CustomPlugins { + err = apply(pluginName, plugin) + if err != nil { + return err + } + } + + for providerName, provider := range p.Plugins.TerraformProviders { + err = apply(providerName, provider) + if err != nil { + return err + } + } + return } func applyGlobal(fs afero.Fs, p plan.Component, repoBox *packr.Box) error { @@ -70,7 +92,7 @@ func applyGlobal(fs afero.Fs, p plan.Component, repoBox *packr.Box) error { if e != nil { return errors.Wrapf(e, "unable to make directory %s", path) } - return applyTree(repoBox, fs, path, p) + return applyTree(fs, repoBox, path, p) } func applyAccounts(fs afero.Fs, p *plan.Plan, accountBox *packr.Box) (e error) { @@ -80,7 +102,7 @@ func applyAccounts(fs afero.Fs, p *plan.Plan, accountBox *packr.Box) (e error) { if e != nil { return errors.Wrap(e, "unable to make directories for accounts") } - e = applyTree(accountBox, fs, path, accountPlan) + e = applyTree(fs, accountBox, path, accountPlan) if e != nil { return errors.Wrap(e, "unable to apply templates to account") } @@ -88,15 +110,14 @@ func applyAccounts(fs afero.Fs, p *plan.Plan, accountBox *packr.Box) (e error) { return nil } -func applyModules(fs afero.Fs, p map[string]plan.Module, moduleBox *packr.Box) error { - var e error +func applyModules(fs afero.Fs, p map[string]plan.Module, moduleBox *packr.Box) (e error) { for module, modulePlan := range p { path := fmt.Sprintf("%s/modules/%s", rootPath, module) e = fs.MkdirAll(path, 0755) if e != nil { return errors.Wrapf(e, "unable to make path %s", path) } - e = applyTree(moduleBox, fs, path, modulePlan) + e = applyTree(fs, moduleBox, path, modulePlan) if e != nil { return errors.Wrap(e, "unable to apply tree") } @@ -111,7 +132,7 @@ func applyEnvs(fs afero.Fs, p *plan.Plan, envBox *packr.Box, componentBox *packr if e != nil { return errors.Wrapf(e, "unable to make directory %s", path) } - e := applyTree(envBox, fs, path, envPlan) + e := applyTree(fs, envBox, path, envPlan) if e != nil { return errors.Wrap(e, "unable to apply templates to env") } @@ -121,7 +142,7 @@ func applyEnvs(fs afero.Fs, p *plan.Plan, envBox *packr.Box, componentBox *packr if e != nil { return errors.Wrap(e, "unable to make directories for component") } - e := applyTree(componentBox, fs, path, componentPlan) + e := applyTree(fs, componentBox, path, componentPlan) if e != nil { return errors.Wrap(e, "unable to apply templates for component") } @@ -138,7 +159,7 @@ func applyEnvs(fs afero.Fs, p *plan.Plan, envBox *packr.Box, componentBox *packr return nil } -func applyTree(source *packr.Box, dest afero.Fs, targetBasePath string, subst interface{}) (e error) { +func applyTree(dest afero.Fs, source *packr.Box, targetBasePath string, subst interface{}) (e error) { return source.Walk(func(path string, sourceFile packr.File) error { extension := filepath.Ext(path) target := getTargetPath(targetBasePath, path) diff --git a/config/config.go b/config/config.go index 36ba129f0..695176f71 100644 --- a/config/config.go +++ b/config/config.go @@ -8,6 +8,7 @@ import ( "reflect" "strings" + "github.com/chanzuckerberg/fogg/plugins" "github.com/hashicorp/go-multierror" "github.com/pkg/errors" "github.com/spf13/afero" @@ -78,15 +79,23 @@ type Component struct { TerraformVersion *string `json:"terraform_version"` } +// Plugins contains configuration around plugins +type Plugins struct { + CustomPlugins map[string]*plugins.CustomPlugin `json:"custom_plugins,omitempty"` + TerraformProviders map[string]*plugins.CustomPlugin `json:"terraform_providers,omitempty"` +} + +// Module is a module type Module struct { TerraformVersion *string `json:"terraform_version"` } type Config struct { - Defaults defaults `json:"defaults"` Accounts map[string]Account `json:"accounts"` + Defaults defaults `json:"defaults"` Envs map[string]Env `json:"envs"` Modules map[string]Module `json:"modules"` + Plugins Plugins `json:"plugins"` } var allRegions = []string{ diff --git a/plan/plan.go b/plan/plan.go index d07667be8..99e956b5b 100644 --- a/plan/plan.go +++ b/plan/plan.go @@ -4,6 +4,7 @@ import ( "fmt" "github.com/chanzuckerberg/fogg/config" + "github.com/chanzuckerberg/fogg/plugins" "github.com/chanzuckerberg/fogg/util" "github.com/pkg/errors" ) @@ -65,11 +66,34 @@ type Env struct { TerraformVersion string } +// Plugins contains a plan around plugins +type Plugins struct { + CustomPlugins map[string]*plugins.CustomPlugin + TerraformProviders map[string]*plugins.CustomPlugin +} + +// SetCustomPluginsPlan determines the plan for customPlugins +func (p *Plugins) SetCustomPluginsPlan(customPlugins map[string]*plugins.CustomPlugin) { + p.CustomPlugins = customPlugins + for _, plugin := range p.CustomPlugins { + plugin.SetTargetPath(plugins.CustomPluginDir) + } +} + +// SetTerraformProvidersPlan determines the plan for customPlugins +func (p *Plugins) SetTerraformProvidersPlan(terraformProviders map[string]*plugins.CustomPlugin) { + p.TerraformProviders = terraformProviders + for _, plugin := range p.TerraformProviders { + plugin.SetTargetPath(plugins.TerraformCustomPluginCacheDir) + } +} + type Plan struct { Accounts map[string]account Envs map[string]Env Global Component Modules map[string]Module + Plugins Plugins Version string } @@ -80,6 +104,8 @@ func Eval(config *config.Config, verbose bool) (*Plan, error) { return nil, errors.Wrap(e, "unable to parse fogg version") } p.Version = v + p.Plugins.SetCustomPluginsPlan(config.Plugins.CustomPlugins) + p.Plugins.SetTerraformProvidersPlan(config.Plugins.TerraformProviders) accounts, err := buildAccounts(config) if err != nil { @@ -152,6 +178,18 @@ func Print(p *Plan) error { fmt.Printf("\tproject: %v\n", p.Global.Project) fmt.Printf("\tterraform_version: %v\n", p.Global.TerraformVersion) + fmt.Println("Plugins:") + for name, customPlugin := range p.Plugins.CustomPlugins { + fmt.Printf("\t%s:\n", name) + fmt.Printf("\t\turl: %s\n", customPlugin.URL) + fmt.Printf("\t\tformat: %s\n", customPlugin.Format) + } + for name, customProvider := range p.Plugins.TerraformProviders { + fmt.Printf("\t%s:\n", name) + fmt.Printf("\t\turl: %s\n", customProvider.URL) + fmt.Printf("\t\tformat: %s\n", customProvider.Format) + } + fmt.Println("Envs:") for name, env := range p.Envs { diff --git a/plugins/constants.go b/plugins/constants.go new file mode 100644 index 000000000..8a5d98e89 --- /dev/null +++ b/plugins/constants.go @@ -0,0 +1,14 @@ +package plugins + +const ( + // TerraformPluginCacheDir is the directory where terraform caches tf approved providers + // See https://www.terraform.io/docs/configuration/providers.html#provider-plugin-cache + TerraformPluginCacheDir = ".terraform.d/plugin-cache" + // TerraformCustomPluginCacheDir is the directory used by terraform to search for custom providers + // We default to linux_amd64 since we're running terraform inside of docker + // We vendor providers here + // See https://www.terraform.io/docs/configuration/providers.html#third-party-plugins + TerraformCustomPluginCacheDir = "terraform.d/plugins/linux_amd64" + // CustomPluginDir where we place custom binaries + CustomPluginDir = ".bin" +) diff --git a/plugins/custom_plugin.go b/plugins/custom_plugin.go new file mode 100644 index 000000000..8f3946859 --- /dev/null +++ b/plugins/custom_plugin.go @@ -0,0 +1,132 @@ +package plugins + +import ( + "archive/tar" + "compress/gzip" + "io" + "io/ioutil" + "net/http" + "os" + "path/filepath" + + "github.com/pkg/errors" + log "github.com/sirupsen/logrus" + "github.com/spf13/afero" +) + +// TypePluginFormat is the plugin format such as binary, zip, tar +type TypePluginFormat string + +const ( + // TypePluginFormatTar is a tar archived plugin + TypePluginFormatTar TypePluginFormat = "tar" +) + +// CustomPlugin is a custom plugin +type CustomPlugin struct { + URL string `json:"url" validate:"required"` + Format TypePluginFormat `json:"format" validate:"required"` + targetDir string +} + +// Install installs the custom plugin +func (cp *CustomPlugin) Install(fs afero.Fs, pluginName string) error { + if cp == nil { + return errors.New("nil CustomPlugin") + } + if fs == nil { + return errors.New("nil fs") + } + + tmpPath, err := cp.fetch(pluginName) + defer os.Remove(tmpPath) // clean up + if err != nil { + return err + } + return cp.process(fs, pluginName, tmpPath) +} + +// SetTargetPath sets the target path for this plugin +func (cp *CustomPlugin) SetTargetPath(path string) { + cp.targetDir = path +} + +// fetch fetches the custom plugin at URL +func (cp *CustomPlugin) fetch(pluginName string) (string, error) { + tmpFile, err := ioutil.TempFile("", pluginName) + if err != nil { + return "", errors.Wrap(err, "could not create temporary directory") + } + resp, err := http.Get(cp.URL) + if err != nil { + return "", errors.Wrapf(err, "could not get %s", cp.URL) + } + defer resp.Body.Close() + _, err = io.Copy(tmpFile, resp.Body) + return tmpFile.Name(), errors.Wrap(err, "could not download file") +} + +// process the custom plugin +func (cp *CustomPlugin) process(fs afero.Fs, pluginName string, path string) error { + switch cp.Format { + case TypePluginFormatTar: + return cp.processTar(fs, path) + default: + return errors.Errorf("Unknown plugin format %s", cp.Format) + } +} + +// https://medium.com/@skdomino/taring-untaring-files-in-go-6b07cf56bc07 +func (cp *CustomPlugin) processTar(fs afero.Fs, path string) error { + err := fs.MkdirAll(cp.targetDir, 0755) + if err != nil { + return errors.Wrapf(err, "Could not create directory %s", cp.targetDir) + } + f, err := os.Open(path) + if err != nil { + return errors.Wrap(err, "could not read staged custom plugin") + } + defer f.Close() + gzr, err := gzip.NewReader(f) + if err != nil { + return errors.Wrap(err, "could not create gzip reader") + } + defer gzr.Close() + tr := tar.NewReader(gzr) + for { + header, err := tr.Next() + if err == io.EOF { + break // no more files are found + } + if err != nil { + return errors.Wrap(err, "Error reading tar") + } + if header == nil { + return errors.New("Nil tar file header") + } + // the target location where the dir/file should be created + target := filepath.Join(cp.targetDir, header.Name) + switch header.Typeflag { + case tar.TypeDir: // if its a dir and it doesn't exist create it + err := fs.MkdirAll(target, 0755) + if err != nil { + return errors.Wrapf(err, "tar: could not create directory %s", target) + } + case tar.TypeReg: // if it is a file create it, preserving the file mode + destFile, err := fs.OpenFile(target, os.O_RDWR|os.O_CREATE|os.O_TRUNC, os.FileMode(header.Mode)) + if err != nil { + return errors.Wrapf(err, "tar: could not open destination file for %s", target) + } + _, err = io.Copy(destFile, tr) + if err != nil { + destFile.Close() + return errors.Wrapf(err, "tar: could not copy file contents") + } + // Manually take care of closing file since defer will pile them up + destFile.Close() + default: + log.Warnf("tar: unrecognized tar.Type %d", header.Typeflag) + } + } + return nil +} diff --git a/plugins/custom_plugin_test.go b/plugins/custom_plugin_test.go new file mode 100644 index 000000000..f26ee9921 --- /dev/null +++ b/plugins/custom_plugin_test.go @@ -0,0 +1,86 @@ +package plugins_test + +import ( + "archive/tar" + "compress/gzip" + "fmt" + "io" + "io/ioutil" + "net/http" + "net/http/httptest" + "os" + "path" + "testing" + + "github.com/chanzuckerberg/fogg/plugins" + "github.com/spf13/afero" + "github.com/stretchr/testify/assert" +) + +func TestCustomPluginTar(t *testing.T) { + a := assert.New(t) + pluginName := "test-provider" + fs := afero.NewMemMapFs() + + files := []string{"test.txt", "terraform-provider-testing"} + tarPath := generateTar(t, files) + defer os.Remove(tarPath) + + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + f, err := os.Open(tarPath) + a.Nil(err) + _, err = io.Copy(w, f) + a.Nil(err) + })) + defer ts.Close() + + customPlugin := &plugins.CustomPlugin{ + URL: ts.URL, + Format: plugins.TypePluginFormatTar, + } + customPlugin.SetTargetPath(plugins.CustomPluginDir) + a.Nil(customPlugin.Install(fs, pluginName)) + + afero.Walk(fs, "", func(path string, info os.FileInfo, err error) error { + a.Nil(err) + return nil + }) + + for _, file := range files { + filePath := path.Join(plugins.CustomPluginDir, file) + fi, err := fs.Stat(filePath) + a.Nil(err) + a.False(fi.IsDir()) + a.Equal(fi.Mode(), os.FileMode(0664)) + + bytes, err := afero.ReadFile(fs, filePath) + a.Nil(err) + a.Equal(bytes, []byte(file)) // We wrote the filename as the contents as well + } +} + +func generateTar(t *testing.T, files []string) string { + a := assert.New(t) + + f, err := ioutil.TempFile("", "testing") + a.Nil(err) + defer f.Close() + gw := gzip.NewWriter(f) + defer gw.Close() + tw := tar.NewWriter(gw) + defer tw.Close() + + for _, file := range files { + header := new(tar.Header) + header.Name = file + header.Size = int64(len([]byte(file))) + header.Mode = int64(0664) + header.Typeflag = tar.TypeReg + + a.Nil(tw.WriteHeader(header)) + _, err = fmt.Fprintf(tw, file) + a.Nil(err) + } + + return f.Name() +} diff --git a/templates/account/Makefile.tmpl b/templates/account/Makefile.tmpl index ba926e4fe..f90d45052 100644 --- a/templates/account/Makefile.tmpl +++ b/templates/account/Makefile.tmpl @@ -11,6 +11,7 @@ IMAGE_VERSION={{ .DockerImageVersion }}_TF{{ .TerraformVersion }} docker_base = \ docker run -it --rm -e HOME=/home -v $$HOME/.aws:/home/.aws -v $(REPO_ROOT):/repo \ + -v $(REPO_ROOT)/.bin:/home/bin -v $(REPO_ROOT)/terraform.d:/repo/$(REPO_RELATIVE_PATH)/terraform.d \ -e GIT_SSH_COMMAND='ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no' \ -e RUN_USER_ID=$(shell id -u) -e RUN_GROUP_ID=$(shell id -g) \ -e TF_PLUGIN_CACHE_DIR="/repo/.terraform.d/plugin-cache" -e TF="$(TF)" \ diff --git a/templates/component/Makefile.tmpl b/templates/component/Makefile.tmpl index ba926e4fe..f90d45052 100644 --- a/templates/component/Makefile.tmpl +++ b/templates/component/Makefile.tmpl @@ -11,6 +11,7 @@ IMAGE_VERSION={{ .DockerImageVersion }}_TF{{ .TerraformVersion }} docker_base = \ docker run -it --rm -e HOME=/home -v $$HOME/.aws:/home/.aws -v $(REPO_ROOT):/repo \ + -v $(REPO_ROOT)/.bin:/home/bin -v $(REPO_ROOT)/terraform.d:/repo/$(REPO_RELATIVE_PATH)/terraform.d \ -e GIT_SSH_COMMAND='ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no' \ -e RUN_USER_ID=$(shell id -u) -e RUN_GROUP_ID=$(shell id -g) \ -e TF_PLUGIN_CACHE_DIR="/repo/.terraform.d/plugin-cache" -e TF="$(TF)" \ diff --git a/templates/global/Makefile.tmpl b/templates/global/Makefile.tmpl index ba926e4fe..f90d45052 100644 --- a/templates/global/Makefile.tmpl +++ b/templates/global/Makefile.tmpl @@ -11,6 +11,7 @@ IMAGE_VERSION={{ .DockerImageVersion }}_TF{{ .TerraformVersion }} docker_base = \ docker run -it --rm -e HOME=/home -v $$HOME/.aws:/home/.aws -v $(REPO_ROOT):/repo \ + -v $(REPO_ROOT)/.bin:/home/bin -v $(REPO_ROOT)/terraform.d:/repo/$(REPO_RELATIVE_PATH)/terraform.d \ -e GIT_SSH_COMMAND='ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no' \ -e RUN_USER_ID=$(shell id -u) -e RUN_GROUP_ID=$(shell id -g) \ -e TF_PLUGIN_CACHE_DIR="/repo/.terraform.d/plugin-cache" -e TF="$(TF)" \ diff --git a/templates/module/Makefile.tmpl b/templates/module/Makefile.tmpl index 56edf7d07..41efd8eea 100644 --- a/templates/module/Makefile.tmpl +++ b/templates/module/Makefile.tmpl @@ -11,6 +11,7 @@ IMAGE_VERSION={{ .DockerImageVersion }}_TF{{ .TerraformVersion }} docker_base = \ docker run -it --rm -e HOME=/home -v $$HOME/.aws:/home/.aws -v $(REPO_ROOT):/repo \ + -v $(REPO_ROOT)/.bin:/home/bin -v $(REPO_ROOT)/terraform.d:/repo/$(REPO_RELATIVE_PATH)/terraform.d \ -e GIT_SSH_COMMAND='ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no' \ -e RUN_USER_ID=$(shell id -u) -e RUN_GROUP_ID=$(shell id -g) \ -e TF_PLUGIN_CACHE_DIR="/repo/.terraform.d/plugin-cache" -e TF="$(TF)" \ diff --git a/templates/repo/.gitignore b/templates/repo/.gitignore index 9c17ace2c..47d0c5ea1 100644 --- a/templates/repo/.gitignore +++ b/templates/repo/.gitignore @@ -9,6 +9,7 @@ # Module directory .terraform/ +terraform/**/terraform.d # legacy .rato_pass diff --git a/templates/repo/.terraform.d/plugin-cache/.gitignore b/templates/repo/.terraform.d/plugin-cache/.gitignore new file mode 100644 index 000000000..504d4d91d --- /dev/null +++ b/templates/repo/.terraform.d/plugin-cache/.gitignore @@ -0,0 +1,5 @@ +# Auto-generated by fogg. Do not edit +# Make improvements in fogg, so that everyone can benefit. + +* +!.gitignore diff --git a/templates/repo/terraform.d/plugins/linux_amd64/.gitignore b/templates/repo/terraform.d/plugins/linux_amd64/.gitignore new file mode 100644 index 000000000..b04e87942 --- /dev/null +++ b/templates/repo/terraform.d/plugins/linux_amd64/.gitignore @@ -0,0 +1,6 @@ +# Auto-generated by fogg. Do not edit +# Make improvements in fogg, so that everyone can benefit. + +* +!.gitignore +!terraform-provider*