diff --git a/api/cmd/build/.dockerignore b/api/cmd/build/.dockerignore deleted file mode 100644 index cd916e75d8..0000000000 --- a/api/cmd/build/.dockerignore +++ /dev/null @@ -1,2 +0,0 @@ -.env -/build diff --git a/api/cmd/build/.gitignore b/api/cmd/build/.gitignore deleted file mode 100644 index cd916e75d8..0000000000 --- a/api/cmd/build/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -.env -/build diff --git a/api/cmd/build/Makefile b/api/cmd/build/Makefile new file mode 100644 index 0000000000..c16788bc1a --- /dev/null +++ b/api/cmd/build/Makefile @@ -0,0 +1,3 @@ +main: + go-bindata data/ + go build main.go \ No newline at end of file diff --git a/api/cmd/build/README.md b/api/cmd/build/README.md index f37acdeb22..afed8834b6 100644 --- a/api/cmd/build/README.md +++ b/api/cmd/build/README.md @@ -1,8 +1,71 @@ -# convox/build +# build -Build and push Docker images for Convox apps +Build, tag and push Docker images from an app's source. -## Development +Source is either a .tgz snapshot of an app directory or an URL to a git +repository and optional commit. + +This command is designed to run in the convox/rack Docker image which is a +basic Linux environment with `docker`, `git`, and `ssh` available for extracting +or cloning the app source. + +## Usage + +``` +$ tar cz | build - +$ build github.com/convox-examples/sinatra.git +``` + +## Docker Run Flags + +* `-v /var/run/docker.sock:/var/run/docker.sock` so it can acess the Docker daemon and execute `docker build` +* `-i` so it can read .tgz data from stdin + +## Environment + +This command tags images in a way that reflects a specific app and build id. Therefore these arguments +are required: + +* `APP` - Name of the app we are building for +* `BUILD` - Id of the build +* `REGISTRY_ADDRESS` - Registry host to tag and push to + +And the tag namespace is optional: + +* `REPOSITORY` - Optional namespace that every image should use + +Without `REPOSITORY`, tags use the app name as the repository, the process name as the image name, and the build id as the tag: + +``` +convox-826133048.us-east-1.elb.amazonaws.com:5000/sinatra/web:BANHPORIOTL +``` + +With `REPOSITORY`, tags always share the same repository, and use the process name and build id as the tag: + +``` +132866487567.dkr.ecr.us-east-1.amazonaws.com/convox-sinatra-soppqmvrdv:web.BDQBBSNVTZD +``` + +This command may pull images that docker-compose.yml references, and will push new images to a remote registry. +These arguments along with `REGISTRY_ADDRESS` offer Docker authentication to push/pull: + +* `DOCKER_AUTH` - Json blob of private registry auth info +* `REGISTRY_EMAIL` - Credentials to `docker push` +* `REGISTRY_USERNAME` - Credentials to `docker push` +* `REGISTRY_PASSWORD` - Credentials to `docker push` + +If this command calls back to Rack to denote build status. Any error calls back to report "failed", +otherwise it reports "complete". These arguments offer Rack authentication to call back: + +* `RACK_HOST` - Hostname to call back on build success or failure +* `RACK_PASSWORD` - Password to call back on build success or failure + +A few options of the build are controlled by a user. These arguments override default assumptions for `docker build`: + +* `MANIFEST_PATH` - Optional path if not docker-compose.yml +* `NO_CACHE` - Option to build without reusing cache + +## Examples # rebuild the API image and build cmd @@ -15,102 +78,4 @@ Build and push Docker images for Convox apps $ cd httpd $ tar cz . | docker run -i -v /var/run/docker.sock:/var/run/docker.sock rack/api \ - build httpd - - - manifest|web: - manifest| image: httpd - manifest| ports: - manifest| - 80:80 - manifest| - 443:80 - manifest|web2: - manifest| image: httpd - manifest| ports: - manifest| - 8000:80 - build|RUNNING: docker tag -f httpd httpd/web - - # build a directory as long as its mounted into the container - - $ cd httpd - $ docker run -i -v /var/run/docker.sock:/var/run/docker.sock -v $(pwd):/tmp/app rack/api \ - build httpd /tmp/app - - manifest|web: - manifest| image: httpd - manifest| ports: - manifest| - 80:80 - manifest| - 443:80 - manifest|web2: - manifest| image: httpd - manifest| ports: - manifest| - 8000:80 - build|RUNNING: docker tag -f httpd httpd/web - - # build an http git repo - - $ docker run -i -v /var/run/docker.sock:/var/run/docker.sock rack/api \ - build httpd https://github.com/nzoschke/httpd.git - - git|Cloning into '/tmp/repo655149862/clone'... - git|POST git-upload-pack (190 bytes) - git|remote: Counting objects: 20, done. - remote: Compressing objects: 100% (16/16), done. - git|remote: Total 20 (delta 11), reused 11 (delta 2), pack-reused 0 - git|Checking connectivity... done. - manifest|web: - manifest| image: httpd - manifest| ports: - manifest| - 80:80 - build|RUNNING: docker tag -f httpd httpd/web - - # build a git repo with a commit-ish - - $ docker run -i -v /var/run/docker.sock:/var/run/docker.sock rack/api \ - build httpd https://github.com/nzoschke/httpd.git#thunk - - git|Cloning into '/tmp/repo824977679/clone'... - git|POST git-upload-pack (190 bytes) - git|remote: Counting objects: 20, done. - remote: Compressing objects: 100% (16/16), done. - git|remote: Total 20 (delta 11), reused 11 (delta 2), pack-reused 0 - git|Checking connectivity... done. - git|error: pathspec 'thunk' did not match any file(s) known to git. - ERROR: exit status 1 - - # try an SSH git repo - - $ docker run -i -v /var/run/docker.sock:/var/run/docker.sock rack/api \ - build httpd ssh://git@gitlab.com:nzoschke/httpd.git - - git|Cloning into '/tmp/repo295143718/clone'... - git|Warning: Permanently added 'gitlab.com,104.210.2.228' (ECDSA) to the list of known hosts. - git|Permission denied (publickey). - git|fatal: Could not read from remote repository. - git| - git|Please make sure you have the correct access rights - git|and the repository exists. - ERROR: exit status 128 - 2016/03/09 22:47:54 exit status 1 - - # build an SSH git repo by passing in a public key that's configured as an SSH deploy key - - $ KEY=$(base64 /tmp/id_rsa) - $ docker run -i -v /var/run/docker.sock:/var/run/docker.sock rack/api \ - build httpd ssh://git:$KEY@gitlab.com:nzoschke/httpd.git - - git|Cloning into '/tmp/repo839273553/clone'... - git|Warning: Permanently added 'gitlab.com,104.210.2.228' (ECDSA) to the list of known hosts. - git|remote: Counting objects: 10, done. - remote: Compressing objects: 100% (8/8), done. - git|remote: Total 10 (delta 3), reused 0 (delta 0) - Receiving objects: 100% (10/10), done. - Resolving deltas: 100% (3/3), done. - git|Checking connectivity... done. - manifest|web: - manifest| image: httpd - manifest| ports: - manifest| - 80:80 - build|RUNNING: docker tag -f httpd httpd/web - -## License - -Apache 2.0 © 2015 Convox, Inc. + build httpd - \ No newline at end of file diff --git a/api/cmd/build/bin/entrypoint b/api/cmd/build/bin/entrypoint deleted file mode 100755 index 2bc1521773..0000000000 --- a/api/cmd/build/bin/entrypoint +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/sh - -if [ ! -S /var/run/docker.sock ]; then - docker -d > /var/log/docker.log 2>&1 & -fi - -build $* diff --git a/api/cmd/build/bindata.go b/api/cmd/build/bindata.go index 36605d1307..0e9dc289a6 100644 --- a/api/cmd/build/bindata.go +++ b/api/cmd/build/bindata.go @@ -1,3 +1,8 @@ +// Code generated by go-bindata. +// sources: +// data/git-restore-mtime +// DO NOT EDIT! + package main import ( @@ -5,15 +10,14 @@ import ( "compress/gzip" "fmt" "io" - "strings" - "os" - "time" "io/ioutil" - "path" + "os" "path/filepath" + "strings" + "time" ) -func bindata_read(data []byte, name string) ([]byte, error) { +func bindataRead(data []byte, name string) ([]byte, error) { gz, err := gzip.NewReader(bytes.NewBuffer(data)) if err != nil { return nil, fmt.Errorf("Read %q: %v", name, err) @@ -21,11 +25,14 @@ func bindata_read(data []byte, name string) ([]byte, error) { var buf bytes.Buffer _, err = io.Copy(&buf, gz) - gz.Close() + clErr := gz.Close() if err != nil { return nil, fmt.Errorf("Read %q: %v", name, err) } + if clErr != nil { + return nil, err + } return buf.Bytes(), nil } @@ -35,129 +42,49 @@ type asset struct { info os.FileInfo } -type bindata_file_info struct { - name string - size int64 - mode os.FileMode +type bindataFileInfo struct { + name string + size int64 + mode os.FileMode modTime time.Time } -func (fi bindata_file_info) Name() string { +func (fi bindataFileInfo) Name() string { return fi.name } -func (fi bindata_file_info) Size() int64 { +func (fi bindataFileInfo) Size() int64 { return fi.size } -func (fi bindata_file_info) Mode() os.FileMode { +func (fi bindataFileInfo) Mode() os.FileMode { return fi.mode } -func (fi bindata_file_info) ModTime() time.Time { +func (fi bindataFileInfo) ModTime() time.Time { return fi.modTime } -func (fi bindata_file_info) IsDir() bool { +func (fi bindataFileInfo) IsDir() bool { return false } -func (fi bindata_file_info) Sys() interface{} { +func (fi bindataFileInfo) Sys() interface{} { return nil } -var _data_app_conf = []byte("\x1f\x8b\x08\x00\x00\x09\x6e\x88\x00\xff\xac\x91\xcf\x4e\xf3\x30\x10\xc4\xef\x7e\x8a\xf9\xaa\x4f\x08\x0e\x8e\xd5\xd2\x22\x51\x89\xbe\x08\xe2\x60\x39\x4b\x31\x35\xb6\x59\x6f\x42\x91\x78\x78\x9c\xa4\xe1\x8f\xc4\x05\x89\xd3\x24\xde\xd1\x6f\x77\x67\x8b\x58\x16\xa4\x08\xee\x62\xa0\x9e\x02\x6e\x57\x97\xeb\xcd\x1d\x6c\x6c\x31\x16\xa9\x45\x9b\xdc\x81\x58\x15\x49\xf9\xbb\xf5\xdf\xe8\x55\x8a\xa9\x64\xfb\x12\x67\x45\xf0\x4f\x5e\x50\x6d\x83\x52\xab\x54\x4e\x45\xf4\xd4\xab\x38\xf6\x59\x14\x06\xba\x3b\xdc\xfc\x3f\x77\x1d\x07\xe8\x82\x07\x91\xbc\x35\x66\x79\x75\xdd\xac\x36\xeb\xe6\xa4\x26\x58\xa1\x22\xa6\x2b\xc4\xba\xb5\x62\xf1\x86\xc7\x67\x68\xc6\xa2\x29\x7e\x1f\x6d\x68\x46\xd0\xe2\xa2\x22\x6b\xff\xd4\xb1\xa3\xbf\xa0\xce\xac\x11\xec\xee\xa3\x9e\xde\x07\xa6\x70\x47\xd0\xfa\xb4\xc0\x28\xf5\xf7\xb3\xf9\xfc\x85\x9d\x69\xa9\x37\xb1\x0b\x01\xab\xdd\xd9\x52\xd1\x10\xea\xb4\xff\x1c\x49\x8d\x94\x8e\xe4\x50\x02\x51\xc6\x52\xa9\x8f\x7c\x7e\xbf\xc1\x74\xa7\xe1\x3e\xc3\x74\x7e\xaf\x33\xa7\xe3\x2b\xb4\x87\xee\x61\x7a\xcb\xa6\x96\xcc\xe4\x6a\x4a\x95\xed\x4f\x8f\x70\x29\xf6\xe9\x68\x7c\xf4\xf2\x75\xe2\xf7\x00\x00\x00\xff\xff\xdc\xca\xb4\x55\x2b\x02\x00\x00") - -func data_app_conf_bytes() ([]byte, error) { - return bindata_read( - _data_app_conf, - "data/app.conf", - ) -} - -func data_app_conf() (*asset, error) { - bytes, err := data_app_conf_bytes() - if err != nil { - return nil, err - } - - info := bindata_file_info{name: "data/app.conf", size: 555, mode: os.FileMode(420), modTime: time.Unix(1431628605, 0)} - a := &asset{bytes: bytes, info: info} - return a, nil -} - -var _data_cloudwatch_logs_conf = []byte("\x1f\x8b\x08\x00\x00\x09\x6e\x88\x00\xff\x6c\x8e\xb1\x0a\xc2\x40\x0c\x86\xf7\x3c\x45\x97\x8c\x9a\xbd\xe0\xe0\xe6\x22\x14\x9c\xa4\x94\x92\xd2\x58\x0a\xbd\xde\x91\x4b\x75\x28\x7d\x77\x63\x41\x70\x70\xc8\x90\xff\xfb\x42\xfe\x7a\x90\x59\x94\xa7\x06\xb2\xb1\x49\xfb\x18\x27\x29\x4e\x05\x3d\x59\x89\x5f\x79\x8a\x43\x26\x76\xc7\x0e\x3b\x07\xa8\x77\xe4\x39\x2d\xc9\x23\x35\xe2\x94\x8e\xbe\x37\xf0\x7b\xfb\x47\x00\x9f\x76\xd0\xb8\xa4\x76\xe6\xf0\x31\xd7\xf5\x5c\x55\xdb\xb6\x83\x6c\x2a\x1c\xbe\x24\xa9\x74\x31\x1a\xf4\xfe\xd3\xc6\xe0\xbd\xa2\x06\x36\x27\xd8\x13\x76\x84\xf7\x12\x2f\x25\x5e\x4b\xbc\xc1\x3b\x00\x00\xff\xff\xe9\xef\x6e\x68\xc3\x00\x00\x00") - -func data_cloudwatch_logs_conf_bytes() ([]byte, error) { - return bindata_read( - _data_cloudwatch_logs_conf, - "data/cloudwatch-logs.conf", - ) -} - -func data_cloudwatch_logs_conf() (*asset, error) { - bytes, err := data_cloudwatch_logs_conf_bytes() - if err != nil { - return nil, err - } +var _dataGitRestoreMtime = []byte("\x1f\x8b\x08\x00\x00\x09\x6e\x88\x00\xff\x7c\x54\x4d\x8f\x1a\x39\x10\xbd\xf7\xaf\xa8\x15\x5a\xd1\x2d\x61\xf7\x30\x3b\xda\xdd\x41\x42\xd1\x24\xe1\x90\x43\xa4\x51\xa4\x9c\x08\x07\xd3\x5d\x80\x07\xb7\xdd\x63\x57\xc3\xf0\xef\x53\xee\x0f\x06\x82\x12\x5f\x6c\x5c\xaf\xde\xab\x7a\x65\x7a\xf4\x57\xde\x04\x9f\xaf\xb5\xcd\xd1\x1e\xa0\x3e\xd1\xce\xd9\x24\x19\xc1\x8e\xa8\x9e\xe5\x79\x20\x55\xec\xdd\x01\xfd\xc6\xb8\xa3\x2c\x5c\x95\xbf\x36\x18\x48\x3b\x1b\xf2\xe9\xe3\xbf\x0f\x0f\xff\xdd\xe5\xc7\x9d\xa2\x20\x68\x87\x02\x5f\x1b\x7d\x50\x06\x2d\x09\xb7\x11\x4d\x40\xc1\x19\x95\x26\x41\xba\xc2\x20\x36\xce\x8b\xad\xa6\x7c\xfa\xcf\xfd\xff\x0f\xf7\xf7\x8f\xa3\xe1\x10\x05\x3f\x2a\x8f\x62\xed\x2c\x06\x60\xb9\xc0\x0a\x12\x3e\x35\xde\x33\x19\x94\xda\x43\xd5\x04\x82\x35\x02\xb9\x5a\x18\x3c\xa0\x01\xb7\x81\xa3\xf3\x7b\x20\x8f\x28\x99\xe1\x7b\x50\x5b\x9c\x01\x2b\x08\xcf\x35\x3a\xe6\xab\xa2\xb0\x58\x33\x35\x2c\x6b\x45\xbb\x50\x63\x11\xa4\x94\xab\x28\x78\x82\x12\x37\xaa\x31\x04\x4d\x5d\x2a\x42\x50\xc6\xc0\x46\x1b\x0c\x1c\x5d\xbc\xa9\xaa\x36\x4c\x47\x0e\x9c\x35\xa7\x01\xd3\x9e\xb9\x57\xf8\xb6\x78\xfa\xfc\x75\x01\xca\x96\x5d\x0e\x68\x0b\x32\x2f\x5d\x31\xe3\xec\xdf\xd4\xd0\xe7\x30\x28\x49\x74\x55\x3b\x4f\x10\x9a\x75\xed\x5d\x81\x21\x4c\x20\xec\x0c\xbe\x9d\x03\x27\xbe\x71\x41\xc6\xb2\x93\x24\x4a\x18\xcd\x0e\xcc\x21\x20\xa5\x59\xc2\x5e\x42\x0c\x45\xd9\x94\xb1\x52\xf9\xed\x61\x39\x9d\xad\x80\x03\xcb\x3e\x4f\x16\x8d\x67\xef\x56\xd9\x2c\x01\x5e\x7a\x33\x10\x4a\x1d\x22\x63\x1a\xcf\x59\xcc\x78\xbf\x37\xda\xee\xbb\xfb\x2e\x29\xae\x41\x5d\xaa\xb2\x4c\x07\xa8\x47\x13\xf7\x0e\x9b\xb5\x58\x06\x5d\x4a\xb0\xf4\x0d\x13\x6b\x79\xe7\x68\x12\x1b\xe7\x38\xf7\x78\x76\x8f\x13\x8f\xca\xdc\x88\xf7\x95\x8f\x25\x9b\x3a\x8e\xb8\x3e\xf3\x1a\x11\x57\x1f\xe0\xca\x2a\x7e\xb2\x69\x97\x91\x5d\xc1\xa2\x7e\x14\x8c\x3c\xad\xf0\x2d\xcb\x1f\x9b\x1d\x7e\xbf\x38\x6d\xd3\xae\x91\x88\xcf\xd8\x80\xa4\x1d\x34\x0f\xe8\x2e\x61\x5d\xb7\x7e\x89\xb3\x3a\x4f\x57\x3e\xbb\x1a\x6d\xda\x8e\x58\x86\xda\x68\x4a\xc7\x0c\x83\xf8\xe7\x29\x76\xca\x6e\xb1\x04\x21\x6a\x8f\x44\xa7\xf9\xdf\x8a\xeb\x9e\xdc\x54\x76\xd1\x29\x95\xae\xa1\xf9\x25\xfd\x97\xe7\x45\xf7\x2a\x78\x80\x6d\x7b\x5d\x11\xb2\x83\x76\x6d\xb6\xa1\x79\xbb\xf1\xbd\xd7\x75\x9a\x0d\x0f\xc3\x3a\x6a\xef\x67\x50\x38\x4b\xda\x36\x98\x0c\xa1\x1e\xae\x3c\x85\xa3\x66\x0f\xc6\xb3\xf1\x2f\x6f\xe3\xcc\xd9\xf5\xf5\x83\xab\x5f\x8a\xe9\x2a\xb9\x18\xdf\xa5\xe9\xd1\xdd\x6b\xdf\xcf\x9e\xf7\xa3\x6b\x3d\xbd\x42\x8c\x6a\xaf\xf9\x4b\xd0\x7a\xdc\x79\x7e\x15\xe6\xb9\x34\x31\xd4\x66\x4e\x20\xed\x71\xed\x76\x7e\x9c\x01\xdf\x55\x87\x61\x19\x67\xb7\x69\x2c\x3e\xeb\xfa\x1d\xc1\xd3\xf0\x15\xe0\xff\xa9\xc5\x0f\x97\x06\xdd\x16\xbf\xf6\xa8\xf6\xc9\xcf\x00\x00\x00\xff\xff\xd5\xa9\x9e\x5b\x46\x05\x00\x00") - info := bindata_file_info{name: "data/cloudwatch-logs.conf", size: 195, mode: os.FileMode(420), modTime: time.Unix(1431483020, 0)} - a := &asset{bytes: bytes, info: info} - return a, nil -} - -var _data_git_restore_mtime = []byte("\x1f\x8b\x08\x00\x00\x09\x6e\x88\x00\xff\x7c\x54\x4d\x8f\x1a\x39\x10\xbd\xf7\xaf\xa8\x15\x5a\xd1\x2d\x61\xf7\x30\x3b\xda\xdd\x41\x42\xd1\x24\xe1\x90\x43\xa4\x51\xa4\x9c\x08\x07\xd3\x5d\x80\x07\xb7\xdd\x63\x57\xc3\xf0\xef\x53\xee\x0f\x06\x82\x12\x5f\x6c\x5c\xaf\xde\xab\x7a\x65\x7a\xf4\x57\xde\x04\x9f\xaf\xb5\xcd\xd1\x1e\xa0\x3e\xd1\xce\xd9\x24\x19\xc1\x8e\xa8\x9e\xe5\x79\x20\x55\xec\xdd\x01\xfd\xc6\xb8\xa3\x2c\x5c\x95\xbf\x36\x18\x48\x3b\x1b\xf2\xe9\xe3\xbf\x0f\x0f\xff\xdd\xe5\xc7\x9d\xa2\x20\x68\x87\x02\x5f\x1b\x7d\x50\x06\x2d\x09\xb7\x11\x4d\x40\xc1\x19\x95\x26\x41\xba\xc2\x20\x36\xce\x8b\xad\xa6\x7c\xfa\xcf\xfd\xff\x0f\xf7\xf7\x8f\xa3\xe1\x10\x05\x3f\x2a\x8f\x62\xed\x2c\x06\x60\xb9\xc0\x0a\x12\x3e\x35\xde\x33\x19\x94\xda\x43\xd5\x04\x82\x35\x02\xb9\x5a\x18\x3c\xa0\x01\xb7\x81\xa3\xf3\x7b\x20\x8f\x28\x99\xe1\x7b\x50\x5b\x9c\x01\x2b\x08\xcf\x35\x3a\xe6\xab\xa2\xb0\x58\x33\x35\x2c\x6b\x45\xbb\x50\x63\x11\xa4\x94\xab\x28\x78\x82\x12\x37\xaa\x31\x04\x4d\x5d\x2a\x42\x50\xc6\xc0\x46\x1b\x0c\x1c\x5d\xbc\xa9\xaa\x36\x4c\x47\x0e\x9c\x35\xa7\x01\xd3\x9e\xb9\x57\xf8\xb6\x78\xfa\xfc\x75\x01\xca\x96\x5d\x0e\x68\x0b\x32\x2f\x5d\x31\xe3\xec\xdf\xd4\xd0\xe7\x30\x28\x49\x74\x55\x3b\x4f\x10\x9a\x75\xed\x5d\x81\x21\x4c\x20\xec\x0c\xbe\x9d\x03\x27\xbe\x71\x41\xc6\xb2\x93\x24\x4a\x18\xcd\x0e\xcc\x21\x20\xa5\x59\xc2\x5e\x42\x0c\x45\xd9\x94\xb1\x52\xf9\xed\x61\x39\x9d\xad\x80\x03\xcb\x3e\x4f\x16\x8d\x67\xef\x56\xd9\x2c\x01\x5e\x7a\x33\x10\x4a\x1d\x22\x63\x1a\xcf\x59\xcc\x78\xbf\x37\xda\xee\xbb\xfb\x2e\x29\xae\x41\x5d\xaa\xb2\x4c\x07\xa8\x47\x13\xf7\x0e\x9b\xb5\x58\x06\x5d\x4a\xb0\xf4\x0d\x13\x6b\x79\xe7\x68\x12\x1b\xe7\x38\xf7\x78\x76\x8f\x13\x8f\xca\xdc\x88\xf7\x95\x8f\x25\x9b\x3a\x8e\xb8\x3e\xf3\x1a\x11\x57\x1f\xe0\xca\x2a\x7e\xb2\x69\x97\x91\x5d\xc1\xa2\x7e\x14\x8c\x3c\xad\xf0\x2d\xcb\x1f\x9b\x1d\x7e\xbf\x38\x6d\xd3\xae\x91\x88\xcf\xd8\x80\xa4\x1d\x34\x0f\xe8\x2e\x61\x5d\xb7\x7e\x89\xb3\x3a\x4f\x57\x3e\xbb\x1a\x6d\xda\x8e\x58\x86\xda\x68\x4a\xc7\x0c\x83\xf8\xe7\x29\x76\xca\x6e\xb1\x04\x21\x6a\x8f\x44\xa7\xf9\xdf\x8a\xeb\x9e\xdc\x54\x76\xd1\x29\x95\xae\xa1\xf9\x25\xfd\x97\xe7\x45\xf7\x2a\x78\x80\x6d\x7b\x5d\x11\xb2\x83\x76\x6d\xb6\xa1\x79\xbb\xf1\xbd\xd7\x75\x9a\x0d\x0f\xc3\x3a\x6a\xef\x67\x50\x38\x4b\xda\x36\x98\x0c\xa1\x1e\xae\x3c\x85\xa3\x66\x0f\xc6\xb3\xf1\x2f\x6f\xe3\xcc\xd9\xf5\xf5\x83\xab\x5f\x8a\xe9\x2a\xb9\x18\xdf\xa5\xe9\xd1\xdd\x6b\xdf\xcf\x9e\xf7\xa3\x6b\x3d\xbd\x42\x8c\x6a\xaf\xf9\x4b\xd0\x7a\xdc\x79\x7e\x15\xe6\xb9\x34\x31\xd4\x66\x4e\x20\xed\x71\xed\x76\x7e\x9c\x01\xdf\x55\x87\x61\x19\x67\xb7\x69\x2c\x3e\xeb\xfa\x1d\xc1\xd3\xf0\x15\xe0\xff\xa9\xc5\x0f\x97\x06\xdd\x16\xbf\xf6\xa8\xf6\xc9\xcf\x00\x00\x00\xff\xff\xd5\xa9\x9e\x5b\x46\x05\x00\x00") - -func data_git_restore_mtime_bytes() ([]byte, error) { - return bindata_read( - _data_git_restore_mtime, +func dataGitRestoreMtimeBytes() ([]byte, error) { + return bindataRead( + _dataGitRestoreMtime, "data/git-restore-mtime", ) } -func data_git_restore_mtime() (*asset, error) { - bytes, err := data_git_restore_mtime_bytes() - if err != nil { - return nil, err - } - - info := bindata_file_info{name: "data/git-restore-mtime", size: 1350, mode: os.FileMode(493), modTime: time.Unix(1433182975, 0)} - a := &asset{bytes: bytes, info: info} - return a, nil -} - -var _data_netrc = []byte("\x1f\x8b\x08\x00\x00\x09\x6e\x88\x00\xff\xca\x4d\x4c\xce\xc8\xcc\x4b\x55\x48\xcf\x2c\xc9\x28\x4d\xd2\x4b\xce\xcf\x55\xc8\xc9\x4f\xcf\xcc\x53\x48\xce\xcf\x2b\xcb\xaf\xd0\x4f\x2a\xcd\xcc\x49\x51\x28\x48\x2c\x2e\x2e\xcf\x2f\x4a\x51\xa8\xae\x76\xf7\x0c\xf1\x08\x75\x8a\x0f\xf1\xf7\x76\xf5\xab\xad\x05\x04\x00\x00\xff\xff\xb8\x71\x2e\x98\x3f\x00\x00\x00") - -func data_netrc_bytes() ([]byte, error) { - return bindata_read( - _data_netrc, - "data/netrc", - ) -} - -func data_netrc() (*asset, error) { - bytes, err := data_netrc_bytes() +func dataGitRestoreMtime() (*asset, error) { + bytes, err := dataGitRestoreMtimeBytes() if err != nil { return nil, err } - info := bindata_file_info{name: "data/netrc", size: 63, mode: os.FileMode(420), modTime: time.Unix(1433182975, 0)} - a := &asset{bytes: bytes, info: info} - return a, nil -} - -var _data_packer_json = []byte("\x1f\x8b\x08\x00\x00\x09\x6e\x88\x00\xff\xbc\x95\x4b\x6f\xda\x40\x10\xc7\xef\x7c\x8a\x91\x2f\xb9\x60\xac\xb6\x52\x55\xe5\xe6\x22\xab\xca\x21\x49\x85\xfb\x38\x34\x91\xb5\xac\x07\x58\x61\xef\xae\xf6\x41\x93\x5a\xfe\xee\xdd\xf5\x03\x8c\x03\x55\xd5\xa0\x5e\x40\xcc\xfc\xbc\x33\xff\xff\xcc\x9a\x6a\x02\x10\xe4\xa8\xa9\x62\xd2\x30\xc1\x83\x6b\x08\xe6\x82\xef\xc4\x13\xc4\x52\x06\x53\x9f\xde\x11\xc5\xc8\xb2\x40\xed\x92\x9e\x77\xa1\xbb\xf8\x36\x71\xbf\xb8\x2d\x8a\x69\x1b\x49\xef\xbf\x2e\xe6\xa3\xd8\xc7\x38\x4d\xb2\xf8\xf6\xc6\x1f\x4a\x4a\x16\x7e\x78\x4f\x49\xfe\x0e\x31\xe8\xf2\xf1\xf7\x34\x5b\x24\x9f\x6e\xee\xef\x3c\x61\x75\x88\x44\x9b\xf0\xcd\x30\x1d\xcf\xe7\x49\x9a\xfa\x74\x55\x21\xdf\xc1\xc3\x20\xfa\x10\xd4\xf5\x90\x4d\x93\xf9\x22\xf9\x32\x66\xdb\x68\xc3\x3a\xb4\x6e\x14\x2d\x2d\x2b\x72\x54\x5e\xd0\x8f\xe6\xf9\x56\x96\xcb\x98\x67\x89\x6d\xbb\xe4\x97\xe0\x21\x2e\x75\x57\xc1\xe5\x14\xae\x3b\x87\xaa\xca\x6a\x54\x5d\x81\x56\xc1\xa0\x19\x87\x12\x4a\x51\xeb\x6c\x8b\xcf\x2f\xf0\x17\xbd\x3b\x5c\x23\x55\x68\x4e\xe2\x83\xf6\x0f\xb8\xb0\x8a\x62\xe6\x2c\x3d\xc2\x7b\xbb\x8f\x61\xc6\xb5\x21\xdc\xe1\xbd\x34\xf3\x76\x56\x32\xaa\xc4\xe0\x3c\xbd\xc9\xfc\x19\x9c\x94\x0d\x61\x97\x96\x1b\x3b\x50\x53\xb2\xac\xcf\xed\xab\xf9\x15\xf0\x95\xc2\xaa\x32\xac\x44\x57\xa4\x94\xc3\xba\xca\xf2\xcc\x90\xf5\x61\x69\xda\xc5\x39\x7b\x4c\x37\x93\xa0\x63\xeb\x49\xff\xf9\xd8\x8c\x4c\x2a\xb1\x63\xda\xf9\xff\xc7\xb1\xe9\x0d\x16\xc5\xa1\x07\x7c\x42\x6a\x0d\x66\x54\x94\x25\xe1\xb9\x27\xfc\x62\x54\x15\xcc\xbe\x11\xa5\xa1\xae\x41\xdb\x5c\x40\x98\x40\x98\x82\xde\xc0\x95\x4f\x7d\x26\x66\xe3\x52\x57\x43\x0f\x0b\xc6\x71\x5f\xb6\x89\x95\xdb\x9c\x29\x88\x9a\xb6\xf7\xa4\x8b\xd3\x8d\xf8\xc9\xa1\xb5\xf0\xba\xfd\xea\xa9\x0e\x7a\x6c\xa5\x4d\x4f\x4b\x58\xb1\x02\xc7\xa3\x3e\x72\xac\xbd\x69\xde\xb3\xe8\xc0\xb9\x2b\x6c\x18\x27\xfd\x15\x1e\x16\x3c\x57\x68\xe4\xd5\x29\x8d\x1a\x0d\x84\x78\xa4\x2e\x3f\x21\x39\xb2\x5a\x45\x85\xa0\xa4\x88\x96\x8c\x47\xb9\xa0\x5b\x54\xa1\x33\x5d\x0a\x8d\x10\x4a\x20\x52\xc2\xbf\x3d\x25\xdd\xdb\xe4\xef\x7c\xfb\x8f\xa3\x57\x25\x84\x6a\x75\x91\xa9\x3a\x8d\x33\x2a\xf8\xea\xfc\x24\x4d\x29\xa3\x3d\xf5\x4a\x07\x2e\xb3\xf8\x3b\x38\x6a\x0a\x22\x34\x34\x62\x9c\x99\x51\x9f\xaf\x9f\xd8\x45\xfa\x6d\x17\xab\xd9\x24\xa0\xcd\x5f\x5b\x44\xd6\xc8\xcd\x70\x17\x4f\x30\x5e\xd0\x48\x87\x7f\x19\x4d\xea\xc9\xef\x00\x00\x00\xff\xff\x5e\x1f\x04\x09\x34\x07\x00\x00") - -func data_packer_json_bytes() ([]byte, error) { - return bindata_read( - _data_packer_json, - "data/packer.json", - ) -} - -func data_packer_json() (*asset, error) { - bytes, err := data_packer_json_bytes() - if err != nil { - return nil, err - } - - info := bindata_file_info{name: "data/packer.json", size: 1844, mode: os.FileMode(420), modTime: time.Unix(1433182975, 0)} - a := &asset{bytes: bytes, info: info} + info := bindataFileInfo{name: "data/git-restore-mtime", size: 1350, mode: os.FileMode(493), modTime: time.Unix(1459609522, 0)} + a := &asset{bytes: bytes, info: info} return a, nil } @@ -176,6 +103,17 @@ func Asset(name string) ([]byte, error) { return nil, fmt.Errorf("Asset %s not found", name) } +// MustAsset is like Asset but panics when Asset would return an error. +// It simplifies safe initialization of global variables. +func MustAsset(name string) []byte { + a, err := Asset(name) + if err != nil { + panic("asset: Asset(" + name + "): " + err.Error()) + } + + return a +} + // AssetInfo loads and returns the asset info for the given name. // It returns an error if the asset could not be found or // could not be loaded. @@ -202,11 +140,7 @@ func AssetNames() []string { // _bindata is a table, holding each asset generator, mapped to its name. var _bindata = map[string]func() (*asset, error){ - "data/app.conf": data_app_conf, - "data/cloudwatch-logs.conf": data_cloudwatch_logs_conf, - "data/git-restore-mtime": data_git_restore_mtime, - "data/netrc": data_netrc, - "data/packer.json": data_packer_json, + "data/git-restore-mtime": dataGitRestoreMtime, } // AssetDir returns the file names below a certain @@ -238,74 +172,66 @@ func AssetDir(name string) ([]string, error) { return nil, fmt.Errorf("Asset %s not found", name) } rv := make([]string, 0, len(node.Children)) - for name := range node.Children { - rv = append(rv, name) + for childName := range node.Children { + rv = append(rv, childName) } return rv, nil } -type _bintree_t struct { - Func func() (*asset, error) - Children map[string]*_bintree_t +type bintree struct { + Func func() (*asset, error) + Children map[string]*bintree } -var _bintree = &_bintree_t{nil, map[string]*_bintree_t{ - "data": &_bintree_t{nil, map[string]*_bintree_t{ - "app.conf": &_bintree_t{data_app_conf, map[string]*_bintree_t{ - }}, - "cloudwatch-logs.conf": &_bintree_t{data_cloudwatch_logs_conf, map[string]*_bintree_t{ - }}, - "git-restore-mtime": &_bintree_t{data_git_restore_mtime, map[string]*_bintree_t{ - }}, - "netrc": &_bintree_t{data_netrc, map[string]*_bintree_t{ - }}, - "packer.json": &_bintree_t{data_packer_json, map[string]*_bintree_t{ - }}, +var _bintree = &bintree{nil, map[string]*bintree{ + "data": &bintree{nil, map[string]*bintree{ + "git-restore-mtime": &bintree{dataGitRestoreMtime, map[string]*bintree{}}, }}, }} -// Restore an asset under the given directory +// RestoreAsset restores an asset under the given directory func RestoreAsset(dir, name string) error { - data, err := Asset(name) - if err != nil { - return err - } - info, err := AssetInfo(name) - if err != nil { - return err - } - err = os.MkdirAll(_filePath(dir, path.Dir(name)), os.FileMode(0755)) - if err != nil { - return err - } - err = ioutil.WriteFile(_filePath(dir, name), data, info.Mode()) - if err != nil { - return err - } - err = os.Chtimes(_filePath(dir, name), info.ModTime(), info.ModTime()) - if err != nil { - return err - } - return nil + data, err := Asset(name) + if err != nil { + return err + } + info, err := AssetInfo(name) + if err != nil { + return err + } + err = os.MkdirAll(_filePath(dir, filepath.Dir(name)), os.FileMode(0755)) + if err != nil { + return err + } + err = ioutil.WriteFile(_filePath(dir, name), data, info.Mode()) + if err != nil { + return err + } + err = os.Chtimes(_filePath(dir, name), info.ModTime(), info.ModTime()) + if err != nil { + return err + } + return nil } -// Restore assets under the given directory recursively +// RestoreAssets restores an asset under the given directory recursively func RestoreAssets(dir, name string) error { - children, err := AssetDir(name) - if err != nil { // File - return RestoreAsset(dir, name) - } else { // Dir - for _, child := range children { - err = RestoreAssets(dir, path.Join(name, child)) - if err != nil { - return err - } - } - } - return nil + children, err := AssetDir(name) + // File + if err != nil { + return RestoreAsset(dir, name) + } + // Dir + for _, child := range children { + err = RestoreAssets(dir, filepath.Join(name, child)) + if err != nil { + return err + } + } + return nil } func _filePath(dir, name string) string { - cannonicalName := strings.Replace(name, "\\", "/", -1) - return filepath.Join(append([]string{dir}, strings.Split(cannonicalName, "/")...)...) + cannonicalName := strings.Replace(name, "\\", "/", -1) + return filepath.Join(append([]string{dir}, strings.Split(cannonicalName, "/")...)...) } diff --git a/api/cmd/build/data/app.conf b/api/cmd/build/data/app.conf deleted file mode 100644 index a3700716cf..0000000000 --- a/api/cmd/build/data/app.conf +++ /dev/null @@ -1,17 +0,0 @@ -start on runlevel [2345] and started docker -stop on runlevel [!2345] - -respawn -respawn limit unlimited - -post-start script - stack=$(curl -s http://169.254.169.254/latest/user-data | jq -r ".signal.stack") - resource=$(curl -s http://169.254.169.254/latest/user-data | jq -r ".signal.resource") - cfn-signal -s true --stack=$stack --resource=$resource >/dev/null 2>&1 -end script - -post-stop exec sleep 1 - -script - curl -s http://169.254.169.254/latest/user-data | docker run --sig-proxy -i -v /var/run/docker.sock:/var/run/docker.sock convox/init -end script diff --git a/api/cmd/build/data/cloudwatch-logs.conf b/api/cmd/build/data/cloudwatch-logs.conf deleted file mode 100644 index dc07d6bae2..0000000000 --- a/api/cmd/build/data/cloudwatch-logs.conf +++ /dev/null @@ -1,8 +0,0 @@ -[general] -state_file = /var/awslogs/agent-state - -[/var/log/upstart/app.log] -file = /var/log/upstart/app.log -log_group_name = {{APP}} -log_stream_name = preboot -datetime_format = %d/%b/%Y:%H:%M:%S diff --git a/api/cmd/build/data/netrc b/api/cmd/build/data/netrc deleted file mode 100644 index 852d5e2148..0000000000 --- a/api/cmd/build/data/netrc +++ /dev/null @@ -1 +0,0 @@ -machine github.com login convox/build password {{GITHUB_TOKEN}} \ No newline at end of file diff --git a/api/cmd/build/data/packer.json b/api/cmd/build/data/packer.json deleted file mode 100644 index cb78630f29..0000000000 --- a/api/cmd/build/data/packer.json +++ /dev/null @@ -1,77 +0,0 @@ -{ - "description": "Convox App", - "variables": { - "NAME": null, - "SOURCE": null, - "BASE_AMI": "ami-86cad3ee", - "AWS_REGION": "us-east-1", - "AWS_ACCESS": "{{env \"AWS_ACCESS\"}}", - "AWS_SECRET": "{{env \"AWS_SECRET\"}}" - }, - "builders": [ - { - "type": "amazon-ebs", - "region": "{{user \"AWS_REGION\"}}", - "access_key": "{{user \"AWS_ACCESS\"}}", - "secret_key": "{{user \"AWS_SECRET\"}}", - "source_ami": "{{user \"BASE_AMI\"}}", - "instance_type": "t2.micro", - "ssh_username": "ubuntu", - "ami_name": "{{user \"NAME\"}}-{{timestamp}}", - "run_tags": { - "Name": "{{user \"NAME\"}}-builder" - } - } - ], - "provisioners": [ - { - "type": "shell", - "execute_command": "env {{ .Vars }} sudo -E -S sh '{{ .Path }}'", - "inline": [ - "mkdir /build", - "chown ubuntu:ubuntu /build" - ] - }, - { - "type": "file", - "source": "{{user \"SOURCE\"}}/", - "destination": "/build" - }, - { - "type": "shell", - "inline": [ - "set -e", - "cd /build", - "/usr/local/bin/docker-compose -p app build", - "/usr/local/bin/docker-compose -p app pull" - ] - }, - { - "type": "shell", - "execute_command": "env {{ .Vars }} sudo -E -S sh '{{ .Path }}'", - "inline": [ - "rm -rf /build" - ] - }, - { - "type": "file", - "source": "app.conf", - "destination": "/tmp/app.conf" - }, - { - "type": "shell", - "execute_command": "{{ .Vars }} sudo -E -S sh '{{ .Path }}'", - "inline": [ - "mv /tmp/app.conf /etc/init/app.conf" - ] - }, - { - "type": "shell", - "execute_command": "{{ .Vars }} sudo -E -S sh '{{ .Path }}'", - "inline": [ - "docker pull convox/agent", - "docker pull convox/init" - ] - } - ] -} diff --git a/api/cmd/build/examples/docker-compose/docker-compose.yml b/api/cmd/build/examples/docker-compose/docker-compose.yml deleted file mode 100644 index 6c4fb07cb7..0000000000 --- a/api/cmd/build/examples/docker-compose/docker-compose.yml +++ /dev/null @@ -1,4 +0,0 @@ -web: - image: httpd - ports: - - 80:80 \ No newline at end of file diff --git a/api/cmd/build/examples/env_file/Dockerfile b/api/cmd/build/examples/env_file/Dockerfile deleted file mode 100644 index ac6be4716b..0000000000 --- a/api/cmd/build/examples/env_file/Dockerfile +++ /dev/null @@ -1 +0,0 @@ -FROM gliderlabs/alpine \ No newline at end of file diff --git a/api/cmd/build/examples/env_file/Gemfile b/api/cmd/build/examples/env_file/Gemfile deleted file mode 100644 index f3e9834d39..0000000000 --- a/api/cmd/build/examples/env_file/Gemfile +++ /dev/null @@ -1,3 +0,0 @@ -source :rubygems - -gem "rack" diff --git a/api/cmd/build/examples/env_file/Gemfile.lock b/api/cmd/build/examples/env_file/Gemfile.lock deleted file mode 100644 index 817ad3670e..0000000000 --- a/api/cmd/build/examples/env_file/Gemfile.lock +++ /dev/null @@ -1,10 +0,0 @@ -GEM - remote: http://rubygems.org/ - specs: - rack (1.6.4) - -PLATFORMS - ruby - -DEPENDENCIES - rack diff --git a/api/cmd/build/examples/env_file/Procfile b/api/cmd/build/examples/env_file/Procfile deleted file mode 100644 index d67308f452..0000000000 --- a/api/cmd/build/examples/env_file/Procfile +++ /dev/null @@ -1 +0,0 @@ -web: bundle exec rackup -p 3000 config.ru diff --git a/api/cmd/build/examples/env_file/config.ru b/api/cmd/build/examples/env_file/config.ru deleted file mode 100644 index 3bc7c21f8c..0000000000 --- a/api/cmd/build/examples/env_file/config.ru +++ /dev/null @@ -1,3 +0,0 @@ -require 'json' - -run lambda { |env| [200, {'Content-Type'=>'application/json'}, StringIO.new(JSON.pretty_generate(ENV.to_h))] } diff --git a/api/cmd/build/helpers.go b/api/cmd/build/helpers.go deleted file mode 100644 index 749bb0a991..0000000000 --- a/api/cmd/build/helpers.go +++ /dev/null @@ -1,11 +0,0 @@ -package main - -import "os" - -func exists(filename string) bool { - if _, err := os.Stat(filename); os.IsNotExist(err) { - return false - } - - return true -} diff --git a/api/cmd/build/main.go b/api/cmd/build/main.go index 344cb1df40..4fb157461f 100644 --- a/api/cmd/build/main.go +++ b/api/cmd/build/main.go @@ -1,346 +1,163 @@ package main import ( - "bufio" - "bytes" "encoding/base64" - "flag" "fmt" - "io" "io/ioutil" "net/url" "os" "os/exec" - "path/filepath" "strings" "github.com/convox/rack/api/manifest" + "github.com/convox/rack/client" +) + +var ( + manifestPath string + app string + cache = true + registryAddress string + buildId string + repository string + rackClient = client.New(os.Getenv("RACK_HOST"), os.Getenv("RACK_PASSWORD"), "build") ) func init() { - flag.Usage = func() { - fmt.Fprintf(os.Stderr, "build: turn a convox application into an ami\n\n") - fmt.Fprintf(os.Stderr, "Usage: %s \n", os.Args[0]) - flag.PrintDefaults() - fmt.Fprintf(os.Stderr, "\nExample:\n build example-sinatra https://github.com/convox-examples/sinatra.git\n") + app = os.Getenv("APP") + buildId = os.Getenv("BUILD") + registryAddress = os.Getenv("REGISTRY_ADDRESS") + repository = os.Getenv("REPOSITORY") + + manifestPath = os.Getenv("MANIFEST_PATH") + if manifestPath == "" { + manifestPath = "docker-compose.yml" + } + + if os.Getenv("NO_CACHE") != "" { + cache = false } } func main() { - id := flag.String("id", "", "tag the build with this id") - push := flag.String("push", "", "push build to this prefix when done") - dockercfg := flag.String("dockercfg", "", "dockercfg auth json for pull") - noCache := flag.Bool("no-cache", false, "skip the docker cache") - manifestPath := flag.String("manifest", "docker-compose.yml", "docker compose filename") - flatten := flag.String("flatten", "", "flatten images into a single namespace") - - flag.Parse() - - l := len(flag.Args()) - - if l < 2 { - flag.Usage() + if len(os.Args) != 2 { + fmt.Println("usage: build ") os.Exit(1) } - args := flag.Args() + src := os.Args[1] - app := positional(args, 0) - source := positional(args, 1) - - dir, err := clone(source, app) - - if err != nil { - die(err) + if src == "-" { + extractTar() + } else { + cloneGit(src) } - m, err := manifest.Read(dir, *manifestPath) + writeDockerAuth() - if err != nil { - die(err) - } + m, err := manifest.Read("src", manifestPath) + handleError(err) data, err := m.Raw() + handleError(err) - if err != nil { - die(err) - } - - scanner := bufio.NewScanner(bytes.NewReader(data)) - - for scanner.Scan() { - fmt.Printf("manifest|%s\n", scanner.Text()) - } - - if err := scanner.Err(); err != nil { - die(err) - } - - manifest.Stdout = prefixWriter("build") - manifest.Stderr = manifest.Stdout - - if err != nil { - die(err) - } - - if *dockercfg != "" { - err := os.MkdirAll("/root/.docker", 0700) - - if err != nil { - die(err) - } - - err = ioutil.WriteFile("/root/.docker/config.json", []byte(*dockercfg), 0400) - - if err != nil { - die(err) - } - } + handleErrors(m.Build(app, "src", cache)) + handleErrors(m.Push(app, registryAddress, buildId, repository)) - cache := !*noCache - - errors := m.Build(app, dir, cache) - - if len(errors) > 0 { - die(errors[0]) - } - - if *push != "" { - manifest.Stdout = prefixWriter("push") - manifest.Stderr = manifest.Stdout - - errors := m.Push(app, *push, *id, *flatten) - - if len(errors) > 0 { - die(errors[0]) - } - } + _, err = rackClient.UpdateBuild(os.Getenv("APP"), os.Getenv("BUILD"), string(data), "complete", "") + handleError(err) } -func die(err error) { - fmt.Fprintf(os.Stderr, "ERROR: %s\n", err) - os.Exit(1) -} - -func clone(source, app string) (string, error) { - tmp, err := ioutil.TempDir("", "repo") - +func handleError(err error) { if err != nil { - return "", err - } - - clone := filepath.Join(tmp, "clone") - - switch { - case isDir(source): - return source, nil - case source == "-": - err := extractTarball(os.Stdin, clone) - - if err != nil { - return "", err - } - default: - u, err := url.Parse(source) - - if err != nil { - return "", err - } - - // if URL has a fragment, i.e. http://github.com/nzoschke/httpd.git#1a2b4aac045609f09de34294de61b45344f419de - // split it off and pass along http://github.com/nzoschke/httpd.git for `git clone` - commitish := u.Fragment - u.Fragment = "" - repo := u.String() - - // if URL is a ssh/git url, i.e. ssh://user:base64(privatekey)@server/project.git - // decode and write private key to disk and pass along user@service:project.git for `git clone` - if u.Scheme == "ssh" { - repo = fmt.Sprintf("%s@%s%s", u.User.Username(), u.Host, u.Path) - - if pass, ok := u.User.Password(); ok { - key, err := base64.StdEncoding.DecodeString(pass) - - if err != nil { - die(err) - } - - err = os.Mkdir("/root/.ssh", 0700) - - if err != nil { - die(err) - } - - err = ioutil.WriteFile("/root/.ssh/id_rsa", key, 0400) - - if err != nil { - die(err) - } - } - - // don't interactive prompt for known hosts and fingerprints - os.Setenv("GIT_SSH_COMMAND", "ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no") - } - - if err = writeFile("/usr/local/bin/git-restore-mtime", "git-restore-mtime", 0755, nil); err != nil { - return "", err - } - - err = run("git", tmp, "git", "clone", "--progress", "-v", repo, clone) + fmt.Println(err.Error()) - if err != nil { - return "", err + _, cerr := rackClient.UpdateBuild(os.Getenv("APP"), os.Getenv("BUILD"), "", "failed", err.Error()) + if cerr != nil { + fmt.Println(cerr.Error()) + os.Exit(2) } - if commitish != "" { - err = run("git", clone, "git", "checkout", commitish) - - if err != nil { - return "", err - } - } - - err = run("git", clone, "/usr/local/bin/git-restore-mtime", ".") - - if err != nil { - return "", err - } - } - - return clone, nil -} - -func extractTarball(r io.Reader, base string) error { - cwd, err := os.Getwd() - - if err != nil { - return err - } - - defer os.Chdir(cwd) - - err = os.MkdirAll(base, 0755) - - if err != nil { - return err - } - - err = os.Chdir(base) - - if err != nil { - return err - } - - cmd := exec.Command("tar", "xz") - cmd.Stdin = r - err = cmd.Run() - - if err != nil { - return err - } - - return nil -} - -func prefixWriter(prefix string) io.Writer { - r, w := io.Pipe() - go prefixReader(r, prefix) - return w -} - -func dropCR(data []byte) []byte { - if len(data) > 0 && data[len(data)-1] == '\r' { - return data[0 : len(data)-1] + os.Exit(1) } - return data } -func scanLinesWithMax(data []byte, atEof bool) (advance int, token []byte, err error) { - if atEof && len(data) == 0 { - return 0, nil, nil +func handleErrors(errs []error) { + for _, err := range errs { + handleError(err) } - - if i := bytes.IndexByte(data, '\n'); i >= 0 { - return i + 1, dropCR(data[0:i]), nil - } - - if len(data) > 2048 { - return 2048, dropCR(data[0:2048]), nil - } - - if atEof { - return len(data), dropCR(data), nil - } - - return 0, nil, nil } -func prefixReader(r io.Reader, prefix string) { - scanner := bufio.NewScanner(r) - - scanner.Split(scanLinesWithMax) - - for scanner.Scan() { - fmt.Printf("%s|%s\n", prefix, scanner.Text()) - } - - if err := scanner.Err(); err != nil { - fmt.Printf("error|%s\n", err.Error()) - } +// extractTar makes a src directory, reads a .tgz from stdin and decompresses it into src +func extractTar() { + handleError(os.MkdirAll("src", 0755)) + run("src", "tar", "xz") } -func run(prefix, dir string, command string, args ...string) error { - cmd := exec.Command(command, args...) - cmd.Dir = dir - - stdout, err := cmd.StdoutPipe() - cmd.Stderr = cmd.Stdout - - if err != nil { - return err - } - - cmd.Start() - prefixReader(stdout, prefix) - err = cmd.Wait() +// cloneGit takes a URL to a git repo with an optional "commit-ish" hash, +// clones it, checks out the right commit-ish, and restores original file creation time +func cloneGit(s string) { + u, err := url.Parse(s) + handleError(err) + + // if URL has a fragment, i.e. http://github.com/nzoschke/httpd.git#1a2b4aac045609f09de34294de61b45344f419de + // split it off and pass along http://github.com/nzoschke/httpd.git for `git clone` + commitish := u.Fragment + u.Fragment = "" + repo := u.String() + + // if URL is a ssh/git url, i.e. ssh://user:base64(privatekey)@server/project.git + // decode and write private key to disk and pass along user@service:project.git for `git clone` + if u.Scheme == "ssh" { + repo = fmt.Sprintf("%s@%s%s", u.User.Username(), u.Host, u.Path) + + if pass, ok := u.User.Password(); ok { + key, err := base64.StdEncoding.DecodeString(pass) + handleError(err) + + handleError(os.Mkdir("/root/.ssh", 0700)) + handleError(ioutil.WriteFile("/root/.ssh/id_rsa", key, 0400)) + } - if err != nil { - writeSystem("error: " + err.Error()) + // don't interactive prompt for known hosts and fingerprints + os.Setenv("GIT_SSH_COMMAND", "ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no") } - return err -} + writeAsset("/usr/local/bin/git-restore-mtime", "git-restore-mtime", 0755, nil) -func isDir(dir string) bool { - fd, err := os.Open(dir) + run(".", "git", "clone", "--progress", repo, "src") - if err != nil { - return false + if commitish != "" { + run("src", "git", "checkout", commitish) } - stat, err := fd.Stat() - - if err != nil { - return false - } - - return stat.IsDir() + run("src", "/usr/local/bin/git-restore-mtime", ".") } -func positional(args []string, n int) string { - if len(args) > n { - return args[n] - } else { - return "" - } +// run optionally changes into a directory then executes the command and args +// connected to the OS stdin/stdout/stderr +func run(dir string, name string, arg ...string) { + sarg := fmt.Sprintf("%v", arg) + fmt.Printf("RUNNING: %s %s\n", name, sarg[1:len(sarg)-1]) + + // optionally change directory and change back at the end of this func + if dir != "" || dir != "." { + cwd, err := os.Getwd() + handleError(err) + defer os.Chdir(cwd) + handleError(os.Chdir(dir)) + } + + cmd := exec.Command(name, arg...) + cmd.Stdin = os.Stdin + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + handleError(cmd.Run()) } -func writeFile(target, name string, perms os.FileMode, replacements map[string]string) error { +func writeAsset(target, name string, perms os.FileMode, replacements map[string]string) { data, err := Asset(fmt.Sprintf("data/%s", name)) - - if err != nil { - return err - } + handleError(err) sdata := string(data) @@ -350,10 +167,13 @@ func writeFile(target, name string, perms os.FileMode, replacements map[string]s } } - return ioutil.WriteFile(target, []byte(sdata), perms) + handleError(ioutil.WriteFile(target, []byte(sdata), perms)) } -func writeSystem(message string) { - system := prefixWriter("system") - system.Write([]byte(message)) +func writeDockerAuth() { + auth := os.Getenv("DOCKER_AUTH") + if auth != "" { + handleError(os.MkdirAll("/root/.docker", 0700)) + handleError(ioutil.WriteFile("/root/.docker/config.json", []byte(auth), 0400)) + } } diff --git a/api/cmd/build/main_test.go b/api/cmd/build/main_test.go deleted file mode 100644 index 1066f888df..0000000000 --- a/api/cmd/build/main_test.go +++ /dev/null @@ -1,90 +0,0 @@ -package main - -import ( - "os" - "os/exec" - "path/filepath" - "strings" - "testing" -) - -/* convox/build integration tests - -This file tests convox/build by invoking the image in docker with -different example apps (in examples/) - -The `TestDockerRunning` test is invoked via make test to ensure -docker is avaible. Make also builds convox/build before the tests. - -*/ - -func TestGitUrl(t *testing.T) { - out := runBuild(t, "worker", "https://github.com/convox-examples/worker.git") - expected := `manifest|worker: -manifest| build: .` - actual := grepPrefix("manifest", out) - if actual != expected { - t.Errorf("Expected:\n %s \n got: \n %s", expected, actual) - } -} - -func TestDockerCompose(t *testing.T) { - out := runBuild(t, "test", "examples/docker-compose/") - expected := `manifest|web: -manifest| image: httpd -manifest| ports: -manifest| - 80:80` - actual := grepPrefix("manifest", out) - if actual != expected { - t.Errorf("Expected:\n %s \n got: \n %s", expected, actual) - } -} - -func TestEnvFile(t *testing.T) { - // cleanup generated docker-compose.yml - defer os.Remove("examples/env_file/docker-compose.yml") - out := runBuild(t, "test", "examples/env_file") - expected := `manifest|main: -manifest| build: . -manifest| ports: []` - actual := grepPrefix("manifest", out) - if actual != expected { - t.Errorf("Expected:\n %s \n got: \n %s", expected, actual) - } -} - -func grepPrefix(prefix, s string) string { - lines := strings.Split(s, "\n") - m := make([]string, 0) - - for i := range lines { - if strings.HasPrefix(lines[i], prefix+"|") { - m = append(m, lines[i]) - } - } - - return strings.Join(m, "\n") -} - -func runBuild(t *testing.T, name, source string) string { - var cmd *exec.Cmd - - fi, err := os.Stat(source) - if err != nil || !fi.IsDir() { - cmd = exec.Command("docker", "run", "-v", "/var/run/docker.sock:/var/run/docker.sock", - "convox/build", name, source) - } else { - // if source is a directory then mount it - hostPath, _ := filepath.Abs(source) - vmPath := "/convox-build" - args := []string{"run", "-v", "/var/run/docker.sock:/var/run/docker.sock", - "-v", hostPath + "/:" + vmPath, "convox/build", name, vmPath} - cmd = exec.Command("docker", args...) - } - - out, err := cmd.CombinedOutput() - if err != nil { - t.Error(err) - } - return string(out) -} diff --git a/api/controllers/builds.go b/api/controllers/builds.go index 4d0c9ad23d..ea69b51fd4 100644 --- a/api/controllers/builds.go +++ b/api/controllers/builds.go @@ -1,16 +1,13 @@ package controllers import ( - "bufio" "encoding/json" "fmt" - "io" "net/http" "os" "strings" "time" - "github.com/convox/rack/Godeps/_workspace/src/github.com/ddollar/logger" docker "github.com/convox/rack/Godeps/_workspace/src/github.com/fsouza/go-dockerclient" "github.com/convox/rack/Godeps/_workspace/src/github.com/gorilla/mux" "github.com/convox/rack/Godeps/_workspace/src/golang.org/x/net/websocket" @@ -18,23 +15,16 @@ import ( "github.com/convox/rack/api/httperr" "github.com/convox/rack/api/models" "github.com/convox/rack/api/provider" + "github.com/convox/rack/api/structs" ) func BuildList(rw http.ResponseWriter, r *http.Request) *httperr.Error { app := mux.Vars(r)["app"] - builds, err := models.ListBuilds(app) - - if err != nil { - return httperr.Server(err) - } - - _, err = models.GetApp(app) - + builds, err := provider.BuildList(app) if awsError(err) == "ValidationError" { return httperr.Errorf(404, "no such app: %s", app) } - if err != nil { return httperr.Server(err) } @@ -47,18 +37,13 @@ func BuildGet(rw http.ResponseWriter, r *http.Request) *httperr.Error { app := vars["app"] build := vars["build"] - _, err := models.GetApp(app) - + b, err := provider.BuildGet(app, build) if awsError(err) == "ValidationError" { return httperr.Errorf(404, "no such app: %s", app) } - - b, err := provider.BuildGet(app, build) - if err != nil && strings.HasPrefix(err.Error(), "no such build") { return httperr.Errorf(404, err.Error()) } - if err != nil { return httperr.Server(err) } @@ -80,166 +65,118 @@ func BuildDelete(rw http.ResponseWriter, r *http.Request) *httperr.Error { } func BuildCreate(rw http.ResponseWriter, r *http.Request) *httperr.Error { - build := models.NewBuild(mux.Vars(r)["app"]) - build.Description = r.FormValue("description") - - manifest := r.FormValue("manifest") // empty value will default "docker-compose.yml" in cmd/build - - // use deprecated "config" param if set and "manifest" is not - if config := r.FormValue("config"); config != "" && manifest == "" { - manifest = config - } + vars := mux.Vars(r) + app := vars["app"] - if build.IsRunning() { - err := fmt.Errorf("Another build is currently running. Please try again later.") - helpers.TrackError("build", err, map[string]interface{}{"at": "build.IsRunning"}) - return httperr.Server(err) - } + cache := !(r.FormValue("cache") == "false") + manifest := r.FormValue("manifest") + description := r.FormValue("description") - err := r.ParseMultipartForm(50 * 1024 * 1024) + repo := r.FormValue("repo") + index := r.FormValue("index") - if err != nil && err != http.ErrNotMultipart { - helpers.TrackError("build", err, map[string]interface{}{"at": "ParseMultipartForm"}) + source, _, err := r.FormFile("source") + if err != nil && err != http.ErrMissingFile && err != http.ErrNotMultipart { + helpers.TrackError("build", err, map[string]interface{}{"at": "FormFile"}) return httperr.Server(err) } - err = build.Save() - + // Log into private registries that we might pull from + // TODO: move to prodiver BuildCreate + err = models.LoginPrivateRegistries() if err != nil { - helpers.TrackError("build", err, map[string]interface{}{"at": "build.Save"}) return httperr.Server(err) } - resources, err := models.ListResources(os.Getenv("RACK")) - + a, err := models.GetApp(app) if err != nil { - helpers.TrackError("build", err, map[string]interface{}{"at": "models.ListResources"}) return httperr.Server(err) } - ch := make(chan error) - - source, _, err := r.FormFile("source") - - if err != nil && err != http.ErrMissingFile && err != http.ErrNotMultipart { - helpers.TrackError("build", err, map[string]interface{}{"at": "FormFile"}) + // Log into registry that we will push to + _, err = models.AppDockerLogin(*a) + if err != nil { return httperr.Server(err) } - cache := !(r.FormValue("cache") == "false") + var b *structs.Build + // if source file was posted, build from tar if source != nil { - err = models.S3PutFile(resources["RegistryBucket"].Id, fmt.Sprintf("builds/%s.tgz", build.Id), source, false) - + b, err = provider.BuildCreateTar(app, source, r.FormValue("manifest"), r.FormValue("description"), cache) + } else if repo != "" { + b, err = provider.BuildCreateRepo(app, repo, r.FormValue("manifest"), r.FormValue("description"), cache) + } else if index != "" { + var i structs.Index + err := json.Unmarshal([]byte(index), &i) if err != nil { - helpers.TrackError("build", err, map[string]interface{}{"at": "models.S3PutFile"}) return httperr.Server(err) } - go build.ExecuteLocal(source, cache, manifest, ch) - - err = <-ch - - if err != nil { - helpers.TrackError("build", err, map[string]interface{}{"at": "models.ExecuteLocal"}) - return httperr.Server(err) - } else { - return RenderJson(rw, build) - } + b, err = provider.BuildCreateIndex(app, i, manifest, description, cache) + } else { + return httperr.Errorf(403, "no source, repo or index") } - if repo := r.FormValue("repo"); repo != "" { - go build.ExecuteRemote(repo, cache, manifest, ch) - - err = <-ch - - if err != nil { - helpers.TrackError("build", err, map[string]interface{}{"at": "build.ExecuteRemote"}) - return httperr.Server(err) - } else { - return RenderJson(rw, build) - } + if err != nil { + return httperr.Server(err) } - if data := r.FormValue("index"); data != "" { - var index models.Index - - err := json.Unmarshal([]byte(data), &index) + return RenderJson(rw, b) +} - if err != nil { - return httperr.Server(err) - } +func BuildUpdate(rw http.ResponseWriter, r *http.Request) *httperr.Error { + vars := mux.Vars(r) + app := vars["app"] + build := vars["build"] - go build.ExecuteIndex(index, cache, manifest, ch) + b, err := provider.BuildGet(app, build) + if err != nil { + return httperr.Server(err) + } - err = <-ch + b.Ended = time.Now() + b.Manifest = r.FormValue("manifest") + b.Reason = r.FormValue("reason") + b.Status = r.FormValue("status") + // if build was successful create a release + if b.Status == "complete" && b.Manifest != "" { + _, err := provider.BuildRelease(b) if err != nil { - helpers.TrackError("build", err, map[string]interface{}{"at": "build.ExecuteIndex"}) return httperr.Server(err) - } else { - return RenderJson(rw, build) } + } else { + provider.BuildSave(b) } - return httperr.Errorf(403, "no source or repo") + return RenderJson(rw, b) } func BuildCopy(rw http.ResponseWriter, r *http.Request) *httperr.Error { vars := mux.Vars(r) - app := vars["app"] + srcApp := vars["app"] build := vars["build"] - dest := r.FormValue("app") - _, err := models.GetApp(app) - - if awsError(err) == "ValidationError" { - return httperr.Errorf(404, "no such source app: %s", app) - } - - srcBuild, err := models.GetBuild(app, build) - - if err != nil && strings.HasPrefix(err.Error(), "no such build") { - return httperr.Errorf(404, err.Error()) - } - + b, err := provider.BuildCopy(srcApp, build, dest) if err != nil { return httperr.Server(err) } - destApp, err := models.GetApp(dest) - - if awsError(err) == "ValidationError" { - return httperr.Errorf(404, "no such destination app: %s", dest) - } - - destBuild, err := srcBuild.CopyTo(*destApp) - - if err != nil { - return httperr.Server(err) - } - - return RenderJson(rw, destBuild) + return RenderJson(rw, b) } func BuildLogs(ws *websocket.Conn) *httperr.Error { vars := mux.Vars(ws.Request()) + app := vars["app"] build := vars["build"] - _, err := models.GetApp(app) - - if awsError(err) == "ValidationError" { - return httperr.Errorf(404, "no such app: %s", app) - } - - _, err = models.GetBuild(app, build) - + _, err := provider.BuildGet(app, build) if err != nil { return httperr.Server(err) } - // default to local docker socket host := "unix:///var/run/docker.sock" @@ -273,8 +210,6 @@ func BuildLogs(ws *websocket.Conn) *httperr.Error { } } - fmt.Printf("host %+v\n", host) - // proxy to docker container logs // https://docs.docker.com/reference/api/docker_remote_api_v1.19/#get-container-logs client, err := docker.NewClient(host) @@ -283,11 +218,8 @@ func BuildLogs(ws *websocket.Conn) *httperr.Error { return httperr.Server(err) } - r, w := io.Pipe() - quit := make(chan bool) - go scanLines(r, ws) go keepAlive(ws, quit) err = client.Logs(docker.LogsOptions{ @@ -297,8 +229,8 @@ func BuildLogs(ws *websocket.Conn) *httperr.Error { Stderr: true, Tail: "all", RawTerminal: false, - OutputStream: w, - ErrorStream: w, + OutputStream: ws, + ErrorStream: ws, }) quit <- true @@ -306,27 +238,6 @@ func BuildLogs(ws *websocket.Conn) *httperr.Error { return httperr.Server(err) } -func scanLines(r io.Reader, ws *websocket.Conn) { - scanner := bufio.NewScanner(r) - - for scanner.Scan() { - parts := strings.SplitN(scanner.Text(), "|", 2) - - if len(parts) < 2 { - ws.Write([]byte(parts[0] + "\n")) - continue - } - - switch parts[0] { - case "manifest": - case "error": - ws.Write([]byte(parts[1] + "\n")) - default: - ws.Write([]byte(parts[1] + "\n")) - } - } -} - func keepAlive(ws *websocket.Conn, quit chan bool) { c := time.Tick(5 * time.Second) b := []byte{} @@ -340,11 +251,3 @@ func keepAlive(ws *websocket.Conn, quit chan bool) { } } } - -func logEvent(log *logger.Logger, build models.Build, step string, err error) { - if err != nil { - log.Log("state=error step=build.%s app=%q build=%q error=%q", step, build.App, build.Id, err) - } else { - log.Success("step=build.%s app=%q build=%q", step, build.App, build.Id) - } -} diff --git a/api/controllers/index.go b/api/controllers/index.go index dbac3427ca..c72bff0ba3 100644 --- a/api/controllers/index.go +++ b/api/controllers/index.go @@ -10,20 +10,19 @@ import ( "github.com/convox/rack/Godeps/_workspace/src/github.com/gorilla/mux" "github.com/convox/rack/api/httperr" - "github.com/convox/rack/api/models" + "github.com/convox/rack/api/provider" + "github.com/convox/rack/api/structs" ) func IndexDiff(rw http.ResponseWriter, r *http.Request) *httperr.Error { - var index models.Index + var index structs.Index err := json.Unmarshal([]byte(r.FormValue("index")), &index) - if err != nil { return httperr.Server(err) } - missing, err := index.Diff() - + missing, err := provider.IndexDiff(&index) if err != nil { return httperr.Server(err) } @@ -32,22 +31,14 @@ func IndexDiff(rw http.ResponseWriter, r *http.Request) *httperr.Error { } func IndexUpload(rw http.ResponseWriter, r *http.Request) *httperr.Error { - err := r.ParseMultipartForm(10 * 1024 * 1024) - hash := mux.Vars(r)["hash"] - if err != nil { - return httperr.Server(err) - } - file, _, err := r.FormFile("data") - if err != nil { return httperr.Server(err) } data, err := ioutil.ReadAll(file) - if err != nil { return httperr.Server(err) } @@ -58,8 +49,7 @@ func IndexUpload(rw http.ResponseWriter, r *http.Request) *httperr.Error { return httperr.New(403, fmt.Errorf("invalid hash")) } - err = models.IndexUpload(hash, data) - + err = provider.IndexUpload(hash, data) if err != nil { return httperr.Server(err) } diff --git a/api/controllers/processes.go b/api/controllers/processes.go index ea0fe396df..d2bbf7a9db 100644 --- a/api/controllers/processes.go +++ b/api/controllers/processes.go @@ -115,6 +115,7 @@ func ProcessRunDetached(rw http.ResponseWriter, r *http.Request) *httperr.Error app := vars["app"] process := vars["process"] command := GetForm(r, "command") + release := GetForm(r, "release") a, err := models.GetApp(app) @@ -122,7 +123,7 @@ func ProcessRunDetached(rw http.ResponseWriter, r *http.Request) *httperr.Error return httperr.Errorf(404, "no such app: %s", app) } - err = a.RunDetached(process, command) + err = a.RunDetached(process, command, release) if err != nil { return httperr.Server(err) @@ -138,6 +139,7 @@ func ProcessRunAttached(ws *websocket.Conn) *httperr.Error { app := vars["app"] process := vars["process"] command := header.Get("Command") + release := header.Get("Release") height, _ := strconv.Atoi(header.Get("Height")) width, _ := strconv.Atoi(header.Get("Width")) @@ -151,7 +153,7 @@ func ProcessRunAttached(ws *websocket.Conn) *httperr.Error { return httperr.Server(err) } - return httperr.Server(a.RunAttached(process, command, height, width, ws)) + return httperr.Server(a.RunAttached(process, command, release, height, width, ws)) } func ProcessStop(rw http.ResponseWriter, r *http.Request) *httperr.Error { diff --git a/api/controllers/releases.go b/api/controllers/releases.go index 5fbd309eb0..db11544f14 100644 --- a/api/controllers/releases.go +++ b/api/controllers/releases.go @@ -1,7 +1,6 @@ package controllers import ( - "fmt" "net/http" "strings" @@ -9,24 +8,16 @@ import ( "github.com/convox/rack/Godeps/_workspace/src/github.com/gorilla/mux" "github.com/convox/rack/api/httperr" "github.com/convox/rack/api/models" + "github.com/convox/rack/api/provider" ) func ReleaseList(rw http.ResponseWriter, r *http.Request) *httperr.Error { - vars := mux.Vars(r) - app := vars["app"] - - _, err := models.GetApp(app) + app := mux.Vars(r)["app"] + releases, err := provider.ReleaseList(app) if awsError(err) == "ValidationError" { return httperr.Errorf(404, "no such app: %s", app) } - - if err != nil { - return httperr.Server(err) - } - - releases, err := models.ListReleases(app) - if err != nil { return httperr.Server(err) } @@ -34,30 +25,23 @@ func ReleaseList(rw http.ResponseWriter, r *http.Request) *httperr.Error { return RenderJson(rw, releases) } -func ReleaseShow(rw http.ResponseWriter, r *http.Request) *httperr.Error { - vars := mux.Vars(r) +func ReleaseGet(rw http.ResponseWriter, req *http.Request) *httperr.Error { + vars := mux.Vars(req) app := vars["app"] release := vars["release"] - _, err := models.GetApp(app) - + r, err := provider.ReleaseGet(app, release) if awsError(err) == "ValidationError" { return httperr.Errorf(404, "no such app: %s", app) } - - rr, err := models.GetRelease(app, release) - if err != nil && strings.HasPrefix(err.Error(), "no such release") { - return httperr.Errorf(404, "no such release: %s", release) + return httperr.Errorf(404, err.Error()) } - - fmt.Printf("err %+v\n", err) - if err != nil { return httperr.Server(err) } - return RenderJson(rw, rr) + return RenderJson(rw, r) } func ReleasePromote(rw http.ResponseWriter, r *http.Request) *httperr.Error { diff --git a/api/controllers/routes.go b/api/controllers/routes.go index 50d0c2a74f..4a98ed3330 100644 --- a/api/controllers/routes.go +++ b/api/controllers/routes.go @@ -21,6 +21,7 @@ func NewRouter() (router *mux.Router) { router.HandleFunc("/apps/{app}/builds", api("build.list", BuildList)).Methods("GET") router.HandleFunc("/apps/{app}/builds", api("build.create", BuildCreate)).Methods("POST") router.HandleFunc("/apps/{app}/builds/{build}", api("build.get", BuildGet)).Methods("GET") + router.HandleFunc("/apps/{app}/builds/{build}", api("build.update", BuildUpdate)).Methods("PUT") router.HandleFunc("/apps/{app}/builds/{build}", api("build.delete", BuildDelete)).Methods("DELETE") router.HandleFunc("/apps/{app}/builds/{build}/copy", api("build.copy", BuildCopy)).Methods("POST") router.HandleFunc("/apps/{app}/environment", api("environment.list", EnvironmentList)).Methods("GET") @@ -35,7 +36,7 @@ func NewRouter() (router *mux.Router) { router.HandleFunc("/apps/{app}/processes/{process}", api("process.stop", ProcessStop)).Methods("DELETE") router.HandleFunc("/apps/{app}/processes/{process}/run", api("process.run.detach", ProcessRunDetached)).Methods("POST") router.HandleFunc("/apps/{app}/releases", api("release.list", ReleaseList)).Methods("GET") - router.HandleFunc("/apps/{app}/releases/{release}", api("release.get", ReleaseShow)).Methods("GET") + router.HandleFunc("/apps/{app}/releases/{release}", api("release.get", ReleaseGet)).Methods("GET") router.HandleFunc("/apps/{app}/releases/{release}/promote", api("release.promote", ReleasePromote)).Methods("POST") router.HandleFunc("/apps/{app}/ssl", api("ssl.list", SSLList)).Methods("GET") router.HandleFunc("/apps/{app}/ssl", api("ssl.create", SSLCreate)).Methods("POST") diff --git a/api/controllers/ssl.go b/api/controllers/ssl.go index 8d6dddab58..9e40ca5950 100644 --- a/api/controllers/ssl.go +++ b/api/controllers/ssl.go @@ -3,6 +3,7 @@ package controllers import ( "net/http" "strconv" + "strings" "github.com/convox/rack/Godeps/_workspace/src/github.com/gorilla/mux" "github.com/convox/rack/api/httperr" @@ -29,6 +30,7 @@ func SSLCreate(rw http.ResponseWriter, r *http.Request) *httperr.Error { a := mux.Vars(r)["app"] process := GetForm(r, "process") port := GetForm(r, "port") + arn := GetForm(r, "arn") chain := GetForm(r, "chain") body := GetForm(r, "body") key := GetForm(r, "key") @@ -44,7 +46,11 @@ func SSLCreate(rw http.ResponseWriter, r *http.Request) *httperr.Error { return httperr.Errorf(403, "port must be numeric") } - ssl, err := models.CreateSSL(a, process, portn, body, key, chain, (secure == "true")) + if (arn != "") && !validateARNFormat(arn) { + return httperr.Errorf(403, "arn must follow the AWS ARN format") + } + + ssl, err := models.CreateSSL(a, process, portn, arn, body, key, chain, (secure == "true")) if awsError(err) == "ValidationError" { return httperr.Errorf(404, "%s", err) @@ -90,6 +96,7 @@ func SSLUpdate(rw http.ResponseWriter, r *http.Request) *httperr.Error { a := mux.Vars(r)["app"] process := GetForm(r, "process") port := GetForm(r, "port") + arn := GetForm(r, "arn") chain := GetForm(r, "chain") body := GetForm(r, "body") key := GetForm(r, "key") @@ -104,7 +111,11 @@ func SSLUpdate(rw http.ResponseWriter, r *http.Request) *httperr.Error { return httperr.Errorf(403, "port must be numeric") } - ssl, err := models.UpdateSSL(a, process, portn, body, key, chain) + if (arn != "") && !validateARNFormat(arn) { + return httperr.Errorf(403, "arn must follow the AWS ARN format") + } + + ssl, err := models.UpdateSSL(a, process, portn, arn, body, key, chain) if awsError(err) == "ValidationError" { return httperr.Errorf(404, "%s", err) @@ -116,3 +127,7 @@ func SSLUpdate(rw http.ResponseWriter, r *http.Request) *httperr.Error { return RenderJson(rw, ssl) } + +func validateARNFormat(arn string) bool { + return strings.HasPrefix(strings.ToLower(arn), "arn:") +} diff --git a/api/controllers/system.go b/api/controllers/system.go index d1f60772ce..890d911d61 100644 --- a/api/controllers/system.go +++ b/api/controllers/system.go @@ -11,17 +11,14 @@ import ( func SystemReleaseList(rw http.ResponseWriter, r *http.Request) *httperr.Error { rack, err := provider.SystemGet() - if awsError(err) == "ValidationError" { return httperr.Errorf(404, "no such stack: %s", rack) } - if err != nil { return httperr.Server(err) } - releases, err := models.ListReleases(rack.Name) - + releases, err := provider.ReleaseList(rack.Name) if err != nil { return httperr.Server(err) } diff --git a/api/helpers/helpers.go b/api/helpers/helpers.go index e0fafe6823..a1af94afce 100644 --- a/api/helpers/helpers.go +++ b/api/helpers/helpers.go @@ -45,6 +45,7 @@ func Error(log *logger.Logger, err error) { if rollbar.Token != "" { extraData := map[string]string{ "AWS_REGION": os.Getenv("AWS_REGION"), + "CLIENT_ID": os.Getenv("CLIENT_ID"), "RACK": os.Getenv("RACK"), "RELEASE": os.Getenv("RELEASE"), "VPC": os.Getenv("VPC"), @@ -62,6 +63,8 @@ func TrackEvent(event string, params map[string]interface{}) { } params["client_id"] = os.Getenv("CLIENT_ID") + params["rack"] = os.Getenv("RACK") + params["release"] = os.Getenv("RELEASE") userId := RackId() diff --git a/api/manifest/manifest.go b/api/manifest/manifest.go index 0e4c2087ee..33e9e894dc 100644 --- a/api/manifest/manifest.go +++ b/api/manifest/manifest.go @@ -8,6 +8,7 @@ import ( "fmt" "io" "io/ioutil" + "log" "math/rand" "net" "net/url" @@ -36,6 +37,11 @@ var ( SignalWaiter = waitForSignal ) +var ( + special = color.New(color.FgWhite).Add(color.Bold).SprintFunc() + command = color.New(color.FgBlack).Add(color.Bold).SprintFunc() +) + var RandomPort = func() int { return 10000 + rand.Intn(50000) } @@ -262,11 +268,28 @@ func (m *Manifest) Build(app, dir string, cache bool) []error { } } + s1 := rand.NewSource(time.Now().UnixNano()) + r1 := rand.New(s1) + for _, image := range pulls { - err := pullSync(image) + var pullErr error + var backOff = 1 + + for i := 0; i < 5; i++ { + if i != 0 { + log.Printf("A pull error occurred for: %s\n", image) + log.Printf("Retrying in %d seconds...\n", backOff) + time.Sleep(time.Duration(backOff) * time.Second) + backOff = ((backOff + r1.Intn(10)) * (i)) + } + pullErr = pullSync(image) + if pullErr == nil { + break + } + } - if err != nil { - return []error{err} + if pullErr != nil { + return []error{pullErr} } } @@ -499,10 +522,26 @@ func (m *Manifest) Push(app, registry, tag string, flatten string) []error { remote = fmt.Sprintf("%s/%s:%s", registry, flatten, fmt.Sprintf("%s.%s", name, tag)) } - err := pushSync(local, remote) + var pushErr error + var backOff = 1 + s1 := rand.NewSource(time.Now().UnixNano()) + r1 := rand.New(s1) - if err != nil { - return []error{err} + for i := 0; i < 5; i++ { + if i != 0 { + log.Printf("A push error occurred for %s/%s\n", app, name) + log.Printf("Retrying in %d seconds...\n", backOff) + time.Sleep(time.Duration(backOff) * time.Second) + backOff = ((backOff + r1.Intn(10)) * (i)) + } + pushErr = pushSync(local, remote) + if pushErr == nil { + break + } + } + + if pushErr != nil { + return []error{pushErr} } } @@ -729,6 +768,7 @@ func (me ManifestEntry) runAsync(m *Manifest, prefix, app, process string, cache switch me.Label(fmt.Sprintf("com.convox.port.%s.protocol", container)) { case "proxy": rnd := RandomPort() + fmt.Println(prefix, special(fmt.Sprintf("proxy protocol enabled for %s:%s", host, container))) go proxyPort(host, fmt.Sprintf("%s:%d", gateway, rnd)) host = strconv.Itoa(rnd) } @@ -762,13 +802,31 @@ func (me ManifestEntry) Label(key string) string { switch labels := me.Labels.(type) { case map[interface{}]interface{}: for k, v := range labels { - if k.(string) == key { - return v.(string) + ks, ok := k.(string) + + if !ok { + return "" + } + + vs, ok := v.(string) + + if !ok { + return "" + } + + if ks == key { + return vs } } case []interface{}: for _, label := range labels { - if parts := strings.SplitN(label.(string), "=", 2); len(parts) == 2 { + ls, ok := label.(string) + + if !ok { + return "" + } + + if parts := strings.SplitN(ls, "=", 2); len(parts) == 2 { if parts[0] == key { return parts[1] } @@ -892,7 +950,7 @@ func run(executable string, args ...string) error { } func runPrefix(prefix, executable string, args ...string) error { - fmt.Printf("%s running: %s %s\n", prefix, executable, strings.Join(args, " ")) + fmt.Println(prefix, command(fmt.Sprintf("%s %s", executable, strings.Join(args, " ")))) cmd := Execer(executable, args...) diff --git a/api/manifest/manifest_test.go b/api/manifest/manifest_test.go index 0f120aca7b..8295e40355 100644 --- a/api/manifest/manifest_test.go +++ b/api/manifest/manifest_test.go @@ -131,7 +131,7 @@ func TestRun(t *testing.T) { stdout, stderr := testRun(m, "compose") cases := Cases{ - {stdout, fmt.Sprintf("\x1b[36mpostgres |\x1b[0m running: docker run -i --name compose-postgres -p 5432:5432 compose/postgres\n\x1b[33mweb |\x1b[0m running: docker run -i --name compose-web -e POSTGRES_HOST=1.1.1.1 -e POSTGRES_PASSWORD= -e POSTGRES_PATH= -e POSTGRES_PORT=5432 -e POSTGRES_SCHEME=tcp -e POSTGRES_URL=tcp://1.1.1.1:5432 -e POSTGRES_USERNAME= -p 5000:3000 -v %s:/app compose/web\n", destDir)}, + {stdout, fmt.Sprintf("\x1b[36mpostgres |\x1b[0m docker run -i --name compose-postgres -p 5432:5432 compose/postgres\n\x1b[33mweb |\x1b[0m docker run -i --name compose-web -e POSTGRES_HOST=1.1.1.1 -e POSTGRES_PASSWORD= -e POSTGRES_PATH= -e POSTGRES_PORT=5432 -e POSTGRES_SCHEME=tcp -e POSTGRES_URL=tcp://1.1.1.1:5432 -e POSTGRES_USERNAME= -p 5000:3000 -v %s:/app compose/web\n", destDir)}, {stderr, ""}, } @@ -267,7 +267,7 @@ func _assert(t *testing.T, cases Cases) { } if !bytes.Equal(j1, j2) { - t.Errorf("Got %q, want %q", c.got, c.want) + t.Errorf("Data Mismatch\nGot: %q\nWant: %q", c.got, c.want) } } } diff --git a/api/models/app.go b/api/models/app.go index ccd2a168d9..aefce5d499 100644 --- a/api/models/app.go +++ b/api/models/app.go @@ -11,6 +11,7 @@ import ( "time" "github.com/convox/rack/api/helpers" + "github.com/convox/rack/api/provider" "github.com/convox/rack/client" "github.com/convox/rack/Godeps/_workspace/src/github.com/aws/aws-sdk-go/aws" @@ -204,24 +205,24 @@ func (a *App) Cleanup() error { return err } - builds, err := ListBuilds(a.Name) - + // FIXME: BuildList and ReleaseList only lists and cleans up the last 20 builds/releases + // FIXME: Should the delete calls happen in a goroutine? + builds, err := provider.BuildList(a.Name) if err != nil { return err } for _, build := range builds { - go cleanupBuild(build) + provider.BuildDelete(a.Name, build.Id) } - releases, err := ListReleases(a.Name) - + releases, err := provider.ReleaseList(a.Name) if err != nil { return err } for _, release := range releases { - go cleanupRelease(release) + provider.ReleaseDelete(a.Name, release.Id) } // monitor and stack deletion state for up to 10 minutes @@ -360,9 +361,9 @@ func (a *App) ForkRelease() (*Release, error) { return release, nil } +// FIXME: Port to provider interface func (a *App) LatestRelease() (*Release, error) { - releases, err := ListReleases(a.Name) - + releases, err := provider.ReleaseList(a.Name) if err != nil { return nil, err } @@ -371,7 +372,16 @@ func (a *App) LatestRelease() (*Release, error) { return nil, nil } - return &releases[0], nil + r := releases[0] + + return &Release{ + Id: r.Id, + App: r.App, + Build: r.Build, + Env: r.Env, + Manifest: r.Manifest, + Created: r.Created, + }, nil } func (a *App) ExecAttached(pid, command string, height, width int, rw io.ReadWriter) error { @@ -460,7 +470,7 @@ func (a *App) ExecAttached(pid, command string, height, width int, rw io.ReadWri return nil } -func (a *App) RunAttached(process, command string, height, width int, rw io.ReadWriter) error { +func (a *App) RunAttached(process, command, releaseId string, height, width int, rw io.ReadWriter) error { resources, err := a.Resources() if err != nil { @@ -489,7 +499,11 @@ func (a *App) RunAttached(process, command string, height, width int, rw io.Read ea = append(ea, fmt.Sprintf("%s=%s", *env.Name, *env.Value)) } - release, err := GetRelease(a.Name, a.Release) + if len(releaseId) == 0 { + releaseId = a.Release + } + + release, err := GetRelease(a.Name, releaseId) if err != nil { return err @@ -658,7 +672,7 @@ func (a *App) RunAttached(process, command string, height, width int, rw io.Read return nil } -func (a *App) RunDetached(process, command string) error { +func (a *App) RunDetached(process, command, releaseId string) error { resources, err := a.Resources() if err != nil { @@ -789,22 +803,6 @@ func cleanupBucketObject(bucket, key, version string) { } } -func cleanupBuild(build Build) { - err := build.Cleanup() - - if err != nil { - fmt.Printf("error: %s\n", err) - } -} - -func cleanupRelease(release Release) { - err := release.Cleanup() - - if err != nil { - fmt.Printf("error: %s\n", err) - } -} - func (s Apps) Len() int { return len(s) } diff --git a/api/models/build.go b/api/models/build.go deleted file mode 100644 index 5b8c494c22..0000000000 --- a/api/models/build.go +++ /dev/null @@ -1,736 +0,0 @@ -package models - -import ( - "bufio" - "bytes" - "fmt" - "io" - "io/ioutil" - "os" - "os/exec" - "strings" - "sync" - "time" - - "github.com/convox/rack/Godeps/_workspace/src/github.com/aws/aws-sdk-go/aws" - "github.com/convox/rack/Godeps/_workspace/src/github.com/aws/aws-sdk-go/service/dynamodb" - "github.com/convox/rack/Godeps/_workspace/src/github.com/aws/aws-sdk-go/service/kinesis" - "github.com/convox/rack/api/helpers" -) - -type Build struct { - Id string `json:"id"` - App string `json:"app"` - Logs string `json:"logs"` - Manifest string `json:"manifest"` - Release string `json:"release"` - Status string `json:"status"` - - Description string `json:"description"` - - Started time.Time `json:"started"` - Ended time.Time `json:"ended"` - - kinesis string `json:"-"` -} - -type Builds []Build - -func NewBuild(app string) Build { - return Build{ - App: app, - Id: generateId("B", 10), - Status: "created", - } -} - -func ListBuilds(app string) (Builds, error) { - req := &dynamodb.QueryInput{ - KeyConditions: map[string]*dynamodb.Condition{ - "app": &dynamodb.Condition{ - AttributeValueList: []*dynamodb.AttributeValue{&dynamodb.AttributeValue{S: aws.String(app)}}, - ComparisonOperator: aws.String("EQ"), - }, - }, - IndexName: aws.String("app.created"), - Limit: aws.Int64(20), - ScanIndexForward: aws.Bool(false), - TableName: aws.String(buildsTable(app)), - } - - res, err := DynamoDB().Query(req) - - if err != nil { - return nil, err - } - - builds := make(Builds, len(res.Items)) - - for i, item := range res.Items { - builds[i] = *buildFromItem(item) - } - - return builds, nil -} - -func GetBuild(app, id string) (*Build, error) { - req := &dynamodb.GetItemInput{ - ConsistentRead: aws.Bool(true), - Key: map[string]*dynamodb.AttributeValue{ - "id": &dynamodb.AttributeValue{S: aws.String(id)}, - }, - TableName: aws.String(buildsTable(app)), - } - - res, err := DynamoDB().GetItem(req) - - if err != nil { - return nil, err - } - - if res.Item == nil { - return nil, fmt.Errorf("no such build: %s", id) - } - - build := buildFromItem(res.Item) - - return build, nil -} - -func (b *Build) Save() error { - if b.Id == "" { - return fmt.Errorf("Id can not be blank") - } - - if b.Started.IsZero() { - b.Started = time.Now() - } - - req := &dynamodb.PutItemInput{ - Item: map[string]*dynamodb.AttributeValue{ - "id": &dynamodb.AttributeValue{S: aws.String(b.Id)}, - "app": &dynamodb.AttributeValue{S: aws.String(b.App)}, - "status": &dynamodb.AttributeValue{S: aws.String(b.Status)}, - "created": &dynamodb.AttributeValue{S: aws.String(b.Started.Format(SortableTime))}, - }, - TableName: aws.String(buildsTable(b.App)), - } - - if b.Description != "" { - req.Item["description"] = &dynamodb.AttributeValue{S: aws.String(b.Description)} - } - - if b.Manifest != "" { - req.Item["manifest"] = &dynamodb.AttributeValue{S: aws.String(b.Manifest)} - } - - if b.Release != "" { - req.Item["release"] = &dynamodb.AttributeValue{S: aws.String(b.Release)} - } - - if !b.Ended.IsZero() { - req.Item["ended"] = &dynamodb.AttributeValue{S: aws.String(b.Ended.Format(SortableTime))} - } - - a, err := GetApp(b.App) - - if err != nil { - return err - } - - err = S3Put(a.Outputs["Settings"], fmt.Sprintf("builds/%s.log", b.Id), []byte(b.Logs), true) - - if err != nil { - return err - } - - _, err = DynamoDB().PutItem(req) - - return err -} - -// Test if another build container is running. -// This is a temporary workaround since the current Docker Registry does not -// handle pushing multiple images at the same time. -// Course grained locking will prevent subtle build errors until a better -// registry and/or Docker image subsystem is integrated -func (b *Build) IsRunning() bool { - out, err := exec.Command("docker", "ps", "-q", "--filter", "name=build-B*").CombinedOutput() - - // log exec errors but optimistically consider builds unlocked - if err != nil { - fmt.Printf("ns=kernel cn=build at=IsRunning state=error step=exec.Command app=%q build=%q error=%q\n", b.App, b.Id, err) - return true - } - - // There are active build-* containers if `docker ps -q` returns a container id, e.g. "930b96f8f3dc\n" - running := string(out) != "" - - fmt.Printf("ns=kernel cn=build at=IsRunning running=%s step=exec.Command app=%q build=%q\n", running, b.App, b.Id) - return running -} - -func (b *Build) Cleanup() error { - return nil -} - -func (b *Build) buildError(err error, ch chan error) { - NotifyError("build:create", err, map[string]string{"id": b.Id, "app": b.App}) - fmt.Printf("ns=kernel cn=build state=error app=%q build=%q error=%q\n", b.App, b.Id, err) - b.Fail(err) - ch <- err -} - -func (b *Build) copyError(err error) { - NotifyError("build:copy", err, map[string]string{"id": b.Id, "app": b.App}) - b.Fail(err) -} - -func (b *Build) buildArgs(cache bool, manifest string) ([]string, error) { - app, err := GetApp(b.App) - - if err != nil { - return nil, err - } - - args := []string{"run", "-i", "--name", fmt.Sprintf("build-%s", b.Id), "-v", "/var/run/docker.sock:/var/run/docker.sock", os.Getenv("DOCKER_IMAGE_API"), "build", "-id", b.Id} - - endpoint, err := AppDockerLogin(*app) - - if err != nil { - return nil, err - } - - args = append(args, "-push", endpoint) - - if repository := app.Outputs["RegistryRepository"]; repository != "" { - args = append(args, "-flatten", repository) - } - - if manifest != "" { - args = append(args, "-manifest", manifest) - } - - if !cache { - args = append(args, "-no-cache") - } - - err = LoginPrivateRegistries() - - if err != nil { - return nil, err - } - - if dockercfg, err := ioutil.ReadFile("/root/.docker/config.json"); err == nil { - args = append(args, "-dockercfg", string(dockercfg)) - } - - return args, nil -} - -func (b *Build) ExecuteLocal(r io.Reader, cache bool, manifest string, ch chan error) { - started := time.Now() - - b.Status = "building" - err := b.Save() - - if err != nil { - helpers.TrackError("build", err, map[string]interface{}{"type": "local", "at": "b.Save"}) - b.buildError(err, ch) - return - } - - args, err := b.buildArgs(cache, manifest) - - if err != nil { - helpers.TrackError("build", err, map[string]interface{}{"type": "local", "at": "b.buildArgs"}) - b.buildError(err, ch) - return - } - - args = append(args, b.App, "-") - - err = b.execute(args, r, ch) - - if err != nil { - helpers.TrackError("build", err, map[string]interface{}{"type": "local", "at": "b.execute"}) - b.buildError(err, ch) - return - } - - NotifySuccess("build:create", map[string]string{"id": b.Id, "app": b.App}) - fmt.Printf("ns=kernel cn=build at=ExecuteLocal state=success step=build.execute app=%q build=%q\n", b.App, b.Id) - helpers.TrackSuccess("build", map[string]interface{}{"type": "local", "elapsed": time.Now().Sub(started).Nanoseconds() / 1000000}) -} - -func (b *Build) ExecuteRemote(repo string, cache bool, manifest string, ch chan error) { - started := time.Now() - - b.Status = "building" - err := b.Save() - - if err != nil { - helpers.TrackError("build", err, map[string]interface{}{"type": "remote", "at": "b.Save"}) - b.buildError(err, ch) - return - } - - args, err := b.buildArgs(cache, manifest) - - if err != nil { - helpers.TrackError("build", err, map[string]interface{}{"type": "remote", "at": "b.buildArgs"}) - b.buildError(err, ch) - return - } - - args = append(args, b.App, repo) - - err = b.execute(args, nil, ch) - - if err != nil { - helpers.TrackError("build", err, map[string]interface{}{"type": "remote", "at": "b.execute"}) - b.buildError(err, ch) - return - } - - NotifySuccess("build:create", map[string]string{"id": b.Id, "app": b.App}) - fmt.Printf("ns=kernel cn=build at=ExecuteRemote state=success step=build.execute app=%q build=%q\n", b.App, b.Id) - helpers.TrackSuccess("build", map[string]interface{}{"type": "remote", "elapsed": time.Now().Sub(started).Nanoseconds() / 1000000}) -} - -func (b *Build) ExecuteIndex(index Index, cache bool, manifest string, ch chan error) { - started := time.Now() - - b.Status = "building" - err := b.Save() - - if err != nil { - helpers.TrackError("build", err, map[string]interface{}{"type": "remote", "at": "b.Save"}) - b.buildError(err, ch) - return - } - - args, err := b.buildArgs(cache, manifest) - - if err != nil { - helpers.TrackError("build", err, map[string]interface{}{"type": "index", "at": "b.buildArgs"}) - b.buildError(err, ch) - return - } - - dir, err := ioutil.TempDir("", "source") - - if err != nil { - helpers.TrackError("build", err, map[string]interface{}{"type": "index", "at": "ioutil.TempDir"}) - b.buildError(err, ch) - return - } - - err = os.Chmod(dir, 0755) - - if err != nil { - helpers.TrackError("build", err, map[string]interface{}{"type": "index", "at": "os.Chmod"}) - b.buildError(err, ch) - return - } - - err = index.Download(dir) - - if err != nil { - helpers.TrackError("build", err, map[string]interface{}{"type": "index", "at": "index.Download"}) - b.buildError(err, ch) - return - } - - tgz, err := createTarball(dir) - - if err != nil { - helpers.TrackError("build", err, map[string]interface{}{"type": "index", "at": "createTarball"}) - b.buildError(err, ch) - return - } - - args = append(args, b.App, "-") - - err = b.execute(args, bytes.NewReader(tgz), ch) - - if err != nil { - helpers.TrackError("build", err, map[string]interface{}{"type": "index", "at": "b.execute"}) - b.buildError(err, ch) - return - } - - NotifySuccess("build:create", map[string]string{"id": b.Id, "app": b.App}) - fmt.Printf("ns=kernel cn=build at=ExecuteIndex state=success step=build.execute app=%q build=%q\n", b.App, b.Id) - helpers.TrackSuccess("build", map[string]interface{}{"type": "index", "elapsed": time.Now().Sub(started).Nanoseconds() / 1000000}) -} - -func (srcBuild *Build) CopyTo(destApp App) (*Build, error) { - started := time.Now() - - srcApp, err := GetApp(srcBuild.App) - - if err != nil { - return nil, err - } - - // generate a new build ID - destBuild := NewBuild(destApp.Name) - - // copy src build properties - destBuild.Manifest = srcBuild.Manifest - - // set copy properties - destBuild.Description = fmt.Sprintf("Copy of %s %s", srcBuild.App, srcBuild.Id) - destBuild.Status = "copying" - - err = destBuild.Save() - - if err != nil { - destBuild.copyError(err) - return nil, err - } - - // pull, tag, push images - manifest, err := LoadManifest(destBuild.Manifest, &destApp) - - if err != nil { - destBuild.copyError(err) - return nil, err - } - - for _, entry := range manifest { - var srcImg, destImg string - - srcImg = entry.RegistryImage(srcApp, srcBuild.Id) - destImg = entry.RegistryImage(&destApp, destBuild.Id) - - destBuild.Logs += fmt.Sprintf("RUNNING: docker pull %s\n", srcImg) - - out, err := exec.Command("docker", "pull", srcImg).CombinedOutput() - - if err != nil { - destBuild.copyError(err) - return nil, err - } - - destBuild.Logs += string(out) - - destBuild.Logs += fmt.Sprintf("RUNNING: docker tag -f %s %s\n", srcImg, destImg) - - out, err = exec.Command("docker", "tag", "-f", srcImg, destImg).CombinedOutput() - - if err != nil { - destBuild.copyError(err) - return nil, err - } - - destBuild.Logs += string(out) - - destBuild.Logs += fmt.Sprintf("RUNNING: docker push %s\n", destImg) - - out, err = exec.Command("docker", "push", destImg).CombinedOutput() - - if err != nil { - destBuild.copyError(err) - return nil, err - } - - destBuild.Logs += string(out) - - } - - // make release for new build - release, err := destApp.ForkRelease() - - if err != nil { - destBuild.copyError(err) - return nil, err - } - - release.Build = destBuild.Id - release.Manifest = destBuild.Manifest - - err = release.Save() - - if err != nil { - destBuild.copyError(err) - return nil, err - } - - // finalize new build - destBuild.Ended = time.Now() - destBuild.Release = release.Id - destBuild.Status = "complete" - - err = destBuild.Save() - - if err != nil { - destBuild.copyError(err) - return nil, err - } - - NotifySuccess("build:copy", map[string]string{"id": destBuild.Id, "app": destBuild.App}) - helpers.TrackSuccess("build-copy", map[string]interface{}{"elapsed": time.Now().Sub(started).Nanoseconds() / 1000000}) - - return &destBuild, nil -} - -func (b *Build) execute(args []string, r io.Reader, ch chan error) error { - app, err := GetApp(b.App) - - if err != nil { - return err - } - - cmd := exec.Command("docker", args...) - - stdin, err := cmd.StdinPipe() - - if err != nil { - return err - } - - stdout, err := cmd.StdoutPipe() - - if err != nil { - return err - } - - stderr, err := cmd.StderrPipe() - - if err != nil { - return err - } - - if err = cmd.Start(); err != nil { - return err - } - - for { - err := exec.Command("docker", "logs", fmt.Sprintf("build-%s", b.Id)).Run() - - time.Sleep(200 * time.Millisecond) - - if err == nil { - break - } - } - - ch <- nil // notify that start was ok - - if r != nil { - _, err := io.Copy(stdin, r) - - if err != nil { - return err - } - } - - stdin.Close() - - var wg sync.WaitGroup - - wg.Add(2) - go b.scanLines(stdout, &wg) - go b.scanLines(stderr, &wg) - wg.Wait() - - if err = cmd.Wait(); err != nil { - return err - } - - err = b.Save() - - if err != nil { - return err - } - - if b.Status == "failed" { - return fmt.Errorf("error from builder") - } - - release, err := app.ForkRelease() - - if err != nil { - return err - } - - release.Build = b.Id - release.Manifest = b.Manifest - - err = release.Save() - - if err != nil { - return err - } - - b.Release = release.Id - b.Status = "complete" - b.Ended = time.Now() - b.Save() - - return nil -} - -func (b *Build) Fail(err error) { - b.Status = "failed" - b.Ended = time.Now() - b.log(fmt.Sprintf("Build Error: %s", err)) - b.Save() -} - -func (b *Build) scanLines(r io.Reader, wg *sync.WaitGroup) { - defer wg.Done() - - scanner := bufio.NewScanner(r) - - for scanner.Scan() { - parts := strings.SplitN(scanner.Text(), "|", 2) - - if len(parts) < 2 { - b.log(parts[0]) - continue - } - - switch parts[0] { - case "manifest": - b.Manifest += fmt.Sprintf("%s\n", parts[1]) - case "error": - b.log(fmt.Sprintf("ERROR: %s", parts[1])) - b.Status = "failed" - default: - b.log(parts[1]) - } - } -} - -func (b *Build) log(line string) { - b.Logs += fmt.Sprintf("%s\n", line) - - if b.kinesis == "" { - app, err := GetApp(b.App) - - if err != nil { - fmt.Fprintf(os.Stderr, "ERROR: %s", err) - return - } - - b.kinesis = app.Outputs["Kinesis"] - } - - _, err := Kinesis().PutRecords(&kinesis.PutRecordsInput{ - StreamName: aws.String(b.kinesis), - Records: []*kinesis.PutRecordsRequestEntry{ - &kinesis.PutRecordsRequestEntry{ - Data: []byte(fmt.Sprintf("build: %s", line)), - PartitionKey: aws.String(string(time.Now().UnixNano())), - }, - }, - }) - - if err != nil { - fmt.Fprintf(os.Stderr, "ERROR: %s\n", err) - } -} - -func buildsTable(app string) string { - return os.Getenv("DYNAMO_BUILDS") -} - -func buildFromItem(item map[string]*dynamodb.AttributeValue) *Build { - started, _ := time.Parse(SortableTime, coalesce(item["created"], "")) - ended, _ := time.Parse(SortableTime, coalesce(item["ended"], "")) - - logs := "" - var err error - - if item["logs"] == nil { - logs, err = getS3BuildLogs(coalesce(item["app"], ""), coalesce(item["id"], "")) - - if err != nil { - logs = "" - } - } - - return &Build{ - Id: coalesce(item["id"], ""), - App: coalesce(item["app"], ""), - Description: coalesce(item["description"], ""), - Logs: coalesce(item["logs"], logs), - Manifest: coalesce(item["manifest"], ""), - Release: coalesce(item["release"], ""), - Status: coalesce(item["status"], ""), - Started: started, - Ended: ended, - } -} - -func createTarball(base string) ([]byte, error) { - cwd, err := os.Getwd() - - if err != nil { - return nil, err - } - - err = os.Chdir(base) - - if err != nil { - return nil, err - } - - args := []string{"cz"} - - // If .dockerignore exists, use it to exclude files from the tarball - if _, err = os.Stat(".dockerignore"); err == nil { - args = append(args, "--exclude-from", ".dockerignore") - } - - args = append(args, ".") - - cmd := exec.Command("tar", args...) - - out, err := cmd.StdoutPipe() - - if err != nil { - return nil, err - } - - cmd.Start() - - bytes, err := ioutil.ReadAll(out) - - if err != nil { - return nil, err - } - - err = cmd.Wait() - - if err != nil { - return nil, err - } - - err = os.Chdir(cwd) - - if err != nil { - return nil, err - } - - return bytes, nil -} - -func getS3BuildLogs(app, build_id string) (string, error) { - a, err := GetApp(app) - - if err != nil { - return "", err - } - - logs, err := s3Get(a.Outputs["Settings"], fmt.Sprintf("builds/%s.log", build_id)) - - if err != nil { - return "", err - } - - return string(logs), nil -} diff --git a/api/models/helpers.go b/api/models/helpers.go index f355f020ff..ba1a5db1a1 100644 --- a/api/models/helpers.go +++ b/api/models/helpers.go @@ -29,16 +29,6 @@ func awserrCode(err error) string { return "" } -func buildEnvironment() string { - env := []string{ - fmt.Sprintf("AWS_REGION=%s", os.Getenv("AWS_REGION")), - fmt.Sprintf("AWS_ACCESS=%s", os.Getenv("AWS_ACCESS")), - fmt.Sprintf("AWS_SECRET=%s", os.Getenv("AWS_SECRET")), - fmt.Sprintf("GITHUB_TOKEN=%s", os.Getenv("GITHUB_TOKEN")), - } - return strings.Join(env, "\n") -} - func cs(s *string, def string) string { if s != nil { return *s diff --git a/api/models/manifest.go b/api/models/manifest.go index ab1787a332..6ad6ef6f8f 100644 --- a/api/models/manifest.go +++ b/api/models/manifest.go @@ -320,13 +320,31 @@ func (me ManifestEntry) Label(key string) string { switch labels := me.Labels.(type) { case map[interface{}]interface{}: for k, v := range labels { - if k.(string) == key { - return v.(string) + ks, ok := k.(string) + + if !ok { + return "" + } + + vs, ok := v.(string) + + if !ok { + return "" + } + + if ks == key { + return vs } } case []interface{}: for _, label := range labels { - if parts := strings.SplitN(label.(string), "=", 2); len(parts) == 2 { + ls, ok := label.(string) + + if !ok { + return "" + } + + if parts := strings.SplitN(ls, "=", 2); len(parts) == 2 { if parts[0] == key { return parts[1] } diff --git a/api/models/notify.go b/api/models/notify.go index 1ee9369863..3316107976 100644 --- a/api/models/notify.go +++ b/api/models/notify.go @@ -54,7 +54,7 @@ func Notify(name, status string, data map[string]string) error { return err } - log.At("Notfiy").Log("message-id=%q", *resp.MessageId) + log.At("Notify").Log("message-id=%q", *resp.MessageId) return nil } diff --git a/api/models/release.go b/api/models/release.go index 1bbc3d8e31..2b00830728 100644 --- a/api/models/release.go +++ b/api/models/release.go @@ -34,37 +34,6 @@ func NewRelease(app string) Release { } } -func ListReleases(app string) (Releases, error) { - req := &dynamodb.QueryInput{ - KeyConditions: map[string]*dynamodb.Condition{ - "app": &dynamodb.Condition{ - AttributeValueList: []*dynamodb.AttributeValue{ - &dynamodb.AttributeValue{S: aws.String(app)}, - }, - ComparisonOperator: aws.String("EQ"), - }, - }, - IndexName: aws.String("app.created"), - Limit: aws.Int64(20), - ScanIndexForward: aws.Bool(false), - TableName: aws.String(releasesTable(app)), - } - - res, err := DynamoDB().Query(req) - - if err != nil { - return nil, err - } - - releases := make(Releases, len(res.Items)) - - for i, item := range res.Items { - releases[i] = *releaseFromItem(item) - } - - return releases, nil -} - func GetRelease(app, id string) (*Release, error) { if id == "" { return nil, fmt.Errorf("no release id") @@ -93,23 +62,6 @@ func GetRelease(app, id string) (*Release, error) { return release, nil } -func (r *Release) Cleanup() error { - app, err := GetApp(r.App) - - if err != nil { - return err - } - - // delete env - err = s3Delete(app.Outputs["Settings"], fmt.Sprintf("releases/%s/env", r.Id)) - - if err != nil { - return err - } - - return nil -} - func (r *Release) Save() error { if r.Id == "" { return fmt.Errorf("Id must not be blank") @@ -368,7 +320,7 @@ func (r *Release) resolveLinks(app App, manifest *Manifest) (Manifest, error) { cmd = exec.Command("docker", "inspect", imageName) out, err = cmd.CombinedOutput() - fmt.Printf("ns=kernel at=release.formation at=entry.inspect imageName=%q out=%q err=%q\n", imageName, string(out), err) + // fmt.Printf("ns=kernel at=release.formation at=entry.inspect imageName=%q out=%q err=%q\n", imageName, string(out), err) if err != nil { return m, fmt.Errorf("could not inspect %q", imageName) diff --git a/api/models/ssl.go b/api/models/ssl.go index 5d4b942823..d0ca5e3fdc 100644 --- a/api/models/ssl.go +++ b/api/models/ssl.go @@ -29,7 +29,7 @@ type SSL struct { type SSLs []SSL -func CreateSSL(app, process string, port int, body, key string, chain string, secure bool) (*SSL, error) { +func CreateSSL(app, process string, port int, arn, body, key, chain string, secure bool) (*SSL, error) { a, err := GetApp(app) if err != nil { @@ -72,10 +72,12 @@ func CreateSSL(app, process string, port int, body, key string, chain string, se return nil, fmt.Errorf("process does not expose port: %d", port) } - arn, err := uploadCert(a, process, port, body, key, chain) + if arn == "" { + arn, err = uploadCert(a, process, port, body, key, chain) - if err != nil { - return nil, err + if err != nil { + return nil, err + } } tmpl, err := release.Formation() @@ -232,7 +234,7 @@ func ListSSLs(a string) (SSLs, error) { return ssls, nil } -func UpdateSSL(app, process string, port int, body, key string, chain string) (*SSL, error) { +func UpdateSSL(app, process string, port int, arn, body, key, chain string) (*SSL, error) { a, err := GetApp(app) if err != nil { @@ -260,11 +262,13 @@ func UpdateSSL(app, process string, port int, body, key string, chain string) (* return nil, fmt.Errorf("Balancer ouptut not found. Please redeploy your app and try again.") } - // upload new cert - arn, err := uploadCert(a, process, port, body, key, chain) + if arn == "" { + // upload new cert + arn, err = uploadCert(a, process, port, body, key, chain) - if err != nil { - return nil, err + if err != nil { + return nil, err + } } // update cloudformation diff --git a/api/provider/aws/apps_test.go b/api/provider/aws/apps_test.go index 591c4612bf..da7a39f040 100644 --- a/api/provider/aws/apps_test.go +++ b/api/provider/aws/apps_test.go @@ -32,7 +32,7 @@ func TestAppGet(t *testing.T) { assert.Nil(t, err) assert.EqualValues(t, &structs.App{ Name: "httpd", - Release: "RLLOVNNXWKR", + Release: "RVFETUHHKKD", Status: "running", Outputs: map[string]string{ "BalancerWebHost": "httpd-web-7E5UPCM-1241527783.us-east-1.elb.amazonaws.com", @@ -46,7 +46,7 @@ func TestAppGet(t *testing.T) { }, Parameters: map[string]string{ "WebMemory": "256", - "Release": "RLLOVNNXWKR", + "Release": "RVFETUHHKKD", "Subnets": "subnet-13de3139,subnet-b5578fc3,subnet-21c13379", "Private": "Yes", "WebPort80ProxyProtocol": "No", @@ -57,7 +57,7 @@ func TestAppGet(t *testing.T) { "Repository": "", "WebPort80Balancer": "80", "SubnetsPrivate": "subnet-d4e85cfe,subnet-103d5a66,subnet-57952a0f", - "Environment": "https://convox-httpd-settings-139bidzalmbtu.s3.amazonaws.com/releases/RLLOVNNXWKR/env", + "Environment": "https://convox-httpd-settings-139bidzalmbtu.s3.amazonaws.com/releases/RVFETUHHKKD/env", "WebPort80Certificate": "", "WebPort80Host": "56694", "WebDesiredCount": "1", diff --git a/api/provider/aws/aws.go b/api/provider/aws/aws.go index c385474e1e..b4d1c3fbfa 100644 --- a/api/provider/aws/aws.go +++ b/api/provider/aws/aws.go @@ -14,6 +14,7 @@ import ( "github.com/convox/rack/Godeps/_workspace/src/github.com/aws/aws-sdk-go/service/ecs" "github.com/convox/rack/Godeps/_workspace/src/github.com/aws/aws-sdk-go/service/kinesis" "github.com/convox/rack/Godeps/_workspace/src/github.com/aws/aws-sdk-go/service/s3" + "github.com/convox/rack/Godeps/_workspace/src/github.com/aws/aws-sdk-go/service/sns" ) var ( @@ -93,3 +94,7 @@ func (p *AWSProvider) kinesis() *kinesis.Kinesis { func (p *AWSProvider) s3() *s3.S3 { return s3.New(session.New(), p.config().WithS3ForcePathStyle(true)) } + +func (p *AWSProvider) sns() *sns.SNS { + return sns.New(session.New(), p.config()) +} diff --git a/api/provider/aws/builds.go b/api/provider/aws/builds.go index 06e44fd839..a5f225f0bb 100644 --- a/api/provider/aws/builds.go +++ b/api/provider/aws/builds.go @@ -1,56 +1,191 @@ package aws import ( + "bufio" + "bytes" + "encoding/base64" "errors" "fmt" + "io" "io/ioutil" "os" + "os/exec" + "path/filepath" "regexp" + "strings" "time" "github.com/convox/rack/Godeps/_workspace/src/github.com/aws/aws-sdk-go/aws" "github.com/convox/rack/Godeps/_workspace/src/github.com/aws/aws-sdk-go/service/dynamodb" "github.com/convox/rack/Godeps/_workspace/src/github.com/aws/aws-sdk-go/service/ecr" + "github.com/convox/rack/Godeps/_workspace/src/github.com/aws/aws-sdk-go/service/kinesis" "github.com/convox/rack/Godeps/_workspace/src/github.com/aws/aws-sdk-go/service/s3" "github.com/convox/rack/Godeps/_workspace/src/gopkg.in/yaml.v2" + "github.com/convox/rack/api/helpers" + "github.com/convox/rack/api/manifest" "github.com/convox/rack/api/structs" ) -type ManifestEntries map[string]interface{} - var regexpECR = regexp.MustCompile(`(\d+)\.dkr\.ecr\.([^.]+)\.amazonaws\.com\/([^:]+):([^ ]+)`) func buildsTable(app string) string { return os.Getenv("DYNAMO_BUILDS") } -func (p *AWSProvider) BuildGet(app, id string) (*structs.Build, error) { +func (p *AWSProvider) BuildCopy(srcApp, id, destApp string) (*structs.Build, error) { + srcA, err := p.AppGet(srcApp) + if err != nil { + return nil, err + } + + srcB, err := p.BuildGet(srcApp, id) + if err != nil { + return nil, err + } + + destA, err := p.AppGet(destApp) + if err != nil { + return nil, err + } + + // make a .tgz file that is the source build manifest + // with build directives removed, and image directives pointing to + // fully qualified URLs of source build images + var m manifest.Manifest + err = yaml.Unmarshal([]byte(srcB.Manifest), &m) + if err != nil { + return nil, err + } + + for name, entry := range m { + entry.Build = "" + entry.Image = registryTag(srcA, name, srcB.Id) + m[name] = entry + } + + data, err := m.Raw() + if err != nil { + return nil, err + } + + dir, err := ioutil.TempDir("", "source") + if err != nil { + return nil, err + } + + err = os.Chmod(dir, 0755) + if err != nil { + return nil, err + } + + err = ioutil.WriteFile(filepath.Join(dir, "docker-compose.yml"), data, 0644) + if err != nil { + return nil, err + } + + tgz, err := createTarball(dir) + if err != nil { + return nil, err + } + + // Build .tgz in context of destApp + return p.BuildCreateTar(destA.Name, bytes.NewReader(tgz), "docker-compose.yml", fmt.Sprintf("Copy of %s %s", srcA.Name, srcB.Id), false) +} + +func (p *AWSProvider) BuildCreateIndex(app string, index structs.Index, manifest, description string, cache bool) (*structs.Build, error) { + dir, err := ioutil.TempDir("", "source") + if err != nil { + return nil, err + } + + err = os.Chmod(dir, 0755) + if err != nil { + return nil, err + } + + err = p.IndexDownload(&index, dir) + if err != nil { + return nil, err + } + + tgz, err := createTarball(dir) + if err != nil { + return nil, err + } + + return p.BuildCreateTar(app, bytes.NewReader(tgz), manifest, description, cache) +} + +func (p *AWSProvider) BuildCreateRepo(app, url, manifest, description string, cache bool) (*structs.Build, error) { a, err := p.AppGet(app) if err != nil { return nil, err } - req := &dynamodb.GetItemInput{ - ConsistentRead: aws.Bool(true), - Key: map[string]*dynamodb.AttributeValue{ - "id": &dynamodb.AttributeValue{S: aws.String(id)}, + b := structs.NewBuild(app) + b.Description = description + + err = p.BuildSave(b) + if err != nil { + return nil, err + } + + args := p.buildArgs(a, b, url) + + env, err := p.buildEnv(a, b, manifest, cache) + if err != nil { + return b, err + } + + err = p.buildRun(a, b, args, env, nil) + + // build create is now complete or failed + p.EventSend(&structs.Event{ + Action: "build:create", + Data: map[string]string{ + "app": b.App, + "id": b.Id, }, - TableName: aws.String(buildsTable(app)), + }, err) + + return b, err +} + +func (p *AWSProvider) BuildCreateTar(app string, src io.Reader, manifest, description string, cache bool) (*structs.Build, error) { + a, err := p.AppGet(app) + if err != nil { + return nil, err } - res, err := p.dynamodb().GetItem(req) + b := structs.NewBuild(app) + b.Description = description + + err = p.BuildSave(b) if err != nil { return nil, err } - if res.Item == nil { - return nil, fmt.Errorf("no such build: %s", id) + // TODO: save the tarball in s3? + + args := p.buildArgs(a, b, "-") + + env, err := p.buildEnv(a, b, manifest, cache) + if err != nil { + return b, err } - build := p.buildFromItem(res.Item, a.Outputs["Settings"]) + err = p.buildRun(a, b, args, env, src) - return build, nil + p.EventSend(&structs.Event{ + Action: "build:create", + Data: map[string]string{ + "app": b.App, + "id": b.Id, + }, + }, err) + + return b, err } func (p *AWSProvider) BuildDelete(app, id string) (*structs.Build, error) { @@ -128,64 +263,270 @@ func (p *AWSProvider) BuildDelete(app, id string) (*structs.Build, error) { return b, nil } -// deleteImages generates a list of fully qualified URLs for images for every process type -// in the build manifest then deletes them. -// Image URLs that point to ECR, e.g. 826133048.dkr.ecr.us-east-1.amazonaws.com/myapp-zridvyqapp:web.BSUSBFCUCSA, -// are deleted with the ECR BatchDeleteImage API. -// Image URLs that point to the convox-hosted registry, e.g. convox-826133048.us-east-1.elb.amazonaws.com:5000/myapp-web:BSUSBFCUCSA, -// are not yet supported and return an error. -func (p *AWSProvider) deleteImages(a *structs.App, b *structs.Build) error { - var entries ManifestEntries +func (p *AWSProvider) BuildGet(app, id string) (*structs.Build, error) { + a, err := p.AppGet(app) + if err != nil { + return nil, err + } + + req := &dynamodb.GetItemInput{ + ConsistentRead: aws.Bool(true), + Key: map[string]*dynamodb.AttributeValue{ + "id": &dynamodb.AttributeValue{S: aws.String(id)}, + }, + TableName: aws.String(buildsTable(a.Name)), + } + + res, err := p.dynamodb().GetItem(req) + if err != nil { + return nil, err + } + + if res.Item == nil { + return nil, fmt.Errorf("no such build: %s", id) + } + + build := p.buildFromItem(res.Item, a.Outputs["Settings"]) + + return build, nil +} + +func (p *AWSProvider) BuildList(app string) (structs.Builds, error) { + a, err := p.AppGet(app) + if err != nil { + return nil, err + } + + req := &dynamodb.QueryInput{ + KeyConditions: map[string]*dynamodb.Condition{ + "app": &dynamodb.Condition{ + AttributeValueList: []*dynamodb.AttributeValue{&dynamodb.AttributeValue{S: aws.String(a.Name)}}, + ComparisonOperator: aws.String("EQ"), + }, + }, + IndexName: aws.String("app.created"), + Limit: aws.Int64(20), + ScanIndexForward: aws.Bool(false), + TableName: aws.String(buildsTable(a.Name)), + } + + res, err := p.dynamodb().Query(req) + if err != nil { + return nil, err + } - err := yaml.Unmarshal([]byte(b.Manifest), &entries) + builds := make(structs.Builds, len(res.Items)) + for i, item := range res.Items { + builds[i] = *p.buildFromItem(item, a.Outputs["Settings"]) + } + + return builds, nil +} + +func (p *AWSProvider) BuildRelease(b *structs.Build) (*structs.Release, error) { + releases, err := p.ReleaseList(b.App) + if err != nil { + return nil, err + } + + r := structs.NewRelease(b.App) + newId := r.Id + + if len(releases) > 0 { + r = &releases[0] + } + + r.Id = newId + r.Created = time.Time{} + r.Build = b.Id + r.Manifest = b.Manifest + + a, err := p.AppGet(b.App) + if err != nil { + return r, err + } + + err = p.ReleaseSave(r, a.Outputs["Settings"], a.Parameters["Key"]) + if err != nil { + return r, err + } + + b.Release = r.Id + err = p.BuildSave(b) + + if err == nil { + p.EventSend(&structs.Event{ + Action: "release:create", + Data: map[string]string{ + "app": r.App, + "id": r.Id, + }, + }, nil) + } + + return r, err +} + +// BuildSave creates or updates a build item in DynamoDB. It takes an optional +// bucket argument, which if set indicates to PUT Log data into S3 +func (p *AWSProvider) BuildSave(b *structs.Build) error { + a, err := p.AppGet(b.App) if err != nil { return err } - // failed builds could have an empty manifest - if len(entries) == 0 { - return nil + if b.Id == "" { + return fmt.Errorf("Id can not be blank") } - urls := []string{} + if b.Started.IsZero() { + b.Started = time.Now() + } - for name, _ := range entries { - img := fmt.Sprintf("%s/%s-%s:%s", os.Getenv("REGISTRY_HOST"), a.Name, name, b.Id) + req := &dynamodb.PutItemInput{ + Item: map[string]*dynamodb.AttributeValue{ + "id": &dynamodb.AttributeValue{S: aws.String(b.Id)}, + "app": &dynamodb.AttributeValue{S: aws.String(b.App)}, + "status": &dynamodb.AttributeValue{S: aws.String(b.Status)}, + "created": &dynamodb.AttributeValue{S: aws.String(b.Started.Format(SortableTime))}, + }, + TableName: aws.String(buildsTable(b.App)), + } - if registryId := a.Outputs["RegistryId"]; registryId != "" { - img = fmt.Sprintf("%s.dkr.ecr.%s.amazonaws.com/%s:%s.%s", registryId, os.Getenv("AWS_REGION"), a.Outputs["RegistryRepository"], name, b.Id) - } + if b.Description != "" { + req.Item["description"] = &dynamodb.AttributeValue{S: aws.String(b.Description)} + } - urls = append(urls, img) + if b.Manifest != "" { + req.Item["manifest"] = &dynamodb.AttributeValue{S: aws.String(b.Manifest)} } - imageIds := []*ecr.ImageIdentifier{} - registryId := "" - repositoryName := "" + if b.Release != "" { + req.Item["release"] = &dynamodb.AttributeValue{S: aws.String(b.Release)} + } - for _, url := range urls { - if match := regexpECR.FindStringSubmatch(url); match != nil { - registryId = match[1] - repositoryName = match[3] + if !b.Ended.IsZero() { + req.Item["ended"] = &dynamodb.AttributeValue{S: aws.String(b.Ended.Format(SortableTime))} + } - imageIds = append(imageIds, &ecr.ImageIdentifier{ - ImageTag: aws.String(match[4]), - }) - } else { - return errors.New("URL not valid ECR") + if b.Logs != "" { + _, err := p.s3().PutObject(&s3.PutObjectInput{ + Body: bytes.NewReader([]byte(b.Logs)), + Bucket: aws.String(a.Outputs["Settings"]), + ContentLength: aws.Int64(int64(len(b.Logs))), + Key: aws.String(fmt.Sprintf("builds/%s.log", b.Id)), + }) + if err != nil { + return err } } - _, err = p.ecr().BatchDeleteImage(&ecr.BatchDeleteImageInput{ - ImageIds: imageIds, - RegistryId: aws.String(registryId), - RepositoryName: aws.String(repositoryName), - }) + _, err = p.dynamodb().PutItem(req) return err } +func (p *AWSProvider) buildArgs(a *structs.App, b *structs.Build, source string) []string { + return []string{ + "run", + "-i", + "--name", fmt.Sprintf("build-%s", b.Id), + "-v", "/var/run/docker.sock:/var/run/docker.sock", + "-e", "APP", + "-e", "BUILD", + "-e", "DOCKER_AUTH", + "-e", "RACK_HOST", + "-e", "RACK_PASSWORD", + "-e", "REGISTRY_EMAIL", + "-e", "REGISTRY_USERNAME", + "-e", "REGISTRY_PASSWORD", + "-e", "REGISTRY_ADDRESS", + "-e", "MANIFEST_PATH", + "-e", "REPOSITORY", + "-e", "NO_CACHE", + os.Getenv("DOCKER_IMAGE_API"), + "build", + source, + } +} + +func (p *AWSProvider) buildEnv(a *structs.App, b *structs.Build, manifest_path string, cache bool) ([]string, error) { + // self-hosted registry auth + email := "user@convox.com" + username := "convox" + password := os.Getenv("PASSWORD") + address := os.Getenv("REGISTRY_HOST") + + // ECR auth + if registryId := a.Outputs["RegistryId"]; registryId != "" { + res, err := p.ecr().GetAuthorizationToken(&ecr.GetAuthorizationTokenInput{ + RegistryIds: []*string{aws.String(registryId)}, + }) + + if err != nil { + return nil, err + } + + if len(res.AuthorizationData) < 1 { + return nil, fmt.Errorf("no authorization data") + } + + endpoint := *res.AuthorizationData[0].ProxyEndpoint + + data, err := base64.StdEncoding.DecodeString(*res.AuthorizationData[0].AuthorizationToken) + + if err != nil { + return nil, err + } + + parts := strings.SplitN(string(data), ":", 2) + + password = parts[1] + address = endpoint[8:] + username = parts[0] + } + + // TODO: The controller logged into private registries and app registry + // Seems like this method should be able to generate docker auth config on its own + dockercfg, err := ioutil.ReadFile("/root/.docker/config.json") + if err != nil { + return nil, err + } + + // Determin callback host. Local Rack should use a variant of localhost + host := os.Getenv("NOTIFICATION_HOST") + + if os.Getenv("DEVELOPMENT") == "true" { + out, err := exec.Command("docker", "run", "convox/docker-gateway").Output() + if err != nil { + return nil, err + } + + host = strings.TrimSpace(string(out)) + } + + env := []string{ + fmt.Sprintf("APP=%s", a.Name), + fmt.Sprintf("BUILD=%s", b.Id), + fmt.Sprintf("MANIFEST_PATH=%s", manifest_path), + fmt.Sprintf("DOCKER_AUTH=%s", dockercfg), + fmt.Sprintf("RACK_HOST=%s", host), + fmt.Sprintf("RACK_PASSWORD=%s", os.Getenv("PASSWORD")), + fmt.Sprintf("REGISTRY_EMAIL=%s", email), + fmt.Sprintf("REGISTRY_USERNAME=%s", username), + fmt.Sprintf("REGISTRY_PASSWORD=%s", password), + fmt.Sprintf("REGISTRY_ADDRESS=%s", address), + fmt.Sprintf("REPOSITORY=%s", a.Outputs["RegistryRepository"]), + } + + if cache == false { + env = append(env, "NO_CACHE=true") + } + + return env, nil +} + // buildFromItem populates a Build struct from a DynamoDB Item. It also populates build.Logs // from an S3 object if a bucket is passed in and a builds/B1234.log object exists. func (p *AWSProvider) buildFromItem(item map[string]*dynamodb.AttributeValue, bucket string) *structs.Build { @@ -207,7 +548,9 @@ func (p *AWSProvider) buildFromItem(item map[string]*dynamodb.AttributeValue, bu res, err := p.s3().GetObject(req) if err != nil { - fmt.Printf("aws buildFromItem s3.GetObject bucket=%s key=%s err=%s\n", bucket, key, err) + if awsError(err) != "NoSuchKey" { + fmt.Printf("aws buildFromItem s3.GetObject bucket=%s key=%s err=%s\n", bucket, key, err) + } } else { body, err := ioutil.ReadAll(res.Body) if err != nil { @@ -230,3 +573,184 @@ func (p *AWSProvider) buildFromItem(item map[string]*dynamodb.AttributeValue, bu Ended: ended, } } + +func (p *AWSProvider) buildRun(a *structs.App, b *structs.Build, args []string, env []string, stdin io.Reader) error { + cmd := exec.Command("docker", args...) + cmd.Env = env + cmd.Stdin = stdin + cmd.Stderr = cmd.Stdout // redirect cmd stderr to stdout + + stdout, err := cmd.StdoutPipe() + if err != nil { + helpers.Error(nil, err) // send internal error to rollbar + return err + } + + // start build command + err = cmd.Start() + if err != nil { + helpers.Error(nil, err) // send internal error to rollbar + return err + } + + go p.buildWait(a, b, cmd, stdout) + + return nil +} + +func (p *AWSProvider) buildWait(a *structs.App, b *structs.Build, cmd *exec.Cmd, stdout io.ReadCloser) { + // scan all output + out := "" + scanner := bufio.NewScanner(stdout) + + for scanner.Scan() { + text := scanner.Text() + out += text + "\n" + + p.kinesis().PutRecord(&kinesis.PutRecordInput{ + Data: []byte(text), + PartitionKey: aws.String(string(time.Now().UnixNano())), + StreamName: aws.String(a.Outputs["Kinesis"]), + }) + } + if err := scanner.Err(); err != nil { + helpers.Error(nil, err) // send internal error to rollbar + } + + // and wait for a return code + werr := cmd.Wait() + + // reload build item to get data from BuildUpdate callback + b, err := p.BuildGet(b.App, b.Id) + if err != nil { + helpers.Error(nil, err) // send internal error to rollbar + return + } + + // Wait / return code are errors, consider the build failed + if werr != nil { + b.Status = "failed" + } + + // save final build logs / status + b.Logs = string(out) + err = p.BuildSave(b) + if err != nil { + helpers.Error(nil, err) // send internal error to rollbar + return + } +} + +func createTarball(base string) ([]byte, error) { + cwd, err := os.Getwd() + + if err != nil { + return nil, err + } + + err = os.Chdir(base) + + if err != nil { + return nil, err + } + + args := []string{"cz"} + + // If .dockerignore exists, use it to exclude files from the tarball + if _, err = os.Stat(".dockerignore"); err == nil { + args = append(args, "--exclude-from", ".dockerignore") + } + + args = append(args, ".") + + cmd := exec.Command("tar", args...) + + out, err := cmd.StdoutPipe() + + if err != nil { + return nil, err + } + + cmd.Start() + + bytes, err := ioutil.ReadAll(out) + + if err != nil { + return nil, err + } + + err = cmd.Wait() + + if err != nil { + return nil, err + } + + err = os.Chdir(cwd) + + if err != nil { + return nil, err + } + + return bytes, nil +} + +// deleteImages generates a list of fully qualified URLs for images for every process type +// in the build manifest then deletes them. +// Image URLs that point to ECR, e.g. 826133048.dkr.ecr.us-east-1.amazonaws.com/myapp-zridvyqapp:web.BSUSBFCUCSA, +// are deleted with the ECR BatchDeleteImage API. +// Image URLs that point to the convox-hosted registry, e.g. convox-826133048.us-east-1.elb.amazonaws.com:5000/myapp-web:BSUSBFCUCSA, +// are not yet supported and return an error. +func (p *AWSProvider) deleteImages(a *structs.App, b *structs.Build) error { + var m manifest.Manifest + + err := yaml.Unmarshal([]byte(b.Manifest), &m) + if err != nil { + return err + } + + // failed builds could have an empty manifest + if len(m) == 0 { + return nil + } + + urls := []string{} + + for name, _ := range m { + urls = append(urls, registryTag(a, name, b.Id)) + } + + imageIds := []*ecr.ImageIdentifier{} + registryId := "" + repositoryName := "" + + for _, url := range urls { + if match := regexpECR.FindStringSubmatch(url); match != nil { + registryId = match[1] + repositoryName = match[3] + + imageIds = append(imageIds, &ecr.ImageIdentifier{ + ImageTag: aws.String(match[4]), + }) + } else { + return errors.New("URL not valid ECR") + } + } + + _, err = p.ecr().BatchDeleteImage(&ecr.BatchDeleteImageInput{ + ImageIds: imageIds, + RegistryId: aws.String(registryId), + RepositoryName: aws.String(repositoryName), + }) + + return err +} + +func registryTag(a *structs.App, serviceName, buildId string) string { + tag := fmt.Sprintf("%s/%s-%s:%s", os.Getenv("REGISTRY_HOST"), a.Name, serviceName, buildId) + + if registryId := a.Outputs["RegistryId"]; registryId != "" { + tag = fmt.Sprintf("%s.dkr.ecr.%s.amazonaws.com/%s:%s.%s", registryId, os.Getenv("AWS_REGION"), a.Outputs["RegistryRepository"], serviceName, buildId) + } + + return tag +} diff --git a/api/provider/aws/builds_test.go b/api/provider/aws/builds_test.go index 7e2b0e8400..38db430e02 100644 --- a/api/provider/aws/builds_test.go +++ b/api/provider/aws/builds_test.go @@ -15,13 +15,53 @@ import ( func init() { os.Setenv("RACK", "convox") os.Setenv("DYNAMO_BUILDS", "convox-builds") + os.Setenv("DYNAMO_RELEASES", "convox-releases") } func TestBuildGet(t *testing.T) { aws := StubAwsProvider( describeStacksCycle, - getItemCycle, - getObjectCycle, + + build1GetItemCycle, + build1GetObjectCycle, + ) + defer aws.Close() + + defer func() { + //TODO: remove: as we arent updating all tests we need to set current provider back to a + //clean default one (I miss rspec before) + provider.CurrentProvider = new(provider.TestProviderRunner) + }() + + b, err := provider.BuildGet("httpd", "BHINCLZYYVN") + + assert.Nil(t, err) + assert.EqualValues(t, &structs.Build{ + Id: "BHINCLZYYVN", + App: "httpd", + Logs: "RUNNING: docker pull httpd", + Manifest: "web:\n image: httpd\n ports:\n - 80:80\n", + Release: "RVFETUHHKKD", + Status: "complete", + Started: time.Unix(1459780456, 178278576).UTC(), + Ended: time.Unix(1459780542, 440881687).UTC(), + }, b) +} + +func TestBuildDelete(t *testing.T) { + aws := StubAwsProvider( + describeStacksCycle, + + build2GetItemCycle, + build2GetObjectCycle, + + describeStacksCycle, + releasesBuild2QueryCycle, + + releasesBuild2BatchWriteItemCycle, + build2DeleteItemCycle, + + build2BatchDeleteImageCycle, ) defer aws.Close() @@ -31,18 +71,84 @@ func TestBuildGet(t *testing.T) { provider.CurrentProvider = new(provider.TestProviderRunner) }() - b, err := provider.BuildGet("httpd", "BVZSXXWEIBT") + b, err := provider.BuildDelete("httpd", "BNOARQMVHUO") assert.Nil(t, err) assert.EqualValues(t, &structs.Build{ - Id: "BVZSXXWEIBT", + Id: "BNOARQMVHUO", App: "httpd", Logs: "RUNNING: docker pull httpd", Manifest: "web:\n image: httpd\n ports:\n - 80:80\n", - Release: "RLLOVNNXWKR", + Release: "RFVZFLKVTYO", Status: "complete", - Started: time.Unix(1459444265, 29372915).UTC(), - Ended: time.Unix(1459444334, 284503073).UTC(), + Started: time.Unix(1459709087, 472025215).UTC(), + Ended: time.Unix(1459709198, 984281955).UTC(), + }, b) +} + +func TestBuildDeleteActive(t *testing.T) { + aws := StubAwsProvider( + describeStacksCycle, + + build1GetItemCycle, + build1GetObjectCycle, + + describeStacksCycle, + releasesBuild1QueryCycle, + ) + defer aws.Close() + + defer func() { + //TODO: remove: as we arent updating all tests we need to set current provider back to a + //clean default one (I miss rspec before) + provider.CurrentProvider = new(provider.TestProviderRunner) + }() + + _, err := provider.BuildDelete("httpd", "BHINCLZYYVN") + + assert.Equal(t, err.Error(), "cant delete build contained in active release") +} + +func TestBuildList(t *testing.T) { + aws := StubAwsProvider( + describeStacksCycle, + buildsQueryCycle, + + build1GetObjectCycle, + build2GetObjectCycle, + ) + defer aws.Close() + + defer func() { + //TODO: remove: as we arent updating all tests we need to set current provider back to a + //clean default one (I miss rspec before) + provider.CurrentProvider = new(provider.TestProviderRunner) + }() + + b, err := provider.BuildList("httpd") + + assert.Nil(t, err) + assert.EqualValues(t, structs.Builds{ + structs.Build{ + Id: "BHINCLZYYVN", + App: "httpd", + Logs: "RUNNING: docker pull httpd", + Manifest: "web:\n image: httpd\n ports:\n - 80:80\n", + Release: "RVFETUHHKKD", + Status: "complete", + Started: time.Unix(1459780456, 178278576).UTC(), + Ended: time.Unix(1459780542, 440881687).UTC(), + }, + structs.Build{ + Id: "BNOARQMVHUO", + App: "httpd", + Logs: "RUNNING: docker pull httpd", + Manifest: "web:\n image: httpd\n ports:\n - 80:80\n", + Release: "RFVZFLKVTYO", + Status: "complete", + Started: time.Unix(1459709087, 472025215).UTC(), + Ended: time.Unix(1459709198, 984281955).UTC(), + }, }, b) } @@ -78,7 +184,7 @@ var describeStacksCycle = awsutil.Cycle{ 2016-03-31T17:09:28.583Z - https://convox-httpd-settings-139bidzalmbtu.s3.amazonaws.com/releases/RLLOVNNXWKR/env + https://convox-httpd-settings-139bidzalmbtu.s3.amazonaws.com/releases/RVFETUHHKKD/env Environment @@ -126,7 +232,7 @@ var describeStacksCycle = awsutil.Cycle{ SubnetsPrivate - RLLOVNNXWKR + RVFETUHHKKD Release @@ -197,21 +303,69 @@ var describeStacksCycle = awsutil.Cycle{ `}, } -var getItemCycle = awsutil.Cycle{ +var buildsQueryCycle = awsutil.Cycle{ + Request: awsutil.Request{ + RequestURI: "/", + Operation: "DynamoDB_20120810.Query", + Body: `{"IndexName":"app.created","KeyConditions":{"app":{"AttributeValueList":[{"S":"httpd"}],"ComparisonOperator":"EQ"}},"Limit":20,"ScanIndexForward":false,"TableName":"convox-builds"}`, + }, + Response: awsutil.Response{ + StatusCode: 200, + Body: `{"Count":2,"Items":[{"id":{"S":"BHINCLZYYVN"},"manifest":{"S":"web:\n image: httpd\n ports:\n - 80:80\n"},"release":{"S":"RVFETUHHKKD"},"ended":{"S":"20160404.143542.440881687"},"app":{"S":"httpd"},"created":{"S":"20160404.143416.178278576"},"status":{"S":"complete"}},{"id":{"S":"BNOARQMVHUO"},"manifest":{"S":"web:\n image: httpd\n ports:\n - 80:80\n"},"release":{"S":"RFVZFLKVTYO"},"ended":{"S":"20160403.184638.984281955"},"app":{"S":"httpd"},"created":{"S":"20160403.184447.472025215"},"status":{"S":"complete"}}],"ScannedCount":2}`, + }, +} + +var build1GetItemCycle = awsutil.Cycle{ + Request: awsutil.Request{ + RequestURI: "/", + Operation: "DynamoDB_20120810.GetItem", + Body: `{"ConsistentRead":true,"Key":{"id":{"S":"BHINCLZYYVN"}},"TableName":"convox-builds"}`, + }, + Response: awsutil.Response{ + StatusCode: 200, + Body: `{"Item":{"id":{"S":"BHINCLZYYVN"},"manifest":{"S":"web:\n image: httpd\n ports:\n - 80:80\n"},"ended":{"S":"20160404.143542.440881687"},"release":{"S":"RVFETUHHKKD"},"app":{"S":"httpd"},"created":{"S":"20160404.143416.178278576"},"status":{"S":"complete"}}}`, + }, +} + +var build1GetObjectCycle = awsutil.Cycle{ + Request: awsutil.Request{ + RequestURI: "/convox-httpd-settings-139bidzalmbtu/builds/BHINCLZYYVN.log", + Operation: "", + Body: ``, + }, + Response: awsutil.Response{ + StatusCode: 200, + Body: `RUNNING: docker pull httpd`, + }, +} + +var build2BatchDeleteImageCycle = awsutil.Cycle{ + Request: awsutil.Request{ + RequestURI: "/", + Operation: "AmazonEC2ContainerRegistry_V20150921.BatchDeleteImage", + Body: `{"imageIds":[{"imageTag":"web.BNOARQMVHUO"}],"registryId":"132866487567","repositoryName":"convox-httpd-hqvvfosgxt"}`, + }, + Response: awsutil.Response{ + StatusCode: 200, + Body: `{"failures":[],"imageIds":[{"imageDigest":"sha256:77f27a1381e53241cd230ca1abf74e33ece2715a51e89ba8bdf8908b9a75aa3d","imageTag":"web.BNOARQMVHUO"}]}`, + }, +} + +var build2GetItemCycle = awsutil.Cycle{ Request: awsutil.Request{ RequestURI: "/", Operation: "DynamoDB_20120810.GetItem", - Body: `{"ConsistentRead":true,"Key":{"id":{"S":"BVZSXXWEIBT"}},"TableName":"convox-builds"}`, + Body: `{"ConsistentRead":true,"Key":{"id":{"S":"BNOARQMVHUO"}},"TableName":"convox-builds"}`, }, Response: awsutil.Response{ StatusCode: 200, - Body: `{"Item":{"id":{"S":"BVZSXXWEIBT"},"manifest":{"S":"web:\n image: httpd\n ports:\n - 80:80\n"},"ended":{"S":"20160331.171214.284503073"},"release":{"S":"RLLOVNNXWKR"},"app":{"S":"httpd"},"created":{"S":"20160331.171105.029372915"},"status":{"S":"complete"}}}`, + Body: `{"Item":{"id":{"S":"BNOARQMVHUO"},"manifest":{"S":"web:\n image: httpd\n ports:\n - 80:80\n"},"ended":{"S":"20160403.184638.984281955"},"release":{"S":"RFVZFLKVTYO"},"app":{"S":"httpd"},"created":{"S":"20160403.184447.472025215"},"status":{"S":"complete"}}}`, }, } -var getObjectCycle = awsutil.Cycle{ +var build2GetObjectCycle = awsutil.Cycle{ Request: awsutil.Request{ - RequestURI: "/convox-httpd-settings-139bidzalmbtu/builds/BVZSXXWEIBT.log", + RequestURI: "/convox-httpd-settings-139bidzalmbtu/builds/BNOARQMVHUO.log", Operation: "", Body: ``, }, @@ -220,3 +374,63 @@ var getObjectCycle = awsutil.Cycle{ Body: `RUNNING: docker pull httpd`, }, } + +var build2DeleteItemCycle = awsutil.Cycle{ + Request: awsutil.Request{ + RequestURI: "/", + Operation: "DynamoDB_20120810.DeleteItem", + Body: `{"Key":{"id":{"S":"BNOARQMVHUO"}},"TableName":"convox-builds"}`, + }, + Response: awsutil.Response{ + StatusCode: 200, + Body: `{}`, + }, +} + +var releasesQueryCycle = awsutil.Cycle{ + Request: awsutil.Request{ + RequestURI: "/", + Operation: "DynamoDB_20120810.Query", + Body: `{"IndexName":"app.created","KeyConditions":{"app":{"AttributeValueList":[{"S":"httpd"}],"ComparisonOperator":"EQ"}},"Limit":20,"ScanIndexForward":false,"TableName":"convox-releases"}`, + }, + Response: awsutil.Response{ + StatusCode: 200, + Body: `{"Count":2,"Items":[{"id":{"S":"RVFETUHHKKD"},"build":{"S":"BHINCLZYYVN"},"app":{"S":"httpd"},"manifest":{"S":"web:\n image: httpd\n ports:\n - 80:80\n"},"env":{"S":"foo=bar"},"created":{"S":"20160404.143542.627770380"}},{"id":{"S":"RFVZFLKVTYO"},"build":{"S":"BNOARQMVHUO"},"app":{"S":"httpd"},"manifest":{"S":"web:\n image: httpd\n ports:\n - 80:80\n"},"env":{"S":"foo=bar"},"created":{"S":"20160403.184639.166694813"}}],"ScannedCount":2}`, + }, +} + +var releasesBuild1QueryCycle = awsutil.Cycle{ + Request: awsutil.Request{ + RequestURI: "/", + Operation: "DynamoDB_20120810.Query", + Body: `{"ExpressionAttributeValues":{":app":{"S":"httpd"},":build":{"S":"BHINCLZYYVN"}},"FilterExpression":"build = :build","IndexName":"app.created","KeyConditionExpression":"app = :app","TableName":"convox-releases"}`, + }, + Response: awsutil.Response{ + StatusCode: 200, + Body: `{"Count":1,"Items":[{"id":{"S":"RVFETUHHKKD"},"build":{"S":"BHINCLZYYVN"},"app":{"S":"httpd"},"manifest":{"S":"web:\n image: httpd\n ports:\n - 80:80\n"},"env":{"S":"foo=bar"},"created":{"S":"20160404.143542.627770380"}}],"ScannedCount":2}`, + }, +} + +var releasesBuild2QueryCycle = awsutil.Cycle{ + Request: awsutil.Request{ + RequestURI: "/", + Operation: "DynamoDB_20120810.Query", + Body: `{"ExpressionAttributeValues":{":app":{"S":"httpd"},":build":{"S":"BNOARQMVHUO"}},"FilterExpression":"build = :build","IndexName":"app.created","KeyConditionExpression":"app = :app","TableName":"convox-releases"}`, + }, + Response: awsutil.Response{ + StatusCode: 200, + Body: `{"Count":1,"Items":[{"id":{"S":"RFVZFLKVTYO"},"build":{"S":"BNOARQMVHUO"},"app":{"S":"httpd"},"manifest":{"S":"web:\n image: httpd\n ports:\n - 80:80\n"},"env":{"S":"foo=bar"},"created":{"S":"20160403.184639.166694813"}}],"ScannedCount":2}`, + }, +} + +var releasesBuild2BatchWriteItemCycle = awsutil.Cycle{ + Request: awsutil.Request{ + RequestURI: "/", + Operation: "DynamoDB_20120810.BatchWriteItem", + Body: `{"RequestItems":{"convox-releases":[{"DeleteRequest":{"Key":{"id":{"S":"RFVZFLKVTYO"}}}}]}}`, + }, + Response: awsutil.Response{ + StatusCode: 200, + Body: `{"UnprocessedItems":{}}`, + }, +} diff --git a/api/provider/aws/event.go b/api/provider/aws/event.go new file mode 100644 index 0000000000..a9b155d5b6 --- /dev/null +++ b/api/provider/aws/event.go @@ -0,0 +1,67 @@ +package aws + +import ( + "encoding/json" + "os" + + "github.com/convox/rack/Godeps/_workspace/src/github.com/aws/aws-sdk-go/aws" + "github.com/convox/rack/Godeps/_workspace/src/github.com/aws/aws-sdk-go/service/sns" + "github.com/convox/rack/Godeps/_workspace/src/github.com/ddollar/logger" + "github.com/convox/rack/api/helpers" + "github.com/convox/rack/api/structs" +) + +// EventSend publishes an important message out to the world. +// +// On AWS messages are published to SNS. The Rack has an HTTP endpoint that is an SNS +// subscription, and when a message is delivered forwards them to all configured +// webhook services. +// +// Often the Rack has a Console webhook which facilitates forwarding events +// to Slack with additional formatting and filtering. +// +// Because these are important system events, they are also published to Segment +// for operational metrics. +func (p *AWSProvider) EventSend(e *structs.Event, err error) error { + log := logger.New("ns=kernel") + + e.Status = "success" + + if err != nil { + e.Data["message"] = err.Error() + e.Status = "error" + } + + msg, err := json.Marshal(e) + if err != nil { + helpers.Error(log, err) // report internal errors to Rollbar + return err + } + + // Publish Event to SNS + resp, err := p.sns().Publish(&sns.PublishInput{ + Message: aws.String(string(msg)), // Required + Subject: aws.String(e.Action), + TargetArn: aws.String(os.Getenv("NOTIFICATION_TOPIC")), + }) + if err != nil { + helpers.Error(log, err) // report internal errors to Rollbar + return err + } + + log.At("EventSend").Log("message-id=%q", *resp.MessageId) + + // report event to Segment + params := map[string]interface{}{ + "action": e.Action, + "status": e.Status, + } + + for k, v := range e.Data { + params[k] = v + } + + helpers.TrackEvent("event", params) + + return nil +} diff --git a/api/provider/aws/helpers.go b/api/provider/aws/helpers.go index 9fdf3b2006..f71c2067e5 100644 --- a/api/provider/aws/helpers.go +++ b/api/provider/aws/helpers.go @@ -1,6 +1,7 @@ package aws import ( + "bytes" "encoding/json" "fmt" "io/ioutil" @@ -10,6 +11,7 @@ import ( "github.com/convox/rack/Godeps/_workspace/src/github.com/aws/aws-sdk-go/aws/awserr" "github.com/convox/rack/Godeps/_workspace/src/github.com/aws/aws-sdk-go/service/cloudformation" "github.com/convox/rack/Godeps/_workspace/src/github.com/aws/aws-sdk-go/service/dynamodb" + "github.com/convox/rack/Godeps/_workspace/src/github.com/aws/aws-sdk-go/service/s3" ) type Template struct { @@ -116,6 +118,66 @@ func stackTags(stack *cloudformation.Stack) map[string]string { return tags } +func (p *AWSProvider) s3Exists(bucket, key string) (bool, error) { + _, err := p.s3().HeadObject(&s3.HeadObjectInput{ + Bucket: aws.String(bucket), + Key: aws.String(key), + }) + + if err != nil { + if aerr, ok := err.(awserr.RequestFailure); ok && aerr.StatusCode() == 404 { + return false, nil + } + + return false, err + } + + return true, nil +} + +func (p *AWSProvider) s3Get(bucket, key string) ([]byte, error) { + req := &s3.GetObjectInput{ + Bucket: aws.String(bucket), + Key: aws.String(key), + } + + res, err := p.s3().GetObject(req) + + if err != nil { + return nil, err + } + + return ioutil.ReadAll(res.Body) +} + +func (p *AWSProvider) s3Delete(bucket, key string) error { + req := &s3.DeleteObjectInput{ + Bucket: aws.String(bucket), + Key: aws.String(key), + } + + _, err := p.s3().DeleteObject(req) + + return err +} + +func (p *AWSProvider) s3Put(bucket, key string, data []byte, public bool) error { + req := &s3.PutObjectInput{ + Body: bytes.NewReader(data), + Bucket: aws.String(bucket), + ContentLength: aws.Int64(int64(len(data))), + Key: aws.String(key), + } + + if public { + req.ACL = aws.String("public-read") + } + + _, err := p.s3().PutObject(req) + + return err +} + func (p *AWSProvider) stackUpdate(name string, templateUrl string, changes map[string]string) error { app, err := p.AppGet(name) diff --git a/api/models/index.go b/api/provider/aws/index.go similarity index 57% rename from api/models/index.go rename to api/provider/aws/index.go index 2f169523aa..031d3acfe5 100644 --- a/api/models/index.go +++ b/api/provider/aws/index.go @@ -1,4 +1,4 @@ -package models +package aws import ( "fmt" @@ -8,25 +8,14 @@ import ( "time" "github.com/convox/rack/api/cache" + "github.com/convox/rack/api/structs" ) var ( IndexOperationConcurrency = 128 ) -type Index map[string]IndexItem - -type IndexItem struct { - Name string `json:"name"` - Mode os.FileMode `json:"mode"` - ModTime time.Time `json:"mtime"` -} - -func IndexUpload(hash string, data []byte) error { - return S3Put(os.Getenv("SETTINGS_BUCKET"), fmt.Sprintf("index/%s", hash), data, false) -} - -func (index Index) Diff() ([]string, error) { +func (p *AWSProvider) IndexDiff(index *structs.Index) ([]string, error) { missing := []string{} bucket := os.Getenv("SETTINGS_BUCKET") @@ -36,16 +25,16 @@ func (index Index) Diff() ([]string, error) { errch := make(chan error) for i := 1; i < IndexOperationConcurrency; i++ { - go missingHashes(bucket, inch, outch, errch) + go p.missingHashes(bucket, inch, outch, errch) } go func() { - for hash := range index { + for hash := range *index { inch <- hash } }() - for range index { + for range *index { select { case hash := <-outch: if hash != "" { @@ -61,23 +50,23 @@ func (index Index) Diff() ([]string, error) { return missing, nil } -func (index Index) Download(dir string) error { +func (p *AWSProvider) IndexDownload(index *structs.Index, dir string) error { bucket := os.Getenv("SETTINGS_BUCKET") inch := make(chan string) errch := make(chan error) for i := 1; i < IndexOperationConcurrency; i++ { - go downloadItems(bucket, index, dir, inch, errch) + go p.downloadItems(bucket, *index, dir, inch, errch) } go func() { - for hash := range index { + for hash := range *index { inch <- hash } }() - for range index { + for range *index { if err := <-errch; err != nil { return err } @@ -86,46 +75,18 @@ func (index Index) Download(dir string) error { return nil } -func missingHashes(bucket string, inch, outch chan string, errch chan error) { - for hash := range inch { - exists, err := hashExists(bucket, hash) - - if err != nil { - errch <- err - } else if !exists { - outch <- hash - } else { - outch <- "" - } - } +func (p *AWSProvider) IndexUpload(hash string, data []byte) error { + return p.s3Put(os.Getenv("SETTINGS_BUCKET"), fmt.Sprintf("index/%s", hash), data, false) } -func hashExists(bucket, hash string) (bool, error) { - if exists, ok := cache.Get("index.missingHash", hash).(bool); ok && exists { - return true, nil - } - - exists, err := s3Exists(bucket, fmt.Sprintf("index/%s", hash)) - - if err != nil { - return false, err - } - - if exists { - cache.Set("index.missingHash", hash, true, 30*24*time.Hour) - } - - return exists, nil -} - -func downloadItems(bucket string, index Index, dir string, inch chan string, errch chan error) { +func (p *AWSProvider) downloadItems(bucket string, index structs.Index, dir string, inch chan string, errch chan error) { for hash := range inch { - errch <- downloadItem(bucket, hash, index[hash], dir) + errch <- p.downloadItem(bucket, hash, index[hash], dir) } } -func downloadItem(bucket, hash string, item IndexItem, dir string) error { - data, err := s3Get(bucket, fmt.Sprintf("index/%s", hash)) +func (p *AWSProvider) downloadItem(bucket, hash string, item structs.IndexItem, dir string) error { + data, err := p.s3Get(bucket, fmt.Sprintf("index/%s", hash)) if err != nil { return err @@ -153,3 +114,35 @@ func downloadItem(bucket, hash string, item IndexItem, dir string) error { return nil } + +func (p *AWSProvider) missingHashes(bucket string, inch, outch chan string, errch chan error) { + for hash := range inch { + exists, err := p.hashExists(bucket, hash) + + if err != nil { + errch <- err + } else if !exists { + outch <- hash + } else { + outch <- "" + } + } +} + +func (p *AWSProvider) hashExists(bucket, hash string) (bool, error) { + if exists, ok := cache.Get("index.missingHash", hash).(bool); ok && exists { + return true, nil + } + + exists, err := p.s3Exists(bucket, fmt.Sprintf("index/%s", hash)) + + if err != nil { + return false, err + } + + if exists { + cache.Set("index.missingHash", hash, true, 30*24*time.Hour) + } + + return exists, nil +} diff --git a/api/provider/aws/releases.go b/api/provider/aws/releases.go index d70ceadf14..3995016c4a 100644 --- a/api/provider/aws/releases.go +++ b/api/provider/aws/releases.go @@ -1,12 +1,15 @@ package aws import ( + "bytes" "fmt" "os" "time" "github.com/convox/rack/Godeps/_workspace/src/github.com/aws/aws-sdk-go/aws" "github.com/convox/rack/Godeps/_workspace/src/github.com/aws/aws-sdk-go/service/dynamodb" + "github.com/convox/rack/Godeps/_workspace/src/github.com/aws/aws-sdk-go/service/s3" + "github.com/convox/rack/api/crypt" "github.com/convox/rack/api/structs" ) @@ -14,13 +17,34 @@ func releasesTable(app string) string { return os.Getenv("DYNAMO_RELEASES") } +func (p *AWSProvider) ReleaseDelete(app, id string) (*structs.Release, error) { + r, err := p.ReleaseGet(app, id) + if err != nil { + return r, err + } + + a, err := p.AppGet(app) + if err != nil { + return r, err + } + + err = p.s3Delete(a.Outputs["Settings"], fmt.Sprintf("releases/%s/env", r.Id)) + + return r, err +} + func (p *AWSProvider) ReleaseGet(app, id string) (*structs.Release, error) { + a, err := p.AppGet(app) + if err != nil { + return nil, err + } + req := &dynamodb.GetItemInput{ ConsistentRead: aws.Bool(true), Key: map[string]*dynamodb.AttributeValue{ "id": &dynamodb.AttributeValue{S: aws.String(id)}, }, - TableName: aws.String(releasesTable(app)), + TableName: aws.String(releasesTable(a.Name)), } res, err := p.dynamodb().GetItem(req) @@ -38,6 +62,107 @@ func (p *AWSProvider) ReleaseGet(app, id string) (*structs.Release, error) { return release, nil } +func (p *AWSProvider) ReleaseList(app string) (structs.Releases, error) { + a, err := p.AppGet(app) + if err != nil { + return nil, err + } + + req := &dynamodb.QueryInput{ + KeyConditions: map[string]*dynamodb.Condition{ + "app": &dynamodb.Condition{ + AttributeValueList: []*dynamodb.AttributeValue{ + &dynamodb.AttributeValue{S: aws.String(a.Name)}, + }, + ComparisonOperator: aws.String("EQ"), + }, + }, + IndexName: aws.String("app.created"), + Limit: aws.Int64(20), + ScanIndexForward: aws.Bool(false), + TableName: aws.String(releasesTable(a.Name)), + } + + res, err := p.dynamodb().Query(req) + if err != nil { + return nil, err + } + + releases := make(structs.Releases, len(res.Items)) + + for i, item := range res.Items { + releases[i] = *releaseFromItem(item) + } + + return releases, nil +} + +func (p *AWSProvider) ReleasePromote(app, id string) (*structs.Release, error) { + _, err := p.AppGet(app) + if err != nil { + return nil, err + } + + return &structs.Release{}, fmt.Errorf("promote not yet implemented for AWS provider") +} + +func (p *AWSProvider) ReleaseSave(r *structs.Release, bucket, key string) error { + if r.Id == "" { + return fmt.Errorf("Id can not be blank") + } + + if r.Created.IsZero() { + r.Created = time.Now() + } + + req := &dynamodb.PutItemInput{ + Item: map[string]*dynamodb.AttributeValue{ + "id": &dynamodb.AttributeValue{S: aws.String(r.Id)}, + "app": &dynamodb.AttributeValue{S: aws.String(r.App)}, + "created": &dynamodb.AttributeValue{S: aws.String(r.Created.Format(SortableTime))}, + }, + TableName: aws.String(releasesTable(r.App)), + } + + if r.Build != "" { + req.Item["build"] = &dynamodb.AttributeValue{S: aws.String(r.Build)} + } + + if r.Env != "" { + req.Item["env"] = &dynamodb.AttributeValue{S: aws.String(r.Env)} + } + + if r.Manifest != "" { + req.Item["manifest"] = &dynamodb.AttributeValue{S: aws.String(r.Manifest)} + } + + var err error + env := []byte(r.Env) + + if key != "" { + cr := crypt.New(os.Getenv("AWS_REGION"), os.Getenv("AWS_ACCESS"), os.Getenv("AWS_SECRET")) + + env, err = cr.Encrypt(key, []byte(env)) + if err != nil { + return err + } + } + + _, err = p.s3().PutObject(&s3.PutObjectInput{ + ACL: aws.String("public-read"), + Body: bytes.NewReader(env), + Bucket: aws.String(bucket), + ContentLength: aws.Int64(int64(len(env))), + Key: aws.String(fmt.Sprintf("releases/%s/env", r.Id)), + }) + if err != nil { + return err + } + + _, err = p.dynamodb().PutItem(req) + return err +} + func releaseFromItem(item map[string]*dynamodb.AttributeValue) *structs.Release { created, _ := time.Parse(SortableTime, coalesce(item["created"], "")) diff --git a/api/provider/aws/releases_test.go b/api/provider/aws/releases_test.go new file mode 100644 index 0000000000..ce655926c0 --- /dev/null +++ b/api/provider/aws/releases_test.go @@ -0,0 +1,88 @@ +package aws_test + +import ( + "testing" + "time" + + "github.com/convox/rack/api/awsutil" + "github.com/convox/rack/api/provider" + "github.com/convox/rack/api/structs" + + "github.com/convox/rack/Godeps/_workspace/src/github.com/stretchr/testify/assert" +) + +func TestReleaseGet(t *testing.T) { + aws := StubAwsProvider( + describeStacksCycle, + + release1GetItemCycle, + ) + defer aws.Close() + + defer func() { + //TODO: remove: as we arent updating all tests we need to set current provider back to a + //clean default one (I miss rspec before) + provider.CurrentProvider = new(provider.TestProviderRunner) + }() + + r, err := provider.ReleaseGet("httpd", "RVFETUHHKKD") + + assert.Nil(t, err) + assert.EqualValues(t, &structs.Release{ + Id: "RVFETUHHKKD", + App: "httpd", + Build: "BHINCLZYYVN", + Env: "foo=bar", + Manifest: "web:\n image: httpd\n ports:\n - 80:80\n", + Created: time.Unix(1459780542, 627770380).UTC(), + }, r) +} + +func TestReleaseList(t *testing.T) { + aws := StubAwsProvider( + describeStacksCycle, + releasesQueryCycle, + ) + defer aws.Close() + + defer func() { + //TODO: remove: as we arent updating all tests we need to set current provider back to a + //clean default one (I miss rspec before) + provider.CurrentProvider = new(provider.TestProviderRunner) + }() + + r, err := provider.ReleaseList("httpd") + + assert.Nil(t, err) + + assert.EqualValues(t, structs.Releases{ + structs.Release{ + Id: "RVFETUHHKKD", + App: "httpd", + Build: "BHINCLZYYVN", + Env: "foo=bar", + Manifest: "web:\n image: httpd\n ports:\n - 80:80\n", + Created: time.Unix(1459780542, 627770380).UTC(), + }, + structs.Release{ + Id: "RFVZFLKVTYO", + App: "httpd", + Build: "BNOARQMVHUO", + Env: "foo=bar", + Manifest: "web:\n image: httpd\n ports:\n - 80:80\n", + Created: time.Unix(1459709199, 166694813).UTC(), + }, + }, r) +} + +var release1GetItemCycle = awsutil.Cycle{ + Request: awsutil.Request{ + RequestURI: "/", + Operation: "DynamoDB_20120810.GetItem", + Body: `{"ConsistentRead":true,"Key":{"id":{"S":"RVFETUHHKKD"}},"TableName":"convox-releases"}`, + }, + Response: awsutil.Response{ + StatusCode: 200, + Body: `{"Item":{"id":{"S":"RVFETUHHKKD"},"build":{"S":"BHINCLZYYVN"},"app":{"S":"httpd"},"manifest":{"S":"web:\n image: httpd\n ports:\n - 80:80\n"},"env":{"S":"foo=bar"},"created":{"S":"20160404.143542.627770380"}}}`, + }, +} diff --git a/api/provider/provider.go b/api/provider/provider.go index c415805054..ada8abfa88 100644 --- a/api/provider/provider.go +++ b/api/provider/provider.go @@ -2,6 +2,7 @@ package provider import ( "fmt" + "io" "os" "github.com/convox/rack/api/provider/aws" @@ -13,14 +14,31 @@ var CurrentProvider Provider type Provider interface { AppGet(name string) (*structs.App, error) - BuildGet(app, id string) (*structs.Build, error) + BuildCopy(srcApp, id, destApp string) (*structs.Build, error) + BuildCreateIndex(app string, index structs.Index, manifest, description string, cache bool) (*structs.Build, error) + BuildCreateRepo(app, url, manifest, description string, cache bool) (*structs.Build, error) + BuildCreateTar(app string, src io.Reader, manifest, description string, cache bool) (*structs.Build, error) BuildDelete(app, id string) (*structs.Build, error) + BuildGet(app, id string) (*structs.Build, error) + BuildList(app string) (structs.Builds, error) + BuildRelease(*structs.Build) (*structs.Release, error) + BuildSave(*structs.Build) error CapacityGet() (*structs.Capacity, error) + EventSend(*structs.Event, error) error + + IndexDiff(*structs.Index) ([]string, error) + IndexDownload(*structs.Index, string) error + IndexUpload(string, []byte) error + InstanceList() (structs.Instances, error) + ReleaseDelete(app, id string) (*structs.Release, error) ReleaseGet(app, id string) (*structs.Release, error) + ReleaseList(app string) (structs.Releases, error) + ReleasePromote(app, id string) (*structs.Release, error) + ReleaseSave(*structs.Release, string, string) error SystemGet() (*structs.System, error) SystemSave(system structs.System) error @@ -49,26 +67,86 @@ func AppGet(name string) (*structs.App, error) { return CurrentProvider.AppGet(name) } -func BuildGet(app, id string) (*structs.Build, error) { - return CurrentProvider.BuildGet(app, id) +func BuildCopy(srcApp, id, destApp string) (*structs.Build, error) { + return CurrentProvider.BuildCopy(srcApp, id, destApp) +} + +func BuildCreateIndex(app string, index structs.Index, manifest, description string, cache bool) (*structs.Build, error) { + return CurrentProvider.BuildCreateIndex(app, index, manifest, description, cache) +} + +func BuildCreateRepo(app, url, manifest, description string, cache bool) (*structs.Build, error) { + return CurrentProvider.BuildCreateRepo(app, url, manifest, description, cache) +} + +func BuildCreateTar(app string, src io.Reader, manifest, description string, cache bool) (*structs.Build, error) { + return CurrentProvider.BuildCreateTar(app, src, manifest, description, cache) } func BuildDelete(app, id string) (*structs.Build, error) { return CurrentProvider.BuildDelete(app, id) } +func BuildGet(app, id string) (*structs.Build, error) { + return CurrentProvider.BuildGet(app, id) +} + +func BuildList(app string) (structs.Builds, error) { + return CurrentProvider.BuildList(app) +} + +func BuildRelease(b *structs.Build) (*structs.Release, error) { + return CurrentProvider.BuildRelease(b) +} + +func BuildSave(b *structs.Build) error { + return CurrentProvider.BuildSave(b) +} + func CapacityGet() (*structs.Capacity, error) { return CurrentProvider.CapacityGet() } +func EventSend(e *structs.Event, err error) error { + return CurrentProvider.EventSend(e, err) +} + +func IndexDiff(i *structs.Index) ([]string, error) { + return CurrentProvider.IndexDiff(i) +} + +func IndexDownload(i *structs.Index, dir string) error { + return CurrentProvider.IndexDownload(i, dir) +} + +func IndexUpload(hash string, data []byte) error { + return CurrentProvider.IndexUpload(hash, data) +} + func InstanceList() (structs.Instances, error) { return CurrentProvider.InstanceList() } +func ReleaseDelete(app, id string) (*structs.Release, error) { + return CurrentProvider.ReleaseDelete(app, id) +} + func ReleaseGet(app, id string) (*structs.Release, error) { return CurrentProvider.ReleaseGet(app, id) } +func ReleaseList(app string) (structs.Releases, error) { + return CurrentProvider.ReleaseList(app) +} + +func ReleasePromote(app, id string) (*structs.Release, error) { + return CurrentProvider.ReleasePromote(app, id) +} + +func ReleaseSave(r *structs.Release, logdir, key string) error { + return CurrentProvider.ReleaseSave(r, logdir, key) +} + func SystemGet() (*structs.System, error) { return CurrentProvider.SystemGet() } diff --git a/api/provider/test.go b/api/provider/test.go index 02c8d134a7..0bc52ab94c 100644 --- a/api/provider/test.go +++ b/api/provider/test.go @@ -1,6 +1,8 @@ package provider import ( + "io" + "github.com/convox/rack/Godeps/_workspace/src/github.com/stretchr/testify/mock" "github.com/convox/rack/api/structs" ) @@ -11,9 +13,11 @@ type TestProviderRunner struct { mock.Mock App structs.App Build structs.Build + Builds structs.Builds Capacity structs.Capacity Instances structs.Instances Release structs.Release + Releases structs.Releases } func (p *TestProviderRunner) AppGet(name string) (*structs.App, error) { @@ -21,8 +25,23 @@ func (p *TestProviderRunner) AppGet(name string) (*structs.App, error) { return &p.App, nil } -func (p *TestProviderRunner) BuildGet(app, id string) (*structs.Build, error) { - p.Called(app, id) +func (p *TestProviderRunner) BuildCopy(srcApp, id, destApp string) (*structs.Build, error) { + p.Called(srcApp, id, destApp) + return &p.Build, nil +} + +func (p *TestProviderRunner) BuildCreateIndex(app string, index structs.Index, manifest, description string, cache bool) (*structs.Build, error) { + p.Called(app, index, manifest, description, cache) + return &p.Build, nil +} + +func (p *TestProviderRunner) BuildCreateRepo(app, url, manifest, description string, cache bool) (*structs.Build, error) { + p.Called(app, url, manifest, description, cache) + return &p.Build, nil +} + +func (p *TestProviderRunner) BuildCreateTar(app string, src io.Reader, manifest, description string, cache bool) (*structs.Build, error) { + p.Called(app, src, manifest, description, cache) return &p.Build, nil } @@ -31,21 +50,81 @@ func (p *TestProviderRunner) BuildDelete(app, id string) (*structs.Build, error) return &p.Build, nil } +func (p *TestProviderRunner) BuildGet(app, id string) (*structs.Build, error) { + p.Called(app, id) + return &p.Build, nil +} + +func (p *TestProviderRunner) BuildList(app string) (structs.Builds, error) { + p.Called(app) + return p.Builds, nil +} + +func (p *TestProviderRunner) BuildRelease(b *structs.Build) (*structs.Release, error) { + p.Called(b) + return &p.Release, nil +} + +func (p *TestProviderRunner) BuildSave(b *structs.Build) error { + p.Called(b) + return nil +} + func (p *TestProviderRunner) CapacityGet() (*structs.Capacity, error) { p.Called() return &p.Capacity, nil } +func (p *TestProviderRunner) EventSend(e *structs.Event, err error) error { + p.Called(e, err) + return nil +} + +func (p *TestProviderRunner) IndexDiff(i *structs.Index) ([]string, error) { + p.Called(i) + return []string{}, nil +} + +func (p *TestProviderRunner) IndexDownload(i *structs.Index, dir string) error { + p.Called(i, dir) + return nil +} + +func (p *TestProviderRunner) IndexUpload(hash string, data []byte) error { + p.Called(hash, data) + return nil +} + func (p *TestProviderRunner) InstanceList() (structs.Instances, error) { p.Called() return p.Instances, nil } +func (p *TestProviderRunner) ReleaseDelete(app, id string) (*structs.Release, error) { + p.Called(app, id) + return &p.Release, nil +} + func (p *TestProviderRunner) ReleaseGet(app, id string) (*structs.Release, error) { p.Called(app, id) return &p.Release, nil } +func (p *TestProviderRunner) ReleaseList(app string) (structs.Releases, error) { + p.Called(app) + return p.Releases, nil +} + +func (p *TestProviderRunner) ReleasePromote(app, id string) (*structs.Release, error) { + p.Called(app, id) + return &p.Release, nil +} + +func (p *TestProviderRunner) ReleaseSave(r *structs.Release, logdir, key string) error { + p.Called(r, logdir, key) + return nil +} + func (p *TestProviderRunner) SystemGet() (*structs.System, error) { p.Called() return nil, nil diff --git a/api/structs/build.go b/api/structs/build.go index 9b5975a993..3a6e8cb87c 100644 --- a/api/structs/build.go +++ b/api/structs/build.go @@ -1,6 +1,9 @@ package structs -import "time" +import ( + "math/rand" + "time" +) type Build struct { Id string `json:"id"` @@ -8,7 +11,9 @@ type Build struct { Logs string `json:"logs"` Manifest string `json:"manifest"` Release string `json:"release"` - Status string `json:"status"` + + Status string `json:"status"` + Reason string `json:"reason"` Description string `json:"description"` @@ -17,3 +22,21 @@ type Build struct { } type Builds []Build + +func NewBuild(app string) *Build { + return &Build{ + App: app, + Id: generateId("B", 10), + Status: "created", + } +} + +var idAlphabet = []rune("ABCDEFGHIJKLMNOPQRSTUVWXYZ") + +func generateId(prefix string, size int) string { + b := make([]rune, size) + for i := range b { + b[i] = idAlphabet[rand.Intn(len(idAlphabet))] + } + return prefix + string(b) +} diff --git a/api/structs/event.go b/api/structs/event.go new file mode 100644 index 0000000000..4bbb6cce58 --- /dev/null +++ b/api/structs/event.go @@ -0,0 +1,10 @@ +package structs + +import "time" + +type Event struct { + Action string `json:"action"` // app:create, release:create, release:promote, etc. + Status string `json:"status"` // success or error + Data map[string]string `json:"data"` // {"rack": "example-rack", "app": "example-app", "id": "R123456789", "message": "unable to load release"} + Timestamp time.Time `json:"timestamp"` +} diff --git a/api/structs/index.go b/api/structs/index.go new file mode 100644 index 0000000000..f3e3fcd018 --- /dev/null +++ b/api/structs/index.go @@ -0,0 +1,14 @@ +package structs + +import ( + "os" + "time" +) + +type Index map[string]IndexItem + +type IndexItem struct { + Name string `json:"name"` + Mode os.FileMode `json:"mode"` + ModTime time.Time `json:"mtime"` +} diff --git a/api/structs/release.go b/api/structs/release.go index 62e14a349b..9af11d6087 100644 --- a/api/structs/release.go +++ b/api/structs/release.go @@ -12,3 +12,10 @@ type Release struct { } type Releases []Release + +func NewRelease(app string) *Release { + return &Release{ + App: app, + Id: generateId("R", 10), + } +} diff --git a/client/builds.go b/client/builds.go index ac6024e91f..57ae3adb89 100644 --- a/client/builds.go +++ b/client/builds.go @@ -140,3 +140,21 @@ func (c *Client) DeleteBuild(app, id string) (*Build, error) { return &build, err } + +func (c *Client) UpdateBuild(app, id, manifest, status, reason string) (*Build, error) { + params := Params{ + "manifest": manifest, + "status": status, + "reason": reason, + } + + var build Build + + err := c.Put(fmt.Sprintf("/apps/%s/builds/%s", app, id), params, &build) + + if err != nil { + return nil, err + } + + return &build, nil +} diff --git a/client/processes.go b/client/processes.go index 02e4849b47..1c430a234d 100644 --- a/client/processes.go +++ b/client/processes.go @@ -79,7 +79,7 @@ func (c *Client) ExecProcessAttached(app, pid, command string, in io.Reader, out return code, nil } -func (c *Client) RunProcessAttached(app, process, command string, height, width int, in io.Reader, out io.WriteCloser) (int, error) { +func (c *Client) RunProcessAttached(app, process, command, release string, height, width int, in io.Reader, out io.WriteCloser) (int, error) { r, w := io.Pipe() defer r.Close() @@ -91,6 +91,7 @@ func (c *Client) RunProcessAttached(app, process, command string, height, width headers := map[string]string{ "Command": command, + "Release": release, "Height": strconv.Itoa(height), "Width": strconv.Itoa(width), } @@ -106,11 +107,12 @@ func (c *Client) RunProcessAttached(app, process, command string, height, width return code, nil } -func (c *Client) RunProcessDetached(app, process, command string) error { +func (c *Client) RunProcessDetached(app, process, command, release string) error { var success interface{} params := map[string]string{ "command": command, + "release": release, } err := c.Post(fmt.Sprintf("/apps/%s/processes/%s/run", app, process), params, &success) diff --git a/client/ssl.go b/client/ssl.go index 41b728596a..01e99589c3 100644 --- a/client/ssl.go +++ b/client/ssl.go @@ -15,8 +15,9 @@ type SSL struct { type SSLs []SSL -func (c *Client) CreateSSL(app, process, port, body, key string, chain string, secure bool) (*SSL, error) { +func (c *Client) CreateSSL(app, process, port, arn, body, key, chain string, secure bool) (*SSL, error) { params := Params{ + "arn": arn, "body": body, "chain": chain, "key": key, @@ -60,8 +61,9 @@ func (c *Client) ListSSL(app string) (*SSLs, error) { return &ssls, nil } -func (c *Client) UpdateSSL(app, process, port, body, key string, chain string) (*SSL, error) { +func (c *Client) UpdateSSL(app, process, port, arn, body, key, chain string) (*SSL, error) { params := Params{ + "arn": arn, "body": body, "chain": chain, "key": key, diff --git a/cmd/convox/builds.go b/cmd/convox/builds.go index b95256cb02..dbd7affaf0 100644 --- a/cmd/convox/builds.go +++ b/cmd/convox/builds.go @@ -22,10 +22,8 @@ import ( var ( IndexOperationConcurrency = 128 -) -func init() { - createFlags := []cli.Flag{ + buildCreateFlags = []cli.Flag{ appFlag, cli.BoolFlag{ Name: "no-cache", @@ -46,13 +44,15 @@ func init() { Usage: "description of the build", }, } +) +func init() { stdcli.RegisterCommand(cli.Command{ Name: "build", Description: "create a new build", Usage: "", Action: cmdBuildsCreate, - Flags: createFlags, + Flags: buildCreateFlags, }) stdcli.RegisterCommand(cli.Command{ Name: "builds", @@ -66,7 +66,7 @@ func init() { Description: "create a new build", Usage: "", Action: cmdBuildsCreate, - Flags: createFlags, + Flags: buildCreateFlags, }, { Name: "copy", @@ -244,7 +244,6 @@ func cmdBuildsCopy(c *cli.Context) { fmt.Print("Copying build... ") b, err := rackClient(c).CopyBuild(app, build, destApp) - if err != nil { stdcli.Error(err) return @@ -252,11 +251,17 @@ func cmdBuildsCopy(c *cli.Context) { fmt.Println("OK") - if b.Release != "" { + releaseId, err := finishBuild(c, destApp, b) + if err != nil { + stdcli.Error(err) + return + } + + if releaseId != "" { if c.Bool("promote") { - fmt.Printf("Promoting %s... ", b.Release) + fmt.Printf("Promoting %s %s... ", destApp, releaseId) - _, err = rackClient(c).PromoteRelease(destApp, b.Release) + _, err = rackClient(c).PromoteRelease(destApp, releaseId) if err != nil { stdcli.Error(err) @@ -265,7 +270,7 @@ func cmdBuildsCopy(c *cli.Context) { fmt.Println("OK") } else { - fmt.Printf("To deploy this copy run `convox releases promote %s --app %s`\n", b.Release, destApp) + fmt.Printf("To deploy this copy run `convox releases promote %s --app %s`\n", releaseId, destApp) } } } @@ -290,20 +295,22 @@ func executeBuild(c *cli.Context, source, app, manifest, description string) (st func createIndex(dir string) (client.Index, error) { index := client.Index{} - ignore, err := readDockerIgnore(dir) + err := warnUnignoredEnv(dir) + if err != nil { + return nil, err + } + ignore, err := readDockerIgnore(dir) if err != nil { return nil, err } resolved, err := filepath.EvalSymlinks(dir) - if err != nil { return nil, err } err = filepath.Walk(resolved, indexWalker(resolved, index, ignore)) - if err != nil { return nil, err } @@ -509,13 +516,17 @@ func executeBuildDirIncremental(c *cli.Context, dir, app, manifest, description } func executeBuildDir(c *cli.Context, dir, app, manifest, description string) (string, error) { - dir, err := filepath.Abs(dir) + err := warnUnignoredEnv(dir) + if err != nil { + return "", err + } + dir, err = filepath.Abs(dir) if err != nil { return "", err } - fmt.Print("Creating tarball... ") + fmt.Println("Creating tarball... ") tar, err := createTarball(dir) @@ -527,7 +538,7 @@ func executeBuildDir(c *cli.Context, dir, app, manifest, description string) (st cache := !c.Bool("no-cache") - fmt.Print("Uploading... ") + fmt.Println("Uploading... ") build, err := rackClient(c).CreateBuildSource(app, tar, cache, manifest, description) @@ -649,3 +660,42 @@ func waitForBuild(c *cli.Context, app, id string) (string, error) { return "", fmt.Errorf("can't get here") } + +func warnUnignoredEnv(dir string) error { + hasDockerIgnore := false + hasDotEnv := false + warn := false + + if _, err := os.Stat(".env"); err == nil { + hasDotEnv = true + } + + if _, err := os.Stat(".dockerignore"); err == nil { + hasDockerIgnore = true + } + + if !hasDockerIgnore && hasDotEnv { + warn = true + } else if hasDockerIgnore && hasDotEnv { + lines, err := readDockerIgnore(dir) + if err != nil { + return err + } + + if len(lines) == 0 { + warn = true + } else { + warn = true + for _, line := range lines { + if line == ".env" { + warn = false + break + } + } + } + } + if warn { + fmt.Println("WARNING: You have a .env file that is not in your .dockerignore, you may be leaking secrets") + } + return nil +} diff --git a/cmd/convox/deploy.go b/cmd/convox/deploy.go index d5dd5c88b4..3b10252d6c 100644 --- a/cmd/convox/deploy.go +++ b/cmd/convox/deploy.go @@ -13,27 +13,7 @@ func init() { Description: "deploy an app to AWS", Usage: "", Action: cmdDeploy, - Flags: []cli.Flag{ - appFlag, - cli.BoolFlag{ - Name: "no-cache", - Usage: "pull fresh image dependencies", - }, - cli.BoolFlag{ - Name: "incremental", - Usage: "use incremental build", - }, - cli.StringFlag{ - Name: "file, f", - Value: "docker-compose.yml", - Usage: "location of docker-compose.yml", - }, - cli.StringFlag{ - Name: "description", - Value: "", - Usage: "description of the build", - }, - }, + Flags: buildCreateFlags, }) } diff --git a/cmd/convox/run.go b/cmd/convox/run.go index 1e18611a14..f2179a0138 100644 --- a/cmd/convox/run.go +++ b/cmd/convox/run.go @@ -24,6 +24,10 @@ func init() { Name: "detach", Usage: "run in the background", }, + cli.StringFlag{ + Name: "release, r", + Usage: "Release Name. Defaults to current release.", + }, }, }) } @@ -50,7 +54,9 @@ func cmdRun(c *cli.Context) { args := strings.Join(c.Args()[1:], " ") - code, err := runAttached(c, app, ps, args) + release := c.String("release") + + code, err := runAttached(c, app, ps, args, release) if err != nil { stdcli.Error(err) @@ -82,9 +88,11 @@ func cmdRunDetached(c *cli.Context) { command = strings.Join(args, " ") } + release := c.String("release") + fmt.Printf("Running `%s` on %s... ", command, ps) - err = rackClient(c).RunProcessDetached(app, ps, command) + err = rackClient(c).RunProcessDetached(app, ps, command, release) if err != nil { stdcli.Error(err) @@ -94,7 +102,7 @@ func cmdRunDetached(c *cli.Context) { fmt.Println("OK") } -func runAttached(c *cli.Context, app string, ps string, args string) (int, error) { +func runAttached(c *cli.Context, app, ps, args, release string) (int, error) { fd := os.Stdin.Fd() if terminal.IsTerminal(int(fd)) { @@ -113,7 +121,7 @@ func runAttached(c *cli.Context, app string, ps string, args string) (int, error return -1, err } - code, err := rackClient(c).RunProcessAttached(app, ps, args, h, w, os.Stdin, os.Stdout) + code, err := rackClient(c).RunProcessAttached(app, ps, args, release, h, w, os.Stdin, os.Stdout) if err != nil { return -1, err diff --git a/cmd/convox/ssl.go b/cmd/convox/ssl.go index 104dfd6a56..89431732ec 100644 --- a/cmd/convox/ssl.go +++ b/cmd/convox/ssl.go @@ -28,7 +28,7 @@ func init() { { Name: "create", Description: "create a new SSL listener", - Usage: " ", + Usage: " [ |]", Action: cmdSSLCreate, Flags: []cli.Flag{ appFlag, @@ -58,7 +58,7 @@ func init() { { Name: "update", Description: "upload a replacement ssl certificate", - Usage: " ", + Usage: " [ |]", Action: cmdSSLUpdate, Flags: []cli.Flag{ appFlag, @@ -96,6 +96,7 @@ func cmdSSLCreate(c *cli.Context) { var pub []byte var key []byte + var arn string switch len(c.Args()) { case 1: @@ -130,6 +131,8 @@ func cmdSSLCreate(c *cli.Context) { stdcli.Usage(c, "create") return } + case 2: + arn = c.Args()[1] case 3: pub, err = ioutil.ReadFile(c.Args()[1]) @@ -164,7 +167,7 @@ func cmdSSLCreate(c *cli.Context) { fmt.Printf("Creating SSL listener %s... ", target) - _, err = rackClient(c).CreateSSL(app, parts[0], parts[1], string(pub), string(key), chain, c.Bool("secure")) + _, err = rackClient(c).CreateSSL(app, parts[0], parts[1], arn, string(pub), string(key), chain, c.Bool("secure")) if err != nil { stdcli.Error(err) @@ -256,18 +259,27 @@ func cmdSSLUpdate(c *cli.Context) { var pub []byte var key []byte + var arn string - pub, err = ioutil.ReadFile(c.Args()[1]) + switch len(c.Args()) { + case 2: + arn = c.Args()[1] + case 3: + pub, err = ioutil.ReadFile(c.Args()[1]) - if err != nil { - stdcli.Error(err) - return - } + if err != nil { + stdcli.Error(err) + return + } - key, err = ioutil.ReadFile(c.Args()[2]) + key, err = ioutil.ReadFile(c.Args()[2]) - if err != nil { - stdcli.Error(err) + if err != nil { + stdcli.Error(err) + return + } + default: + stdcli.Usage(c, "update") return } @@ -286,7 +298,7 @@ func cmdSSLUpdate(c *cli.Context) { fmt.Printf("Updating SSL listener %s... ", target) - _, err = rackClient(c).UpdateSSL(app, parts[0], parts[1], string(pub), string(key), chain) + _, err = rackClient(c).UpdateSSL(app, parts[0], parts[1], arn, string(pub), string(key), chain) if err != nil { stdcli.Error(err)