Merge pull request #76 from cpg1111/remote_config_load

config: Remote config load

Dockerfile_c

@@ -1,13 +1,13 @@
 FROM golang:1.7
 RUN apt-get -qq update && apt-get install -y cmake build-essential pkg-config openssl zlib1g curl wget && \
-    curl -L -o zlib-1.2.8.tar.gz -z zlib-1.2.8.tar.gz http://zlib.net/zlib-1.2.8.tar.gz && \
-    tar xzvf zlib-1.2.8.tar.gz && \
-    cd zlib-1.2.8 && \
+    curl -L -o zlib-1.2.11.tar.gz http://zlib.net/zlib-1.2.11.tar.gz && \
+    tar xzvf zlib-1.2.11.tar.gz && \
+    cd zlib-1.2.11 && \
     ./configure && \
     make && make install && \
     cd / && \
-    rm -rf zlib-1.2.8 && \
-    curl -L -o openssl-1.0.2h.tar.gz -z openssl-1.0.2h.tar.gz https://openssl.org/source/openssl-1.0.2h.tar.gz && \
+    rm -rf zlib-1.2.11 && \
+    curl -L -o openssl-1.0.2h.tar.gz https://openssl.org/source/openssl-1.0.2h.tar.gz && \
     tar xzvf openssl-1.0.2h.tar.gz && \
     cd openssl-1.0.2h && \
     ./config --prefix=/usr \

Makefile

@@ -32,12 +32,12 @@ all: build
 get-deps:
 	mkdir -p tmp && \
 	cd tmp && \
-	curl -o zlib-1.2.8.tar.gz -z zlib-1.2.8.tar.gz http://zlib.net/zlib-1.2.8.tar.gz && \
+	curl -o zlib-1.2.8.tar.gz http://zlib.net/zlib-1.2.8.tar.gz && \
     tar xzvf zlib-1.2.8.tar.gz && \
     cd zlib-1.2.8 && \
     ./configure && \
     make && make install && \
-    curl -o openssl-1.0.2h.tar.gz -z openssl-1.0.2h.tar.gz https://openssl.org/source/openssl-1.0.2h.tar.gz && \
+    curl -o openssl-1.0.2h.tar.gz https://openssl.org/source/openssl-1.0.2h.tar.gz && \
     tar xzvf openssl-1.0.2h.tar.gz && \
     cd openssl-1.0.2h && \
     ./config --prefix=/usr \

config/config.go

@@ -14,10 +14,19 @@ limitations under the License.
 package config
 
 import (
-	"io/ioutil"
+	"context"
+	"fmt"
+	"io"
+	"os"
 	"strings"
 
+	gs "cloud.google.com/go/storage"
 	"github.com/BurntSushi/toml"
+	"github.com/aws/aws-sdk-go/aws"
+	awscreds "github.com/aws/aws-sdk-go/aws/credentials"
+	awssession "github.com/aws/aws-sdk-go/aws/session"
+	"github.com/aws/aws-sdk-go/service/s3"
+	"google.golang.org/api/option"
 )
 
 // HealthCheck is a struct to check for a service's 'upness'
@@ -89,24 +98,108 @@ type Config struct {
 	CleanUp     CleanUp
 }
 
-func readConfig(path string) ([]byte, error) {
-	return ioutil.ReadFile(path)
+type remoteConfig struct {
+	Storage string
+	Bucket  string
+	Object  string
+}
+
+func parseRemote(path string) *remoteConfig {
+	storageIdx := strings.Index(path, "://")
+	pathSlice := strings.Split(path[storageIdx+1:], "/")
+	obj := pathSlice[1]
+	if len(pathSlice) > 2 {
+		for i := 2; i < len(pathSlice); i++ {
+			obj = fmt.Sprintf("%s/%s", obj, pathSlice[i])
+		}
+	}
+	return &remoteConfig{
+		Storage: path[0:storageIdx],
+		Bucket:  pathSlice[0],
+		Object:  obj,
+	}
+}
+
+func decode(r io.Reader) (*Config, error) {
+	var conf Config
+	if _, pErr := toml.DecodeReader(r, &conf); pErr != nil {
+		return &conf, pErr
+	}
+	return &conf, nil
+}
+
+func loadLocal(path string) (*Config, error) {
+	conf, err := os.OpenFile(path, os.O_RDONLY, 0644)
+	if err != nil {
+		return nil, err
+	}
+	return decode(conf)
+}
+
+func loadS3(path string) (*Config, error) {
+	remote := parseRemote(path)
+	creds := awscreds.NewEnvCredentials()
+	_, err := creds.Get()
+	if err != nil {
+		return nil, err
+	}
+	config := &aws.Config{
+		Region:           aws.String(os.Getenv("AWS_S3_REGION")),
+		Endpoint:         aws.String("s3.amazonaws.com"),
+		S3ForcePathStyle: aws.Bool(true),
+		Credentials:      creds,
+		LogLevel:         aws.LogLevel(aws.LogLevelType(0)),
+	}
+	session := awssession.New(config)
+	s3Client := s3.New(session)
+	query := &s3.GetObjectInput{
+		Bucket: aws.String(remote.Bucket),
+		Key:    aws.String(remote.Object),
+	}
+	resp, err := s3Client.GetObject(query)
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+	return decode(resp.Body)
+}
+
+func loadGStorage(path string) (*Config, error) {
+	remote := parseRemote(path)
+	opts := option.WithServiceAccountFile(os.Getenv("GCLOUD_SVC_ACCNT_FILE"))
+	ctx := context.Background()
+	gsClient, err := gs.NewClient(ctx, opts)
+	if err != nil {
+		return nil, err
+	}
+	bucket := gsClient.Bucket(remote.Bucket)
+	obj := bucket.Object(remote.Object)
+	rdr, err := obj.NewReader(ctx)
+	if err != nil {
+		return nil, err
+	}
+	defer rdr.Close()
+	return decode(rdr)
 }
 
 // Load reads the config and returns a Config struct
 func Load(path, clonePath string) (Config, error) {
-	var conf Config
-	confData, readErr := readConfig(path)
-	if readErr != nil {
-		return conf, readErr
+	var (
+		conf *Config
+		rErr error
+	)
+	if strings.Contains(path, "s3://") {
+		conf, rErr = loadS3(path)
+	} else {
+		conf, rErr = loadLocal(path)
 	}
-	if _, pErr := toml.Decode((string)(confData), &conf); pErr != nil {
-		return conf, pErr
+	if rErr != nil {
+		return *conf, rErr
 	}
 	for i := range conf.Services {
 		if strings.Contains(conf.Services[i].Path, ".") {
 			conf.Services[i].Path = strings.Replace(conf.Services[i].Path, ".", clonePath, 1)
 		}
 	}
-	return conf, nil
+	return *conf, nil
 }

glide.yaml

@@ -1,12 +1,39 @@
 package: github.com/cpg1111/maestro
 import:
 - package: github.com/BurntSushi/toml
-  version: ^0.2.0
+  version: 0.2.0
 - package: github.com/fatih/structs
 - package: github.com/segmentio/go-prompt
-  version: ^1.2.0
+  version: f3218e418a3d6dbe1dcab5809d80fa6e15c05cb4
 - package: gopkg.in/cheggaaa/pb.v1
-  version: ^1.0.5
+  version: 1.0.5
 - package: gopkg.in/libgit2/git2go.v24
+- package: github.com/aws/aws-sdk-go
+  version: 1.6.13
+  subpackages:
+  - aws
+- package: cloud.google.com/go
+  version: ^0.6.0
+  subpackages:
+  - storage
+  - compute/metadata
+- package: google.golang.org/api
+  version: b9d03e6e091e36f9a089515bc84cf14c2d199c4c
+  subpackages:
+  - option
+  - iterator
+  - transport
+- package: golang.org/x/oauth2
+  version: 314dd2c0bf3ebd592ec0d20847d27e79d0dbe8dd
+  subpackages:
+  - google
+- package: github.com/googleapis/gax-go
+- package: google.golang.org/cloud
+  version: ^0.6.0
+  subpackages:
+  - compute/metadata
+- package: golang.org/x/net
+  subpackages:
+  - context
 testImport:
 - package: github.com/fatih/structs

pipeline/clone.go

@@ -123,6 +123,7 @@ func New(conf *config.Config, creds *credentials.RawCredentials, clonePath, bran
 func (p *Project) Clone() (resRepo *git.Repository, resErr error) {
 	log.Println("Cloning Repo...")
 	resRepo, resErr = git.Clone(p.conf.RepoURL, fmt.Sprintf("%s/", p.clonePath), p.CloneOpts)
+	os.Chdir(p.clonePath)
 	return
 }