Add Google Cloud Storage Support

.gitignore

@@ -1 +1,4 @@
 test_env
+.env
+go.mod
+go.sum

gcs/README.md

@@ -0,0 +1,38 @@
+# Google Cloud Storage
+
+[GCS](https://pkg.go.dev/cloud.google.com/go/storage) backend for [QOR OSS](https://github.com/qor/oss)
+
+## Usage
+
+> Set ENV `GOOGLE_APPLICATION_CREDENTIALS` to service account
+
+```go
+import "github.com/qor/oss/gcs"
+
+func main() {
+  storage := gcs.New(gcs.Config{
+    Bucket: "bucket",
+    Endpoint: "https://storage.googleapis.com/",
+  })
+
+  // Save a reader interface into storage
+  storage.Put("/sample.txt", reader)
+
+  // Get file with path
+  storage.Get("/sample.txt")
+
+  // Get object as io.ReadCloser
+  storage.GetStream("/sample.txt")
+
+  // Delete file with path
+  storage.Delete("/sample.txt")
+
+  // List all objects under path
+  storage.List("/")
+
+  // Get Public Accessible URL (useful if current file saved privately)
+  storage.GetURL("/sample.txt")
+}
+```
+
+

gcs/gcs.go

@@ -0,0 +1,179 @@
+package gcs
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"log"
+	"mime"
+	"net/http"
+	"net/url"
+	"os"
+	"path"
+	"path/filepath"
+	"regexp"
+	"strings"
+	"time"
+
+	"cloud.google.com/go/storage"
+	"github.com/qor/oss"
+	"google.golang.org/api/iterator"
+)
+
+// Client GCS storage
+type Client struct {
+	*storage.Client
+	Config *Config
+}
+
+// Config GCS client config
+type Config struct {
+	Bucket   string
+	Endpoint string
+}
+
+// New initialize GCS storage
+func New(config *Config) *Client {
+
+	client := &Client{Config: config}
+	ctx := context.Background()
+	gcsclient, err := storage.NewClient(ctx)
+	if err != nil {
+		log.Println(err)
+	}
+
+	client.Client = gcsclient
+
+	return client
+}
+
+// Get receive file with given path
+func (client Client) Get(path string) (file *os.File, err error) {
+	readCloser, err := client.GetStream(path)
+
+	ext := filepath.Ext(path)
+	pattern := fmt.Sprintf("gcs*%s", ext)
+
+	if err == nil {
+		if file, err = ioutil.TempFile("/tmp", pattern); err == nil {
+			defer readCloser.Close()
+			_, err = io.Copy(file, readCloser)
+			file.Seek(0, 0)
+		}
+	}
+
+	return file, err
+}
+
+// GetStream get file as stream
+func (client Client) GetStream(path string) (io.ReadCloser, error) {
+	bkt := client.Client.Bucket(client.Config.Bucket)
+	obj := bkt.Object(client.ToRelativePath(path))
+	r, err := obj.NewReader(context.TODO())
+	_, err = obj.Attrs(context.TODO())
+	return r, err
+}
+
+// Put store a reader into given path
+func (client Client) Put(urlPath string, reader io.Reader) (*oss.Object, error) {
+	if seeker, ok := reader.(io.ReadSeeker); ok {
+		seeker.Seek(0, 0)
+	}
+
+	urlPath = client.ToRelativePath(urlPath)
+	buffer, err := ioutil.ReadAll(reader)
+
+	fileType := mime.TypeByExtension(path.Ext(urlPath))
+	if fileType == "" {
+		fileType = http.DetectContentType(buffer)
+	}
+
+	bkt := client.Client.Bucket(client.Config.Bucket)
+	obj := bkt.Object(urlPath)
+	w := obj.NewWriter(context.TODO())
+	w.ContentType = fileType
+	w.Write(buffer)
+
+	if err := w.Close(); err != nil {
+		return nil, err
+	}
+
+	now := time.Now()
+	return &oss.Object{
+		Path:             urlPath,
+		Name:             filepath.Base(urlPath),
+		LastModified:     &now,
+		StorageInterface: client,
+	}, err
+}
+
+// Delete delete file
+func (client Client) Delete(path string) error {
+	path = strings.TrimPrefix(path, "/")
+	bkt := client.Client.Bucket(client.Config.Bucket)
+	obj := bkt.Object(client.ToRelativePath(path))
+	err := obj.Delete(context.TODO())
+	return err
+}
+
+// List list all objects under current path
+func (client Client) List(path string) ([]*oss.Object, error) {
+	var objects []*oss.Object
+	var prefix string
+
+	if path != "" {
+		prefix = strings.Trim(path, "/")
+	}
+
+	query := &storage.Query{Prefix: prefix}
+	bkt := client.Client.Bucket(client.Config.Bucket)
+	it := bkt.Objects(context.TODO(), query)
+	for {
+		attrs, err := it.Next()
+		if err == iterator.Done {
+			break
+		}
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		objects = append(objects, &oss.Object{
+			Path:             "/" + client.ToRelativePath(attrs.Name),
+			Name:             filepath.Base(attrs.Name),
+			LastModified:     &attrs.Created,
+			StorageInterface: client,
+		})
+	}
+
+	return objects, nil
+}
+
+// GetEndpoint get endpoint, FileSystem's endpoint is /
+func (client Client) GetEndpoint() string {
+	u, err := url.Parse(client.Config.Endpoint)
+	if err != nil {
+		log.Println(err)
+	}
+	u.Path = path.Join(u.Path, client.Config.Bucket)
+	return u.String()
+}
+
+var urlRegexp = regexp.MustCompile(`(https?:)?//((\w+).)+(\w+)/`)
+
+// ToRelativePath process path to relative path
+func (client Client) ToRelativePath(urlPath string) string {
+	if urlRegexp.MatchString(urlPath) {
+		if u, err := url.Parse(urlPath); err == nil {
+			urlPath = strings.TrimPrefix(u.Path, "/"+client.Config.Bucket+"/")
+			urlPath = strings.TrimPrefix(urlPath, "/")
+			return urlPath
+		}
+	}
+	return strings.TrimPrefix(urlPath, "/")
+}
+
+// GetURL get public accessible URL
+func (client Client) GetURL(path string) (string, error) {
+	return path, nil
+}

gcs/gcs_test.go

@@ -0,0 +1,50 @@
+package gcs_test
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/dilip640/oss/gcs"
+	"github.com/dilip640/oss/tests"
+	"github.com/jinzhu/configor"
+)
+
+type Config struct {
+	Bucket   string `env:"QOR_GCS_BUCKET"`
+	Endpoint string `env:"QOR_GCS_ENDPOINT"`
+}
+
+var (
+	client *gcs.Client
+	config = Config{}
+)
+
+func init() {
+	configor.Load(&config)
+
+	client = gcs.New(&gcs.Config{Bucket: config.Bucket, Endpoint: config.Endpoint})
+}
+
+func TestAll(t *testing.T) {
+	fmt.Println("testing GCS with public ACL")
+	tests.TestAll(client, t)
+
+	fmt.Println("testing GCS with private ACL")
+	privateClient := gcs.New(&gcs.Config{Bucket: config.Bucket, Endpoint: config.Endpoint})
+	tests.TestAll(privateClient, t)
+}
+
+func TestToRelativePath(t *testing.T) {
+	urlMap := map[string]string{
+		"https://storage.googleapis.com/pelto-test/myobject.ext": "myobject.ext",
+		"//storage.googleapis.com/pelto-test/myobject.ext":       "myobject.ext",
+		"gs://pelt-test/myobject.ext":                            "myobject.ext",
+		"myobject.ext":                                           "myobject.ext",
+	}
+
+	for url, path := range urlMap {
+		if client.ToRelativePath(url) != path {
+			t.Errorf("%v's relative path should be %v, but got %v", url, path, client.ToRelativePath(url))
+		}
+	}
+}

gcs/sample.txt

@@ -0,0 +1 @@
+sample