working s3 integration

.github/workflows/go.yml

@@ -26,3 +26,10 @@ jobs:
 
     - name: Test
       run: go test -v ./...
+
+    - name: Build
+      run: go build -o potatos3.exe ./cmd/potatos3
+    - uses: actions/upload-artifact@v4
+      with:
+        name: potatos3.exe
+        path: potatos3.exe

cmd/potatos3/basepathfs.go

@@ -0,0 +1,255 @@
+package main
+
+import (
+	"io/fs"
+	"os"
+	"path"
+	"runtime"
+	"strings"
+	"time"
+
+	"github.com/spf13/afero"
+)
+
+var (
+	_ afero.Lstater  = (*BasePathFs)(nil)
+	_ fs.ReadDirFile = (*BasePathFile)(nil)
+)
+
+// The BasePathFs restricts all operations to a given path within an Fs.
+// The given file name to the operations on this Fs will be prepended with
+// the base path before calling the base Fs.
+// Any file name (after filepath.Clean()) outside this base path will be
+// treated as non existing file.
+//
+// Note that it does not clean the error messages on return, so you may
+// reveal the real path on errors.
+type BasePathFs struct {
+	source afero.Fs
+	path   string
+}
+
+type BasePathFile struct {
+	afero.File
+	path string
+}
+
+type readDirFile struct {
+	afero.File
+}
+
+var _ fs.ReadDirFile = readDirFile{}
+
+func (r readDirFile) ReadDir(n int) ([]fs.DirEntry, error) {
+	items, err := r.File.Readdir(n)
+	if err != nil {
+		return nil, err
+	}
+
+	ret := make([]fs.DirEntry, len(items))
+	for i := range items {
+		ret[i] = FileInfoDirEntry{FileInfo: items[i]}
+	}
+
+	return ret, nil
+}
+
+// FileInfoDirEntry provides an adapter from os.FileInfo to fs.DirEntry
+type FileInfoDirEntry struct {
+	fs.FileInfo
+}
+
+var _ fs.DirEntry = FileInfoDirEntry{}
+
+func (d FileInfoDirEntry) Type() fs.FileMode { return d.FileInfo.Mode().Type() }
+
+func (d FileInfoDirEntry) Info() (fs.FileInfo, error) { return d.FileInfo, nil }
+
+func (f *BasePathFile) Name() string {
+	sourcename := f.File.Name()
+	return strings.TrimPrefix(sourcename, path.Clean(f.path))
+}
+
+func (f *BasePathFile) ReadDir(n int) ([]fs.DirEntry, error) {
+	if rdf, ok := f.File.(fs.ReadDirFile); ok {
+		return rdf.ReadDir(n)
+	}
+	return readDirFile{f.File}.ReadDir(n)
+}
+
+func NewBasePathFs(source afero.Fs, path string) afero.Fs {
+	return &BasePathFs{source: source, path: path}
+}
+
+// on a file outside the base path it returns the given file name and an error,
+// else the given file with the base path prepended
+func (b *BasePathFs) RealPath(name string) (rpath string, err error) {
+	if err := validateBasePathName(name); err != nil {
+		return name, err
+	}
+
+	bpath := path.Clean(b.path)
+	rpath = path.Clean(path.Join(bpath, name))
+	if !strings.HasPrefix(rpath, bpath) {
+		return name, os.ErrNotExist
+	}
+
+	return rpath, nil
+}
+
+func validateBasePathName(name string) error {
+	if runtime.GOOS != "windows" {
+		// Not much to do here;
+		// the virtual file paths all look absolute on *nix.
+		return nil
+	}
+
+	// On Windows a common mistake would be to provide an absolute OS path
+	// We could strip out the base part, but that would not be very portable.
+	if path.IsAbs(name) {
+		return os.ErrNotExist
+	}
+
+	return nil
+}
+
+func (b *BasePathFs) Chtimes(name string, atime, mtime time.Time) (err error) {
+	if name, err = b.RealPath(name); err != nil {
+		return &os.PathError{Op: "chtimes", Path: name, Err: err}
+	}
+	return b.source.Chtimes(name, atime, mtime)
+}
+
+func (b *BasePathFs) Chmod(name string, mode os.FileMode) (err error) {
+	if name, err = b.RealPath(name); err != nil {
+		return &os.PathError{Op: "chmod", Path: name, Err: err}
+	}
+	return b.source.Chmod(name, mode)
+}
+
+func (b *BasePathFs) Chown(name string, uid, gid int) (err error) {
+	if name, err = b.RealPath(name); err != nil {
+		return &os.PathError{Op: "chown", Path: name, Err: err}
+	}
+	return b.source.Chown(name, uid, gid)
+}
+
+func (b *BasePathFs) Name() string {
+	return "BasePathFs"
+}
+
+func (b *BasePathFs) Stat(name string) (fi os.FileInfo, err error) {
+	if name, err = b.RealPath(name); err != nil {
+		return nil, &os.PathError{Op: "stat", Path: name, Err: err}
+	}
+	return b.source.Stat(name)
+}
+
+func (b *BasePathFs) Rename(oldname, newname string) (err error) {
+	if oldname, err = b.RealPath(oldname); err != nil {
+		return &os.PathError{Op: "rename", Path: oldname, Err: err}
+	}
+	if newname, err = b.RealPath(newname); err != nil {
+		return &os.PathError{Op: "rename", Path: newname, Err: err}
+	}
+	return b.source.Rename(oldname, newname)
+}
+
+func (b *BasePathFs) RemoveAll(name string) (err error) {
+	if name, err = b.RealPath(name); err != nil {
+		return &os.PathError{Op: "remove_all", Path: name, Err: err}
+	}
+	return b.source.RemoveAll(name)
+}
+
+func (b *BasePathFs) Remove(name string) (err error) {
+	if name, err = b.RealPath(name); err != nil {
+		return &os.PathError{Op: "remove", Path: name, Err: err}
+	}
+	return b.source.Remove(name)
+}
+
+func (b *BasePathFs) OpenFile(name string, flag int, mode os.FileMode) (f afero.File, err error) {
+	if name, err = b.RealPath(name); err != nil {
+		return nil, &os.PathError{Op: "openfile", Path: name, Err: err}
+	}
+	sourcef, err := b.source.OpenFile(name, flag, mode)
+	if err != nil {
+		return nil, err
+	}
+	return &BasePathFile{sourcef, b.path}, nil
+}
+
+func (b *BasePathFs) Open(name string) (f afero.File, err error) {
+	if name, err = b.RealPath(name); err != nil {
+		return nil, &os.PathError{Op: "open", Path: name, Err: err}
+	}
+	sourcef, err := b.source.Open(name)
+	if err != nil {
+		return nil, err
+	}
+	return &BasePathFile{File: sourcef, path: b.path}, nil
+}
+
+func (b *BasePathFs) Mkdir(name string, mode os.FileMode) (err error) {
+	if name, err = b.RealPath(name); err != nil {
+		return &os.PathError{Op: "mkdir", Path: name, Err: err}
+	}
+	return b.source.Mkdir(name, mode)
+}
+
+func (b *BasePathFs) MkdirAll(name string, mode os.FileMode) (err error) {
+	if name, err = b.RealPath(name); err != nil {
+		return &os.PathError{Op: "mkdir", Path: name, Err: err}
+	}
+	return b.source.MkdirAll(name, mode)
+}
+
+func (b *BasePathFs) Create(name string) (f afero.File, err error) {
+	if name, err = b.RealPath(name); err != nil {
+		return nil, &os.PathError{Op: "create", Path: name, Err: err}
+	}
+	sourcef, err := b.source.Create(name)
+	if err != nil {
+		return nil, err
+	}
+	return &BasePathFile{File: sourcef, path: b.path}, nil
+}
+
+func (b *BasePathFs) LstatIfPossible(name string) (os.FileInfo, bool, error) {
+	name, err := b.RealPath(name)
+	if err != nil {
+		return nil, false, &os.PathError{Op: "lstat", Path: name, Err: err}
+	}
+	if lstater, ok := b.source.(afero.Lstater); ok {
+		return lstater.LstatIfPossible(name)
+	}
+	fi, err := b.source.Stat(name)
+	return fi, false, err
+}
+
+func (b *BasePathFs) SymlinkIfPossible(oldname, newname string) error {
+	oldname, err := b.RealPath(oldname)
+	if err != nil {
+		return &os.LinkError{Op: "symlink", Old: oldname, New: newname, Err: err}
+	}
+	newname, err = b.RealPath(newname)
+	if err != nil {
+		return &os.LinkError{Op: "symlink", Old: oldname, New: newname, Err: err}
+	}
+	if linker, ok := b.source.(afero.Linker); ok {
+		return linker.SymlinkIfPossible(oldname, newname)
+	}
+	return &os.LinkError{Op: "symlink", Old: oldname, New: newname, Err: afero.ErrNoSymlink}
+}
+
+func (b *BasePathFs) ReadlinkIfPossible(name string) (string, error) {
+	name, err := b.RealPath(name)
+	if err != nil {
+		return "", &os.PathError{Op: "readlink", Path: name, Err: err}
+	}
+	if reader, ok := b.source.(afero.LinkReader); ok {
+		return reader.ReadlinkIfPossible(name)
+	}
+	return "", &os.PathError{Op: "readlink", Path: name, Err: afero.ErrNoReadlink}
+}

cmd/potatos3/main.go

@@ -0,0 +1,62 @@
+package main
+
+import (
+	"flag"
+	"log"
+	"os"
+	"os/signal"
+	"syscall"
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/credentials"
+	"github.com/aws/aws-sdk-go/aws/session"
+	"github.com/balazsgrill/projfero/filesystem"
+	s3 "github.com/fclairamb/afero-s3"
+)
+
+func main() {
+	endpoint := flag.String("endpoint", "", "S3 endpoint")
+	accessKeyID := flag.String("keyid", "", "Access Key ID")
+	secretAccessKey := flag.String("secred", "", "Access Key Secret")
+	useSSL := flag.Bool("useSSL", false, "Use SSL encryption for S3 connection")
+	region := flag.String("region", "", "Region")
+	bucket := flag.String("bucket", "", "Bucket")
+	localpath := flag.String("localpath", "", "Local folder")
+	flag.Parse()
+
+	sess, _ := session.NewSession(&aws.Config{
+		Region:           aws.String(*region),
+		Endpoint:         aws.String(*endpoint),
+		DisableSSL:       aws.Bool(!*useSSL),
+		S3ForcePathStyle: aws.Bool(true),
+		Credentials:      credentials.NewStaticCredentials(*accessKeyID, *secretAccessKey, ""),
+	})
+
+	// Initialize the file system
+	fs := s3.NewFs(*bucket, sess)
+	fs.MkdirAll("root", 0777)
+	rootfs := NewBasePathFs(fs, "root")
+
+	c := make(chan os.Signal, 1)
+	signal.Notify(c, os.Interrupt, syscall.SIGTERM)
+	closer, err := filesystem.StartProjecting(*localpath, rootfs)
+	if err != nil {
+		log.Panic(err)
+	}
+
+	t := time.NewTicker(30 * time.Second)
+	go func() {
+		for range t.C {
+			err = closer.PerformSynchronization()
+			if err != nil {
+				log.Panic(err)
+			}
+		}
+	}()
+
+	<-c
+	t.Stop()
+	closer.Close()
+	os.Exit(1)
+}

example/main.go

@@ -6,15 +6,15 @@ import (
 	"os/signal"
 	"syscall"
 
-	"github.com/balazsgrill/projfero"
+	"github.com/balazsgrill/projfero/filesystem"
 	"github.com/spf13/afero"
 )
 
 func main() {
 	fs := afero.NewBasePathFs(afero.NewOsFs(), "C:\\work\\vfsbase")
 	c := make(chan os.Signal, 1)
 	signal.Notify(c, os.Interrupt, syscall.SIGTERM)
-	closer, err := projfero.StartProjecting("C:\\work\\vfs", fs)
+	closer, err := filesystem.StartProjecting("C:\\work\\vfs", fs)
 	if err != nil {
 		log.Panic(err)
 	}