thrift-based proxy

.github/workflows/go.yml → .github/workflows/client.yml

@@ -54,6 +54,7 @@ jobs:
         go install github.com/akavel/rsrc@latest 
         go install github.com/minio/minio@latest
         go install github.com/minio/mc@latest
+        wget -o "${{ env.GOBIN}}\thrift.exe" https://dlcdn.apache.org/thrift/0.21.0/thrift-0.21.0.exe
 
     - name: Test
       run: go test -v ./...

.github/workflows/proxy.yml

@@ -0,0 +1,48 @@
+# This workflow will build a golang project
+# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-go
+
+name: Go
+
+on:
+  push:
+    branches: [ "main" ]
+    tags:
+      - '*'
+  pull_request:
+    branches: [ "main" ]
+
+jobs:
+
+  build:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+
+    - name: Get release info
+      id: release_info
+      if: github.ref_type == 'tag'
+      uses: revam/gh-action-get-tag-and-version@v1
+      with:
+        tag: ${{ github.ref }}
+        prefix: v
+        prefixRegex: "[vV]?"
+
+    - id: set_version
+      uses: marcdomain/[email protected]
+      name: Set version
+      with:
+        variables: |
+          VERSION: '${{ github.ref_type }}' == 'tag' ? "${{ steps.release_info.outputs.version }}" : "0.0.0.0"
+
+    - name: Set up Go
+      uses: actions/setup-go@v4
+      with:
+        go-version: '1.23'
+    - name: Generate
+      run: go generate ./...
+    - name: Build
+      run: go build -o potatoproxy -X 'main.Version=${{ env.VERSION }}'" ./cmd/proxy
+    - uses: actions/upload-artifact@v4
+      with:
+        name: potatoproxy
+        path: potatoproxy

bindings/proxy/.gitignore

@@ -0,0 +1,5 @@
+filesystem-remote
+file_service-remote
+GoUnusedProtection__.go
+proxy.go
+proxy-consts.go

bindings/proxy/client/client.go

@@ -0,0 +1,29 @@
+package client
+
+import (
+	"time"
+
+	"github.com/apache/thrift/lib/go/thrift"
+	"github.com/balazsgrill/potatodrive/bindings/proxy"
+	"github.com/spf13/afero"
+)
+
+func Connect(url string) (afero.Fs, error) {
+	conf := &thrift.TConfiguration{
+		ConnectTimeout: time.Second,
+		SocketTimeout:  time.Second,
+
+		MaxFrameSize: 1024 * 1024 * 256,
+
+		TBinaryStrictRead:  thrift.BoolPtr(true),
+		TBinaryStrictWrite: thrift.BoolPtr(true),
+	}
+	protocol := thrift.NewTCompactProtocolFactoryConf(conf)
+	clientfactory := thrift.NewTHttpClientTransportFactory(url)
+	transport, err := clientfactory.GetTransport(nil)
+	if err != nil {
+		return nil, err
+	}
+	client := proxy.NewFilesystemClientFactory(transport, protocol)
+	return New(client), nil
+}

bindings/proxy/client/file.go

@@ -0,0 +1,108 @@
+package client
+
+import (
+	"context"
+	"io/fs"
+
+	"github.com/balazsgrill/potatodrive/bindings/proxy"
+	"github.com/spf13/afero"
+)
+
+type file struct {
+	fs     *filesystemClient
+	handle proxy.FileHandle
+}
+
+// Close implements afero.File.
+func (f *file) Close() error {
+	return f.fs.client.Fclose(context.Background(), f.handle)
+}
+
+// Name implements afero.File.
+func (f *file) Name() string {
+	name, _ := f.fs.client.Fname(context.Background(), f.handle)
+	return name
+}
+
+// Read implements afero.File.
+func (f *file) Read(p []byte) (n int, err error) {
+	data, err := f.fs.client.Fread(context.Background(), f.handle, int64(len(p)))
+	if err != nil {
+		return 0, err
+	}
+	return copy(p, data), nil
+}
+
+// ReadAt implements afero.File.
+func (f *file) ReadAt(p []byte, off int64) (n int, err error) {
+	data, err := f.fs.client.FreadAt(context.Background(), f.handle, int64(len(p)), off)
+	if err != nil {
+		return 0, err
+	}
+	return copy(p, data), nil
+}
+
+// Readdir implements afero.File.
+func (f *file) Readdir(count int) ([]fs.FileInfo, error) {
+	dirs, err := f.fs.client.Freaddir(context.Background(), f.handle, int32(count))
+	if err != nil {
+		return nil, err
+	}
+
+	fileInfos := make([]fs.FileInfo, len(dirs))
+	for i, dir := range dirs {
+		fileInfos[i] = dir
+	}
+
+	return fileInfos, nil
+}
+
+// Readdirnames implements afero.File.
+func (f *file) Readdirnames(n int) ([]string, error) {
+	return f.fs.client.Freaddirnames(context.Background(), f.handle, int32(n))
+}
+
+// Seek implements afero.File.
+func (f *file) Seek(offset int64, whence int) (int64, error) {
+	return f.fs.client.Fseek(context.Background(), f.handle, offset, int32(whence))
+}
+
+// Stat implements afero.File.
+func (f *file) Stat() (fs.FileInfo, error) {
+	return f.fs.client.Fstat(context.Background(), f.handle)
+}
+
+// Sync implements afero.File.
+func (f *file) Sync() error {
+	return f.fs.client.Fsync(context.Background(), f.handle)
+}
+
+// Truncate implements afero.File.
+func (f *file) Truncate(size int64) error {
+	return f.fs.client.Ftruncate(context.Background(), f.handle, size)
+}
+
+// Write implements afero.File.
+func (f *file) Write(p []byte) (n int, err error) {
+	r, err := f.fs.client.Fwrite(context.Background(), f.handle, p)
+	return int(r), err
+}
+
+// WriteAt implements afero.File.
+func (f *file) WriteAt(p []byte, off int64) (n int, err error) {
+	r, err := f.fs.client.FwriteAt(context.Background(), f.handle, p, off)
+	return int(r), err
+}
+
+// WriteString implements afero.File.
+func (f *file) WriteString(s string) (ret int, err error) {
+	r, err := f.fs.client.FwriteString(context.Background(), f.handle, s)
+	return int(r), err
+}
+
+func toFile(fs *filesystemClient, handle proxy.FileHandle) afero.File {
+	return &file{
+		fs:     fs,
+		handle: handle,
+	}
+}

bindings/proxy/client/filesystem.go

@@ -0,0 +1,101 @@
+package client
+
+import (
+	"context"
+	"io/fs"
+	"time"
+
+	"github.com/balazsgrill/potatodrive/bindings/proxy"
+	"github.com/spf13/afero"
+)
+
+type filesystemClient struct {
+	client proxy.Filesystem
+}
+
+func New(fs proxy.Filesystem) afero.Fs {
+	return &filesystemClient{
+		client: fs,
+	}
+}
+
+// Chmod implements afero.Fs.
+func (f *filesystemClient) Chmod(name string, mode fs.FileMode) error {
+	return f.client.Chmod(context.Background(), name, proxy.FileMode(mode))
+}
+
+// Chown implements afero.Fs.
+func (f *filesystemClient) Chown(name string, uid int, gid int) error {
+	return f.client.Chown(context.Background(), name, int32(uid), int32(gid))
+}
+
+// Chtimes implements afero.Fs.
+func (f *filesystemClient) Chtimes(name string, atime time.Time, mtime time.Time) error {
+	return f.client.Chtimes(context.Background(), name, proxy.Timestamp(atime.UnixMicro()), proxy.Timestamp(mtime.UnixMicro()))
+}
+
+// Create implements afero.Fs.
+func (f *filesystemClient) Create(name string) (afero.File, error) {
+	h, err := f.client.Create(context.Background(), name)
+	if err != nil {
+		return nil, err
+	}
+	return toFile(f, h), nil
+}
+
+// Mkdir implements afero.Fs.
+func (f *filesystemClient) Mkdir(name string, perm fs.FileMode) error {
+	return f.client.Mkdir(context.Background(), name, proxy.FileMode(perm))
+}
+
+// MkdirAll implements afero.Fs.
+func (f *filesystemClient) MkdirAll(path string, perm fs.FileMode) error {
+	return f.client.MkdirAll(context.Background(), path, proxy.FileMode(perm))
+}
+
+// Name implements afero.Fs.
+func (f *filesystemClient) Name() string {
+	name, err := f.client.Name(context.Background())
+	if err != nil {
+		return ""
+	}
+	return name
+}
+
+// Open implements afero.Fs.
+func (f *filesystemClient) Open(name string) (afero.File, error) {
+	h, err := f.client.Open(context.Background(), name)
+	if err != nil {
+		return nil, err
+	}
+	return toFile(f, h), nil
+}
+
+// OpenFile implements afero.Fs.
+func (f *filesystemClient) OpenFile(name string, flag int, perm fs.FileMode) (afero.File, error) {
+	h, err := f.client.OpenFile(context.Background(), name, int32(flag), proxy.FileMode(perm))
+	if err != nil {
+		return nil, err
+	}
+	return toFile(f, h), nil
+}
+
+// Remove implements afero.Fs.
+func (f *filesystemClient) Remove(name string) error {
+	return f.client.Remove(context.Background(), name)
+}
+
+// RemoveAll implements afero.Fs.
+func (f *filesystemClient) RemoveAll(path string) error {
+	return f.client.RemoveAll(context.Background(), path)
+}
+
+// Rename implements afero.Fs.
+func (f *filesystemClient) Rename(oldname string, newname string) error {
+	return f.client.Rename(context.Background(), oldname, newname)
+}
+
+// Stat implements afero.Fs.
+func (f *filesystemClient) Stat(name string) (fs.FileInfo, error) {
+	return f.client.Stat(context.Background(), name)
+}

bindings/proxy/fileinfo.go

@@ -0,0 +1,29 @@
+package proxy
+
+import (
+	"io/fs"
+	"time"
+)
+
+func (f *FileInfo) Name() string {
+	return f.Fname
+}
+
+func (f *FileInfo) Size() int64 {
+	return f.Fsize
+}
+func (f *FileInfo) Mode() fs.FileMode {
+	return fs.FileMode(f.Fmode)
+}
+
+func (f *FileInfo) ModTime() time.Time {
+	return time.UnixMicro(int64(f.Ftime))
+}
+func (f *FileInfo) Sys() interface{} {
+	return nil
+}
+func (f *FileInfo) IsDir() bool {
+	return f.FisDir
+}
+
+var _ fs.FileInfo = (*FileInfo)(nil)

bindings/proxy/generate.go

@@ -0,0 +1,3 @@
+package proxy
+
+//go:generate thrift --gen go:package_prefix=github.com/balazsgrill/potatodrive/bindings/ --out .. proxy.thrift

bindings/proxy/proxy.thrift

@@ -0,0 +1,53 @@
+typedef i32 FileHandle
+typedef i32 FileMode
+typedef i64 Timestamp
+
+exception FilesystemException {
+    1: string message
+  }
+
+struct FileInfo {
+  1: string fname
+  2: i64 fsize
+  3: FileMode fmode
+  4: Timestamp ftime
+  5: bool fisDir
+}
+
+service Filesystem{
+    FileHandle create(1:string name) throws(1:FilesystemException error)
+    void mkdir(1:string path, 2:FileMode perm) throws(1:FilesystemException error)
+    void mkdirAll(1:string path, 2:FileMode perm) throws(1:FilesystemException error)
+    FileHandle open(1:string name) throws(1:FilesystemException error)
+    FileHandle openFile(1:string name, 2:i32 flag, 3:FileMode perm) throws(1:FilesystemException error)
+    void remove(1:string name) throws(1:FilesystemException error)
+    void removeAll(1:string name) throws(1:FilesystemException error)
+    void rename(1:string oldname, 2:string newname) throws(1:FilesystemException error)
+    FileInfo stat(1:string name) throws(1:FilesystemException error)
+    string name()
+    void chmod(1:string name, 2:FileMode mode) throws(1:FilesystemException error)
+    void chown(1:string name, 2:i32 uid, 3:i32 gid) throws(1:FilesystemException error)
+    void chtimes(1:string name, 2:Timestamp atime, 3:Timestamp mtime) throws(1:FilesystemException error)
+
+    // File operations
+    // Closer
+    void fclose(1:FileHandle file) throws(1:FilesystemException error)
+    // Reader
+    binary fread(1:FileHandle file, 2:i64 bufferSize) throws(1:FilesystemException error)
+    // ReaderAt
+    binary freadAt(1:FileHandle file, 2:i64 bufferSize, 3:i64 offset) throws(1:FilesystemException error)
+    // Seeker
+    i64 fseek(1:FileHandle file, 2:i64 offset, 3:i32 whence) throws(1:FilesystemException error)
+    // Writer
+    i32 fwrite(1:FileHandle file, 2:binary buffer) throws(1:FilesystemException error)
+    // WriterAt
+    i32 fwriteAt(1:FileHandle file, 2:binary buffer, 3:i64 offset) throws(1:FilesystemException error)
+
+    string fname(1:FileHandle file)
+    list<FileInfo> freaddir(1:FileHandle file, 2:i32 count) throws(1:FilesystemException error)
+    list<string> freaddirnames(1:FileHandle file, 2:i32 count) throws(1:FilesystemException error)
+    FileInfo fstat(1:FileHandle file) throws(1:FilesystemException error)
+    void fsync(1:FileHandle file) throws(1:FilesystemException error)
+    void ftruncate(1:FileHandle file, 2:i64 size) throws(1:FilesystemException error)
+    i32 fwriteString(1:FileHandle file, 2:string value) throws(1:FilesystemException error)
+}