From c1e13c5717ec4934397412b76abedd9810263d43 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bal=C3=A1zs=20Grill?= Date: Sat, 21 Sep 2024 05:36:41 +0200 Subject: [PATCH] E2E test with minio --- .github/workflows/go.yml | 2 + bindings/bindings.go | 38 ++++++++-- cmd/main/mgr.go | 2 +- test/minio/minio_test.go | 150 +++++++++++++++++++++++++++++++++++++++ 4 files changed, 185 insertions(+), 7 deletions(-) create mode 100644 test/minio/minio_test.go diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index 66a8aae..6ee1f6d 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -52,6 +52,8 @@ jobs: - name: Get tools run: | go install github.com/akavel/rsrc@latest + go install github.com/minio/minio@latest + go install github.com/minio/mc@latest - name: Test run: go test -v ./... diff --git a/bindings/bindings.go b/bindings/bindings.go index 1d17811..077d8df 100644 --- a/bindings/bindings.go +++ b/bindings/bindings.go @@ -10,6 +10,7 @@ import ( "syscall" "time" + "github.com/google/uuid" "github.com/rs/zerolog" "github.com/balazsgrill/potatodrive/bindings/s3" @@ -28,9 +29,24 @@ type BindingConfig interface { ToFileSystem() (afero.Fs, error) } +const ( + APIType_CFAPI = "cfapi" + APIType_PRJFS = "prjfs" + APIType_CFAPI_Simplfied = "cfapi-simplified" +) + type BaseConfig struct { LocalPath string `flag:"localpath,Local folder" reg:"LocalPath"` Type string `flag:"type,Type of binding" reg:"Type"` + API string `flag:"api,Type of API to be used of" reg:"API"` +} + +func (config *BaseConfig) IsCFAPI() bool { + return config.API == "" || config.API == APIType_CFAPI || config.API == APIType_CFAPI_Simplfied +} + +func (config *BaseConfig) IsSimplfied() bool { + return config.API == APIType_CFAPI_Simplfied } func ConfigToFlags(config any) { @@ -134,17 +150,23 @@ func (context InstanceContext) ConnectionStateChanged(id string, syninprogress b context.StateCallback(state) } -func BindVirtualizationInstance(id string, localpath string, remotefs afero.Fs, context InstanceContext) (io.Closer, error) { +func BindVirtualizationInstance(id string, config *BaseConfig, remotefs afero.Fs, context InstanceContext) (io.Closer, error) { var closer core.Virtualization var err error - if UseCFAPI { - err = cfapi.RegisterRootPath(id, localpath) + if config.IsCFAPI() { + if config.IsSimplfied() { + uid := uuid.NewMD5(uuid.UUID{}, []byte(id)) + gid := core.BytesToGuid(uid[:]) + err = cfapi.RegisterRootPathSimple(*gid, config.LocalPath) + } else { + err = cfapi.RegisterRootPath(id, config.LocalPath) + } if err != nil { return nil, err } - closer, err = cfapi.StartProjecting(localpath, remotefs, context.Logger) + closer, err = cfapi.StartProjecting(config.LocalPath, remotefs, context.Logger) } else { - closer, err = prjfs.StartProjecting(localpath, remotefs, context.Logger) + closer, err = prjfs.StartProjecting(config.LocalPath, remotefs, context.Logger) } if err != nil { return nil, err @@ -174,6 +196,10 @@ func BindVirtualizationInstance(id string, localpath string, remotefs afero.Fs, return (closerFunc)(func() error { t.Stop() - return closer.Close() + err := closer.Close() + if config.IsCFAPI() && config.IsSimplfied() { + cfapi.UnregisterRootPathSimple(config.LocalPath) + } + return err }), nil } diff --git a/cmd/main/mgr.go b/cmd/main/mgr.go index 2858ed2..2c6c386 100644 --- a/cmd/main/mgr.go +++ b/cmd/main/mgr.go @@ -72,7 +72,7 @@ func startInstance(parentkey registry.Key, keyname string, context bindings.Inst context.Logger.Info().Msgf("Starting %s on %s", keyname, basec.LocalPath) innercontext := context innercontext.Logger = context.Logger.With().Str("instance", keyname).Logger() - c, err := bindings.BindVirtualizationInstance(keyname, basec.LocalPath, fs, innercontext) + c, err := bindings.BindVirtualizationInstance(keyname, &basec, fs, innercontext) if err != nil { return nil, err } diff --git a/test/minio/minio_test.go b/test/minio/minio_test.go new file mode 100644 index 0000000..a1bf43e --- /dev/null +++ b/test/minio/minio_test.go @@ -0,0 +1,150 @@ +package test + +import ( + "io" + "os" + "os/exec" + "path/filepath" + "testing" + "time" + + "github.com/balazsgrill/potatodrive/bindings" + "github.com/balazsgrill/potatodrive/bindings/s3" + "github.com/balazsgrill/potatodrive/core" + "github.com/balazsgrill/potatodrive/core/cfapi/filesystem" + "github.com/google/uuid" + "github.com/rs/zerolog" +) + +type Write func(p []byte) (n int, err error) + +var _ io.Writer = Write(nil) + +func (w Write) Write(p []byte) (n int, err error) { + return w(p) +} + +func stdlog(t *testing.T) Write { + return func(p []byte) (n int, err error) { + t.Log(string(p)) + return len(p), nil + } +} + +var MINIO_ACCESS_KEY = "minioadmin" +var MINIO_SECRET_KEY = "minioadmin" + +func runAsync(t *testing.T, name string, args ...string) *exec.Cmd { + cmd := exec.Command(name, args...) + cmd.Stdout = stdlog(t) + cmd.Stderr = stdlog(t) + err := cmd.Start() + if err != nil { + t.Fatal(err) + } + return cmd +} + +func runSync(t *testing.T, name string, args ...string) { + cmd := exec.Command(name, args...) + cmd.Stdout = stdlog(t) + cmd.Stderr = stdlog(t) + err := cmd.Run() + if err != nil { + t.Fatal(err) + } +} + +type testInstance struct { + fsdir string + tempdir string + miniop *exec.Cmd + virtualization core.Virtualization +} + +func setup(t *testing.T) *testInstance { + instance := &testInstance{} + instance.fsdir = t.TempDir() + instance.tempdir = t.TempDir() + minioworkdir := t.TempDir() + instance.miniop = runAsync(t, "minio", "server", minioworkdir) + time.Sleep(1 * time.Second) + runSync(t, "mc", "alias", "set", "testminio", "http://localhost:9000", MINIO_ACCESS_KEY, MINIO_SECRET_KEY) + runSync(t, "mc", "mb", "testminio/test") + return instance +} + +func (instance *testInstance) start(t *testing.T) { + config := s3.Config{ + Endpoint: "localhost:9000", + Region: "us-east-1", // default region + Bucket: "test", + KeyId: MINIO_ACCESS_KEY, + KeySecret: MINIO_SECRET_KEY, + UseSSL: false, + } + fs, err := config.ToFileSystem() + if err != nil { + t.Fatal(err) + } + instancecontext := bindings.InstanceContext{ + Logger: zerolog.New(zerolog.NewTestWriter(t)), + } + uid := uuid.NewMD5(uuid.UUID{}, []byte("test")) + gid := core.BytesToGuid(uid[:]) + err = filesystem.RegisterRootPathSimple(*gid, instance.fsdir) + if err != nil { + t.Fatal(err) + } + instance.virtualization, err = filesystem.StartProjecting(instance.fsdir, fs, instancecontext.Logger) + if err != nil { + t.Fatal(err) + } +} + +func (instance *testInstance) Close() { + if instance.virtualization != nil { + instance.virtualization.Close() + } + filesystem.UnregisterRootPathSimple(instance.fsdir) + instance.miniop.Process.Signal(os.Kill) + instance.miniop.Wait() +} + +func generateTestData(size int, seed string) []byte { + data := make([]byte, size) + for i := 0; i < size; i++ { + data[i] = byte(seed[i%len(seed)]) + } + return data +} + +func TestDownloadingLargeFile(t *testing.T) { + instance := setup(t) + defer instance.Close() + + // generate data and upload it + data := generateTestData(2*1024*1024, "2megabytesof2megabytes") + inputfile := filepath.Join(instance.tempdir, "inputfile.dat") + err := os.WriteFile(inputfile, data, 0644) + if err != nil { + t.Fatal(err) + } + runSync(t, "mc", "cp", inputfile, "testminio/test/root/inputfile.dat") + + instance.start(t) + + err = instance.virtualization.PerformSynchronization() + if err != nil { + t.Fatal(err) + } + + outputfile := filepath.Join(instance.fsdir, "inputfile.dat") + outputdata, err := os.ReadFile(outputfile) + if err != nil { + t.Fatal(err) + } + if string(outputdata) != string(data) { + t.Fatal("data mismatch") + } +}