From 53d8e8fdef21befacf0189c08b399d3407f3afd3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bal=C3=A1zs=20Grill?= Date: Sat, 10 Aug 2024 06:46:13 +0200 Subject: [PATCH] FS implementation based on Cloud File API --- .github/workflows/go.yml | 2 +- bindings/bindings.go | 14 +- win/cfapi/callbacks.go | 106 ++++ win/cfapi/cfapi_test.go | 16 + win/cfapi/filesystem/fetchdata.go | 69 +++ win/cfapi/filesystem/filesystem.go | 312 ++++++++++ win/cfapi/filesystem/filesystem_test.go | 670 ++++++++++++++++++++++ win/cfapi/filesystem/fswatch.go | 22 + win/cfapi/filesystem/pathutils.go | 70 +++ win/cfapi/filesystem/placeholders.go | 76 +++ win/cfapi/filesystem/synclocaltoremote.go | 97 ++++ win/cfapi/filesystem/syncremotetolocal.go | 102 ++++ win/cfapi/functions.go | 132 ++++- win/cfapi/types.go | 195 ++++++- win/errorcode.go | 20 + win/projfs/filesystem/filesystem.go | 36 +- win/projfs/filesystem/filesystem_test.go | 3 +- win/projfs/functions.go | 9 - win/virtualization.go | 21 + 19 files changed, 1922 insertions(+), 50 deletions(-) create mode 100644 win/cfapi/callbacks.go create mode 100644 win/cfapi/cfapi_test.go create mode 100644 win/cfapi/filesystem/fetchdata.go create mode 100644 win/cfapi/filesystem/filesystem.go create mode 100644 win/cfapi/filesystem/filesystem_test.go create mode 100644 win/cfapi/filesystem/fswatch.go create mode 100644 win/cfapi/filesystem/pathutils.go create mode 100644 win/cfapi/filesystem/placeholders.go create mode 100644 win/cfapi/filesystem/synclocaltoremote.go create mode 100644 win/cfapi/filesystem/syncremotetolocal.go create mode 100644 win/errorcode.go create mode 100644 win/virtualization.go diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index face849..c4eb378 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -25,7 +25,7 @@ jobs: go-version: '1.21' - name: Test - run: go test -v ./... + run: go test -v ./.. - name: Build run: go build -o potatodrive.exe ./cmd/main diff --git a/bindings/bindings.go b/bindings/bindings.go index e0f29d7..5b0c3cc 100644 --- a/bindings/bindings.go +++ b/bindings/bindings.go @@ -11,11 +11,15 @@ import ( "syscall" "time" - "github.com/balazsgrill/potatodrive/win/projfs/filesystem" + "github.com/balazsgrill/potatodrive/win" + cfapi "github.com/balazsgrill/potatodrive/win/cfapi/filesystem" + prjfs "github.com/balazsgrill/potatodrive/win/projfs/filesystem" "github.com/spf13/afero" "golang.org/x/sys/windows/registry" ) +const UseCFAPI bool = true + func ConfigToFlags(config any) { structPtrValue := reflect.ValueOf(config) structValue := structPtrValue.Elem() @@ -90,7 +94,13 @@ func (f closerFunc) Close() error { } func BindVirtualizationInstance(localpath string, remotefs afero.Fs) (io.Closer, error) { - closer, err := filesystem.StartProjecting(localpath, remotefs) + var closer win.Virtualization + var err error + if UseCFAPI { + closer, err = cfapi.StartProjecting(localpath, remotefs) + } else { + closer, err = prjfs.StartProjecting(localpath, remotefs) + } if err != nil { return nil, err } diff --git a/win/cfapi/callbacks.go b/win/cfapi/callbacks.go new file mode 100644 index 0000000..8f94511 --- /dev/null +++ b/win/cfapi/callbacks.go @@ -0,0 +1,106 @@ +package cfapi + +import "syscall" +import "C" + +type Callback_FetchData func(*CF_CALLBACK_INFO, *CF_CALLBACK_PARAMETERS_FetchData) uintptr +type Callback_ValidateData func(*CF_CALLBACK_INFO, *CF_CALLBACK_PARAMETERS_ValidateData) uintptr +type Callback_CancelFetchData func(*CF_CALLBACK_INFO, *CF_CALLBACK_PARAMETERS_Cancel) uintptr +type Callback_FetchPlaceholders func(*CF_CALLBACK_INFO, *CF_CALLBACK_PARAMETERS_FetchPlaceholders) uintptr +type Callback_CancelFetchPlaceholders func(*CF_CALLBACK_INFO, *CF_CALLBACK_PARAMETERS_Cancel) uintptr +type Callback_OpenCompletion func(*CF_CALLBACK_INFO, *CF_CALLBACK_PARAMETERS_OpenCompletion) uintptr +type Callback_CloseCompletion func(*CF_CALLBACK_INFO, *CF_CALLBACK_PARAMETERS_CloseCompletion) uintptr +type Callback_Dehydrate func(*CF_CALLBACK_INFO, *CF_CALLBACK_PARAMETERS_Dehydrate) uintptr +type Callback_DehydrateCompletion func(*CF_CALLBACK_INFO, *CF_CALLBACK_PARAMETERS_DehydrateCompletion) uintptr +type Callback_Delete func(*CF_CALLBACK_INFO, *CF_CALLBACK_PARAMETERS_Delete) uintptr +type Callback_DeleteCompletion func(*CF_CALLBACK_INFO, *CF_CALLBACK_PARAMETERS_DeleteCompletion) uintptr +type Callback_Rename func(*CF_CALLBACK_INFO, *CF_CALLBACK_PARAMETERS_Rename) uintptr +type Callback_RenameCompletion func(*CF_CALLBACK_INFO, *CF_CALLBACK_PARAMETERS_RenameCompletion) uintptr + +type Callbacks struct { + FetchData Callback_FetchData + ValidateData Callback_ValidateData + CancelFetchData Callback_CancelFetchData + FetchPlaceholders Callback_FetchPlaceholders + CancelFetchPlaceholders Callback_CancelFetchPlaceholders + OpenCompletion Callback_OpenCompletion + CloseCompletion Callback_CloseCompletion + Dehydrate Callback_Dehydrate + DehydrateCompletion Callback_DehydrateCompletion + Delete Callback_Delete + DeleteCompletion Callback_DeleteCompletion + Rename Callback_Rename + RenameCompletion Callback_RenameCompletion +} + +func (cb *Callbacks) CreateCallbackTable() []CF_CALLBACK_REGISTRATION { + result := make([]CF_CALLBACK_REGISTRATION, 14) + count := 0 + if cb.FetchData != nil { + result[count].Callback = syscall.NewCallback(cb.FetchData) + result[count].Type = CF_CALLBACK_TYPE_FETCH_DATA + count++ + } + if cb.ValidateData != nil { + result[count].Callback = syscall.NewCallback(cb.ValidateData) + result[count].Type = CF_CALLBACK_TYPE_VALIDATE_DATA + count++ + } + if cb.CancelFetchData != nil { + result[count].Callback = syscall.NewCallback(cb.CancelFetchData) + result[count].Type = CF_CALLBACK_TYPE_CANCEL_FETCH_DATA + count++ + } + if cb.FetchPlaceholders != nil { + result[count].Callback = syscall.NewCallback(cb.FetchPlaceholders) + result[count].Type = CF_CALLBACK_TYPE_FETCH_PLACEHOLDERS + count++ + } + if cb.CancelFetchPlaceholders != nil { + result[count].Callback = syscall.NewCallback(cb.CancelFetchPlaceholders) + result[count].Type = CF_CALLBACK_TYPE_CANCEL_FETCH_PLACEHOLDERS + count++ + } + if cb.OpenCompletion != nil { + result[count].Callback = syscall.NewCallback(cb.OpenCompletion) + result[count].Type = CF_CALLBACK_TYPE_NOTIFY_FILE_OPEN_COMPLETION + count++ + } + if cb.CloseCompletion != nil { + result[count].Callback = syscall.NewCallback(cb.CloseCompletion) + result[count].Type = CF_CALLBACK_TYPE_NOTIFY_FILE_CLOSE_COMPLETION + count++ + } + if cb.Dehydrate != nil { + result[count].Callback = syscall.NewCallback(cb.Dehydrate) + result[count].Type = CF_CALLBACK_TYPE_NOTIFY_DEHYDRATE + count++ + } + if cb.DehydrateCompletion != nil { + result[count].Callback = syscall.NewCallback(cb.DehydrateCompletion) + result[count].Type = CF_CALLBACK_TYPE_NOTIFY_DEHYDRATE_COMPLETION + count++ + } + if cb.Delete != nil { + result[count].Callback = syscall.NewCallback(cb.Delete) + result[count].Type = CF_CALLBACK_TYPE_NOTIFY_DELETE + count++ + } + if cb.DeleteCompletion != nil { + result[count].Callback = syscall.NewCallback(cb.DeleteCompletion) + result[count].Type = CF_CALLBACK_TYPE_NOTIFY_DELETE_COMPLETION + count++ + } + if cb.Rename != nil { + result[count].Callback = syscall.NewCallback(cb.Rename) + result[count].Type = CF_CALLBACK_TYPE_NOTIFY_RENAME + count++ + } + if cb.RenameCompletion != nil { + result[count].Callback = syscall.NewCallback(cb.RenameCompletion) + result[count].Type = CF_CALLBACK_TYPE_NOTIFY_RENAME_COMPLETION + count++ + } + result[count].Type = CF_CALLBACK_TYPE_NONE + return result +} diff --git a/win/cfapi/cfapi_test.go b/win/cfapi/cfapi_test.go new file mode 100644 index 0000000..3ffd4e6 --- /dev/null +++ b/win/cfapi/cfapi_test.go @@ -0,0 +1,16 @@ +package cfapi_test + +import ( + "testing" + + "github.com/balazsgrill/potatodrive/win/cfapi" +) + +func Test_PlatformVersion(t *testing.T) { + var platforminfo cfapi.CF_PLATFORM_INFO + hr := cfapi.CfGetPlatformInfo(&platforminfo) + if hr != 0 { + t.Fatal(hr) + } + t.Logf("platform version: b%d i%d, r%d", platforminfo.BuildNumber, platforminfo.IntegrationNumber, platforminfo.RevisionNumber) +} diff --git a/win/cfapi/filesystem/fetchdata.go b/win/cfapi/filesystem/fetchdata.go new file mode 100644 index 0000000..b4bb0a9 --- /dev/null +++ b/win/cfapi/filesystem/fetchdata.go @@ -0,0 +1,69 @@ +package filesystem + +import ( + "io" + "log" + "syscall" + "unsafe" + + "github.com/balazsgrill/potatodrive/win" + "github.com/balazsgrill/potatodrive/win/cfapi" +) + +func (instance *VirtualizationInstance) callback_getRemoteFilePath(info *cfapi.CF_CALLBACK_INFO) string { + return instance.path_localToRemote(win.GetString(info.VolumeDosName) + win.GetString(info.NormalizedPath)) +} + +func (instance *VirtualizationInstance) fetchData(info *cfapi.CF_CALLBACK_INFO, data *cfapi.CF_CALLBACK_PARAMETERS_FetchData) uintptr { + instance.lock.Lock() + defer instance.lock.Unlock() + filename := instance.callback_getRemoteFilePath(info) + length := data.RequiredLength + byteOffset := data.RequiredFileOffset + if length == 0 || length < 0 { + length = info.FileSize + } + if data.OptionalLength > data.RequiredLength { + length = data.OptionalLength + byteOffset = data.OptionalFileOffset + } + log.Printf("Fetch data: %s %d bytes at %d", filename, length, byteOffset) + log.Printf("Optional %d at %d", data.OptionalLength, data.OptionalFileOffset) + file, err := instance.fs.Open(filename) + if err != nil { + log.Printf("Error opening file %s: %s", filename, err) + return uintptr(syscall.EIO) + } + defer file.Close() + buffer := make([]byte, length) + + var n int + var count int64 + for count < length { + n, err = file.ReadAt(buffer[count:], byteOffset+count) + count += int64(n) + if err == io.EOF { + err = nil + break + } + } + + log.Printf("Read %d bytes", count) + if err != nil { + log.Printf("Error reading file %s: %s", filename, err) + return uintptr(syscall.EIO) + } + + var transfer cfapi.CF_OPERATION_PARAMETERS_TransferData + transfer.Buffer = uintptr(unsafe.Pointer(&buffer[0])) + transfer.Length = count + transfer.Offset = byteOffset + transfer.ParamSize = uint32(unsafe.Sizeof(transfer)) + transfer.Flags = cfapi.CF_OPERATION_TRANSFER_DATA_FLAG_NONE + hr := instance.transferData(info, &transfer) + if hr != 0 { + log.Printf("Error transferring data: %s", win.ErrorByCode(hr)) + return hr + } + return 0 +} diff --git a/win/cfapi/filesystem/filesystem.go b/win/cfapi/filesystem/filesystem.go new file mode 100644 index 0000000..2a651b2 --- /dev/null +++ b/win/cfapi/filesystem/filesystem.go @@ -0,0 +1,312 @@ +package filesystem + +import ( + "crypto/md5" + "errors" + "io" + "io/fs" + "log" + "os" + "path/filepath" + "strings" + "sync" + "syscall" + "unsafe" + + "github.com/balazsgrill/potatodrive/win" + "github.com/balazsgrill/potatodrive/win/cfapi" + "github.com/fsnotify/fsnotify" + "github.com/spf13/afero" + "golang.org/x/sys/windows" +) + +type VirtualizationInstance struct { + rootPath string + shortprefix string + longprefix string + fs afero.Fs + + connectionKey cfapi.CF_CONNECTION_KEY + lock sync.Mutex + watcher *fsnotify.Watcher +} + +func StartProjecting(rootPath string, filesystem afero.Fs) (win.Virtualization, error) { + instance := &VirtualizationInstance{ + rootPath: rootPath, + fs: filesystem, + } + + instance.longprefix = toLongPath(rootPath) + instance.shortprefix = toShortPath(rootPath) + + return instance, instance.start() +} + +func (instance *VirtualizationInstance) start() error { + var registration cfapi.CF_SYNC_REGISTRATION + registration.ProviderName = win.GetPointer("PotatoDrive") + registration.ProviderVersion = win.GetPointer("0.1") + registration.StructSize = uint32(unsafe.Sizeof(registration)) + var policies cfapi.CF_SYNC_POLICIES + policies.StructSize = uint32(unsafe.Sizeof(policies)) + policies.Hydration.Primary = cfapi.CF_HYDRATION_POLICY_FULL + policies.Hydration.Modifier = cfapi.CF_HYDRATION_POLICY_MODIFIER_AUTO_DEHYDRATION_ALLOWED + policies.Population.Primary = cfapi.CF_POPULATION_POLICY_ALWAYS_FULL + policies.InSync = cfapi.CF_INSYNC_POLICY_TRACK_ALL + policies.HardLink = cfapi.CF_HARDLINK_POLICY_NONE + policies.PlaceholderManagement = cfapi.CF_PLACEHOLDER_MANAGEMENT_POLICY_DEFAULT + log.Println("Registering sync root") + hr := cfapi.CfRegisterSyncRoot(win.GetPointer(instance.rootPath), ®istration, &policies, cfapi.CF_REGISTER_FLAG_NONE) + if hr != 0 { + return win.ErrorByCode(hr) + } + + callbacks := &cfapi.Callbacks{ + FetchData: instance.fetchData, + //FetchPlaceholders: instance.fetchPlaceholders, + //DeleteCompletion: instance.deleteCompletion, + } + + log.Println("Connecting sync root") + hr = cfapi.CfConnectSyncRoot(win.GetPointer(instance.rootPath), callbacks.CreateCallbackTable(), uintptr(unsafe.Pointer(instance)), cfapi.CF_CONNECT_FLAG_REQUIRE_FULL_FILE_PATH, &instance.connectionKey) + + err := win.ErrorByCode(hr) + if err != nil { + return err + } + + instance.watcher, err = fsnotify.NewWatcher() + if err != nil { + return err + } + instance.watcher.Add(instance.rootPath) + go instance.watch() + + err = instance.PerformSynchronization() + if err != nil { + log.Printf("Initial synchronization failed %v", err) + } + return nil +} + +func (instance *VirtualizationInstance) readRemoteHash(remotepath string) ([]byte, error) { + hashpath := instance.path_hashFile(remotepath) + exists, err := afero.Exists(instance.fs, hashpath) + if err != nil { + return nil, err + } + if !exists { + return nil, nil + } + return afero.ReadFile(instance.fs, hashpath) +} + +func getFileNameFromIdentity(info *cfapi.CF_CALLBACK_INFO) string { + name := unsafe.Slice((*byte)(unsafe.Pointer(info.FileIdentity)), info.FileIdentityLength) + return string(name) +} + +func getPlaceholder(f fs.FileInfo) cfapi.CF_PLACEHOLDER_CREATE_INFO { + var placeholder cfapi.CF_PLACEHOLDER_CREATE_INFO + filename := f.Name() + log.Println(filename) + placeholder.RelativeFileName = win.GetPointer(filename) + placeholder.FsMetadata.BasicInfo = toBasicInfo(f) + identity := []byte(filename) + placeholder.FileIdentity = uintptr(unsafe.Pointer(&identity[0])) + log.Printf("Identity address %x", placeholder.FileIdentity) + placeholder.FileIdentityLength = uint32(len(identity)) + if !f.IsDir() { + placeholder.FsMetadata.FileSize = int64(f.Size()) + placeholder.Flags = cfapi.CF_PLACEHOLDER_CREATE_FLAG_DISABLE_ON_DEMAND_POPULATION | cfapi.CF_PLACEHOLDER_CREATE_FLAG_MARK_IN_SYNC + } else { + placeholder.FsMetadata.FileSize = 0 + } + return placeholder +} + +func toBasicInfo(file fs.FileInfo) cfapi.FILE_BASIC_INFO { + ftime := syscall.NsecToFiletime(file.ModTime().UnixNano()) + var attributes int32 + if file.IsDir() { + attributes |= syscall.FILE_ATTRIBUTE_DIRECTORY + } else { + attributes |= syscall.FILE_ATTRIBUTE_NORMAL + } + return cfapi.FILE_BASIC_INFO{ + CreationTime: ftime, + LastAccessTime: ftime, + LastWriteTime: ftime, + ChangeTime: ftime, + FileAttributes: attributes, + } +} + +func (instance *VirtualizationInstance) getOperationInfo(info *cfapi.CF_CALLBACK_INFO) cfapi.CF_OPERATION_INFO { + operation := cfapi.CF_OPERATION_INFO{} + operation.StructSize = uint32(unsafe.Sizeof(operation)) + operation.ConnectionKey = instance.connectionKey + operation.TransferKey = info.TransferKey + operation.CorrelationVector = info.CorrelationVector + operation.RequestKey = info.RequestKey + return operation +} + +func (instance *VirtualizationInstance) transferPlaceholders(info *cfapi.CF_CALLBACK_INFO, parameters *cfapi.CF_OPERATION_PARAMETERS_TransferPlaceholders) uintptr { + operation := instance.getOperationInfo(info) + operation.Type = cfapi.CF_OPERATION_TYPE_TRANSFER_PLACEHOLDERS + return cfapi.CfExecute(&operation, uintptr(unsafe.Pointer(parameters))) +} + +func (instance *VirtualizationInstance) transferData(info *cfapi.CF_CALLBACK_INFO, parameters *cfapi.CF_OPERATION_PARAMETERS_TransferData) uintptr { + operation := instance.getOperationInfo(info) + operation.Type = cfapi.CF_OPERATION_TYPE_TRANSFER_DATA + return cfapi.CfExecute(&operation, uintptr(unsafe.Pointer(parameters))) +} + +func (instance *VirtualizationInstance) Close() error { + if instance.connectionKey == 0 { + return errors.New("not started") + } + + instance.watcher.Close() + hr := cfapi.CfDisconnectSyncRoot(instance.connectionKey) + if hr != 0 { + return win.ErrorByCode(hr) + } + + hr = cfapi.CfUnregisterSyncRoot(win.GetPointer(instance.rootPath)) + return win.ErrorByCode(hr) +} + +func (instance *VirtualizationInstance) PerformSynchronization() error { + err := instance.syncRemoteToLocal() + if err != nil { + return err + } + return instance.syncLocalToRemote() +} + +func (instance *VirtualizationInstance) streamLocalToRemote(filename string) error { + file, err := os.Open(instance.path_remoteToLocal(filename)) + if err != nil { + return err + } + defer file.Close() + data := make([]byte, 1024*1024) + targetfile, err := instance.fs.OpenFile(filename, os.O_WRONLY|os.O_CREATE, 0x666) + if err != nil { + return err + } + defer targetfile.Close() + + hash := md5.New() + for { + n, err := file.Read(data) + if err != nil { + if err == io.EOF { + break + } + return err + } + _, err = hash.Write(data[:n]) + if err != nil { + return err + } + _, err = targetfile.Write(data[:n]) + if err != nil { + return err + } + } + + return afero.WriteFile(instance.fs, instance.path_hashFile(filename), hash.Sum(nil), 0666) +} + +func (instance *VirtualizationInstance) localHash(remotepath string) ([]byte, error) { + localpath := instance.path_remoteToLocal(remotepath) + // only calculate hash if file is available on local disk + localstate, err := getPlaceholderState(localpath) + if err != nil { + return nil, err + } + if (localstate | (cfapi.CF_PLACEHOLDER_STATE_IN_SYNC)) == 0 { + return nil, nil + } + hash := md5.New() + f, err := os.Open(localpath) + if err != nil { + return nil, err + } + defer f.Close() + _, err = io.Copy(hash, f) + if err != nil { + return nil, err + } + return hash.Sum(nil), nil +} + +func getPlaceholderInfo(localpath string) (*cfapi.CF_PLACEHOLDER_BASIC_INFO, error) { + localpathstr, err := syscall.UTF16PtrFromString(localpath) + fileHandle, err := syscall.CreateFile(localpathstr, syscall.GENERIC_READ, syscall.FILE_SHARE_READ, nil, syscall.OPEN_EXISTING, // existing file only + syscall.FILE_ATTRIBUTE_NORMAL|syscall.FILE_FLAG_OVERLAPPED, + 0) + if err != nil { + return nil, err + } + defer syscall.CloseHandle(fileHandle) + var placeholderInfo cfapi.CF_PLACEHOLDER_BASIC_INFO + var ReturnedLength uint32 + hr := cfapi.CfGetPlaceholderInfo(fileHandle, cfapi.CF_PLACEHOLDER_INFO_BASIC, uintptr(unsafe.Pointer(&placeholderInfo)), uint32(unsafe.Sizeof(placeholderInfo)), &ReturnedLength) + return &placeholderInfo, win.ErrorByCode(hr) +} + +const FileAttributeTagInfo uint32 = 9 + +func getPlaceholderState(localpath string) (cfapi.CF_PLACEHOLDER_STATE, error) { + localpathstr, err := syscall.UTF16PtrFromString(localpath) + if err != nil { + return cfapi.CF_PLACEHOLDER_STATE_INVALID, err + } + var finddata windows.Win32finddata + findhandle, err := windows.FindFirstFile(localpathstr, &finddata) + if err != nil { + return cfapi.CF_PLACEHOLDER_STATE_INVALID, err + } + defer windows.FindClose(findhandle) + + result := cfapi.CfGetPlaceholderStateFromFindData(uintptr(unsafe.Pointer(&finddata))) + return cfapi.CF_PLACEHOLDER_STATE(result), nil +} + +func (instance *VirtualizationInstance) handleDeletion(localpath string) { + instance.lock.Lock() + defer instance.lock.Unlock() + parentpath := filepath.Dir(localpath) + remoteparent := instance.path_localToRemote(parentpath) + remotepath := remoteparent + "/" + filepath.Base(localpath) + remotepath = strings.TrimPrefix(remotepath, "/") + err := instance.fs.Remove(remotepath) + if err != nil { + log.Printf("deleteCompletion: remove %s failed: %v", remotepath, err) + } +} + +func (instance *VirtualizationInstance) deleteCompletion(info *cfapi.CF_CALLBACK_INFO, data *cfapi.CF_CALLBACK_PARAMETERS_DeleteCompletion) uintptr { + instance.lock.Lock() + defer instance.lock.Unlock() + filename := instance.callback_getRemoteFilePath(info) + log.Printf("deleteCompletion: %s", filename) + //hashfilename := instance.path_hashFile(filename) + + err := instance.fs.Remove(filename) + if err != nil { + log.Printf("deleteCompletion: remove %s failed: %v", filename, err) + } + /* + err = instance.fs.Remove(hashfilename) + if err != nil { + log.Printf("deleteCompletion: remove %s failed: %v", hashfilename, err) + }*/ + return 0 +} diff --git a/win/cfapi/filesystem/filesystem_test.go b/win/cfapi/filesystem/filesystem_test.go new file mode 100644 index 0000000..4991882 --- /dev/null +++ b/win/cfapi/filesystem/filesystem_test.go @@ -0,0 +1,670 @@ +package filesystem_test + +import ( + "bytes" + "log" + "os" + "os/exec" + "reflect" + "strings" + "testing" + "time" + + "github.com/balazsgrill/potatodrive/win" + "github.com/balazsgrill/potatodrive/win/cfapi/filesystem" + "github.com/spf13/afero" +) + +type testInstance struct { + t *testing.T + location string + fs afero.Fs + closer win.Virtualization + closechan chan bool +} + +func newTestInstance(t *testing.T) *testInstance { + location := t.TempDir() + os.RemoveAll(location) + os.MkdirAll(location, 0x777) + return &testInstance{ + t: t, + location: location, + fs: afero.NewMemMapFs(), + closechan: make(chan bool), + } +} + +func (i *testInstance) start() { + started := make(chan bool) + var err error + go func() { + i.closer, err = filesystem.StartProjecting(i.location, i.fs) + started <- true + <-i.closechan + i.closer.Close() + }() + <-started + if err != nil { + log.Fatal(err) + } +} + +func (i *testInstance) osWriteFile(filename string, content string) error { + return exec.Command("cmd", "/c", "echo", content, ">", i.location+"\\"+filename).Run() +} + +func (i *testInstance) osRemoveFile(filename string) error { + return exec.Command("cmd", "/c", "del", i.location+"\\"+filename).Run() +} + +func (i *testInstance) osCreateDir(filename string) error { + return exec.Command("cmd", "/c", "mkdir", i.location+"\\"+filename).Run() +} + +func (i *testInstance) osRemoveDir(filename string) error { + return exec.Command("cmd", "/c", "rmdir", i.location+"\\"+filename).Run() +} + +func (i *testInstance) stop() { + i.closechan <- true +} + +func TestExistingFileOnBackend(t *testing.T) { + instance := newTestInstance(t) + + data := []byte("something") + filename := "test.txt" + err := afero.WriteFile(instance.fs, filename, data, 0x777) + if err != nil { + t.Fatal(err) + } + + instance.start() + defer instance.stop() + err = instance.closer.PerformSynchronization() + if err != nil { + t.Fatal(err) + } + + data2, err := os.ReadFile(instance.location + "\\" + filename) + if err != nil { + t.Fatal(err) + } + + if !bytes.Equal(data, data2) { + t.Errorf("expected %v, got %v", data, data2) + } +} + +func TestFileCreation(t *testing.T) { + instance := newTestInstance(t) + instance.start() + defer instance.stop() + + filename := "test.txt" + data := "something" + log.Printf("Writing %s to %s\n", data, filename) + err := instance.osWriteFile(filename, data) + if err != nil { + t.Fatal(err) + } + + log.Printf("Sycnhronizing\n") + err = instance.closer.PerformSynchronization() + if err != nil { + t.Fatal(err) + } + + log.Printf("Reading %s from %s\n", data, filename) + data2, err := afero.ReadFile(instance.fs, filename) + if err != nil { + t.Fatal(err) + } + + if data != strings.TrimSpace(string(data2)) { + t.Errorf("expected '%s', got '%s'", data, string(data2)) + } +} + +func TestUpdateExistingFileOnBackend(t *testing.T) { + instance := newTestInstance(t) + + data := "something" + filename := "test.txt" + err := afero.WriteFile(instance.fs, filename, []byte(data), 0x777) + if err != nil { + t.Fatal(err) + } + + instance.start() + defer instance.stop() + + // sleep to make sure that the file is newer + time.Sleep(1 * time.Second) + log.Println("Changing file") + data = "somethingelse" + err = instance.osWriteFile(filename, data) + if err != nil { + t.Fatal(err) + } + + log.Println("Synchronizing") + err = instance.closer.PerformSynchronization() + if err != nil { + t.Fatal(err) + } + + log.Println("Check content on remote") + data2, err := afero.ReadFile(instance.fs, filename) + if err != nil { + t.Fatal(err) + } + + if data != strings.TrimSpace(string(data2)) { + t.Errorf("expected %s, got %s", data, string(data2)) + } +} + +func TestDeleteExistingFileOnBackend(t *testing.T) { + instance := newTestInstance(t) + data := "something" + filename := "test.txt" + err := afero.WriteFile(instance.fs, filename, []byte(data), 0x777) + if err != nil { + t.Fatal(err) + } + + instance.start() + defer instance.stop() + + err = instance.osRemoveFile(filename) + if err != nil { + t.Fatal(err) + } + + log.Println("Synchronizing") + err = instance.closer.PerformSynchronization() + if err != nil { + t.Fatal(err) + } + + _, err = instance.fs.Stat(filename) + if err != nil { + if os.IsNotExist(err) { + //ok + return + } + t.Fatal(err) + } else { + t.Error("File exists") + } +} + +func TestListFiles(t *testing.T) { + instance := newTestInstance(t) + instance.start() + defer instance.stop() + + data := "something" + filename := "test.txt" + err := afero.WriteFile(instance.fs, filename, []byte(data), 0x777) + if err != nil { + t.Fatal(err) + } + + filename2 := "test2.txt" + err = instance.osWriteFile(filename2, data) + if err != nil { + t.Fatal(err) + } + + log.Println("Synchronizing") + err = instance.closer.PerformSynchronization() + if err != nil { + t.Fatal(err) + } + + expected := make(map[string]bool) + expected[filename] = true + expected[filename2] = true + + entries, err := os.ReadDir(instance.location) + if err != nil { + t.Fatal(err) + } + + actual := make(map[string]bool) + for _, entry := range entries { + if strings.HasPrefix(entry.Name(), ".") { + continue + } + actual[entry.Name()] = true + } + + if !reflect.DeepEqual(expected, actual) { + t.Errorf("expected %v, got %v", expected, actual) + } +} + +func TestExistingFolderOnBackend(t *testing.T) { + instance := newTestInstance(t) + + foldername := "test" + instance.fs.Mkdir(foldername, 0x777) + + instance.start() + defer instance.stop() + + stat, err := os.Stat(instance.location + "\\" + foldername) + if err != nil { + t.Fatal(err) + } + + if stat.IsDir() != true { + t.Error("Not a directory") + } +} + +func TestFolderCreation(t *testing.T) { + instance := newTestInstance(t) + instance.start() + defer instance.stop() + + foldername := "test" + err := instance.osCreateDir(foldername) + if err != nil { + t.Fatal(err) + } + + log.Println("Synchronizing") + err = instance.closer.PerformSynchronization() + if err != nil { + t.Fatal(err) + } + + stat, err := instance.fs.Stat(foldername) + if err != nil { + t.Fatal(err) + } + if stat.IsDir() != true { + t.Error("Not a directory") + } +} + +func TestCreatedOnBackend(t *testing.T) { + instance := newTestInstance(t) + instance.start() + defer instance.stop() + + data := []byte("something") + filename := "test.txt" + err := afero.WriteFile(instance.fs, filename, data, 0x777) + if err != nil { + t.Fatal(err) + } + + err = instance.closer.PerformSynchronization() + if err != nil { + t.Fatal(err) + } + + data2, err := os.ReadFile(instance.location + "\\" + filename) + if err != nil { + t.Fatal(err) + } + + if !bytes.Equal(data, data2) { + t.Errorf("expected %v, got %v", data, data2) + } + +} + +func TestChangedOnBackend(t *testing.T) { + instance := newTestInstance(t) + + data := []byte("something") + filename := "test.txt" + err := afero.WriteFile(instance.fs, filename, data, 0x777) + if err != nil { + t.Fatal(err) + } + + instance.start() + defer instance.stop() + + data2, err := os.ReadFile(instance.location + "\\" + filename) + if err != nil { + t.Fatal(err) + } + + if !bytes.Equal(data, data2) { + t.Errorf("expected %v, got %v", data, data2) + } + + // sleep for a bit to ensure that the file timestamp is different + time.Sleep(time.Second) + data = []byte("somethingelse") + err = afero.WriteFile(instance.fs, filename, data, 0x777) + if err != nil { + t.Fatal(err) + } + + err = instance.closer.PerformSynchronization() + if err != nil { + t.Fatal(err) + } + + data2, err = os.ReadFile(instance.location + "\\" + filename) + if err != nil { + t.Fatal(err) + } + + if !bytes.Equal(data, data2) { + t.Errorf("expected %v, got %v", data, data2) + } +} + +func TestDeletedOnBackend(t *testing.T) { + instance := newTestInstance(t) + instance.start() + defer instance.stop() + data := []byte("something") + filename := "test.txt" + err := afero.WriteFile(instance.fs, filename, data, 0x777) + if err != nil { + t.Fatal(err) + } + + err = instance.closer.PerformSynchronization() + if err != nil { + t.Fatal(err) + } + err = instance.osRemoveFile(filename) + if err != nil { + t.Fatal(err) + } + err = instance.closer.PerformSynchronization() + if err != nil { + t.Fatal(err) + } + _, err = instance.fs.Stat(filename) + if err != nil { + if os.IsNotExist(err) { + //ok + return + } + t.Fatal(err) + } else { + t.Error("File exists") + } +} + +func TestUpdatedLocallyWhileOffline(t *testing.T) { + instance := newTestInstance(t) + instance.start() + + data := []byte("something") + filename := "test.txt" + err := instance.osWriteFile(filename, string(data)) + + if err != nil { + t.Fatal(err) + } + + instance.stop() + time.Sleep(time.Second) + + data = []byte("somethingelse") + err = instance.osWriteFile(filename, string(data)) + if err != nil { + t.Fatal(err) + } + + instance.start() + err = instance.closer.PerformSynchronization() + if err != nil { + t.Fatal(err) + } + + data2, err := afero.ReadFile(instance.fs, filename) + if err != nil { + t.Fatal(err) + } + if string(data) != strings.TrimSpace(string(data2)) { + t.Errorf("expected '%s', got '%s'", string(data), string(data2)) + } + instance.stop() +} + +func TestRemoveFolder(t *testing.T) { + foldername := "test" + instance := newTestInstance(t) + err := instance.fs.Mkdir(foldername, 0x777) + if err != nil { + t.Fatal(err) + } + instance.start() + defer instance.stop() + + file, err := os.Stat(instance.location + "\\" + foldername) + if err != nil { + t.Fatal(err) + } + if file.IsDir() != true { + t.Error("Not a directory") + } + + log.Printf("Removing folder %s", foldername) + err = instance.osRemoveDir(foldername) + if err != nil { + t.Fatal(err) + } + err = instance.closer.PerformSynchronization() + if err != nil { + t.Fatal(err) + } + _, err = instance.fs.Stat(foldername) + if err != nil { + if os.IsNotExist(err) { + //ok + return + } + t.Fatal(err) + } else { + t.Error("File exists") + } + +} + +func TestDeletedOnBackendWhileOffline(t *testing.T) { + instance := newTestInstance(t) + instance.start() + + data := []byte("something") + filename := "test.txt" + err := instance.osWriteFile(filename, string(data)) + + if err != nil { + t.Fatal(err) + } + + err = instance.closer.PerformSynchronization() + if err != nil { + t.Fatal(err) + } + instance.stop() + time.Sleep(time.Second) + + err = instance.fs.Remove(filename) + if err != nil { + t.Fatal(err) + } + _, err = instance.fs.Stat(filename) + if !os.IsNotExist(err) { + t.Error("remote file exists") + } + + instance.start() + err = instance.closer.PerformSynchronization() + if err != nil { + t.Fatal(err) + } + + _, err = os.Stat(instance.location + "\\" + filename) + if !os.IsNotExist(err) { + t.Error("local file exists") + } + _, err = instance.fs.Stat(filename) + if !os.IsNotExist(err) { + t.Error("remote file exists") + } + + instance.stop() +} + +func TestDeletedLocallyWhileOffline(t *testing.T) { + instance := newTestInstance(t) + instance.start() + + data := []byte("something") + filename := "test.txt" + err := instance.osWriteFile(filename, string(data)) + + if err != nil { + t.Fatal(err) + } + + err = instance.closer.PerformSynchronization() + if err != nil { + t.Fatal(err) + } + instance.stop() + time.Sleep(time.Second) + + err = instance.osRemoveFile(filename) + if err != nil { + t.Fatal(err) + } + + instance.start() + err = instance.closer.PerformSynchronization() + if err != nil { + t.Fatal(err) + } + + _, err = os.Stat(instance.location + "\\" + filename) + if os.IsNotExist(err) { + t.Error("File should be restored locally") + } + _, err = instance.fs.Stat(filename) + if os.IsNotExist(err) { + t.Error("remote file should not be removed") + } + + instance.stop() +} + +func TestConflictWhileOfflineLocalNewer(t *testing.T) { + instance := newTestInstance(t) + instance.start() + + data := []byte("something") + filename := "test.txt" + err := instance.osWriteFile(filename, string(data)) + + if err != nil { + t.Fatal(err) + } + + instance.stop() + time.Sleep(time.Second) + + data2 := []byte("something2") + data3 := []byte("something3") + + err = afero.WriteFile(instance.fs, filename, data3, 0x777) + if err != nil { + t.Fatal(err) + } + time.Sleep(time.Second) + err = instance.osWriteFile(filename, string(data2)) + if err != nil { + t.Fatal(err) + } + + instance.start() + err = instance.closer.PerformSynchronization() + if err != nil { + t.Fatal(err) + } + + data4, err := afero.ReadFile(instance.fs, filename) + if err != nil { + t.Fatal(err) + } + if string(data2) != strings.TrimSpace(string(data4)) { + t.Errorf("expected '%s', got '%s'", string(data2), string(data4)) + } + data5, err := os.ReadFile(instance.location + "\\" + filename) + if err != nil { + t.Fatal(err) + } + if string(data2) != strings.TrimSpace(string(data5)) { + t.Errorf("expected '%s', got '%s'", string(data2), string(data5)) + } + + instance.stop() +} + +func TestConflictWhileOfflineRemoteNewer(t *testing.T) { + instance := newTestInstance(t) + instance.start() + + data := []byte("something") + filename := "test.txt" + err := instance.osWriteFile(filename, string(data)) + + if err != nil { + t.Fatal(err) + } + + instance.stop() + time.Sleep(time.Second) + + data2 := []byte("something2") + data3 := []byte("something3") + + err = instance.osWriteFile(filename, string(data2)) + if err != nil { + t.Fatal(err) + } + time.Sleep(time.Second) + err = afero.WriteFile(instance.fs, filename, data3, 0x777) + if err != nil { + t.Fatal(err) + } + + instance.start() + err = instance.closer.PerformSynchronization() + if err != nil { + t.Fatal(err) + } + + data4, err := afero.ReadFile(instance.fs, filename) + if err != nil { + t.Fatal(err) + } + if string(data3) != strings.TrimSpace(string(data4)) { + t.Errorf("expected '%s', got '%s'", string(data3), string(data4)) + } + data5, err := os.ReadFile(instance.location + "\\" + filename) + if err != nil { + t.Fatal(err) + } + if string(data3) != strings.TrimSpace(string(data5)) { + t.Errorf("expected '%s', got '%s'", string(data3), string(data5)) + } + + instance.stop() +} diff --git a/win/cfapi/filesystem/fswatch.go b/win/cfapi/filesystem/fswatch.go new file mode 100644 index 0000000..2908c74 --- /dev/null +++ b/win/cfapi/filesystem/fswatch.go @@ -0,0 +1,22 @@ +package filesystem + +import ( + "log" + + "github.com/fsnotify/fsnotify" +) + +func (instance *VirtualizationInstance) watch() { + log.Println("Watching for changes") + go func() { + for err := range instance.watcher.Errors { + log.Printf("Received error: %s", err) + } + }() + for event := range instance.watcher.Events { + log.Printf("Received event: %s", event) + if event.Op&fsnotify.Remove == fsnotify.Remove { + instance.handleDeletion(event.Name) + } + } +} diff --git a/win/cfapi/filesystem/pathutils.go b/win/cfapi/filesystem/pathutils.go new file mode 100644 index 0000000..66beb91 --- /dev/null +++ b/win/cfapi/filesystem/pathutils.go @@ -0,0 +1,70 @@ +package filesystem + +import ( + "path/filepath" + "strings" + + "log" + + "golang.org/x/sys/windows" +) + +func toLongPath(localpath string) string { + shortpathp, err := windows.UTF16FromString(localpath) + if err != nil { + log.Printf("Failed to convert path '%s' to UTF16: %v", localpath, err) + return localpath + } + longpathp := make([]uint16, windows.MAX_PATH) + _, err = windows.GetLongPathName(&shortpathp[0], &longpathp[0], uint32(len(longpathp))) + if err != nil { + log.Printf("Failed to convert path '%s' to long path: %v", localpath, err) + return localpath + } + return windows.UTF16ToString(longpathp) +} + +func toShortPath(localpath string) string { + shortpathp, err := windows.UTF16FromString(localpath) + if err != nil { + log.Printf("Failed to convert path '%s' to UTF16: %v", localpath, err) + return localpath + } + longpathp := make([]uint16, windows.MAX_PATH) + _, err = windows.GetShortPathName(&shortpathp[0], &longpathp[0], uint32(len(longpathp))) + if err != nil { + log.Printf("Failed to convert path '%s' to short path: %v", localpath, err) + return localpath + } + return windows.UTF16ToString(longpathp) +} + +func (instance *VirtualizationInstance) path_localToRemote(path string) string { + p := toLongPath(path) + p = strings.TrimPrefix(p, instance.shortprefix) + p = strings.TrimPrefix(p, instance.longprefix) + p = strings.ReplaceAll(p, "\\", "/") + p = strings.TrimPrefix(p, "/") + return p +} + +func (instance *VirtualizationInstance) path_remoteToLocal(path string) string { + p := strings.TrimPrefix(path, "/") + p = strings.ReplaceAll(p, "/", "\\") + return filepath.Join(instance.rootPath, "\\", p) +} + +func (instance *VirtualizationInstance) path_getNameRemote(path string) string { + p := strings.TrimPrefix(path, "/") + return filepath.Base(p) +} + +func (instance *VirtualizationInstance) path_getNameLocal(path string) string { + return filepath.Base(strings.ReplaceAll(path, "\\", "/")) +} + +func (instance *VirtualizationInstance) path_hashFile(remotepath string) string { + fname := filepath.Base(remotepath) + dir := filepath.Dir(remotepath) + return dir + "/.md5_" + fname +} diff --git a/win/cfapi/filesystem/placeholders.go b/win/cfapi/filesystem/placeholders.go new file mode 100644 index 0000000..bdb4f84 --- /dev/null +++ b/win/cfapi/filesystem/placeholders.go @@ -0,0 +1,76 @@ +package filesystem + +import ( + "log" + "strings" + "syscall" + "unsafe" + + "github.com/balazsgrill/potatodrive/win" + "github.com/balazsgrill/potatodrive/win/cfapi" + "github.com/spf13/afero" +) + +func (instance *VirtualizationInstance) fetchPlaceholders(info *cfapi.CF_CALLBACK_INFO, data *cfapi.CF_CALLBACK_PARAMETERS_FetchPlaceholders) uintptr { + instance.lock.Lock() + defer instance.lock.Unlock() + name := getFileNameFromIdentity(info) + log.Printf("Fetch placeholders: %s / %s", win.GetString(info.NormalizedPath), name) + remotepath := instance.path_localToRemote(win.GetString(info.NormalizedPath)) + files, err := afero.ReadDir(instance.fs, remotepath) + if err != nil { + log.Printf("Error reading directory %s: %s", remotepath, err) + return uintptr(syscall.EIO) + } + transfer := cfapi.CF_OPERATION_PARAMETERS_TransferPlaceholders{} + transfer.ParamSize = uint32(unsafe.Sizeof(transfer)) + transfer.CompletionStatus = 0 //success + transfer.Flags = cfapi.CF_OPERATION_TRANSFER_PLACEHOLDERS_FLAG_NONE + count := 0 + placeholders := make([]cfapi.CF_PLACEHOLDER_CREATE_INFO, len(files)) + for _, f := range files { + if !strings.HasPrefix(f.Name(), ".") { + log.Println(f.Name()) + placeholders[count] = getPlaceholder(f) + + count += 1 + } + } + + for i := 0; i < count; i++ { + log.Printf("Sending %d", i) + var placeholder cfapi.CF_PLACEHOLDER_CREATE_INFO + transfer.PlaceholderTotalCount = int64(count) + transfer.EntriesProcessed = 0 + transfer.PlaceholderCount = 1 + placeholder = placeholders[i] + transfer.PlaceholderArray = &placeholder + if i == count-1 { + transfer.Flags = cfapi.CF_OPERATION_TRANSFER_PLACEHOLDERS_FLAG_DISABLE_ON_DEMAND_POPULATION + } + + hr := instance.transferPlaceholders(info, &transfer) + + if hr != 0 { + log.Printf("Error transferring placeholders: %s", win.ErrorByCode(hr)) + return hr + } + } + + if count == 0 { + // send empty placeholder array + transfer.PlaceholderTotalCount = 0 + transfer.EntriesProcessed = 0 + transfer.PlaceholderCount = 0 + transfer.PlaceholderArray = nil + transfer.Flags = cfapi.CF_OPERATION_TRANSFER_PLACEHOLDERS_FLAG_DISABLE_ON_DEMAND_POPULATION + hr := instance.transferPlaceholders(info, &transfer) + + if hr != 0 { + log.Printf("Error transferring placeholders: %s", win.ErrorByCode(hr)) + return hr + } + } + log.Printf("Sent %d entries", count) + return 0 +} diff --git a/win/cfapi/filesystem/synclocaltoremote.go b/win/cfapi/filesystem/synclocaltoremote.go new file mode 100644 index 0000000..10d22a6 --- /dev/null +++ b/win/cfapi/filesystem/synclocaltoremote.go @@ -0,0 +1,97 @@ +package filesystem + +import ( + "bytes" + "io/fs" + "log" + "os" + "path/filepath" + "strings" + + "github.com/balazsgrill/potatodrive/win/cfapi" + "github.com/spf13/afero" +) + +// isDeletedRemotely check whether file was deleted remotely +// if it was, it compares local hash with remote hash. Returns true only if the file has been deleted remotely and was not changed locally +func (instance *VirtualizationInstance) isDeletedRemotely(remotepath string, localpath string) (bool, error) { + _, err := instance.fs.Stat(remotepath) + if os.IsNotExist(err) { + // chek if hash file exists on remote + hashpath := instance.path_hashFile(remotepath) + exists, err := afero.Exists(instance.fs, hashpath) + if err != nil { + return false, err + } + if exists { + // on remote file existed before, upload only if hash is different + hash, err := afero.ReadFile(instance.fs, hashpath) + if err != nil { + return false, err + } + localhash, err := instance.localHash(remotepath) + if err != nil { + return false, err + } + if localhash == nil { + // local file does not exist, no need to upload + // TODO is this a tombstone? + return false, nil + } + if bytes.Equal(hash, localhash) { + // hash is the same this file has been removed remotely, delete local file + return true, nil + } + } + + } + return false, nil +} + +func (instance *VirtualizationInstance) syncLocalToRemote() error { + return filepath.Walk(instance.rootPath, func(localpath string, localinfo fs.FileInfo, err error) error { + log.Printf("Syncing local file '%s'", localpath) + if os.IsNotExist(err) { + return nil + } + if err != nil { + return err + } + + path := instance.path_localToRemote(localpath) + if localinfo.IsDir() { + return instance.fs.MkdirAll(path, 0777) + } + if strings.HasPrefix(path, ".") { + return nil + } + + localstate, err := getPlaceholderState(localpath) + if err != nil { + return err + } + log.Printf("Local state %x", localstate) + + deleted, err := instance.isDeletedRemotely(path, localpath) + if err != nil { + return err + } + + if ((localstate & cfapi.CF_PLACEHOLDER_STATE_IN_SYNC) == 0) && (!deleted) { + // local file is a placeholder, but not in sync, upload it if local is newer + + remoteinfo, err := instance.fs.Stat(path) + localisnewer := os.IsNotExist(err) || (localinfo.ModTime().UTC().Unix() > remoteinfo.ModTime().UTC().Unix()) + + if localisnewer { + log.Printf("Updating remote file '%s'", path) + return instance.streamLocalToRemote(path) + } + } + + if deleted { + return os.Remove(localpath) + } + return nil + }) +} diff --git a/win/cfapi/filesystem/syncremotetolocal.go b/win/cfapi/filesystem/syncremotetolocal.go new file mode 100644 index 0000000..3a51981 --- /dev/null +++ b/win/cfapi/filesystem/syncremotetolocal.go @@ -0,0 +1,102 @@ +package filesystem + +import ( + "fmt" + "io/fs" + "log" + "os" + "path/filepath" + "strings" + "syscall" + + "github.com/balazsgrill/potatodrive/win" + "github.com/balazsgrill/potatodrive/win/cfapi" + "github.com/spf13/afero" +) + +func (instance *VirtualizationInstance) syncRemoteToLocal() error { + return afero.Walk(instance.fs, "", func(path string, remoteinfo fs.FileInfo, err error) error { + log.Printf("Syncing remote file '%s'", path) + if os.IsNotExist(err) { + return nil + } + if err != nil { + return err + } + + filename := instance.path_getNameRemote(path) + if strings.HasPrefix(filename, ".") { + return nil + } + localpath := instance.path_remoteToLocal(path) + placeholderstate, err := getPlaceholderState(localpath) + log.Printf("Placeholder state for '%s' is %x", localpath, placeholderstate) + if os.IsNotExist(err) { + if remoteinfo.IsDir() { + // local dir does not exist, create it + return os.MkdirAll(localpath, 0777) + } else { + localdir := filepath.Dir(localpath) + // placeholder does not exists, create it + placeholder := getPlaceholder(remoteinfo) + var EntriesProcessed uint32 + hr := cfapi.CfCreatePlaceholders(win.GetPointer(localdir), &placeholder, 1, cfapi.CF_CREATE_FLAG_NONE, &EntriesProcessed) + if hr != 0 { + return win.ErrorByCode(hr) + } + if EntriesProcessed != 1 { + return fmt.Errorf("unexpected number of entries processed: %d", EntriesProcessed) + } + // done here, return + return nil + } + } + if err != nil { + return err + } + + insync := (placeholderstate & cfapi.CF_PLACEHOLDER_STATE_IN_SYNC) != 0 + isaplacehoder := (placeholderstate & cfapi.CF_PLACEHOLDER_STATE_PLACEHOLDER) != 0 + + // is not a placeholder, or an in-sync hydrated placeholder + //if (insync) || (!isaplacehoder) { + // check if remote is newer + localinfo, _ := os.Stat(localpath) + if localinfo.ModTime().UTC().Unix() < remoteinfo.ModTime().UTC().Unix() { + log.Printf("Updating local file '%s'", path) + + var handle syscall.Handle + hr := cfapi.CfOpenFileWithOplock(win.GetPointer(localpath), cfapi.CF_OPEN_FILE_FLAG_WRITE_ACCESS|cfapi.CF_OPEN_FILE_FLAG_EXCLUSIVE, &handle) + if hr != 0 { + return win.ErrorByCode(hr) + } + defer cfapi.CfCloseHandle(handle) + placeholder := getPlaceholder(remoteinfo) + + if !isaplacehoder { + log.Printf("Converting to placeholder '%s'", path) + hr = cfapi.CfConvertToPlaceholder(handle, placeholder.FileIdentity, placeholder.FileIdentityLength, cfapi.CF_CONVERT_FLAG_NONE, 0, 0) + if hr != 0 { + return win.ErrorByCode(hr) + } + } + if !insync { + hr = cfapi.CfSetInSyncState(handle, cfapi.CF_IN_SYNC_STATE_IN_SYNC, cfapi.CF_SET_IN_SYNC_FLAG_NONE, nil) + if hr != 0 { + return win.ErrorByCode(hr) + } + } + var fileRange cfapi.CF_FILE_RANGE + fileRange.StartingOffset = 0 + fileRange.Length = localinfo.Size() + hr = cfapi.CfUpdatePlaceholder(handle, &placeholder.FsMetadata, placeholder.FileIdentity, placeholder.FileIdentityLength, &fileRange, 1, cfapi.CF_UPDATE_FLAG_CLEAR_IN_SYNC|cfapi.CF_UPDATE_FLAG_DEHYDRATE, nil, 0) + if hr != 0 { + return win.ErrorByCode(hr) + } + + } + //} + + return nil + }) +} diff --git a/win/cfapi/functions.go b/win/cfapi/functions.go index d3f83fb..e4fec8f 100644 --- a/win/cfapi/functions.go +++ b/win/cfapi/functions.go @@ -65,8 +65,8 @@ func CfConvertToPlaceholder(FileHandle syscall.Handle, FileIdentity uintptr, Fil return ret } -func CfCreatePlaceholders(BaseDirectoryPath uintptr, PlaceholderArray []CF_PLACEHOLDER_CREATE_INFO, PlaceholderCount uint32, CreateFlags CF_CREATE_FLAGS, EntriesProcessed *uint32) uintptr { - ret, _, _ := cfCreatePlaceholders.Call(BaseDirectoryPath, uintptr(unsafe.Pointer(&PlaceholderArray[0])), uintptr(PlaceholderCount), uintptr(CreateFlags), uintptr(unsafe.Pointer(EntriesProcessed))) +func CfCreatePlaceholders(BaseDirectoryPath uintptr, PlaceholderArray *CF_PLACEHOLDER_CREATE_INFO, PlaceholderCount uint32, CreateFlags CF_CREATE_FLAGS, EntriesProcessed *uint32) uintptr { + ret, _, _ := cfCreatePlaceholders.Call(BaseDirectoryPath, uintptr(unsafe.Pointer(PlaceholderArray)), uintptr(PlaceholderCount), uintptr(CreateFlags), uintptr(unsafe.Pointer(EntriesProcessed))) return ret } @@ -89,3 +89,131 @@ func CfGetPlaceholderInfo(FileHandle syscall.Handle, InfoClass CF_PLACEHOLDER_IN ret, _, _ := cfGetPlaceholderInfo.Call(uintptr(FileHandle), uintptr(InfoClass), InfoBuffer, uintptr(InfoBufferLength), uintptr(unsafe.Pointer(ReturnedLength))) return ret } + +func CfGetPlaceholderRangeInfo(FileHandle syscall.Handle, InfoClass CF_PLACEHOLDER_RANGE_INFO_CLASS, StartingOffset int64, Length int64, InfoBuffer uintptr, InfoBufferLength uint32, ReturnedLength *uint32) uintptr { + ret, _, _ := cfGetPlaceholderRangeInfo.Call(uintptr(FileHandle), uintptr(InfoClass), uintptr(StartingOffset), uintptr(Length), InfoBuffer, uintptr(InfoBufferLength), uintptr(unsafe.Pointer(ReturnedLength))) + return ret +} + +func CfGetPlaceholderRangeInfoForHydration(ConnectionKey CF_CONNECTION_KEY, TransferKey CF_TRANSFER_KEY, FileId int64, InfoClass CF_PLACEHOLDER_RANGE_INFO_CLASS, StartingOffset int64, RangeLength int64, InfoBuffer uintptr, InfoBufferSize uint32, InfoBufferWritten *uint32) uintptr { + ret, _, _ := cfGetPlaceholderRangeInfoForHydration.Call(uintptr(ConnectionKey), uintptr(TransferKey), uintptr(FileId), uintptr(InfoClass), uintptr(StartingOffset), uintptr(RangeLength), InfoBuffer, uintptr(InfoBufferSize), uintptr(unsafe.Pointer(InfoBufferWritten))) + return ret +} + +func CfGetPlaceholderStateFromAttributeTag(FileAttributes uint32, ReparseTag uint32) uintptr { + ret, _, _ := cfGetPlaceholderStateFromAttributeTag.Call(uintptr(FileAttributes), uintptr(ReparseTag)) + return ret +} + +func CfGetPlaceholderStateFromFileInfo(InfoBuffer uintptr, InfoClass FILE_INFO_BY_HANDLE_CLASS) uintptr { + ret, _, _ := cfGetPlaceholderStateFromFileInfo.Call(InfoBuffer, uintptr(InfoClass)) + return ret +} + +func CfGetPlaceholderStateFromFindData(FindData uintptr) uintptr { + ret, _, _ := cfGetPlaceholderStateFromFindData.Call(FindData) + return ret +} + +func CfGetPlatformInfo(PlatformVersion *CF_PLATFORM_INFO) uintptr { + ret, _, _ := cfGetPlatformInfo.Call(uintptr(unsafe.Pointer(PlatformVersion))) + return ret +} + +func CfGetSyncRootInfoByHandle(FileHandle syscall.Handle, InfoClass CF_SYNC_ROOT_INFO_CLASS, InfoBuffer uintptr, InfoBufferLength uint32, ReturnedLength *uint32) uintptr { + ret, _, _ := cfGetSyncRootInfoByHandle.Call(uintptr(FileHandle), uintptr(InfoClass), InfoBuffer, uintptr(InfoBufferLength), uintptr(unsafe.Pointer(ReturnedLength))) + return ret +} + +func CfGetSyncRootInfoByPath(SyncRootPath uintptr, InfoClass CF_SYNC_ROOT_INFO_CLASS, InfoBuffer uintptr, InfoBufferLength uint32, ReturnedLength *uint32) uintptr { + ret, _, _ := cfGetSyncRootInfoByPath.Call(SyncRootPath, uintptr(InfoClass), InfoBuffer, uintptr(InfoBufferLength), uintptr(unsafe.Pointer(ReturnedLength))) + return ret +} + +func CfGetTransferKey(FileHandle syscall.Handle, TransferKey *CF_TRANSFER_KEY) uintptr { + ret, _, _ := cfGetTransferKey.Call(uintptr(FileHandle), uintptr(unsafe.Pointer(TransferKey))) + return ret +} +func CfGetWin32HandleFromProtectedHandle(ProtectedHandle uintptr) uintptr { + ret, _, _ := cfGetWin32HandleFromProtectedHandle.Call(ProtectedHandle) + return ret +} + +func CfHydratePlaceholder(FileHandle syscall.Handle, StartingOffset int64, Length int64, HydrateFlags CF_HYDRATE_FLAGS, Overlapped uintptr) uintptr { + ret, _, _ := cfHydratePlaceholder.Call(uintptr(FileHandle), uintptr(StartingOffset), uintptr(Length), uintptr(HydrateFlags)) + return ret +} + +func CfOpenFileWithOplock(FilePath uintptr, Flags CF_OPEN_FILE_FLAGS, ProtectedHandle *syscall.Handle) uintptr { + ret, _, _ := cfOpenFileWithOplock.Call(FilePath, uintptr(Flags), uintptr(unsafe.Pointer(ProtectedHandle))) + return ret +} + +func CfQuerySyncProviderStatus(ConnectionKey CF_CONNECTION_KEY, SyncProviderStatus *CF_SYNC_PROVIDER_STATUS) uintptr { + ret, _, _ := cfQuerySyncProviderStatus.Call(uintptr(ConnectionKey), uintptr(unsafe.Pointer(SyncProviderStatus))) + return ret +} + +func CfReferenceProtectedHandle(ProtectedHandle syscall.Handle) uintptr { + ret, _, _ := cfReferenceProtectedHandle.Call(uintptr(ProtectedHandle)) + return ret +} + +func CfReleaseProtectedHandle(ProtectedHandle syscall.Handle) uintptr { + ret, _, _ := cfReleaseProtectedHandle.Call(uintptr(ProtectedHandle)) + return ret +} + +func CfReleaseTransferKey(FileHandle syscall.Handle, TransferKey CF_TRANSFER_KEY) uintptr { + ret, _, _ := cfReleaseTransferKey.Call(uintptr(FileHandle), uintptr(TransferKey)) + return ret +} + +func CfReportProviderProgress(ConnectionKey CF_CONNECTION_KEY, TransferKey CF_TRANSFER_KEY, ProviderProgressTotal int64, ProviderProgressCompleted int64) uintptr { + ret, _, _ := cfReportProviderProgress.Call(uintptr(ConnectionKey), uintptr(TransferKey), uintptr(ProviderProgressTotal), uintptr(ProviderProgressCompleted)) + return ret +} + +func CfReportProviderProgress2(ConnectionKey CF_CONNECTION_KEY, TransferKey CF_TRANSFER_KEY, RequestKey CF_REQUEST_KEY, ProviderProgressTotal int64, ProviderProgressCompleted int64, TargetSessionId uint32) uintptr { + ret, _, _ := cfReportProviderProgress2.Call(uintptr(ConnectionKey), uintptr(TransferKey), uintptr(RequestKey), uintptr(ProviderProgressTotal), uintptr(ProviderProgressCompleted), uintptr(TargetSessionId)) + return ret +} + +func CfReportSyncStatus(SyncRootPath uintptr, SyncStatus *CF_SYNC_STATUS) uintptr { + ret, _, _ := cfReportSyncStatus.Call(SyncRootPath, uintptr(unsafe.Pointer(SyncStatus))) + return ret +} + +func CfRevertPlaceholder(FileHandle syscall.Handle, RevertFlags CF_REVERT_FLAGS, Overlapped uintptr) uintptr { + ret, _, _ := cfRevertPlaceholder.Call(uintptr(FileHandle), uintptr(RevertFlags)) + return ret +} + +func CfSetCorrelationVector(FileHandle syscall.Handle, CorrelationVector *CorrelationVector) uintptr { + ret, _, _ := cfSetCorrelationVector.Call(uintptr(FileHandle), uintptr(unsafe.Pointer(CorrelationVector))) + return ret +} + +func CfSetInSyncState(FileHandle syscall.Handle, InSyncState CF_IN_SYNC_STATE, InSyncFlags CF_SET_IN_SYNC_FLAGS, InSyncUsn *USN) uintptr { + ret, _, _ := cfSetInSyncState.Call(uintptr(FileHandle), uintptr(InSyncState), uintptr(InSyncFlags), uintptr(unsafe.Pointer(InSyncUsn))) + return ret +} +func CfSetPinState(FileHandle syscall.Handle, PinState CF_PIN_STATE, PinFlags CF_SET_PIN_FLAGS, Overlapped uintptr) uintptr { + ret, _, _ := cfSetPinState.Call(uintptr(FileHandle), uintptr(PinState), uintptr(PinFlags), Overlapped) + return ret +} + +func CfUnregisterSyncRoot(SyncRootPath uintptr) uintptr { + ret, _, _ := cfUnregisterSyncRoot.Call(SyncRootPath) + return ret +} + +func CfUpdatePlaceholder(FileHandle syscall.Handle, FsMetadata *CF_FS_METADATA, FileIdentity uintptr, FileIdentityLength uint32, DehydrateRangeArray *CF_FILE_RANGE, DehydrateRangeCount uint32, UpdateFlags CF_UPDATE_FLAGS, UpdateUsn *USN, Overlapped uintptr) uintptr { + ret, _, _ := cfUpdatePlaceholder.Call(uintptr(FileHandle), uintptr(unsafe.Pointer(FsMetadata)), FileIdentity, uintptr(FileIdentityLength), uintptr(unsafe.Pointer(DehydrateRangeArray)), uintptr(DehydrateRangeCount), uintptr(UpdateFlags), uintptr(unsafe.Pointer(UpdateUsn)), Overlapped) + return ret +} + +func CfUpdateSyncProviderStatus(ConnectionKey CF_CONNECTION_KEY, SyncProviderStatus *CF_SYNC_PROVIDER_STATUS) uintptr { + ret, _, _ := cfUpdateSyncProviderStatus.Call(uintptr(ConnectionKey), uintptr(unsafe.Pointer(SyncProviderStatus))) + return ret +} diff --git a/win/cfapi/types.go b/win/cfapi/types.go index a5a2794..4a3f23d 100644 --- a/win/cfapi/types.go +++ b/win/cfapi/types.go @@ -104,12 +104,14 @@ const ( ) type CF_CALLBACK_PARAMETERS_FetchData struct { - ParamSize uint32 - Flags CF_CALLBACK_FETCH_DATA_FLAGS - RequiredFileOffset int64 - RequiredLength int64 - OptionalFileOffset int64 - OptionalLength int64 + ParamSize uint32 + Flags CF_CALLBACK_FETCH_DATA_FLAGS + // TODO, reordered fields? + RequiredLength int64 + RequiredFileOffset int64 + OptionalLength int64 + OptionalFileOffset int64 + LastDehydrationTime int64 LastDehydrationReason CF_CALLBACK_DEHYDRATION_REASON } @@ -389,10 +391,10 @@ type CF_FS_METADATA struct { // https://learn.microsoft.com/en-us/windows/win32/api/winbase/ns-winbase-file_basic_info type FILE_BASIC_INFO struct { - CreationTime int64 - LastAccessTime int64 - LastWriteTime int64 - ChangeTime int64 + CreationTime syscall.Filetime + LastAccessTime syscall.Filetime + LastWriteTime syscall.Filetime + ChangeTime syscall.Filetime FileAttributes int32 } @@ -568,3 +570,176 @@ const ( CF_PLACEHOLDER_INFO_BASIC CF_PLACEHOLDER_INFO_CLASS = 0 CF_PLACEHOLDER_INFO_STANDARD CF_PLACEHOLDER_INFO_CLASS = 1 ) + +type CF_PLACEHOLDER_RANGE_INFO_CLASS uint32 + +const ( + CF_PLACEHOLDER_RANGE_INFO_ONDISK CF_PLACEHOLDER_RANGE_INFO_CLASS = 1 + CF_PLACEHOLDER_RANGE_INFO_VALIDATED CF_PLACEHOLDER_RANGE_INFO_CLASS = 2 + CF_PLACEHOLDER_RANGE_INFO_MODIFIED CF_PLACEHOLDER_RANGE_INFO_CLASS = 3 +) + +// https://learn.microsoft.com/en-us/windows/win32/api/minwinbase/ne-minwinbase-file_info_by_handle_class +type FILE_INFO_BY_HANDLE_CLASS uint32 + +const ( + FileBasicInfo FILE_INFO_BY_HANDLE_CLASS = 0 + FileStandardInfo FILE_INFO_BY_HANDLE_CLASS = 1 + FileNameInfo FILE_INFO_BY_HANDLE_CLASS = 2 + FileRenameInfo FILE_INFO_BY_HANDLE_CLASS = 3 + FileDispositionInfo FILE_INFO_BY_HANDLE_CLASS = 4 + FileAllocationInfo FILE_INFO_BY_HANDLE_CLASS = 5 + FileEndOfFileInfo FILE_INFO_BY_HANDLE_CLASS = 6 + FileStreamInfo FILE_INFO_BY_HANDLE_CLASS = 7 + FileCompressionInfo FILE_INFO_BY_HANDLE_CLASS = 8 + FileAttributeTagInfo FILE_INFO_BY_HANDLE_CLASS = 9 + FileIdBothDirectoryInfo FILE_INFO_BY_HANDLE_CLASS = 10 + FileIdBothDirectoryRestartInfo FILE_INFO_BY_HANDLE_CLASS = 11 + FileIoPriorityHintInfo FILE_INFO_BY_HANDLE_CLASS = 12 + FileRemoteProtocolInfo FILE_INFO_BY_HANDLE_CLASS = 13 + FileFullDirectoryInfo FILE_INFO_BY_HANDLE_CLASS = 14 + FileFullDirectoryRestartInfo FILE_INFO_BY_HANDLE_CLASS = 15 + FileStorageInfo FILE_INFO_BY_HANDLE_CLASS = 16 + FileAlignmentInfo FILE_INFO_BY_HANDLE_CLASS = 17 + FileIdInfo FILE_INFO_BY_HANDLE_CLASS = 18 + FileIdExtdDirectoryInfo FILE_INFO_BY_HANDLE_CLASS = 19 + FileIdExtdDirectoryRestartInfo FILE_INFO_BY_HANDLE_CLASS = 20 + FileDispositionInfoEx FILE_INFO_BY_HANDLE_CLASS = 21 + FileRenameInfoEx FILE_INFO_BY_HANDLE_CLASS = 22 + FileCaseSensitiveInfo FILE_INFO_BY_HANDLE_CLASS = 23 + FileNormalizedNameInfo FILE_INFO_BY_HANDLE_CLASS = 24 + MaximumFileInfoByHandleClass FILE_INFO_BY_HANDLE_CLASS = 25 +) + +type CF_PLATFORM_INFO struct { + BuildNumber uint32 + RevisionNumber uint32 + IntegrationNumber uint32 +} + +type CF_SYNC_ROOT_INFO_CLASS uint32 + +const ( + CF_SYNC_ROOT_INFO_BASIC CF_SYNC_ROOT_INFO_CLASS = 0 + CF_SYNC_ROOT_INFO_STANDARD CF_SYNC_ROOT_INFO_CLASS = 1 + CF_SYNC_ROOT_INFO_PROVIDER CF_SYNC_ROOT_INFO_CLASS = 2 +) + +type CF_HYDRATE_FLAGS uint32 + +const ( + CF_HYDRATE_FLAG_NONE CF_HYDRATE_FLAGS = 0x00000000 +) + +type CF_OPEN_FILE_FLAGS uint32 + +const ( + CF_OPEN_FILE_FLAG_NONE CF_OPEN_FILE_FLAGS = 0x00000000 + CF_OPEN_FILE_FLAG_EXCLUSIVE CF_OPEN_FILE_FLAGS = 0x00000001 + CF_OPEN_FILE_FLAG_WRITE_ACCESS CF_OPEN_FILE_FLAGS = 0x00000002 + CF_OPEN_FILE_FLAG_DELETE_ACCESS CF_OPEN_FILE_FLAGS = 0x00000004 + CF_OPEN_FILE_FLAG_FOREGROUND_SYNC CF_OPEN_FILE_FLAGS = 0x00000008 +) + +type CF_SYNC_PROVIDER_STATUS uint32 + +const ( + CF_PROVIDER_STATUS_DISCONNECTED CF_SYNC_PROVIDER_STATUS = 0x00000000 + CF_PROVIDER_STATUS_IDLE CF_SYNC_PROVIDER_STATUS = 0x00000001 + CF_PROVIDER_STATUS_POPULATE_NAMESPACE CF_SYNC_PROVIDER_STATUS = 0x00000002 + CF_PROVIDER_STATUS_POPULATE_METADATA CF_SYNC_PROVIDER_STATUS = 0x00000004 + CF_PROVIDER_STATUS_POPULATE_CONTENT CF_SYNC_PROVIDER_STATUS = 0x00000008 + CF_PROVIDER_STATUS_SYNC_INCREMENTAL CF_SYNC_PROVIDER_STATUS = 0x00000010 + CF_PROVIDER_STATUS_SYNC_FULL CF_SYNC_PROVIDER_STATUS = 0x00000020 + CF_PROVIDER_STATUS_CONNECTIVITY_LOST CF_SYNC_PROVIDER_STATUS = 0x00000040 + CF_PROVIDER_STATUS_CLEAR_FLAGS CF_SYNC_PROVIDER_STATUS = 0x00000080 + CF_PROVIDER_STATUS_TERMINATED CF_SYNC_PROVIDER_STATUS = 0xC0000001 + CF_PROVIDER_STATUS_ERROR CF_SYNC_PROVIDER_STATUS = 0xC0000002 +) + +type CF_REVERT_FLAGS uint32 + +const ( + CF_REVERT_FLAG_NONE CF_REVERT_FLAGS = 0x00000000 +) + +type CF_IN_SYNC_STATE uint32 + +const ( + CF_IN_SYNC_STATE_NOT_IN_SYNC CF_IN_SYNC_STATE = 0x00000000 + CF_IN_SYNC_STATE_IN_SYNC CF_IN_SYNC_STATE = 0x00000001 +) + +type CF_SET_IN_SYNC_FLAGS uint32 + +const ( + CF_SET_IN_SYNC_FLAG_NONE CF_SET_IN_SYNC_FLAGS = 0x00000000 +) + +type CF_PIN_STATE uint32 + +const ( + CF_PIN_STATE_UNSPECIFIED CF_PIN_STATE = 0 + CF_PIN_STATE_PINNED CF_PIN_STATE = 1 + CF_PIN_STATE_UNPINNED CF_PIN_STATE = 2 + CF_PIN_STATE_EXCLUDED CF_PIN_STATE = 3 + CF_PIN_STATE_INHERIT CF_PIN_STATE = 4 +) + +type CF_SET_PIN_FLAGS uint32 + +const ( + CF_SET_PIN_FLAG_NONE CF_SET_PIN_FLAGS = 0x00000000 + CF_SET_PIN_FLAG_RECURSE CF_SET_PIN_FLAGS = 0x00000001 + CF_SET_PIN_FLAG_RECURSE_ONLY CF_SET_PIN_FLAGS = 0x00000002 + CF_SET_PIN_FLAG_RECURSE_STOP_ON_ERROR CF_SET_PIN_FLAGS = 0x00000004 +) + +type CF_FILE_RANGE struct { + StartingOffset int64 + Length int64 +} + +type CF_UPDATE_FLAGS uint32 + +const ( + CF_UPDATE_FLAG_NONE CF_UPDATE_FLAGS = 0x00000000 + CF_UPDATE_FLAG_VERIFY_IN_SYNC CF_UPDATE_FLAGS = 0x00000001 + CF_UPDATE_FLAG_MARK_IN_SYNC CF_UPDATE_FLAGS = 0x00000002 + CF_UPDATE_FLAG_DEHYDRATE CF_UPDATE_FLAGS = 0x00000004 + CF_UPDATE_FLAG_ENABLE_ON_DEMAND_POPULATION CF_UPDATE_FLAGS = 0x00000008 + CF_UPDATE_FLAG_DISABLE_ON_DEMAND_POPULATION CF_UPDATE_FLAGS = 0x00000010 + CF_UPDATE_FLAG_REMOVE_FILE_IDENTITY CF_UPDATE_FLAGS = 0x00000020 + CF_UPDATE_FLAG_CLEAR_IN_SYNC CF_UPDATE_FLAGS = 0x00000040 + CF_UPDATE_FLAG_REMOVE_PROPERTY CF_UPDATE_FLAGS = 0x00000080 + CF_UPDATE_FLAG_PASSTHROUGH_FS_METADATA CF_UPDATE_FLAGS = 0x00000100 + CF_UPDATE_FLAG_ALWAYS_FULL CF_UPDATE_FLAGS = 0x00000200 + CF_UPDATE_FLAG_ALLOW_PARTIAL CF_UPDATE_FLAGS = 0x00000400 +) + +type CF_PLACEHOLDER_BASIC_INFO struct { + PinState CF_PIN_STATE + InSyncState CF_IN_SYNC_STATE + FileId int64 + SyncRootFileId int64 + FileIdentityLength uint32 + FileIdentity uintptr +} + +type FILE_ATTRIBUTE_TAG_INFO struct { + FileAttributes uint32 + ReparseTag uint32 +} + +type CF_PLACEHOLDER_STATE uint32 + +const ( + CF_PLACEHOLDER_STATE_NO_STATES CF_PLACEHOLDER_STATE = 0x00000000 + CF_PLACEHOLDER_STATE_PLACEHOLDER CF_PLACEHOLDER_STATE = 0x00000001 + CF_PLACEHOLDER_STATE_SYNC_ROOT CF_PLACEHOLDER_STATE = 0x00000002 + CF_PLACEHOLDER_STATE_ESSENTIAL_PROP_PRESENT CF_PLACEHOLDER_STATE = 0x00000004 + CF_PLACEHOLDER_STATE_IN_SYNC CF_PLACEHOLDER_STATE = 0x00000008 + CF_PLACEHOLDER_STATE_PARTIAL_FILE_IN_SYNC CF_PLACEHOLDER_STATE = 0x00000010 + CF_PLACEHOLDER_STATE_PARTIALLY_ON_DISK CF_PLACEHOLDER_STATE = 0x00000020 + CF_PLACEHOLDER_STATE_INVALID CF_PLACEHOLDER_STATE = 0xffffffff +) diff --git a/win/errorcode.go b/win/errorcode.go new file mode 100644 index 0000000..d5b500a --- /dev/null +++ b/win/errorcode.go @@ -0,0 +1,20 @@ +package win + +import ( + "fmt" + + "golang.org/x/sys/windows" +) + +func ErrorByCode(result uintptr) error { + if result == 0 { + return nil + } else { + message := make([]uint16, 256) + _, err := windows.FormatMessage(windows.FORMAT_MESSAGE_IGNORE_INSERTS|windows.FORMAT_MESSAGE_FROM_SYSTEM, 0, uint32(result), 0, message, nil) + if err != nil { + return fmt.Errorf("can't extract message of %x: %v", result, err) + } + return fmt.Errorf("error result: %x - %s", result, windows.UTF16ToString(message)) + } +} diff --git a/win/projfs/filesystem/filesystem.go b/win/projfs/filesystem/filesystem.go index 562ce36..14e8e15 100644 --- a/win/projfs/filesystem/filesystem.go +++ b/win/projfs/filesystem/filesystem.go @@ -33,11 +33,6 @@ type VirtualizationInstance struct { enumerations map[syscall.GUID]*enumerationSession } -type Virtualization interface { - io.Closer - PerformSynchronization() error -} - type enumerationSession struct { searchstr uintptr countget int @@ -55,7 +50,7 @@ func (instance *VirtualizationInstance) Close() error { return nil } -func StartProjecting(rootPath string, filesystem afero.Fs) (Virtualization, error) { +func StartProjecting(rootPath string, filesystem afero.Fs) (win.Virtualization, error) { instance := &VirtualizationInstance{ enumerations: make(map[syscall.GUID]*enumerationSession), } @@ -76,8 +71,8 @@ func (instance *VirtualizationInstance) start(rootPath string, filesystem afero. hr := projfs.PrjMarkDirectoryAsPlaceholder(rootPath, "", nil, id) if hr != 0 { - log.Printf("Error marking directory as placeholder: %s", projfs.ErrorByCode(hr)) - return projfs.ErrorByCode(hr) + log.Printf("Error marking directory as placeholder: %s", win.ErrorByCode(hr)) + return win.ErrorByCode(hr) } log.Printf("Starting virtualization of '%s' (%v)", rootPath, *id) options := &projfs.PRJ_STARTVIRTUALIZING_OPTIONS{ @@ -90,7 +85,7 @@ func (instance *VirtualizationInstance) start(rootPath string, filesystem afero. ConcurrentThreadCount: 4, } hr = projfs.PrjStartVirtualizing(rootPath, instance.get_callbacks(), instance, options, &instance._instanceHandle) - err = projfs.ErrorByCode(hr) + err = win.ErrorByCode(hr) if err != nil { log.Printf("Error starting virtualization: %s", err) return err @@ -159,7 +154,7 @@ func (instance *VirtualizationInstance) syncRemoteToLocal() error { var localstate projfs.PRJ_FILE_STATE hr := projfs.PrjGetOnDiskFileState(localpath, &localstate) if hr != 0 { - return projfs.ErrorByCode(hr) + return win.ErrorByCode(hr) } if (localstate | (projfs.PRJ_FILE_STATE_FULL & projfs.PRJ_FILE_STATE_HYDRATED_PLACEHOLDER)) != 0 { @@ -169,7 +164,7 @@ func (instance *VirtualizationInstance) syncRemoteToLocal() error { log.Printf("Updating local file '%s'", path) var placeholderInfo projfs.PRJ_PLACEHOLDER_INFO FillInPlaceholderInfo(&placeholderInfo, remoteinfo) - //err = projfs.ErrorByCode(projfs.PrjWritePlaceholderInfo(instance._instanceHandle, path, &placeholderInfo, uint32(unsafe.Sizeof(placeholderInfo)))) + //err = win.ErrorByCode(projfs.PrjWritePlaceholderInfo(instance._instanceHandle, path, &placeholderInfo, uint32(unsafe.Sizeof(placeholderInfo)))) err = instance.UpdateFileIfNeeded(path, &placeholderInfo, uint32(unsafe.Sizeof(placeholderInfo)), projfs.PRJ_UPDATE_ALLOW_DIRTY_METADATA|projfs.PRJ_UPDATE_ALLOW_DIRTY_DATA) if err != nil { return err @@ -186,7 +181,7 @@ func (instance *VirtualizationInstance) localHash(remotepath string) ([]byte, er var localstate projfs.PRJ_FILE_STATE hr := projfs.PrjGetOnDiskFileState(instance.path_remoteToLocal(remotepath), &localstate) if hr != 0 { - return nil, projfs.ErrorByCode(hr) + return nil, win.ErrorByCode(hr) } if (localstate | (projfs.PRJ_FILE_STATE_FULL & projfs.PRJ_FILE_STATE_HYDRATED_PLACEHOLDER)) == 0 { return nil, nil @@ -225,7 +220,7 @@ func (instance *VirtualizationInstance) syncLocalToRemote() error { var localstate projfs.PRJ_FILE_STATE hr := projfs.PrjGetOnDiskFileState(localpath, &localstate) if hr != 0 { - return projfs.ErrorByCode(hr) + return win.ErrorByCode(hr) } if (localstate | (projfs.PRJ_FILE_STATE_FULL & projfs.PRJ_FILE_STATE_HYDRATED_PLACEHOLDER)) != 0 { @@ -286,15 +281,6 @@ func (instance *VirtualizationInstance) getVirtualizationInfoFileName() string { return instance.rootPath + "\\.virtualization" } -func bytesToGuid(b []byte) *syscall.GUID { - return &syscall.GUID{ - Data1: binary.LittleEndian.Uint32(b[0:4]), - Data2: binary.LittleEndian.Uint16(b[4:6]), - Data3: binary.LittleEndian.Uint16(b[6:8]), - Data4: ([8]byte)(b[8:16]), - } -} - func (instance *VirtualizationInstance) ensureVirtualizationFolderExists() (*syscall.GUID, error) { err := os.MkdirAll(instance.rootPath, 0777) if err != nil { @@ -303,7 +289,7 @@ func (instance *VirtualizationInstance) ensureVirtualizationFolderExists() (*sys if _, err := os.Stat(instance.getVirtualizationInfoFileName()); errors.Is(err, os.ErrNotExist) { uuid, _ := uuid.NewRandom() - id := bytesToGuid(uuid[:]) + id := win.BytesToGuid(uuid[:]) err = os.WriteFile(instance.getVirtualizationInfoFileName(), uuid[:], 0666) if err != nil { return nil, err @@ -319,7 +305,7 @@ func (instance *VirtualizationInstance) ensureVirtualizationFolderExists() (*sys return nil, errors.New("invalid virtualization info file") } - return bytesToGuid(bytes), nil + return win.BytesToGuid(bytes), nil } func (instance *VirtualizationInstance) get_callbacks() *projfs.PRJ_CALLBACKS { @@ -337,7 +323,7 @@ func (instance *VirtualizationInstance) get_callbacks() *projfs.PRJ_CALLBACKS { func (instance *VirtualizationInstance) UpdateFileIfNeeded(relativePath string, placeholderInfo *projfs.PRJ_PLACEHOLDER_INFO, length uint32, updateFlags projfs.PRJ_UPDATE_TYPES) error { var failureReason projfs.PRJ_UPDATE_FAILURE_CAUSES - err := projfs.ErrorByCode(projfs.PrjUpdateFileIfNeeded(instance._instanceHandle, relativePath, placeholderInfo, length, updateFlags, &failureReason)) + err := win.ErrorByCode(projfs.PrjUpdateFileIfNeeded(instance._instanceHandle, relativePath, placeholderInfo, length, updateFlags, &failureReason)) if err != nil { err = fmt.Errorf("UpdateFileIfNeeded failed: %w (reason: %d)", err, failureReason) } diff --git a/win/projfs/filesystem/filesystem_test.go b/win/projfs/filesystem/filesystem_test.go index aabe3e7..8fb121c 100644 --- a/win/projfs/filesystem/filesystem_test.go +++ b/win/projfs/filesystem/filesystem_test.go @@ -10,6 +10,7 @@ import ( "testing" "time" + "github.com/balazsgrill/potatodrive/win" "github.com/balazsgrill/potatodrive/win/projfs/filesystem" "github.com/spf13/afero" ) @@ -18,7 +19,7 @@ type testInstance struct { t *testing.T location string fs afero.Fs - closer filesystem.Virtualization + closer win.Virtualization closechan chan bool } diff --git a/win/projfs/functions.go b/win/projfs/functions.go index 2b106d1..d3c6490 100644 --- a/win/projfs/functions.go +++ b/win/projfs/functions.go @@ -3,7 +3,6 @@ package projfs import ( - "fmt" "syscall" "unsafe" @@ -33,14 +32,6 @@ var ( prjWritePlaceholderInfo2 = projectedfslib.NewProc("PrjWritePlaceholderInfo2") ) -func ErrorByCode(result uintptr) error { - if result == 0 { - return nil - } else { - return fmt.Errorf("error result: %x", result) - } -} - func PrjAllocateAlignedBuffer(namespaceVirtualizationContext PRJ_NAMESPACE_VIRTUALIZATION_CONTEXT, size uint32) uintptr { res, _, _ := prjAllocateAlignedBuffer.Call(uintptr(namespaceVirtualizationContext), uintptr(size)) return res diff --git a/win/virtualization.go b/win/virtualization.go new file mode 100644 index 0000000..6b04a80 --- /dev/null +++ b/win/virtualization.go @@ -0,0 +1,21 @@ +package win + +import ( + "encoding/binary" + "io" + "syscall" +) + +type Virtualization interface { + io.Closer + PerformSynchronization() error +} + +func BytesToGuid(b []byte) *syscall.GUID { + return &syscall.GUID{ + Data1: binary.LittleEndian.Uint32(b[0:4]), + Data2: binary.LittleEndian.Uint16(b[4:6]), + Data3: binary.LittleEndian.Uint16(b[6:8]), + Data4: ([8]byte)(b[8:16]), + } +}