From e14beaca2fdb111c88b8515a070dfcaaa37e3c38 Mon Sep 17 00:00:00 2001 From: Mario Lubenka Date: Sun, 28 May 2023 22:39:20 +0200 Subject: [PATCH] feat: implement backups for external files during deployment --- go.mod | 2 +- go.sum | 4 +- modules/container/docker/deploy.go | 6 +- modules/proxy/caddy/deploy.go | 2 +- modules/proxy/nginx/deploy.go | 2 +- routines/implementations.go | 13 ++- system/context.go | 4 +- system/resource.go | 5 +- system/resource_manager.go | 142 +++++++++++++++++++++++++---- 9 files changed, 149 insertions(+), 31 deletions(-) diff --git a/go.mod b/go.mod index b75e5c5a..cf5c214f 100644 --- a/go.mod +++ b/go.mod @@ -20,7 +20,7 @@ require ( github.com/knadh/koanf v1.4.4 github.com/phayes/freeport v0.0.0-20220201140144-74d24b5ae9f5 github.com/saitho/diff-docker-compose v1.1.3 - github.com/saitho/golang-extended-fs/v2 v2.0.2 + github.com/saitho/golang-extended-fs/v2 v2.1.0 github.com/shibukawa/configdir v0.0.0-20170330084843-e180dbdc8da0 github.com/sirupsen/logrus v1.9.2 github.com/spf13/cast v1.3.1 diff --git a/go.sum b/go.sum index d06fa094..02d5e14f 100644 --- a/go.sum +++ b/go.sum @@ -862,8 +862,8 @@ github.com/ryanuber/go-glob v1.0.0/go.mod h1:807d1WSdnB0XRJzKNil9Om6lcp/3a0v4qIH github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46 h1:GHRpF1pTW19a8tTFrMLUcfWwyC0pnifVo2ClaLq+hP8= github.com/saitho/diff-docker-compose v1.1.3 h1:ZKx+F7yMmoAKzI76BFpr7kY+Thd7quk7d/ETug0qNY8= github.com/saitho/diff-docker-compose v1.1.3/go.mod h1:O3V+mwGtlXQ7UERA4+yunV319kyNLwIKeNSw8PODyEU= -github.com/saitho/golang-extended-fs/v2 v2.0.2 h1:QWGOIP4wFTaSODIU54qDCuH1IQXWcjL6irLezJiHqOA= -github.com/saitho/golang-extended-fs/v2 v2.0.2/go.mod h1:aTmESz9Z7Rwbx80zxGWTeuBmW6MLdzYrBZMJOfXsJVw= +github.com/saitho/golang-extended-fs/v2 v2.1.0 h1:j1X3hu5LQRUM0WQzis8Kyh6zZhqCjYQox7Gw1h/ruX0= +github.com/saitho/golang-extended-fs/v2 v2.1.0/go.mod h1:aTmESz9Z7Rwbx80zxGWTeuBmW6MLdzYrBZMJOfXsJVw= github.com/saitho/jsonschema-validator v1.2.0 h1:bWSvTla54F5cziZsxLBR0mlN3gTw9hjrkFuZU55kO+E= github.com/saitho/jsonschema-validator v1.2.0/go.mod h1:W/1Q1xQ+vyYxANlTEpi6hQBEHJPEi4wJmCBkdIehFvQ= github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= diff --git a/modules/container/docker/deploy.go b/modules/container/docker/deploy.go index 51b151e5..f18757eb 100644 --- a/modules/container/docker/deploy.go +++ b/modules/container/docker/deploy.go @@ -147,7 +147,7 @@ func (m Module) Deploy(modulesSettings interface{}) error { oldComposeFilePath := "" var remoteComposeObjMap map[string]interface{} if system.Context.LatestDeployment != nil { - dockerComposeFilePathOld, err := system.Context.LatestDeployment.GetResourcePath(dockerComposeResource) + dockerComposeFilePathOld, err := system.Context.LatestDeployment.GetResourcePath(&dockerComposeResource) if err != nil { return err } @@ -166,7 +166,7 @@ func (m Module) Deploy(modulesSettings interface{}) error { if err != nil { return fmt.Errorf("unable to process remote docker-compose.yaml file from previous deployment: " + err.Error()) } - } else if err != nil && err.Error() != "file does not exist" { + } else if err != nil { return fmt.Errorf("Unable to check state of remote docker-compose.yaml from previous deployment: " + err.Error()) } } @@ -190,7 +190,7 @@ func (m Module) Deploy(modulesSettings interface{}) error { } dockerComposeResource.Content = composeFileContent - dockerComposeFilePathNew, _ := system.Context.CurrentDeployment.GetResourcePath(dockerComposeResource) + dockerComposeFilePathNew, _ := system.Context.CurrentDeployment.GetResourcePath(&dockerComposeResource) system.Context.CurrentDeployment.ResourceGroups = append(system.Context.CurrentDeployment.ResourceGroups, system.ResourceGroup{ Name: "container-docker-" + system.Context.Project.Name + "-composefile", diff --git a/modules/proxy/caddy/deploy.go b/modules/proxy/caddy/deploy.go index 2e516a3f..56a2d4b2 100644 --- a/modules/proxy/caddy/deploy.go +++ b/modules/proxy/caddy/deploy.go @@ -24,7 +24,7 @@ func (Module) Deploy(modulesSettings interface{}) error { Content: caddyDirectives, } - caddyFilePath, err := system.Context.CurrentDeployment.GetResourcePath(caddyFileResource) + caddyFilePath, err := system.Context.CurrentDeployment.GetResourcePath(&caddyFileResource) if err != nil { return err } diff --git a/modules/proxy/nginx/deploy.go b/modules/proxy/nginx/deploy.go index ab7d6a62..0c71de7e 100644 --- a/modules/proxy/nginx/deploy.go +++ b/modules/proxy/nginx/deploy.go @@ -124,7 +124,7 @@ func (Module) Deploy(_modulesSettings interface{}) error { Name: "nginx.conf", Content: serverConfig, } - nginxConfigResourcePath, _ := system.Context.CurrentDeployment.GetResourcePath(nginxConfigResource) + nginxConfigResourcePath, _ := system.Context.CurrentDeployment.GetResourcePath(&nginxConfigResource) system.Context.CurrentDeployment.ResourceGroups = append(system.Context.CurrentDeployment.ResourceGroups, system.ResourceGroup{ Name: "proxy-nginx-" + system.Context.Project.Name, Resources: []system.Resource{ diff --git a/routines/implementations.go b/routines/implementations.go index c2f1473d..fe8af2d8 100644 --- a/routines/implementations.go +++ b/routines/implementations.go @@ -147,9 +147,9 @@ func processResourceGroup(taskRunner *TaskRunner, resourceGroup system.ResourceG var err error var processed bool if isRollbackMode { - processed, err = system.RollbackResourceOperation(resource, ignoreBackup) + processed, err = system.RollbackResourceOperation(&resource, ignoreBackup) } else { - processed, err = system.ApplyResourceOperation(resource, ignoreBackup) + processed, err = system.ApplyResourceOperation(&resource, ignoreBackup) } if err != nil { if spinner != nil { @@ -260,8 +260,15 @@ var FinalizeDeployment = Task{ return err } - // update current symlink if deployment was successful if !system.Context.CurrentDeployment.RolledBack { + // Remove external backups + for _, resourceGroup := range system.Context.CurrentDeployment.ResourceGroups { + for _, resource := range resourceGroup.Resources { + fmt.Println(resource.BackupFilePath) // todo: remove + } + } + + // update current symlink if deployment was successful if _, err := system.SimpleRemoteRun("ln", system.RemoteRunOpts{Args: []string{"-sfn " + system.Context.CurrentDeployment.GetPath() + " " + path.Join(system.Context.CurrentDeployment.Project.GetDeploymentsPath(), "current")}}); err != nil { return fmt.Errorf("Unable to symlink current deployment: " + err.Error()) } diff --git a/system/context.go b/system/context.go index 1a6598ed..977b110d 100644 --- a/system/context.go +++ b/system/context.go @@ -43,9 +43,9 @@ type Deployment struct { ResourceGroups []ResourceGroup } -func (d Deployment) GetResourcePath(resource Resource) (string, error) { +func (d Deployment) GetResourcePath(resource *Resource) (string, error) { if resource.Type != TypeFile && resource.Type != TypeFolder && resource.Type != TypeLink { - return "", fmt.Errorf("not a file, folder or link resource") + return "", fmt.Errorf("unsupported resouce type \"%s\". expected file, folder or link", resource.Type) } if resource.ExternalResource { if !path.IsAbs(resource.Name) { diff --git a/system/resource.go b/system/resource.go index cbbbad18..1b04536c 100644 --- a/system/resource.go +++ b/system/resource.go @@ -35,8 +35,9 @@ type ResourceGroup struct { } type Resource struct { - Type Type - Operation Operation `yaml:"-"` + Type Type + Operation Operation `yaml:"-"` + BackupFilePath string `yaml:"-"` // if set the Name refers to an external resource. for files an absolute path is expected ExternalResource bool `yaml:"externalResource,omitempty"` diff --git a/system/resource_manager.go b/system/resource_manager.go index 1b894522..1ad20045 100644 --- a/system/resource_manager.go +++ b/system/resource_manager.go @@ -2,35 +2,147 @@ package system import ( "fmt" + log "github.com/sirupsen/logrus" xfs "github.com/saitho/golang-extended-fs/v2" ) -func ApplyResourceOperation(resource Resource, ignoreBackup bool) (bool, error) { - return PerformOperation(resource, ignoreBackup) +func ApplyResourceOperation(resource *Resource, ignoreBackup bool) (bool, error) { + if !ignoreBackup { + // Backup existing file + backupPath, err := backupResource(resource) + if err != nil { + return true, err + } + fmt.Println(backupPath) + } + return PerformOperation(resource) } -func RollbackResourceOperation(resource Resource, ignoreBackup bool) (bool, error) { +func RollbackResourceOperation(resource *Resource, ignoreBackup bool) (bool, error) { if resource.Operation == OperationCreate { resource.Operation = OperationDelete - return PerformOperation(resource, ignoreBackup) + found, err := PerformOperation(resource) + if err != nil { + return found, err + } + if !ignoreBackup { + // Restore backup + if err = restoreBackup(resource); err != nil { + return found, err + } + } + return found, err } return true, fmt.Errorf(fmt.Sprintf("unupported rollback for operation %s", resource.Operation)) } -func PerformOperation(resource Resource, ignoreBackup bool) (bool, error) { +func backupResource(resource *Resource) (string, error) { + // && resource.Type != TypeLink todo: make it available for symlinks again + // issue with symlinks: cannot stat symlink: permission denied + if resource.Type != TypeFile && resource.Type != TypeFolder { + return "", nil + } + if !resource.ExternalResource { + return "", nil + } + resourceFilePath, err := Context.CurrentDeployment.GetResourcePath(resource) + if err != nil { + return "", err + } + log.Info("Creating backup of resource " + resourceFilePath) + backupFilePath := resourceFilePath + ".bak" + xfsFilePath := "ssh://" + resourceFilePath + switch resource.Type { + case TypeFile: + hasFile, err := xfs.HasFile(xfsFilePath) + if err != nil { + return "", fmt.Errorf("unable to check status of file %s: %s", resourceFilePath, err) + } + if !hasFile { + return "", nil + } + if _, err = SimpleRemoteRun("cp", RemoteRunOpts{Args: []string{resourceFilePath, backupFilePath}}); err != nil { + return backupFilePath, fmt.Errorf("unable to backup file %s: %s", resourceFilePath, err) + } + return backupFilePath, nil + case TypeLink: + hasFile, err := xfs.HasLink(xfsFilePath) + if err != nil { + return "", fmt.Errorf("unable to check status of link %s: %s", resourceFilePath, err) + } + if !hasFile { + return "", nil + } + if _, err = SimpleRemoteRun("cp", RemoteRunOpts{Args: []string{resourceFilePath, backupFilePath}}); err != nil { + return backupFilePath, fmt.Errorf("unable to backup link %s: %s", resourceFilePath, err) + } + return backupFilePath, nil + case TypeFolder: + hasFolder, err := xfs.HasFolder(xfsFilePath) + if err != nil { + return "", fmt.Errorf("unable to check status of folder %s: %s", resourceFilePath, err) + } + if !hasFolder { + return "", nil + } + if _, err = SimpleRemoteRun("cp", RemoteRunOpts{Args: []string{"-R", resourceFilePath, backupFilePath}}); err != nil { + return backupFilePath, fmt.Errorf("unable to backup folder %s: %s", resourceFilePath, err) + } + return backupFilePath, nil + } + return "", fmt.Errorf("unknown backup handler for resource type %s", resource.Type) +} + +func restoreBackup(resource *Resource) error { + if resource.Type != TypeFile && resource.Type != TypeFolder && resource.Type != TypeLink { + return nil + } + if resource.BackupFilePath == "" { + return nil + } resourceFilePath, _ := Context.CurrentDeployment.GetResourcePath(resource) + xfsBackupFilePath := "ssh://" + resource.BackupFilePath + log.Info("Restoring backup of resource " + resourceFilePath) + + switch resource.Type { + case TypeFile, TypeLink: + hasFile, err := xfs.HasFile(xfsBackupFilePath) + if err != nil { + return err + } + if !hasFile { + return fmt.Errorf("backup not found for " + resource.Name) + } + return xfs.CopyFile(xfsBackupFilePath, "ssh://"+resourceFilePath) + case TypeFolder: + backupFileName := "ssh://" + resourceFilePath + ".bak" + hasFolder, err := xfs.HasFolder(backupFileName) + if err != nil { + return err + } + if !hasFolder { + return fmt.Errorf("backup not found for " + resource.Name) + } + if _, err = SimpleRemoteRun("cp", RemoteRunOpts{Args: []string{"-R", backupFileName, resourceFilePath}}); err != nil { + return err + } + return nil + } + return fmt.Errorf("unknown restore backup handler for resource type %s", resource.Type) +} + +func PerformOperation(resource *Resource) (bool, error) { + resourceFilePath, _ := Context.CurrentDeployment.GetResourcePath(resource) + xfsResourceFilePath := "ssh://" + resourceFilePath switch resource.Type { case TypeFile: if resource.Operation == OperationCreate { - // TODO: backup if file exists - if err := xfs.WriteFile("ssh://"+resourceFilePath, resource.Content); err != nil { + if err := xfs.WriteFile(xfsResourceFilePath, resource.Content); err != nil { return true, fmt.Errorf("unable to create file at %s: %s", resource.Name, err) } } else if resource.Operation == OperationDelete { - // TODO: restore backup if file exists - resourcePath, _ := Context.CurrentDeployment.GetResourcePath(resource) - if err := xfs.DeleteFile("ssh://" + resourcePath); err != nil { + if err := xfs.DeleteFile(xfsResourceFilePath); err != nil { if err.Error() == "file does not exist" { return true, nil } @@ -40,13 +152,11 @@ func PerformOperation(resource Resource, ignoreBackup bool) (bool, error) { return true, nil case TypeFolder: if resource.Operation == OperationCreate { - // TODO: backup if file exists - if err := xfs.CreateFolder("ssh://" + resourceFilePath); err != nil { + if err := xfs.CreateFolder(xfsResourceFilePath); err != nil { return true, fmt.Errorf("unable to create folder at %s: %s", resource.Name, err) } } else if resource.Operation == OperationDelete { - // TODO: restore backup if file exists - if err := xfs.DeleteFolder("ssh://"+resourceFilePath, true); err != nil { + if err := xfs.DeleteFolder(xfsResourceFilePath, true); err != nil { return true, fmt.Errorf("unable to remove folder at %s: %s", resource.Name, err) } } @@ -61,14 +171,14 @@ func PerformOperation(resource Resource, ignoreBackup bool) (bool, error) { return true, fmt.Errorf("Unable to symlink " + resource.LinkSource + " -> " + resourceFilePath + ": " + err.Error()) } } else if resource.Operation == OperationDelete { - // TODO: restore backup if file exists - if err := xfs.DeleteFile("ssh://" + resourceFilePath); err != nil { + if err := xfs.DeleteFile(xfsResourceFilePath); err != nil { if err.Error() == "file does not exist" { return true, nil } return true, fmt.Errorf("unable to remove symlink at %s: %s", resource.Name, err) } } + return true, nil } // CONTAINER via ResourceGroup (see StackHead container module) return false, nil