feat: implement backups for external files during deployment

go.mod

@@ -20,7 +20,7 @@ require (
 	github.com/knadh/koanf v1.4.4
 	github.com/phayes/freeport v0.0.0-20220201140144-74d24b5ae9f5
 	github.com/saitho/diff-docker-compose v1.1.3
-	github.com/saitho/golang-extended-fs/v2 v2.0.2
+	github.com/saitho/golang-extended-fs/v2 v2.1.0
 	github.com/shibukawa/configdir v0.0.0-20170330084843-e180dbdc8da0
 	github.com/sirupsen/logrus v1.9.2
 	github.com/spf13/cast v1.3.1

go.sum

@@ -862,8 +862,8 @@ github.com/ryanuber/go-glob v1.0.0/go.mod h1:807d1WSdnB0XRJzKNil9Om6lcp/3a0v4qIH
 github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46 h1:GHRpF1pTW19a8tTFrMLUcfWwyC0pnifVo2ClaLq+hP8=
 github.com/saitho/diff-docker-compose v1.1.3 h1:ZKx+F7yMmoAKzI76BFpr7kY+Thd7quk7d/ETug0qNY8=
 github.com/saitho/diff-docker-compose v1.1.3/go.mod h1:O3V+mwGtlXQ7UERA4+yunV319kyNLwIKeNSw8PODyEU=
-github.com/saitho/golang-extended-fs/v2 v2.0.2 h1:QWGOIP4wFTaSODIU54qDCuH1IQXWcjL6irLezJiHqOA=
-github.com/saitho/golang-extended-fs/v2 v2.0.2/go.mod h1:aTmESz9Z7Rwbx80zxGWTeuBmW6MLdzYrBZMJOfXsJVw=
+github.com/saitho/golang-extended-fs/v2 v2.1.0 h1:j1X3hu5LQRUM0WQzis8Kyh6zZhqCjYQox7Gw1h/ruX0=
+github.com/saitho/golang-extended-fs/v2 v2.1.0/go.mod h1:aTmESz9Z7Rwbx80zxGWTeuBmW6MLdzYrBZMJOfXsJVw=
 github.com/saitho/jsonschema-validator v1.2.0 h1:bWSvTla54F5cziZsxLBR0mlN3gTw9hjrkFuZU55kO+E=
 github.com/saitho/jsonschema-validator v1.2.0/go.mod h1:W/1Q1xQ+vyYxANlTEpi6hQBEHJPEi4wJmCBkdIehFvQ=
 github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0=

modules/container/docker/deploy.go

@@ -147,7 +147,7 @@ func (m Module) Deploy(modulesSettings interface{}) error {
 	oldComposeFilePath := ""
 	var remoteComposeObjMap map[string]interface{}
 	if system.Context.LatestDeployment != nil {
-		dockerComposeFilePathOld, err := system.Context.LatestDeployment.GetResourcePath(dockerComposeResource)
+		dockerComposeFilePathOld, err := system.Context.LatestDeployment.GetResourcePath(&dockerComposeResource)
 		if err != nil {
 			return err
 		}
@@ -166,7 +166,7 @@ func (m Module) Deploy(modulesSettings interface{}) error {
 			if err != nil {
 				return fmt.Errorf("unable to process remote docker-compose.yaml file from previous deployment: " + err.Error())
 			}
-		} else if err != nil && err.Error() != "file does not exist" {
+		} else if err != nil {
 			return fmt.Errorf("Unable to check state of remote docker-compose.yaml from previous deployment: " + err.Error())
 		}
 	}
@@ -190,7 +190,7 @@ func (m Module) Deploy(modulesSettings interface{}) error {
 	}
 	dockerComposeResource.Content = composeFileContent
 
-	dockerComposeFilePathNew, _ := system.Context.CurrentDeployment.GetResourcePath(dockerComposeResource)
+	dockerComposeFilePathNew, _ := system.Context.CurrentDeployment.GetResourcePath(&dockerComposeResource)
 
 	system.Context.CurrentDeployment.ResourceGroups = append(system.Context.CurrentDeployment.ResourceGroups, system.ResourceGroup{
 		Name:      "container-docker-" + system.Context.Project.Name + "-composefile",

modules/proxy/caddy/deploy.go

@@ -24,7 +24,7 @@ func (Module) Deploy(modulesSettings interface{}) error {
 		Content:   caddyDirectives,
 	}
 
-	caddyFilePath, err := system.Context.CurrentDeployment.GetResourcePath(caddyFileResource)
+	caddyFilePath, err := system.Context.CurrentDeployment.GetResourcePath(&caddyFileResource)
 	if err != nil {
 		return err
 	}

modules/proxy/nginx/deploy.go

@@ -124,7 +124,7 @@ func (Module) Deploy(_modulesSettings interface{}) error {
 		Name:      "nginx.conf",
 		Content:   serverConfig,
 	}
-	nginxConfigResourcePath, _ := system.Context.CurrentDeployment.GetResourcePath(nginxConfigResource)
+	nginxConfigResourcePath, _ := system.Context.CurrentDeployment.GetResourcePath(&nginxConfigResource)
 	system.Context.CurrentDeployment.ResourceGroups = append(system.Context.CurrentDeployment.ResourceGroups, system.ResourceGroup{
 		Name: "proxy-nginx-" + system.Context.Project.Name,
 		Resources: []system.Resource{

routines/implementations.go

@@ -147,9 +147,9 @@ func processResourceGroup(taskRunner *TaskRunner, resourceGroup system.ResourceG
 		var err error
 		var processed bool
 		if isRollbackMode {
-			processed, err = system.RollbackResourceOperation(resource, ignoreBackup)
+			processed, err = system.RollbackResourceOperation(&resource, ignoreBackup)
 		} else {
-			processed, err = system.ApplyResourceOperation(resource, ignoreBackup)
+			processed, err = system.ApplyResourceOperation(&resource, ignoreBackup)
 		}
 		if err != nil {
 			if spinner != nil {
@@ -260,8 +260,15 @@ var FinalizeDeployment = Task{
 			return err
 		}
 
-		// update current symlink if deployment was successful
 		if !system.Context.CurrentDeployment.RolledBack {
+			// Remove external backups
+			for _, resourceGroup := range system.Context.CurrentDeployment.ResourceGroups {
+				for _, resource := range resourceGroup.Resources {
+					fmt.Println(resource.BackupFilePath) // todo: remove
+				}
+			}
+
+			// update current symlink if deployment was successful
 			if _, err := system.SimpleRemoteRun("ln", system.RemoteRunOpts{Args: []string{"-sfn " + system.Context.CurrentDeployment.GetPath() + " " + path.Join(system.Context.CurrentDeployment.Project.GetDeploymentsPath(), "current")}}); err != nil {
 				return fmt.Errorf("Unable to symlink current deployment: " + err.Error())
 			}

system/context.go

@@ -43,9 +43,9 @@ type Deployment struct {
 	ResourceGroups []ResourceGroup
 }
 
-func (d Deployment) GetResourcePath(resource Resource) (string, error) {
+func (d Deployment) GetResourcePath(resource *Resource) (string, error) {
 	if resource.Type != TypeFile && resource.Type != TypeFolder && resource.Type != TypeLink {
-		return "", fmt.Errorf("not a file, folder or link resource")
+		return "", fmt.Errorf("unsupported resouce type \"%s\". expected file, folder or link", resource.Type)
 	}
 	if resource.ExternalResource {
 		if !path.IsAbs(resource.Name) {

system/resource.go

@@ -35,8 +35,9 @@ type ResourceGroup struct {
 }
 
 type Resource struct {
-	Type      Type
-	Operation Operation `yaml:"-"`
+	Type           Type
+	Operation      Operation `yaml:"-"`
+	BackupFilePath string    `yaml:"-"`
 
 	// if set the Name refers to an external resource. for files an absolute path is expected
 	ExternalResource bool `yaml:"externalResource,omitempty"`

system/resource_manager.go

@@ -2,35 +2,147 @@ package system
 
 import (
 	"fmt"
+	log "github.com/sirupsen/logrus"
 
 	xfs "github.com/saitho/golang-extended-fs/v2"
 )
 
-func ApplyResourceOperation(resource Resource, ignoreBackup bool) (bool, error) {
-	return PerformOperation(resource, ignoreBackup)
+func ApplyResourceOperation(resource *Resource, ignoreBackup bool) (bool, error) {
+	if !ignoreBackup {
+		// Backup existing file
+		backupPath, err := backupResource(resource)
+		if err != nil {
+			return true, err
+		}
+		fmt.Println(backupPath)
+	}
+	return PerformOperation(resource)
 }
 
-func RollbackResourceOperation(resource Resource, ignoreBackup bool) (bool, error) {
+func RollbackResourceOperation(resource *Resource, ignoreBackup bool) (bool, error) {
 	if resource.Operation == OperationCreate {
 		resource.Operation = OperationDelete
-		return PerformOperation(resource, ignoreBackup)
+		found, err := PerformOperation(resource)
+		if err != nil {
+			return found, err
+		}
+		if !ignoreBackup {
+			// Restore backup
+			if err = restoreBackup(resource); err != nil {
+				return found, err
+			}
+		}
+		return found, err
 	}
 	return true, fmt.Errorf(fmt.Sprintf("unupported rollback for operation %s", resource.Operation))
 }
 
-func PerformOperation(resource Resource, ignoreBackup bool) (bool, error) {
+func backupResource(resource *Resource) (string, error) {
+	//  && resource.Type != TypeLink todo: make it available for symlinks again
+	// issue with symlinks: cannot stat symlink: permission denied
+	if resource.Type != TypeFile && resource.Type != TypeFolder {
+		return "", nil
+	}
+	if !resource.ExternalResource {
+		return "", nil
+	}
+	resourceFilePath, err := Context.CurrentDeployment.GetResourcePath(resource)
+	if err != nil {
+		return "", err
+	}
+	log.Info("Creating backup of resource " + resourceFilePath)
+	backupFilePath := resourceFilePath + ".bak"
+	xfsFilePath := "ssh://" + resourceFilePath
+	switch resource.Type {
+	case TypeFile:
+		hasFile, err := xfs.HasFile(xfsFilePath)
+		if err != nil {
+			return "", fmt.Errorf("unable to check status of file %s: %s", resourceFilePath, err)
+		}
+		if !hasFile {
+			return "", nil
+		}
+		if _, err = SimpleRemoteRun("cp", RemoteRunOpts{Args: []string{resourceFilePath, backupFilePath}}); err != nil {
+			return backupFilePath, fmt.Errorf("unable to backup file %s: %s", resourceFilePath, err)
+		}
+		return backupFilePath, nil
+	case TypeLink:
+		hasFile, err := xfs.HasLink(xfsFilePath)
+		if err != nil {
+			return "", fmt.Errorf("unable to check status of link %s: %s", resourceFilePath, err)
+		}
+		if !hasFile {
+			return "", nil
+		}
+		if _, err = SimpleRemoteRun("cp", RemoteRunOpts{Args: []string{resourceFilePath, backupFilePath}}); err != nil {
+			return backupFilePath, fmt.Errorf("unable to backup link %s: %s", resourceFilePath, err)
+		}
+		return backupFilePath, nil
+	case TypeFolder:
+		hasFolder, err := xfs.HasFolder(xfsFilePath)
+		if err != nil {
+			return "", fmt.Errorf("unable to check status of folder %s: %s", resourceFilePath, err)
+		}
+		if !hasFolder {
+			return "", nil
+		}
+		if _, err = SimpleRemoteRun("cp", RemoteRunOpts{Args: []string{"-R", resourceFilePath, backupFilePath}}); err != nil {
+			return backupFilePath, fmt.Errorf("unable to backup folder %s: %s", resourceFilePath, err)
+		}
+		return backupFilePath, nil
+	}
+	return "", fmt.Errorf("unknown backup handler for resource type %s", resource.Type)
+}
+
+func restoreBackup(resource *Resource) error {
+	if resource.Type != TypeFile && resource.Type != TypeFolder && resource.Type != TypeLink {
+		return nil
+	}
+	if resource.BackupFilePath == "" {
+		return nil
+	}
 	resourceFilePath, _ := Context.CurrentDeployment.GetResourcePath(resource)
+	xfsBackupFilePath := "ssh://" + resource.BackupFilePath
+	log.Info("Restoring backup of resource " + resourceFilePath)
+
+	switch resource.Type {
+	case TypeFile, TypeLink:
+		hasFile, err := xfs.HasFile(xfsBackupFilePath)
+		if err != nil {
+			return err
+		}
+		if !hasFile {
+			return fmt.Errorf("backup not found for " + resource.Name)
+		}
+		return xfs.CopyFile(xfsBackupFilePath, "ssh://"+resourceFilePath)
+	case TypeFolder:
+		backupFileName := "ssh://" + resourceFilePath + ".bak"
+		hasFolder, err := xfs.HasFolder(backupFileName)
+		if err != nil {
+			return err
+		}
+		if !hasFolder {
+			return fmt.Errorf("backup not found for " + resource.Name)
+		}
+		if _, err = SimpleRemoteRun("cp", RemoteRunOpts{Args: []string{"-R", backupFileName, resourceFilePath}}); err != nil {
+			return err
+		}
+		return nil
+	}
+	return fmt.Errorf("unknown restore backup handler for resource type %s", resource.Type)
+}
+
+func PerformOperation(resource *Resource) (bool, error) {
+	resourceFilePath, _ := Context.CurrentDeployment.GetResourcePath(resource)
+	xfsResourceFilePath := "ssh://" + resourceFilePath
 	switch resource.Type {
 	case TypeFile:
 		if resource.Operation == OperationCreate {
-			// TODO: backup if file exists
-			if err := xfs.WriteFile("ssh://"+resourceFilePath, resource.Content); err != nil {
+			if err := xfs.WriteFile(xfsResourceFilePath, resource.Content); err != nil {
 				return true, fmt.Errorf("unable to create file at %s: %s", resource.Name, err)
 			}
 		} else if resource.Operation == OperationDelete {
-			// TODO: restore backup if file exists
-			resourcePath, _ := Context.CurrentDeployment.GetResourcePath(resource)
-			if err := xfs.DeleteFile("ssh://" + resourcePath); err != nil {
+			if err := xfs.DeleteFile(xfsResourceFilePath); err != nil {
 				if err.Error() == "file does not exist" {
 					return true, nil
 				}
@@ -40,13 +152,11 @@ func PerformOperation(resource Resource, ignoreBackup bool) (bool, error) {
 		return true, nil
 	case TypeFolder:
 		if resource.Operation == OperationCreate {
-			// TODO: backup if file exists
-			if err := xfs.CreateFolder("ssh://" + resourceFilePath); err != nil {
+			if err := xfs.CreateFolder(xfsResourceFilePath); err != nil {
 				return true, fmt.Errorf("unable to create folder at %s: %s", resource.Name, err)
 			}
 		} else if resource.Operation == OperationDelete {
-			// TODO: restore backup if file exists
-			if err := xfs.DeleteFolder("ssh://"+resourceFilePath, true); err != nil {
+			if err := xfs.DeleteFolder(xfsResourceFilePath, true); err != nil {
 				return true, fmt.Errorf("unable to remove folder at %s: %s", resource.Name, err)
 			}
 		}
@@ -61,14 +171,14 @@ func PerformOperation(resource Resource, ignoreBackup bool) (bool, error) {
 				return true, fmt.Errorf("Unable to symlink " + resource.LinkSource + " -> " + resourceFilePath + ": " + err.Error())
 			}
 		} else if resource.Operation == OperationDelete {
-			// TODO: restore backup if file exists
-			if err := xfs.DeleteFile("ssh://" + resourceFilePath); err != nil {
+			if err := xfs.DeleteFile(xfsResourceFilePath); err != nil {
 				if err.Error() == "file does not exist" {
 					return true, nil
 				}
 				return true, fmt.Errorf("unable to remove symlink at %s: %s", resource.Name, err)
 			}
 		}
+		return true, nil
 	}
 	// CONTAINER via ResourceGroup (see StackHead container module)
 	return false, nil