From e2053dcf70e34065abdfdc030452508c0435aa7b Mon Sep 17 00:00:00 2001 From: Mario Lubenka Date: Sat, 27 May 2023 21:18:08 +0200 Subject: [PATCH] feat: project deployment versions --- commands/project/deploy.go | 21 +++- .../container/docker/definitions/volume.go | 11 +- modules/container/docker/deploy.go | 89 +++++++++----- modules/container/docker/destroy.go | 2 +- .../container/docker/docker-compose/yaml.go | 9 +- modules/container/docker/system/hooks.go | 8 +- modules/container/docker/system/naming.go | 13 ++- modules/container/docker/system/ports.go | 2 +- .../docker/templates/project.tf.tmpl | 17 --- modules/proxy/caddy/deploy.go | 27 +++-- modules/proxy/nginx/certificates.go | 4 - modules/proxy/nginx/deploy.go | 110 +++++++++++------- modules/proxy/rendering.go | 2 +- project/funcs.go | 6 +- routines/implementations.go | 92 +++++++++++++-- system/context.go | 45 ++++++- system/deployments.go | 40 +++++++ system/resource.go | 24 ++-- system/resource_manager.go | 44 +++++-- 19 files changed, 409 insertions(+), 157 deletions(-) delete mode 100644 modules/container/docker/templates/project.tf.tmpl create mode 100644 system/deployments.go diff --git a/commands/project/deploy.go b/commands/project/deploy.go index 2b03473a..4ad37bf8 100644 --- a/commands/project/deploy.go +++ b/commands/project/deploy.go @@ -2,6 +2,7 @@ package project import ( "fmt" + xfs "github.com/saitho/golang-extended-fs/v2" "github.com/spf13/cobra" "golang.org/x/exp/slices" @@ -50,11 +51,11 @@ var DeployApplication = func() *cobra.Command { } if autoConfirm { - _ = taskRunner.RunTask(routines.CreateResources) + err = taskRunner.RunTask(routines.CreateResources) } else { // Confirm resource creation fmt.Println("\nStackHead will try to create or update the following resources:") - for _, resourceGroup := range system.Context.Resources { + for _, resourceGroup := range system.Context.CurrentDeployment.ResourceGroups { for _, resource := range resourceGroup.Resources { fmt.Println(fmt.Sprintf("- %s", resource.ToString(false))) } @@ -62,14 +63,28 @@ var DeployApplication = func() *cobra.Command { fmt.Println("") fmt.Print("Please confirm with \"y\" or \"yes\": ") if askForConfirmation() { - _ = taskRunner.RunTask(routines.CreateResources) + err = taskRunner.RunTask(routines.CreateResources) + } else { + // Abort deployment -> delete + if err := xfs.DeleteFolder("ssh://"+system.Context.CurrentDeployment.GetPath(), true); err != nil { + fmt.Print("Unable to remove deployment directory.") + return + } + fmt.Print("Deployment aborted.") + return } } + if err != nil { + // ensure rollback is performed due to errors caused by StackHead module Run scripts not resource creation + routines.RollbackResources.Disabled = false + } + if !noRollback { // Rollback may be skipped if CreateResources does not trigger a rollback _ = taskRunner.RunTask(routines.RollbackResources) } + _ = taskRunner.RunTask(routines.FinalizeDeployment) }, } command.PersistentFlags().BoolVar(&autoConfirm, "autoconfirm", false, "Whether to auto-confirm resource changes") diff --git a/modules/container/docker/definitions/volume.go b/modules/container/docker/definitions/volume.go index 190d27bd..5bfdbb1f 100644 --- a/modules/container/docker/definitions/volume.go +++ b/modules/container/docker/definitions/volume.go @@ -3,24 +3,27 @@ package container_docker_definitions import ( "github.com/getstackhead/stackhead/project" "github.com/getstackhead/stackhead/system" + "path" ) type DockerPaths struct { - BaseDir string + DataDir string + DeploymentDir string } func GetDockerPaths() DockerPaths { return DockerPaths{ - BaseDir: system.Context.Project.GetRuntimeDataDirectoryPath() + "/container", + DataDir: path.Join(system.Context.Project.GetRuntimeDataDirectoryPath(), "container"), + DeploymentDir: path.Join(system.Context.CurrentDeployment.GetPath(), "container"), } } func (p DockerPaths) GetHooksDir() string { - return p.BaseDir + "/hooks" + return path.Join(p.DeploymentDir, "hooks") } func (p DockerPaths) getDataDir() string { - return p.BaseDir + "/data" + return path.Join(p.DataDir, "data") } func (p DockerPaths) GetServiceDataDir(service project.ContainerService, volume project.ContainerServiceVolume) string { diff --git a/modules/container/docker/deploy.go b/modules/container/docker/deploy.go index 3f5b4dbd..51b151e5 100644 --- a/modules/container/docker/deploy.go +++ b/modules/container/docker/deploy.go @@ -138,25 +138,36 @@ func (m Module) Deploy(modulesSettings interface{}) error { return err } - composeFileRemotePath := system.Context.Project.GetDirectoryPath() + "/docker-compose.yaml" - - hasRemoteFile, err := xfs.HasFile("ssh://" + composeFileRemotePath) - if err != nil && err.Error() == "file does not exist" { - hasRemoteFile = false - } else if err != nil { - return fmt.Errorf("Unable to check state of remote docker-compose.yaml from previous deployment: " + err.Error()) + dockerComposeResource := system.Resource{ + Type: system.TypeFile, + Operation: system.OperationCreate, + Name: "docker-compose.yaml", } + oldComposeFilePath := "" var remoteComposeObjMap map[string]interface{} - if hasRemoteFile { - remoteComposeObj := docker_compose.DockerCompose{} - remoteComposeContent, err := xfs.ReadFile("ssh://" + composeFileRemotePath) - if err := yaml.Unmarshal([]byte(remoteComposeContent), &remoteComposeObj); err != nil { - return fmt.Errorf("unable to read remote docker-compose.yaml file from previous deployment: " + err.Error()) - } - remoteComposeObjMap, err = remoteComposeObj.Map() + if system.Context.LatestDeployment != nil { + dockerComposeFilePathOld, err := system.Context.LatestDeployment.GetResourcePath(dockerComposeResource) if err != nil { - return fmt.Errorf("unable to process remote docker-compose.yaml file from previous deployment: " + err.Error()) + return err + } + oldComposeFilePath = dockerComposeFilePathOld + hasRemoteFile, err := xfs.HasFile("ssh://" + oldComposeFilePath) + if err != nil && hasRemoteFile { + remoteComposeContent, err := xfs.ReadFile("ssh://" + oldComposeFilePath) + if err != nil { + return fmt.Errorf("unable to read remote docker-compose.yaml file from previous deployment: " + err.Error()) + } + remoteComposeObj := docker_compose.DockerCompose{} + if err := yaml.Unmarshal([]byte(remoteComposeContent), &remoteComposeObj); err != nil { + return fmt.Errorf("unable to read remote docker-compose.yaml file from previous deployment: " + err.Error()) + } + remoteComposeObjMap, err = remoteComposeObj.Map() + if err != nil { + return fmt.Errorf("unable to process remote docker-compose.yaml file from previous deployment: " + err.Error()) + } + } else if err != nil && err.Error() != "file does not exist" { + return fmt.Errorf("Unable to check state of remote docker-compose.yaml from previous deployment: " + err.Error()) } } @@ -177,17 +188,13 @@ func (m Module) Deploy(modulesSettings interface{}) error { if err != nil { return err } + dockerComposeResource.Content = composeFileContent - system.Context.Resources = append(system.Context.Resources, system.ResourceGroup{ - Name: "container-docker-" + system.Context.Project.Name + "-composefile", - Resources: []system.Resource{ - { - Type: system.TypeFile, - Operation: system.OperationCreate, - Name: composeFileRemotePath, - Content: composeFileContent, - }, - }, + dockerComposeFilePathNew, _ := system.Context.CurrentDeployment.GetResourcePath(dockerComposeResource) + + system.Context.CurrentDeployment.ResourceGroups = append(system.Context.CurrentDeployment.ResourceGroups, system.ResourceGroup{ + Name: "container-docker-" + system.Context.Project.Name + "-composefile", + Resources: []system.Resource{dockerComposeResource}, }) var containerResources []system.Resource @@ -202,14 +209,24 @@ func (m Module) Deploy(modulesSettings interface{}) error { }) } - system.Context.Resources = append(system.Context.Resources, system.ResourceGroup{ + system.Context.CurrentDeployment.ResourceGroups = append(system.Context.CurrentDeployment.ResourceGroups, system.ResourceGroup{ Name: "container-docker-" + system.Context.Project.Name + "-containers", Resources: containerResources, ApplyResourceFunc: func() error { + if oldComposeFilePath != "" { + // Stop old Docker Compose containers + // todo: allow using either docker-compose or "docker compose" whichever is available (prefer "docker compose") + if _, stderr, err := system.RemoteRun("docker compose", system.RemoteRunOpts{Args: []string{"down"}, WorkingDir: path.Dir(oldComposeFilePath)}); err != nil { + if stderr.Len() > 0 { + return fmt.Errorf("Unable to stop old Docker containers: " + stderr.String()) + } + return fmt.Errorf("Unable to stop old Docker containers: " + err.Error()) + } + } + // Start Docker Compose // todo: allow using either docker-compose or "docker compose" whichever is available (prefer "docker compose") - _, stderr, err := system.RemoteRun("docker compose", system.RemoteRunOpts{Args: []string{"up", "-d"}, WorkingDir: system.Context.Project.GetDirectoryPath()}) - if err != nil { + if _, stderr, err := system.RemoteRun("docker compose", system.RemoteRunOpts{Args: []string{"up", "-d"}, WorkingDir: path.Dir(dockerComposeFilePathNew)}); err != nil { if stderr.Len() > 0 { return fmt.Errorf("Unable to start Docker containers: " + stderr.String()) } @@ -223,14 +240,24 @@ func (m Module) Deploy(modulesSettings interface{}) error { return nil }, RollbackResourceFunc: func() error { + // Start old containers again + if oldComposeFilePath != "" { + // todo: allow using either docker-compose or "docker compose" whichever is available (prefer "docker compose") + if _, stderr, err := system.RemoteRun("docker compose", system.RemoteRunOpts{Args: []string{"up", "-d"}, WorkingDir: path.Dir(oldComposeFilePath)}); err != nil { + if stderr.Len() > 0 { + return fmt.Errorf("Unable to stop Docker containers: " + stderr.String()) + } + return fmt.Errorf("Unable to start old Docker containers: " + err.Error()) + } + } + // Stop Docker Compose // todo: allow using either docker-compose or "docker compose" whichever is available (prefer "docker compose") - _, stderr, err := system.RemoteRun("docker compose", system.RemoteRunOpts{Args: []string{"down"}, WorkingDir: system.Context.Project.GetDirectoryPath()}) - if err != nil { + if _, stderr, err := system.RemoteRun("docker compose", system.RemoteRunOpts{Args: []string{"down"}, WorkingDir: path.Dir(dockerComposeFilePathNew)}); err != nil { if stderr.Len() > 0 { return fmt.Errorf("Unable to stop Docker containers: " + stderr.String()) } - return fmt.Errorf("Unable to stop Docker containers: " + err.Error()) + return fmt.Errorf("Unable to stop new Docker containers: " + err.Error()) } return nil }, diff --git a/modules/container/docker/destroy.go b/modules/container/docker/destroy.go index 8a276ff2..8203b706 100644 --- a/modules/container/docker/destroy.go +++ b/modules/container/docker/destroy.go @@ -15,7 +15,7 @@ func (m Module) Destroy(modulesSettings interface{}) error { // Stop and remove containers // todo: allow using either docker-compose or "docker compose" whichever is available (prefer "docker compose") - _, stderr, err := system.RemoteRun("docker compose", system.RemoteRunOpts{Args: []string{"down"}, WorkingDir: system.Context.Project.GetDirectoryPath()}) + _, stderr, err := system.RemoteRun("docker compose", system.RemoteRunOpts{Args: []string{"down"}, WorkingDir: system.Context.CurrentDeployment.GetPath()}) if err != nil { if stderr.Len() > 0 { return fmt.Errorf("Unable to stop Docker containers: " + stderr.String()) diff --git a/modules/container/docker/docker-compose/yaml.go b/modules/container/docker/docker-compose/yaml.go index 2b5292a4..8b6a311a 100644 --- a/modules/container/docker/docker-compose/yaml.go +++ b/modules/container/docker/docker-compose/yaml.go @@ -17,7 +17,7 @@ func BuildDockerCompose(project *project.Project) (DockerCompose, error) { dockerPaths := container_docker_definitions.GetDockerPaths() compose := DockerCompose{ Version: "2.4", - Networks: map[string]Network{"stackhead-network-" + project.Name: {}}, + Networks: map[string]Network{docker_system.NetworkName(project.Name, system.Context.CurrentDeployment): {}}, Services: map[string]Services{}, Volumes: map[string]Volume{}, } @@ -36,7 +36,6 @@ func BuildDockerCompose(project *project.Project) (DockerCompose, error) { continue } vol := Volume{} - serviceName := service.Name vol.DriverOpts.Type = "none" vol.DriverOpts.O = "bind" if volume.Type == "local" { @@ -46,7 +45,7 @@ func BuildDockerCompose(project *project.Project) (DockerCompose, error) { } else if volume.Type == "custom" { vol.DriverOpts.Device = volume.Src } - compose.Volumes[GetVolumeSrcKey(project.Name, serviceName, volume)] = vol + compose.Volumes[GetVolumeSrcKey(project.Name, service.Name, volume)] = vol } } @@ -89,12 +88,12 @@ func addService(compose *DockerCompose, project *project.Project, service projec } compose.Services[service.Name] = Services{ - ContainerName: docker_system.ContainerName(project.Name, service.Name), + ContainerName: docker_system.ContainerName(project.Name, service.Name, system.Context.CurrentDeployment), Image: service.Image, Restart: "unless-stopped", Labels: map[string]string{"stackhead.project": project.Name}, User: service.User, - Networks: map[string]ServiceNetwork{"stackhead-network-" + project.Name: {Aliases: []string{service.Name}}}, + Networks: map[string]ServiceNetwork{docker_system.NetworkName(project.Name, system.Context.CurrentDeployment): {Aliases: []string{service.Name}}}, Volumes: volumes, Ports: ports, Environment: service.Environment, diff --git a/modules/container/docker/system/hooks.go b/modules/container/docker/system/hooks.go index 78c709b0..3b71ff83 100644 --- a/modules/container/docker/system/hooks.go +++ b/modules/container/docker/system/hooks.go @@ -45,14 +45,14 @@ func ExecuteHook(hookName string) error { // copy file onto container and run it..... containerLocation := path.Join("/", file.File) - containerName := ContainerName(system.Context.Project.Name, file.Service) + containerName := ContainerName(system.Context.Project.Name, file.Service, system.Context.CurrentDeployment) _, err := system.SimpleRemoteRun("docker", system.RemoteRunOpts{ Args: []string{ "cp", filePath, containerName + ":" + containerLocation, }, - WorkingDir: system.Context.Project.GetDirectoryPath(), + WorkingDir: system.Context.CurrentDeployment.GetPath(), }) if err != nil { return fmt.Errorf("Unable to copy file %s to container %s: \"%s\"", file.File, containerName, err.Error()) @@ -64,7 +64,7 @@ func ExecuteHook(hookName string) error { containerName, "chmod +x " + containerLocation, }, - WorkingDir: system.Context.Project.GetDirectoryPath(), + WorkingDir: system.Context.CurrentDeployment.GetPath(), }) if err != nil { return fmt.Errorf("Unable to copy file %s to container %s: \"%s\"", file.File, containerName, err.Error()) @@ -76,7 +76,7 @@ func ExecuteHook(hookName string) error { containerName, containerLocation, }, - WorkingDir: system.Context.Project.GetDirectoryPath(), + WorkingDir: system.Context.CurrentDeployment.GetPath(), }) if err != nil { return fmt.Errorf("Unable to run %s on container %s: \"%s\"", file, containerName, err.Error()) diff --git a/modules/container/docker/system/naming.go b/modules/container/docker/system/naming.go index 90be2817..1a31a86e 100644 --- a/modules/container/docker/system/naming.go +++ b/modules/container/docker/system/naming.go @@ -1,7 +1,14 @@ package docker_system -import "fmt" +import ( + "fmt" + "github.com/getstackhead/stackhead/system" +) -func ContainerName(projectName string, serviceName string) string { - return fmt.Sprintf("stackhead-%s-%s", projectName, serviceName) +func ContainerName(projectName string, serviceName string, deployment system.Deployment) string { + return fmt.Sprintf("stackhead-%s-%s-v%d", projectName, serviceName, deployment.Version) +} + +func NetworkName(projectName string, deployment system.Deployment) string { + return fmt.Sprintf("stackhead-network-%s-v%d", projectName, deployment.Version) } diff --git a/modules/container/docker/system/ports.go b/modules/container/docker/system/ports.go index 9cb0acf1..ce021a7d 100644 --- a/modules/container/docker/system/ports.go +++ b/modules/container/docker/system/ports.go @@ -16,7 +16,7 @@ func GetPortMap(project *project.Project) (map[string]int, error) { // find ports for running containers for _, service := range project.Container.Services { - res, _, err := system.RemoteRun("docker", system.RemoteRunOpts{Args: []string{"port", "stackhead-" + project.Name + "-" + service.Name}}) + res, _, err := system.RemoteRun("docker", system.RemoteRunOpts{Args: []string{"port", ContainerName(project.Name, service.Name, system.Context.CurrentDeployment)}}) if err == nil { // ignore error (container not running) // e.g. 80/tcp -> 0.0.0.0:49155 re := regexp.MustCompile(`(?P\d+)\/tcp -> 0\.0\.0\.0:(?P\d+)`) diff --git a/modules/container/docker/templates/project.tf.tmpl b/modules/container/docker/templates/project.tf.tmpl deleted file mode 100644 index 2c92eec8..00000000 --- a/modules/container/docker/templates/project.tf.tmpl +++ /dev/null @@ -1,17 +0,0 @@ -# This file was generated by StackHead. Do not modify it. - -resource "docker_network" "stackhead-network-{{ $.Context.Project.Name }}" { - provider = docker.{{ $.Context.Project.Name }} - name = "stackhead-network-{{ $.Context.Project.Name }}" -} - -resource "docker_container" "stackhead-{{ $.Context.Project.Name }}-{{ $service.Name }}" { -{{- if $service.Hooks }} - {{- if $service.Hooks.ExecuteAfterSetup }} - provisioner "local-exec" { - command = "docker cp {{ $.DockerPaths.GetHooksDir }}/afterSetup_{{ base $service.Hooks.ExecuteAfterSetup }} stackhead-{{ $.Context.Project.Name }}-{{ $service.Name }}:/afterSetup_{{ base $service.Hooks.ExecuteAfterSetup }} && docker exec stackhead-{{ $.Context.Project.Name }}-{{ $service.Name }} sh /afterSetup_{{ base $service.Hooks.ExecuteAfterSetup }}" - } - {{- end }} -{{- end }} -} -{{end}} diff --git a/modules/proxy/caddy/deploy.go b/modules/proxy/caddy/deploy.go index 24bc7655..2e516a3f 100644 --- a/modules/proxy/caddy/deploy.go +++ b/modules/proxy/caddy/deploy.go @@ -17,20 +17,23 @@ func (Module) Deploy(modulesSettings interface{}) error { return err } - projectCaddyLocation := system.Context.Project.GetDirectoryPath() + "/Caddyfile" + caddyFileResource := system.Resource{ + Type: system.TypeFile, + Operation: system.OperationCreate, + Name: "Caddyfile", + Content: caddyDirectives, + } - system.Context.Resources = append(system.Context.Resources, system.ResourceGroup{ - Name: "proxy-caddy-" + system.Context.Project.Name + "-caddyfile", - Resources: []system.Resource{ - { - Type: system.TypeFile, - Operation: system.OperationCreate, - Name: projectCaddyLocation, - Content: caddyDirectives, - }, - }, + caddyFilePath, err := system.Context.CurrentDeployment.GetResourcePath(caddyFileResource) + if err != nil { + return err + } + + system.Context.CurrentDeployment.ResourceGroups = append(system.Context.CurrentDeployment.ResourceGroups, system.ResourceGroup{ + Name: "proxy-caddy-" + system.Context.Project.Name + "-caddyfile", + Resources: []system.Resource{caddyFileResource}, ApplyResourceFunc: func() error { - if _, err := system.SimpleRemoteRun("ln", system.RemoteRunOpts{Args: []string{"-sf " + projectCaddyLocation + " /etc/caddy/conf.d/stackhead_" + system.Context.Project.Name + ".conf"}}); err != nil { + if _, err := system.SimpleRemoteRun("ln", system.RemoteRunOpts{Args: []string{"-sf " + caddyFilePath + " /etc/caddy/conf.d/stackhead_" + system.Context.Project.Name + ".conf"}}); err != nil { return fmt.Errorf("Unable to symlink project Caddyfile: " + err.Error()) } if _, err := system.SimpleRemoteRun("systemctl", system.RemoteRunOpts{Args: []string{"reload", "caddy"}, Sudo: true}); err != nil { diff --git a/modules/proxy/nginx/certificates.go b/modules/proxy/nginx/certificates.go index b47c3d5a..91d6d9b2 100644 --- a/modules/proxy/nginx/certificates.go +++ b/modules/proxy/nginx/certificates.go @@ -14,10 +14,6 @@ func GetSnakeoilPaths() (string, string) { return path.Join(CertificatesDirectory, "fullchain_snakeoil.pem"), path.Join(CertificatesDirectory, "privkey_snakeoil.pem") } -func GetCertificateDirectoryPath(p *project.Project) string { - return path.Join(config.ProjectsRootDirectory, p.Name, "certificates") -} - func GetCertificatesDirectory(p *project.Project) string { return path.Join(CertificatesDirectory, p.Name) } diff --git a/modules/proxy/nginx/deploy.go b/modules/proxy/nginx/deploy.go index 8be7bed5..5c0e74d1 100644 --- a/modules/proxy/nginx/deploy.go +++ b/modules/proxy/nginx/deploy.go @@ -2,9 +2,10 @@ package proxy_nginx import ( "fmt" - xfs "github.com/saitho/golang-extended-fs/v2" "path" + xfs "github.com/saitho/golang-extended-fs/v2" + "github.com/getstackhead/stackhead/config" "github.com/getstackhead/stackhead/modules/proxy" "github.com/getstackhead/stackhead/project" @@ -14,7 +15,6 @@ import ( type Paths struct { RootDirectory string CertificatesProjectDirectory string - ProjectCertificatesDirectory string ProjectsRootDirectory string AcmeChallengesDirectory string SnakeoilFullchainPath string @@ -30,7 +30,6 @@ func getPaths() Paths { return Paths{ RootDirectory: config.RootDirectory, CertificatesProjectDirectory: GetCertificatesDirectory(system.Context.Project), - ProjectCertificatesDirectory: GetCertificateDirectoryPath(system.Context.Project), ProjectsRootDirectory: config.ProjectsRootDirectory, AcmeChallengesDirectory: AcmeChallengesDirectory, SnakeoilFullchainPath: SnakeoilFullchainPath, @@ -118,50 +117,79 @@ func (Module) Deploy(_modulesSettings interface{}) error { fmt.Println("Deploy step") paths := getPaths() - if err := xfs.CreateFolder("ssh://" + paths.ProjectCertificatesDirectory); err != nil { - return err - } if err := xfs.CreateFolder("ssh://" + paths.CertificatesProjectDirectory); err != nil { return err } serverConfig := buildServerConfig(system.Context.Project, proxy.Context.AllPorts) - nginxProjectLocation := system.Context.Project.GetDirectoryPath() + "/nginx.conf" - if err = xfs.WriteFile("ssh://"+nginxProjectLocation, serverConfig); err != nil { - return err - } - - // Symlink project certificate files to snakeoil files after initial creation - if _, err := system.SimpleRemoteRun("ln", system.RemoteRunOpts{Args: []string{"-s " + paths.SnakeoilFullchainPath + " " + paths.CertificatesProjectDirectory + "/fullchain.pem"}, AllowFail: true}); err != nil { - return fmt.Errorf("Unable to symlink snakeoil full chain: " + err.Error()) - } - if _, err := system.SimpleRemoteRun("ln", system.RemoteRunOpts{Args: []string{"-s " + paths.SnakeoilPrivkeyPath + " " + paths.CertificatesProjectDirectory + "/privkey.pem"}, AllowFail: true}); err != nil { - return fmt.Errorf("Unable to symlink snakeoil privkey: " + err.Error()) - } - - if _, err := system.SimpleRemoteRun("ln", system.RemoteRunOpts{Args: []string{"-sf " + nginxProjectLocation + " /etc/nginx/sites-available/stackhead_" + system.Context.Project.Name + ".conf"}}); err != nil { - return fmt.Errorf("Unable to symlink project Nginx file: " + err.Error()) - } - if _, err := system.SimpleRemoteRun("ln", system.RemoteRunOpts{Args: []string{"-sf /etc/nginx/sites-available/stackhead_" + system.Context.Project.Name + ".conf " + moduleSettings.Config.VhostPath + "/stackhead_" + system.Context.Project.Name + ".conf"}}); err != nil { - return fmt.Errorf("Unable to enable project Nginx file: " + err.Error()) - } - // first reload so webserver config works for ACME request - if _, err := system.SimpleRemoteRun("systemctl", system.RemoteRunOpts{Args: []string{"reload", "nginx"}, Sudo: true}); err != nil { - return fmt.Errorf("Unable to reload Nginx service: " + err.Error()) - } - - certMail := "certificates-noreply@stackhead.io" - if len(moduleSettings.CertificatesEmail) > 0 { - certMail = moduleSettings.CertificatesEmail - } - if err := generateCertificates(paths, certMail); err != nil { - return fmt.Errorf("Unable to generate certificates: " + err.Error()) - } - // reload Nginx again so certificates take effect - if _, err := system.SimpleRemoteRun("systemctl", system.RemoteRunOpts{Args: []string{"reload", "nginx"}, Sudo: true}); err != nil { - return fmt.Errorf("Unable to reload Nginx service: " + err.Error()) - } + nginxConfigResource := system.Resource{ + Type: system.TypeFile, + Operation: system.OperationCreate, + Name: "nginx.conf", + Content: serverConfig, + } + nginxConfigResourcePath, _ := system.Context.CurrentDeployment.GetResourcePath(nginxConfigResource) + system.Context.CurrentDeployment.ResourceGroups = append(system.Context.CurrentDeployment.ResourceGroups, system.ResourceGroup{ + Name: "proxy-nginx-" + system.Context.Project.Name, + Resources: []system.Resource{ + system.Resource{ + Type: system.TypeFolder, + Operation: system.OperationCreate, + Name: "certificates", + }, + nginxConfigResource, + // Symlink project certificate files to snakeoil files after initial creation + system.Resource{ + Type: system.TypeLink, + Operation: system.OperationCreate, + Name: paths.CertificatesProjectDirectory + "/fullchain.pem", + ExternalResource: true, + LinkSource: paths.SnakeoilFullchainPath, + }, + system.Resource{ + Type: system.TypeLink, + Operation: system.OperationCreate, + Name: paths.CertificatesProjectDirectory + "/privkey.pem", + ExternalResource: true, + LinkSource: paths.SnakeoilPrivkeyPath, + }, + system.Resource{ + Type: system.TypeLink, + Operation: system.OperationCreate, + Name: "/etc/nginx/sites-available/stackhead_" + system.Context.Project.Name + ".conf", + ExternalResource: true, + LinkSource: nginxConfigResourcePath, + EnforceLink: true, + }, + system.Resource{ + Type: system.TypeLink, + Operation: system.OperationCreate, + Name: moduleSettings.Config.VhostPath + "/stackhead_" + system.Context.Project.Name + ".conf", + ExternalResource: true, + LinkSource: "/etc/nginx/sites-available/stackhead_" + system.Context.Project.Name + ".conf", + EnforceLink: true, + }, + }, + ApplyResourceFunc: func() error { + // first reload so webserver config works for ACME request + if _, err := system.SimpleRemoteRun("systemctl", system.RemoteRunOpts{Args: []string{"reload", "nginx"}, Sudo: true}); err != nil { + return fmt.Errorf("Unable to reload Nginx service: " + err.Error()) + } + certMail := "certificates-noreply@stackhead.io" + if len(moduleSettings.CertificatesEmail) > 0 { + certMail = moduleSettings.CertificatesEmail + } + if err := generateCertificates(paths, certMail); err != nil { + return fmt.Errorf("Unable to generate certificates: " + err.Error()) + } + // reload Nginx again so certificates take effect + if _, err := system.SimpleRemoteRun("systemctl", system.RemoteRunOpts{Args: []string{"reload", "nginx"}, Sudo: true}); err != nil { + return fmt.Errorf("Unable to reload Nginx service: " + err.Error()) + } + return nil + }, + }) return nil } diff --git a/modules/proxy/rendering.go b/modules/proxy/rendering.go index e7cbc55f..b789c333 100644 --- a/modules/proxy/rendering.go +++ b/modules/proxy/rendering.go @@ -38,7 +38,7 @@ var FuncMap = template.FuncMap{ return auths }, "getExternalPort": func(service string, internalPort int) string { - for _, resourceGroup := range system.Context.Resources { + for _, resourceGroup := range system.Context.CurrentDeployment.ResourceGroups { for _, resource := range resourceGroup.Resources { if resource.Type != system.TypeContainer { continue diff --git a/project/funcs.go b/project/funcs.go index 21b4416d..d5862ba0 100644 --- a/project/funcs.go +++ b/project/funcs.go @@ -10,6 +10,10 @@ func (project *Project) GetDirectoryPath() string { return path.Join(config.ProjectsRootDirectory, project.Name) } +func (project *Project) GetDeploymentsPath() string { + return path.Join(config.ProjectsRootDirectory, project.Name, "deployments") +} + func (project *Project) GetRuntimeDataDirectoryPath() string { - return path.Join(config.ProjectsRootDirectory, project.Name, "runtime") + return path.Join(config.ProjectsRootDirectory, project.Name, "data") } diff --git a/routines/implementations.go b/routines/implementations.go index e0952881..cdd9a202 100644 --- a/routines/implementations.go +++ b/routines/implementations.go @@ -2,11 +2,14 @@ package routines import ( "fmt" - "strings" - "github.com/chelnak/ysmrr" xfs "github.com/saitho/golang-extended-fs/v2" logger "github.com/sirupsen/logrus" + "gopkg.in/yaml.v3" + "path" + "sort" + "strings" + "time" "github.com/getstackhead/stackhead/project" "github.com/getstackhead/stackhead/system" @@ -40,6 +43,45 @@ var PrepareProjectTask = func(projectDefinition *project.Project) Task { if err := xfs.CreateFolder("ssh://" + projectDefinition.GetDirectoryPath()); err != nil { return err } + if err := xfs.CreateFolder("ssh://" + projectDefinition.GetDeploymentsPath()); err != nil { + return err + } + + // Find latest deployment + files, err := xfs.ListFolders("ssh://" + projectDefinition.GetDeploymentsPath()) + if err != nil { + return err + } + if files != nil { + // newest files at the top + sort.Slice(files, func(i, j int) bool { + return files[i].ModTime().After(files[j].ModTime()) + }) + for _, file := range files { + if file.IsDir() && system.MatchDeploymentNaming(file.Name()) { + latestDeployment, err := system.DeploymentFromFolder(file) + if err != nil { + return err + } + system.Context.LatestDeployment = &latestDeployment + break + } + } + } + newVersion := 1 + if system.Context.LatestDeployment != nil { + newVersion = system.Context.LatestDeployment.Version + 1 + } + system.Context.CurrentDeployment = system.Deployment{ + Version: newVersion, + Date: time.Now(), + Project: *system.Context.Project, + } + + // Create folder for new deployment + if err := xfs.CreateFolder("ssh://" + system.Context.CurrentDeployment.GetPath()); err != nil { + return err + } return nil }, ErrorAsErrorMessage: true, @@ -88,21 +130,34 @@ var RollbackResources = Task{ spinner := r.TaskRunner.GetNewSubtaskSpinner(resource.ToString(true)) matched, err := system.RollbackResourceOperation(resource) if !matched || err == nil { - spinner.Complete() + if spinner != nil { + spinner.Complete() + } } else if err != nil { errors = append(errors, fmt.Errorf("Rollback error: %s", err)) - spinner.Error() + if spinner != nil { + spinner.Error() + } } } } - if len(errors) == 0 { - return nil - } - errorMessages := []string{"The following errors occurred:"} + + // Mark deployment as rolled back + system.Context.CurrentDeployment.RolledBack = true for _, err2 := range errors { - errorMessages = append(errorMessages, "- "+err2.Error()) + system.Context.CurrentDeployment.RollbackErrors = append(system.Context.CurrentDeployment.RollbackErrors, err2.Error()) } - return fmt.Errorf(strings.Join(errorMessages, "\n")) + + if len(system.Context.CurrentDeployment.RollbackErrors) > 0 { + return fmt.Errorf("The following errors occurred:\n" + strings.Join(system.Context.CurrentDeployment.RollbackErrors, "\n")) + } + + // Delete deployment version + //if err := xfs.DeleteFolder("ssh://"+system.Context.CurrentDeployment.GetPath(), true); err != nil { + // return fmt.Errorf("unable to remove deployment folder: " + err.Error()) + //} + + return nil }, } @@ -112,7 +167,7 @@ var CreateResources = Task{ var errors []error var uncompletedSpinners []*ysmrr.Spinner - for _, resourceGroup := range system.Context.Resources { + for _, resourceGroup := range system.Context.CurrentDeployment.ResourceGroups { for _, resource := range resourceGroup.Resources { spinner := r.TaskRunner.GetNewSubtaskSpinner(resource.ToString(false)) processed, err := system.ApplyResourceOperation(resource) @@ -165,3 +220,18 @@ var CreateResources = Task{ return fmt.Errorf(strings.Join(errorMessages, "\n")) }, } + +var FinalizeDeployment = Task{ + Name: "Finalizing deployment", + Run: func(r *Task) error { + resourcesPath := path.Join(system.Context.CurrentDeployment.GetPath(), "deployment.yaml") + yamlString, err := yaml.Marshal(system.Context.CurrentDeployment) + if err != nil { + return err + } + if err = xfs.WriteFile("ssh://"+resourcesPath, string(yamlString)); err != nil { + return err + } + return nil + }, +} diff --git a/system/context.go b/system/context.go index fbb97525..3b33af16 100644 --- a/system/context.go +++ b/system/context.go @@ -2,12 +2,14 @@ package system import ( "fmt" - logger "github.com/sirupsen/logrus" "net" "os" "path" + "strconv" + "time" "github.com/saitho/golang-extended-fs/v2/sftp" + logger "github.com/sirupsen/logrus" "github.com/getstackhead/stackhead/config" "github.com/getstackhead/stackhead/project" @@ -29,13 +31,48 @@ func (c ContextAuthenticationStruct) GetPublicKeyPath() string { return path.Join(c.LocalAuthenticationDir, "public_key.pem") } +type Deployment struct { + Version int + Date time.Time + Project project.Project `yaml:"-"` + + RolledBack bool + RollbackErrors []string + + ResourceGroups []ResourceGroup +} + +func (d Deployment) GetResourcePath(resource Resource) (string, error) { + if resource.Type != TypeFile && resource.Type != TypeFolder && resource.Type != TypeLink { + return "", fmt.Errorf("not a file, folder or link resource") + } + if resource.ExternalResource { + if !path.IsAbs(resource.Name) { + return "", fmt.Errorf("expected absolute path in Name as ExternalResource is set to true") + } + return resource.Name, nil + } + return path.Join(d.GetPath(), resource.Name), nil +} + +func (d Deployment) GetPath() string { + return path.Join(d.Project.GetDeploymentsPath(), "v"+strconv.Itoa(d.Version)) +} + +func (d Deployment) Serialize() string { + return "" +} + type ContextStruct struct { - TargetHost net.IP - CurrentAction string + TargetHost net.IP + + CurrentAction string + LatestDeployment *Deployment + CurrentDeployment Deployment + Project *project.Project IsCI bool Authentication ContextAuthenticationStruct - Resources []ResourceGroup ProxyModule Module ContainerModule Module diff --git a/system/deployments.go b/system/deployments.go new file mode 100644 index 00000000..a8fd92b1 --- /dev/null +++ b/system/deployments.go @@ -0,0 +1,40 @@ +package system + +import ( + "fmt" + "os" + "regexp" + "strconv" +) + +func MatchDeploymentNaming(folderName string) bool { + pattern := regexp.MustCompile(`(?m)^v(\d+)$`) + return pattern.MatchString(folderName) +} + +func ExtractVersionNumberFromFolderName(folderName string) (int, error) { + pattern := regexp.MustCompile(`(?m)^v(\d+)$`) + submatch := pattern.FindStringSubmatch(folderName) + if len(submatch) < 2 { + return 0, fmt.Errorf("unable to extract version number") + } + intVal, err := strconv.Atoi(submatch[1]) + if err != nil { + return 0, err + } + return intVal, nil +} + +func DeploymentFromFolder(file os.FileInfo) (Deployment, error) { + version, err := ExtractVersionNumberFromFolderName(file.Name()) + if err != nil { + return Deployment{}, err + } + + return Deployment{ + Version: version, + Date: file.ModTime(), + Project: *Context.Project, + ResourceGroups: nil, + }, nil +} diff --git a/system/resource.go b/system/resource.go index 0dba1ed4..8e3edff0 100644 --- a/system/resource.go +++ b/system/resource.go @@ -11,6 +11,8 @@ type Type string const ( TypeFile Type = "file" + TypeFolder Type = "folder" + TypeLink Type = "link" TypeContainer Type = "container" ) @@ -27,26 +29,34 @@ type ResourceGroup struct { Name string Resources []Resource - ApplyResourceFunc ApplyResourceFuncType - RollbackResourceFunc RollbackResourceFuncType + ApplyResourceFunc ApplyResourceFuncType `yaml:"-"` + RollbackResourceFunc RollbackResourceFuncType `yaml:"-"` } type Resource struct { Type Type - Operation Operation + Operation Operation `yaml:"-"` + + // if set the Name refers to an external resource. for files an absolute path is expected + ExternalResource bool `yaml:"externalResource,omitempty"` // name of the resource (e.g. file name, container name) Name string // contents of resource, if any - Content string + Content string `yaml:"-"` // name of the associated service - ServiceName string + ServiceName string `yaml:"serviceName,omitempty"` // for container resources - Ports []string - ImageName string + Ports []string `yaml:"ports,omitempty"` + ImageName string `yaml:"imageName,omitempty"` + + // for link resources + LinkSource string `yaml:"linkSource,omitempty"` + // EnforceLink is true, then symlink is created with force and will not ignore errors + EnforceLink bool `yaml:"-"` } func (r Resource) GetOperationLabel(invertOperation bool) string { diff --git a/system/resource_manager.go b/system/resource_manager.go index 4344f2fd..245ec414 100644 --- a/system/resource_manager.go +++ b/system/resource_manager.go @@ -2,35 +2,65 @@ package system import ( "fmt" - xfs "github.com/saitho/golang-extended-fs/v2" ) func ApplyResourceOperation(resource Resource) (bool, error) { - // FILE - if resource.Type == TypeFile { + resourceFilePath, _ := Context.CurrentDeployment.GetResourcePath(resource) + + switch resource.Type { + case TypeFile: if resource.Operation == OperationCreate { // TODO: backup if file exists - if err := xfs.WriteFile("ssh://"+resource.Name, resource.Content); err != nil { + if err := xfs.WriteFile("ssh://"+resourceFilePath, resource.Content); err != nil { return true, fmt.Errorf("unable to create file at %s: %s", resource.Name, err) } } return true, nil + case TypeFolder: + if resource.Operation == OperationCreate { + // TODO: backup if file exists + if err := xfs.CreateFolder("ssh://" + resourceFilePath); err != nil { + return true, fmt.Errorf("unable to create folder at %s: %s", resource.Name, err) + } + } + return true, nil + case TypeLink: + if resource.Operation == OperationCreate { + args := RemoteRunOpts{Args: []string{"-s " + resource.LinkSource + " " + resourceFilePath}, AllowFail: true} + if resource.EnforceLink { + args = RemoteRunOpts{Args: []string{"-sf " + resource.LinkSource + " " + resourceFilePath}} + } + if _, err := SimpleRemoteRun("ln", args); err != nil { + return true, fmt.Errorf("Unable to symlink " + resource.LinkSource + " -> " + resourceFilePath + ": " + err.Error()) + } + } } // CONTAINER via ResourceGroup (see StackHead container module) return false, nil } func RollbackResourceOperation(resource Resource) (bool, error) { - // FILE - if resource.Type == TypeFile { + switch resource.Type { + case TypeFile: + case TypeLink: if resource.Operation == OperationCreate { // TODO: restore backup if file exists - if err := xfs.DeleteFile("ssh://" + resource.Name); err != nil { + resourcePath, _ := Context.CurrentDeployment.GetResourcePath(resource) + if err := xfs.DeleteFile("ssh://" + resourcePath); err != nil { return true, fmt.Errorf("unable to remove file at %s: %s", resource.Name, err) } } return true, nil + case TypeFolder: + if resource.Operation == OperationCreate { + // TODO: restore backup if file exists + resourcePath, _ := Context.CurrentDeployment.GetResourcePath(resource) + if err := xfs.DeleteFolder("ssh://"+resourcePath, true); err != nil { + return true, fmt.Errorf("unable to remove folder at %s: %s", resource.Name, err) + } + } + return true, nil } // CONTAINER via ResourceGroup (see StackHead container module) return false, nil