feat: Fine-grained stack update and etcd and network stacks separation (

#1233)

* Prompt before updating

* feat: Support updating a subset of cfn stacks only

* Vendor changes for the stack subset update feature

* Rename some func-scoped variables for clarity

* feat: Extract etcd and network stack for more fine-grained cluster update

* feat: Ability to specify just one or more node pools for updates

* fix: Make update work with `--targets all`

Ref #1112

build

@@ -26,6 +26,16 @@ go run ../../../codegen/files_gen.go Etcdadm=../../../etcdadm/etcdadm
 gofmt -w files.go
 popd
 
+pushd core/network/config
+go run ../../../codegen/templates_gen.go StackTemplateTemplate=stack-template.json
+gofmt -w templates.go
+popd
+
+pushd core/etcd/config
+go run ../../../codegen/templates_gen.go StackTemplateTemplate=stack-template.json
+gofmt -w templates.go
+popd
+
 pushd core/nodepool/config
 go run ../../../codegen/templates_gen.go StackTemplateTemplate=stack-template.json
 gofmt -w templates.go

cfnstack/assets.go

@@ -14,6 +14,12 @@ type Assets interface {
 	FindAssetByStackAndFileName(string, string) (model.Asset, error)
 }
 
+func EmptyAssets() assetsImpl {
+	return assetsImpl{
+		underlying: map[model.AssetID]model.Asset{},
+	}
+}
+
 type assetsImpl struct {
 	underlying map[model.AssetID]model.Asset
 }

cfnstack/provisioner.go

@@ -223,7 +223,7 @@ func (c *Provisioner) ValidateStackAtURL(templateURL string) (string, error) {
 	cfSvc := cloudformation.New(c.session)
 	validationReport, err := cfSvc.ValidateTemplate(&validateInput)
 	if err != nil {
-		return "", fmt.Errorf("invalid cloudformation stack: %v", err)
+		return "", fmt.Errorf("invalid cloudformation stack template %s: %v", templateURL, err)
 	}
 
 	return validationReport.String(), nil

cmd/update.go

@@ -3,8 +3,11 @@ package cmd
 import (
 	"fmt"
 
+	"bufio"
 	"github.com/kubernetes-incubator/kube-aws/core/root"
 	"github.com/spf13/cobra"
+	"os"
+	"strings"
 )
 
 var (
@@ -18,6 +21,8 @@ var (
 
 	updateOpts = struct {
 		awsDebug, prettyPrint, skipWait bool
+		force                           bool
+		targets                         []string
 	}{}
 )
 
@@ -26,21 +31,30 @@ func init() {
 	cmdUpdate.Flags().BoolVar(&updateOpts.awsDebug, "aws-debug", false, "Log debug information from aws-sdk-go library")
 	cmdUpdate.Flags().BoolVar(&updateOpts.prettyPrint, "pretty-print", false, "Pretty print the resulting CloudFormation")
 	cmdUpdate.Flags().BoolVar(&updateOpts.skipWait, "skip-wait", false, "Don't wait the resources finish")
+	cmdUpdate.Flags().BoolVar(&updateOpts.force, "force", false, "Don't ask for confirmation")
+	cmdUpdate.Flags().StringSliceVar(&updateOpts.targets, "targets", root.AllOperationTargetsAsStringSlice(), "Update nothing but specified sub-stacks.  Specify `all` or any combination of `etcd`, `control-plane`, and node pool names. Defaults to `all`")
 }
 
 func runCmdUpdate(_ *cobra.Command, _ []string) error {
+	if !updateOpts.force && !updateConfirmation() {
+		fmt.Println("Operation cancelled")
+		return nil
+	}
+
 	opts := root.NewOptions(updateOpts.prettyPrint, updateOpts.skipWait)
 
 	cluster, err := root.ClusterFromFile(configPath, opts, updateOpts.awsDebug)
 	if err != nil {
 		return fmt.Errorf("Failed to read cluster config: %v", err)
 	}
 
-	if _, err := cluster.ValidateStack(); err != nil {
+	targets := root.OperationTargetsFromStringSlice(updateOpts.targets)
+
+	if _, err := cluster.ValidateStack(targets); err != nil {
 		return err
 	}
 
-	report, err := cluster.Update()
+	report, err := cluster.Update(targets)
 	if err != nil {
 		return fmt.Errorf("Error updating cluster: %v", err)
 	}
@@ -61,3 +75,12 @@ func runCmdUpdate(_ *cobra.Command, _ []string) error {
 
 	return nil
 }
+
+func updateConfirmation() bool {
+	reader := bufio.NewReader(os.Stdin)
+	fmt.Print("This operation will update the cluster. Are you sure? [y,n]: ")
+	text, _ := reader.ReadString('\n')
+	text = strings.TrimSuffix(strings.ToLower(text), "\n")
+
+	return text == "y" || text == "yes"
+}

cmd/validate.go

@@ -19,6 +19,7 @@ var (
 
 	validateOpts = struct {
 		awsDebug, skipWait bool
+		targets            []string
 	}{}
 )
 
@@ -30,6 +31,11 @@ func init() {
 		false,
 		"Log debug information from aws-sdk-go library",
 	)
+	cmdValidate.Flags().StringSliceVar(
+		&validateOpts.targets,
+		"targets",
+		root.AllOperationTargetsAsStringSlice(),
+		"Validate nothing but specified sub-stacks. Specify `all` or any combination of `etcd`, `control-plane`, and node pool names. Defaults to `all`")
 }
 
 func runCmdValidate(_ *cobra.Command, _ []string) error {
@@ -41,7 +47,10 @@ func runCmdValidate(_ *cobra.Command, _ []string) error {
 	}
 
 	fmt.Printf("Validating UserData and stack template...\n")
-	report, err := cluster.ValidateStack()
+
+	targets := root.OperationTargetsFromStringSlice(validateOpts.targets)
+
+	report, err := cluster.ValidateStack(targets)
 	if report != "" {
 		fmt.Fprintf(os.Stderr, "Validation Report: %s\n", report)
 	}

core/controlplane/cluster/cluster.go

@@ -13,6 +13,7 @@ import (
 	"github.com/kubernetes-incubator/kube-aws/cfnstack"
 	"github.com/kubernetes-incubator/kube-aws/core/controlplane/config"
 	"github.com/kubernetes-incubator/kube-aws/model"
+	"github.com/kubernetes-incubator/kube-aws/naming"
 	"github.com/kubernetes-incubator/kube-aws/plugin/clusterextension"
 	"github.com/kubernetes-incubator/kube-aws/plugin/pluginmodel"
 )
@@ -121,12 +122,24 @@ func (c *ClusterRef) validateExistingVPCState(ec2Svc ec2Service) error {
 	return nil
 }
 
-func NewCluster(cfg *config.Cluster, opts config.StackTemplateOptions, plugins []*pluginmodel.Plugin, awsDebug bool) (*Cluster, error) {
+func NewCluster(cfgRef *config.Cluster, opts config.StackTemplateOptions, plugins []*pluginmodel.Plugin, awsDebug bool) (*Cluster, error) {
+	cfg := &config.Cluster{}
+	*cfg = *cfgRef
+
+	// Import all the managed subnets from the network stack
+	var err error
+	cfg.Subnets, err = cfg.Subnets.ImportFromNetworkStackRetainingNames()
+	if err != nil {
+		return nil, fmt.Errorf("failed to import subnets from network stack: %v", err)
+	}
+	cfg.VPC = cfg.VPC.ImportFromNetworkStack()
+	cfg.SetDefaults()
+
 	clusterRef := NewClusterRef(cfg, awsDebug)
 	// TODO Do this in a cleaner way e.g. in config.go
 	clusterRef.KubeResourcesAutosave.S3Path = model.NewS3Folders(cfg.DeploymentSettings.S3URI, clusterRef.ClusterName).ClusterBackups().Path()
 
-	stackConfig, err := clusterRef.StackConfig(opts, plugins)
+	stackConfig, err := clusterRef.StackConfig(config.ControlPlaneStackName, opts, plugins)
 	if err != nil {
 		return nil, err
 	}
@@ -161,14 +174,6 @@ func NewCluster(cfg *config.Cluster, opts config.StackTemplateOptions, plugins [
 		c.StackConfig.Controller.NodeLabels[k] = v
 	}
 
-	extraEtcd, err := extras.Etcd()
-	if err != nil {
-		return nil, fmt.Errorf("failed to load controller node extras from plugins: %v", err)
-	}
-	c.StackConfig.Etcd.CustomSystemdUnits = append(c.StackConfig.Etcd.CustomSystemdUnits, extraEtcd.SystemdUnits...)
-	c.StackConfig.Etcd.CustomFiles = append(c.StackConfig.Etcd.CustomFiles, extraEtcd.Files...)
-	c.StackConfig.Etcd.IAMConfig.Policy.Statements = append(c.StackConfig.Etcd.IAMConfig.Policy.Statements, extraEtcd.IAMPolicyStatements...)
-
 	c.assets, err = c.buildAssets()
 
 	return c, err
@@ -180,27 +185,19 @@ func (c *Cluster) Assets() cfnstack.Assets {
 
 func (c *Cluster) buildAssets() (cfnstack.Assets, error) {
 	var err error
-	assets := cfnstack.NewAssetsBuilder(c.StackName(), c.StackConfig.ClusterExportedStacksS3URI(), c.StackConfig.Region)
+	assets := cfnstack.NewAssetsBuilder(c.StackName, c.StackConfig.ClusterExportedStacksS3URI(), c.StackConfig.Region)
 
 	if c.StackConfig.UserDataController, err = model.NewUserData(c.StackTemplateOptions.ControllerTmplFile, c.StackConfig.Config); err != nil {
 		return nil, fmt.Errorf("failed to render controller cloud config: %v", err)
 	}
 
-	if c.StackConfig.UserDataEtcd, err = model.NewUserData(c.StackTemplateOptions.EtcdTmplFile, c.StackConfig.Config); err != nil {
-		return nil, fmt.Errorf("failed to render etcd cloud config: %v", err)
-	}
-
 	if err = assets.AddUserDataPart(c.UserDataController, model.USERDATA_S3, "userdata-controller"); err != nil {
 		return nil, fmt.Errorf("failed to render controller cloud config: %v", err)
 	}
 
-	if err = assets.AddUserDataPart(c.UserDataEtcd, model.USERDATA_S3, "userdata-etcd"); err != nil {
-		return nil, fmt.Errorf("failed to render etcd cloud config: %v", err)
-	}
-
 	stackTemplate, err := c.RenderStackTemplateAsString()
 	if err != nil {
-		return nil, fmt.Errorf("Error while rendering template: %v", err)
+		return nil, fmt.Errorf("failed to render control-plane template: %v", err)
 	}
 
 	assets.Add(STACK_TEMPLATE_FILENAME, stackTemplate)
@@ -210,7 +207,7 @@ func (c *Cluster) buildAssets() (cfnstack.Assets, error) {
 
 func (c *Cluster) TemplateURL() (string, error) {
 	assets := c.Assets()
-	asset, err := assets.FindAssetByStackAndFileName(c.StackName(), STACK_TEMPLATE_FILENAME)
+	asset, err := assets.FindAssetByStackAndFileName(c.StackName, STACK_TEMPLATE_FILENAME)
 	if err != nil {
 		return "", fmt.Errorf("failed to get template URL: %v", err)
 	}
@@ -239,7 +236,7 @@ func (c *Cluster) stackProvisioner() *cfnstack.Provisioner {
 }
 `
 	return cfnstack.NewProvisioner(
-		c.StackName(),
+		c.StackName,
 		c.StackTags,
 		c.ClusterExportedStacksS3URI(),
 		c.Region,
@@ -272,12 +269,17 @@ func (c *Cluster) Validate() error {
 	return nil
 }
 
+// NestedStackName returns a sanitized name of this control-plane which is usable as a valid cloudformation nested stack name
+func (c Cluster) NestedStackName() string {
+	return naming.FromStackToCfnResource(config.ControlPlaneStackName)
+}
+
 func (c *Cluster) String() string {
 	return fmt.Sprintf("{Config:%+v}", *c.StackConfig.Config)
 }
 
 func (c *ClusterRef) Destroy() error {
-	return cfnstack.NewDestroyer(c.StackName(), c.session, c.CloudFormation.RoleARN).Destroy()
+	return cfnstack.NewDestroyer(config.ControlPlaneStackName, c.session, c.CloudFormation.RoleARN).Destroy()
 }
 
 func (c *ClusterRef) validateKeyPair(ec2Svc ec2Service) error {