Stricter validation of cloud configuration changes (#310)

 * Stricter validation of cloud configuration changes:
 * Do not allow to change/set cloud provider on the installed cluster
 * Keep cloud provider CLI parameter and cloud config block from cluster configuration in sync
 * Only allow to update cloud configuration if the cluster has cloud integration
 * Address review comments. Make sure to keep clusterconfig in sync with cloud provider on reset
 * Require cloud provider when changing cloud-config as a sanity check
 * Fix reversing environment variables: use the same approach as cluster configuration. Fix opshandler handlers/opsclient for RuntimeEnvironment/ClusterConfiguration.
* Turn off preemption by default

Jenkinsfile

@@ -40,7 +40,7 @@ properties([
            defaultValue: 'stable-gce',
            description: 'Robotest tag to use.'),
     choice(choices: ["true", "false"].join("\n"),
-           defaultValue: 'true',
+           defaultValue: 'false',
            description: 'Whether to use preemptible VMs.',
            name: 'GCE_PREEMPTIBLE'),
     choice(choices: ["custom-4-8192", "custom-8-8192"].join("\n"),

Makefile

@@ -42,7 +42,7 @@ RELEASE_OUT ?=
 TELEPORT_TAG = 3.0.5
 # TELEPORT_REPOTAG adapts TELEPORT_TAG to the teleport tagging scheme
 TELEPORT_REPOTAG := v$(TELEPORT_TAG)
-PLANET_TAG := 5.5.12-$(K8S_VER_SUFFIX)-1-ga3d3820
+PLANET_TAG := 5.5.13-$(K8S_VER_SUFFIX)
 PLANET_BRANCH := $(PLANET_TAG)
 K8S_APP_TAG := $(GRAVITY_TAG)
 TELEKUBE_APP_TAG := $(GRAVITY_TAG)

docs/5.x/changelog.md

@@ -47,6 +47,7 @@ LTS starts with `3.51.0` with minor backwards compatible changes added over time
 #### Improvements
 
 * Introduce `ClusterConfiguration` resource, see [Configuring Cluster](/cluster/#cluster-configuration) for details.
+* Introduce `RuntimeEnvironment` resource, see [Configuring Runtime Environment Variables](/cluster/#configuring-runtime-environment-variables) for details.
 * Update 'gravity plan' to support all cluster operations.
 
 ### 5.5.0-beta.2

lib/install/flow.go

@@ -111,7 +111,6 @@ func (i *Installer) NewClusterRequest() ops.NewSiteRequest {
 		Email:        fmt.Sprintf("installer@%v", i.SiteDomain),
 		Provider:     i.CloudProvider,
 		DomainName:   i.SiteDomain,
-		Resources:    i.Resources,
 		InstallToken: i.Config.Token,
 		ServiceUser: storage.OSUser{
 			Name: i.Config.ServiceUser.Name,

lib/install/install.go

@@ -162,12 +162,14 @@ type Config struct {
 	Role string
 	// AppPackage is the application being installed
 	AppPackage *loc.Locator
-	// Resources specifies optional resources to create.
-	// This is assumed to be either JSON- or YAML-encoded list of resources
-	Resources []byte
 	// RuntimeResources specifies optional Kubernetes resources to create
 	// If specified, will be combined with Resources
 	RuntimeResources []runtime.Object
+	// ClusterResources specifies optional cluster resources to create
+	// If specified, will be combined with Resources
+	// TODO(dmitri): externalize the ClusterConfiguration resource and create
+	// default provider-specific cloud-config on Gravity side
+	ClusterResources []storage.UnknownResource
 	// EventsC is channel with events indicating install progress
 	EventsC chan Event
 	// SystemDevice is a device for gravity data
@@ -283,6 +285,7 @@ func (c *Config) validateCloudConfig() error {
 	if c.CloudProvider != schema.ProviderGCE {
 		return nil
 	}
+	// TODO(dmitri): skip validations if user provided custom cloud configuration
 	if err := cloudgce.ValidateTag(c.SiteDomain); err != nil {
 		log.WithError(err).Warnf("Failed to validate cluster name %v as node tag on GCE.", c.SiteDomain)
 		if len(c.GCENodeTags) == 0 {

lib/install/plan_test.go

@@ -125,7 +125,6 @@ func (s *PlanSuite) SetUpSuite(c *check.C) {
 			DomainName: "example.com",
 			AppPackage: appPackage.String(),
 			Provider:   schema.ProviderAWS,
-			Resources:  []byte(resourceBytes),
 			DNSConfig:  s.dnsConfig,
 		})
 	_, err = s.services.Users.CreateClusterAdminAgent(s.cluster.Domain,
@@ -178,13 +177,16 @@ func (s *PlanSuite) SetUpSuite(c *check.C) {
 		UID:  999,
 		GID:  999,
 	}
+	runtimeResources, clusterResources, err := resources.Split(bytes.NewReader(resourceBytes))
+	c.Assert(err, check.IsNil)
 	s.installer = &Installer{
 		Config: Config{
-			Resources:   resourceBytes,
-			ServiceUser: s.serviceUser,
-			Mode:        constants.InstallModeCLI,
-			DNSConfig:   s.dnsConfig,
-			Process:     &mockProcess{},
+			RuntimeResources: runtimeResources,
+			ClusterResources: clusterResources,
+			ServiceUser:      s.serviceUser,
+			Mode:             constants.InstallModeCLI,
+			DNSConfig:        s.dnsConfig,
+			Process:          &mockProcess{},
 		},
 		FieldLogger: logrus.WithField(trace.Component, "plan-suite"),
 		AppPackage:  appPackage,
@@ -452,12 +454,12 @@ func (s *PlanSuite) verifyResourcesPhase(c *check.C, phase storage.OperationPhas
 	expected := []byte(`
 {
   "apiVersion": "v1",
-  "data": {
-    "test-key": "test-value"
-  },
   "kind": "ConfigMap",
   "metadata": {
     "name": "test-config"
+  },
+  "data": {
+    "test-key": "test-value"
   }
 }
 {
@@ -726,8 +728,7 @@ func decode(c *check.C, resources []storage.UnknownResource) (result []resource)
 		result = append(result, resource)
 	}
 	sort.Slice(result, func(i, j int) bool {
-		return result[i].Kind < result[j].Kind &&
-			result[i].Metadata.Name < result[j].Metadata.Name
+		return result[i].Metadata.Name < result[j].Metadata.Name
 	})
 	return result
 }

lib/install/planbuilder.go

@@ -602,7 +602,7 @@ func (i *Installer) GetPlanBuilder(cluster ops.Site, op ops.SiteOperation) (*Pla
 		},
 		InstallerTrustedCluster: trustedCluster,
 	}
-	err = addResources(builder, cluster.Resources, i.Config.RuntimeResources)
+	err = addResources(builder, cluster.Resources, i.Config.RuntimeResources, i.Config.ClusterResources)
 	if err != nil {
 		return nil, trace.Wrap(err)
 	}
@@ -664,12 +664,13 @@ func (b *PlanBuilder) skipDependency(dep loc.Locator) bool {
 	return schema.ShouldSkipApp(b.Application.Manifest, dep)
 }
 
-func addResources(builder *PlanBuilder, resourceBytes []byte, runtimeResources []runtime.Object) error {
+func addResources(builder *PlanBuilder, resourceBytes []byte, runtimeResources []runtime.Object, clusterResources []storage.UnknownResource) error {
 	kubernetesResources, gravityResources, err := resourceutil.Split(bytes.NewReader(resourceBytes))
 	if err != nil {
 		return trace.Wrap(err)
 	}
-	rs := gravityResources[:0]
+	gravityResources = append(gravityResources, clusterResources...)
+	rest := gravityResources[:0]
 	for _, res := range gravityResources {
 		switch res.Kind {
 		case storage.KindRuntimeEnvironment:
@@ -686,10 +687,10 @@ func addResources(builder *PlanBuilder, resourceBytes []byte, runtimeResources [
 			kubernetesResources = append(kubernetesResources, configmap)
 		default:
 			// Filter out resources that are created using the regular workflow
-			rs = append(rs, res)
+			rest = append(rest, res)
 		}
 	}
-	builder.gravityResources = rs
+	builder.gravityResources = rest
 	kubernetesResources = append(kubernetesResources, runtimeResources...)
 	if len(kubernetesResources) != 0 {
 		var buf bytes.Buffer

lib/ops/operatoracl.go

@@ -801,6 +801,15 @@ func (o *OperatorACL) GetClusterEnvironmentVariables(key SiteKey) (storage.Envir
 	return o.operator.GetClusterEnvironmentVariables(key)
 }
 
+// UpdateClusterEnvironmentVariables updates the cluster runtime environment variables
+// from the specified request
+func (o *OperatorACL) UpdateClusterEnvironmentVariables(req UpdateClusterEnvironRequest) error {
+	if err := o.ClusterAction(req.ClusterKey.SiteDomain, storage.KindClusterConfiguration, teleservices.VerbUpdate); err != nil {
+		return trace.Wrap(err)
+	}
+	return o.operator.UpdateClusterEnvironmentVariables(req)
+}
+
 // GetClusterConfiguration retrieves the cluster configuration
 func (o *OperatorACL) GetClusterConfiguration(key SiteKey) (clusterconfig.Interface, error) {
 	if err := o.ClusterAction(key.SiteDomain, storage.KindClusterConfiguration, teleservices.VerbList); err != nil {

lib/ops/ops.go

@@ -485,6 +485,9 @@ type RuntimeEnvironment interface {
 	CreateUpdateEnvarsOperation(CreateUpdateEnvarsOperationRequest) (*SiteOperationKey, error)
 	// GetClusterEnvironmentVariables retrieves the cluster runtime environment variables
 	GetClusterEnvironmentVariables(SiteKey) (storage.EnvironmentVariables, error)
+	// UpdateClusterEnvironmentVariables updates the cluster runtime environment variables
+	// from the specified request
+	UpdateClusterEnvironmentVariables(UpdateClusterEnvironRequest) error
 }
 
 // ClusterConfiguration manages configuration in cluster
@@ -1204,13 +1207,22 @@ type CreateUpdateConfigOperationRequest struct {
 	Config []byte `json:"config"`
 }
 
+// UpdateClusterEnvironRequest is a request
+// to update cluster runtime environment
+type UpdateClusterEnvironRequest struct {
+	// ClusterKey identifies the cluster
+	ClusterKey SiteKey `json:"cluster_key"`
+	// Env specifies the new runtime environment
+	Env map[string]string `json:"env,omitempty"`
+}
+
 // UpdateClusterConfigRequest is a request
 // to update cluster configuration
 type UpdateClusterConfigRequest struct {
 	// ClusterKey identifies the cluster
 	ClusterKey SiteKey `json:"cluster_key"`
 	// Config specifies the new configuration as JSON-encoded payload
-	Config []byte `json:"config"`
+	Config []byte `json:"config,omitempty"`
 }
 
 // AgentService coordinates install agents that are started on every server

lib/ops/opsclient/opsclient.go

@@ -1140,6 +1140,18 @@ func (c *Client) GetClusterEnvironmentVariables(key ops.SiteKey) (storage.Enviro
 	return env, nil
 }
 
+// UpdateClusterEnvironmentVariables updates the cluster runtime environment variables
+// from the specified request
+func (c *Client) UpdateClusterEnvironmentVariables(req ops.UpdateClusterEnvironRequest) error {
+	_, err := c.PutJSON(c.Endpoint(
+		"accounts", req.ClusterKey.AccountID, "sites", req.ClusterKey.SiteDomain, "envars"),
+		&req)
+	if err != nil {
+		return trace.Wrap(err)
+	}
+	return nil
+}
+
 // GetClusterConfiguration retrieves the cluster configuration
 func (c *Client) GetClusterConfiguration(key ops.SiteKey) (clusterconfig.Interface, error) {
 	response, err := c.Get(c.Endpoint(
@@ -1162,7 +1174,7 @@ func (c *Client) GetClusterConfiguration(key ops.SiteKey) (clusterconfig.Interfa
 func (c *Client) UpdateClusterConfiguration(req ops.UpdateClusterConfigRequest) error {
 	_, err := c.PutJSON(c.Endpoint(
 		"accounts", req.ClusterKey.AccountID, "sites", req.ClusterKey.SiteDomain, "config"),
-		&UpsertResourceRawReq{Resource: req.Config})
+		&req)
 	if err != nil {
 		return trace.Wrap(err)
 	}

lib/ops/opshandler/environment.go

@@ -78,3 +78,34 @@ func (h *WebHandler) getEnvironmentVariables(w http.ResponseWriter, r *http.Requ
 	bytes, err := storage.MarshalEnvironment(env)
 	return trace.Wrap(rawMessage(w, bytes, err))
 }
+
+/* updateEnvironmentVariables updates the cluster runtime environment
+
+   PUT /portal/v1/accounts/:account_id/sites/:site_domain/envars
+
+   {
+      "account_id": "account id",
+      "site_id": "site_id",
+      "env": "<new environ>"
+   }
+
+Success response:
+
+   {
+      "message": "cluster runtime environment updated",
+   }
+*/
+func (h *WebHandler) updateEnvironmentVariables(w http.ResponseWriter, r *http.Request, p httprouter.Params, context *HandlerContext) error {
+	d := json.NewDecoder(r.Body)
+	var req ops.UpdateClusterEnvironRequest
+	if err := d.Decode(&req); err != nil {
+		return trace.BadParameter(err.Error())
+	}
+	req.ClusterKey = siteKey(p)
+	err := context.Operator.UpdateClusterEnvironmentVariables(req)
+	if err != nil {
+		return trace.Wrap(err)
+	}
+	roundtrip.ReplyJSON(w, http.StatusOK, statusOK("cluster runtime environment updated"))
+	return nil
+}

lib/ops/opshandler/opshandler.go

@@ -236,6 +236,7 @@ func NewWebHandler(cfg WebHandlerConfig) (*WebHandler, error) {
 
 	// environment variables
 	h.GET("/portal/v1/accounts/:account_id/sites/:site_domain/envars", h.needsAuth(h.getEnvironmentVariables))
+	h.PUT("/portal/v1/accounts/:account_id/sites/:site_domain/envars", h.needsAuth(h.updateEnvironmentVariables))
 	h.POST("/portal/v1/accounts/:account_id/sites/:site_domain/operations/envars", h.needsAuth(h.createUpdateEnvarsOperation))
 
 	// cluster configuration

lib/ops/opsroute/forward.go

@@ -674,6 +674,16 @@ func (r *Router) GetClusterEnvironmentVariables(key ops.SiteKey) (storage.Enviro
 	return client.GetClusterEnvironmentVariables(key)
 }
 
+// UpdateClusterEnvironmentVariables updates the cluster runtime environment variables
+// from the specified request
+func (r *Router) UpdateClusterEnvironmentVariables(req ops.UpdateClusterEnvironRequest) error {
+	client, err := r.RemoteClient(req.ClusterKey.SiteDomain)
+	if err != nil {
+		return trace.Wrap(err)
+	}
+	return client.UpdateClusterEnvironmentVariables(req)
+}
+
 // GetClusterConfiguration retrieves the cluster configuration
 func (r *Router) GetClusterConfiguration(key ops.SiteKey) (clusterconfig.Interface, error) {
 	client, err := r.RemoteClient(key.SiteDomain)

lib/ops/opsservice/clusterconfig.go

@@ -68,7 +68,7 @@ func (o *Operator) GetClusterConfiguration(ops.SiteKey) (config clusterconfig.In
 			return nil, trace.Wrap(err)
 		}
 	} else {
-		config = clusterconfig.New()
+		config = clusterconfig.NewEmpty()
 	}
 	return config, nil
 }