diff --git a/Jenkinsfile b/Jenkinsfile index d980a0827c..8aa48a8eb7 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -40,7 +40,7 @@ properties([ defaultValue: 'stable-gce', description: 'Robotest tag to use.'), choice(choices: ["true", "false"].join("\n"), - defaultValue: 'true', + defaultValue: 'false', description: 'Whether to use preemptible VMs.', name: 'GCE_PREEMPTIBLE'), choice(choices: ["custom-4-8192", "custom-8-8192"].join("\n"), diff --git a/Makefile b/Makefile index 9a919191e5..c87e1aad7f 100644 --- a/Makefile +++ b/Makefile @@ -42,7 +42,7 @@ RELEASE_OUT ?= TELEPORT_TAG = 3.0.5 # TELEPORT_REPOTAG adapts TELEPORT_TAG to the teleport tagging scheme TELEPORT_REPOTAG := v$(TELEPORT_TAG) -PLANET_TAG := 5.5.12-$(K8S_VER_SUFFIX)-1-ga3d3820 +PLANET_TAG := 5.5.13-$(K8S_VER_SUFFIX) PLANET_BRANCH := $(PLANET_TAG) K8S_APP_TAG := $(GRAVITY_TAG) TELEKUBE_APP_TAG := $(GRAVITY_TAG) diff --git a/docs/5.x/changelog.md b/docs/5.x/changelog.md index 3619201dc7..9a8bdb1f73 100644 --- a/docs/5.x/changelog.md +++ b/docs/5.x/changelog.md @@ -47,6 +47,7 @@ LTS starts with `3.51.0` with minor backwards compatible changes added over time #### Improvements * Introduce `ClusterConfiguration` resource, see [Configuring Cluster](/cluster/#cluster-configuration) for details. +* Introduce `RuntimeEnvironment` resource, see [Configuring Runtime Environment Variables](/cluster/#configuring-runtime-environment-variables) for details. * Update 'gravity plan' to support all cluster operations. ### 5.5.0-beta.2 diff --git a/e b/e index 20a7fbdf85..6746011588 160000 --- a/e +++ b/e @@ -1 +1 @@ -Subproject commit 20a7fbdf85408418f8b3c54b93d942e4d5997c4d +Subproject commit 6746011588464f916f3fd6b45b7f5ac2a58e111b diff --git a/lib/install/flow.go b/lib/install/flow.go index b47162067a..206fc6a14e 100644 --- a/lib/install/flow.go +++ b/lib/install/flow.go @@ -111,7 +111,6 @@ func (i *Installer) NewClusterRequest() ops.NewSiteRequest { Email: fmt.Sprintf("installer@%v", i.SiteDomain), Provider: i.CloudProvider, DomainName: i.SiteDomain, - Resources: i.Resources, InstallToken: i.Config.Token, ServiceUser: storage.OSUser{ Name: i.Config.ServiceUser.Name, diff --git a/lib/install/install.go b/lib/install/install.go index a091f180b0..f4728c322c 100644 --- a/lib/install/install.go +++ b/lib/install/install.go @@ -162,12 +162,14 @@ type Config struct { Role string // AppPackage is the application being installed AppPackage *loc.Locator - // Resources specifies optional resources to create. - // This is assumed to be either JSON- or YAML-encoded list of resources - Resources []byte // RuntimeResources specifies optional Kubernetes resources to create // If specified, will be combined with Resources RuntimeResources []runtime.Object + // ClusterResources specifies optional cluster resources to create + // If specified, will be combined with Resources + // TODO(dmitri): externalize the ClusterConfiguration resource and create + // default provider-specific cloud-config on Gravity side + ClusterResources []storage.UnknownResource // EventsC is channel with events indicating install progress EventsC chan Event // SystemDevice is a device for gravity data @@ -283,6 +285,7 @@ func (c *Config) validateCloudConfig() error { if c.CloudProvider != schema.ProviderGCE { return nil } + // TODO(dmitri): skip validations if user provided custom cloud configuration if err := cloudgce.ValidateTag(c.SiteDomain); err != nil { log.WithError(err).Warnf("Failed to validate cluster name %v as node tag on GCE.", c.SiteDomain) if len(c.GCENodeTags) == 0 { diff --git a/lib/install/plan_test.go b/lib/install/plan_test.go index dc89922438..dc561bd04a 100644 --- a/lib/install/plan_test.go +++ b/lib/install/plan_test.go @@ -125,7 +125,6 @@ func (s *PlanSuite) SetUpSuite(c *check.C) { DomainName: "example.com", AppPackage: appPackage.String(), Provider: schema.ProviderAWS, - Resources: []byte(resourceBytes), DNSConfig: s.dnsConfig, }) _, err = s.services.Users.CreateClusterAdminAgent(s.cluster.Domain, @@ -178,13 +177,16 @@ func (s *PlanSuite) SetUpSuite(c *check.C) { UID: 999, GID: 999, } + runtimeResources, clusterResources, err := resources.Split(bytes.NewReader(resourceBytes)) + c.Assert(err, check.IsNil) s.installer = &Installer{ Config: Config{ - Resources: resourceBytes, - ServiceUser: s.serviceUser, - Mode: constants.InstallModeCLI, - DNSConfig: s.dnsConfig, - Process: &mockProcess{}, + RuntimeResources: runtimeResources, + ClusterResources: clusterResources, + ServiceUser: s.serviceUser, + Mode: constants.InstallModeCLI, + DNSConfig: s.dnsConfig, + Process: &mockProcess{}, }, FieldLogger: logrus.WithField(trace.Component, "plan-suite"), AppPackage: appPackage, @@ -452,12 +454,12 @@ func (s *PlanSuite) verifyResourcesPhase(c *check.C, phase storage.OperationPhas expected := []byte(` { "apiVersion": "v1", - "data": { - "test-key": "test-value" - }, "kind": "ConfigMap", "metadata": { "name": "test-config" + }, + "data": { + "test-key": "test-value" } } { @@ -726,8 +728,7 @@ func decode(c *check.C, resources []storage.UnknownResource) (result []resource) result = append(result, resource) } sort.Slice(result, func(i, j int) bool { - return result[i].Kind < result[j].Kind && - result[i].Metadata.Name < result[j].Metadata.Name + return result[i].Metadata.Name < result[j].Metadata.Name }) return result } diff --git a/lib/install/planbuilder.go b/lib/install/planbuilder.go index 741de9fa0e..0662e07242 100644 --- a/lib/install/planbuilder.go +++ b/lib/install/planbuilder.go @@ -602,7 +602,7 @@ func (i *Installer) GetPlanBuilder(cluster ops.Site, op ops.SiteOperation) (*Pla }, InstallerTrustedCluster: trustedCluster, } - err = addResources(builder, cluster.Resources, i.Config.RuntimeResources) + err = addResources(builder, cluster.Resources, i.Config.RuntimeResources, i.Config.ClusterResources) if err != nil { return nil, trace.Wrap(err) } @@ -664,12 +664,13 @@ func (b *PlanBuilder) skipDependency(dep loc.Locator) bool { return schema.ShouldSkipApp(b.Application.Manifest, dep) } -func addResources(builder *PlanBuilder, resourceBytes []byte, runtimeResources []runtime.Object) error { +func addResources(builder *PlanBuilder, resourceBytes []byte, runtimeResources []runtime.Object, clusterResources []storage.UnknownResource) error { kubernetesResources, gravityResources, err := resourceutil.Split(bytes.NewReader(resourceBytes)) if err != nil { return trace.Wrap(err) } - rs := gravityResources[:0] + gravityResources = append(gravityResources, clusterResources...) + rest := gravityResources[:0] for _, res := range gravityResources { switch res.Kind { case storage.KindRuntimeEnvironment: @@ -686,10 +687,10 @@ func addResources(builder *PlanBuilder, resourceBytes []byte, runtimeResources [ kubernetesResources = append(kubernetesResources, configmap) default: // Filter out resources that are created using the regular workflow - rs = append(rs, res) + rest = append(rest, res) } } - builder.gravityResources = rs + builder.gravityResources = rest kubernetesResources = append(kubernetesResources, runtimeResources...) if len(kubernetesResources) != 0 { var buf bytes.Buffer diff --git a/lib/ops/operatoracl.go b/lib/ops/operatoracl.go index 13c74e625b..e936b022c9 100644 --- a/lib/ops/operatoracl.go +++ b/lib/ops/operatoracl.go @@ -801,6 +801,15 @@ func (o *OperatorACL) GetClusterEnvironmentVariables(key SiteKey) (storage.Envir return o.operator.GetClusterEnvironmentVariables(key) } +// UpdateClusterEnvironmentVariables updates the cluster runtime environment variables +// from the specified request +func (o *OperatorACL) UpdateClusterEnvironmentVariables(req UpdateClusterEnvironRequest) error { + if err := o.ClusterAction(req.ClusterKey.SiteDomain, storage.KindClusterConfiguration, teleservices.VerbUpdate); err != nil { + return trace.Wrap(err) + } + return o.operator.UpdateClusterEnvironmentVariables(req) +} + // GetClusterConfiguration retrieves the cluster configuration func (o *OperatorACL) GetClusterConfiguration(key SiteKey) (clusterconfig.Interface, error) { if err := o.ClusterAction(key.SiteDomain, storage.KindClusterConfiguration, teleservices.VerbList); err != nil { diff --git a/lib/ops/ops.go b/lib/ops/ops.go index b80e0ab5b0..c161dde6c3 100644 --- a/lib/ops/ops.go +++ b/lib/ops/ops.go @@ -485,6 +485,9 @@ type RuntimeEnvironment interface { CreateUpdateEnvarsOperation(CreateUpdateEnvarsOperationRequest) (*SiteOperationKey, error) // GetClusterEnvironmentVariables retrieves the cluster runtime environment variables GetClusterEnvironmentVariables(SiteKey) (storage.EnvironmentVariables, error) + // UpdateClusterEnvironmentVariables updates the cluster runtime environment variables + // from the specified request + UpdateClusterEnvironmentVariables(UpdateClusterEnvironRequest) error } // ClusterConfiguration manages configuration in cluster @@ -1204,13 +1207,22 @@ type CreateUpdateConfigOperationRequest struct { Config []byte `json:"config"` } +// UpdateClusterEnvironRequest is a request +// to update cluster runtime environment +type UpdateClusterEnvironRequest struct { + // ClusterKey identifies the cluster + ClusterKey SiteKey `json:"cluster_key"` + // Env specifies the new runtime environment + Env map[string]string `json:"env,omitempty"` +} + // UpdateClusterConfigRequest is a request // to update cluster configuration type UpdateClusterConfigRequest struct { // ClusterKey identifies the cluster ClusterKey SiteKey `json:"cluster_key"` // Config specifies the new configuration as JSON-encoded payload - Config []byte `json:"config"` + Config []byte `json:"config,omitempty"` } // AgentService coordinates install agents that are started on every server diff --git a/lib/ops/opsclient/opsclient.go b/lib/ops/opsclient/opsclient.go index d3bee52a22..3acc31f590 100644 --- a/lib/ops/opsclient/opsclient.go +++ b/lib/ops/opsclient/opsclient.go @@ -1140,6 +1140,18 @@ func (c *Client) GetClusterEnvironmentVariables(key ops.SiteKey) (storage.Enviro return env, nil } +// UpdateClusterEnvironmentVariables updates the cluster runtime environment variables +// from the specified request +func (c *Client) UpdateClusterEnvironmentVariables(req ops.UpdateClusterEnvironRequest) error { + _, err := c.PutJSON(c.Endpoint( + "accounts", req.ClusterKey.AccountID, "sites", req.ClusterKey.SiteDomain, "envars"), + &req) + if err != nil { + return trace.Wrap(err) + } + return nil +} + // GetClusterConfiguration retrieves the cluster configuration func (c *Client) GetClusterConfiguration(key ops.SiteKey) (clusterconfig.Interface, error) { response, err := c.Get(c.Endpoint( @@ -1162,7 +1174,7 @@ func (c *Client) GetClusterConfiguration(key ops.SiteKey) (clusterconfig.Interfa func (c *Client) UpdateClusterConfiguration(req ops.UpdateClusterConfigRequest) error { _, err := c.PutJSON(c.Endpoint( "accounts", req.ClusterKey.AccountID, "sites", req.ClusterKey.SiteDomain, "config"), - &UpsertResourceRawReq{Resource: req.Config}) + &req) if err != nil { return trace.Wrap(err) } diff --git a/lib/ops/opshandler/environment.go b/lib/ops/opshandler/environment.go index 23aa44579f..ee0aa9be72 100644 --- a/lib/ops/opshandler/environment.go +++ b/lib/ops/opshandler/environment.go @@ -78,3 +78,34 @@ func (h *WebHandler) getEnvironmentVariables(w http.ResponseWriter, r *http.Requ bytes, err := storage.MarshalEnvironment(env) return trace.Wrap(rawMessage(w, bytes, err)) } + +/* updateEnvironmentVariables updates the cluster runtime environment + + PUT /portal/v1/accounts/:account_id/sites/:site_domain/envars + + { + "account_id": "account id", + "site_id": "site_id", + "env": "" + } + +Success response: + + { + "message": "cluster runtime environment updated", + } +*/ +func (h *WebHandler) updateEnvironmentVariables(w http.ResponseWriter, r *http.Request, p httprouter.Params, context *HandlerContext) error { + d := json.NewDecoder(r.Body) + var req ops.UpdateClusterEnvironRequest + if err := d.Decode(&req); err != nil { + return trace.BadParameter(err.Error()) + } + req.ClusterKey = siteKey(p) + err := context.Operator.UpdateClusterEnvironmentVariables(req) + if err != nil { + return trace.Wrap(err) + } + roundtrip.ReplyJSON(w, http.StatusOK, statusOK("cluster runtime environment updated")) + return nil +} diff --git a/lib/ops/opshandler/opshandler.go b/lib/ops/opshandler/opshandler.go index c919fde96b..25a4827751 100644 --- a/lib/ops/opshandler/opshandler.go +++ b/lib/ops/opshandler/opshandler.go @@ -236,6 +236,7 @@ func NewWebHandler(cfg WebHandlerConfig) (*WebHandler, error) { // environment variables h.GET("/portal/v1/accounts/:account_id/sites/:site_domain/envars", h.needsAuth(h.getEnvironmentVariables)) + h.PUT("/portal/v1/accounts/:account_id/sites/:site_domain/envars", h.needsAuth(h.updateEnvironmentVariables)) h.POST("/portal/v1/accounts/:account_id/sites/:site_domain/operations/envars", h.needsAuth(h.createUpdateEnvarsOperation)) // cluster configuration diff --git a/lib/ops/opsroute/forward.go b/lib/ops/opsroute/forward.go index 577adba54a..3dd824a2da 100644 --- a/lib/ops/opsroute/forward.go +++ b/lib/ops/opsroute/forward.go @@ -674,6 +674,16 @@ func (r *Router) GetClusterEnvironmentVariables(key ops.SiteKey) (storage.Enviro return client.GetClusterEnvironmentVariables(key) } +// UpdateClusterEnvironmentVariables updates the cluster runtime environment variables +// from the specified request +func (r *Router) UpdateClusterEnvironmentVariables(req ops.UpdateClusterEnvironRequest) error { + client, err := r.RemoteClient(req.ClusterKey.SiteDomain) + if err != nil { + return trace.Wrap(err) + } + return client.UpdateClusterEnvironmentVariables(req) +} + // GetClusterConfiguration retrieves the cluster configuration func (r *Router) GetClusterConfiguration(key ops.SiteKey) (clusterconfig.Interface, error) { client, err := r.RemoteClient(key.SiteDomain) diff --git a/lib/ops/opsservice/clusterconfig.go b/lib/ops/opsservice/clusterconfig.go index 67a2a3c980..8ccc5b6a42 100644 --- a/lib/ops/opsservice/clusterconfig.go +++ b/lib/ops/opsservice/clusterconfig.go @@ -68,7 +68,7 @@ func (o *Operator) GetClusterConfiguration(ops.SiteKey) (config clusterconfig.In return nil, trace.Wrap(err) } } else { - config = clusterconfig.New() + config = clusterconfig.NewEmpty() } return config, nil } diff --git a/lib/ops/opsservice/configure.go b/lib/ops/opsservice/configure.go index 3ba3259eec..e39422114e 100644 --- a/lib/ops/opsservice/configure.go +++ b/lib/ops/opsservice/configure.go @@ -325,6 +325,9 @@ func (s *site) configurePackages(ctx *operationContext, req ops.ConfigurePackage if err != nil { return trace.Wrap(err) } + if s.cloudProviderName() != "" { + clusterConfig.SetCloudProvider(s.cloudProviderName()) + } } for i, master := range masters { @@ -923,7 +926,8 @@ func (s *site) getPlanetConfig(config planetConfig) (args []string, err error) { args = append(args, fmt.Sprintf("--env=%v=%v", k, v)) } - args = append(args, s.addClusterConfig(config, overrideArgs)...) + args = append(args, s.addCloudConfig(config.config)...) + args = append(args, s.addClusterConfig(config.config, overrideArgs)...) if node.IsMaster() { args = append(args, "--role=master") @@ -1027,7 +1031,7 @@ func (s *site) getPlanetConfig(config planetConfig) (args []string, err error) { args = append(args, fmt.Sprintf("--%v=%v", k, v)) } - log.WithField("args", args).Debug("Runtime configuration.") + log.WithField("args", args).Info("Runtime configuration.") return args, nil } @@ -1478,44 +1482,40 @@ func (s *site) serverPackages(server *ProvisionedServer) ([]loc.Locator, error) }, nil } -func (s *site) addClusterConfig(planetConfig planetConfig, overrideArgs map[string]string) (args []string) { - if planetConfig.config == nil { - if s.cloudProviderName() != "" { - args = append(args, fmt.Sprintf("--cloud-provider=%v", s.cloudProviderName())) - if s.cloudProviderName() == schema.ProviderGCE { - args = append(args, fmt.Sprintf("--gce-node-tags=%v", s.gceNodeTags())) - } - return args - } +func (s *site) addCloudConfig(config clusterconfig.Interface) (args []string) { + if s.cloudProviderName() == "" { return nil } - - if config := planetConfig.config.GetKubeletConfig(); config != nil { - args = append(args, fmt.Sprintf("--kubelet-config=%v", - base64.StdEncoding.EncodeToString(config.Config))) + args = append(args, fmt.Sprintf("--cloud-provider=%v", s.cloudProviderName())) + var cloudConfig string + if config != nil { + if globalConfig := config.GetGlobalConfig(); globalConfig != nil { + cloudConfig = globalConfig.CloudConfig + } } + if cloudConfig != "" { + args = append(args, fmt.Sprintf("--cloud-config=%v", + base64.StdEncoding.EncodeToString([]byte(cloudConfig)))) + } else if s.cloudProviderName() == schema.ProviderGCE { + args = append(args, fmt.Sprintf("--gce-node-tags=%v", s.gceNodeTags())) + } + return args +} - globalConfig := planetConfig.config.GetGlobalConfig() - cloudProvider := s.cloudProviderName() - if globalConfig != nil { - cloudProvider = globalConfig.CloudProvider +func (s *site) addClusterConfig(config clusterconfig.Interface, overrideArgs map[string]string) (args []string) { + if config == nil { + return nil } - if cloudProvider != "" { - args = append(args, fmt.Sprintf("--cloud-provider=%v", cloudProvider)) - if cloudProvider == schema.ProviderGCE { - args = append(args, fmt.Sprintf("--gce-node-tags=%v", s.gceNodeTags())) - } - if globalConfig != nil { - args = append(args, fmt.Sprintf("--cloud-config=%v", - base64.StdEncoding.EncodeToString([]byte(globalConfig.CloudConfig)))) - } + if config := config.GetKubeletConfig(); config != nil { + args = append(args, fmt.Sprintf("--kubelet-config=%v", + base64.StdEncoding.EncodeToString(config.Config))) } + globalConfig := config.GetGlobalConfig() if globalConfig == nil { return args } - if globalConfig.ServiceCIDR != "" { overrideArgs["service-subnet"] = globalConfig.ServiceCIDR } diff --git a/lib/ops/opsservice/configure_test.go b/lib/ops/opsservice/configure_test.go index f781a6c479..2b0afc19ee 100644 --- a/lib/ops/opsservice/configure_test.go +++ b/lib/ops/opsservice/configure_test.go @@ -46,6 +46,7 @@ var _ = check.Suite(&ConfigureSuite{}) func (s *ConfigureSuite) SetUpTest(c *check.C) { s.cluster = &site{ domainName: "example.com", + provider: "gce", backendSite: &storage.Site{ CloudConfig: storage.CloudConfig{ GCENodeTags: []string{"example-com"}, @@ -154,8 +155,8 @@ func (s *ConfigureSuite) TestGeneratesPlanetConfigPackage(c *check.C) { CloudProvider: "gce", CloudConfig: ` [global] -username=user -password=pass`, +node-tags=example-com +multizone=true`, FeatureGates: map[string]bool{ "FeatureA": true, "FeatureB": false, @@ -168,20 +169,20 @@ password=pass`, c.Assert(err, check.IsNil) features, args := stripItem(args, "--feature-gates") c.Assert(sort.StringSlice(args), compare.SortedSliceEquals, mapToArgs(map[string][]string{ - "node-name": []string{"172.12.13.0"}, - "hostname": []string{"node-1"}, - "master-ip": []string{"172.12.13.0"}, - "public-ip": []string{"172.12.13.0"}, - "cluster-id": []string{"example.com"}, - "etcd-proxy": []string{"off"}, - "etcd-member-name": []string{"172_12_13_0.example.com"}, - "initial-cluster": []string{"172.12.13.0.example.com"}, - "etcd-initial-cluster-state": []string{"new"}, - "secrets-dir": []string{"/var/lib/gravity/secrets"}, - "election-enabled": []string{"true"}, - "service-uid": []string{"1000"}, - "env": []string{"VAR=value", "VAR2=value2"}, - "volume": []string{ + "node-name": {"172.12.13.0"}, + "hostname": {"node-1"}, + "master-ip": {"172.12.13.0"}, + "public-ip": {"172.12.13.0"}, + "cluster-id": {"example.com"}, + "etcd-proxy": {"off"}, + "etcd-member-name": {"172_12_13_0.example.com"}, + "initial-cluster": {"172.12.13.0.example.com"}, + "etcd-initial-cluster-state": {"new"}, + "secrets-dir": {"/var/lib/gravity/secrets"}, + "election-enabled": {"true"}, + "service-uid": {"1000"}, + "env": {"VAR=value", "VAR2=value2"}, + "volume": { "/var/lib/gravity/planet/etcd:/ext/etcd", "/var/lib/gravity/planet/docker:/ext/docker", "/var/lib/gravity/planet/registry:/ext/registry", @@ -190,20 +191,19 @@ password=pass`, "/var/lib/gravity/planet/log:/var/log", "/var/lib/gravity:/var/lib/gravity", }, - "cloud-provider": []string{"gce"}, - "cloud-config": []string{base64.StdEncoding.EncodeToString([]byte("\n[global]\nusername=user\npassword=pass"))}, - "gce-node-tags": []string{"example-com"}, - "role": []string{"node"}, - "docker-promiscuous-mode": []string{"true"}, - "dns-listen-addr": []string{"127.0.0.2"}, - "dns-port": []string{"53"}, - "docker-backend": []string{"overlay2"}, - "docker-options": []string{"--storage-opt=overlay2.override_kernel_check=1"}, - "kubelet-options": []string{"--hairpin-mode=none"}, - "kubelet-config": []string{base64.StdEncoding.EncodeToString(configBytes)}, - "node-label": []string{"gravitational.io/advertise-ip=172.12.13.0"}, - "service-subnet": []string{"10.0.0.1/8"}, - "pod-subnet": []string{"10.0.1.1/8"}, + "cloud-provider": {"gce"}, + "cloud-config": {base64.StdEncoding.EncodeToString([]byte("\n[global]\nnode-tags=example-com\nmultizone=true"))}, + "role": {"node"}, + "docker-promiscuous-mode": {"true"}, + "dns-listen-addr": {"127.0.0.2"}, + "dns-port": {"53"}, + "docker-backend": {"overlay2"}, + "docker-options": {"--storage-opt=overlay2.override_kernel_check=1"}, + "kubelet-options": {"--hairpin-mode=none"}, + "kubelet-config": {base64.StdEncoding.EncodeToString(configBytes)}, + "node-label": {"gravitational.io/advertise-ip=172.12.13.0"}, + "service-subnet": {"10.0.0.1/8"}, + "pod-subnet": {"10.0.1.1/8"}, })) assertFeatures(features, []string{"FeatureA=true", "FeatureB=false"}, c) } @@ -264,19 +264,19 @@ func (s *ConfigureSuite) TestCanSetCloudProviderWithoutCloudConfig(c *check.C) { args, err := s.cluster.getPlanetConfig(config) c.Assert(err, check.IsNil) c.Assert(sort.StringSlice(args), compare.SortedSliceEquals, mapToArgs(map[string][]string{ - "node-name": []string{"172.12.13.0"}, - "hostname": []string{"node-1"}, - "master-ip": []string{"172.12.13.0"}, - "public-ip": []string{"172.12.13.0"}, - "cluster-id": []string{"example.com"}, - "etcd-proxy": []string{"off"}, - "etcd-member-name": []string{"172_12_13_0.example.com"}, - "initial-cluster": []string{"172.12.13.0.example.com"}, - "etcd-initial-cluster-state": []string{"new"}, - "secrets-dir": []string{"/var/lib/gravity/secrets"}, - "election-enabled": []string{"true"}, - "service-uid": []string{"1000"}, - "volume": []string{ + "node-name": {"172.12.13.0"}, + "hostname": {"node-1"}, + "master-ip": {"172.12.13.0"}, + "public-ip": {"172.12.13.0"}, + "cluster-id": {"example.com"}, + "etcd-proxy": {"off"}, + "etcd-member-name": {"172_12_13_0.example.com"}, + "initial-cluster": {"172.12.13.0.example.com"}, + "etcd-initial-cluster-state": {"new"}, + "secrets-dir": {"/var/lib/gravity/secrets"}, + "election-enabled": {"true"}, + "service-uid": {"1000"}, + "volume": { "/var/lib/gravity/planet/etcd:/ext/etcd", "/var/lib/gravity/planet/docker:/ext/docker", "/var/lib/gravity/planet/registry:/ext/registry", @@ -285,18 +285,18 @@ func (s *ConfigureSuite) TestCanSetCloudProviderWithoutCloudConfig(c *check.C) { "/var/lib/gravity/planet/log:/var/log", "/var/lib/gravity:/var/lib/gravity", }, - "cloud-provider": []string{"gce"}, - "gce-node-tags": []string{"example-com"}, - "role": []string{"node"}, - "docker-promiscuous-mode": []string{"true"}, - "dns-listen-addr": []string{"127.0.0.2"}, - "dns-port": []string{"53"}, - "docker-backend": []string{"overlay2"}, - "docker-options": []string{"--storage-opt=overlay2.override_kernel_check=1"}, - "kubelet-options": []string{"--hairpin-mode=none"}, - "node-label": []string{"gravitational.io/advertise-ip=172.12.13.0"}, - "service-subnet": []string{"10.0.0.1/8"}, - "pod-subnet": []string{"10.0.1.1/8"}, + "cloud-provider": {"gce"}, + "gce-node-tags": {"example-com"}, + "role": {"node"}, + "docker-promiscuous-mode": {"true"}, + "dns-listen-addr": {"127.0.0.2"}, + "dns-port": {"53"}, + "docker-backend": {"overlay2"}, + "docker-options": {"--storage-opt=overlay2.override_kernel_check=1"}, + "kubelet-options": {"--hairpin-mode=none"}, + "node-label": {"gravitational.io/advertise-ip=172.12.13.0"}, + "service-subnet": {"10.0.0.1/8"}, + "pod-subnet": {"10.0.1.1/8"}, })) } diff --git a/lib/ops/opsservice/environment.go b/lib/ops/opsservice/environment.go index af2ccce968..bdd2f87c2e 100644 --- a/lib/ops/opsservice/environment.go +++ b/lib/ops/opsservice/environment.go @@ -40,13 +40,15 @@ func (o *Operator) CreateUpdateEnvarsOperation(r ops.CreateUpdateEnvarsOperation if err != nil { return nil, trace.Wrap(err) } - cluster, err := o.openSite(r.ClusterKey) if err != nil { return nil, trace.Wrap(err) } - - key, err := cluster.createUpdateEnvarsOperation(r) + env, err := o.getClusterEnvironment() + if err != nil { + return nil, trace.Wrap(err) + } + key, err := cluster.createUpdateEnvarsOperation(r, env) if err != nil { return nil, trace.Wrap(err) } @@ -73,44 +75,69 @@ func (o *Operator) GetClusterEnvironmentVariables(key ops.SiteKey) (env storage. return env, nil } -// NewEnvironmentConfigMap creates the backing ConfigMap to host cluster runtime environment variables -func NewEnvironmentConfigMap(data map[string]string) *v1.ConfigMap { - return &v1.ConfigMap{ - TypeMeta: metav1.TypeMeta{ - Kind: constants.KindConfigMap, - APIVersion: metav1.SchemeGroupVersion.Version, - }, - ObjectMeta: metav1.ObjectMeta{ - Name: constants.ClusterEnvironmentMap, - Namespace: defaults.KubeSystemNamespace, - }, - Data: data, +func (o *Operator) getClusterEnvironment() (env map[string]string, err error) { + client, err := o.GetKubeClient() + if err != nil { + return nil, trace.Wrap(err) + } + configmap, err := client.CoreV1().ConfigMaps(defaults.KubeSystemNamespace). + Get(constants.ClusterEnvironmentMap, metav1.GetOptions{}) + err = rigging.ConvertError(err) + if err != nil && !trace.IsNotFound(err) { + return nil, trace.Wrap(err) } + return configmap.Data, nil } -// createUpdateEnvarsOperation creates a new operation to update cluster environment variables -func (s *site) createUpdateEnvarsOperation(req ops.CreateUpdateEnvarsOperationRequest) (*ops.SiteOperationKey, error) { - client, err := s.service.GetKubeClient() +// UpdateClusterEnvironmentVariables updates the cluster runtime environment variables +// from the specified request +func (o *Operator) UpdateClusterEnvironmentVariables(req ops.UpdateClusterEnvironRequest) error { + client, err := o.GetKubeClient() if err != nil { - return nil, trace.Wrap(err) + return trace.Wrap(err) } configmaps := client.CoreV1().ConfigMaps(defaults.KubeSystemNamespace) configmap, err := getOrCreateEnvironmentConfigMap(configmaps) if err != nil { - return nil, trace.Wrap(err) + return trace.Wrap(err) } var previousKeyValues []byte if len(configmap.Data) != 0 { var err error previousKeyValues, err = json.Marshal(configmap.Data) if err != nil { - return nil, trace.Wrap(err, "failed to marshal previous key/values") + return trace.Wrap(err, "failed to marshal previous key/values") } if configmap.Annotations == nil { configmap.Annotations = make(map[string]string) } configmap.Annotations[constants.PreviousKeyValuesAnnotationKey] = string(previousKeyValues) } + configmap.Data = req.Env + err = kubernetes.Retry(context.TODO(), func() error { + _, err := configmaps.Update(configmap) + return trace.Wrap(err) + }) + return trace.Wrap(err) +} + +// NewEnvironmentConfigMap creates the backing ConfigMap to host cluster runtime environment variables +func NewEnvironmentConfigMap(data map[string]string) *v1.ConfigMap { + return &v1.ConfigMap{ + TypeMeta: metav1.TypeMeta{ + Kind: constants.KindConfigMap, + APIVersion: metav1.SchemeGroupVersion.Version, + }, + ObjectMeta: metav1.ObjectMeta{ + Name: constants.ClusterEnvironmentMap, + Namespace: defaults.KubeSystemNamespace, + }, + Data: data, + } +} + +// createUpdateEnvarsOperation creates a new operation to update cluster environment variables +func (s *site) createUpdateEnvarsOperation(req ops.CreateUpdateEnvarsOperationRequest, prevEnv map[string]string) (*ops.SiteOperationKey, error) { op := ops.SiteOperation{ ID: uuid.New(), AccountID: s.key.AccountID, @@ -120,21 +147,14 @@ func (s *site) createUpdateEnvarsOperation(req ops.CreateUpdateEnvarsOperationRe Updated: s.clock().UtcNow(), State: ops.OperationUpdateRuntimeEnvironInProgress, UpdateEnviron: &storage.UpdateEnvarsOperationState{ - Env: req.Env, + PrevEnv: prevEnv, + Env: req.Env, }, } key, err := s.getOperationGroup().createSiteOperation(op) if err != nil { return nil, trace.Wrap(err) } - configmap.Data = req.Env - err = kubernetes.Retry(context.TODO(), func() error { - _, err := configmaps.Update(configmap) - return trace.Wrap(err) - }) - if err != nil { - return nil, trace.Wrap(err) - } return key, nil } diff --git a/lib/ops/resources/gravity/collection.go b/lib/ops/resources/gravity/collection.go index d576ba9fc3..0a0514d996 100644 --- a/lib/ops/resources/gravity/collection.go +++ b/lib/ops/resources/gravity/collection.go @@ -631,11 +631,14 @@ func (r configCollection) WriteText(w io.Writer) error { fmt.Fprintf(t, "%v\n", string(config.Config)) } if config := r.GetGlobalConfig(); config != nil { - common.PrintCustomTableHeader(t, []string{"Cloud"}, "-") - if len(config.CloudProvider) != 0 { - fmt.Fprintf(t, "Provider:\t%v\n", config.CloudProvider) + displayCloudConfig := config.CloudProvider != "" || config.CloudConfig != "" + if displayCloudConfig { + common.PrintCustomTableHeader(t, []string{"Cloud"}, "-") + if len(config.CloudProvider) != 0 { + fmt.Fprintf(t, "Provider:\t%v\n", config.CloudProvider) + } + formatCloudConfig(t, config.CloudConfig) } - formatCloudConfig(t, config.CloudConfig) if len(config.ServiceNodePortRange) != 0 { fmt.Fprintf(t, "Service Node Port Range:\t%v\n", config.ServiceNodePortRange) } diff --git a/lib/ops/resources/resources.go b/lib/ops/resources/resources.go index c20f6d89b4..958d77181d 100644 --- a/lib/ops/resources/resources.go +++ b/lib/ops/resources/resources.go @@ -155,6 +155,8 @@ func (r *RemoveRequest) Check() error { switch kind { case storage.KindAlertTarget: case storage.KindSMTPConfig: + case storage.KindRuntimeEnvironment: + case storage.KindClusterConfiguration: default: if r.Name == "" { return trace.BadParameter("resource name is mandatory") diff --git a/lib/storage/clusterconfig/clusterconfig.go b/lib/storage/clusterconfig/clusterconfig.go index 24a2376f6a..0131d14f04 100644 --- a/lib/storage/clusterconfig/clusterconfig.go +++ b/lib/storage/clusterconfig/clusterconfig.go @@ -39,18 +39,21 @@ type Interface interface { GetKubeletConfig() *Kubelet // GetGlobalConfig returns the global configuration GetGlobalConfig() *Global + // SetCloudProvider sets the cloud provider for this configuration + SetCloudProvider(provider string) } -// New returns a new instance of the resource initialized to defaults -func New() *Resource { - return &Resource{ - Kind: storage.KindClusterConfiguration, - Version: "v1", - Metadata: teleservices.Metadata{ - Name: constants.ClusterConfigurationMap, - Namespace: defaults.KubeSystemNamespace, - }, - } +// New returns a new instance of the resource initialized to specified spec +func New(spec Spec) *Resource { + res := newEmpty() + res.Spec = spec + return res +} + +// NewEmpty returns a new instance of the resource initialized to defaults +func NewEmpty() *Resource { + res := newEmpty() + return res } // Resource describes the cluster configuration resource @@ -106,6 +109,14 @@ func (r *Resource) GetGlobalConfig() *Global { return r.Spec.Global } +// SetCloudProvider sets the cloud provider for this configuration +func (r *Resource) SetCloudProvider(provider string) { + if r.Spec.Global == nil { + r.Spec.Global = &Global{} + } + r.Spec.Global.CloudProvider = provider +} + // Unmarshal unmarshals the resource from either YAML- or JSON-encoded data func Unmarshal(data []byte) (*Resource, error) { if len(data) == 0 { @@ -145,6 +156,23 @@ func Marshal(config Interface, opts ...teleservices.MarshalOption) ([]byte, erro return json.Marshal(config) } +// ToUnknown returns this resource as a storage.UnknownResource +func ToUnknown(config Interface) (*storage.UnknownResource, error) { + bytes, err := Marshal(config) + if err != nil { + return nil, trace.Wrap(err) + } + res := newEmpty() + return &storage.UnknownResource{ + ResourceHeader: teleservices.ResourceHeader{ + Kind: res.Kind, + Version: res.Version, + Metadata: res.Metadata, + }, + Raw: bytes, + }, nil +} + // Spec defines the cluster configuration resource type Spec struct { // ComponentsConfigs groups component configurations @@ -356,3 +384,14 @@ func getSpecSchema() string { return fmt.Sprintf(specSchemaTemplate, constants.ClusterConfigurationMap, defaults.KubeSystemNamespace) } + +func newEmpty() *Resource { + return &Resource{ + Kind: storage.KindClusterConfiguration, + Version: "v1", + Metadata: teleservices.Metadata{ + Name: constants.ClusterConfigurationMap, + Namespace: defaults.KubeSystemNamespace, + }, + } +} diff --git a/lib/storage/storage.go b/lib/storage/storage.go index abe531b429..4d28e93a07 100644 --- a/lib/storage/storage.go +++ b/lib/storage/storage.go @@ -1996,8 +1996,10 @@ type UpdateOperationState struct { // UpdateEnvarsOperationState describes the state of the operation to update cluster environment variables. type UpdateEnvarsOperationState struct { + // PrevEnv specifies the previous environment state + PrevEnv map[string]string `json:"prev_env,omitempty"` // Env defines new cluster environment variables - Env map[string]string `json:"env"` + Env map[string]string `json:"env,omitempty"` } // Package returns the update package locator @@ -2012,9 +2014,9 @@ func (s UpdateOperationState) Package() (*loc.Locator, error) { // UpdateConfigOperationState describes the state of the operation to update cluster configuration type UpdateConfigOperationState struct { // PrevConfig specifies the previous configuration state - PrevConfig []byte `json:"prev_config"` + PrevConfig []byte `json:"prev_config,omitempty"` // Config specifies the raw configuration resource - Config []byte `json:"config"` + Config []byte `json:"config,omitempty"` } // ServerUpdate represents server that is being updated diff --git a/lib/update/clusterconfig/plan_test.go b/lib/update/clusterconfig/plan_test.go index c87d5e5874..b1b07e1ca7 100644 --- a/lib/update/clusterconfig/plan_test.go +++ b/lib/update/clusterconfig/plan_test.go @@ -47,7 +47,7 @@ func (S) TestSingleNodePlan(c *C) { {Hostname: "node-1", ClusterRole: string(schema.ServiceRoleMaster)}, } app := loc.MustParseLocator("gravitational.io/app:0.0.1") - clusterConfig := clusterconfig.New() + clusterConfig := clusterconfig.NewEmpty() plan, err := newOperationPlan(app, storage.DefaultDNSConfig, operation, clusterConfig, servers) c.Assert(err, IsNil) @@ -154,7 +154,7 @@ func (S) TestMultiNodePlan(c *C) { {Hostname: "node-3", ClusterRole: string(schema.ServiceRoleMaster)}, } app := loc.MustParseLocator("gravitational.io/app:0.0.1") - clusterConfig := clusterconfig.New() + clusterConfig := clusterconfig.NewEmpty() plan, err := newOperationPlan(app, storage.DefaultDNSConfig, operation, clusterConfig, servers) c.Assert(err, IsNil) @@ -358,7 +358,7 @@ func (S) TestBuildsPlanWithNodes(c *C) { {Hostname: "node-2", ClusterRole: string(schema.ServiceRoleNode)}, } app := loc.MustParseLocator("gravitational.io/app:0.0.1") - clusterConfig := clusterconfig.New() + clusterConfig := clusterconfig.NewEmpty() clusterConfig.Spec.ComponentConfigs.Kubelet = &clusterconfig.Kubelet{ Config: []byte(`apiVersion: v1 kind: KubeletConfiguration diff --git a/lib/update/environ/environ.go b/lib/update/environ/environ.go index 3b8e6d09f3..4aba49765b 100644 --- a/lib/update/environ/environ.go +++ b/lib/update/environ/environ.go @@ -20,23 +20,29 @@ import ( "context" "github.com/gravitational/gravity/lib/app" - "github.com/gravitational/gravity/lib/ops" + "github.com/gravitational/gravity/lib/fsm" "github.com/gravitational/gravity/lib/pack" "github.com/gravitational/gravity/lib/update" + "github.com/gravitational/gravity/lib/update/environ/phases" "github.com/gravitational/gravity/lib/update/internal/rollingupdate" + libphase "github.com/gravitational/gravity/lib/update/internal/rollingupdate/phases" "github.com/gravitational/trace" + log "github.com/sirupsen/logrus" "k8s.io/client-go/kubernetes" ) // New returns new cluster runtime environment updater for the specified configuration func New(ctx context.Context, config Config) (*update.Updater, error) { + dispatcher := &dispatcher{ + Dispatcher: rollingupdate.NewDefaultDispatcher(), + } machine, err := rollingupdate.NewMachine(ctx, rollingupdate.Config{ Config: config.Config, Apps: config.Apps, ClusterPackages: config.ClusterPackages, Client: config.Client, - RequestAdaptor: rollingupdate.RequestAdaptorFunc(updateRequest), + Dispatcher: dispatcher, }) if err != nil { return nil, trace.Wrap(err) @@ -59,8 +65,18 @@ type Config struct { Client *kubernetes.Clientset } -func updateRequest(req ops.RotatePlanetConfigRequest, operation ops.SiteOperation) ops.RotatePlanetConfigRequest { - result := req - result.Env = operation.UpdateEnviron.Env - return result +// Dispatch returns the appropriate phase executor based on the provided parameters +func (r *dispatcher) Dispatch(config rollingupdate.Config, params fsm.ExecutorParams, remote fsm.Remote, logger log.FieldLogger) (fsm.PhaseExecutor, error) { + switch params.Phase.Executor { + case libphase.UpdateConfig: + return phases.NewUpdateConfig(params, + config.Operator, *config.Operation, config.Apps, config.ClusterPackages, + logger) + default: + return r.Dispatcher.Dispatch(config, params, remote, logger) + } +} + +type dispatcher struct { + rollingupdate.Dispatcher } diff --git a/lib/update/environ/phases/update.go b/lib/update/environ/phases/update.go new file mode 100644 index 0000000000..aab73e24d0 --- /dev/null +++ b/lib/update/environ/phases/update.go @@ -0,0 +1,139 @@ +/* +Copyright 2019 Gravitational, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package phases + +import ( + "context" + "io" + + "github.com/gravitational/gravity/lib/app" + libfsm "github.com/gravitational/gravity/lib/fsm" + "github.com/gravitational/gravity/lib/loc" + "github.com/gravitational/gravity/lib/ops" + "github.com/gravitational/gravity/lib/pack" + "github.com/gravitational/gravity/lib/schema" + "github.com/gravitational/gravity/lib/storage" + + "github.com/gravitational/trace" + log "github.com/sirupsen/logrus" +) + +// NewUpdateConfig returns a new executor to update runtime configuration on the specified node +func NewUpdateConfig( + params libfsm.ExecutorParams, + operator operator, + operation ops.SiteOperation, + apps appGetter, + packages packageService, + logger log.FieldLogger, +) (*updateConfig, error) { + if params.Phase.Data == nil || params.Phase.Data.Package == nil { + return nil, trace.NotFound("no installed application package specified for phase %q", + params.Phase.ID) + } + app, err := apps.GetApp(*params.Phase.Data.Package) + if err != nil { + return nil, trace.Wrap(err, "failed to query installed application") + } + servers := params.Plan.Servers + if params.Phase.Data.Update != nil && len(params.Phase.Data.Update.Servers) != 0 { + servers = params.Phase.Data.Update.Servers + } + return &updateConfig{ + FieldLogger: logger, + operator: operator, + operation: operation, + packages: packages, + servers: servers, + manifest: app.Manifest, + }, nil +} + +// Execute generates new runtime configuration with the specified environment +func (r *updateConfig) Execute(ctx context.Context) error { + for _, server := range r.servers { + r.Infof("Generate new runtime configuration package for %v.", server) + runtimePackage, err := r.manifest.RuntimePackageForProfile(server.Role) + if err != nil { + return trace.Wrap(err) + } + req := ops.RotatePlanetConfigRequest{ + Key: r.operation.Key(), + Server: server, + Manifest: r.manifest, + Package: *runtimePackage, + Env: r.operation.UpdateEnviron.Env, + } + resp, err := r.operator.RotatePlanetConfig(req) + if err != nil { + return trace.Wrap(err) + } + _, err = r.packages.UpsertPackage(resp.Locator, resp.Reader, + pack.WithLabels(resp.Labels)) + if err != nil { + return trace.Wrap(err) + } + } + err := r.operator.UpdateClusterEnvironmentVariables(ops.UpdateClusterEnvironRequest{ + ClusterKey: r.operation.ClusterKey(), + Env: r.operation.UpdateEnviron.Env, + }) + return trace.Wrap(err) +} + +// Rollback resets the cluster configuration to the previous value +func (r *updateConfig) Rollback(context.Context) error { + err := r.operator.UpdateClusterEnvironmentVariables(ops.UpdateClusterEnvironRequest{ + ClusterKey: r.operation.ClusterKey(), + Env: r.operation.UpdateEnviron.PrevEnv, + }) + return trace.Wrap(err) +} + +// PreCheck is a no-op +func (r *updateConfig) PreCheck(context.Context) error { + return nil +} + +// PostCheck is a no-op +func (r *updateConfig) PostCheck(context.Context) error { + return nil +} + +type updateConfig struct { + // FieldLogger specifies the logger for the phase + log.FieldLogger + operator operator + operation ops.SiteOperation + packages packageService + servers []storage.Server + manifest schema.Manifest +} + +type operator interface { + RotatePlanetConfig(ops.RotatePlanetConfigRequest) (*ops.RotatePackageResponse, error) + UpdateClusterEnvironmentVariables(ops.UpdateClusterEnvironRequest) error +} + +type appGetter interface { + GetApp(loc.Locator) (*app.Application, error) +} + +type packageService interface { + UpsertPackage(loc.Locator, io.Reader, ...pack.PackageOption) (*pack.PackageEnvelope, error) + DeletePackage(loc.Locator) error +} diff --git a/tool/gravity/cli/clusterconfig.go b/tool/gravity/cli/clusterconfig.go index 0055be2f6c..67ad476db5 100644 --- a/tool/gravity/cli/clusterconfig.go +++ b/tool/gravity/cli/clusterconfig.go @@ -34,11 +34,14 @@ import ( // resetConfig executes the loop to reset cluster configuration to defaults func resetConfig(ctx context.Context, localEnv, updateEnv *localenv.LocalEnvironment, manual, confirmed bool) error { - config := libclusterconfig.New() + config := libclusterconfig.NewEmpty() return trace.Wrap(updateConfig(ctx, localEnv, updateEnv, config, manual, confirmed)) } func updateConfig(ctx context.Context, localEnv, updateEnv *localenv.LocalEnvironment, config libclusterconfig.Interface, manual, confirmed bool) error { + if err := validateCloudConfig(localEnv, config); err != nil { + return trace.Wrap(err) + } if !confirmed { if manual { localEnv.Println(updateConfigBannerManual) @@ -218,6 +221,51 @@ type configInitializer struct { config libclusterconfig.Interface } +func validateCloudConfig(localEnv *localenv.LocalEnvironment, config libclusterconfig.Interface) error { + if newGlobalConfig := config.GetGlobalConfig(); !isCloudConfigEmpty(newGlobalConfig) { + // TODO(dmitri): require cloud provider if cloud-config is being updated + // This is more a sanity check than a hard requirement so users are explicit about changes + // in the cloud configuration + if newGlobalConfig.CloudConfig != "" && newGlobalConfig.CloudProvider == "" { + return trace.BadParameter("cloud provider is required when updating cloud configuration") + } + } + operator, err := localEnv.SiteOperator() + if err != nil { + return trace.Wrap(err) + } + cluster, err := operator.GetLocalSite() + if err != nil { + return trace.Wrap(err) + } + clusterConfig, err := operator.GetClusterConfiguration(cluster.Key()) + if err != nil { + return trace.Wrap(err) + } + globalConfig := clusterConfig.GetGlobalConfig() + if isCloudConfigEmpty(globalConfig) { + if newGlobalConfig := config.GetGlobalConfig(); !isCloudConfigEmpty(newGlobalConfig) { + return trace.BadParameter("cannot change cloud configuration: cluster does not have cloud provider configured") + } + } + if globalConfig != nil { + if newGlobalConfig := config.GetGlobalConfig(); newGlobalConfig != nil { + if newGlobalConfig.CloudProvider != "" && globalConfig.CloudProvider != newGlobalConfig.CloudProvider { + return trace.BadParameter("changing cloud provider is not supported (%q -> %q)", + newGlobalConfig.CloudProvider, globalConfig.CloudProvider) + } + if globalConfig.CloudProvider == "" && newGlobalConfig.CloudConfig != "" { + return trace.BadParameter("cannot set cloud configuration: cluster does not have cloud provider configured") + } + } + } + return nil +} + +func isCloudConfigEmpty(global *libclusterconfig.Global) bool { + return global == nil || (global.CloudProvider == "" && global.CloudConfig == "") +} + const ( updateConfigBanner = `Updating cluster configuration might require restart of runtime containers on master nodes. The operation might take a few minutes to complete. diff --git a/tool/gravity/cli/config.go b/tool/gravity/cli/config.go index afea74b8c9..60444e19e7 100644 --- a/tool/gravity/cli/config.go +++ b/tool/gravity/cli/config.go @@ -17,7 +17,6 @@ limitations under the License. package cli import ( - "bytes" "context" "io/ioutil" "net" @@ -40,6 +39,7 @@ import ( teleutils "github.com/gravitational/teleport/lib/utils" "github.com/gravitational/trace" + "k8s.io/apimachinery/pkg/runtime" ) // InstallConfig is the gravity install command configuration @@ -288,16 +288,17 @@ func (i *InstallConfig) getDNSOverrides() (*storage.DNSOverrides, error) { } // ToInstallerConfig converts CLI config to installer format -func (i *InstallConfig) ToInstallerConfig(env *localenv.LocalEnvironment) (*install.Config, error) { +func (i *InstallConfig) ToInstallerConfig(env *localenv.LocalEnvironment, validator resources.Validator) (*install.Config, error) { advertiseAddr, err := i.GetAdvertiseAddr() if err != nil { return nil, trace.Wrap(err) } - var resources []byte + var kubernetesResources []runtime.Object + var gravityResources []storage.UnknownResource if i.ResourcesPath != "" { - resources, err = i.GetResources() + kubernetesResources, gravityResources, err = i.splitResources(validator) if err != nil { - return nil, trace.Wrap(err, "failed to load resources file") + return nil, trace.Wrap(err) } } appPackage, err := i.GetAppPackage() @@ -308,7 +309,7 @@ func (i *InstallConfig) ToInstallerConfig(env *localenv.LocalEnvironment) (*inst if err != nil { return nil, trace.Wrap(err) } - err = i.updateFromClusterConfig(resources) + gravityResources, err = i.updateClusterConfig(gravityResources) if err != nil { return nil, trace.Wrap(err) } @@ -318,7 +319,6 @@ func (i *InstallConfig) ToInstallerConfig(env *localenv.LocalEnvironment) (*inst Cancel: cancel, EventsC: make(chan install.Event, 100), AdvertiseAddr: advertiseAddr, - Resources: resources, AppPackage: appPackage, LocalPackages: env.Packages, LocalApps: env.Apps, @@ -331,6 +331,7 @@ func (i *InstallConfig) ToInstallerConfig(env *localenv.LocalEnvironment) (*inst SystemLogFile: i.SystemLogFile, Token: i.InstallToken, CloudProvider: i.CloudProvider, + GCENodeTags: i.NodeTags, Flavor: i.Flavor, Role: i.Role, SystemDevice: i.SystemDevice, @@ -346,56 +347,75 @@ func (i *InstallConfig) ToInstallerConfig(env *localenv.LocalEnvironment) (*inst Insecure: i.Insecure, Manual: i.Manual, ServiceUser: i.ServiceUser, - GCENodeTags: i.NodeTags, NewProcess: i.NewProcess, LocalClusterClient: env.SiteOperator, + RuntimeResources: kubernetesResources, + ClusterResources: gravityResources, }, nil } -// ValidateResources validates the resources specified in ResourcePath -// using the given validator -func (i *InstallConfig) ValidateResources(validator resources.Validator) error { +// splitResources validates the resources specified in ResourcePath +// using the given validator and splits them into Kubernetes and Gravity-specific +func (i *InstallConfig) splitResources(validator resources.Validator) (runtimeResources []runtime.Object, clusterResources []storage.UnknownResource, err error) { if i.ResourcesPath == "" { - return trace.NotFound("no resources provided") + return nil, nil, trace.NotFound("no resources provided") } rc, err := utils.ReaderForPath(i.ResourcesPath) if err != nil { - return trace.Wrap(err, "failed to read resources") + return nil, nil, trace.Wrap(err, "failed to read resources") } defer rc.Close() // TODO(dmitri): validate kubernetes resources as well - _, gravityResources, err := resources.Split(rc) + runtimeResources, clusterResources, err = resources.Split(rc) if err != nil { - return trace.BadParameter("failed to validate %q: %v", i.ResourcesPath, err) + return nil, nil, trace.BadParameter("failed to validate %q: %v", i.ResourcesPath, err) } - for _, res := range gravityResources { + for _, res := range clusterResources { log.WithField("resource", res.ResourceHeader).Info("Validating.") if err := validator.Validate(res); err != nil { - return trace.Wrap(err, "resource %q is invalid", res.Kind) + return nil, nil, trace.Wrap(err, "resource %q is invalid", res.Kind) } } - return nil + return runtimeResources, clusterResources, nil } -func (i *InstallConfig) updateFromClusterConfig(resourceBytes []byte) error { - if len(resourceBytes) == 0 { - return nil - } - err := resources.ForEach(bytes.NewReader(resourceBytes), func(res storage.UnknownResource) error { - if res.Kind != storage.KindClusterConfiguration { - return nil +func (i *InstallConfig) updateClusterConfig(resources []storage.UnknownResource) (updated []storage.UnknownResource, err error) { + var clusterConfig *storage.UnknownResource + updated = resources[:0] + for _, res := range resources { + if res.Kind == storage.KindClusterConfiguration { + clusterConfig = &res + continue } - config, err := clusterconfig.Unmarshal(res.Raw) + updated = append(updated, res) + } + if clusterConfig == nil && i.CloudProvider == "" { + // Return the resources unchanged + return resources, nil + } + var config clusterconfig.Interface + if clusterConfig == nil { + config = clusterconfig.New(clusterconfig.Spec{ + Global: &clusterconfig.Global{CloudProvider: i.CloudProvider}, + }) + } else { + config, err = clusterconfig.Unmarshal(clusterConfig.Raw) if err != nil { - return trace.Wrap(err) + return nil, trace.Wrap(err) } - if config := config.GetGlobalConfig(); config != nil && config.CloudProvider != "" { + } + if config := config.GetGlobalConfig(); config != nil { + if config.CloudProvider != "" { i.CloudProvider = config.CloudProvider - return utils.Abort(nil) } - return nil - }) - return trace.Wrap(err) + } + // Serialize the cluster configuration and add to resources + configResource, err := clusterconfig.ToUnknown(config) + if err != nil { + return nil, trace.Wrap(err) + } + updated = append(updated, *configResource) + return updated, nil } func (i *InstallConfig) validateDNSConfig() error { diff --git a/tool/gravity/cli/install.go b/tool/gravity/cli/install.go index 793fa3fec8..0b993bbae3 100644 --- a/tool/gravity/cli/install.go +++ b/tool/gravity/cli/install.go @@ -59,14 +59,7 @@ func startInstall(env *localenv.LocalEnvironment, i InstallConfig) error { return trace.Wrap(err) } - if i.ResourcesPath != "" { - err = i.ValidateResources(resources.ValidateFunc(gravity.Validate)) - if err != nil { - return trace.Wrap(err) - } - } - - installerConfig, err := i.ToInstallerConfig(env) + installerConfig, err := i.ToInstallerConfig(env, resources.ValidateFunc(gravity.Validate)) if err != nil { return trace.Wrap(err) }