From da41044f67829980442101883c64efb4e2f737d9 Mon Sep 17 00:00:00 2001 From: Roman Tkachenko Date: Mon, 20 May 2019 09:48:51 -0700 Subject: [PATCH] Kubernetes proxy fixes. (#407) --- Gopkg.lock | 4 +- assets/site-app/resources/site.yaml | 18 +-- lib/ops/endpoints.go | 105 ++++++++++++++++ lib/ops/endpoints_test.go | 115 ++++++++++++++++++ lib/ops/opsservice/endpoints.go | 2 +- lib/process/process.go | 15 +-- lib/process/teleport.go | 21 ++++ lib/status/status.go | 21 ++-- lib/storage/authgateway.go | 19 +-- lib/utils/parse.go | 5 + lib/webapi/clusterinfo.go | 41 +------ tool/gravity/cli/status.go | 4 +- .../teleport/lib/kube/proxy/forwarder.go | 14 ++- 13 files changed, 305 insertions(+), 79 deletions(-) create mode 100644 lib/ops/endpoints.go create mode 100644 lib/ops/endpoints_test.go diff --git a/Gopkg.lock b/Gopkg.lock index fb7529d6cf..24422f39cc 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -894,7 +894,7 @@ [[projects]] branch = "branch/3.2-g" - digest = "1:dc70d53cfb938ba2490891306441c151e2256eb957d06a5e6dab3d4f3c7e4874" + digest = "1:2d9aec3c832481bd817f951e43ab87e03417519a086279a48767629d94f0284b" name = "github.com/gravitational/teleport" packages = [ ".", @@ -949,7 +949,7 @@ "lib/web/ui", ] pruneopts = "UT" - revision = "495a59bc1737cf62aa9850f5eed7e6fc3117c0c2" + revision = "d88eaecef78779b0211cbe458230fd3ce0989bde" [[projects]] digest = "1:22c8951489f0e27ec3b2d50d9f74086ff3f08554ce5965d418d6a10296b9ea04" diff --git a/assets/site-app/resources/site.yaml b/assets/site-app/resources/site.yaml index 928acae435..9c6d4883b8 100644 --- a/assets/site-app/resources/site.yaml +++ b/assets/site-app/resources/site.yaml @@ -66,21 +66,15 @@ rules: - list - watch # The following permissions are required for Teleport's Kubernetes proxy -# functionality which uses Kubernetes CSR API for signing client certs. +# functionality which uses Kubernetes Impersonation API. - apiGroups: - - certificates.k8s.io - resources: - - certificatesigningrequests - verbs: - - create - - watch - - delete -- apiGroups: - - certificates.k8s.io + - "" resources: - - certificatesigningrequests/approval + - users + - groups + - serviceaccounts verbs: - - update + - impersonate --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding diff --git a/lib/ops/endpoints.go b/lib/ops/endpoints.go new file mode 100644 index 0000000000..805c05b86f --- /dev/null +++ b/lib/ops/endpoints.go @@ -0,0 +1,105 @@ +/* +Copyright 2019 Gravitational, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package ops + +import ( + "strconv" + + "github.com/gravitational/gravity/lib/defaults" + "github.com/gravitational/gravity/lib/storage" + "github.com/gravitational/gravity/lib/utils" + + "github.com/gravitational/trace" +) + +// ClusterEndpoints contains system cluster endpoints such as Teleport +// proxy address or cluster control panel URL. +type ClusterEndpoints struct { + // Internal contains internal cluster endpoints. + Internal clusterEndpoints + // Public contains public cluster endpoints. + Public clusterEndpoints +} + +// AuthGateways returns all auth gateway endpoints. +func (e ClusterEndpoints) AuthGateways() []string { + if len(e.Public.AuthGateways) > 0 { + return e.Public.AuthGateways + } + return e.Internal.AuthGateways +} + +// FirstAuthGateway returns the first auth gateway endpoint. +func (e ClusterEndpoints) FirstAuthGateway() string { + gateways := e.AuthGateways() + if len(gateways) > 0 { + return gateways[0] + } + return "" +} + +// ManagementURLs returns all cluster management URLs. +func (e ClusterEndpoints) ManagementURLs() []string { + if len(e.Public.ManagementURLs) > 0 { + return e.Public.ManagementURLs + } + return e.Internal.ManagementURLs +} + +// clusterEndpoints combines various types of cluster endpoints. +type clusterEndpoints struct { + // AuthGateways is a list of Teleport proxy addresses. + AuthGateways []string + // ManagementURLs is a list of URLs pointing to cluster dashboard. + ManagementURLs []string +} + +// GetClusterEndpoints returns system endpoints for the specified cluster. +func GetClusterEndpoints(operator Operator, key SiteKey) (*ClusterEndpoints, error) { + cluster, err := operator.GetSite(key) + if err != nil { + return nil, trace.Wrap(err) + } + gateway, err := operator.GetAuthGateway(key) + if err != nil { + return nil, trace.Wrap(err) + } + return getClusterEndpoints(cluster, gateway) +} + +func getClusterEndpoints(cluster *Site, gateway storage.AuthGateway) (*ClusterEndpoints, error) { + // Internal endpoints point directly to master nodes. + var internal clusterEndpoints + for _, master := range cluster.Masters() { + internal.AuthGateways = append(internal.AuthGateways, + utils.EnsurePort(master.AdvertiseIP, strconv.Itoa(defaults.GravitySiteNodePort))) + internal.ManagementURLs = append(internal.ManagementURLs, + utils.EnsurePortURL(master.AdvertiseIP, strconv.Itoa(defaults.GravitySiteNodePort))) + } + // Public endpoints are configured via auth gateway resource. + var public clusterEndpoints + for _, address := range gateway.GetWebPublicAddrs() { + public.AuthGateways = append(public.AuthGateways, + utils.EnsurePort(address, defaults.HTTPSPort)) + public.ManagementURLs = append(public.ManagementURLs, + utils.EnsurePortURL(address, defaults.HTTPSPort)) + } + return &ClusterEndpoints{ + Internal: internal, + Public: public, + }, nil +} diff --git a/lib/ops/endpoints_test.go b/lib/ops/endpoints_test.go new file mode 100644 index 0000000000..a13dd4bf0b --- /dev/null +++ b/lib/ops/endpoints_test.go @@ -0,0 +1,115 @@ +/* +Copyright 2019 Gravitational, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package ops + +import ( + "fmt" + "strconv" + + "github.com/gravitational/gravity/lib/defaults" + "github.com/gravitational/gravity/lib/schema" + "github.com/gravitational/gravity/lib/storage" + "github.com/gravitational/gravity/lib/utils" + + "gopkg.in/check.v1" +) + +type EndpointsSuite struct{} + +var _ = check.Suite(&EndpointsSuite{}) + +func (s *EndpointsSuite) TestClusterEndpoints(c *check.C) { + master1 := storage.Server{ + AdvertiseIP: "192.168.1.1", + ClusterRole: string(schema.ServiceRoleMaster), + } + node := storage.Server{ + AdvertiseIP: "192.168.1.2", + ClusterRole: string(schema.ServiceRoleNode), + } + master2 := storage.Server{ + AdvertiseIP: "192.168.1.3", + ClusterRole: string(schema.ServiceRoleMaster), + } + cluster := &Site{ + ClusterState: storage.ClusterState{ + Servers: []storage.Server{ + master1, node, master2, + }, + }, + } + gateway := storage.DefaultAuthGateway() + + endpoints, err := getClusterEndpoints(cluster, gateway) + c.Assert(err, check.IsNil) + c.Assert(endpoints, check.DeepEquals, &ClusterEndpoints{ + Internal: clusterEndpoints{ + AuthGateways: []string{ + utils.EnsurePort(master1.AdvertiseIP, strconv.Itoa(defaults.GravitySiteNodePort)), + utils.EnsurePort(master2.AdvertiseIP, strconv.Itoa(defaults.GravitySiteNodePort)), + }, + ManagementURLs: []string{ + utils.EnsurePortURL(master1.AdvertiseIP, strconv.Itoa(defaults.GravitySiteNodePort)), + utils.EnsurePortURL(master2.AdvertiseIP, strconv.Itoa(defaults.GravitySiteNodePort)), + }, + }, + Public: clusterEndpoints{}, + }) + c.Assert(endpoints.AuthGateways(), check.DeepEquals, []string{ + utils.EnsurePort(master1.AdvertiseIP, strconv.Itoa(defaults.GravitySiteNodePort)), + utils.EnsurePort(master2.AdvertiseIP, strconv.Itoa(defaults.GravitySiteNodePort)), + }) + c.Assert(endpoints.FirstAuthGateway(), check.Equals, + utils.EnsurePort(master1.AdvertiseIP, strconv.Itoa(defaults.GravitySiteNodePort))) + c.Assert(endpoints.ManagementURLs(), check.DeepEquals, []string{ + utils.EnsurePortURL(master1.AdvertiseIP, strconv.Itoa(defaults.GravitySiteNodePort)), + utils.EnsurePortURL(master2.AdvertiseIP, strconv.Itoa(defaults.GravitySiteNodePort)), + }) + + publicAddr := "cluster.example.com:444" + gateway.SetPublicAddrs([]string{publicAddr}) + + endpoints, err = getClusterEndpoints(cluster, gateway) + c.Assert(err, check.IsNil) + c.Assert(endpoints, check.DeepEquals, &ClusterEndpoints{ + Internal: clusterEndpoints{ + AuthGateways: []string{ + utils.EnsurePort(master1.AdvertiseIP, strconv.Itoa(defaults.GravitySiteNodePort)), + utils.EnsurePort(master2.AdvertiseIP, strconv.Itoa(defaults.GravitySiteNodePort)), + }, + ManagementURLs: []string{ + utils.EnsurePortURL(master1.AdvertiseIP, strconv.Itoa(defaults.GravitySiteNodePort)), + utils.EnsurePortURL(master2.AdvertiseIP, strconv.Itoa(defaults.GravitySiteNodePort)), + }, + }, + Public: clusterEndpoints{ + AuthGateways: []string{ + publicAddr, + }, + ManagementURLs: []string{ + fmt.Sprintf("https://%v", publicAddr), + }, + }, + }) + c.Assert(endpoints.AuthGateways(), check.DeepEquals, []string{ + publicAddr, + }) + c.Assert(endpoints.FirstAuthGateway(), check.Equals, publicAddr) + c.Assert(endpoints.ManagementURLs(), check.DeepEquals, []string{ + fmt.Sprintf("https://%v", publicAddr), + }) +} diff --git a/lib/ops/opsservice/endpoints.go b/lib/ops/opsservice/endpoints.go index 892ae61ff8..3e29826a3e 100644 --- a/lib/ops/opsservice/endpoints.go +++ b/lib/ops/opsservice/endpoints.go @@ -25,7 +25,7 @@ import ( "github.com/gravitational/gravity/lib/utils" "github.com/gravitational/trace" - "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" ) diff --git a/lib/process/process.go b/lib/process/process.go index 3c8e7ee656..f743d9f57c 100644 --- a/lib/process/process.go +++ b/lib/process/process.go @@ -996,13 +996,14 @@ func (p *Process) initService(ctx context.Context) (err error) { } p.handlers.WebProxy, err = teleweb.NewHandler(teleweb.Config{ - Proxy: reverseTunnel, - AuthServers: p.teleportConfig.AuthServers[0], - DomainName: p.teleportConfig.Hostname, - ProxyClient: proxyConn.Client, - DisableUI: true, - ProxySSHAddr: p.teleportConfig.Proxy.SSHAddr, - ProxyWebAddr: p.teleportConfig.Proxy.WebAddr, + Proxy: reverseTunnel, + AuthServers: p.teleportConfig.AuthServers[0], + DomainName: p.teleportConfig.Hostname, + ProxyClient: proxyConn.Client, + DisableUI: true, + ProxySSHAddr: p.teleportConfig.Proxy.SSHAddr, + ProxyWebAddr: p.teleportConfig.Proxy.WebAddr, + ProxySettings: p.proxySettings(), }) if err != nil { return trace.Wrap(err) diff --git a/lib/process/teleport.go b/lib/process/teleport.go index 0901cdc42d..37d70dc796 100644 --- a/lib/process/teleport.go +++ b/lib/process/teleport.go @@ -22,7 +22,9 @@ import ( "github.com/gravitational/gravity/lib/processconfig" "github.com/gravitational/gravity/lib/storage" + "github.com/gravitational/teleport/lib/client" "github.com/gravitational/teleport/lib/config" + teledefaults "github.com/gravitational/teleport/lib/defaults" "github.com/gravitational/teleport/lib/service" "github.com/gravitational/trace" @@ -74,6 +76,9 @@ func (p *Process) buildTeleportConfig(authGatewayConfig storage.AuthGateway) (*s serviceConfig.Access = p.identity serviceConfig.Console = logrus.StandardLogger().Writer() serviceConfig.ClusterConfiguration = p.identity + // Use high-res polling period so principal changes are detected + // faster when auth gateway settings are updated. + serviceConfig.PollingPeriod = teledefaults.HighResPollingPeriod return serviceConfig, nil } @@ -139,3 +144,19 @@ func (p *Process) getAuthGatewayConfig() (storage.AuthGateway, error) { } return opsservice.GetAuthGateway(client, p.identity) } + +// proxySettings returns Teleport proxy settings based on the Teleport config. +func (p *Process) proxySettings() client.ProxySettings { + settings := client.ProxySettings{ + Kube: client.KubeProxySettings{ + Enabled: p.teleportConfig.Proxy.Kube.Enabled, + }, + SSH: client.SSHProxySettings{ + ListenAddr: p.teleportConfig.Proxy.SSHAddr.String(), + }, + } + if len(p.teleportConfig.Proxy.Kube.PublicAddrs) > 0 { + settings.Kube.PublicAddr = p.teleportConfig.Proxy.Kube.PublicAddrs[0].String() + } + return settings +} diff --git a/lib/status/status.go b/lib/status/status.go index 66a447cc0a..efd5776f90 100644 --- a/lib/status/status.go +++ b/lib/status/status.go @@ -63,28 +63,29 @@ func FromCluster(ctx context.Context, operator ops.Operator, cluster ops.Site, o } // Collect application endpoints. - endpoints, err := operator.GetApplicationEndpoints(cluster.Key()) + appEndpoints, err := operator.GetApplicationEndpoints(cluster.Key()) if err != nil { logrus.WithError(err).Warn("Failed to fetch application endpoints.") status.Endpoints.Applications.Error = err } - if len(endpoints) != 0 { + if len(appEndpoints) != 0 { // Right now only 1 application is supported, in the future there // will be many applications each with its own endpoints. status.Endpoints.Applications.Endpoints = append(status.Endpoints.Applications.Endpoints, ApplicationEndpoints{ Application: cluster.App.Package, - Endpoints: endpoints, + Endpoints: appEndpoints, }) } - // For cluster endpoints, they point to gravity-site service on master nodes. - masters := cluster.ClusterState.Servers.Masters() - for _, master := range masters { - status.Endpoints.Cluster.AuthGateway = append(status.Endpoints.Cluster.AuthGateway, - fmt.Sprintf("%v:%v", master.AdvertiseIP, defaults.GravitySiteNodePort)) - status.Endpoints.Cluster.UI = append(status.Endpoints.Cluster.UI, - fmt.Sprintf("https://%v:%v", master.AdvertiseIP, defaults.GravitySiteNodePort)) + // Fetch cluster endpoints. + clusterEndpoints, err := ops.GetClusterEndpoints(operator, cluster.Key()) + if err != nil { + logrus.WithError(err).Warn("Failed to fetch cluster endpoints.") + } + if clusterEndpoints != nil { + status.Endpoints.Cluster.AuthGateway = clusterEndpoints.AuthGateways() + status.Endpoints.Cluster.UI = clusterEndpoints.ManagementURLs() } // FIXME: have status extension accept the operator/environment diff --git a/lib/storage/authgateway.go b/lib/storage/authgateway.go index 97335749a7..58b1c57a37 100644 --- a/lib/storage/authgateway.go +++ b/lib/storage/authgateway.go @@ -295,14 +295,17 @@ func (gw *AuthGatewayV1) ApplyToTeleportConfig(config *teleconfig.FileConfig) { U2F: u2f, } } - config.Auth.PublicAddr = append(config.Auth.PublicAddr, - gw.GetSSHPublicAddrs()...) - config.Proxy.SSHPublicAddr = append(config.Proxy.SSHPublicAddr, - gw.GetSSHPublicAddrs()...) - config.Proxy.PublicAddr = append(config.Proxy.PublicAddr, - gw.GetWebPublicAddrs()...) - config.Proxy.Kube.PublicAddr = append(config.Proxy.Kube.PublicAddr, - gw.GetKubernetesPublicAddrs()...) + // Make sure user-set values take precedence as Teleport may just + // grab first value from the list, for example when advertising + // Kubernetes proxy public address. + config.Auth.PublicAddr = append(gw.GetSSHPublicAddrs(), + config.Auth.PublicAddr...) + config.Proxy.SSHPublicAddr = append(gw.GetSSHPublicAddrs(), + config.Proxy.SSHPublicAddr...) + config.Proxy.PublicAddr = append(gw.GetWebPublicAddrs(), + config.Proxy.PublicAddr...) + config.Proxy.Kube.PublicAddr = append(gw.GetKubernetesPublicAddrs(), + config.Proxy.Kube.PublicAddr...) } // GetMaxConnections returns max connections setting. diff --git a/lib/utils/parse.go b/lib/utils/parse.go index 91dd2c9344..3deecac128 100644 --- a/lib/utils/parse.go +++ b/lib/utils/parse.go @@ -111,6 +111,11 @@ func EnsurePort(address, defaultPort string) string { return net.JoinHostPort(address, defaultPort) } +// EnsurePortURL is like EnsurePort but for URLs. +func EnsurePortURL(url, defaultPort string) string { + return ParseOpsCenterAddress(url, defaultPort) +} + // Hosts returns a list of hosts from the provided host:port addresses func Hosts(addrs []string) (hosts []string) { for _, addr := range addrs { diff --git a/lib/webapi/clusterinfo.go b/lib/webapi/clusterinfo.go index 802fbfdb50..3a4d00f698 100644 --- a/lib/webapi/clusterinfo.go +++ b/lib/webapi/clusterinfo.go @@ -18,14 +18,11 @@ package webapi import ( "bytes" - "fmt" "strconv" - "strings" "text/template" "github.com/gravitational/gravity/lib/defaults" "github.com/gravitational/gravity/lib/ops" - "github.com/gravitational/gravity/lib/utils" "github.com/gravitational/trace" ) @@ -56,32 +53,16 @@ type webClusterCommands struct { // getClusterInfo collects information for the specified cluster. func getClusterInfo(operator ops.Operator, cluster ops.Site) (*webClusterInfo, error) { - authGateway, err := operator.GetAuthGateway(cluster.Key()) + masterNode, err := cluster.FirstMaster() if err != nil { return nil, trace.Wrap(err) } - masterNode, err := cluster.FirstMaster() + endpoints, err := ops.GetClusterEndpoints(operator, cluster.Key()) if err != nil { return nil, trace.Wrap(err) } - var internalAddrs []string - for _, node := range cluster.Masters() { - internalAddrs = append(internalAddrs, fmt.Sprintf("%v:%v", - node.AdvertiseIP, defaults.GravitySiteNodePort)) - } - var publicAddrs []string - for _, webAddr := range authGateway.GetWebPublicAddrs() { - publicAddrs = append(publicAddrs, utils.EnsurePort( - webAddr, defaults.HTTPSPort)) - } - var proxyAddr string - if len(publicAddrs) != 0 { - proxyAddr = publicAddrs[0] - } else { - proxyAddr = internalAddrs[0] - } tshLoginCommand, err := renderCommand(tshLoginTpl, map[string]string{ - "proxyAddr": proxyAddr, + "proxyAddr": endpoints.FirstAuthGateway(), }) if err != nil { return nil, trace.Wrap(err) @@ -111,8 +92,8 @@ func getClusterInfo(operator ops.Operator, cluster ops.Site) (*webClusterInfo, e } return &webClusterInfo{ ClusterState: cluster.State, - PublicURLs: makeURLs(publicAddrs), - InternalURLs: makeURLs(internalAddrs), + PublicURLs: endpoints.Public.ManagementURLs, + InternalURLs: endpoints.Internal.ManagementURLs, Commands: webClusterCommands{ TshLogin: tshLoginCommand, GravityDownload: gravityDownloadCommand, @@ -130,18 +111,6 @@ func renderCommand(tpl *template.Template, params map[string]string) (string, er return b.String(), nil } -// makeURLs converts provided addresses into URLs. -func makeURLs(addrs []string) (urls []string) { - for _, addr := range addrs { - if !strings.HasPrefix(addr, "https://") { - urls = append(urls, fmt.Sprintf("https://%v", addr)) - } else { - urls = append(urls, addr) - } - } - return urls -} - var ( // gravityJoinTpl is the gravity join command template. gravityJoinTpl = template.Must(template.New("join").Parse( diff --git a/tool/gravity/cli/status.go b/tool/gravity/cli/status.go index 7f9d529d66..0eaa8abce2 100644 --- a/tool/gravity/cli/status.go +++ b/tool/gravity/cli/status.go @@ -362,9 +362,9 @@ func printAgentStatus(status statusapi.Agent, w io.Writer) { func printNodeStatus(node statusapi.ClusterServer, w io.Writer) { description := node.AdvertiseIP if node.Profile != "" { - description = fmt.Sprintf("%v, %v", description, node.Profile) + description = fmt.Sprintf("%v / %v", description, node.Profile) } - fmt.Fprintf(w, " * %v (%v)\n", unknownFallback(node.Hostname), description) + fmt.Fprintf(w, " * %v / %v\n", unknownFallback(node.Hostname), description) switch node.Status { case statusapi.NodeOffline: fmt.Fprintf(w, " Status:\t%v\n", color.YellowString("offline")) diff --git a/vendor/github.com/gravitational/teleport/lib/kube/proxy/forwarder.go b/vendor/github.com/gravitational/teleport/lib/kube/proxy/forwarder.go index 8b0764bd19..592019f40b 100644 --- a/vendor/github.com/gravitational/teleport/lib/kube/proxy/forwarder.go +++ b/vendor/github.com/gravitational/teleport/lib/kube/proxy/forwarder.go @@ -878,9 +878,21 @@ func (f *Forwarder) newClusterSession(ctx authContext) (*clusterSession, error) tlsConfig: tlsConfig, } + var transport http.RoundTripper = f.newTransport(sess.Dial, tlsConfig) + + // when running inside Kubernetes cluster, kubeconfig provides a + // transport wrapper that adds service account token to requests + // + // when forwarding request to a remote cluster, this is not needed + // as the proxy uses client cert auth to reach out to remote proxy + // which will then use its own transport wrapper + if !ctx.cluster.isRemote && f.creds.cfg.WrapTransport != nil { + transport = f.creds.cfg.WrapTransport(transport) + } + fwd, err := forward.New( forward.FlushInterval(100*time.Millisecond), - forward.RoundTripper(f.newTransport(sess.Dial, tlsConfig)), + forward.RoundTripper(transport), forward.WebsocketDial(sess.Dial), forward.Logger(log.StandardLogger()), )