Kubernetes proxy fixes. (#407)

assets/site-app/resources/site.yaml

@@ -66,21 +66,15 @@ rules:
   - list
   - watch
 # The following permissions are required for Teleport's Kubernetes proxy
-# functionality which uses Kubernetes CSR API for signing client certs.
+# functionality which uses Kubernetes Impersonation API.
 - apiGroups:
-  - certificates.k8s.io
-  resources:
-  - certificatesigningrequests
-  verbs:
-  - create
-  - watch
-  - delete
-- apiGroups:
-  - certificates.k8s.io
+  - ""
   resources:
-  - certificatesigningrequests/approval
+  - users
+  - groups
+  - serviceaccounts
   verbs:
-  - update
+  - impersonate
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding

lib/ops/endpoints.go

@@ -0,0 +1,105 @@
+/*
+Copyright 2019 Gravitational, Inc.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package ops
+
+import (
+	"strconv"
+
+	"github.com/gravitational/gravity/lib/defaults"
+	"github.com/gravitational/gravity/lib/storage"
+	"github.com/gravitational/gravity/lib/utils"
+
+	"github.com/gravitational/trace"
+)
+
+// ClusterEndpoints contains system cluster endpoints such as Teleport
+// proxy address or cluster control panel URL.
+type ClusterEndpoints struct {
+	// Internal contains internal cluster endpoints.
+	Internal clusterEndpoints
+	// Public contains public cluster endpoints.
+	Public clusterEndpoints
+}
+
+// AuthGateways returns all auth gateway endpoints.
+func (e ClusterEndpoints) AuthGateways() []string {
+	if len(e.Public.AuthGateways) > 0 {
+		return e.Public.AuthGateways
+	}
+	return e.Internal.AuthGateways
+}
+
+// FirstAuthGateway returns the first auth gateway endpoint.
+func (e ClusterEndpoints) FirstAuthGateway() string {
+	gateways := e.AuthGateways()
+	if len(gateways) > 0 {
+		return gateways[0]
+	}
+	return ""
+}
+
+// ManagementURLs returns all cluster management URLs.
+func (e ClusterEndpoints) ManagementURLs() []string {
+	if len(e.Public.ManagementURLs) > 0 {
+		return e.Public.ManagementURLs
+	}
+	return e.Internal.ManagementURLs
+}
+
+// clusterEndpoints combines various types of cluster endpoints.
+type clusterEndpoints struct {
+	// AuthGateways is a list of Teleport proxy addresses.
+	AuthGateways []string
+	// ManagementURLs is a list of URLs pointing to cluster dashboard.
+	ManagementURLs []string
+}
+
+// GetClusterEndpoints returns system endpoints for the specified cluster.
+func GetClusterEndpoints(operator Operator, key SiteKey) (*ClusterEndpoints, error) {
+	cluster, err := operator.GetSite(key)
+	if err != nil {
+		return nil, trace.Wrap(err)
+	}
+	gateway, err := operator.GetAuthGateway(key)
+	if err != nil {
+		return nil, trace.Wrap(err)
+	}
+	return getClusterEndpoints(cluster, gateway)
+}
+
+func getClusterEndpoints(cluster *Site, gateway storage.AuthGateway) (*ClusterEndpoints, error) {
+	// Internal endpoints point directly to master nodes.
+	var internal clusterEndpoints
+	for _, master := range cluster.Masters() {
+		internal.AuthGateways = append(internal.AuthGateways,
+			utils.EnsurePort(master.AdvertiseIP, strconv.Itoa(defaults.GravitySiteNodePort)))
+		internal.ManagementURLs = append(internal.ManagementURLs,
+			utils.EnsurePortURL(master.AdvertiseIP, strconv.Itoa(defaults.GravitySiteNodePort)))
+	}
+	// Public endpoints are configured via auth gateway resource.
+	var public clusterEndpoints
+	for _, address := range gateway.GetWebPublicAddrs() {
+		public.AuthGateways = append(public.AuthGateways,
+			utils.EnsurePort(address, defaults.HTTPSPort))
+		public.ManagementURLs = append(public.ManagementURLs,
+			utils.EnsurePortURL(address, defaults.HTTPSPort))
+	}
+	return &ClusterEndpoints{
+		Internal: internal,
+		Public:   public,
+	}, nil
+}

lib/ops/endpoints_test.go

@@ -0,0 +1,115 @@
+/*
+Copyright 2019 Gravitational, Inc.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package ops
+
+import (
+	"fmt"
+	"strconv"
+
+	"github.com/gravitational/gravity/lib/defaults"
+	"github.com/gravitational/gravity/lib/schema"
+	"github.com/gravitational/gravity/lib/storage"
+	"github.com/gravitational/gravity/lib/utils"
+
+	"gopkg.in/check.v1"
+)
+
+type EndpointsSuite struct{}
+
+var _ = check.Suite(&EndpointsSuite{})
+
+func (s *EndpointsSuite) TestClusterEndpoints(c *check.C) {
+	master1 := storage.Server{
+		AdvertiseIP: "192.168.1.1",
+		ClusterRole: string(schema.ServiceRoleMaster),
+	}
+	node := storage.Server{
+		AdvertiseIP: "192.168.1.2",
+		ClusterRole: string(schema.ServiceRoleNode),
+	}
+	master2 := storage.Server{
+		AdvertiseIP: "192.168.1.3",
+		ClusterRole: string(schema.ServiceRoleMaster),
+	}
+	cluster := &Site{
+		ClusterState: storage.ClusterState{
+			Servers: []storage.Server{
+				master1, node, master2,
+			},
+		},
+	}
+	gateway := storage.DefaultAuthGateway()
+
+	endpoints, err := getClusterEndpoints(cluster, gateway)
+	c.Assert(err, check.IsNil)
+	c.Assert(endpoints, check.DeepEquals, &ClusterEndpoints{
+		Internal: clusterEndpoints{
+			AuthGateways: []string{
+				utils.EnsurePort(master1.AdvertiseIP, strconv.Itoa(defaults.GravitySiteNodePort)),
+				utils.EnsurePort(master2.AdvertiseIP, strconv.Itoa(defaults.GravitySiteNodePort)),
+			},
+			ManagementURLs: []string{
+				utils.EnsurePortURL(master1.AdvertiseIP, strconv.Itoa(defaults.GravitySiteNodePort)),
+				utils.EnsurePortURL(master2.AdvertiseIP, strconv.Itoa(defaults.GravitySiteNodePort)),
+			},
+		},
+		Public: clusterEndpoints{},
+	})
+	c.Assert(endpoints.AuthGateways(), check.DeepEquals, []string{
+		utils.EnsurePort(master1.AdvertiseIP, strconv.Itoa(defaults.GravitySiteNodePort)),
+		utils.EnsurePort(master2.AdvertiseIP, strconv.Itoa(defaults.GravitySiteNodePort)),
+	})
+	c.Assert(endpoints.FirstAuthGateway(), check.Equals,
+		utils.EnsurePort(master1.AdvertiseIP, strconv.Itoa(defaults.GravitySiteNodePort)))
+	c.Assert(endpoints.ManagementURLs(), check.DeepEquals, []string{
+		utils.EnsurePortURL(master1.AdvertiseIP, strconv.Itoa(defaults.GravitySiteNodePort)),
+		utils.EnsurePortURL(master2.AdvertiseIP, strconv.Itoa(defaults.GravitySiteNodePort)),
+	})
+
+	publicAddr := "cluster.example.com:444"
+	gateway.SetPublicAddrs([]string{publicAddr})
+
+	endpoints, err = getClusterEndpoints(cluster, gateway)
+	c.Assert(err, check.IsNil)
+	c.Assert(endpoints, check.DeepEquals, &ClusterEndpoints{
+		Internal: clusterEndpoints{
+			AuthGateways: []string{
+				utils.EnsurePort(master1.AdvertiseIP, strconv.Itoa(defaults.GravitySiteNodePort)),
+				utils.EnsurePort(master2.AdvertiseIP, strconv.Itoa(defaults.GravitySiteNodePort)),
+			},
+			ManagementURLs: []string{
+				utils.EnsurePortURL(master1.AdvertiseIP, strconv.Itoa(defaults.GravitySiteNodePort)),
+				utils.EnsurePortURL(master2.AdvertiseIP, strconv.Itoa(defaults.GravitySiteNodePort)),
+			},
+		},
+		Public: clusterEndpoints{
+			AuthGateways: []string{
+				publicAddr,
+			},
+			ManagementURLs: []string{
+				fmt.Sprintf("https://%v", publicAddr),
+			},
+		},
+	})
+	c.Assert(endpoints.AuthGateways(), check.DeepEquals, []string{
+		publicAddr,
+	})
+	c.Assert(endpoints.FirstAuthGateway(), check.Equals, publicAddr)
+	c.Assert(endpoints.ManagementURLs(), check.DeepEquals, []string{
+		fmt.Sprintf("https://%v", publicAddr),
+	})
+}

lib/ops/opsservice/endpoints.go

@@ -25,7 +25,7 @@ import (
 	"github.com/gravitational/gravity/lib/utils"
 
 	"github.com/gravitational/trace"
-	"k8s.io/api/core/v1"
+	v1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/client-go/kubernetes"
 )

lib/process/process.go

@@ -996,13 +996,14 @@ func (p *Process) initService(ctx context.Context) (err error) {
 	}
 
 	p.handlers.WebProxy, err = teleweb.NewHandler(teleweb.Config{
-		Proxy:        reverseTunnel,
-		AuthServers:  p.teleportConfig.AuthServers[0],
-		DomainName:   p.teleportConfig.Hostname,
-		ProxyClient:  proxyConn.Client,
-		DisableUI:    true,
-		ProxySSHAddr: p.teleportConfig.Proxy.SSHAddr,
-		ProxyWebAddr: p.teleportConfig.Proxy.WebAddr,
+		Proxy:         reverseTunnel,
+		AuthServers:   p.teleportConfig.AuthServers[0],
+		DomainName:    p.teleportConfig.Hostname,
+		ProxyClient:   proxyConn.Client,
+		DisableUI:     true,
+		ProxySSHAddr:  p.teleportConfig.Proxy.SSHAddr,
+		ProxyWebAddr:  p.teleportConfig.Proxy.WebAddr,
+		ProxySettings: p.proxySettings(),
 	})
 	if err != nil {
 		return trace.Wrap(err)

lib/process/teleport.go

@@ -22,7 +22,9 @@ import (
 	"github.com/gravitational/gravity/lib/processconfig"
 	"github.com/gravitational/gravity/lib/storage"
 
+	"github.com/gravitational/teleport/lib/client"
 	"github.com/gravitational/teleport/lib/config"
+	teledefaults "github.com/gravitational/teleport/lib/defaults"
 	"github.com/gravitational/teleport/lib/service"
 
 	"github.com/gravitational/trace"
@@ -74,6 +76,9 @@ func (p *Process) buildTeleportConfig(authGatewayConfig storage.AuthGateway) (*s
 	serviceConfig.Access = p.identity
 	serviceConfig.Console = logrus.StandardLogger().Writer()
 	serviceConfig.ClusterConfiguration = p.identity
+	// Use high-res polling period so principal changes are detected
+	// faster when auth gateway settings are updated.
+	serviceConfig.PollingPeriod = teledefaults.HighResPollingPeriod
 	return serviceConfig, nil
 }
 
@@ -139,3 +144,19 @@ func (p *Process) getAuthGatewayConfig() (storage.AuthGateway, error) {
 	}
 	return opsservice.GetAuthGateway(client, p.identity)
 }
+
+// proxySettings returns Teleport proxy settings based on the Teleport config.
+func (p *Process) proxySettings() client.ProxySettings {
+	settings := client.ProxySettings{
+		Kube: client.KubeProxySettings{
+			Enabled: p.teleportConfig.Proxy.Kube.Enabled,
+		},
+		SSH: client.SSHProxySettings{
+			ListenAddr: p.teleportConfig.Proxy.SSHAddr.String(),
+		},
+	}
+	if len(p.teleportConfig.Proxy.Kube.PublicAddrs) > 0 {
+		settings.Kube.PublicAddr = p.teleportConfig.Proxy.Kube.PublicAddrs[0].String()
+	}
+	return settings
+}

lib/status/status.go

@@ -63,28 +63,29 @@ func FromCluster(ctx context.Context, operator ops.Operator, cluster ops.Site, o
 	}
 
 	// Collect application endpoints.
-	endpoints, err := operator.GetApplicationEndpoints(cluster.Key())
+	appEndpoints, err := operator.GetApplicationEndpoints(cluster.Key())
 	if err != nil {
 		logrus.WithError(err).Warn("Failed to fetch application endpoints.")
 		status.Endpoints.Applications.Error = err
 	}
-	if len(endpoints) != 0 {
+	if len(appEndpoints) != 0 {
 		// Right now only 1 application is supported, in the future there
 		// will be many applications each with its own endpoints.
 		status.Endpoints.Applications.Endpoints = append(status.Endpoints.Applications.Endpoints,
 			ApplicationEndpoints{
 				Application: cluster.App.Package,
-				Endpoints:   endpoints,
+				Endpoints:   appEndpoints,
 			})
 	}
 
-	// For cluster endpoints, they point to gravity-site service on master nodes.
-	masters := cluster.ClusterState.Servers.Masters()
-	for _, master := range masters {
-		status.Endpoints.Cluster.AuthGateway = append(status.Endpoints.Cluster.AuthGateway,
-			fmt.Sprintf("%v:%v", master.AdvertiseIP, defaults.GravitySiteNodePort))
-		status.Endpoints.Cluster.UI = append(status.Endpoints.Cluster.UI,
-			fmt.Sprintf("https://%v:%v", master.AdvertiseIP, defaults.GravitySiteNodePort))
+	// Fetch cluster endpoints.
+	clusterEndpoints, err := ops.GetClusterEndpoints(operator, cluster.Key())
+	if err != nil {
+		logrus.WithError(err).Warn("Failed to fetch cluster endpoints.")
+	}
+	if clusterEndpoints != nil {
+		status.Endpoints.Cluster.AuthGateway = clusterEndpoints.AuthGateways()
+		status.Endpoints.Cluster.UI = clusterEndpoints.ManagementURLs()
 	}
 
 	// FIXME: have status extension accept the operator/environment