From 07cc49efdbf3c1f7a22a73b2daab7aab1a9cd1be Mon Sep 17 00:00:00 2001 From: Ygal Blum Date: Sun, 1 Jan 2023 09:49:08 +0200 Subject: [PATCH] Kube Play - allow setting and overriding published host ports Add a new flag --publish Remote - Pass PublishPorts as a string array ABI - translate the string array to Ports and merge with the ports in the spec Add e2e tests Add option to man doc Signed-off-by: Ygal Blum --- cmd/podman/kube/play.go | 4 + docs/source/markdown/podman-kube-play.1.md.in | 7 + pkg/api/handlers/libpod/kube.go | 44 +-- pkg/bindings/kube/types.go | 2 + pkg/bindings/kube/types_play_options.go | 15 + pkg/domain/entities/play.go | 2 + pkg/domain/infra/abi/play.go | 40 +++ pkg/domain/infra/tunnel/kube.go | 1 + test/e2e/play_kube_test.go | 301 ++++++++++++++++++ 9 files changed, 395 insertions(+), 21 deletions(-) diff --git a/cmd/podman/kube/play.go b/cmd/podman/kube/play.go index d4ee11f737..4a62f02759 100644 --- a/cmd/podman/kube/play.go +++ b/cmd/podman/kube/play.go @@ -151,6 +151,10 @@ func playFlags(cmd *cobra.Command) { replaceFlagName := "replace" flags.BoolVar(&playOptions.Replace, replaceFlagName, false, "Delete and recreate pods defined in the YAML file") + publishPortsFlagName := "publish" + flags.StringSliceVar(&playOptions.PublishPorts, publishPortsFlagName, []string{}, "Publish a container's port, or a range of ports, to the host") + _ = cmd.RegisterFlagCompletionFunc(publishPortsFlagName, completion.AutocompleteNone) + if !registry.IsRemote() { certDirFlagName := "cert-dir" flags.StringVar(&playOptions.CertDir, certDirFlagName, "", "`Pathname` of a directory containing TLS certificates and keys") diff --git a/docs/source/markdown/podman-kube-play.1.md.in b/docs/source/markdown/podman-kube-play.1.md.in index 886b2a07ff..59a8589fbd 100644 --- a/docs/source/markdown/podman-kube-play.1.md.in +++ b/docs/source/markdown/podman-kube-play.1.md.in @@ -170,6 +170,13 @@ When no network option is specified and *host* network mode is not configured in This option conflicts with host added in the Kubernetes YAML. +#### **--publish**=*[[ip:][hostPort]:]containerPort[/protocol]* + +Define or override a port definition in the YAML file. + +The lists of ports in the YAML file and the command line are merged. Matching is done by using the **containerPort** field. +If **containerPort** exists in both the YAML file and the option, the latter takes precedence. + #### **--quiet**, **-q** Suppress output information when pulling images diff --git a/pkg/api/handlers/libpod/kube.go b/pkg/api/handlers/libpod/kube.go index 4715286020..77dddaf02d 100644 --- a/pkg/api/handlers/libpod/kube.go +++ b/pkg/api/handlers/libpod/kube.go @@ -19,15 +19,16 @@ func KubePlay(w http.ResponseWriter, r *http.Request) { runtime := r.Context().Value(api.RuntimeKey).(*libpod.Runtime) decoder := r.Context().Value(api.DecoderKey).(*schema.Decoder) query := struct { - Annotations map[string]string `schema:"annotations"` - Network []string `schema:"network"` - TLSVerify bool `schema:"tlsVerify"` - LogDriver string `schema:"logDriver"` - LogOptions []string `schema:"logOptions"` - Start bool `schema:"start"` - StaticIPs []string `schema:"staticIPs"` - StaticMACs []string `schema:"staticMACs"` - NoHosts bool `schema:"noHosts"` + Annotations map[string]string `schema:"annotations"` + Network []string `schema:"network"` + TLSVerify bool `schema:"tlsVerify"` + LogDriver string `schema:"logDriver"` + LogOptions []string `schema:"logOptions"` + Start bool `schema:"start"` + StaticIPs []string `schema:"staticIPs"` + StaticMACs []string `schema:"staticMACs"` + NoHosts bool `schema:"noHosts"` + PublishPorts []string `schema:"publishPorts"` }{ TLSVerify: true, Start: true, @@ -82,18 +83,19 @@ func KubePlay(w http.ResponseWriter, r *http.Request) { containerEngine := abi.ContainerEngine{Libpod: runtime} options := entities.PlayKubeOptions{ - Annotations: query.Annotations, - Authfile: authfile, - Username: username, - Password: password, - Networks: query.Network, - NoHosts: query.NoHosts, - Quiet: true, - LogDriver: logDriver, - LogOptions: query.LogOptions, - StaticIPs: staticIPs, - StaticMACs: staticMACs, - IsRemote: true, + Annotations: query.Annotations, + Authfile: authfile, + Username: username, + Password: password, + Networks: query.Network, + NoHosts: query.NoHosts, + Quiet: true, + LogDriver: logDriver, + LogOptions: query.LogOptions, + StaticIPs: staticIPs, + StaticMACs: staticMACs, + IsRemote: true, + PublishPorts: query.PublishPorts, } if _, found := r.URL.Query()["tlsVerify"]; found { options.SkipTLSVerify = types.NewOptionalBool(!query.TLSVerify) diff --git a/pkg/bindings/kube/types.go b/pkg/bindings/kube/types.go index c41a578a58..8cd7c79262 100644 --- a/pkg/bindings/kube/types.go +++ b/pkg/bindings/kube/types.go @@ -48,6 +48,8 @@ type PlayOptions struct { Userns *string // Force - remove volumes on --down Force *bool + // PublishPorts - configure how to expose ports configured inside the K8S YAML file + PublishPorts []string } // ApplyOptions are optional options for applying kube YAML files to a k8s cluster diff --git a/pkg/bindings/kube/types_play_options.go b/pkg/bindings/kube/types_play_options.go index 6db3241577..522fac85d8 100644 --- a/pkg/bindings/kube/types_play_options.go +++ b/pkg/bindings/kube/types_play_options.go @@ -302,3 +302,18 @@ func (o *PlayOptions) GetForce() bool { } return *o.Force } + +// WithPublishPorts set field PublishPorts to given value +func (o *PlayOptions) WithPublishPorts(value []string) *PlayOptions { + o.PublishPorts = value + return o +} + +// GetPublishPorts returns value of field PublishPorts +func (o *PlayOptions) GetPublishPorts() []string { + if o.PublishPorts == nil { + var z []string + return z + } + return o.PublishPorts +} diff --git a/pkg/domain/entities/play.go b/pkg/domain/entities/play.go index a74fdbef4d..bd14b29680 100644 --- a/pkg/domain/entities/play.go +++ b/pkg/domain/entities/play.go @@ -62,6 +62,8 @@ type PlayKubeOptions struct { IsRemote bool // Force - remove volumes on --down Force bool + // PublishPorts - configure how to expose ports configured inside the K8S YAML file + PublishPorts []string } // PlayKubePod represents a single pod and associated containers created by play kube diff --git a/pkg/domain/infra/abi/play.go b/pkg/domain/infra/abi/play.go index e3752c0a61..7a351d2417 100644 --- a/pkg/domain/infra/abi/play.go +++ b/pkg/domain/infra/abi/play.go @@ -465,6 +465,14 @@ func (ic *ContainerEngine) playKubePod(ctx context.Context, podName string, podY } *ipIndex++ + if len(options.PublishPorts) > 0 { + publishPorts, err := specgenutil.CreatePortBindings(options.PublishPorts) + if err != nil { + return nil, nil, err + } + mergePublishPorts(&podOpt, publishPorts) + } + p := specgen.NewPodSpecGenerator() if err != nil { return nil, nil, err @@ -1001,6 +1009,38 @@ func (ic *ContainerEngine) playKubePVC(ctx context.Context, pvcYAML *v1.Persiste return &report, nil } +func mergePublishPorts(p *entities.PodCreateOptions, publishPortsOption []nettypes.PortMapping) { + for _, publishPortSpec := range p.Net.PublishPorts { + if !portAlreadyPublished(publishPortSpec, publishPortsOption) { + publishPortsOption = append(publishPortsOption, publishPortSpec) + } + } + p.Net.PublishPorts = publishPortsOption +} + +func portAlreadyPublished(port nettypes.PortMapping, publishedPorts []nettypes.PortMapping) bool { + for _, publishedPort := range publishedPorts { + if port.ContainerPort >= publishedPort.ContainerPort && + port.ContainerPort < publishedPort.ContainerPort+publishedPort.Range && + isSamePortProtocol(port.Protocol, publishedPort.Protocol) { + return true + } + } + return false +} + +func isSamePortProtocol(a, b string) bool { + if len(a) == 0 { + a = string(v1.ProtocolTCP) + } + if len(b) == 0 { + b = string(v1.ProtocolTCP) + } + + ret := strings.EqualFold(a, b) + return ret +} + func (ic *ContainerEngine) importVolume(ctx context.Context, vol *libpod.Volume, tarFile *os.File) error { volumeConfig, err := vol.Config() if err != nil { diff --git a/pkg/domain/infra/tunnel/kube.go b/pkg/domain/infra/tunnel/kube.go index 7942833dcb..6e42240fe2 100644 --- a/pkg/domain/infra/tunnel/kube.go +++ b/pkg/domain/infra/tunnel/kube.go @@ -72,6 +72,7 @@ func (ic *ContainerEngine) PlayKube(ctx context.Context, body io.Reader, opts en if start := opts.Start; start != types.OptionalBoolUndefined { options.WithStart(start == types.OptionalBoolTrue) } + options.WithPublishPorts(opts.PublishPorts) return play.KubeWithBody(ic.ClientCtx, body, options) } diff --git a/test/e2e/play_kube_test.go b/test/e2e/play_kube_test.go index c37a87c5b5..336e3b47ef 100644 --- a/test/e2e/play_kube_test.go +++ b/test/e2e/play_kube_test.go @@ -1,12 +1,15 @@ package integration import ( + "bufio" "bytes" "context" "encoding/base64" "encoding/json" "fmt" + "io" "net" + "net/http" "net/url" "os" "os/user" @@ -850,6 +853,94 @@ spec: {{ end }} ` +var publishPortsPodWithoutPorts = ` +apiVersion: v1 +kind: Pod +metadata: + name: nginx +spec: + containers: + - name: nginx + image: quay.io/libpod/alpine_nginx:latest +` + +var publishPortsPodWithContainerPort = ` +apiVersion: v1 +kind: Pod +metadata: + name: nginx +spec: + containers: + - name: nginx + image: quay.io/libpod/alpine_nginx:latest + ports: + - containerPort: 80 +` + +var publishPortsPodWithContainerHostPort = ` +apiVersion: v1 +kind: Pod +metadata: + name: nginx +spec: + containers: + - name: nginx + image: quay.io/libpod/alpine_nginx:latest + ports: + - containerPort: 80 + hostPort: 19001 +` + +var publishPortsEchoWithHostPortUDP = ` +apiVersion: v1 +kind: Pod +metadata: + name: network-echo +spec: + containers: + - name: udp-echo + image: quay.io/libpod/busybox:latest + command: + - "/bin/sh" + - "-c" + - "nc -ulk -p 19008 -e /bin/cat" + ports: + - containerPort: 19008 + hostPort: 19009 + protocol: udp + - name: tcp-echo + image: quay.io/libpod/busybox:latest + command: + - "/bin/sh" + - "-c" + - "nc -lk -p 19008 -e /bin/cat" +` + +var publishPortsEchoWithHostPortTCP = ` +apiVersion: v1 +kind: Pod +metadata: + name: network-echo +spec: + containers: + - name: udp-echo + image: quay.io/libpod/busybox:latest + command: + - "/bin/sh" + - "-c" + - "nc -ulk -p 19008 -e /bin/cat" + - name: tcp-echo + image: quay.io/libpod/busybox:latest + command: + - "/bin/sh" + - "-c" + - "nc -lk -p 19008 -e /bin/cat" + ports: + - containerPort: 19008 + hostPort: 19011 + protocol: tcp +` + var ( defaultCtrName = "testCtr" defaultCtrCmd = []string{"top"} @@ -1569,6 +1660,108 @@ func testPodWithSecret(podmanTest *PodmanTestIntegration, podYamlString, fileNam Expect(podRm).Should(Exit(0)) } +func testHTTPServer(port string, shouldErr bool, expectedResponse string) { + address := url.URL{ + Scheme: "http", + Host: net.JoinHostPort("localhost", port), + } + + interval := 250 * time.Millisecond + var err error + var resp *http.Response + for i := 0; i < 6; i++ { + resp, err = http.Get(address.String()) + if err != nil && shouldErr { + Expect(err.Error()).To(ContainSubstring(expectedResponse)) + return + } + if err == nil { + defer resp.Body.Close() + break + } + time.Sleep(interval) + interval *= 2 + } + Expect(err).To(BeNil()) + + body, err := io.ReadAll(resp.Body) + Expect(err).To(BeNil()) + Expect(string(body)).Should(Equal(expectedResponse)) +} + +func testEchoServer(connection io.ReadWriter) { + stringToSend := "hello world" + var err error + var bytesSent int + interval := 250 * time.Millisecond + for i := 0; i < 6; i++ { + bytesSent, err = fmt.Fprint(connection, stringToSend) + if err == nil { + break + } + time.Sleep(interval) + interval *= 2 + } + Expect(err).To(BeNil()) + Expect(bytesSent).To(Equal(len(stringToSend))) + + stringReceived := make([]byte, bytesSent) + var bytesRead int + interval = 250 * time.Millisecond + for i := 0; i < 6; i++ { + bytesRead, err = bufio.NewReader(connection).Read(stringReceived) + if err == nil { + break + } + time.Sleep(interval) + interval *= 2 + } + Expect(err).To(BeNil()) + Expect(bytesRead).To(Equal(bytesSent)) + + Expect(stringToSend).To(Equal(string(stringReceived))) +} + +func testEchoServerUDP(address string) { + udpServer, err := net.ResolveUDPAddr("udp", address) + Expect(err).To(BeNil()) + + interval := 250 * time.Millisecond + var conn *net.UDPConn + for i := 0; i < 6; i++ { + conn, err = net.DialUDP("udp", nil, udpServer) + if err == nil { + break + } + time.Sleep(interval) + interval *= 2 + } + Expect(err).To(BeNil()) + defer conn.Close() + + testEchoServer(conn) +} + +func testEchoServerTCP(address string) { + tcpServer, err := net.ResolveTCPAddr("tcp", address) + Expect(err).To(BeNil()) + + interval := 250 * time.Millisecond + var conn *net.TCPConn + for i := 0; i < 6; i++ { + conn, err = net.DialTCP("tcp", nil, tcpServer) + if err == nil { + break + } + time.Sleep(interval) + interval *= 2 + } + Expect(err).To(BeNil()) + defer conn.Close() + + testEchoServer(conn) +} + var _ = Describe("Podman play kube", func() { var ( tempdir string @@ -4677,4 +4870,112 @@ spec: Expect(exec.OutputToString()).Should(ContainSubstring("BAR")) // we want to check that we can mount a subpath but not replace the entire dir }) + + It("podman play kube without Ports - curl should fail", func() { + err := writeYaml(publishPortsPodWithoutPorts, kubeYaml) + Expect(err).ToNot(HaveOccurred()) + + kube := podmanTest.Podman([]string{"play", "kube", kubeYaml}) + kube.WaitWithDefaultTimeout() + Expect(kube).Should(Exit(0)) + + curlTest := podmanTest.Podman([]string{"run", "--network", "host", NGINX_IMAGE, "curl", "-s", "localhost:19000"}) + curlTest.WaitWithDefaultTimeout() + Expect(curlTest).Should(Exit(7)) + }) + + It("podman play kube without Ports, publish in command line - curl should succeed", func() { + err := writeYaml(publishPortsPodWithoutPorts, kubeYaml) + Expect(err).ToNot(HaveOccurred()) + + kube := podmanTest.Podman([]string{"play", "kube", "--publish", "19002:80", kubeYaml}) + kube.WaitWithDefaultTimeout() + Expect(kube).Should(Exit(0)) + + testHTTPServer("19002", false, "podman rulez") + }) + + It("podman play kube with privileged container ports - should fail", func() { + SkipIfNotRootless("rootlessport can expose privileged port 80, no point in checking for failure") + err := writeYaml(publishPortsPodWithContainerPort, kubeYaml) + Expect(err).ToNot(HaveOccurred()) + + kube := podmanTest.Podman([]string{"play", "kube", kubeYaml}) + kube.WaitWithDefaultTimeout() + Expect(kube).Should(Exit(125)) + // The error message is printed only on local call + if !IsRemote() { + Expect(kube.OutputToString()).Should(ContainSubstring("rootlessport cannot expose privileged port 80")) + } + }) + + It("podman play kube with privileged containers ports and publish in command line - curl should succeed", func() { + err := writeYaml(publishPortsPodWithContainerPort, kubeYaml) + Expect(err).ToNot(HaveOccurred()) + + kube := podmanTest.Podman([]string{"play", "kube", "--publish", "19003:80", kubeYaml}) + kube.WaitWithDefaultTimeout() + Expect(kube).Should(Exit(0)) + + testHTTPServer("19003", false, "podman rulez") + }) + + It("podman play kube with Host Ports - curl should succeed", func() { + err := writeYaml(publishPortsPodWithContainerHostPort, kubeYaml) + Expect(err).ToNot(HaveOccurred()) + + kube := podmanTest.Podman([]string{"play", "kube", "--publish", "19004:80", kubeYaml}) + kube.WaitWithDefaultTimeout() + Expect(kube).Should(Exit(0)) + + testHTTPServer("19004", false, "podman rulez") + }) + + It("podman play kube with Host Ports and publish in command line - curl should succeed only on overriding port", func() { + err := writeYaml(publishPortsPodWithContainerHostPort, kubeYaml) + Expect(err).ToNot(HaveOccurred()) + + kube := podmanTest.Podman([]string{"play", "kube", "--publish", "19005:80", kubeYaml}) + kube.WaitWithDefaultTimeout() + Expect(kube).Should(Exit(0)) + + testHTTPServer("19001", true, "connection refused") + testHTTPServer("19005", false, "podman rulez") + }) + + It("podman play kube multiple publish ports", func() { + err := writeYaml(publishPortsPodWithoutPorts, kubeYaml) + Expect(err).ToNot(HaveOccurred()) + + kube := podmanTest.Podman([]string{"play", "kube", "--publish", "19006:80", "--publish", "19007:80", kubeYaml}) + kube.WaitWithDefaultTimeout() + Expect(kube).Should(Exit(0)) + + testHTTPServer("19006", false, "podman rulez") + testHTTPServer("19007", false, "podman rulez") + }) + + It("podman play kube override with tcp should keep udp from YAML file", func() { + err := writeYaml(publishPortsEchoWithHostPortUDP, kubeYaml) + Expect(err).ToNot(HaveOccurred()) + + kube := podmanTest.Podman([]string{"play", "kube", "--publish", "19010:19008/tcp", kubeYaml}) + kube.WaitWithDefaultTimeout() + Expect(kube).Should(Exit(0)) + + testEchoServerUDP(":19009") + testEchoServerTCP(":19010") + }) + + It("podman play kube override with udp should keep tcp from YAML file", func() { + err := writeYaml(publishPortsEchoWithHostPortTCP, kubeYaml) + Expect(err).ToNot(HaveOccurred()) + + kube := podmanTest.Podman([]string{"play", "kube", "--publish", "19012:19008/udp", kubeYaml}) + kube.WaitWithDefaultTimeout() + Expect(kube).Should(Exit(0)) + + testEchoServerUDP(":19012") + testEchoServerTCP(":19011") + }) })