From 786081c13d9f484983ec0143574d88393416bd24 Mon Sep 17 00:00:00 2001 From: David Hadas Date: Sat, 21 Sep 2024 13:42:41 -0500 Subject: [PATCH] SecureComms: Add support for inbound namespace Support opening inbound ports in configrable network namespaces. This allows opening an inbound port inside the pod network namespace of the podvm. The traffic will be forwarded via the SecureComms ssh channel to the outbound at the worker node. Signed-off-by: David Hadas --- src/cloud-api-adaptor/docs/SecureComms.md | 39 +++--- .../pkg/securecomms/sshproxy/sshproxy.go | 52 +++++++- .../pkg/securecomms/sshproxy/sshproxy_test.go | 118 ++++++++++++++++++ .../test/securecomms/test/httpClient.go | 39 ++++++ 4 files changed, 228 insertions(+), 20 deletions(-) diff --git a/src/cloud-api-adaptor/docs/SecureComms.md b/src/cloud-api-adaptor/docs/SecureComms.md index e9737dd8d..be320eda6 100644 --- a/src/cloud-api-adaptor/docs/SecureComms.md +++ b/src/cloud-api-adaptor/docs/SecureComms.md @@ -27,7 +27,7 @@ Once the "Kubernetes Phase" SSH channel is established, Secure Comms connects th See [Secure Comms Architecture Slides](./SecureComms.pdf) for more details. -## Setup +## Setup for CoCo with Trustee ### Deploy CAA Use any of the option for installing CAA depending on the cloud driver used. @@ -53,8 +53,7 @@ kubectl get secret kbs-client -n trustee-operator-system -o json|jq --arg ns "co For a testing environment, you may need to change the policy of the KBS and AS using the KBS Client to allow all or fit your own policy. One way to do that is: ```sh -kubectl -n trustee-operator-system exec deployment/trustee-deployment --container as -it -- /bin/bash - sed -i.bak 's/^default allow = false/default allow = true/' /opt/confidential-containers/attestation-service/opa/default.rego +kubectl -n trustee-operator-system exec deployment/trustee-deployment --container as -it -- sed -i.bak 's/^default allow = false/default allow = true/' /opt/confidential-containers/attestation-service/opa/default.rego kubectl -n trustee-operator-system get cm resource-policy -o yaml | sed "s/default allow = false/default allow = true/"|kubectl apply -f - ``` @@ -66,19 +65,21 @@ Change the `src/cloud-api-adaptor/podvm/files/etc/systemd/system/agent-protocol- ExecStart=/usr/local/bin/agent-protocol-forwarder -pod-namespace /run/netns/podns -secure-comms -kata-agent-socket /run/kata-containers/agent.sock $TLS_OPTIONS $OPTIONS ``` -You may also include additional Inbounds and Outbounds configurations to the Forwarder using the `-secure-comms-inbounds` and `-secure-comms-outbounds` flags. See more details regarding Inbounds and Outbounds below. +You may also include additional Inbounds and Outbounds configurations to the Forwarder using the `-secure-comms-inbounds` and `-secure-comms-outbounds` flags. [See more details regarding Inbounds and Outbounds below.](#adding-named-tunnels-to-the-ssh-channel) + +For example: +```sh +ExecStart=/usr/local/bin/agent-protocol-forwarder -kata-agent-namespace /run/netns/podns -secure-comms -secure-comms-inbounds KUBERNETES_PHASE:mytunnel:podns:6666 -kata-agent-socket /run/kata-containers/agent.sock $TLS_OPTIONS $OPTIONS +``` Once you changed `podvm/files/etc/systemd/system/agent-protocol-forwarder.service`, you will need to [rebuild the podvm](./../podvm/README.md). ### Activate CAA Secure-Comms feature -Use `kubectl edit cm peer-pods-cm -n confidential-containers-system` to add to the `peer-pods-cm` config map at the `confidential-containers-system` namespace: +Activate Secure-Comms of CAA by changing the `SECURE_COMMS` parameter of the `peer-pods-cm` configMap in the `confidential-containers-system` namespace to `"true"`. + ```sh -apiVersion: v1 -data: - ... - SECURE_COMMS: "true" - ... +kubectl -n confidential-containers-system get cm peer-pods-cm -o yaml | sed "s/SECURE_COMMS: \"false\"/SECURE_COMMS: \"true\"/"|kubectl apply -f - ``` Set InitData to point KBC services to IP address 127.0.0.1 @@ -105,10 +106,19 @@ url = 'http://127.0.0.1:8080' EOF export INITDATA=`base64 -w 0 /tmp/initdata.txt` kubectl -n confidential-containers-system get cm peer-pods-cm -o yaml | sed 's/^\s*INITDATA: .*/ INITDATA: '$INITDATA'/'|kubectl apply -f - - ``` -You may also include additional Inbounds and Outbounds configurations to the Adaptor using the `SECURE_COMMS_INBOUNDS` and `SECURE_COMMS_OUTBOUNDS` config points. See more details regarding Inbounds and Outbounds below. +You may also include additional Inbounds and Outbounds configurations to the Adaptor using the `SECURE_COMMS_INBOUNDS` and `SECURE_COMMS_OUTBOUNDS` config points. [See more details regarding Inbounds and Outbounds below.](#adding-named-tunnels-to-the-ssh-channel) + +Use `kubectl edit cm peer-pods-cm -n confidential-containers-system` to make such changes in the configMap, for example: +```sh +apiVersion: v1 +data: + ... + SECURE_COMMS: "true" + SECURE_COMMS_OUTBOUNDS: "KUBERNETES_PHASE:mytunnel:149.81.64.62:7777" + ... +``` You may also set the KBS address using the `SECURE_COMMS_KBS_ADDR` config point. @@ -120,7 +130,7 @@ You may also set the KBS address using the `SECURE_COMMS_KBS_ADDR` config point. > -### Adding named tunnels to the SSH channel +## Adding named tunnels to the SSH channel Named tunnels can be added to the SSH channel. Adding a named tunnel requires adding an Inbound at one of the SSH channel peers and an Outbound at the other SSH channel peer. The Inbound and Outbound both carry the name of the tunnel being created. |---------Tunnel----------| @@ -129,9 +139,10 @@ Named tunnels can be added to the SSH channel. Adding a named tunnel requires ad Inbounds and Outbounds take the form of a comma separated inbound/outbound tags such that Inbounds are formed as "InboundTag1,InboundTag2,InboundTag3,..." and Outbounds are formed as "OutboundTag1,OutboundTag2,outboundTag3,..." -Each Inbound tag is structured as `Phase:Name:Port` where: +Each Inbound tag is structured as `Phase:Name:Namespace:Port` or `Phase:Name:Port` where: - Phase can be 'KUBERNETES_PHASE' to represent an outbound available during the Kubernetes phase, 'ATTESTATION_PHASE' to represent an outbound available during the Attestation phase, or 'BOTH_PHASES' to represent an outbound available during both phases. - Name is the name of the tunnel +- Namespace (if available) is a linux network namespace where the local service should be available. - Port is the local service port being opened to serve as ingress of the tunnel. Each outbound tag is structured as `Phase:Name:Host:Port` or `Phase:Name:Port` where: diff --git a/src/cloud-api-adaptor/pkg/securecomms/sshproxy/sshproxy.go b/src/cloud-api-adaptor/pkg/securecomms/sshproxy/sshproxy.go index 972eac8c9..aae9f313b 100644 --- a/src/cloud-api-adaptor/pkg/securecomms/sshproxy/sshproxy.go +++ b/src/cloud-api-adaptor/pkg/securecomms/sshproxy/sshproxy.go @@ -9,11 +9,13 @@ import ( "net/http" "net/http/httputil" "net/url" + "path/filepath" "strconv" "strings" "sync" "github.com/confidential-containers/cloud-api-adaptor/src/cloud-api-adaptor/pkg/securecomms/sshutil" + "github.com/confidential-containers/cloud-api-adaptor/src/cloud-api-adaptor/pkg/util/netops" "golang.org/x/crypto/ssh" ) @@ -102,11 +104,11 @@ func (inbounds *Inbounds) AddTags(tags []string, inboundPorts map[string]string, if tag == "" { continue } - inPort, _, name, phase, err := ParseTag(tag) + inPort, namespace, name, phase, err := ParseTag(tag) if err != nil { return fmt.Errorf("failed to parse inbound tag %s: %v", tag, err) } - retPort, err := inbounds.Add(inPort, name, phase, wg) + retPort, err := inbounds.Add(namespace, inPort, name, phase, wg) if err != nil { return fmt.Errorf("failed to add inbound: %v", err) } @@ -117,12 +119,34 @@ func (inbounds *Inbounds) AddTags(tags []string, inboundPorts map[string]string, return nil } -func (inbounds *Inbounds) Add(inPort int, name, phase string, wg *sync.WaitGroup) (string, error) { +func (inbounds *Inbounds) listen(namespace string, tcpAddr *net.TCPAddr, name string) (tcpListener *net.TCPListener, err error) { + if namespace == "" { + return net.ListenTCP("tcp", tcpAddr) + } + + var ns netops.Namespace + ns, netopsErr := netops.OpenNamespace(filepath.Join("/run/netns", namespace)) + if netopsErr != nil { + return nil, fmt.Errorf("inbound %s failed to OpenNamespace '%s': %w", name, namespace, netopsErr) + } + defer ns.Close() + + netopsErr = ns.Run(func() error { + tcpListener, err = net.ListenTCP("tcp", tcpAddr) + return err + }) + if netopsErr != nil { + return nil, fmt.Errorf("inbound %s failed to ListenTCP '%s': %w", name, namespace, netopsErr) + } + return +} + +func (inbounds *Inbounds) Add(namespace string, inPort int, name, phase string, wg *sync.WaitGroup) (string, error) { tcpAddr := &net.TCPAddr{ IP: net.IPv4(127, 0, 0, 1), Port: inPort, } - tcpListener, err := net.ListenTCP("tcp", tcpAddr) + tcpListener, err := inbounds.listen(namespace, tcpAddr, name) if err != nil { return "", fmt.Errorf("inbound failed to listen to host: %s port '%d' - err: %v", name, inPort, err) } @@ -130,7 +154,7 @@ func (inbounds *Inbounds) Add(inPort int, name, phase string, wg *sync.WaitGroup if err != nil { panic(err) } - logger.Printf("Inbound listening to port %s", retPort) + logger.Printf("Inbound listening to port %s in namespace %s", retPort, namespace) inbound := &Inbound{ Phase: phase, @@ -162,6 +186,22 @@ func (inbounds *Inbounds) DelAll() { inbounds.list = [](*Inbound){} } +// ParseTag() parses an inbound or outbound tag +// Outbound tags with structure :: +// +// are interperted to approach 127.0.0.1: +// +// Outbound tags with structure ::: +// +// are interperted to approach : +// +// Inbound tags with structure :: +// +// are interperted to serve 127.0.0.1: on host network namespace +// +// Inbound tags with structure ::: +// +// are interperted to serve 127.0.0.1: on network namepsace func ParseTag(tag string) (port int, host, name, phase string, err error) { var inPort string var uint64port uint64 @@ -174,7 +214,7 @@ func ParseTag(tag string) (port int, host, name, phase string, err error) { } else if len(splits) == 4 { phase = splits[0] name = splits[1] - host = splits[2] + host = splits[2] // host for outbound or network namespace for inbound inPort = splits[3] } else { err = fmt.Errorf("illegal tag: %s", tag) diff --git a/src/cloud-api-adaptor/pkg/securecomms/sshproxy/sshproxy_test.go b/src/cloud-api-adaptor/pkg/securecomms/sshproxy/sshproxy_test.go index 62c44b726..486fd360b 100644 --- a/src/cloud-api-adaptor/pkg/securecomms/sshproxy/sshproxy_test.go +++ b/src/cloud-api-adaptor/pkg/securecomms/sshproxy/sshproxy_test.go @@ -4,6 +4,8 @@ import ( "context" "crypto/rand" "crypto/rsa" + "errors" + "io/fs" "net" "sync" "testing" @@ -11,6 +13,9 @@ import ( "github.com/confidential-containers/cloud-api-adaptor/src/cloud-api-adaptor/pkg/securecomms/sshutil" "github.com/confidential-containers/cloud-api-adaptor/src/cloud-api-adaptor/test/securecomms/test" + "github.com/google/uuid" + "github.com/vishvananda/netlink" + "github.com/vishvananda/netns" "golang.org/x/crypto/ssh" ) @@ -126,6 +131,78 @@ func TestSshProxy(t *testing.T) { inbounds.DelAll() } +func TestSshProxyWithNamespace(t *testing.T) { + var wg sync.WaitGroup + + clientSshPeer, serverSshPeer := getPeers(t) + + namespace := uuid.NewString() + nsPath := "/run/netns/" + namespace + // Create a new network namespace + newns, err := netns.NewNamed(namespace) + if err != nil && !errors.Is(err, fs.ErrExist) { + if errors.Is(err, fs.ErrPermission) { + t.Skip("Skip due to missing permissions - run privileged!") + } + t.Errorf("netns.NewNamed(%s) returned err %s", namespace, err.Error()) + } + defer func() { + newns.Close() + if err := netns.DeleteNamed(namespace); err != nil { + t.Errorf("failed to delete a named network namespace %s: %v", namespace, err) + } + }() + + link, err := netlink.LinkByName("lo") + if err != nil { + t.Fatal(err) + } + + // bring the interface up + if err := netlink.LinkSetUp(link); err != nil { + t.Fatal(err) + } + + outbounds := Outbounds{} + if err := outbounds.AddTags([]string{"ATTESTATION_PHASE:ABC:127.0.0.1:7020", " "}); err != nil { + t.Error(err) + return + } + inboundPorts := map[string]string{} + inbounds := Inbounds{} + if err := inbounds.AddTags([]string{"ATTESTATION_PHASE:ABC:" + namespace + ":7010", " "}, inboundPorts, &wg); err != nil { + t.Error(err) + return + } + + serverSshPeer.AddOutbounds(outbounds) + clientSshPeer.AddInbounds(inbounds) + + clientSshPeer.Ready() + serverSshPeer.Ready() + + s := test.HttpServer("7020") + success := test.HttpClientInNamespace("http://127.0.0.1:7010", nsPath) + if !success { + t.Error("Failed - not successful") + return + } + if err := s.Shutdown(context.Background()); err != nil { + t.Error(err) + return + } + + serverSshPeer.Upgrade() + serverSshPeer.Close("Test Finish") + + clientSshPeer.Wait() + if !clientSshPeer.IsUpgraded() { + t.Errorf("attestation phase closed without being upgraded") + return + } + inbounds.DelAll() +} + func TestSshProxyReverse(t *testing.T) { var wg sync.WaitGroup @@ -203,3 +280,44 @@ func TestSshProxyReverseKBS(t *testing.T) { t.Errorf("attestation phase closed without being upgraded") } } + +func TestParseTag(t *testing.T) { + + tests := []struct { + name string + tag string + wantPort int + wantHost string + wantName string + wantPhase string + wantErr bool + }{ + {name: "::", tag: "KUBERNETES_PHASE:nn:12", wantPort: 12, wantHost: "", wantName: "nn", wantPhase: "KUBERNETES_PHASE", wantErr: false}, + {name: ":::", tag: "ATTESTATION_PHASE:nn:12", wantPort: 12, wantHost: "", wantName: "nn", wantPhase: "ATTESTATION_PHASE", wantErr: false}, + {name: "::", tag: "MY_PHASE:nn:12", wantPort: 12, wantHost: "", wantName: "nn", wantPhase: "MY_PHASE", wantErr: true}, + {name: ":", tag: "ATTESTATION_PHASE:12", wantPort: 0, wantHost: "", wantName: "", wantPhase: "", wantErr: true}, + {name: "", tag: "ATTESTATION_PHASE", wantPort: 0, wantHost: "", wantName: "", wantPhase: "", wantErr: true}, + {name: "::", tag: "ATTESTATION_PHASE:12", wantPort: 0, wantHost: "", wantName: "", wantPhase: "", wantErr: true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + gotPort, gotHost, gotName, gotPhase, err := ParseTag(tt.tag) + if (err != nil) != tt.wantErr { + t.Errorf("ParseTag() error = %v, wantErr %v", err, tt.wantErr) + return + } + if gotPort != tt.wantPort { + t.Errorf("ParseTag() gotPort = %v, want %v", gotPort, tt.wantPort) + } + if gotHost != tt.wantHost { + t.Errorf("ParseTag() gotHost = %v, want %v", gotHost, tt.wantHost) + } + if gotName != tt.wantName { + t.Errorf("ParseTag() gotName = %v, want %v", gotName, tt.wantName) + } + if gotPhase != tt.wantPhase { + t.Errorf("ParseTag() gotPhase = %v, want %v", gotPhase, tt.wantPhase) + } + }) + } +} diff --git a/src/cloud-api-adaptor/test/securecomms/test/httpClient.go b/src/cloud-api-adaptor/test/securecomms/test/httpClient.go index 1bd9f2a2c..010078bd9 100644 --- a/src/cloud-api-adaptor/test/securecomms/test/httpClient.go +++ b/src/cloud-api-adaptor/test/securecomms/test/httpClient.go @@ -1,9 +1,13 @@ package test import ( + "context" "fmt" "io" + "net" "net/http" + + "github.com/confidential-containers/cloud-api-adaptor/src/cloud-api-adaptor/pkg/util/netops" ) func HttpClient(dest string) bool { @@ -25,3 +29,38 @@ func HttpClient(dest string) bool { fmt.Printf("HttpClient %s Body : %s\n", dest, body) return (resp.StatusCode == 200) } + +func HttpClientInNamespace(dest string, nsPath string) bool { + fmt.Printf("HttpClient start : %s in namepspace: %s\n", dest, nsPath) + + c := http.Client{ + Transport: &http.Transport{ + // Override the DialContext to enable connecting to a service at the pod namespace + DialContext: func(ctx context.Context, network, addr string) (conn net.Conn, err error) { + runErr := netops.RunAsNsPath(nsPath, func() error { + fmt.Printf("HttpClient dialing req: %s in namepspace: %s\n", addr, nsPath) + conn, err = (&net.Dialer{}).DialContext(ctx, network, addr) + return nil + }) + if runErr != nil { + return nil, runErr + } + return + }, + }, + } + + resp, err := c.Get(dest) + if err != nil { + fmt.Printf("HttpClient %s Get Error %s\n", dest, err) + return false + } + defer resp.Body.Close() + body, err := io.ReadAll(resp.Body) + if err != nil { + fmt.Printf("HttpClient %s ReadAll Error %s\n", dest, err) + return false + } + fmt.Printf("HttpClient %s StatusCode %d Body : %s\n", dest, resp.StatusCode, body) + return (resp.StatusCode == 200) +}