From 3ea7b8c90e74c1211efe4a606f23686694f3abf4 Mon Sep 17 00:00:00 2001 From: Thomas Tendyck Date: Tue, 16 Jul 2024 13:03:38 +0200 Subject: [PATCH] test: use go api in integration test --- test/framework/framework.go | 210 +++++++----------------------------- test/integration_test.go | 89 ++++----------- 2 files changed, 62 insertions(+), 237 deletions(-) diff --git a/test/framework/framework.go b/test/framework/framework.go index 938b2210..a6e8d298 100644 --- a/test/framework/framework.go +++ b/test/framework/framework.go @@ -9,7 +9,6 @@ package framework import ( "bufio" - "bytes" "context" "crypto/rsa" "crypto/tls" @@ -19,8 +18,6 @@ import ( "fmt" "io" "net" - "net/http" - "net/url" "os" "os/exec" "path/filepath" @@ -28,13 +25,13 @@ import ( "testing" "time" + "github.com/edgelesssys/marblerun/api" "github.com/edgelesssys/marblerun/coordinator/constants" "github.com/edgelesssys/marblerun/coordinator/manifest" "github.com/edgelesssys/marblerun/coordinator/store/stdstore" mconfig "github.com/edgelesssys/marblerun/marble/config" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "github.com/tidwall/gjson" ) // IntegrationTest is a testing framework for MarbleRun tests. @@ -43,16 +40,15 @@ type IntegrationTest struct { assert *assert.Assertions require *require.Assertions - Ctx context.Context - TestManifest manifest.Manifest - UpdatedManifest manifest.Manifest - BuildDir string - SimulationFlag string - NoEnclave bool - MeshServerAddr string - ClientServerAddr string - MarbleTestAddr string - transportSkipVerify http.RoundTripper + Ctx context.Context + TestManifest manifest.Manifest + UpdatedManifest manifest.Manifest + BuildDir string + SimulationFlag string + NoEnclave bool + MeshServerAddr string + ClientServerAddr string + MarbleTestAddr string } // New creates a new IntegrationTest. @@ -68,14 +64,13 @@ func New(t *testing.T, buildDir, simulation string, noenclave bool, assert: assert.New(t), require: require.New(t), - Ctx: ctx, - BuildDir: buildDir, - SimulationFlag: simulation, - NoEnclave: noenclave, - MeshServerAddr: meshServerAddr, - ClientServerAddr: clientServerAddr, - MarbleTestAddr: marbleTestAddr, - transportSkipVerify: &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}}, + Ctx: ctx, + BuildDir: buildDir, + SimulationFlag: simulation, + NoEnclave: noenclave, + MeshServerAddr: meshServerAddr, + ClientServerAddr: clientServerAddr, + MarbleTestAddr: marbleTestAddr, } i.require.NoError(json.Unmarshal([]byte(testManifest), &i.TestManifest)) @@ -154,9 +149,6 @@ func (i IntegrationTest) StartCoordinator(ctx context.Context, cfg CoordinatorCo cmd.Env = append(cmd.Env, cfg.extraEnv...) cmdErr := i.StartCommand("coor", cmd) - client := http.Client{Transport: i.transportSkipVerify} - url := url.URL{Scheme: "https", Host: i.ClientServerAddr, Path: "status"} - i.t.Log("Coordinator starting...") for { time.Sleep(10 * time.Millisecond) @@ -167,14 +159,8 @@ func (i IntegrationTest) StartCoordinator(ctx context.Context, cfg CoordinatorCo default: } - req, err := http.NewRequestWithContext(context.Background(), http.MethodGet, url.String(), http.NoBody) - i.require.NoError(err) - - resp, err := client.Do(req) - if err == nil { + if _, _, err := api.GetStatus(context.Background(), i.ClientServerAddr, nil); err == nil { i.t.Log("Coordinator started") - resp.Body.Close() - i.require.Equal(http.StatusOK, resp.StatusCode) return func() { _ = cmd.Cancel() <-cmdErr @@ -214,38 +200,14 @@ func (i IntegrationTest) StartCommand(friendlyName string, cmd *exec.Cmd) chan e } // SetManifest sets the manifest of the Coordinator. -func (i IntegrationTest) SetManifest(manifest manifest.Manifest) ([]byte, error) { - // Use ClientAPI to set Manifest - client := http.Client{Transport: i.transportSkipVerify} - clientAPIURL := url.URL{ - Scheme: "https", - Host: i.ClientServerAddr, - Path: "manifest", - } - +func (i IntegrationTest) SetManifest(manifest manifest.Manifest) (map[string][]byte, error) { manifestRaw, err := json.Marshal(manifest) i.require.NoError(err) - - req, err := http.NewRequestWithContext(context.Background(), http.MethodPost, clientAPIURL.String(), bytes.NewReader(manifestRaw)) - i.require.NoError(err) - req.Header.Set("Content-Type", "application/json") - - resp, err := client.Do(req) - i.require.NoError(err) - defer resp.Body.Close() - - body, err := io.ReadAll(resp.Body) - i.require.NoError(err) - - if resp.StatusCode != http.StatusOK { - return nil, fmt.Errorf("expected %v, but /manifest returned %v: %v", http.StatusOK, resp.Status, string(body)) - } - - return body, nil + return api.ManifestSet(context.Background(), i.ClientServerAddr, nil, manifestRaw) } // SetUpdateManifest sets performs a manifest update for the Coordinator. -func (i IntegrationTest) SetUpdateManifest(manifest manifest.Manifest, certPEM []byte, key *rsa.PrivateKey) ([]byte, error) { +func (i IntegrationTest) SetUpdateManifest(manifest manifest.Manifest, certPEM []byte, key *rsa.PrivateKey) error { // Setup requied client certificate for authentication privk, err := x509.MarshalPKCS8PrivateKey(key) i.require.NoError(err) @@ -253,92 +215,22 @@ func (i IntegrationTest) SetUpdateManifest(manifest manifest.Manifest, certPEM [ cert, err := tls.X509KeyPair(certPEM, pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: privk})) i.require.NoError(err) - tlsConfig := &tls.Config{ - Certificates: []tls.Certificate{cert}, - InsecureSkipVerify: true, - } - - transport := &http.Transport{TLSClientConfig: tlsConfig} - - // Use ClientAPI to set Manifest - client := http.Client{Transport: transport} - clientAPIURL := url.URL{ - Scheme: "https", - Host: i.ClientServerAddr, - Path: "update", - } - manifestRaw, err := json.Marshal(manifest) i.require.NoError(err) - req, err := http.NewRequestWithContext(context.Background(), http.MethodPost, clientAPIURL.String(), bytes.NewReader(manifestRaw)) - i.require.NoError(err) - req.Header.Set("Content-Type", "application/json") - - resp, err := client.Do(req) - i.require.NoError(err) - defer resp.Body.Close() - - body, err := io.ReadAll(resp.Body) - i.require.NoError(err) - - if resp.StatusCode != http.StatusOK { - return nil, fmt.Errorf("expected %v, but /manifest returned %v: %v", http.StatusOK, resp.Status, string(body)) - } - - return body, nil + return api.ManifestUpdateApply(context.Background(), i.ClientServerAddr, nil, manifestRaw, &cert) } // SetRecover sets the recovery key of the Coordinator. func (i IntegrationTest) SetRecover(recoveryKey []byte) error { - client := http.Client{Transport: i.transportSkipVerify} - clientAPIURL := url.URL{ - Scheme: "https", - Host: i.ClientServerAddr, - Path: "recover", - } - - req, err := http.NewRequestWithContext(context.Background(), http.MethodPost, clientAPIURL.String(), bytes.NewReader(recoveryKey)) - i.require.NoError(err) - req.Header.Set("Content-Type", "application/octet-stream") - - resp, err := client.Do(req) - i.require.NoError(err) - defer resp.Body.Close() - - body, err := io.ReadAll(resp.Body) - i.require.NoError(err) - - if resp.StatusCode != http.StatusOK { - return fmt.Errorf("expected %v, but /recover returned %v: %v", http.StatusOK, resp.Status, string(body)) - } - - return nil + _, _, err := api.Recover(context.Background(), i.ClientServerAddr, api.VerifyOptions{InsecureSkipVerify: true}, recoveryKey) + return err } // GetStatus returns the status of the Coordinator. -func (i IntegrationTest) GetStatus() (string, error) { - client := http.Client{Transport: i.transportSkipVerify} - clientAPIURL := url.URL{ - Scheme: "https", - Host: i.ClientServerAddr, - Path: "status", - } - - req, err := http.NewRequestWithContext(context.Background(), http.MethodGet, clientAPIURL.String(), http.NoBody) - i.require.NoError(err) - resp, err := client.Do(req) - i.require.NoError(err) - defer resp.Body.Close() - - body, err := io.ReadAll(resp.Body) - i.require.NoError(err) - - if resp.StatusCode != http.StatusOK { - return "", fmt.Errorf("expected %v, but /status returned %v: %v", http.StatusOK, resp.Status, string(body)) - } - - return string(body), nil +func (i IntegrationTest) GetStatus() (int, error) { + code, _, err := api.GetStatus(context.Background(), i.ClientServerAddr, nil) + return code, err } // MarbleConfig contains the configuration for a Marble. @@ -438,21 +330,11 @@ func (i IntegrationTest) StartMarbleClient(ctx context.Context, cfg MarbleConfig } // TriggerRecovery triggers a recovery. -func (i IntegrationTest) TriggerRecovery(coordinatorCfg CoordinatorConfig, cancelCoordinator func()) (func(), string) { +func (i IntegrationTest) TriggerRecovery(coordinatorCfg CoordinatorConfig, cancelCoordinator func()) (func(), *x509.Certificate) { // get certificate i.t.Log("Save certificate before we try to recover.") - client := http.Client{Transport: i.transportSkipVerify} - clientAPIURL := url.URL{Scheme: "https", Host: i.ClientServerAddr, Path: "quote"} - req, err := http.NewRequestWithContext(context.Background(), http.MethodGet, clientAPIURL.String(), http.NoBody) - i.require.NoError(err) - resp, err := client.Do(req) + cert, _, _, err := api.VerifyCoordinator(context.Background(), i.ClientServerAddr, api.VerifyOptions{InsecureSkipVerify: true}) i.require.NoError(err) - i.require.Equal(http.StatusOK, resp.StatusCode) - quote, err := io.ReadAll(resp.Body) - resp.Body.Close() - i.require.NoError(err) - cert := gjson.GetBytes(quote, "data.Cert").String() - i.require.NotEmpty(cert) // simulate restart of coordinator i.t.Log("Simulating a restart of the coordinator enclave...") @@ -469,27 +351,19 @@ func (i IntegrationTest) TriggerRecovery(coordinatorCfg CoordinatorConfig, cance // Query status API, check if status response begins with Code 1 (recovery state) i.t.Log("Checking status...") - statusResponse, err := i.GetStatus() + statusCode, err := i.GetStatus() i.require.NoError(err) - i.assert.EqualValues(1, gjson.Get(statusResponse, "data.StatusCode").Int(), "Server is not in recovery state, but should be.") + i.assert.EqualValues(1, statusCode, "Server is not in recovery state, but should be.") return cancelCoordinator, cert } // VerifyCertAfterRecovery verifies the certificate after a recovery. -func (i IntegrationTest) VerifyCertAfterRecovery(cert string, cancelCoordinator func(), cfg CoordinatorConfig) func() { +func (i IntegrationTest) VerifyCertAfterRecovery(cert *x509.Certificate, cancelCoordinator func(), cfg CoordinatorConfig) func() { // Test with certificate i.t.Log("Verifying certificate after recovery, without a restart.") - pool := x509.NewCertPool() - i.require.True(pool.AppendCertsFromPEM([]byte(cert))) - client := http.Client{Transport: &http.Transport{TLSClientConfig: &tls.Config{RootCAs: pool}}} - clientAPIURL := url.URL{Scheme: "https", Host: i.ClientServerAddr, Path: "status"} - req, err := http.NewRequestWithContext(context.Background(), http.MethodGet, clientAPIURL.String(), http.NoBody) - i.require.NoError(err) - resp, err := client.Do(req) + _, _, err := api.GetStatus(context.Background(), i.ClientServerAddr, cert) i.require.NoError(err) - resp.Body.Close() - i.require.Equal(http.StatusOK, resp.StatusCode) // Simulate restart of coordinator i.t.Log("Simulating a restart of the coordinator enclave...") @@ -502,18 +376,14 @@ func (i IntegrationTest) VerifyCertAfterRecovery(cert string, cancelCoordinator // Finally, check if we survive a restart. i.t.Log("Restarted instance, now let's see if the state can be restored again successfully.") - statusResponse, err := i.GetStatus() + statusCode, err := i.GetStatus() i.require.NoError(err) - i.assert.EqualValues(3, gjson.Get(statusResponse, "data.StatusCode").Int(), "Server is in wrong status after recovery.") + i.assert.EqualValues(3, statusCode, "Server is in wrong status after recovery.") // test with certificate i.t.Log("Verifying certificate after restart.") - req, err = http.NewRequestWithContext(context.Background(), http.MethodGet, clientAPIURL.String(), http.NoBody) - i.require.NoError(err) - resp, err = client.Do(req) + _, _, err = api.GetStatus(context.Background(), i.ClientServerAddr, cert) i.require.NoError(err) - resp.Body.Close() - i.require.Equal(http.StatusOK, resp.StatusCode) return cancelCoordinator } @@ -522,9 +392,9 @@ func (i IntegrationTest) VerifyCertAfterRecovery(cert string, cancelCoordinator func (i IntegrationTest) VerifyResetAfterRecovery(cancelCoordinator func(), cfg CoordinatorConfig) func() { // Check status after setting a new manifest, we should be able i.t.Log("Check if the manifest was accepted and we are ready to accept Marbles") - statusResponse, err := i.GetStatus() + statusCode, err := i.GetStatus() i.require.NoError(err) - i.assert.EqualValues(3, gjson.Get(statusResponse, "data.StatusCode").Int(), "Server is in wrong status after recovery.") + i.assert.EqualValues(3, statusCode, "Server is in wrong status after recovery.") // simulate restart of coordinator i.t.Log("Simulating a restart of the coordinator enclave...") @@ -537,9 +407,9 @@ func (i IntegrationTest) VerifyResetAfterRecovery(cancelCoordinator func(), cfg // Finally, check if we survive a restart. i.t.Log("Restarted instance, now let's see if the new state can be decrypted successfully...") - statusResponse, err = i.GetStatus() + statusCode, err = i.GetStatus() i.require.NoError(err) - i.assert.EqualValues(3, gjson.Get(statusResponse, "data.StatusCode").Int(), "Server is in wrong status after recovery.") + i.assert.EqualValues(3, statusCode, "Server is in wrong status after recovery.") return cancelCoordinator } diff --git a/test/integration_test.go b/test/integration_test.go index a29852b7..6d7321bf 100644 --- a/test/integration_test.go +++ b/test/integration_test.go @@ -12,23 +12,20 @@ import ( "context" "crypto/tls" "crypto/x509" - "encoding/base64" + "encoding/json" "encoding/pem" "flag" - "io" "log" "net" - "net/http" - "net/url" "os" - "strings" "testing" + "github.com/edgelesssys/marblerun/api" + "github.com/edgelesssys/marblerun/coordinator/manifest" "github.com/edgelesssys/marblerun/test/framework" "github.com/edgelesssys/marblerun/util" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "github.com/tidwall/gjson" ) var ( @@ -36,7 +33,6 @@ var ( simulationMode = flag.Bool("s", false, "Execute test in simulation mode (without real quoting)") noenclave = flag.Bool("noenclave", false, "Do not run with erthost") meshServerAddr, clientServerAddr, marbleTestAddr string - transportSkipVerify = &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}} simFlag string ) @@ -182,44 +178,19 @@ func TestClientAPI(t *testing.T) { f.StartCoordinator(f.Ctx, cfg) // get certificate - client := http.Client{Transport: transportSkipVerify} - clientAPIURL := url.URL{Scheme: "https", Host: clientServerAddr, Path: "quote"} - req, err := http.NewRequestWithContext(context.Background(), http.MethodGet, clientAPIURL.String(), http.NoBody) + cert, _, _, err := api.VerifyCoordinator(context.Background(), clientServerAddr, api.VerifyOptions{InsecureSkipVerify: true}) require.NoError(err) - resp, err := client.Do(req) - require.NoError(err) - - require.Equal(http.StatusOK, resp.StatusCode) - quote, err := io.ReadAll(resp.Body) - resp.Body.Close() - require.NoError(err) - cert := gjson.Get(string(quote), "data.Cert").String() - require.NotEmpty(cert) - // create client with certificates - pool := x509.NewCertPool() - require.True(pool.AppendCertsFromPEM([]byte(cert))) + // create client certificate privk, err := x509.MarshalPKCS8PrivateKey(RecoveryPrivateKey) require.NoError(err) clCert, err := tls.X509KeyPair(AdminCert, pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: privk})) require.NoError(err) - client = http.Client{Transport: &http.Transport{TLSClientConfig: &tls.Config{ - Certificates: []tls.Certificate{clCert}, - RootCAs: pool, - }}} // test with certificate - clientAPIURL.Path = "manifest" - req, err = http.NewRequestWithContext(context.Background(), http.MethodGet, clientAPIURL.String(), http.NoBody) - require.NoError(err) - resp, err = client.Do(req) - require.NoError(err) - - require.Equal(http.StatusOK, resp.StatusCode) - manifest, err := io.ReadAll(resp.Body) - resp.Body.Close() + statusCode, _, err := api.GetStatus(context.Background(), clientServerAddr, cert) require.NoError(err) - assert.JSONEq(`{"status":"success","data":{"ManifestSignatureRootECDSA":null,"ManifestSignature":"","Manifest":null}}`, string(manifest)) + assert.Equal(2, statusCode) t.Log("Setting the Manifest") _, err = f.SetManifest(f.TestManifest) @@ -227,18 +198,12 @@ func TestClientAPI(t *testing.T) { // test reading of secrets t.Log("Requesting a secret from the Coordinator") - clientAPIURL.Path = "secrets" - clientAPIURL.RawQuery = "s=symmetricKeyShared" - req, err = http.NewRequestWithContext(context.Background(), http.MethodGet, clientAPIURL.String(), http.NoBody) - require.NoError(err) - resp, err = client.Do(req) - require.NoError(err) - - require.Equal(http.StatusOK, resp.StatusCode) - secret, err := io.ReadAll(resp.Body) - defer resp.Body.Close() + const secretName = "symmetricKeyShared" + secrets, err := api.SecretGet(context.Background(), clientServerAddr, cert, &clCert, []string{secretName}) require.NoError(err) - assert.Contains(string(secret), `{"status":"success","data":{"symmetricKeyShared":{"Type":"symmetric-key","Size":128,`) + secret := secrets[secretName] + assert.Equal("symmetric-key", secret.Type) + assert.EqualValues(128, secret.Size) } func TestSettingSecrets(t *testing.T) { @@ -260,10 +225,6 @@ func TestSettingSecrets(t *testing.T) { require.NoError(err) clCert, err := tls.X509KeyPair(AdminCert, pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: privk})) require.NoError(err) - client := http.Client{Transport: &http.Transport{TLSClientConfig: &tls.Config{ - Certificates: []tls.Certificate{clCert}, - InsecureSkipVerify: true, - }}} // start server t.Log("Starting a Server-Marble...") @@ -279,12 +240,9 @@ func TestSettingSecrets(t *testing.T) { // test setting a secret t.Log("Setting a custom secret") - clientAPIURL := url.URL{Scheme: "https", Host: clientServerAddr, Path: "secrets"} - req, err := http.NewRequestWithContext(context.Background(), http.MethodPost, clientAPIURL.String(), strings.NewReader(UserSecrets)) - require.NoError(err) - resp, err := client.Do(req) - require.NoError(err) - resp.Body.Close() + var userSecrets map[string]manifest.UserSecret + require.NoError(json.Unmarshal([]byte(UserSecrets), &userSecrets)) + require.NoError(api.SecretSet(context.Background(), clientServerAddr, nil, &clCert, userSecrets)) // start the marble again t.Log("Starting the Client-Marble again, with the secret now set...") @@ -307,7 +265,7 @@ func TestRecoveryRestoreKey(t *testing.T) { // set Manifest t.Log("Setting the Manifest") f.TestManifest.Config.SealMode = sealMode - recoveryResponse, err := f.SetManifest(f.TestManifest) + recoveryData, err := f.SetManifest(f.TestManifest) require.NoError(err, "failed to set Manifest") // start server @@ -319,19 +277,16 @@ func TestRecoveryRestoreKey(t *testing.T) { // Trigger recovery mode cancelCoordinator, cert := f.TriggerRecovery(cfg, cancelCoordinator) - // Decode & Decrypt recovery data from when we set the manifest - key := gjson.GetBytes(recoveryResponse, "data.RecoverySecrets.testRecKey1").String() - recoveryDataEncrypted, err := base64.StdEncoding.DecodeString(key) - require.NoError(err, "Failed to base64 decode recovery data.") - recoveryKey, err := util.DecryptOAEP(RecoveryPrivateKey, recoveryDataEncrypted) - require.NoError(err, "Failed to RSA OAEP decrypt the recovery data.") + // Decrypt recovery data from when we set the manifest + recoveryKey, err := api.DecryptRecoveryData(recoveryData["testRecKey1"], RecoveryPrivateKey) + require.NoError(err, "Failed to decrypt the recovery data.") // Perform recovery require.NoError(f.SetRecover(recoveryKey)) t.Log("Performed recovery, now checking status again...") - statusResponse, err := f.GetStatus() + statusCode, err := f.GetStatus() require.NoError(err) - assert.EqualValues(3, gjson.Get(statusResponse, "data.StatusCode").Int(), "Server is in wrong status after recovery.") + assert.EqualValues(3, statusCode, "Server is in wrong status after recovery.") // Verify if old certificate is still valid f.VerifyCertAfterRecovery(cert, cancelCoordinator, cfg) @@ -414,7 +369,7 @@ func TestManifestUpdate(t *testing.T) { // Set the update manifest t.Log("Setting the Update Manifest") - _, err = f.SetUpdateManifest(f.UpdatedManifest, AdminCert, RecoveryPrivateKey) + err = f.SetUpdateManifest(f.UpdatedManifest, AdminCert, RecoveryPrivateKey) require.NoError(err, "failed to set Update Manifest") // Try to start marbles again, should fail now due to increased minimum SecurityVersion