From 1af845201c7567118b59a687c3f68358b191a5a5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Wei=C3=9Fe?= Date: Fri, 19 Jan 2024 11:51:33 +0100 Subject: [PATCH] Add versioned docs for v1.4 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Daniel Weiße --- .../version-1.4/_media/cert-chain.svg | 3 + .../_media/coordinator_deployment.svg | 37 + .../_media/enc-state-distributed.svg | 3 + .../version-1.4/_media/enc-state-single.svg | 3 + .../version-1.4/_media/marble_deployment.svg | 53 + .../version-1.4/_media/overview.svg | 41 + .../_media/security_architecture.svg | 3 + .../version-1.4/_media/service_mesh.svg | 36 + .../version-1.4/_media/verify_cluster.svg | 37 + .../version-1.4/architecture/concepts.md | 35 + .../version-1.4/architecture/coordinator.md | 27 + .../version-1.4/architecture/marbles.md | 6 + .../version-1.4/architecture/security.md | 111 +++ .../version-1.4/building-marbles/ego.md | 20 + .../version-1.4/building-marbles/gramine.md | 164 ++++ .../version-1.4/building-marbles/occlum.md | 81 ++ .../version-1.4/deployment/kubernetes.md | 212 ++++ .../deployment/platforms/alibaba.md | 71 ++ .../version-1.4/deployment/platforms/azure.md | 29 + .../deployment/platforms/on-prem.md | 90 ++ .../deployment/platforms/platforms.md | 16 + .../version-1.4/deployment/standalone.md | 56 ++ .../version-1.4/features/attestation.md | 51 + .../features/kubernetes-integration.md | 56 ++ .../version-1.4/features/manifest.md | 32 + .../version-1.4/features/recovery.md | 53 + .../version-1.4/features/runtimes.md | 20 + .../features/secrets-management.md | 31 + .../version-1.4/features/transparent-TLS.md | 12 + .../version-1.4/getting-started/examples.md | 30 + .../getting-started/installation.md | 134 +++ .../version-1.4/getting-started/quickstart.md | 174 ++++ docs/versioned_docs/version-1.4/intro.md | 45 + .../version-1.4/reference/cli.md | 919 ++++++++++++++++++ .../version-1.4/reference/coordinator.md | 5 + .../version-1.4/workflows/add-service.md | 167 ++++ .../version-1.4/workflows/define-manifest.md | 485 +++++++++ .../version-1.4/workflows/managing-secrets.md | 92 ++ .../version-1.4/workflows/monitoring.md | 54 + .../workflows/recover-coordinator.md | 136 +++ .../version-1.4/workflows/set-manifest.md | 30 + .../version-1.4/workflows/update-manifest.md | 176 ++++ .../version-1.4/workflows/updates.md | 56 ++ .../version-1.4/workflows/verification.md | 40 + .../version-1.4-sidebars.json | 247 +++++ docs/versions.json | 1 + 46 files changed, 4180 insertions(+) create mode 100644 docs/versioned_docs/version-1.4/_media/cert-chain.svg create mode 100644 docs/versioned_docs/version-1.4/_media/coordinator_deployment.svg create mode 100644 docs/versioned_docs/version-1.4/_media/enc-state-distributed.svg create mode 100644 docs/versioned_docs/version-1.4/_media/enc-state-single.svg create mode 100644 docs/versioned_docs/version-1.4/_media/marble_deployment.svg create mode 100644 docs/versioned_docs/version-1.4/_media/overview.svg create mode 100644 docs/versioned_docs/version-1.4/_media/security_architecture.svg create mode 100644 docs/versioned_docs/version-1.4/_media/service_mesh.svg create mode 100644 docs/versioned_docs/version-1.4/_media/verify_cluster.svg create mode 100644 docs/versioned_docs/version-1.4/architecture/concepts.md create mode 100644 docs/versioned_docs/version-1.4/architecture/coordinator.md create mode 100644 docs/versioned_docs/version-1.4/architecture/marbles.md create mode 100644 docs/versioned_docs/version-1.4/architecture/security.md create mode 100644 docs/versioned_docs/version-1.4/building-marbles/ego.md create mode 100644 docs/versioned_docs/version-1.4/building-marbles/gramine.md create mode 100644 docs/versioned_docs/version-1.4/building-marbles/occlum.md create mode 100644 docs/versioned_docs/version-1.4/deployment/kubernetes.md create mode 100644 docs/versioned_docs/version-1.4/deployment/platforms/alibaba.md create mode 100644 docs/versioned_docs/version-1.4/deployment/platforms/azure.md create mode 100644 docs/versioned_docs/version-1.4/deployment/platforms/on-prem.md create mode 100644 docs/versioned_docs/version-1.4/deployment/platforms/platforms.md create mode 100644 docs/versioned_docs/version-1.4/deployment/standalone.md create mode 100644 docs/versioned_docs/version-1.4/features/attestation.md create mode 100644 docs/versioned_docs/version-1.4/features/kubernetes-integration.md create mode 100644 docs/versioned_docs/version-1.4/features/manifest.md create mode 100644 docs/versioned_docs/version-1.4/features/recovery.md create mode 100644 docs/versioned_docs/version-1.4/features/runtimes.md create mode 100644 docs/versioned_docs/version-1.4/features/secrets-management.md create mode 100644 docs/versioned_docs/version-1.4/features/transparent-TLS.md create mode 100644 docs/versioned_docs/version-1.4/getting-started/examples.md create mode 100644 docs/versioned_docs/version-1.4/getting-started/installation.md create mode 100644 docs/versioned_docs/version-1.4/getting-started/quickstart.md create mode 100644 docs/versioned_docs/version-1.4/intro.md create mode 100644 docs/versioned_docs/version-1.4/reference/cli.md create mode 100644 docs/versioned_docs/version-1.4/reference/coordinator.md create mode 100644 docs/versioned_docs/version-1.4/workflows/add-service.md create mode 100644 docs/versioned_docs/version-1.4/workflows/define-manifest.md create mode 100644 docs/versioned_docs/version-1.4/workflows/managing-secrets.md create mode 100644 docs/versioned_docs/version-1.4/workflows/monitoring.md create mode 100644 docs/versioned_docs/version-1.4/workflows/recover-coordinator.md create mode 100644 docs/versioned_docs/version-1.4/workflows/set-manifest.md create mode 100644 docs/versioned_docs/version-1.4/workflows/update-manifest.md create mode 100644 docs/versioned_docs/version-1.4/workflows/updates.md create mode 100644 docs/versioned_docs/version-1.4/workflows/verification.md create mode 100644 docs/versioned_sidebars/version-1.4-sidebars.json diff --git a/docs/versioned_docs/version-1.4/_media/cert-chain.svg b/docs/versioned_docs/version-1.4/_media/cert-chain.svg new file mode 100644 index 00000000..3f9325d5 --- /dev/null +++ b/docs/versioned_docs/version-1.4/_media/cert-chain.svg @@ -0,0 +1,3 @@ + + +
Leaf Certificates
Leaf Certificates
Intermediate CA
Intermediate CA
Root CA
Root CA
signs
signs
Root Private Key
Root...
signs
signs
signs
signs
Intermediate Private Key
Int...
Marble Private Key
Mar...
Root Certificate
Root Certificate
Intermediate Certificate
Intermediate Cer...
Marble Root Certificate
Marble Root Cert...
Marble Certificate
Marble Certifica...Text is not SVG - cannot display \ No newline at end of file diff --git a/docs/versioned_docs/version-1.4/_media/coordinator_deployment.svg b/docs/versioned_docs/version-1.4/_media/coordinator_deployment.svg new file mode 100644 index 00000000..313d5135 --- /dev/null +++ b/docs/versioned_docs/version-1.4/_media/coordinator_deployment.svg @@ -0,0 +1,37 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/docs/versioned_docs/version-1.4/_media/enc-state-distributed.svg b/docs/versioned_docs/version-1.4/_media/enc-state-distributed.svg new file mode 100644 index 00000000..646659e4 --- /dev/null +++ b/docs/versioned_docs/version-1.4/_media/enc-state-distributed.svg @@ -0,0 +1,3 @@ + + +
Kubernetes Secret
Kubernetes Secret
Encrypted State
Encrypted State
Encrypted DEK
Encrypted DEK
Coordinator State
Coordinator State
Data Encryption Key
Data Encryption K...
Key Encryption Key
Key Encryption Key
Generated by the first Coordinator
Generated by the fir...
Generated by the first Coordinator
Generated by the fir...
Kubernetes ConfigMap
Kubernetes ConfigMap
Node-1: enc(KEK)
Node-1: enc(KEK)
Node-2: enc(KEK)
Node-2: enc(KEK)
Node-X: enc(KEK)
Node-X: enc(KEK)
Node-1 SGX 'Product' Sealing Key
Node-1 SGX 'Product' Sealing Key
Node-2 SGX 'Product' Sealing Key
Node-2 SGX 'Product' Sealing Key
Node-X SGX 'Product' Sealing Key
Node-X SGX 'Product' Sealing Key
...
...
...
...Text is not SVG - cannot display \ No newline at end of file diff --git a/docs/versioned_docs/version-1.4/_media/enc-state-single.svg b/docs/versioned_docs/version-1.4/_media/enc-state-single.svg new file mode 100644 index 00000000..50df40aa --- /dev/null +++ b/docs/versioned_docs/version-1.4/_media/enc-state-single.svg @@ -0,0 +1,3 @@ + + +
Persistent Storage
Persistent Storage
Encrypted State
Encrypted State
Encrypted DEK
Encrypted DEK
Coordinator State
Coordinator State
Data Encryption Key
Data Encryption K...
Key Encryption Key
Key Encryption Key
Generated by the Coordinator
Generated by the Coordi...
SGX 'Product' Sealing Key
SGX 'Product' Sealing K...Text is not SVG - cannot display \ No newline at end of file diff --git a/docs/versioned_docs/version-1.4/_media/marble_deployment.svg b/docs/versioned_docs/version-1.4/_media/marble_deployment.svg new file mode 100644 index 00000000..af2712da --- /dev/null +++ b/docs/versioned_docs/version-1.4/_media/marble_deployment.svg @@ -0,0 +1,53 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/docs/versioned_docs/version-1.4/_media/overview.svg b/docs/versioned_docs/version-1.4/_media/overview.svg new file mode 100644 index 00000000..f36004ff --- /dev/null +++ b/docs/versioned_docs/version-1.4/_media/overview.svg @@ -0,0 +1,41 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/docs/versioned_docs/version-1.4/_media/security_architecture.svg b/docs/versioned_docs/version-1.4/_media/security_architecture.svg new file mode 100644 index 00000000..21970277 --- /dev/null +++ b/docs/versioned_docs/version-1.4/_media/security_architecture.svg @@ -0,0 +1,3 @@ + + +
MarbleRun Deployment
MarbleRun Deployment
aTLS
gRPC
aTLS...
Coordinator
Coordinator
Manifest
Manifest
Packages
Packages
Marbles
Marbles
Secrets
Secrets
Recovery Keys
Recovery Keys
Certificate Authority
Certificate Authority
mTLS
HTTP, gRPC, TCP
mTLS...
Marble A
Marble A
TLS
HTTP, gRPC, TCP
TLS...
aTLS
gRPC
aTLS...
Marble B
Marble B
Marble
Certificate A
Marble...
Root Certificate
Root Certificate
Intermediate Certificate
Intermediate Certifi...
Marble
Certificate B
Marble...
aTLS
HTTP
aTLS...
aTLS
HTTP
aTLS...
Parameters A
Parameters A
 Secrets A
 Secrets A
Parameters B
Parameters B
 Secrets B
 Secrets B
Private Keys
Private Keys
Marble Root Certificate
Marble Root Certific...
Marble Root Certificate
Marble Root Certific...
Marble Root Certificate
Marble Root Certific...
Administrator
Administrator
Application user
Application u...Text is not SVG - cannot display \ No newline at end of file diff --git a/docs/versioned_docs/version-1.4/_media/service_mesh.svg b/docs/versioned_docs/version-1.4/_media/service_mesh.svg new file mode 100644 index 00000000..fd29779c --- /dev/null +++ b/docs/versioned_docs/version-1.4/_media/service_mesh.svg @@ -0,0 +1,36 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/docs/versioned_docs/version-1.4/_media/verify_cluster.svg b/docs/versioned_docs/version-1.4/_media/verify_cluster.svg new file mode 100644 index 00000000..4cffea30 --- /dev/null +++ b/docs/versioned_docs/version-1.4/_media/verify_cluster.svg @@ -0,0 +1,37 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/docs/versioned_docs/version-1.4/architecture/concepts.md b/docs/versioned_docs/version-1.4/architecture/concepts.md new file mode 100644 index 00000000..71635e8e --- /dev/null +++ b/docs/versioned_docs/version-1.4/architecture/concepts.md @@ -0,0 +1,35 @@ +# Concepts + +This article describes the concepts of [confidential computing](#confidential-computing) and [service meshes](#service-meshes) which are key to the [MarbleRun approach](#marblerun-approach). + +## Confidential computing + +Confidential computing protects data in use by performing computations in hardware-based secure enclaves. The most prominent enclave to date is probably [Intel SGX](https://www.intel.de/content/www/de/de/architecture-and-technology/software-guard-extensions.html). +Enclaves prevent unauthorized access or modification of applications and data while in use, thereby increasing the security assurances for organizations that manage sensitive and regulated data. +For information about confidential computing, see the Confidential Computing Consortium [white paper](https://confidentialcomputing.io/white-papers/). + +## Service meshes + +A service mesh is an infrastructure layer for managing, observing, and securing communications in a container-based cluster. In the Kubernetes world, [Istio](https://istio.io), [HashiCorp Consul](https://www.consul.io/), and [Linkerd](https://linkerd.io/) are the most popular general-purpose service meshes. + +When we started looking into the concept of *confidential microservices*, we realized that there are additional challenges and requirements for service meshes in the context of confidential computing. + +* How to make an entire cluster or distributed app verifiable in a meaningful way from the outside? +* How to establish secure connections to a distributed app based on this? +* How to establish secure connections between services within a cluster based on remote attestation? +* How to securely and safely restart and migrate services between nodes? + +## MarbleRun approach + +Most general-purpose service meshes are implemented using so-called *sidecars*. The most prevalent sidecar is probably [Envoy](https://www.envoyproxy.io/). +In essence, sidecars are network proxies that are injected into *pods* running application containers. Sidecars observe, control, and often encrypt the network communication between application containers. Sidecars are often referred to as the data plane, in relation to the so-called control plane. +The control plane manages and configures the sidecars to route traffic, enforce policies, and collect stats. + +Security-wise, conventional service meshes focus on protecting data in transit between application containers. +In contrast, distributed confidential apps require a more comprehensive approach and careful consideration of security implications. + +In summary, MarbleRun takes the following approach. + +* Instead of relying on separate sidecars, MarbleRun injects the data-plane logic directly into the application logic running inside secure enclaves. Through this tight coupling, secure connections always terminate inside secure enclaves. We refer to containers running such enclaves as *Marbles*. +* Before bootstrapping Marbles, MarbleRun verifies their integrity using Intel SGX remote attestation primitives. This way, MarbleRun is able to guarantee that the topology of a distributed confidential app adheres to the cluster's effective [manifest](../features/manifest.md). +* MarbleRun acts as a certificate authority for all Marble-based services and issues one concise remote attestation statement for the entire cluster. This can be used by anyone to verify the integrity of a distributed confidential app. diff --git a/docs/versioned_docs/version-1.4/architecture/coordinator.md b/docs/versioned_docs/version-1.4/architecture/coordinator.md new file mode 100644 index 00000000..bd08260a --- /dev/null +++ b/docs/versioned_docs/version-1.4/architecture/coordinator.md @@ -0,0 +1,27 @@ +# Coordinator + +The Coordinator is MarbleRun's control plane. +Conceptually, it operates as trusted controller in your confidential deployment. +It securely holds the deployment's state, including the [manifest](../features/manifest.md) and the defined [secrets](../features/secrets-management.md). +Based on the manifest, the Coordinator uses [remote attestation](../features/attestation.md) to authenticate the application's enclaves. + + + +## API and Configuration + +It communicates with the Marble's data plane through gRPC and provides an HTTP REST interface on the client-side. +The Coordinator can be configured with several environment variables: + +* `EDG_COORDINATOR_MESH_ADDR`: The listener address for the gRPC server +* `EDG_COORDINATOR_CLIENT_ADDR`: The listener address for the HTTP REST server +* `EDG_COORDINATOR_DNS_NAMES`: The DNS names and IPs for the cluster's root certificate +* `EDG_COORDINATOR_SEAL_DIR`: The file path for storing sealed data + +When you use MarbleRun [with Kubernetes](../deployment/kubernetes.md), you can [scale the Coordinator to multiple instances](../features/recovery.md#distributed-coordinator) to increase availability and reduce the occurrence of events that require [manual recovery](../workflows/recover-coordinator.md). + +The Coordinator clients can be divided into two major groups. + +* The owners/providers/administrators who need to interact with the Coordinator for deploying their confidential application and administrative tasks +* The users/customers who use the Coordinator for remote attestation and establishing trust with the application + +The [Client API](../reference/coordinator.md) serves both use-cases with a compact HTTP REST API. diff --git a/docs/versioned_docs/version-1.4/architecture/marbles.md b/docs/versioned_docs/version-1.4/architecture/marbles.md new file mode 100644 index 00000000..d34f04ec --- /dev/null +++ b/docs/versioned_docs/version-1.4/architecture/marbles.md @@ -0,0 +1,6 @@ +# Marbles + +Marbles are MarbleRun's data planes and run your application code in secure enclaves. +Marbles communicate with the Coordinator via gRPC over TLS. +MarbleRun [supports multiple enclave runtimes](../features/runtimes.md) as the data plane. +See the [Add a Service](../workflows/add-service.md) section on how to build and configure an enclave as Marble. diff --git a/docs/versioned_docs/version-1.4/architecture/security.md b/docs/versioned_docs/version-1.4/architecture/security.md new file mode 100644 index 00000000..214f6fa6 --- /dev/null +++ b/docs/versioned_docs/version-1.4/architecture/security.md @@ -0,0 +1,111 @@ +# Key management and cryptographic primitives + +MarbleRun protects and isolates your deployments and workloads. To that end, cryptography is the foundation that ensures the confidentiality and integrity of all components. +Evaluating the security of MarbleRun requires a precise understanding of the cryptographic primitives and keys used. +The following gives an overview of the architecture and explains the technical details. + +## High-level architecture + +The [Coordinator](coordinator.md) and the [Marbles](marbles.md) run inside SGX Enclaves. See [Intel's documentation](https://www.intel.com/content/www/us/en/developer/tools/software-guard-extensions/overview.html) on the architecture details and cryptographic primitives of SGX. + +MarbleRun uses cryptography for the following tasks. + +* Authentication and authorization +* Establishment of a public key infrastructure (PKI) for MarbleRun +* Encryption of network traffic via mutual TLS between enclaves +* Encrypting persistent state + +The following diagram gives an overview of the architecture and the components. + +![Security architecture](../_media/security_architecture.svg) + +## Authentication and authorization + +MarbleRun uses the SGX [remote attestation](../features/attestation.md) capability to authenticate the Coordinator and the Marble enclaves. +For authorization, the [manifest](../features/manifest.md) defines the Marble's access to secrets and keys after successful attestation. +Furthermore, MarbleRun's [RBAC](../workflows/define-manifest.md#roles) attaches [users'](../workflows/define-manifest.md#users) identities to roles in the manifest. +Each role is associated with a set of operations the user can perform in the deployment. +Users are authenticated by the Coordinator using an RSA or ECDSA public key defined in the manifest. + +## Public key infrastructure and certificate authority + +The Coordinator establishes a public key infrastructure (PKI) for MarbleRun and acts as the certificate authority (CA). +The goal of the PKI is to make authentication of confidential applications based on remote attestation accessible and usable. +The Coordinator provides an [API](../reference/coordinator.md) for retrieving an SGX attestation statement that embeds its *Root CA Certificate* in the user-defined body. +By verifying the statement, clients can verify the certificate's authenticity and, thereby, the MarbleRun CA. +See the [attested TLS](#attested-tls-atls) section for details behind that concept. +All MarbleRun clients and Marbles can then use the attested *Root CA Certificate* for authenticating TLS connections. +This is further illustrated conceptually in the [attestation](../features/attestation.md) section. The following focuses on the cryptographic primitives. +The Coordinator generates a root X.509 certificate and corresponding asymmetric key pair during initialization. +The [Elliptic Curve Digital Signature Algorithm (ECDSA)](https://www.secg.org/sec1-v2.pdf#page=49) is used with curve [P256](https://nvlpubs.nist.gov/nistpubs/FIPS/NIST.FIPS.186-4.pdf#page=111). +The *Root CA Certificate* has no expiry date and lives as long as the MarbleRun deployment. + +Alongside the *Root CA Certificate*, the Coordinator generates an X.509 *Intermediate Certificate* and corresponding asymmetric key pair, again using ECDSA with P256. +The *Intermediate Certificate* is signed by the Coordinator's *Root CA Certificate* and rotated with every manifest update. +When you push an update to the manifest (for example, bump up the *SecurityVersion* of a Marble), the *Intermediate Certificate* will change. +Marble instances of the new version won't authenticate with instances of the old version and vice versa. +Hence, no data flow is happening between different *SecurityVersions* of your application. +However, the *Root CA Certificate* doesn't change. So you can still verify the Coordinator and your application from the outside and ensure it's the same instance you might have interacted with. +Applications interacting with the MarbleRun deployment should use the intermediate certificate as CA to make manifest updates observable: +If the manifest changes, connections to the deployment will fail. +The application user can then review the changes and install the new intermediate certificate to approve them. +If such observability isn't required, the application can use the root certificate as CA. +In that case, the application will continue to work even if the manifest changes. + +The Coordinator creates a second certificate with the same key material as the *Intermediate Certificate* called the *Marble Root Certificate*. +In that sense, they're siblings containing the same public key. +However, while the *Intermediate Certificate* is signed by the *Root Certificate*, the *Marble Root Certificate* is self-signed using its private key. +The goal here is to implement a [cross-signed certificate chain](https://www.ssltrust.com.au/blog/understanding-certificate-cross-signing). +In that way, the Marbles see the *Marble Root Certificate* as a self-signed root certificate. Hence, they're dealing with a terminating certificate chain without knowing about the Coordinator's *Root CA Certificate*. +The "outside world" sees an intermediate certificate signed by the Coordinator's *Root CA Certificate*. +The Coordinator generates a unique leaf *Marble Certificate* and corresponding key pair using ECDSA with P256 for every Marble. +The *Marble Root Certificate* signs the *Marble Certificate*. +The *Marble Certificate* is provisioned to the Marble's enclave via the secure channel established during the [attestation procedure](../features/attestation.md). +Depending on the Marble's runtime, the certificate can be used [manually](../workflows/add-service.md#make-your-service-use-the-provided-tls-credentials) or [automatically](../features/transparent-TLS.md) to establish mutually authenticated TLS connections. + +![PKI Certificate chain](../_media/cert-chain.svg) + + +## Attested TLS (aTLS) + +In a confidential computing environment, attested TLS (aTLS) can establish secure connections between two parties using the remote attestation features of the confidential computing components. +With aTLS, the party to be authenticated binds its TLS certificate to an attestation statement. +For example, it embeds the certificate's public key into the attestation statement. +Instead of relying on a certificate authority, aTLS uses this attestation statement to establish trust in the certificate. +The protocol can be used by clients to verify a server certificate, by a server to verify a client certificate, or for mutual verification (mutual aTLS). + + +## Encryption of state + +The Coordinator holds MarbleRun's state, which consists of the [manifest](../features/manifest.md), the [managed secrets](../features/secrets-management.md), and the [certificates for its CA](../features/attestation.md). +The state is stored encrypted in persistent storage. For this, MarbleRun uses [AES128-GCM](https://www.rfc-editor.org/rfc/rfc5116#section-5.1) and a generated 16-byte data encryption key (DEK). +The DEK is also sealed to persistent storage to recover the state in case of a restart autonomously. +[SGX sealing](https://www.intel.com/content/www/us/en/developer/articles/technical/introduction-to-intel-sgx-sealing.html) is used for that purpose. +The Coordinator encrypts the DEK with a key encryption key (KEK). +MarbleRun uses the SGX sealing key called `Product key` as its KEK, which is bound to its `Product ID` and the enclave's author `MRSIGNER` identity. +In other words, a fresh and benign enclave instance of the same identity can recover that key. +Hence, if the Coordinator is restarted on the same CPU, it can obtain the same KEK from the CPU, decrypt the DEK, and recover its state. + +![Encrypted state single instance](../_media/enc-state-single.svg) + +If the Coordinator is restarted on a different CPU, it won't be able to obtain the same SGX sealing key from the CPU. +To address this, MarbleRun provides a [recovery feature](../features/recovery.md#recovery). +The manifest allows for specifying a designated Recovery Key. The Recovery Key is a RSA public key. Upon startup, the Coordinator encrypts the DEK with this public key and returns it to the user. +In case of a recovery event, the user decrypts the DEK locally and [uploads it to the Coordinator](../workflows/recover-coordinator.md). +The Coordinator will decrypt the state with the DEK and proceed with operations. + +For [multi-party use cases](../features/recovery.md#multi-party-recovery), MarbleRun allows splitting the Recovery Key between parties. +Every recovery party is defined in the manifest with its own public RSA key. +The Coordinator generates a share of the recovery secret for every party and encrypts it with the corresponding RSA key. +During a recovery event, every party will upload their share of the secret, which are all XORed together by the Coordinator to receive the combined key for decrypting the DEK. + + +### Distributed Coordinator + +The [distributed Coordinator](../features/recovery.md#distributed-coordinator) works similarly. However, all Coordinators share the same state stored encrypted in the Kubernetes [Secret](https://kubernetes.io/docs/concepts/configuration/secret/) called *marble-state*. +In contrast to the single instance, the KEK is generated at start-up by the first instance. +The existing Coordinators authenticate every new Coordinator instance via remote attestation, and the KEK is subsequently shared via the secure and attested TLS connection. +Every Coordinator instance uses its own SGX Product (Sealing) Key to seal the KEK into a Kubernetes [ConfigMap](https://kubernetes.io/docs/concepts/configuration/configmap/) structure called "KEK." + + +![Encrypted state distributed](../_media/enc-state-distributed.svg) diff --git a/docs/versioned_docs/version-1.4/building-marbles/ego.md b/docs/versioned_docs/version-1.4/building-marbles/ego.md new file mode 100644 index 00000000..c6eb6341 --- /dev/null +++ b/docs/versioned_docs/version-1.4/building-marbles/ego.md @@ -0,0 +1,20 @@ +# Building a service: EGo +To get your Go service ready for MarbleRun, we provide two solutions. With [Transparent TLS (TTLS)](../features/transparent-TLS.md) you are able to use your existing applications without any code changes. Alternatively, you can adapt and recompile your application to manually handle the TLS credentials. Details are given in the following. + +## TTLS +Simply follow the steps on [adding a service](../workflows/add-service.md). No code changes and no recompiling needed. + +## Manual TLS credentials handling + +If your service already uses TLS and gets the credentials from, e.g., a file, you just need to [adapt the manifest](../workflows/add-service.md#make-your-service-use-the-provided-tls-credentials). Otherwise, you need to make small code changes. + +We provide a convenience package called [`github.com/edgelesssys/ego/marble`](https://pkg.go.dev/github.com/edgelesssys/ego/marble#GetTLSConfig). With it, a service can automatically get and use its MarbleRun TLS credentials. The following gives an example. +```Go + serverCfg, err := marble.GetTLSConfig(false) + if err != nil { + log.Fatalf("Failed to retrieve server TLS config") + } + // use serverCfg, e.g., to create an HTTPS server +``` + +Finally, you need to re-build your service for the enclave environment with `ego-go` and sign it with `ego sign`. Please follow the build instructions for Go provided in our [Go sample](https://github.com/edgelesssys/marblerun/blob/master/samples/helloworld). diff --git a/docs/versioned_docs/version-1.4/building-marbles/gramine.md b/docs/versioned_docs/version-1.4/building-marbles/gramine.md new file mode 100644 index 00000000..91c94232 --- /dev/null +++ b/docs/versioned_docs/version-1.4/building-marbles/gramine.md @@ -0,0 +1,164 @@ +# Building a service: Gramine + +Running a Gramine app with MarbleRun requires some changes to its manifest. These are explained in the following. See also the [hello world example](https://github.com/edgelesssys/marblerun/tree/master/samples/gramine-hello) for a simple introduction, or the [nginx](https://github.com/edgelesssys/marblerun/tree/master/samples/gramine-nginx) and [Redis](https://github.com/edgelesssys/marblerun/tree/master/samples/gramine-redis) examples for more detailed applications. + +## Requirements + +First, get Gramine up and running. Gramine is available [as a Debian package](https://github.com/gramineproject/gramine/releases). Alternatively you can follow either the [Building](https://gramine.readthedocs.io/en/latest/devel/building.html) or [Cloud Deployment](https://gramine.readthedocs.io/en/latest/installation.html) guide to build and install Gramine from source. + +Before running your application, make sure you got the prerequisites for ECDSA remote attestation installed on your system. You can collectively install them with the following command: + +```sh +sudo apt install libsgx-quote-ex-dev +``` + +## Configuration + +### Entrypoint and argv + +We provide the `premain-libos` executable with the [MarbleRun Releases](https://github.com/edgelesssys/marblerun/releases). It will contact the Coordinator, set up the environment, and run the actual application. + +Set the premain executable as [the entry point](https://gramine.readthedocs.io/en/v1.3/manifest-syntax.html#libos-entrypoint) of the Gramine application and place the actual entry point [in argv0](https://gramine.readthedocs.io/en/v1.3/manifest-syntax.html#command-line-arguments): + +```toml +libos.entrypoint = "file:premain-libos" + +# argv0 needs to contain the name of your executable +loader.argv = ["hello"] + +# add the premain to the list of trusted files +sgx.trusted_files = [ + # ... + "file:premain-libos" +] +``` + +After the premain is done running, it will automatically spawn your application. + +### Host environment variables + +By default, environment variables from the host won't be passed to the application. +Gramine allows to [pass through whitelisted environment variables from the host](https://gramine.readthedocs.io/en/v1.3/manifest-syntax.html#environment-variables). +The premain needs access to the following [environment variables for configuration](../workflows/add-service.md#step-3-start-your-service): + +```toml +loader.env.EDG_MARBLE_TYPE = { passthrough = true } +loader.env.EDG_MARBLE_COORDINATOR_ADDR = { passthrough = true } +loader.env.EDG_MARBLE_UUID_FILE = { passthrough = true } +loader.env.EDG_MARBLE_DNS_NAMES = { passthrough = true } +``` + +### UUID file + +The Marble must be able to store its UUID: + +```toml +sgx.allowed_files = [ + # ... + "file:uuid" +] +``` + +### Remote attestation + +The Marble will send an SGX quote to the Coordinator for remote attestation using [DCAP attestation](https://gramine.readthedocs.io/en/v1.3/manifest-syntax.html#attestation-and-quotes): + +```toml +sgx.remote_attestation = "dcap" +``` + +### Enclave size and threads + +The premain process is written in Go. The enclave needs to have enough resources for the Go runtime: + +```toml +sgx.enclave_size = "1024M" +sgx.thread_num = 16 +``` + +If your application has high memory demands, you may need to increase the size even further. + +### Secret files + +A Marble's secrets, e.g. a certificate and private key, can be provisioned as files. You can utilize Gramine's in-memory filesystem [`tmpfs`](https://gramine.readthedocs.io/en/latest/manifest-syntax.html#fs-mount-points), so the secrets will never show up on the host's file system : + +```toml +fs.mounts = [ + # ... + { type = "tmpfs", path = "/secrets" }, + # ... +] +``` + +You can specify the files' content in the MarbleRun manifest: + +```javascript +... + "Parameters": { + "Files": { + "/secrets/server.crt": "{{ pem .Secrets.serverCert.Cert }}", + "/secrets/server.key": "{{ pem .Secrets.serverCert.Private }}" + } + } +... +``` + +Gramine also allows to store files [encrypted on the host's file system](https://gramine.readthedocs.io/en/v1.3/manifest-syntax.html#encrypted-files). + +```toml +fs.mounts = [ + # ... + { type = "encrypted", path = "/secrets", uri = "file:/path/to/local/directory", key_name = "[KEY_NAME]" }, + # ... +] +``` + +Gramine provides access to a pseudo filesystem for [setting the encryption key](https://gramine.readthedocs.io/en/v1.3/attestation.html#low-level-dev-attestation-interface). +MarbleRun can set up your enclave with keys at runtime by specifying them in the MarbleRun manifest: + +```javascript +... + "Parameters": { + "Files": { + "/dev/attestation/[KEY_NAME]": "{{ raw .Secrets.encryptedFilesKey }}" + } + } +... +``` + +You can see how this is done in the [nginx example](https://github.com/edgelesssys/marblerun/tree/master/samples/gramine-nginx). + +## Troubleshooting + +### aesm_service returned error: 30 + +If you receive the following error message on launch: + +```sh +aesm_service returned error: 30 +load_enclave() failed with error -1 +``` + +Make sure you installed the Intel AESM ECDSA plugins on your machine. You can do this by installing the `libsgx-quote-dev` package mentioned in the requirements above. + +If you are running your application in a container, you will need to mount the aesm socket. The socket is located at `/var/run/aesmd/`. + +If you are deploying your application on Kubernetes with the [Intel SGX device plugin](https://intel.github.io/intel-device-plugins-for-kubernetes/cmd/sgx_plugin/README.html) installed, the socket is automatically mounted by setting the `sgx.intel.com/quote-provider: aesmd` annotation for your deployment: + +```yaml +apiVersion: v1 +kind: Pod +metadata: + name: gramine-marble + labels: + marblerun/marbletype: gramine-marble + annotations: + sgx.intel.com/quote-provider: aesmd +spec: + container: + - name: gramine-marble + image: localhost/gramine-marble + resources: + limits: + sgx.intel.com/epc: 10Mi +``` diff --git a/docs/versioned_docs/version-1.4/building-marbles/occlum.md b/docs/versioned_docs/version-1.4/building-marbles/occlum.md new file mode 100644 index 00000000..d35bfe91 --- /dev/null +++ b/docs/versioned_docs/version-1.4/building-marbles/occlum.md @@ -0,0 +1,81 @@ +# Building a service: Occlum +Running an Occlum app with MarbleRun requires some changes to its manifest. + +## Requirements +Set up an environment to create Occlum images. For an easy start, we recommend using either the official [Occlum Docker image](https://hub.docker.com/r/occlum/occlum) or [our provided Dockerfile](https://github.com/edgelesssys/marblerun/blob/master/samples/occlum-hello/Dockerfile). For a working DCAP remote attestation environment, we recommend [our platform deployment guide](../deployment/platforms/platforms.md). + +To build your service, you can start with [Occlum's Introduction](https://github.com/occlum/occlum#introduction) to get your application up and running and then come back here to adapt it for use with MarbleRun. + +## Configuration +### Premain executable +Add our pre-built [premain-libos](https://github.com/edgelesssys/marblerun/releases/latest/download/premain-libos) executable to your Occlum image, e.g., by copying it to `image/bin/premain-libos`. By default, Occlum restricts executable files to the `/bin` directory. If you placed the `premain-libos` binary at a different path, you must adjust this setting accordingly. + +Finally, define the original entry point for your Occlum instance as the first `Argv` parameter for your Marble in MarbleRun's `manifest.json`. See [Defining a manifest](../workflows/define-manifest.md) for more information on how to define the `Argv` parameters. This lets MarbleRun launch your application after it succeeded in authenticating with the Coordinator and provides entrypoint pinning similar to the one offered in `Occlum.json`. + +### Environment variables +The Marble needs to retrieve the MarbleRun-specific configuration parameters via environment variables, as [described under Step 3 in "Adding a service."](../workflows/add-service.md) + +To pass environment variables to the enclave, Occlum requires them to be specified in the `env` section in `Occlum.json`. + +You can provide default (hardcoded) values under `default`, and you may also define them additionally as `untrusted` in case you want to allow changes to the Marble configuration after build time. + +For example, consider this configuration: +```javascript +"env": { + "default": [ + "OCCLUM=yes", + "EDG_MARBLE_COORDINATOR_ADDR=localhost:2001", + "EDG_MARBLE_TYPE=hello", + "EDG_MARBLE_UUID_FILE=uuid", + "EDG_MARBLE_DNS_NAMES=localhost" + ], + "untrusted": [ + "EDG_MARBLE_COORDINATOR_ADDR", + "EDG_MARBLE_TYPE", + "EDG_MARBLE_UUID_FILE", + "EDG_MARBLE_DNS_NAMES" + ] +}, +``` + +This will allow you to embed the expected default values during build time. It also lets the user/host system change them during run time when a non-default Coordinator configuration is used. + +### Resource limits +The premain process is written in Go. The enclave must have enough resources for the Go runtime and additional memory to launch your application. + +We recommend starting with the following values, which should work fine for light-weight to medium memory-demanding applications: +```javascript +"user_space_size": "2048MB", +"default_mmap_size": "900MB" +"max_num_of_threads": 64 +``` + +In case you are running into issues with memory demands, check out the [Resource Configuration Guide](https://github.com/occlum/occlum/blob/master/docs/resource_config_guide.md) provided by the Occlum team to debug and resolve issues related to resource limits. + +## Troubleshooting + +### Error message: `fatal error: failed to reserve page summary memory` + +If you receive this error during the launch of your Occlum image, make sure you allocated enough memory in `Occlum.json` [as described above](#resource-limits). The most important parameters are `user_space_size` and `default_mmap_size`. + +### Error message: `Error returned from the p_sgx_get_quote_config API` + +If Occlum crashes during the quote generation with the following error message: +``` +[get_platform_quote_cert_data ../qe_logic.cpp:346] Error returned from the p_sgx_get_quote_config API. 0xe019 +thread '' panicked at 'assertion failed: `(left == right)` +left: `SGX_QL_SUCCESS`, +right: `SGX_QL_NETWORK_ERROR`: fail to launch QE', src/util/sgx/dcap/quote_generator.rs:22:13 +``` + +You might need to check the DCAP configuration on your system. Note that when using the Docker image, the local Intel DCAP configuration must be correctly set **inside the container.** + +If you use an Azure Confidential Computing machine, you can use our [provided Dockerfile](https://github.com/edgelesssys/marblerun/blob/master/samples/occlum-hello/Dockerfile). It patches the official Occlum image to use the Azure DCAP client, which handles the configuration automatically. + +For other DCAP setups, please consult the documentation of your Intel Provisioning Certificate Caching Service (PCCS) service running locally or remotely. + +### Other issues +If you are running into other issues, Occlum's error logging might help: +```bash +OCCLUM_LOG_LEVEL=error occlum run /bin/premain-libos +``` diff --git a/docs/versioned_docs/version-1.4/deployment/kubernetes.md b/docs/versioned_docs/version-1.4/deployment/kubernetes.md new file mode 100644 index 00000000..3e5a4b04 --- /dev/null +++ b/docs/versioned_docs/version-1.4/deployment/kubernetes.md @@ -0,0 +1,212 @@ +# Kubernetes MarbleRun deployment + +This guide walks you through setting up MarbleRun in your Kubernetes cluster. + +The Kubernetes deployment is managed through the use of a [Helm chart](https://helm.sh/), which can be found in our [source repository](https://github.com/edgelesssys/marblerun/tree/master/charts) and installed via our [Helm repository.](https://helm.edgeless.systems) +The installation consists of a deployment for the Coordinator and an admission controller. +For more details see our section on [Kubernetes Integration](../features/kubernetes-integration.md). + +## Prerequisites + +### SGX device plugin on Kubernetes + +Kubernetes manages hardware resources like Intel SGX through its [device plugin framework](https://kubernetes.io/docs/concepts/extend-kubernetes/compute-storage-net/device-plugins/). +The SGX device plugin can either be deployed manually or as a DaemonSet in the cluster. Different vendors provide open-source device plugins for SGX: + +* [Intel](https://intel.github.io/intel-device-plugins-for-kubernetes/cmd/sgx_plugin/README.html) +* [Azure](https://github.com/Azure/aks-engine/blob/master/docs/topics/sgx.md#deploying-the-sgx-device-plugin) +* [Alibaba Cloud](https://github.com/AliyunContainerService/sgx-device-plugin) + +:::info + +If you are using a confidential computing-enlightened, managed Kubernetes cluster, you will usually already have an SGX device plugin installed. +For example, creating a confidential computing cluster on AKS has a pre-configured SGX device plugin. + +::: + +### Manually deploying an SGX device plugin + +For different reasons, you may want to deploy the device plugin manually. Intel provides [a guide](https://intel.github.io/intel-device-plugins-for-kubernetes/cmd/sgx_plugin/README.html#installation) to install their SGX plugin. +In any case, you will need to adjust your deployments to request the SGX resources provided by the plugin: + +```yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: oe-deployment +spec: + selector: + matchLabels: + app: oe-app + replicas: 1 + template: + metadata: + labels: + app: oe-app + spec: + tolerations: + - key: sgx.intel.com/epc + operator: Exists + effect: NoSchedule + containers: + - name: + image: + command: + resources: + limits: + sgx.intel.com/epc: 10Mi + sgx.intel.com/enclave: 1 + sgx.intel.com/provision: 1 +``` + +Note, that every plugin uses its own way of injecting SGX resources into deployments. Please refer to the documentation for your plugin of choice. This is an example of the Intel plugin. + +MarbleRun supports [automatic injection](../features/kubernetes-integration.md) of those values for a selection of popular plugins: + +* [Intel](https://intel.github.io/intel-device-plugins-for-kubernetes/cmd/sgx_plugin/README.html) using `sgx.intel.com/epc`, `sgx.intel.com/enclave`, and `sgx.intel.com/provision` +* [Azure](https://github.com/Azure/aks-engine/blob/master/docs/topics/sgx.md#deploying-the-sgx-device-plugin) using `kubernetes.azure.com/sgx_epc_mem_in_MiB` +* [Alibaba Cloud](https://github.com/AliyunContainerService/sgx-device-plugin) using `alibabacloud.com/sgx_epc_MiB` +* You can use the `--resource-key` flag, during installation with the CLI, to declare your own SGX resource key for injection + +:::tip + +If you are using a different plugin please let us know, so we can add support! + +::: + +### Out-of-process attestation + +Intel SGX supports two modes for obtaining remote attestation quotes: + +* In-process: The software generating the quote is part of the enclave application +* Out-of-process: The software generating the quote isn't part of the actual enclave application. This requires the Intel SGX Architectural Enclave Service Manager (AESM) to run on the system + +While Marbles built with [Ego](../building-marbles/ego.md) perform in-process attestation, other frameworks, such as [Gramine](../building-marbles/gramine.md), use out-of-process attestation. +If your confidential application uses out-of-process attestation, you will need to expose the AESM device to your container. + +You can follow [the AKS guide](https://docs.microsoft.com/en-us/azure/confidential-computing/confidential-nodes-aks-addon) to make your deployments able to use AESM for quote generation. Note, that in this case, your Kubernetes nodes need the AESM service installed. See the [Intel installation guide](https://download.01.org/intel-sgx/sgx-linux/2.12/docs/Intel_SGX_Installation_Guide_Linux_2.12_Open_Source.pdf) for more information. + +## Option 1: Install with the MarbleRun CLI + +Use MarbleRun's [CLI](../reference/cli.md) that facilitates the administrative tasks. +You can install MarbleRun using the CLI as follows: + +* For a cluster with SGX support: + + ```bash + marblerun install --domain=mycluster.uksouth.cloudapp.azure.com + ``` + +* For a cluster without SGX support: + + ```bash + marblerun install --domain=mycluster.uksouth.cloudapp.azure.com --simulation + ``` + +This command will pull the latest Helm chart from [our repository](https:/helm.edgeless.systems) and manages the installation of said chart. + +By default `--domain` is set to `localhost`. +The domain is used as the CommonName in the Coordinator's TLS certificate. +This certificate is used for the HTTPS communication of the Coordinator's client API. +The HTTPS endpoint is exposed via a [Kubernetes ClusterIP Service](https://kubernetes.io/docs/concepts/services-networking/service/#publishing-services-service-types). +If you plan on exposing the endpoint on a public IP, make sure that the domain set via `--domain` matches the one configured for the public IPs provisioned in your cluster. +On Azure, you can [use a static public IP address with the Azure Kubernetes Service (AKS) load balancer](https://docs.microsoft.com/en-us/azure/aks/static-ip#create-a-static-ip-address). +The client API can be used by users/clients of your application to obtain one concise remote attestation statement for your cluster. + +The Coordinator is now in a pending state, waiting for a manifest. +See the [how to add a service](../workflows/add-service.md) documentation for more information on how to create and set a manifest. + +## Option 2: Install with Helm + +Make sure that you are using the latest release of Helm and have access to the MarbleRun Helm repositories. For upgrade instructions, see the [Helm install docs](https://docs.helm.sh/using_helm/#installing-helm). For more information on configuring and using Helm, see [Install applications with Helm in Azure Kubernetes Service (AKS)](https://docs.microsoft.com/en-us/azure/aks/kubernetes-helm). + +### Adding MarbleRun's Helm repository + +```bash +helm repo add edgeless https://helm.edgeless.systems/stable +helm repo update +``` + +### Installing the chart + +Note that installing MarbleRun with the [marble-injector webhook](../features/kubernetes-integration.md) enabled using Helm requires [cert-manager](https://cert-manager.io/docs/) to be installed in your cluster. +Review the `values.yaml` file of the chart for a full list of available configuration options. + +Update the hostname with your cluster's FQDN. + +* For a cluster with SGX support: + + ```bash + helm install marblerun edgeless/marblerun \ + --create-namespace \ + -n marblerun \ + --set coordinator.hostname=mycluster.uksouth.cloudapp.azure.com \ + --set marbleInjector.start=true \ + --set marbleInjector.useCertManager=true + ``` + +* For a cluster without SGX support: + + ```bash + helm install marblerun edgeless/marblerun \ + --create-namespace \ + -n marblerun \ + --set coordinator.resources=null \ + --set coordinator.simulation=1 \ + --set tolerations=null \ + --set coordinator.hostname=mycluster.uksouth.cloudapp.azure.com \ + --set marbleInjector.start=true \ + --set marbleInjector.useCertManager=true + ``` + +By default `coordinator.hostname` is set to `localhost`. +The domain is used as the CommonName in the Coordinator's TLS certificate. +This certificate is used for the HTTPS communication of the Coordinator's client API. +The HTTPS endpoint is exposed via a [Kubernetes ClusterIP Service](https://kubernetes.io/docs/concepts/services-networking/service/#publishing-services-service-types). +If you plan on exposing the endpoint on a public IP, make sure that the domain set via `--domain` matches the one configured for the public IPs provisioned in your cluster. +On Azure, you can [use a static public IP address with the Azure Kubernetes Service (AKS) load balancer](https://docs.microsoft.com/en-us/azure/aks/static-ip#create-a-static-ip-address). +The client API can be used by users/clients of your application to obtain one concise remote attestation statement for your cluster. + +The Coordinator is now in a pending state, waiting for a manifest. +See the [how to add a service](../workflows/add-service.md) documentation for more information on how to create and set a manifest. + +## (Optional) Exposing the client API + +The Coordinator creates a [`ClusterIP`](https://kubernetes.io/docs/concepts/services-networking/service/#publishing-services-service-types) service called `coordinator-client-api` exposing the client API on the default port 4433. +Depending on your deployment type you may want to deploy an Ingress Gateway forwarding the traffic or create a [`LoadBalancer`](https://kubernetes.io/docs/concepts/services-networking/service/#loadbalancer) service to expose the endpoint on a public IP. + +You can also use `kubectl` to forward the port to your local system: + +```bash +kubectl -n marblerun port-forward svc/coordinator-client-api 4433:4433 --address localhost +``` + +### Ingress/Gateway configuration + +If you're using an ingress-controller or gateway for managing access to the `coordinator-client-api` service, make sure you're enabling SNI for your TLS connections. + +* For the nginx ingress controller add the [`nginx.ingress.kubernetes.io/ssl-passthrough`](https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/annotations/#ssl-passthrough) annotation. +* For Istio Gateways set the [tls-mode PASSTHROUGH](https://istio.io/latest/docs/tasks/traffic-management/ingress/ingress-sni-passthrough/#configure-an-ingress-gateway) + +## DCAP configuration + +By default, the Coordinator's quote provider is configured to generate its quote using the Azure PCCS. +If you're running on Azure, no additional steps are required. +Otherwise, set the [necessary configuration](https://github.com/intel/SGXDataCenterAttestationPrimitives/blob/master/QuoteGeneration/qpl/README.md#configuration) during installation: + +* Using the CLI + + ```bash + marblerun install --dcap-pccs-url --dcap-secure-cert + ``` + +* Using Helm + + ```bash + helm install marblerun edgeless/marblerun \ + --create-namespace \ + -n marblerun \ + --set coordinator.hostname=mycluster.uksouth.cloudapp.azure.com \ + --set dcap.pccsUrl= \ + --set dcap.useSecureCert= + ``` diff --git a/docs/versioned_docs/version-1.4/deployment/platforms/alibaba.md b/docs/versioned_docs/version-1.4/deployment/platforms/alibaba.md new file mode 100644 index 00000000..04a3dbf3 --- /dev/null +++ b/docs/versioned_docs/version-1.4/deployment/platforms/alibaba.md @@ -0,0 +1,71 @@ +# Alibaba Cloud MarbleRun deployment + +## Alibaba Cloud Container Service for Kubernetes (ACK) + +Alibaba Cloud Container Service for Kubernetes (ACK) offers a popular deployment technique relying on Alibaba's cloud resources. +[ACK hosts Kubernetes pods in SGX-capable Alibaba VMs](https://www.alibabacloud.com/help/en/ack/ack-managed-and-ack-dedicated/user-guide/tee-based-confidential-computing) and exposes the underlying SGX hardware. + +### Prerequisites + +* Follow the instructions on the [ACK Confidential Computing Quick Start guide](https://www.alibabacloud.com/help/en/ack/ack-managed-and-ack-dedicated/user-guide/create-an-ack-managed-cluster-for-confidential-computing) to provision an ACK cluster with Intel SGX-enabled worker nodes. + +### Deploy MarbleRun + +See the [Kubernetes guide](../kubernetes.md) on installing MarbleRun in your ACK cluster. + +## Alibaba Cloud Elastic Compute Service + +With 7th-generation [security-enhanced ECS instances](https://www.alibabacloud.com/help/en/ecs/user-guide/overview-25), users can use Intel SGX on Alibaba Cloud. +You can follow the guide for creating a [g7t, c7t, or r7t](https://www.alibabacloud.com/help/en/ecs/user-guide/create-a-security-enhanced-instance) instance. + +The description below uses a VM running Ubuntu 18.04. + +### Prerequisites + +1. Install Intel DCAP Quote Provider Library + + Add the Intel SGX APT repository: + + ```bash + wget -qO - https://download.01.org/intel-sgx/sgx_repo/ubuntu/intel-sgx-deb.key | sudo apt-key add - + echo 'deb [arch=amd64] https://download.01.org/intel-sgx/sgx_repo/ubuntu bionic main' | sudo tee /etc/apt/sources.list.d/intel-sgx.list + ``` + + Download and install the QPL: + + ```bash + sudo apt update + sudo apt install libsgx-dcap-default-qpl + ``` + +1. Set configuration for Alibaba Cloud SGX remote attestation service + + Alibaba Cloud provides a PCCS for remote attestation deployed per region. For optimal stability, accessing the service in your instance's region is recommended. + The configuration is set in `/etc/sgx_default_qcnl.conf`. + + * If your instance is assigned a public IP address, change the configuration to the following, where `[Region-ID]` is the ID of your instance's region: + + ``` + PCCS_URL=https://sgx-dcap-server.[Region-ID].aliyuncs.com/sgx/certification/v3/ + USE_SECURE_CERT=TRUE + ``` + + * If your instance is in a virtual private cloud and has only internal IP addresses, change the configuration to the following, where `[Region-ID]` is the ID of your instance's region: + + ``` + PCCS_URL=https://sgx-dcap-server-vpc.[Region-ID].aliyuncs.com/sgx/certification/v3/ + USE_SECURE_CERT=TRUE + ``` + + :::note + + Currently, the Alibaba Cloud SGX remote attestation service is only supported within [mainland China regions, Singapore, and Indonesia](https://www.alibabacloud.com/help/en/elastic-compute-service/latest/build-an-sgx-encrypted-computing-environment) + + ::: + +1. [Update and install EGo](https://github.com/edgelesssys/ego#install) + +### Deploy MarbleRun + +You can run MarbleRun standalone on your Alibaba Cloud ECS VM. See the [standalone guide](../standalone.md). +Alternatively, you can install a Kubernetes cluster. Probably the simplest option would be [minikube](https://minikube.sigs.k8s.io/docs/start/). See the [Kubernetes guide](../kubernetes.md) on how to install MarbleRun in minikube. diff --git a/docs/versioned_docs/version-1.4/deployment/platforms/azure.md b/docs/versioned_docs/version-1.4/deployment/platforms/azure.md new file mode 100644 index 00000000..dd8984ae --- /dev/null +++ b/docs/versioned_docs/version-1.4/deployment/platforms/azure.md @@ -0,0 +1,29 @@ +# Azure MarbleRun deployment + +## Azure Kubernetes Services (AKS) + +Azure Kubernetes Service (AKS) offers a popular deployment technique relying on +Azure's cloud resources. AKS hosts Kubernetes pods in SGX-capabale Azure VMs and exposes the underlying SGX hardware. + +### Prerequisites + +* Follow the instructions on the [AKS Confidential Computing Quick Start guide](https://docs.microsoft.com/en-us/azure/confidential-computing/confidential-enclave-nodes-aks-get-started) to provision an AKS cluster with Intel SGX enabled worker nodes. + +### Deploy MarbleRun + +See our [Kubernetes guide](../kubernetes.md) on how to install MarbleRun in your AKS cluster. + +## Azure confidential computing VMs + +[Azure confidential computing services](https://learn.microsoft.com/en-us/azure/confidential-computing/virtual-machine-solutions-sgx) provide access to VMs with Intel SGX enabled. +You can follow their [quickstart](https://learn.microsoft.com/en-us/azure/confidential-computing/quick-create-portal) to create a [DCsv2](https://docs.microsoft.com/en-us/azure/virtual-machines/dcv2-series) (Coffee Lake) or [DCsv3](https://learn.microsoft.com/en-us/azure/virtual-machines/dcv3-series) (Ice Lake) VM. + +### Prerequisites + +* [Update and install EGo](https://github.com/edgelesssys/ego#install) +* [Update and install the Azure DCAP client](https://learn.microsoft.com/en-us/azure/confidential-computing/quick-create-portal#install-azure-dcap-client) + +### Deploy MarbleRun + +You can run MarbleRun standalone on your Azure DCsv2/3 VM, see our [standalone guide](../standalone.md). +Alternatively, you can install a Kubernetes cluster, probably the simplest option would be [minikube](https://minikube.sigs.k8s.io/docs/start/), see our [Kubernetes guide](../kubernetes.md) on how to install MarbleRun in minikube. diff --git a/docs/versioned_docs/version-1.4/deployment/platforms/on-prem.md b/docs/versioned_docs/version-1.4/deployment/platforms/on-prem.md new file mode 100644 index 00000000..15d4c24c --- /dev/null +++ b/docs/versioned_docs/version-1.4/deployment/platforms/on-prem.md @@ -0,0 +1,90 @@ +# On-premises MarbleRun deployment + +This guide walks you through setting up MarbleRun for your on-premises deployment. + +## Prerequisites + +### Hardware and firmware + +#### CPU + +To deploy MarbleRun with Intel SGX, the machine or VM has to support Intel SGX. +Particularly, MarbleRun requires support for the SGX Data Center Attestation Primitives (DCAP). +You can verify [if your CPU supports DCAP](https://www.intel.com/content/www/us/en/support/articles/000057420/software/intel-security-products.html). + +#### BIOS + +BIOS support is required for Intel SGX to provide the capability to enable and configure the Intel SGX feature in the system. +Currently, most of the SGX-capable systems have SGX disabled by default in the BIOS. This default setting might change, but for now, you need to manually enable it if it's not already enabled. + +#### Updates + +As with any modern technology, Intel SGX has been affected by security vulnerabilities. Intel addresses these vulnerabilities by updating the microcode of CPUs, changing the hardware of new CPUs, and updating the system software. Each microcode update that patches an SGX vulnerability requires a BIOS update. During remote attestation, it's checked that the microcode of the CPU, which is deployed by the BIOS, is up to date. The microcode and platform enclaves are commonly called the platform `Trusted Computing Base (TCB)`. + +If your BIOS/firmware is outdated, you will see errors as `Platform TCB (2) is not up-to-date (oe_result_t=OE_TCB_LEVEL_INVALID)` during remote attestation procedures. + +#### Hypervisor + +If you are using VMs for your MarbleRun deployment, you need to make sure your hypervisor has SGX enabled. +Most of the popular hypervisors support SGX: + +* [QEMU/KVM](https://www.intel.com/content/www/us/en/developer/articles/technical/virtualizing-intel-software-guard-extensions-with-kvm-and-qemu.html) +* [XEN](https://wiki.xenproject.org/wiki/Xen_and_Intel_Hardware-Assisted_Virtualization_Security) +* Hyper-V: Hyper-V will only expose SGX to Gen 2 VMs +* [VMWare vSphere](https://blogs.vmware.com/vsphere/2020/04/vsphere-7-vsgx-secure-enclaves.html) +* [ACRN](https://projectacrn.github.io/2.7/tutorials/sgx_virtualization.html) + +#### Driver + +You need to install the [DCAP SGX Driver](https://download.01.org/intel-sgx/sgx-dcap/1.11/linux/docs/Intel_SGX_SW_Installation_Guide_for_Linux.pdf). +Azure provides instructions on [how to install this driver](https://docs.microsoft.com/en-us/azure/confidential-computing/quick-create-portal#2-install-the-intel-sgx-dcap-driver) that you can use for your on-premises machines. + +### SGX Data Center Attestation Primitives (DCAP) + +DCAP is the new attestation mechanism for SGX, [replacing EPID](https://www.intel.com/content/www/us/en/developer/articles/technical/an-update-on-3rd-party-attestation.html). +You can find an overview of DCAP in the [official Intel docs](https://download.01.org/intel-sgx/sgx-dcap/1.11/linux/docs/DCAP_ECDSA_Orientation.pdf). +MarbleRun only supports DCAP and requires DCAP libraries installed and configured on your system. + +From the perspective of MarbleRun and your workloads, DCAP is accessed with a [Quote Generation Library (QGL)](https://github.com/intel/SGXDataCenterAttestationPrimitives/blob/master/QuoteGeneration/README.md) and a [Quote Verification Library (QVL)](https://github.com/intel/SGXDataCenterAttestationPrimitives/blob/master/QuoteVerification/README.md) for generating and verifying quotes respectively. +The QGL and QVL libraries need to be configured to talk to a [Provisioning Certificate Caching Service (PCCS)](https://download.01.org/intel-sgx/sgx-dcap/1.11/linux/docs/DCAP_ECDSA_Orientation.pdf). +You currently have two options regarding PCCS for your on-premises machines and clusters: + +1. Use a public PCCS service by configuring your QGL and QVL to point to the public endpoints. Currently, Azure and Alibaba Cloud provide such a service but require using infrastructure by these providers to make full use of the service. + +1. Run your own PCCS and expose it to your machine or cluster. See [Intel's demo reference implementation](https://github.com/intel/SGXDataCenterAttestationPrimitives/blob/master/QuoteGeneration/pccs/README.md) and [design guide](https://download.01.org/intel-sgx/latest/dcap-latest/linux/docs/SGX_DCAP_Caching_Service_Design_Guide.pdf) for more information. + + Follow these steps to set up your machines for your PCCS: + + * Install the [DCAP client libraries](https://download.01.org/intel-sgx/sgx-dcap/1.11/linux/docs/Intel_SGX_SW_Installation_Guide_for_Linux.pdf) + * Install a [configuration that points to your PCCS](https://github.com/intel/SGXDataCenterAttestationPrimitives/blob/master/QuoteGeneration/qpl/README.md#configuration) + + The PCCS is a cache, so you need to make sure it stays up to date. In case your cache is outdated, you might see error messages as: + + ```bash + coordinator-enclave.signed:mbedtls_x509_crt_verify failed with The CRL is expired (flags=0x20) (oe_result_t=OE_VERIFY_CRL_EXPIRED) + ``` + + You can inspect the Intel Root CA CRL of your PCCS: + + ```bash + curl --insecure --request GET --url https://:/sgx/certification/v3/rootcacrl > rootca.crl + openssl crl -inform DER -text -noout -in rootca.crl + ``` + + You can refresh all SGX collaterals for your PCCS: + + ```bash + curl --insecure --request GET -H "admin-token: " --url https://:/sgx/certification/v3/refresh + ``` + + If refreshing CRL fails, you can manually delete the `pckcache.db` database (default location `/opt/intel/sgx-dcap-pccs/pckcache.db`) and restart your PCCS. + +The docker image for the [MarbleRun Coordinator](https://github.com/edgelesssys/marblerun/pkgs/container/marblerun%2Fcoordinator) comes with the default quote provider library by Intel. +Mount the desired configuration to `/etc/sgx_default_qcnl.conf`. + +## Deploy MarbleRun + +You have made sure your hardware supports SGX, updated all firmware, installed the SGX driver, and configured DCAP on all your machines and VMs? +Great! Now it's time to install MarbleRun and get going. + +You can either [use MarbleRun in standalone mode](../standalone.md) or [install it in your Kubernetes cluster](../kubernetes.md). diff --git a/docs/versioned_docs/version-1.4/deployment/platforms/platforms.md b/docs/versioned_docs/version-1.4/deployment/platforms/platforms.md new file mode 100644 index 00000000..4e74677d --- /dev/null +++ b/docs/versioned_docs/version-1.4/deployment/platforms/platforms.md @@ -0,0 +1,16 @@ +# Supported platforms + +MarbleRun supports all platforms where Intel SGX and the [Data Center Attestation Primitives (DCAP)](https://download.01.org/intel-sgx/latest/dcap-latest/linux/docs/DCAP_ECDSA_Orientation.pdf) are available. + +The following gives you an updated list of supported platforms: + + +| Platform | Service | DCAP | +|---------------|------------|-------------------| +| Azure | VMs | Managed | +| | AKS | Managed | +| Alibaba Cloud | VMs | Managed | +| | ACK | Managed | +| Equinix | Bare-Metal | Customer-operated | +| IBM Cloud | Bare-Metal | Customer-operated | +| On-premises | * | Customer-operated | diff --git a/docs/versioned_docs/version-1.4/deployment/standalone.md b/docs/versioned_docs/version-1.4/deployment/standalone.md new file mode 100644 index 00000000..f3677ac4 --- /dev/null +++ b/docs/versioned_docs/version-1.4/deployment/standalone.md @@ -0,0 +1,56 @@ +# Standalone MarbleRun deployment + +This guide walks you through deploying MarbleRun standalone. + +## Prerequisites + +* [EGo](https://github.com/edgelesssys/ego#install) is installed. + +## Setup the Coordinator control plane + +You can download the [latest release](https://github.com/edgelesssys/marblerun/releases/latest/download/coordinator-enclave.signed) of our Coordinator control plane on GitHub. +If you prefer to build from source please refer to our [build guide](https://github.com/edgelesssys/marblerun/blob/master/BUILD.md). + +You can then run the Coordinator's enclave: + +```bash +erthost build/coordinator-enclave.signed +``` + +Per default, the Coordinator starts with the following default values. You can set your desired configuration by setting the environment variables. + +| Setting | Default Value | Environment Variable | +| --- | --- | --- | +| the listener address for the gRPC server | localhost:2001 | EDG_COORDINATOR_MESH_ADDR | +| the listener address for the HTTP server | localhost: 4433 | EDG_COORDINATOR_CLIENT_ADDR | +| the DNS names and IPs for the cluster’s root certificate | localhost | EDG_COORDINATOR_DNS_NAMES | +| the file path for storing sealed data | $PWD/marblerun-coordinator-data | EDG_COORDINATOR_SEAL_DIR | + +:::tip + +The Coordinator's state is sealed to `$PWD/marblerun-coordinator-data/sealed_data`. If you want a fresh restart remove this file first: `rm $PWD/marblerun-coordinator-data/sealed_data`. + +::: + +The Coordinator is now in a pending state, waiting for a manifest. +See the [how to add a service](../workflows/add-service.md) documentation for more information on how to create and set a manifest. + +### Run your workloads + +You first need to build your workloads together with our Marble data plane. +See our guides for building [EGo](../building-marbles/ego.md), [Gramine](../building-marbles/gramine.md), and [Occlum](../building-marbles/occlum.md) workloads. + +You can then run your Marble as follows: + +```bash +EDG_MARBLE_TYPE= erthost +``` + +Per default, a Marble starts with the following default values. You can set your desired configuration by setting the environment variables. + +| Setting | Default Value | Environment Variable | +| --- | --- | --- | +| network address of the Coordinator’s API for Marbles | `localhost:2001` | EDG_MARBLE_COORDINATOR_ADDR | +| reference on one entry from your manifest’s `Marbles` section | - (this needs to be set every time) | EDG_MARBLE_TYPE | +| local file path where the Marble stores its UUID | `$PWD/uuid` | EDG_MARBLE_UUID_FILE | +| DNS names and IPs the Coordinator will issue the Marble’s certificate for | `$EDG_MARBLE_TYPE` | EDG_MARBLE_DNS_NAMES | diff --git a/docs/versioned_docs/version-1.4/features/attestation.md b/docs/versioned_docs/version-1.4/features/attestation.md new file mode 100644 index 00000000..c5a853aa --- /dev/null +++ b/docs/versioned_docs/version-1.4/features/attestation.md @@ -0,0 +1,51 @@ +# Attestation + +Hardware-rooted *remote attestation* is a key ingredient for distributed confidential apps. MarbleRun relies on the [*Data Center Attestation Primitives* (DCAP)](https://download.01.org/intel-sgx/sgx-dcap/1.11/linux/docs/DCAP_ECDSA_Orientation.pdf) of the latest SGX-enabled Intel Xeon processors. +At the time of writing, only Microsoft Azure had a public DCAP service deployed in their data centers. Hence, our demos are mainly tested and deployed on Azure Kubernetes Service (AKS). +However, MarbleRun works with any DCAP service complying with the SGX specification. You can read more about setting up your own DCAP infrastructure [in the Intel SGX development articles](https://www.intel.com/content/www/us/en/developer/articles/guide/intel-software-guard-extensions-data-center-attestation-primitives-quick-install-guide.html). + +## Coordinator deployment + +Initially, the MarbleRun Coordinator is deployed to the cluster. +The Coordinator generates an X.509 certificate chain with a root and intermediate certificate authority (CA). +It generates a remote attestation quote that contains the hash of the root certificate (as `report_data`) as well as a hardware measurement of its enclave (`MRENCLAVE` of the report). + +The admin receives the quote and verifies the Coordinator's integrity via the measurement. Next, they verify that the quote contains the root certificate’s hash. This ensures that the certificate was indeed generated by this Coordinator and wasn't manipulated during transport. + +After the successful verification of the Coordinator, the manifest can be uploaded to the cluster. +The [manifest](../workflows/define-manifest.md) describes which Marbles (services) are allowed to join the confidential mesh. The manifest contains their enclaves’ build-time measurements. + +![Coordinator deployment](../_media/coordinator_deployment.svg) + +## Marble deployment and attestation + +The Coordinator enforces the manifest and only admits the desired Marbles to the cluster. +Each Marble registers itself with a quote via gRPC to the Coordinator. This quote is tied to the TLS session of the gRPC call. It contains the Marble's measurements. The Coordinator compares them to the manifest to ensure the Marble's identity and integrity. +The [Marble's TLS credentials](../features/secrets-management.md) are generated by the Coordinator and signed with the intermediate certificate. +The Coordinator's certificate chain and the freshly generated Marble certificate are then delivered to the Marble. + +![Marble deployment](../_media/marble_deployment.svg) + +## Confidential service mesh + +The Marble and the Coordinator can now communicate securely and authenticated over TLS (with client authentication), using the Coordinator's CA as the root of trust. +In the same way, Marbles can communicate via mutual TLS with other Marbles, because the X.509 certificates of both Marbles were signed by the same intermediate certificate. + +If a client wants to connect to a Marble, the Coordinator acts as a trusted CA to establish a TLS connection and verify the Marble's certificate. + +![service mesh](../_media/service_mesh.svg) + +When the Coordinator receives an update to the manifest, e.g., a Marble is updated to a newer security version, a new intermediate certificate is generated. The communication between Marbles relies on the intermediate certificate as the root of trust. This ensures that Marbles which were deployed before the update can't communicate with Marbles that have the newer security version. + +## Cluster attestation + +The Coordinator issues one concise remote attestation statement for your whole distributed app. +The Coordinator's build-time measurement is distributed to the relying party (this must be done by the admin or operator). +The relying party requests the Coordinator's quote and certificate chain. +The quote contains the hash of the Coordinator's root CA certificate, which is verified against the received certificate chain. +The quote also contains the Coordinator's measurement, which is verified against the build-time measurement. + +The relying party then requests the manifest from the Coordinator and ensures it contains the expected Marbles and their expected measurements. +The steps required on the client side are described in the [verification hands-on](../workflows/verification.md). + +![cluster attestation](../_media/verify_cluster.svg) diff --git a/docs/versioned_docs/version-1.4/features/kubernetes-integration.md b/docs/versioned_docs/version-1.4/features/kubernetes-integration.md new file mode 100644 index 00000000..4a52db4a --- /dev/null +++ b/docs/versioned_docs/version-1.4/features/kubernetes-integration.md @@ -0,0 +1,56 @@ +# Kubernetes integration + +MarbleRun provides its data-plane configuration through Kubernetes resource definitions. Like regular service meshes, MarbleRun uses Kubernetes' [admission controllers](https://kubernetes.io/docs/reference/access-authn-authz/admission-controllers/#mutatingadmissionwebhook). + +MarbleRun optionally injects [tolerations](https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/) and [resources](https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/) for its SGX device plugin. See the [Kubernetes deployment](../deployment/kubernetes.md#sgx-device-plugin-on-kubernetes) section for more information. + +You can enable auto-injection of the data-plane configuration using Pod labels. + +## The `marbletype` label +In MarbleRun, Marbles (i.e., secure enclaves) are defined in the [manifest](../workflows/define-manifest.md). You need to reference Marbles in your Kubernetes resource description as follows using the `marblerun/marbletype` label: + +```javascript +{ + "Marbles": { + "voting-svc": { + // ... +``` + +```yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: voting + namespace: emojivoto + labels: + app.kubernetes.io/name: voting + app.kubernetes.io/part-of: emojivoto + app.kubernetes.io/version: v1 + marblerun/marbletype: voting-svc +``` + +The label is used to map Kubernetes Pods to MarbleRun Marbles. +When you deploy your application with the `marblerun/marbletype` label, the Pod's creation is intercepted by MarbleRun. +It will then inject environment variables and SGX resources into the Pod containers based on the label's value. +The Pod's injection is skipped if the `marblerun/marbletype` label is missing. + +## The `marblecontainer` label +By default, MarbleRun will inject environment variables and resource requests into all containers of the Pod. +You can use the `marblerun/marblecontainer=` label to limit injection to the specified container. +This is useful if your configuration uses multiple containers in the same Pod, e.g., a sidecar proxy, and you wish to prevent non-enclave containers from taking up resources. + +## The `resource-injection` label + +To prevent MarbleRun from injecting SGX resource requests, you can set the label `marblerun/resource-injection=disabled`. +Use this if you want to set your own SGX resource requests or if you need to start a Marble in simulation mode without any SGX resources. + +## Injected environment variables + +The webhook will inject the following environment variables into each container of a pod: + +* `EDG_MARBLE_TYPE`: The value of the `marblerun/marbletype` label +* `EDG_MARBLE_COORDINATOR_ADDR`: The address of the MarbleRun Coordinator running on the cluster +* `EDG_MARBLE_DNS_NAMES`: DNS names of the pod are derived from `marbletype` and namespace: `marbletype, marbletype.namespace, marbletype.namespace.svc.cluster.local` +* `EDG_MARBLE_UUID_FILE`: The mounted UUID of the Marble + +If an environment variable is already set before the webhook handles the creation request, the variable won't be overwritten, and the custom value is used instead. diff --git a/docs/versioned_docs/version-1.4/features/manifest.md b/docs/versioned_docs/version-1.4/features/manifest.md new file mode 100644 index 00000000..7c525d42 --- /dev/null +++ b/docs/versioned_docs/version-1.4/features/manifest.md @@ -0,0 +1,32 @@ +# Manifest and deployment updates + +The manifest is a JSON document that defines the essential properties of your deployment: allowed software packages, secrets, access control, update policy, etc. +Particularly, the manifest contains all information required by the [Coordinator](../architecture/coordinator.md) to verify the confidentiality and integrity of a newly spawned [Marble](../architecture/marbles.md). +On successful verification, the Coordinator provisions the Marble with configuration and secrets. + +As a cluster owner, you can define in the manifest how rigid you want your deployment to be. + +## Immutable deployment + +In an immutable deployment, you initialize MarbleRun with a permanent manifest that won't allow any changes later on. +Clients of such a deployment don't need to trust any other party because they can audit the manifest and verify its enforcement via [remote attestation](attestation.md). +Choose this approach for deployments with a limited lifetime. + +## Deployment with updatable packages + +The manifest allows to [permit a user to update existing packages of a deployment](../workflows/define-manifest.md#roles). +Clients of such a deployment need to trust the vendors of these packages to provide legitimate software updates. +Choose this approach for deployments with a well-defined scope, but a possibly longer lifetime. + +## Fully updatable deployment + + + +The manifest allows to [permit a user to update the full manifest](../workflows/define-manifest.md#roles). +With such a deployment, this user usually needs to be a trusted party. +Choose this approach for deployments that require full flexibility. + +### Multi-party update + +Depending on the use case, it may not be acceptable that a single user can update the full manifest. +MarbleRun supports [defining a group of users](../workflows/define-manifest.md#roles) that must acknowledge the newly uploaded manifest before it's applied. diff --git a/docs/versioned_docs/version-1.4/features/recovery.md b/docs/versioned_docs/version-1.4/features/recovery.md new file mode 100644 index 00000000..f4a096cc --- /dev/null +++ b/docs/versioned_docs/version-1.4/features/recovery.md @@ -0,0 +1,53 @@ +# State and recovery + +Persistent storage for confidential applications in the cloud requires attention. +If an application should be able to restart without manual intervention, it needs a way to automatically and securely obtain a secret to decrypt its state. + +The SGX programming model considers a single, local application running on a specific CPU. +The application can use the SGX seal key as its root secret. +However, this binds the application and its state to the physical machine because seal keys are unique to a single CPU. +In sum, the usual SGX programming model isn't suited for virtual environments or distributed applications. + +With MarbleRun, the Coordinator [manages the Marbles' secrets](../features/secrets-management.md), and Marbles obtain them securely on start. +Thus, Marbles can be distributed and rescheduled on arbitrary machines. +This narrows the challenge of persistent storage down to the Coordinator itself. + +### Single Coordinator + +The straightforward way to run MarbleRun is with a single Coordinator. +In this case, the state is encrypted with the SGX seal key and stored on disk. +When pinned to a single host, the Coordinator can unseal its state automatically. +However, a [manual step](#recovery) is required to recover the Coordinator's state when the Coordinator is moved to another physical host. + +### Distributed Coordinator + + + +When you use MarbleRun [with Kubernetes](../deployment/kubernetes.md), you can scale the Coordinator to multiple instances. +The instances share a common state, encrypted and stored as a Kubernetes secret. +The encryption key is securely distributed among the Coordinator instances via attested TLS. +Additionally, each Coordinator encrypts the encryption key with its SGX seal key and stores it in a ConfigMap. + +In this mode of operation, manual recovery is only required when + +* all Coordinator instances are stopped at the same time, and +* all new instances are scheduled on new physical hosts. +In other words, if at least one instance is scheduled on a host where a previous instance was running, the state can be recovered automatically. + +## Recovery + +The manifest allows for specifying a designated *Recovery Key*. The Recovery Key is a public RSA key. Upon startup, the Coordinator encrypts its symmetric state-encryption key with this public key. The holder of the corresponding private key can recover the Coordinator, as is described [in the recovery workflow](../workflows/recover-coordinator.md). + +:::caution + +The Recovery Key's owner can access the Coordinator's raw state. + +::: + +### Multi-party recovery + + + +Depending on the use case, it may not be acceptable that the owner has full control over the cluster. +MarbleRun supports splitting the Recovery Key between parties. +Only if all parties agree can they recover a cluster or access the raw state. diff --git a/docs/versioned_docs/version-1.4/features/runtimes.md b/docs/versioned_docs/version-1.4/features/runtimes.md new file mode 100644 index 00000000..1fe7b0ca --- /dev/null +++ b/docs/versioned_docs/version-1.4/features/runtimes.md @@ -0,0 +1,20 @@ +# Supported runtimes + +MarbleRun strives to be runtime-agnostic. Currently, supported runtimes are described below. More will follow in the future. + +## EGo +[EGo](https://github.com/edgelesssys/ego) is the preferred way for writing confidential Go applications from scratch as well as porting existing ones. Usage is similar to conventional Go programming. Start [building your service with EGo](../building-marbles/ego.md) to use it with MarbleRun. + +## Edgeless RT +With [Edgeless RT](https://github.com/edgelesssys/edgelessrt), you can create confidential C++ applications with a low TCB. Please follow the build instructions provided [in the C++ sample](https://github.com/edgelesssys/marblerun/blob/master/samples/helloc%2B%2B) to use it with MarbleRun. + +## Gramine +[Gramine](https://gramineproject.io/) is a popular choice for wrapping unmodified applications into enclaves. +This approach, commonly known as "lift and shift," facilitates the process of bringing existing applications into the confidential space. +Gramine further adds support for dynamically linked libraries and multi-process applications in SGX. +[Running a Gramine app](../building-marbles/gramine.md) with MarbleRun requires minor changes to its manifest. + +## Occlum +[Occlum](https://github.com/occlum/occlum) is another popular solution that allows wrapping existing applications with minimal to no changes inside an enclave. It requires you to, at best, recompile existing applications with the provided toolchains with support for common languages such as C, C++, Go, Java, and Rust. +Its core is written in the memory-safe programming language Rust, and it provides a separated environment under which your application is running. This provides a safe yet powerful way to build your applications. +[Running an Occlum app](../building-marbles/occlum.md) with MarbleRun requires minor changes to its manifest. diff --git a/docs/versioned_docs/version-1.4/features/secrets-management.md b/docs/versioned_docs/version-1.4/features/secrets-management.md new file mode 100644 index 00000000..171f0836 --- /dev/null +++ b/docs/versioned_docs/version-1.4/features/secrets-management.md @@ -0,0 +1,31 @@ +# Secrets management + +The generation and the management of cryptographic keys and certificates for Marbles (i.e., containers running enclaves) are central duties of the Coordinator. Keys and certificates are passed to Marbles on startup via placeholders defined in the manifest. You can learn more about this mechanism in the `Secrets` section from our [manifest definition hands-on](../workflows/define-manifest.md?id=secrets). Specifically, the Coordinator provides the following to Marbles. + +* [Symmetric keys](#symmetric-keys) +* [Certificates](#certificates) +* [User-defined secrets](#user-defined-secrets) + +## Symmetric keys +MarbleRun allows you to define symmetric keys that are generated by the Coordinator and distributed to your Marbles. +Such keys can either be shared among Marbles or individually generated for each instance. +This is useful whenever data needs to leave the protected memory of your enclave, hence, needs to be encrypted. For example, when sharing data between multiple Marbles or storing data on the disk. + +To pass the generated keys to your application, you can use a template placeholder definition like `{{ raw .Secrets.MySecretKey }}` in your manifest. Those templates can be utilized when specifying files and environment variables for your application. From these locations, your application can retrieve the generated secret keys at runtime and use them for cryptographic purposes. + +If you use one key throughout multiple Marbles, care has to be taken to not repeat nonces between Marbles when using shared keys with AES-GCM or similar encryption algorithms. + +## Certificates +Upon launch of a Marble, the Coordinator will generate TLS credentials for each new Marble in the form of a private key and a corresponding X.509 certificate. The certificate is signed with the Coordinator's intermediate CA. These TLS credentials represent [the notion of identity used for authentication in MarbleRun](../features/attestation.md). Marbles can use this certificate to establish secure communication channels with other Marbles and external clients (i.e., users of your app). Clients only need to verify the Coordinator's root CA once before they can securely communicate with any Marble, as is described in more detail in our [verification hands-on](../workflows/verification.md). By default, they're available to your application via the environment variables `MARBLE_PREDEFINED_MARBLE_CERTIFICATE_CHAIN` and `MARBLE_PREDEFINED_PRIVATE_KEY`. In addition, you can expose them via custom environment variables or files by defining template placeholders such as `{{ pem .MarbleRun.MarbleCert.Cert }}` or `{{ pem .MarbleRun.MarbleCert.Private }}` in the manifest. + +Next to the default credentials, you can use `Secrets` to define custom certificates in your manifest. These are also generated upon each launch of your Marble and signed by the Coordinator's intermediate CA. You can include them into environment variables or files via placeholders such as `{{ pem .Secrets.MySecretCertificate }}` in your manifest. + +:::note + +Certificate secrets are always regenerated upon launch of a Marble, making them not suitable for certificate pinning. + +::: + +## User-defined secrets + +In addition to secrets generated by the Coordinator, MarbleRun also allows you to define secrets that can be uploaded and updated after the initial deployment of your manifest. This can be used to store sensitive data, such as passwords, tokens, certificates, or other types of credentials that shouldn't be stored in the manifest directly. They can be injected via the placeholders in the same way as other secret types. Changes to user-defined secrets are logged and can be audited over via the `/update` or `/secrets` HTTP REST API endpoint of the Coordinator. You can learn more about this in the section about [Managing secrets](../workflows/managing-secrets.md). diff --git a/docs/versioned_docs/version-1.4/features/transparent-TLS.md b/docs/versioned_docs/version-1.4/features/transparent-TLS.md new file mode 100644 index 00000000..0ed9e71e --- /dev/null +++ b/docs/versioned_docs/version-1.4/features/transparent-TLS.md @@ -0,0 +1,12 @@ +# Transparent TLS + +Authenticated and encrypted connections between services are essential for the security and verifiability of confidential applications. These properties are provided by mutual TLS authentication (mTLS). Normally, the applications inside the Marbles must support mTLS, be configured correctly, and be provisioned with the necessary secrets. + +Transparent TLS (TTLS) can wrap any connection in TLS on the MarbleRun layer. MarbleRun adds secure communication to your cluster even if your application doesn't support the required TLS features. Just define the desired [connections in the manifest](../workflows/define-manifest.md#tls). + +TTLS is currently available with [EGo Marbles](../building-marbles/ego.md). Other [runtimes](../features/runtimes.md) will be supported in future. + +## Authentication and credentials +By default the Marble's credentials are automatically configured. Connections between two Marbles are mutually authenticated. + +You can use custom credentials defined in the manifest's secrets, as described in the [manifest workflow](../workflows/define-manifest.md#tls). This can be useful when connecting from outside the cluster, to always serve the same certificate. diff --git a/docs/versioned_docs/version-1.4/getting-started/examples.md b/docs/versioned_docs/version-1.4/getting-started/examples.md new file mode 100644 index 00000000..65133f4f --- /dev/null +++ b/docs/versioned_docs/version-1.4/getting-started/examples.md @@ -0,0 +1,30 @@ +# Examples + +## Hello world + +There are two basic examples of how to build confidential applications with MarbleRun. + +* See [`helloworld`](https://github.com/edgelesssys/marblerun/blob/master/samples/helloworld) for how to integrate MarbleRun with your Golang application. 🌎 +* See [`helloc++`](https://github.com/edgelesssys/marblerun/blob/master/samples/helloc%2B%2B) for how to integrate MarbleRun with your C++ application. + +## Emoji voting + +The popular [Linkerd](https://linkerd.io) service mesh uses the simple and fun scalable *emojivoto* app as the default demo. You can find our confidential variant [on GitHub](https://github.com/edgelesssys/emojivoto). Your emoji votes have never been safer! + +## Confidential multi-stakeholder AI with Tensorflow + +We provide a hands-on example for a confidential multi-stakeholder inference service. + +* Read the [blog post](https://www.edgeless.systems/blog/confidential-multi-stakeholder-ai/) +* Check it out [on GitHub](https://github.com/edgelesssys/marblerun-tensorflow-demo) + +## Gramine-based examples + +We provide two examples for Gramine-based Marbles: + +* A [helloworld](https://github.com/edgelesssys/marblerun/tree/master/samples/gramine-hello) example to get you started. 🤓 +* An [nginx web server](https://github.com/edgelesssys/marblerun/tree/master/samples/gramine-nginx) for an example of converting an existing Gramine application to a Marble. :rocket: + +## Occlum-based examples + +* A ["hello world"](https://github.com/edgelesssys/marblerun/tree/master/samples/occlum-hello) app to get you started. 👋 diff --git a/docs/versioned_docs/version-1.4/getting-started/installation.md b/docs/versioned_docs/version-1.4/getting-started/installation.md new file mode 100644 index 00000000..94a60b2a --- /dev/null +++ b/docs/versioned_docs/version-1.4/getting-started/installation.md @@ -0,0 +1,134 @@ +# Installation and setup + +To install MarbleRun into your Kubernetes cluster and manage it, there's a dedicated command-line interface (CLI). +The following guides you through the steps of installing the CLI on your machine and the configuration required to verify the [Coordinator's attestation reports](../workflows/verification.md). + +## Prerequisites + +Make sure the following requirements are met: + +* Your machine is running Linux on an x86-64 CPU +* You have access to a Kubernetes cluster and kubectl installed and configured + +An easy way to get started is to run Kubernetes on your local machine using [minikube](https://minikube.sigs.k8s.io/docs/start/). +Check the [prerequisites](../deployment/kubernetes.md#prerequisites) if you want to set up an SGX-enabled cluster. +Another easy way is to use [Azure Kubernetes Service (AKS)](https://docs.microsoft.com/en-us/azure/aks/kubernetes-walkthrough-portal), which offers SGX-enabled nodes. + +You can validate your setup by running the following: + +```bash +kubectl version --short +``` + +You should see an output with both a Client Version and a Server Version component. +Now your cluster is ready and you can install the MarbleRun CLI. + +## Install the MarbleRun CLI + +CLI executables for different platforms are available at [GitHub](https://github.com/edgelesssys/marblerun/releases). +The CLI needs an SGX [quote provider](https://download.01.org/intel-sgx/latest/dcap-latest/linux/docs/DCAP_ECDSA_Orientation.pdf) to verify attestation reports. + + + + +The AppImage runs on all x86-64 Linux distributions with glibc v2.29 or higher. +It includes the quote provider. +Install it with the following commands: + +```bash +wget https://github.com/edgelesssys/marblerun/releases/latest/download/marblerun-x86_64.AppImage +sudo install marblerun-x86_64.AppImage /usr/local/bin/marblerun +``` + + + + +Install the CLI and the quote provider with the following commands: + +```bash +# install CLI +wget https://github.com/edgelesssys/marblerun/releases/latest/download/marblerun-ubuntu-20.04 +sudo install marblerun-ubuntu-20.04 /usr/local/bin/marblerun + +# install quote provider +sudo mkdir -p /etc/apt/keyrings +wget -qO- https://download.01.org/intel-sgx/sgx_repo/ubuntu/intel-sgx-deb.key | sudo tee /etc/apt/keyrings/intel-sgx-keyring.asc > /dev/null +echo "deb [signed-by=/etc/apt/keyrings/intel-sgx-keyring.asc arch=amd64] https://download.01.org/intel-sgx/sgx_repo/ubuntu focal main" | sudo tee /etc/apt/sources.list.d/intel-sgx.list +sudo apt update +sudo apt install libsgx-dcap-default-qpl +``` + + + + +Install the CLI and the quote provider with the following commands: + +```bash +# install CLI +wget https://github.com/edgelesssys/marblerun/releases/latest/download/marblerun-ubuntu-22.04 +sudo install marblerun-ubuntu-22.04 /usr/local/bin/marblerun + +# install quote provider +wget -qO- https://download.01.org/intel-sgx/sgx_repo/ubuntu/intel-sgx-deb.key | sudo tee /etc/apt/keyrings/intel-sgx-keyring.asc > /dev/null +echo "deb [signed-by=/etc/apt/keyrings/intel-sgx-keyring.asc arch=amd64] https://download.01.org/intel-sgx/sgx_repo/ubuntu jammy main" | sudo tee /etc/apt/sources.list.d/intel-sgx.list +sudo apt update +sudo apt install libsgx-dcap-default-qpl +``` + + + + +Once installed, verify the CLI is running correctly with the following: + +```bash +marblerun +``` + +You can use the CLI to check if your cluster is configured to run SGX workloads: + +```bash +marblerun precheck +``` + +## Configure the quote provider + +The CLI uses the quote provider to retrieve collaterals required for verifying attestation reports. +Locate its configuration file at `/etc/sgx_default_qcnl.conf`. +If the file doesn't exist or is outdated, download it with the following command: + +```bash +wget -qO- https://raw.githubusercontent.com/intel/SGXDataCenterAttestationPrimitives/master/QuoteGeneration/qcnl/linux/sgx_default_qcnl.conf | sudo tee /etc/sgx_default_qcnl.conf > /dev/null +``` + +You can configure the quote provider to get the collaterals from the Intel PCS, the PCCS of your cloud service provider (CSP), or your own PCCS. + +### Intel PCS + +Using the Intel PCS is the simplest and most generic way to get the collaterals, but it may be slower and less reliable than a PCCS. +Configure it by uncommenting the `"collateral_service"` key: + +```json + ,"collateral_service": "https://api.trustedservices.intel.com/sgx/certification/v4/" +``` + +### PCCS of your CSP + +If you're running MarbleRun in the cloud, it's recommended to use the PCCS of your CSP. +Set the `"pccs_url"` value to the respective address: + +* Azure: `https://global.acccache.azure.net/sgx/certification/v4/` + + See the [Azure documentation](https://learn.microsoft.com/en-us/azure/security/fundamentals/trusted-hardware-identity-management#how-do-i-use-intel-qpl-with-trusted-hardware-identity-management) for more information on configuring the quote provider. + +* Alibaba: `https://sgx-dcap-server.[Region-ID].aliyuncs.com/sgx/certification/v4/` + + See the [Alibaba documentation](https://www.alibabacloud.com/help/en/ecs/user-guide/build-an-sgx-encrypted-computing-environment) for supported Region-ID values and more information on configuring the quote provider. + +### Your own PCCS + +If you're running MarbleRun on premises and have set up your own PCCS for quote generation, you can also use it for quote verification. +Set the `"pccs_url"` value to the address of your PCCS. + +If your PCCS runs with a certificate not signed by a trusted CA, you need to set `"use_secure_cert"` to `false`. +This instructs the quote provider to accept a self-signed certificate of the PCCS. +It doesn't affect the security of the remote attestation process itself. diff --git a/docs/versioned_docs/version-1.4/getting-started/quickstart.md b/docs/versioned_docs/version-1.4/getting-started/quickstart.md new file mode 100644 index 00000000..f5866e3b --- /dev/null +++ b/docs/versioned_docs/version-1.4/getting-started/quickstart.md @@ -0,0 +1,174 @@ +# First steps with MarbleRun + +The following steps guide you through the process of deploying MarbleRun in your cluster and deploying a sample app. This example assumes that you have successfully [installed MarbleRun](./installation.md), and have access to a Kubernetes cluster. + +A working SGX DCAP environment is required for MarbleRun. For ease of exploring and testing, we provide a simulation mode with `--simulation` that runs without SGX hardware. +Depending on your setup, you may follow the quickstart for SGX-enabled clusters. Alternatively, if your setup doesn't support SGX, you can follow the quickstart in simulation mode by selecting the respective tabs. + +## Step 0: Clone the demo repository + +To get a feel for how MarbleRun would work for one of your services, you can install a demo application. +The emojivoto application is a standalone Kubernetes application that uses a mix of gRPC and HTTP calls to allow users to vote on their favorite emojis. +Created as a demo application for the popular [Linkerd](https://linkerd.io) service mesh, we've made a confidential variant that uses a confidential service mesh for all gRPC and HTTP connections. +Clone the [demo application's repository](https://github.com/edgelesssys/emojivoto.git) from GitHub by running the following: + +```bash +git clone https://github.com/edgelesssys/emojivoto.git && cd emojivoto +``` + +## Step 1: Install the control plane onto your cluster + +Install MarbleRun's *Coordinator* control plane by running: + + + + +```bash +marblerun install +``` + +The `marblerun install` command generates a Kubernetes manifest with all the necessary control plane resources. +This includes a deployment for the Coordinator and for MarbleRun's [admission controller.](../features/kubernetes-integration.md) + + + + +```bash +marblerun install --simulation +``` + +The `marblerun install` command generates a Kubernetes manifest with all the necessary control plane resources. +This includes a deployment for the Coordinator and for MarbleRun's [admission controller.](../features/kubernetes-integration.md) +The simulation flag tells MarbleRun that real SGX hardware might not be present and the SGX layer should be emulated. + + + + +Wait for the control plane to finish installing: + +```bash +marblerun check +``` + +This command will wait until all components of MarbleRun are ready to be used or return an error after a timeout period is reached. + +Port forward the Coordinator's Client API: + +```bash +kubectl -n marblerun port-forward svc/coordinator-client-api 4433:4433 --address localhost >/dev/null & +export MARBLERUN=localhost:4433 +``` + +### Step 2: Configure MarbleRun + +MarbleRun guarantees that the topology of your distributed app adheres to a manifest specified in simple JSON. +MarbleRun verifies the integrity of services, bootstraps them, and sets up encrypted connections between them. +The emojivoto demo already comes with a [manifest](https://github.com/edgelesssys/emojivoto/blob/main/tools/manifest.json), which you can deploy onto MarbleRun by running the following: + + + + +```bash +marblerun manifest set tools/manifest.json $MARBLERUN +``` + +The CLI will obtain the Coordinator's remote attestation quote and verify it against the configuration on our [release page](https://github.com/edgelesssys/marblerun/releases/latest/download/coordinator-era.json). +The SGX quote proves the integrity of the Coordinator pod. +The CLI saves the TLS certificate of the Coordinator as `coordinator-cert.pem` in your config directory. +The certificate is bound to the quote and is used by the CLI for future verification. +It can also be used as a root of trust for [authenticating your confidential applications](../features/attestation.md). + +:::info + +By default the certificate is saved to `$XDG_CONFIG_HOME/marblerun/coordinator-cert.pem`, +or `$HOME/.config/marblerun/coordinator-cert.pem` if `$XDG_CONFIG_HOME` is not set. +Subsequent CLI commands will try loading the certificate from that location. +Use the `--coordinator-cert` flag to choose your own location to save or load the certificate. + +::: + + + + +```bash +marblerun manifest set tools/manifest.json $MARBLERUN --insecure +``` + +The insecure flag tells MarbleRun that real SGX hardware might not be present and the quote verification should be omitted. +Since this bypasses all security checks on the TLS connection to the Coordinator, you MUST NOT use it in production. + + + + +You can check that the state of MarbleRun has changed and is now ready to authenticate your services by running: + + + + +```bash +marblerun status $MARBLERUN +``` + + + + +```bash +marblerun status $MARBLERUN --insecure +``` + + + + +## Step 3: Deploy the demo application + +Finally, install the demo application onto your cluster. +Please make sure you have [Helm](https://helm.sh/docs/intro/install/) ("the package manager for Kubernetes") installed at least at Version v3.2.0. +Install emojivoto into the emojivoto namespace by running: + + + + +```bash +helm install -f ./kubernetes/sgx_values.yaml emojivoto ./kubernetes --create-namespace -n emojivoto +``` + + + + +```bash +helm install -f ./kubernetes/nosgx_values.yaml emojivoto ./kubernetes --create-namespace -n emojivoto +``` + + + + +## Step 4: Watch it run + +You can now check the MarbleRun log and see the services being authenticated by the Coordinator. + +```bash +kubectl -n marblerun logs -f -l edgeless.systems/control-plane-component=coordinator +``` + +Port forward the front-end web service to access it on your local machine by running: + +```bash +kubectl -n emojivoto port-forward svc/web-svc 8443:443 --address 0.0.0.0 +``` + +Now visit [https://localhost:8443](https://localhost:8443). +You'll be presented with a certificate warning because your browser, by default, doesn't trust certificates signed by MarbleRun. +You can ignore this error for now and proceed to the website.\ +Voila! Your emoji votes have never been safer! + +## That’s it 👏 + +Congratulations, you’re now a MarbleRun user! Here are some suggested next steps: + +* Explore how [MarbleRun takes care of your secrets](../features/secrets-management.md) +* [Add your own service](../workflows/add-service.md) to MarbleRun +* Learn more about [MarbleRun’s architecture](../architecture/concepts.md) +* Chat with us on [Discord](https://discord.gg/rH8QTH56JN) +* Try out the full demo on [GitHub](https://github.com/edgelesssys/emojivoto) + +Welcome to the MarbleRun community! diff --git a/docs/versioned_docs/version-1.4/intro.md b/docs/versioned_docs/version-1.4/intro.md new file mode 100644 index 00000000..85094883 --- /dev/null +++ b/docs/versioned_docs/version-1.4/intro.md @@ -0,0 +1,45 @@ +--- +slug: / +--- + +# Welcome to MarbleRun + +MarbleRun is a framework for deploying distributed confidential computing applications. +MarbleRun acts as a confidential operator for your deployment. Think of a trusted party in the control plane. + +Build your confidential microservices with [EGo, Gramine, or similar runtimes](./features/runtimes.md), orchestrate them with Kubernetes on an SGX-enabled cluster, and let MarbleRun take care of the rest. +Deploy end-to-end secure and verifiable AI pipelines or crunch on sensitive big data in the cloud. Confidential computing at scale has never been easier. + +MarbleRun simplifies the process by handling much of the groundwork. +It ensures that your app's topology adheres to your specified manifest. +It verifies the identity and integrity of all your services, bootstraps them, and establishes secure, encrypted communication channels. +As your app needs to scale, MarbleRun manages the addition of new instances, ensuring their secure verification. + +MarbleRun provides comprehensive access and privilege management through its role-based access control (RBAC) feature. You can use this to enforce security policies such as four-eyes principles and other verification procedures in your operations. + +To keep things simple, MarbleRun offers a single remote attestation statement for your entire app. Anyone can use this to verify the integrity of your distributed app, making it easier to assure stakeholders of your app's security. + +## Key features + +🔒 Authentication and integrity verification of microservices based on the manifest + + +🔑 Secrets management for confidential microservices + + +📃 Provisioning of certificates, configurations, and parameters + + +🔬 Remote attestation of the entire deployment + + +👥 Role-based access control (RBAC), verifiable updates, and multi-stakeholder support + +## Overview + +Logically, MarbleRun consists of two parts, the control plane called *Coordinator* and the data plane called *Marbles*. +The Coordinator needs to be deployed once in your cluster, and the Marble layer needs to be integrated with each service. +MarbleRun is configured with a simple JSON document called the *manifest*. +It specifies the distributed app's topology and infrastructure properties and provides configuration parameters for each service. + +![overview](_media/overview.svg) diff --git a/docs/versioned_docs/version-1.4/reference/cli.md b/docs/versioned_docs/version-1.4/reference/cli.md new file mode 100644 index 00000000..da2580a0 --- /dev/null +++ b/docs/versioned_docs/version-1.4/reference/cli.md @@ -0,0 +1,919 @@ +# Command Line Interface (CLI) + +The MarbleRun CLI allows you to install MarbleRun on your cluster and interacts with the control plane through the Client API for all administrative tasks in the service mesh. + +## Reference + +Usage: + +```terminal +marblerun [command] +``` +Commands: + +* [install](#marblerun-install): Installs MarbleRun on a Kubernetes cluster +* [uninstall](#marblerun-uninstall): Remove MarbleRun from a Kubernetes cluster +* [precheck](#marblerun-precheck): Check if your Kubernetes cluster supports SGX +* [check](#marblerun-check): Check the status of MarbleRun's control plane +* [manifest](#marblerun-manifest): Manages manifest for the MarbleRun Coordinator + * [get](#marblerun-manifest-get): Get the manifest from the MarbleRun Coordinator + * [log](#marblerun-manifest-log): Get the update log from the MarbleRun Coordinator + * [set](#marblerun-manifest-set): Sets the manifest for the MarbleRun Coordinator + * [signature](#marblerun-manifest-signature): Prints the signature of a MarbleRun manifest + * [update](#marblerun-manifest-update): Manage manifest updates for the MarbleRun Coordinator + * [apply](#marblerun-manifest-update-apply): Update the MarbleRun Coordinator with the specified manifest + * [acknowledge](#marblerun-manifest-update-acknowledge): Acknowledge a pending update for the MarbleRun Coordinator (Enterprise feature) + * [cancel](#marblerun-manifest-update-cancel): Cancel a pending manifest update for the MarbleRun Coordinator (Enterprise feature) + * [get](#marblerun-manifest-update-get): View a pending manifest update (Enterprise feature) + * [verify](#marblerun-manifest-verify): Verify the signature of a MarbleRun manifest +* [certificate](#marblerun-certificate): Retrieves the certificate of the MarbleRun Coordinator + * [root](#marblerun-certificate-root): Returns the root certificate of the MarbleRun Coordinator + * [intermediate](#marblerun-certificate-intermediate): Returns the intermediate certificate of the MarbleRun Coordinator + * [chain](#marblerun-certificate-chain): Returns the certificate chain of the MarbleRun Coordinator +* [secret](#marblerun-secret): Manage secrets for the MarbleRun Coordinator + * [set](#marblerun-secret-set): Set a secret for the MarbleRun Coordinator + * [get](#marblerun-secret-get): Retrieve secrets from the MarbleRun Coordinator +* [status](#marblerun-status): Retrieve information about the status of the MarbleRun Coordinator +* [recover](#marblerun-recover): Recover the MarbleRun Coordinator from a sealed state +* [package-info](#marblerun-package-info): Print the package signature properties of an enclave +* [version](#marblerun-version): Display version of this CLI and (if running) the MarbleRun Coordinator + +## marblerun install + +Installs MarbleRun on a Kubernetes cluster + +### Synopsis + +Installs MarbleRun on a Kubernetes cluster + +``` +marblerun install [flags] +``` + +### Examples + +``` +# Install MarbleRun in simulation mode +marblerun install --simulation + +# Install MarbleRun using a custom PCCS +marblerun install --dcap-pccs-url https://pccs.example.com/sgx/certification/v4/ --dcap-secure-cert FALSE +``` + +### Options + +``` + --client-server-port int Set the client server port. Needs to be configured to the same port as in your client tool stack (default 4433) + --dcap-pccs-url string Provisioning Certificate Caching Service (PCCS) server address. Defaults to Azure PCCS. (default "https://global.acccache.azure.net/sgx/certification/v4/") + --dcap-secure-cert string To accept insecure HTTPS certificate from the PCCS, set this option to FALSE (default "TRUE") + --disable-auto-injection Install MarbleRun without auto-injection webhook + --domain string Sets the CNAME for the Coordinator certificate (default "localhost") + --enterprise-access-token string Access token for Enterprise Coordinator. Leave empty for default installation + -h, --help help for install + --marblerun-chart-path string Path to MarbleRun helm chart + --mesh-server-port int Set the mesh server port. Needs to be configured to the same port as in the data-plane marbles (default 2001) + --resource-key string Resource providing SGX, different depending on used device plugin. Use this to set tolerations/resources if your device plugin is not supported by MarbleRun + --simulation Set MarbleRun to start in simulation mode + --version string Version of the Coordinator to install, latest by default + --wait Wait for MarbleRun installation to complete before returning +``` + +### Options inherited from parent commands + +``` + --accepted-tcb-statuses strings Comma-separated list of user accepted TCB statuses (e.g. ConfigurationNeeded,ConfigurationAndSWHardeningNeeded) (default [UpToDate]) + --coordinator-cert string Path to MarbleRun Coordinator's root certificate to use for TLS connections (default "/github/home/.config/marblerun/coordinator-cert.pem") + --era-config string Path to remote attestation config file in json format, if none provided the newest configuration will be loaded from github + -i, --insecure Set to skip quote verification, needed when running in simulation mode + -n, --namespace string Kubernetes namespace of the MarbleRun installation (default "marblerun") +``` + +## marblerun uninstall + +Remove MarbleRun from a Kubernetes cluster + +### Synopsis + +Remove MarbleRun from a Kubernetes cluster + +``` +marblerun uninstall [flags] +``` + +### Options + +``` + -h, --help help for uninstall + --wait Wait for the uninstallation to complete before returning +``` + +### Options inherited from parent commands + +``` + --accepted-tcb-statuses strings Comma-separated list of user accepted TCB statuses (e.g. ConfigurationNeeded,ConfigurationAndSWHardeningNeeded) (default [UpToDate]) + --coordinator-cert string Path to MarbleRun Coordinator's root certificate to use for TLS connections (default "/github/home/.config/marblerun/coordinator-cert.pem") + --era-config string Path to remote attestation config file in json format, if none provided the newest configuration will be loaded from github + -i, --insecure Set to skip quote verification, needed when running in simulation mode + -n, --namespace string Kubernetes namespace of the MarbleRun installation (default "marblerun") +``` + +## marblerun precheck + +Check if your Kubernetes cluster supports SGX + +### Synopsis + +Check if your Kubernetes cluster supports SGX + +``` +marblerun precheck [flags] +``` + +### Options + +``` + -h, --help help for precheck +``` + +### Options inherited from parent commands + +``` + --accepted-tcb-statuses strings Comma-separated list of user accepted TCB statuses (e.g. ConfigurationNeeded,ConfigurationAndSWHardeningNeeded) (default [UpToDate]) + --coordinator-cert string Path to MarbleRun Coordinator's root certificate to use for TLS connections (default "/github/home/.config/marblerun/coordinator-cert.pem") + --era-config string Path to remote attestation config file in json format, if none provided the newest configuration will be loaded from github + -i, --insecure Set to skip quote verification, needed when running in simulation mode + -n, --namespace string Kubernetes namespace of the MarbleRun installation (default "marblerun") +``` + +## marblerun check + +Check the status of MarbleRun's control plane + +### Synopsis + +Check the status of MarbleRun's control plane + +``` +marblerun check [flags] +``` + +### Options + +``` + -h, --help help for check + --timeout uint Time to wait before aborting in seconds (default 60) +``` + +### Options inherited from parent commands + +``` + --accepted-tcb-statuses strings Comma-separated list of user accepted TCB statuses (e.g. ConfigurationNeeded,ConfigurationAndSWHardeningNeeded) (default [UpToDate]) + --coordinator-cert string Path to MarbleRun Coordinator's root certificate to use for TLS connections (default "/github/home/.config/marblerun/coordinator-cert.pem") + --era-config string Path to remote attestation config file in json format, if none provided the newest configuration will be loaded from github + -i, --insecure Set to skip quote verification, needed when running in simulation mode + -n, --namespace string Kubernetes namespace of the MarbleRun installation (default "marblerun") +``` + +## marblerun manifest + +Manages manifest for the MarbleRun Coordinator + +### Synopsis + + +Manages manifests for the MarbleRun Coordinator. +Used to either set the manifest, update an already set manifest, +or return a signature of the currently set manifest to the user + +### Examples + +``` +manifest set manifest.json example.com:4433 [--era-config=config.json] [--insecure] +``` + +### Options + +``` + -h, --help help for manifest +``` + +### Options inherited from parent commands + +``` + --accepted-tcb-statuses strings Comma-separated list of user accepted TCB statuses (e.g. ConfigurationNeeded,ConfigurationAndSWHardeningNeeded) (default [UpToDate]) + --coordinator-cert string Path to MarbleRun Coordinator's root certificate to use for TLS connections (default "/github/home/.config/marblerun/coordinator-cert.pem") + --era-config string Path to remote attestation config file in json format, if none provided the newest configuration will be loaded from github + -i, --insecure Set to skip quote verification, needed when running in simulation mode + -n, --namespace string Kubernetes namespace of the MarbleRun installation (default "marblerun") +``` + +## marblerun manifest get + +Get the manifest from the MarbleRun Coordinator + +### Synopsis + +Get the manifest from the MarbleRun Coordinator. +Optionally get the manifests signature or merge updates into the displayed manifest. + +``` +marblerun manifest get [flags] +``` + +### Examples + +``` +marblerun manifest get $MARBLERUN -s --era-config=era.json +``` + +### Options + +``` + -u, --display-update Set to merge updates into the displayed manifest + -h, --help help for get + --keep-cert Set to keep the certificate of the Coordinator and save it to the location specified by --coordinator-cert + -o, --output string Save output to file instead of printing to stdout + -s, --signature Set to additionally display the manifests signature +``` + +### Options inherited from parent commands + +``` + --accepted-tcb-statuses strings Comma-separated list of user accepted TCB statuses (e.g. ConfigurationNeeded,ConfigurationAndSWHardeningNeeded) (default [UpToDate]) + --coordinator-cert string Path to MarbleRun Coordinator's root certificate to use for TLS connections (default "/github/home/.config/marblerun/coordinator-cert.pem") + --era-config string Path to remote attestation config file in json format, if none provided the newest configuration will be loaded from github + -i, --insecure Set to skip quote verification, needed when running in simulation mode + -n, --namespace string Kubernetes namespace of the MarbleRun installation (default "marblerun") +``` + +## marblerun manifest log + +Get the update log from the MarbleRun Coordinator + +### Synopsis + +Get the update log from the MarbleRun Coordinator. + The log is list of all successful changes to the Coordinator, + including a timestamp and user performing the operation. + +``` +marblerun manifest log [flags] +``` + +### Examples + +``` +marblerun manifest log $MARBLERUN +``` + +### Options + +``` + -h, --help help for log + -o, --output string Save log to file instead of printing to stdout +``` + +### Options inherited from parent commands + +``` + --accepted-tcb-statuses strings Comma-separated list of user accepted TCB statuses (e.g. ConfigurationNeeded,ConfigurationAndSWHardeningNeeded) (default [UpToDate]) + --coordinator-cert string Path to MarbleRun Coordinator's root certificate to use for TLS connections (default "/github/home/.config/marblerun/coordinator-cert.pem") + --era-config string Path to remote attestation config file in json format, if none provided the newest configuration will be loaded from github + -i, --insecure Set to skip quote verification, needed when running in simulation mode + -n, --namespace string Kubernetes namespace of the MarbleRun installation (default "marblerun") +``` + +## marblerun manifest set + +Sets the manifest for the MarbleRun Coordinator + +### Synopsis + +Sets the manifest for the MarbleRun Coordinator + +``` +marblerun manifest set [flags] +``` + +### Examples + +``` +marblerun manifest set manifest.json $MARBLERUN --recovery-data=recovery-secret.json --era-config=era.json +``` + +### Options + +``` + -h, --help help for set + -r, --recoverydata string File to write recovery data to, print to stdout if non specified +``` + +### Options inherited from parent commands + +``` + --accepted-tcb-statuses strings Comma-separated list of user accepted TCB statuses (e.g. ConfigurationNeeded,ConfigurationAndSWHardeningNeeded) (default [UpToDate]) + --coordinator-cert string Path to MarbleRun Coordinator's root certificate to use for TLS connections (default "/github/home/.config/marblerun/coordinator-cert.pem") + --era-config string Path to remote attestation config file in json format, if none provided the newest configuration will be loaded from github + -i, --insecure Set to skip quote verification, needed when running in simulation mode + -n, --namespace string Kubernetes namespace of the MarbleRun installation (default "marblerun") +``` + +## marblerun manifest signature + +Prints the signature of a MarbleRun manifest + +### Synopsis + +Prints the signature of a MarbleRun manifest + +``` +marblerun manifest signature [flags] +``` + +### Options + +``` + -h, --help help for signature +``` + +### Options inherited from parent commands + +``` + --accepted-tcb-statuses strings Comma-separated list of user accepted TCB statuses (e.g. ConfigurationNeeded,ConfigurationAndSWHardeningNeeded) (default [UpToDate]) + --coordinator-cert string Path to MarbleRun Coordinator's root certificate to use for TLS connections (default "/github/home/.config/marblerun/coordinator-cert.pem") + --era-config string Path to remote attestation config file in json format, if none provided the newest configuration will be loaded from github + -i, --insecure Set to skip quote verification, needed when running in simulation mode + -n, --namespace string Kubernetes namespace of the MarbleRun installation (default "marblerun") +``` + +## marblerun manifest update + +Manage manifest updates for the MarbleRun Coordinator + +### Synopsis + +Manage manifest updates for the MarbleRun Coordinator. + +### Options + +``` + -h, --help help for update +``` + +### Options inherited from parent commands + +``` + --accepted-tcb-statuses strings Comma-separated list of user accepted TCB statuses (e.g. ConfigurationNeeded,ConfigurationAndSWHardeningNeeded) (default [UpToDate]) + --coordinator-cert string Path to MarbleRun Coordinator's root certificate to use for TLS connections (default "/github/home/.config/marblerun/coordinator-cert.pem") + --era-config string Path to remote attestation config file in json format, if none provided the newest configuration will be loaded from github + -i, --insecure Set to skip quote verification, needed when running in simulation mode + -n, --namespace string Kubernetes namespace of the MarbleRun installation (default "marblerun") +``` + +## marblerun manifest update apply + +Update the MarbleRun Coordinator with the specified manifest + +### Synopsis + + +Update the MarbleRun Coordinator with the specified manifest. +An admin certificate specified in the original manifest is needed to verify the authenticity of the update manifest. + + +``` +marblerun manifest update apply [flags] +``` + +### Examples + +``` +marblerun manifest update apply update-manifest.json $MARBLERUN --cert=admin-cert.pem --key=admin-key.pem --era-config=era.json +``` + +### Options + +``` + -c, --cert string PEM encoded admin certificate file (required) + -h, --help help for apply + -k, --key string PEM encoded admin key file (required) +``` + +### Options inherited from parent commands + +``` + --accepted-tcb-statuses strings Comma-separated list of user accepted TCB statuses (e.g. ConfigurationNeeded,ConfigurationAndSWHardeningNeeded) (default [UpToDate]) + --coordinator-cert string Path to MarbleRun Coordinator's root certificate to use for TLS connections (default "/github/home/.config/marblerun/coordinator-cert.pem") + --era-config string Path to remote attestation config file in json format, if none provided the newest configuration will be loaded from github + -i, --insecure Set to skip quote verification, needed when running in simulation mode + -n, --namespace string Kubernetes namespace of the MarbleRun installation (default "marblerun") +``` + +## marblerun manifest update acknowledge + +Acknowledge a pending update for the MarbleRun Coordinator (Enterprise feature) + +### Synopsis + +Acknowledge a pending update for the MarbleRun Coordinator (Enterprise feature). + +In case of multi-party updates, the Coordinator will wait for all participants to acknowledge the update before applying it. +All participants must use the same manifest to acknowledge the pending update. + + +``` +marblerun manifest update acknowledge [flags] +``` + +### Examples + +``` +marblerun manifest update acknowledge update-manifest.json $MARBLERUN --cert=admin-cert.pem --key=admin-key.pem --era-config=era.json +``` + +### Options + +``` + -c, --cert string PEM encoded admin certificate file (required) + -h, --help help for acknowledge + -k, --key string PEM encoded admin key file (required) +``` + +### Options inherited from parent commands + +``` + --accepted-tcb-statuses strings Comma-separated list of user accepted TCB statuses (e.g. ConfigurationNeeded,ConfigurationAndSWHardeningNeeded) (default [UpToDate]) + --coordinator-cert string Path to MarbleRun Coordinator's root certificate to use for TLS connections (default "/github/home/.config/marblerun/coordinator-cert.pem") + --era-config string Path to remote attestation config file in json format, if none provided the newest configuration will be loaded from github + -i, --insecure Set to skip quote verification, needed when running in simulation mode + -n, --namespace string Kubernetes namespace of the MarbleRun installation (default "marblerun") +``` + +## marblerun manifest update cancel + +Cancel a pending manifest update for the MarbleRun Coordinator (Enterprise feature) + +### Synopsis + +Cancel a pending manifest update for the MarbleRun Coordinator (Enterprise feature). + +``` +marblerun manifest update cancel [flags] +``` + +### Examples + +``` +marblerun manifest update cancel $MARBLERUN --cert=admin-cert.pem --key=admin-key.pem --era-config=era.json +``` + +### Options + +``` + -c, --cert string PEM encoded admin certificate file (required) + -h, --help help for cancel + -k, --key string PEM encoded admin key file (required) +``` + +### Options inherited from parent commands + +``` + --accepted-tcb-statuses strings Comma-separated list of user accepted TCB statuses (e.g. ConfigurationNeeded,ConfigurationAndSWHardeningNeeded) (default [UpToDate]) + --coordinator-cert string Path to MarbleRun Coordinator's root certificate to use for TLS connections (default "/github/home/.config/marblerun/coordinator-cert.pem") + --era-config string Path to remote attestation config file in json format, if none provided the newest configuration will be loaded from github + -i, --insecure Set to skip quote verification, needed when running in simulation mode + -n, --namespace string Kubernetes namespace of the MarbleRun installation (default "marblerun") +``` + +## marblerun manifest update get + +View a pending manifest update (Enterprise feature) + +### Synopsis + +View a pending manifest update (Enterprise feature). + +``` +marblerun manifest update get [flags] +``` + +### Examples + +``` +marblerun manifest update get $MARBLERUN --era-config=era.json +``` + +### Options + +``` + -h, --help help for get + --missing Display number of missing acknowledgements instead of the manifest + -o, --output string Save output to file instead of printing to stdout +``` + +### Options inherited from parent commands + +``` + --accepted-tcb-statuses strings Comma-separated list of user accepted TCB statuses (e.g. ConfigurationNeeded,ConfigurationAndSWHardeningNeeded) (default [UpToDate]) + --coordinator-cert string Path to MarbleRun Coordinator's root certificate to use for TLS connections (default "/github/home/.config/marblerun/coordinator-cert.pem") + --era-config string Path to remote attestation config file in json format, if none provided the newest configuration will be loaded from github + -i, --insecure Set to skip quote verification, needed when running in simulation mode + -n, --namespace string Kubernetes namespace of the MarbleRun installation (default "marblerun") +``` + +## marblerun manifest verify + +Verify the signature of a MarbleRun manifest + +### Synopsis + +Verify that the signature returned by the Coordinator is equal to a local signature + +``` +marblerun manifest verify [flags] +``` + +### Examples + +``` +marblerun manifest verify manifest.json $MARBLERUN +``` + +### Options + +``` + -h, --help help for verify +``` + +### Options inherited from parent commands + +``` + --accepted-tcb-statuses strings Comma-separated list of user accepted TCB statuses (e.g. ConfigurationNeeded,ConfigurationAndSWHardeningNeeded) (default [UpToDate]) + --coordinator-cert string Path to MarbleRun Coordinator's root certificate to use for TLS connections (default "/github/home/.config/marblerun/coordinator-cert.pem") + --era-config string Path to remote attestation config file in json format, if none provided the newest configuration will be loaded from github + -i, --insecure Set to skip quote verification, needed when running in simulation mode + -n, --namespace string Kubernetes namespace of the MarbleRun installation (default "marblerun") +``` + +## marblerun certificate + +Retrieves the certificate of the MarbleRun Coordinator + +### Synopsis + +Retrieves the certificate of the MarbleRun Coordinator + +### Options + +``` + -h, --help help for certificate +``` + +### Options inherited from parent commands + +``` + --accepted-tcb-statuses strings Comma-separated list of user accepted TCB statuses (e.g. ConfigurationNeeded,ConfigurationAndSWHardeningNeeded) (default [UpToDate]) + --coordinator-cert string Path to MarbleRun Coordinator's root certificate to use for TLS connections (default "/github/home/.config/marblerun/coordinator-cert.pem") + --era-config string Path to remote attestation config file in json format, if none provided the newest configuration will be loaded from github + -i, --insecure Set to skip quote verification, needed when running in simulation mode + -n, --namespace string Kubernetes namespace of the MarbleRun installation (default "marblerun") +``` + +## marblerun certificate root + +Returns the root certificate of the MarbleRun Coordinator + +### Synopsis + +Returns the root certificate of the MarbleRun Coordinator + +``` +marblerun certificate root [flags] +``` + +### Options + +``` + -h, --help help for root + -o, --output string File to save the certificate to (default "marblerunRootCA.crt") +``` + +### Options inherited from parent commands + +``` + --accepted-tcb-statuses strings Comma-separated list of user accepted TCB statuses (e.g. ConfigurationNeeded,ConfigurationAndSWHardeningNeeded) (default [UpToDate]) + --coordinator-cert string Path to MarbleRun Coordinator's root certificate to use for TLS connections (default "/github/home/.config/marblerun/coordinator-cert.pem") + --era-config string Path to remote attestation config file in json format, if none provided the newest configuration will be loaded from github + -i, --insecure Set to skip quote verification, needed when running in simulation mode + -n, --namespace string Kubernetes namespace of the MarbleRun installation (default "marblerun") +``` + +## marblerun certificate intermediate + +Returns the intermediate certificate of the MarbleRun Coordinator + +### Synopsis + +Returns the intermediate certificate of the MarbleRun Coordinator + +``` +marblerun certificate intermediate [flags] +``` + +### Options + +``` + -h, --help help for intermediate + -o, --output string File to save the certificate to (default "marblerunIntermediateCA.crt") +``` + +### Options inherited from parent commands + +``` + --accepted-tcb-statuses strings Comma-separated list of user accepted TCB statuses (e.g. ConfigurationNeeded,ConfigurationAndSWHardeningNeeded) (default [UpToDate]) + --coordinator-cert string Path to MarbleRun Coordinator's root certificate to use for TLS connections (default "/github/home/.config/marblerun/coordinator-cert.pem") + --era-config string Path to remote attestation config file in json format, if none provided the newest configuration will be loaded from github + -i, --insecure Set to skip quote verification, needed when running in simulation mode + -n, --namespace string Kubernetes namespace of the MarbleRun installation (default "marblerun") +``` + +## marblerun certificate chain + +Returns the certificate chain of the MarbleRun Coordinator + +### Synopsis + +Returns the certificate chain of the MarbleRun Coordinator + +``` +marblerun certificate chain [flags] +``` + +### Options + +``` + -h, --help help for chain + -o, --output string File to save the certificate to (default "marblerunChainCA.crt") +``` + +### Options inherited from parent commands + +``` + --accepted-tcb-statuses strings Comma-separated list of user accepted TCB statuses (e.g. ConfigurationNeeded,ConfigurationAndSWHardeningNeeded) (default [UpToDate]) + --coordinator-cert string Path to MarbleRun Coordinator's root certificate to use for TLS connections (default "/github/home/.config/marblerun/coordinator-cert.pem") + --era-config string Path to remote attestation config file in json format, if none provided the newest configuration will be loaded from github + -i, --insecure Set to skip quote verification, needed when running in simulation mode + -n, --namespace string Kubernetes namespace of the MarbleRun installation (default "marblerun") +``` + +## marblerun secret + +Manage secrets for the MarbleRun Coordinator + +### Synopsis + + +Manage secrets for the MarbleRun Coordinator. +Set or retrieve a secret defined in the manifest. + +### Options + +``` + -c, --cert string PEM encoded MarbleRun user certificate file (required) + -h, --help help for secret + -k, --key string PEM encoded MarbleRun user key file (required) +``` + +### Options inherited from parent commands + +``` + --accepted-tcb-statuses strings Comma-separated list of user accepted TCB statuses (e.g. ConfigurationNeeded,ConfigurationAndSWHardeningNeeded) (default [UpToDate]) + --coordinator-cert string Path to MarbleRun Coordinator's root certificate to use for TLS connections (default "/github/home/.config/marblerun/coordinator-cert.pem") + --era-config string Path to remote attestation config file in json format, if none provided the newest configuration will be loaded from github + -i, --insecure Set to skip quote verification, needed when running in simulation mode + -n, --namespace string Kubernetes namespace of the MarbleRun installation (default "marblerun") +``` + +## marblerun secret set + +Set a secret for the MarbleRun Coordinator + +### Synopsis + + +Set a secret for the MarbleRun Coordinator. +Secrets are loaded from a file in JSON format or directly from a PEM +encoded certificate and/or key. In the later case, the name of the secret +has to be set with the flag [--from-pem]. +Users have to authenticate themselves using a certificate and private key +and need permissions in the manifest to write the requested secrets. + + +``` +marblerun secret set [flags] +``` + +### Examples + +``` +# Set a secret from a JSON file +marblerun secret set secret.json $MARBLERUN -c admin.crt -k admin.key + +# Set a secret from a PEM encoded file +marblerun secret set certificate.pem $MARBLERUN -c admin.crt -k admin.key --from-pem certificateSecret +``` + +### Options + +``` + --from-pem string name of the secret from a PEM encoded file + -h, --help help for set +``` + +### Options inherited from parent commands + +``` + --accepted-tcb-statuses strings Comma-separated list of user accepted TCB statuses (e.g. ConfigurationNeeded,ConfigurationAndSWHardeningNeeded) (default [UpToDate]) + -c, --cert string PEM encoded MarbleRun user certificate file (required) + --coordinator-cert string Path to MarbleRun Coordinator's root certificate to use for TLS connections (default "/github/home/.config/marblerun/coordinator-cert.pem") + --era-config string Path to remote attestation config file in json format, if none provided the newest configuration will be loaded from github + -i, --insecure Set to skip quote verification, needed when running in simulation mode + -k, --key string PEM encoded MarbleRun user key file (required) + -n, --namespace string Kubernetes namespace of the MarbleRun installation (default "marblerun") +``` + +## marblerun secret get + +Retrieve secrets from the MarbleRun Coordinator + +### Synopsis + + +Retrieve one or more secrets from the MarbleRun Coordinator. +Users have to authenticate themselves using a certificate and private key, +and need permissions in the manifest to read the requested secrets. + + +``` +marblerun secret get SECRETNAME ... [flags] +``` + +### Examples + +``` +marblerun secret get genericSecret symmetricKeyShared $MARBLERUN -c admin.crt -k admin.key +``` + +### Options + +``` + -h, --help help for get + -o, --output string File to save the secret to +``` + +### Options inherited from parent commands + +``` + --accepted-tcb-statuses strings Comma-separated list of user accepted TCB statuses (e.g. ConfigurationNeeded,ConfigurationAndSWHardeningNeeded) (default [UpToDate]) + -c, --cert string PEM encoded MarbleRun user certificate file (required) + --coordinator-cert string Path to MarbleRun Coordinator's root certificate to use for TLS connections (default "/github/home/.config/marblerun/coordinator-cert.pem") + --era-config string Path to remote attestation config file in json format, if none provided the newest configuration will be loaded from github + -i, --insecure Set to skip quote verification, needed when running in simulation mode + -k, --key string PEM encoded MarbleRun user key file (required) + -n, --namespace string Kubernetes namespace of the MarbleRun installation (default "marblerun") +``` + +## marblerun status + +Retrieve information about the status of the MarbleRun Coordinator + +### Synopsis + + +This command provides information about the currently running MarbleRun Coordinator. +Information is obtained from the /status endpoint of the Coordinators REST API. + +The Coordinator will be in one of these 4 states: + 0 recovery mode: Found a sealed state of an old seal key. Waiting for user input on /recovery. + The Coordinator is currently sealed, it can be recovered using the [marblerun recover] command. + + 1 uninitialized: Fresh start, initializing the Coordinator. + The Coordinator is in its starting phase. + + 2 waiting for manifest: Waiting for user input on /manifest. + Send a manifest to the Coordinator using [marblerun manifest set] to start. + + 3 accepting marble: The Coordinator is running, you can add marbles to the mesh or update the + manifest using [marblerun manifest update]. + + +``` +marblerun status [flags] +``` + +### Options + +``` + -h, --help help for status +``` + +### Options inherited from parent commands + +``` + --accepted-tcb-statuses strings Comma-separated list of user accepted TCB statuses (e.g. ConfigurationNeeded,ConfigurationAndSWHardeningNeeded) (default [UpToDate]) + --coordinator-cert string Path to MarbleRun Coordinator's root certificate to use for TLS connections (default "/github/home/.config/marblerun/coordinator-cert.pem") + --era-config string Path to remote attestation config file in json format, if none provided the newest configuration will be loaded from github + -i, --insecure Set to skip quote verification, needed when running in simulation mode + -n, --namespace string Kubernetes namespace of the MarbleRun installation (default "marblerun") +``` + +## marblerun recover + +Recover the MarbleRun Coordinator from a sealed state + +### Synopsis + +Recover the MarbleRun Coordinator from a sealed state + +``` +marblerun recover [flags] +``` + +### Examples + +``` +marblerun recover recovery_key_decrypted $MARBLERUN +``` + +### Options + +``` + -h, --help help for recover +``` + +### Options inherited from parent commands + +``` + --accepted-tcb-statuses strings Comma-separated list of user accepted TCB statuses (e.g. ConfigurationNeeded,ConfigurationAndSWHardeningNeeded) (default [UpToDate]) + --coordinator-cert string Path to MarbleRun Coordinator's root certificate to use for TLS connections (default "/github/home/.config/marblerun/coordinator-cert.pem") + --era-config string Path to remote attestation config file in json format, if none provided the newest configuration will be loaded from github + -i, --insecure Set to skip quote verification, needed when running in simulation mode + -n, --namespace string Kubernetes namespace of the MarbleRun installation (default "marblerun") +``` + +## marblerun package-info + +Print the package signature properties of an enclave + +### Synopsis + +Print the package signature properties of an enclave + +``` +marblerun package-info [flags] +``` + +### Options + +``` + -h, --help help for package-info +``` + +### Options inherited from parent commands + +``` + --accepted-tcb-statuses strings Comma-separated list of user accepted TCB statuses (e.g. ConfigurationNeeded,ConfigurationAndSWHardeningNeeded) (default [UpToDate]) + --coordinator-cert string Path to MarbleRun Coordinator's root certificate to use for TLS connections (default "/github/home/.config/marblerun/coordinator-cert.pem") + --era-config string Path to remote attestation config file in json format, if none provided the newest configuration will be loaded from github + -i, --insecure Set to skip quote verification, needed when running in simulation mode + -n, --namespace string Kubernetes namespace of the MarbleRun installation (default "marblerun") +``` + +## marblerun version + +Display version of this CLI and (if running) the MarbleRun Coordinator + +### Synopsis + +Display version of this CLI and (if running) the MarbleRun Coordinator + +``` +marblerun version [flags] +``` + +### Options + +``` + -h, --help help for version +``` + +### Options inherited from parent commands + +``` + --accepted-tcb-statuses strings Comma-separated list of user accepted TCB statuses (e.g. ConfigurationNeeded,ConfigurationAndSWHardeningNeeded) (default [UpToDate]) + --coordinator-cert string Path to MarbleRun Coordinator's root certificate to use for TLS connections (default "/github/home/.config/marblerun/coordinator-cert.pem") + --era-config string Path to remote attestation config file in json format, if none provided the newest configuration will be loaded from github + -i, --insecure Set to skip quote verification, needed when running in simulation mode + -n, --namespace string Kubernetes namespace of the MarbleRun installation (default "marblerun") +``` + diff --git a/docs/versioned_docs/version-1.4/reference/coordinator.md b/docs/versioned_docs/version-1.4/reference/coordinator.md new file mode 100644 index 00000000..1c505abf --- /dev/null +++ b/docs/versioned_docs/version-1.4/reference/coordinator.md @@ -0,0 +1,5 @@ +import ApiDocMdx from '@theme/ApiDocMdx'; + +# Coordinator Client API + + diff --git a/docs/versioned_docs/version-1.4/workflows/add-service.md b/docs/versioned_docs/version-1.4/workflows/add-service.md new file mode 100644 index 00000000..568362d1 --- /dev/null +++ b/docs/versioned_docs/version-1.4/workflows/add-service.md @@ -0,0 +1,167 @@ +# Adding a service + +Adding a service to your application requires three steps, which are described in the following. + +## **Step 1:** Get your service ready for MarbleRun + +To get your service ready for MarbleRun, you need to rebuild it with one of the supported [runtimes](../features/runtimes.md): +* [EGo](../building-marbles/ego.md) +* [Edgeless RT](https://github.com/edgelesssys/marblerun/blob/master/samples/helloc%2B%2B) +* [Gramine](../building-marbles/gramine.md) +* [Occlum](../building-marbles/occlum.md) + +### Make your service use the provided TLS credentials + +Skip this step, when using EGo with [TTLS](../features/transparent-TLS.md). + +Quick refresher: MarbleRun's Coordinator issues TLS credentials for each verified Marble (i.e., a service running in a secure enclave) as is described in the [secrets management chapter](../features/secrets-management.md). + +The TLS X.509 certificate and the corresponding private key can be securely passed to a service through files, environment variables, or command line arguments. This is defined in the manifest as is described in the [writing a manifest hands-on](../workflows/define-manifest.md#marbles). + +Make sure that your service reads the credentials from one of these sources, e.g., the file `/tmp/mycert.cert` or the environment variable `MY_PRIVATE_KEY`, and uses them for internal and external connections. If you're lucky, your service already does this and you don't need to change a thing in the code. + +## **Step 2:** Define your service in the manifest + +Now that your service is ready, you need to make two types of entries in the manifest regarding its properties and parameters. + +### **Step 2.1:** Define the enclave software package + +As is described in more detail in the [writing a manifest hands-on](../workflows/define-manifest.md#packages), the manifest contains a section `Packages`, in which allowed enclave software packages are defined. + +#### EGo / EdgelessRT +To add an entry for your EGo / EdgelessRT service, run the `oesign` tool on the enclave file you built in the previous step as follows. (`oesign` is installed with [Edgeless RT](https://github.com/edgelesssys/edgelessrt).) + +```bash +oesign eradump -e enclave.signed +``` + +The tool's output is similar to the following. + +```json +{ + "UniqueID": "6b2822ac2585040d4b9397675d54977a71ef292ab5b3c0a6acceca26074ae585", + "SignerID": "5826218dbe96de0d7b3b1ccf70ece51457e71e886a3d4c1f18b27576d22cdc74", + "SecurityVersion": 1, + "ProductID": 3 +} +``` + +#### Gramine + +To add an entry for your Gramine service, run the `gramine-sgx-get-token` tool on the `.sig` file you built in the previous step as follows. (`gramine-sgx-get-token` is installed with [Gramine](https://github.com/gramineproject/gramine/).) + + +```bash +gramine-sgx-get-token --sig hello.sig +``` + +The tool's output is similar to the following. + +```json +Attributes: + mr_enclave: 72612ea17be998f098459ff3cdc0daa0d592cec85115db8e152b10fc6df033a7 + mr_signer: ed81204cd726dcff2dc4c498bdfcef63a2b02009ef188e7e2914c37a7e99b547 + isv_prod_id: 1 + isv_svn: 3 + attr.flags: 0600000000000000 + attr.xfrm: 1f00000000000000 + misc_select: 00000000 + misc_mask: 00000000 + modulus: 09d6497ec75a05a2280974b7e5b39422... + exponent: 3 + signature: 4b6db90216e6a5e8447812f7f0107317... + date: 2021-08-18 +``` + +#### Occlum + +To add an entry for your Occlum service, run the MarbleRun CLI on the Occlum instance you built in the previous step as follows. + +```bash +marblerun sgxsdk-package-info ./occlum-instance +``` + +The output is similar to the following. + +```json +PackageProperties for Occlum image at './occlum-instance': +UniqueID (MRENCLAVE) : ccad2391e0b79d9108209135c26b2c276c5a24f4f55bc67ccf5ab90fd3f5fc22 +SignerID (MRSIGNER) : 83d719e77deaca1470f6baf62a4d774303c899db69020f9c70ee1dfc08c7ce9e +ProductID (ISVPRODID) : 1 +SecurityVersion (ISVSVN) : 3 +``` + + +Use `UniqueID` (i.e., `MRENCLAVE` in Intel SGX speak) or the triplet of `SignerID` (i.e., `MRSIGNER`), `SecurityVersion`, and `ProductID` to add an entry in the `Packages` section. + +### **Step 2.2:** Define the parameters + +Now you can define with which parameters (i.e., files, environment variables, and command line arguments) your service is allowed to run. This is done in the `Marbles` section of the manifest as is described in the [writing a manifest hands-on](../workflows/define-manifest.md#marbles). When using EGo, define all TTLS connections as described in the [manifest hands-on](../workflows/define-manifest.md#tls). + +Otherwise, as discussed in [Step #1](#make-your-service-use-the-provided-tls-credentials), make sure that the TLS credentials for your service (i.e., `MarbleRun.MarbleCert.Cert` and `MarbleRun.MarbleCert.Private`) are injected such that your service finds them at runtime. + +## **Step 3:** Start your service + +When you start your service, you need to pass in a couple of configuration parameters through environment variables. Here is an example: + +```bash +EDG_MARBLE_COORDINATOR_ADDR=coordinator-mesh-api.marblerun:2001 \ +EDG_MARBLE_TYPE=mymarble \ +EDG_MARBLE_UUID_FILE=$PWD/uuid \ +EDG_MARBLE_DNS_NAMES=localhost,myservice \ +erthost enclave.signed +``` + +`erthost` is the generic host for EdgelessRT Marbles, which will load your `enclave.signed`. +For EGo (`ego marblerun`), Gramine (`gramine-sgx`), and Occlum (`occlum run`) use their particular launch mechanism instead. + +The environment variables have the following purposes. + +* `EDG_MARBLE_COORDINATOR_ADDR` is the network address of the Coordinator's API for Marbles. When you deploy the Coordinator using the Helm repository as is described in the [deploying MarbleRun hands-on](../deployment/kubernetes.md), the default address is `coordinator-mesh-api.marblerun:2001`. + +* `EDG_MARBLE_TYPE` needs to reference one entry from your manifest's `Marbles` section. + +* `EDG_MARBLE_UUID_FILE` is the local file path where the Marble stores its UUID. Every instance of a Marble has its unique and public UUID. The file is needed to allow a Marble to restart under its UUID. + +* `EDG_MARBLE_DNS_NAMES` is the list of DNS names and IPs the Coordinator will issue the Marble's certificate for. + +## **Step 4:** Deploy your service with Kubernetes + +Typically, you'll write a Kubernetes resource definition for your service, which you'll deploy with the Kubernetes CLI, Helm, or similar tools. + +For your services to take advantage of MarbleRun, they need to be "added to the mesh" by having the data plane configuration injected into their pods. +This is typically done by labeling the deployment, or pod with the `marblerun/marbletype` Kubernetes label. +This label triggers automatic configuration injection when the resources are created. (See [auto injection](../features/kubernetes-integration.md) for more on how this works.) + +An example for a Marble of type `web` could look like this: + +```yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: web + namespace: emojivoto + labels: + app.kubernetes.io/name: web + app.kubernetes.io/part-of: emojivoto + app.kubernetes.io/version: v1 + marblerun/marbletype: web +``` + +This will result in the following configuration being injected when your resources are created: + +```yaml +spec: + containers: + - env: + - name: EDG_MARBLE_COORDINATOR_ADDR + value: coordinator-mesh-api.marblerun:2001 + - name: EDG_MARBLE_TYPE + value: web + - name: EDG_MARBLE_DNS_NAMES + value: "web,web.emojivoto,web.emojivoto.svc.cluster.local" + - name: EDG_MARBLE_UUID_FILE + value: "$PWD/uuid" +``` + +Refer to the [emojivoto](https://github.com/edgelesssys/emojivoto) app for complete Helm chart examples. diff --git a/docs/versioned_docs/version-1.4/workflows/define-manifest.md b/docs/versioned_docs/version-1.4/workflows/define-manifest.md new file mode 100644 index 00000000..f598b074 --- /dev/null +++ b/docs/versioned_docs/version-1.4/workflows/define-manifest.md @@ -0,0 +1,485 @@ +# Defining a manifest + +The manifest is a simple JSON file that determines the key properties of your cluster: [`Packages`](#packages), [`Marbles`](#marbles), [`Secrets`](#secrets), [`Users`](#users), [`Roles`](#roles), [`RecoveryKeys`](#recoverykeys), and [`TLS`](#tls). +This article describes how to define these in your `manifest.json`. + +For a working example see the manifest of the [emojivoto demo](https://github.com/edgelesssys/emojivoto/blob/main/tools/manifest.json). See also the [sample and template manifests](https://github.com/edgelesssys/marblerun/tree/master/samples). + +## Packages + +The `Packages` section of the manifest lists all the secure enclave software packages that your application uses. A package is defined by the following properties. + +* `UniqueID`: this value will pin this package to one specific release build of an application. It represents the globally unique ID of the enclave software package; on SGX, this corresponds to the `MRENCLAVE` value, which is the SHA-256 hash of the enclave's initial contents and its configuration. +* `SignerID`: this value limits MarbleRun to only accept releases signed by a given public key. On SGX, this corresponds to the `MRSIGNER` value, which is the SHA-256 hash of the enclave issuer's RSA-3072 public key. +* `ProductID`: an integer that uniquely identifies the enclave software for a given `SignerID`. Can only be used in conjunction with `SignerID`. +* `SecurityVersion`: an integer that reflects the security-patch level of the enclave software. Can only be used in conjunction with `SignerID`. +* `Debug`: set to `true` if the enclave is to be run in debug mode. This allows you to experiment with deploying your application with MarbleRun without having to worry about setting correct values for the above properties, but note that enclaves in debug mode aren't secure. +* `AcceptedTCBStatuses`: a list of acceptable TCB statuses a Marble is allowed to start with. You can use this option to allow Marbles to run on machines whose TCB is out-of-date. + +The following gives an example of a simple `Packages` section with made-up values. + +```javascript +{ + // ... + "Packages": { + "backend": { + "UniqueID": "6b2822ac2585040d4b9397675d54977a71ef292ab5b3c0a6acceca26074ae585", + "Debug": false, + "AcceptedTCBStatuses": [ + "ConfigurationNeeded", + "ConfigurationAndSWHardeningNeeded" + ] + }, + "frontend": { + "SignerID": "43361affedeb75affee9baec7e054a5e14883213e5a121b67d74a0e12e9d2b7a", + "ProductID": 43, + "SecurityVersion": 3, + "Debug": true + } + } + // ... +} +``` + +In this example, `backend` is identified through `UniqueID`. Since `UniqueID` is the hash of the enclave software package, this means that `backend` can't be updated. (That' s because any update to the package will change the hash.) + +In contrast, `frontend` is identified through the triplet `SignerID`, `ProductID`, and `SecurityVersion`. `SignerID` cryptographically identifies the vendor of the package; `ProductID` is an arbitrary product ID chosen by the vendor, and `SecurityVersion` is the security-patch level of the product. See our [adding a service hands-on](../workflows/add-service.md#step-21-define-the-enclave-software-package) on how to get these values for a given service. + +## Marbles + +Marbles represent the actual services in your mesh. They're defined in the `Marbles` section, which typically looks somewhat like the following example. + +```javascript +{ + // ... + "Marbles": { + "backendFirst": { + "Package": "backend", + "MaxActivations": 1, + "Parameters": { + "Files": { + "/tmp/defg.txt": "foo", + "/tmp/jkl.mno": "bar", + "/tmp/pqr.ust": { + "Data": "Zm9vCmJhcg==", + "Encoding": "base64", + "NoTemplates": true + } + }, + "Env": { + "IS_FIRST": "true", + "ROOT_CA": "{{ pem .MarbleRun.RootCA.Cert }}", + "MARBLE_CERT": "{{ pem .MarbleRun.MarbleCert.Cert }}", + "MARBLE_KEY": "{{ pem .MarbleRun.MarbleCert.Private }}" + }, + "Argv": [ + "--first", + "serve" + ] + }, + "TLS": [ + "backendFirstTLS" + ] + }, + "frontend": { + "Package": "frontend", + "Parameters": { + "Env": { + "ROOT_CA": "{{ pem .MarbleRun.RootCA.Cert }}", + "MARBLE_CERT": "{{ pem .MarbleRun.MarbleCert.Cert }}", + "MARBLE_KEY": "{{ pem .MarbleRun.MarbleCert.Private }}" + } + }, + "TLS": [ + "frontendTLS1", "frontendTLS2" + ] + } + } + //... +} +``` + +Each Marble corresponds to a [`Package`](#packages) and defines a set of optional [`TLS` Tags](#tls) and `Parameters`: + +* `Files`: Files and their contents +* `Env`: Environment variables +* `Argv`: Command line arguments + +### Files and Env +Entries for these types can be defined in two ways: +* By using a direct mapping of filename to content: `"": ""` +* By specifying an encoding for the content, and optionally, if the content contains templates: + ```javascript + "": { + "Data": "", + "Encoding": "", + "NoTemplates": true/false + } + ``` + * `Data`: The file content + * `Encoding`: Allows users to encode the `Data` field of the manifest. Marbles receive the decoded value. The following options are available: + * `string`: No encoding. This is the default + * `base64`: The manifest contains `Data` in [Base64](https://pkg.go.dev/encoding/base64). This can be useful to set content that can otherwise not be parsed in JSON format, or to avoid having to worry about correctly escaping newlines in a multi-line document + * `hex`: Same as `base64`, but [Hex Encoding](https://pkg.go.dev/encoding/hex) is used instead + * `NoTemplates`: If this flag is set, content in `Data` isn't processed for templates. Use this if your file contains [Go Templates](https://golang.org/pkg/text/template/) structures that shouldn't be interpreted by MarbleRun. + +### Argv +Command line arguments are defined as an array. Entries are passed to the Marble in order, with the first being `argv[0]`. +Usually, `argv[0]` is expected to be the name of the executable. +Templates aren't supported. + +The general format is the following: +```javascript +"Argv": [ + "" + "" + "" + //... + "" +] +``` + +### Templates + +`Parameters` are passed from the Coordinator to secure enclaves (i.e., Marbles) after successful initial remote attestation. In the remote attestation step, the Coordinator ensures that enclaves run the software defined in the `Packages` section. It's important to note that `Parameters` are only accessible from within the corresponding secure enclave. `Parameters` may contain arbitrary static data. However, they can also be used to securely communicate different types of dynamically generated cryptographic keys and certificates to Marbles. For this, we use [Go Templates](https://golang.org/pkg/text/template/) with the following syntax. + +`{{ }}` + +The following encoding types are available to both `Files` and `Env`: + +* `hex`: hex string +* `base64`: Base64 encoding +* `pem`: PEM encoding with a header matching the type of the requested key or certificate + + +The following encoding types are only available to `Files`: + +* `raw`: raw bytes + +The following encoding types are only available to `Env`: + +* `string`: similar to `raw`, but doesn't allow [NULL bytes](https://man7.org/linux/man-pages/man7/environ.7.html). Since the content in non-user-defined secrets is random, and can't be controlled, only secrets with `UserDefined` set to `true` are allowed to use this encoding. + +The following named keys and certificates are always available. + +* `.MarbleRun.RootCA.Cert`: the root certificate of the cluster issued by the Coordinator; this can be used to verify the certificates of all Marbles in the cluster. +* `.MarbleRun.MarbleCert.Cert`: the Marble's certificate; this is issued by the `.MarbleRun.RootCA.Cert` and is for Marble-to-Marble and Marble-to-client authentication. +* `.MarbleRun.MarbleCert.Private`: the Marble's private key corresponding to `.MarbleRun.MarbleCert.Cert` + +Finally, the optional field `MaxActivations` can be used to restrict the number of distinct instances that can be created of a Marble. + +## Secrets + +In the [previous section](#marbles), we discussed how certain cryptographic keys and certificates can be injected into a Marble's `Parameters` using Go Templates. In addition, MarbleRun also allows for the specification of custom cryptographic keys and certificates in the `Secrets` section. A typical `Secrets` section looks like the following. + +```javascript +{ + //... + "Secrets": { + "secretAESKey": { + "Type": "symmetric-key", + "Size": 128, + "Shared": true + }, + "rsaCert": { + "Type": "cert-rsa", + "Size": 2048, + "Shared": false, + "ValidFor": 7, + "Cert": { + "Subject": { + "CommonName": "MarbleRun Unit Test" + } + } + }, + "secretKeyUnset": { + "Type": "symmetric-key", + "Size": 128, + "UserDefined": true + } + } + //... +} +``` + +When defining a custom key or certificate, the following fields are available. + +* `Type`: can be either `symmetric-key` for a symmetric encryption key, `cert-rsa`, `cert-ecdsa`, `cert-ed25519` or `plain`. Secrets of type `plain` contain arbitrary data uploaded by users, and are never generated by the Coordinator. +* `Size`: the size of the key in bits. For symmetric keys, this needs to be a multiple of `8`. For ECDSA, this needs to map to a curve supported by Go's `crypto` library, currently: `224`, `256`, `384`, or `521`. For Ed25519, this should be omitted. +* `Shared` (default: `false`): specifies if the secret should be shared across all Marbles (`true`), or if the secret should be uniquely generated for each Marble (`false`). See [Secrets management](../features/secrets-management.md) for more info. +* `ValidFor` (only for certificates, default: `365`): validity of the certificate in days; can't be specified in combination with the `NotAfter`. +* `Cert` (only for certificates): allows for the specification of additional X.509 certificate properties. See below for details. +* `UserDefined` (default: `false`): specifies if the secret should be generated by MarbleRun (`false`), or if it will be [uploaded by a user](../workflows/managing-secrets.md) later (`true`). + +### Available `Cert` fields + +When specifying a custom certificate in the `Secrets` section, the following properties can be set. These map directly to Go's `x509.Certificate` structure. (This is because the Coordinator is written in Go.) + +```javascript +"Cert": { + "SignatureAlgorithm": 0, + "Subject": { + "Country": null, + "Organization": null, + "OrganizationalUnit": null, + "Locality": null, + "Province": null, + "StreetAddress": null, + "PostalCode": null, + "CommonName": "", + "Names": null, + "ExtraNames": null + }, + "NotAfter": "0001-01-01T00:00:00Z", + "KeyUsage": 0, + "ExtKeyUsage": null, + "UnknownExtKeyUsage": null, + "MaxPathLen": 0, + "MaxPathLenZero": false, + "SubjectKeyId": null, + "AuthorityKeyId": null, + "OCSPServer": null, + "IssuingCertificateURL": null, + "DNSNames": null, + "EmailAddresses": null, + "IPAddresses": null, + "URIs": null, + "PermittedDNSDomainsCritical": false, + "PermittedDNSDomains": null, + "ExcludedDNSDomains": null, + "PermittedIPRanges": null, + "ExcludedIPRanges": null, + "PermittedEmailAddresses": null, + "ExcludedEmailAddresses": null, + "PermittedURIDomains": null, + "ExcludedURIDomains": null, + "CRLDistributionPoints": null, + "PolicyIdentifiers": null + } +``` +Typically, you only define a subset of these. Commonly used properties include for example: +* `DNSNames` +* `IPAdresses` +* `KeyUsage` & `ExtKeyUsage` +* `Subject` (+ children) + +The following X.509 properties can't be specified because they're set by the Coordinator when creating a certificate. +* `Issuer`: always set to "MarbleRun Coordinator" +* `SerialNumber`: always set to a unique, random value +* `BasicConstraintsValid`: always set to "true" +* `NotBefore`: always set to the host time at creation + +### Injecting custom secrets + +Keys and certificates defined in the `Secrets` section can be injected via `Parameters` using the following syntax. + +`{{ .Secrets.. }}` + +Refer to the [previous section](#marbles) for a list of supported encodings. `` can be any of the following. + +* *empty*: for secret type `symmetric-key`, returns the symmetric key. For secret type `plain`, returns the secret data. For other types, returns the public key. +* `Cert`: returns the certificate. +* `Public`: returns the public key. +* `Private`: returns the private key. + +The following gives some examples. + +* Inject the certificate of custom secret `rsaCert` in PEM format: `{{ pem .Secrets.rsaCert.Cert }}` +* Inject the corresponding private key in PKCS#8 format: `{{ raw .Secrets.rsaCert.Private }}` +* Inject the corresponding public key PKIX-encoded and in PEM format: `{{ pem .Secrets.rsaCert.Public }}` +* Inject a symmetric key in hex format: `{{ hex .Secrets.secretAESKey }}` + +## Users +The optional entry `Users` defines user credentials and role bindings for authentication and access control. +Each user is authenticated via a client certificate. The certificate needs to be specified as a PEM-encoded self-signed X.509 certificate. +Users with the appropriate roles can [update a manifest](../workflows/update-manifest.md) and [read or write secrets](../workflows/managing-secrets.md). + +```javascript +{ + //... + "Users": { + "alice": { + "Certificate": "-----BEGIN CERTIFICATE-----\nMIIFPjCCA...", + "Roles": [ + "secretManager", + "updateFrontend" + ] + }, + "bob": { + "Certificate": "-----BEGIN CERTIFICATE-----\nMIIFP...", + "Roles": [ + "secretManager", + "updateBackend" + ] + } + } + //... +} +``` +When verifying certificates in this context, MarbleRun ignores their `issuer`, `subject`, and `expiration date` fields. Thus, users can't lock themselves out through expired certificates. + +Use OpenSSL to generate a compatible certificate. + +```bash +openssl req -x509 -newkey rsa:4096 -sha256 -days 3650 -nodes -keyout admin_private.key -out admin_certificate.crt +``` + +Use the following command to preserve newlines correctly: + +```bash +awk 'NF {sub(/\r/, ""); printf "%s\\n",$0;}' admin_certificate.pem +``` +## Roles + +MarbleRun supports Role-based access control (RBAC). +An RBAC Role represents a set of permissions for a MarbleRun `User`. Permissions are purely additive (there are no "deny" rules). +Each role defines a `ResourceType` (one of `Packages` or `Secrets`), a list of `ResourceNames` of that type, and a list of `Actions` that role permits on the listed resources. + +Valid `Actions` are: +* For `"ResourceType": "Secrets"`: `ReadSecret` and `WriteSecret`, allowing reading and writing a secret respectively +* For `"ResourceType": "Packages"`: `UpdateSecurityVersion`, allowing to update the `SecurityVersion` of a given package +* For `"ResourceType": "Manifest"`: `UpdateManifest`, allowing to update the full manifest (MarbleRun Enterprise only) + +:::note + +Assigning a role with the `UpdateManifest` action to multiple users enables [multi-party manifest update](../features/manifest.md#multi-party-update): each of the users can upload a manifest, but all other users must [acknowledge this manifest](update-manifest.md#acknowledging-a-multi-party-update). + +::: + +```javascript +{ + //... + "Roles": { + "updateFrontend": { + "ResourceType": "Packages", + "ResourceNames": ["frontend"], + "Actions": ["UpdateSecurityVersion"] + }, + "updateBackend": { + "ResourceType": "Packages", + "ResourceNames": ["backend"], + "Actions": ["UpdateSecurityVersion"] + }, + "secretManager": { + "ResourceType": "Secrets", + "ResourceNames": [ + "secretKeyUnset", + "genericSecret" + ], + "Actions": [ + "ReadSecret", + "WriteSecret" + ] + }, + "updateManifest": { + "ResourceType": "Manifest", + "Actions": ["UpdateManifest"] + } + } + //... +} +``` + +:::note + +Deployment updates will only be possible if you create a role with permissions to update the particular packages and bind a user to that role. +The same applies for setting user-defined secrets. + +::: + +## RecoveryKeys + +The optional entry `RecoveryKeys` holds PEM-encoded RSA public keys that can be used to [recover](../features/recovery.md) a failed MarbleRun deployment. + +```javascript +{ + //... + "RecoveryKeys": + { + "recoveryKey1": "-----BEGIN PUBLIC KEY-----\nMIIBpTANBgk..." + } + //... +} +``` + +You can generate this key with OpenSSL: + +```bash +openssl genrsa -out private_key.pem 4096 +openssl rsa -in private_key.pem -pubout -out public_key.pem +``` + +Use the following command to preserve newlines correctly: + +```bash +awk 1 ORS='\\n' public_key.pem +``` + +### Multi-party recovery + + + +To enable [multi-party recovery](../features/recovery.md#multi-party-recovery), first ask the other parties to generate key pairs as described above and receive their public keys via an authenticated channel. + +Add all public keys to the manifest: + +```javascript +{ + //... + "RecoveryKeys": + { + "admin": "-----BEGIN PUBLIC KEY-----\n...", + "dataProtectionOfficer": "-----BEGIN PUBLIC KEY-----\n...", + "collaborator": "-----BEGIN PUBLIC KEY-----\n..." + } + //... +} +``` + +## TLS + +:::note + +The Transparent TLS feature is currently only available for EGo and Edgeless RT Marbles. Gramine and Occlum aren't supported yet. + +::: + +The TLS entry holds a list of tags which can be used in a Marble's definition. Each tag can define multiple `Incoming` and `Outgoing` connections. To elevate the connection between two marbles to TLS, the client needs to set the server under `Outgoing` and the server needs to define its service under `Incoming`. + +Outgoing connections are defined by `Port` and `Addr`. For `Addr`, you can use both IP addresses and domains, e.g., the DNS names of other services. + +Incoming connections are defined by `Port`. For services used by external clients, you must disable client authentication by setting `DisableClientAuth` to `true` and set `Cert`. Use the name of a certificate defined in the [Secrets section](#secrets). + +```javascript +{ + //... + "TLS": + { + "frontendTLS1": { + "Outgoing": [ + { + "Port": "8080", + "Addr": "service.name" + }, + { + "Port": "4443", + "Addr": "10.111.37.164" + } + ], + "Incoming": [ + { + "Port": "8443" + }, + { + "Port": "8080", + "Cert": "rsaCert", + "DisableClientAuth": true + } + ] + }, + "backendFirstTLS": { + // ... + } + } +} +``` diff --git a/docs/versioned_docs/version-1.4/workflows/managing-secrets.md b/docs/versioned_docs/version-1.4/workflows/managing-secrets.md new file mode 100644 index 00000000..d31e43df --- /dev/null +++ b/docs/versioned_docs/version-1.4/workflows/managing-secrets.md @@ -0,0 +1,92 @@ +# Managing secrets + +Users might need to access secrets generated by the Coordinator or pass in secrets to MarbleRun themselves. + +For example, an application might require an access token, to establish secure connections to an external service. +While setting the token directly in the manifest is an option, this would mean exposing the confidential token to everyone with access to the manifest. + +For such cases, MarbleRun allows predefined users to upload secrets that have been predefined in the manifest. +Similarly, predefined users can retrieve particular secrets from the Coordinator via the `/secrets` endpoint. + + +## Reading a secret + +Authorized users can use the client API to read secrets in JSON format. + +For example, the following command requests a secret named `symmetricKeyShared`: +```bash +marblerun secret get symmetricKeyShared $MARBLERUN --cert=admin-cert.pem --key=admin-key.pem --era-config=era.json +``` +If the user has the required permissions, the secret is returned: +``` +symmetricKeyShared: + Type: symmetric-key + UserDefined: false + Size: 128 + Key: uVGpoJZTRICLccJiVNt9jA== +``` + +## Setting a secret + +A user can upload secrets via the HTTP REST API in JSON format. The secrets need to be marked as `UserDefined`. + +For secrets of the types `cert-rsa`, `cert-ecdsa` and `cert-ed25519` the CLI supports directly uploading a PEM encoded certificate and key from a file. +Take for example the `secret_file.pem` containing a certificate and corresponding private key: +``` +-----BEGIN CERTIFICATE----- +MIICpjC ... zrNxzg== +-----END CERTIFICATE----- + +-----BEGIN PRIVATE KEY----- +MIICeAI ... 8bu8Z+Fe +-----END PRIVATE KEY----- +``` +The CLI will look for the first instance of an encoded certificate and/or key in the file, perform a conversion to JSON format, and upload them to the Coordinator. +The name of the secret you are trying to set is specified using the `--from-pem` flag. + +The following command uploads the key and certificate pair stored in the `secret_file.pem` as a secret named `user-certificate`: +```bash +marblerun secret set secret_file.pem $MARBLERUN --from-pem user-certificate --cert=admin-cert.pem --key=admin-key.pem --era-config=era.json +``` + +To upload different types, or a batch of secrets, a file with the secrets in JSON format is needed. +Each entry in the file is a JSON object labeled by the name of the secret and contains values depending on its type: + +* For the types `cert-rsa`, `cert-ecdsa` and `cert-ed25519`: + * `Cert`: A PEM encoded certificate. + To preserve newlines correctly set `Cert` to the output of the following command: + ```bash + echo -n $(cat .pem | sed '1,1d;$ d' | awk 'NF {sub(/\r/, ""); printf "%s\\n",$0;}') + ``` + * `Private`: A PEM encoded private key. This may be omitted. + To preserve newlines correctly set `Private` to the output of the following command: + ```bash + echo -n $(cat .pem | sed '1,1d;$ d' | awk 'NF {sub(/\r/, ""); printf "%s\\n",$0;}') + ``` +* For type `symmetric-key`: + * Key: a base64 encoded key, with length matching the size defined in the original manifest +* For the type `plain`: + * `Key`: arbitrary data in base64 encoding. This may be a key of unspecified length, a confidential config file, or any kind of data only some Marbles and/or users should have access to. + +The following gives an example for a file to set the secrets `userDefinedSymmetricKey`, `userDefinedCert`, and `genericSecret`: + +```javascript +{ + "userDefinedSymmetricKey": { + "Key": "AAECAwQFBgcICQoLDA0ODw==" + }, + "userDefinedCert": { + "Cert": "MIIBjDCCATOgAwIBAgICBTkwCgYIKoZIzj0EAwIwMjEwMC4GA1UEAxMnTWFyYmxlcnVuIENvb3JkaW5hdG9yIC0gSW50ZXJtZWRpYXRlIENBMB4XDTIxMDYxNTA4NTY0M1oXDTIxMDYyMjA4NTY0M1owLTEcMBoGA1UEAxMTTWFyYmxlcnVuIFVuaXQgVGVzdDENMAsGA1UEBRMEMTMzNzAqMAUGAytlcAMhAEPOc066G5XmvLizOKTENSR+U9lv3geZ0/a2+XkhJRvDo20wazAOBgNVHQ8BAf8EBAMCAoQwHQYDVR0lBBYwFAYIKwYBBQUHAwEGCCsGAQUFBwMCMAwGA1UdEwEB/wQCMAAwLAYDVR0RBCUwI4IJbG9jYWxob3N0hwR/AAABhxAAAAAAAAAAAAAAAAAAAAABMAoGCCqGSM49BAMCA0cAMEQCIGOlRcynaPaj/flSr2ZEvmTmhuvtmTb4QkwPFtxFz3EJAiB77ijxAcJNxPKcKmgMB+c8NORC+6N/St2iP/oX/vqQvg==", + "Private": "MC4CAQAwBQYDK2VwBCIEIPlmAOOhAStk8ytxzvekPr8zLaQa9+lxnHK+CizDrMds" + }, + "genericSecret": { + "Key": "SGVsbG8gZnJvbSB0aGUgTWFyYmxlcnVuIERvY3MhCg==" + } +} +``` + +This file can then be uploaded using the CLI and the users credentials: +```bash +marblerun secret set secret_file.json $MARBLERUN --cert=admin-cert.pem --key=admin-key.pem --era-config=era.json +``` +Subsequently, the secrets can be accessed by marbles and authorized users. If the user lacks permission to set one of the secrets, or a secret the user tried to upload was malformed an error occurs instead. diff --git a/docs/versioned_docs/version-1.4/workflows/monitoring.md b/docs/versioned_docs/version-1.4/workflows/monitoring.md new file mode 100644 index 00000000..5d90566a --- /dev/null +++ b/docs/versioned_docs/version-1.4/workflows/monitoring.md @@ -0,0 +1,54 @@ +# Monitoring and logging + + +## /status endpoint of client API + +For status information, the Coordinator provides the `/status` endpoint in the client API. +It returns the following information: + +- **1 recovery mode**: Either upload a key to unseal the saved state, or set a new manifest. Waiting for user input on [`/recover`](../features/recovery.md). +- **2 waiting for a manifest**: Coordinator is ready to accept a manifest. Waiting for user input on [`/manifest`](../workflows/set-manifest.md) +- **3 accepting marbles**: Coordinator is running correctly and ready to accept Marbles through the [Marble API](../workflows/add-service.md) + +## Kubernetes logs + +If you are running on a Kubernetes cluster, details about the Coordinator can be retrieved through its log as follows. + +```bash +kubectl -n marblerun logs -f marblerun-coordinator-xxxxxxxxxx-xxxxx +``` + +## Prometheus + +To enable the Coordinators [Prometheus](https://prometheus.io/) instrumentation, set an environment variable with the address the Prometheus exporter should run at: + +```bash +EDG_COORDINATOR_PROMETHEUS_ADDR=0.0.0.0:9944 +``` + +The exporter then serves the `/metrics` endpoint under that address. + +Here are the most important metrics for your monitoring setup: + +- The state of the Coordinator. See the listing above for interpretation of status codes. + ```bash + coordinator_state 1 + ``` + +- Build and version information of the Coordinator. + ```bash + coordinator_version_info{commit="bd505dca04f78da57e2330263f9076839d906267",version="0.4.0"} 0 + ``` + +- Marble activations are labeled with the Marbles type and UUID. + ```bash + marble_activations_total{type="frontend",uuid="4aac36cc-bbea-4db3-89ff-078ccad738f6"} 1 + marble_activations_success_total{type="frontend",uuid="4aac36cc-bbea-4db3-89ff-078ccad738f6"} 1 + ``` + +- gRPC metrics of the Marble API with prefix `grpc_server` + +- HTTP metrics of the client API with prefix `server_client_api_http` + + +Going forward, plans are having a web dashboard with status and health information accessible. diff --git a/docs/versioned_docs/version-1.4/workflows/recover-coordinator.md b/docs/versioned_docs/version-1.4/workflows/recover-coordinator.md new file mode 100644 index 00000000..23da81b0 --- /dev/null +++ b/docs/versioned_docs/version-1.4/workflows/recover-coordinator.md @@ -0,0 +1,136 @@ +# Recovering the Coordinator + +Different situations require the [recovery](../features/recovery.md) of the Coordinator. +If the Coordinator fails to unseal its state, it will enter recovery mode. + +:::tip + +Use [distributed Coordinator instances](../features/recovery.md#distributed-coordinator) to reduce the occurrence of events that require manual recovery. + +::: + +You need the corresponding private key to the [`RecoveryKeys` defined in the manifest](define-manifest.md#recoverykeys) and the [recovery secret returned to you during the initial upload of the manifest](set-manifest.md). + +:::info + +If you don't want or can't recover the old state, you can also dismiss it by [uploading a new manifest](set-manifest.md). +The old state will be overwritten on disk, and the `/recover` endpoint won't be available anymore. + +::: + +You can upload the recovery secret through the `/recover` client API endpoint. To do so, you need to: + +* Decode the Base64-encoded output returned to you during the initial upload of the manifest +* Decrypt the decoded output with the corresponding RSA private key of the key defined in the manifest +* Get the temporary root certificate (valid only during recovery mode) +* Upload the decrypted key to the `/recover` endpoint + +Assuming you saved the output from the manifest upload step in a file called `recovery_data` and the recovery private key in a file called `private_key.pem`, perform recovery like this: + +```bash +base64 -d recovery_data \ + | openssl pkeyutl -inkey private_key.pem -decrypt \ + -pkeyopt rsa_padding_mode:oaep -pkeyopt rsa_oaep_md:sha256 \ + -out recovery_key_decrypted +``` + +You can then upload the extracted secret using the MarbleRun CLI: + +```bash +marblerun recover recovery_key_decrypted $MARBLERUN +``` + +Alternatively, you can use `curl`: +```bash +era -c coordinator-era.json -h $MARBLERUN -output-root marblerun-temp.pem +curl --cacert marblerun-temp.pem --data-binary @recovery_key_decrypted https://$MARBLERUN/recover +``` + +On success, the Coordinator applies the sealed state again. If the Coordinator can't restore the state with the uploaded key, an error will be returned in the logs, and the `/recover` endpoint will stay open for further interaction. + +## Multi-party recovery + + + +If you've [configured your MarbleRun deployment for multi-party recovery](define-manifest.md#multi-party-recovery), send each party the corresponding [recovery secret](set-manifest.md). Ask them to perform the steps above. Once all parties have uploaded their secrets, the Coordinator recovers the sealed state and continues its operations. + +:::note + +If your MarbleRun deployment uses [distributed Coordinator instances](../features/recovery.md#distributed-coordinator), make sure all parties send their secrets to the same Coordinator instance. +One way to achieve this is scaling the Coordinator to a single instance: + +```bash +kubectl scale --replicas=1 -n marblerun deployment/marblerun-coordinator +``` + +After recovery, you can scale the Coordinator back to the desired number of instances. + +::: + +### Example + +The following gives an example of a multi-party recovery workflow. + +Assume the following `RecoveryKeys` were set in the manifest: + +```javascript + "RecoveryKeys": { + "alice": "-----BEGIN PUBLIC KEY-----\nMIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAk/6gfFF+cbcTlj8MT+4M\njjpM+suTwNM9gjv47EAAQ==\n-----END PUBLIC KEY-----\n", + "bob": "-----BEGIN PUBLIC KEY-----\nMIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAsnOEAvynVrbgLdp0lwcp\nk2k04+n4op6tp1Yw2OaDbEAAQ==\n-----END PUBLIC KEY-----\n" + } +``` + +1. The Coordinator returned the following `RecoverySecrets`: + + ```shell-session + $ marblerun manifest set manifest.json $MARBLERUN + ... + {"RecoverySecrets":{"alice":"EbkX/skIPrJISf8PiXdzRIKnwQyJ+VejtGzHGfES5NIPuCeEFedqgCVDk=","bob":"bvPzio4A4SvzeHajsb+dFDpDarErcU9wMR0V9hyHtG2lC4ZfyrYjDBE7wtis3eOPgDaMG/HCt="}} + ``` + +2. Alice decrypts and uploads her recovery key using her private key `private_key.pem` as follows: + + ```shell-session + $ marblerun status $MARBLERUN + 1: Coordinator is in recovery mode. Either upload a key to unseal the saved state, or set a new manifest. For more information on how to proceed, consult the documentation. + $ echo "EbkX/skIPrJISf8PiXdzRIKnwQyJ+VejtGzHGfES5NIPuCeEFedqgCVDk=" > recovery_data + $ base64 -d recovery_data \ + | openssl pkeyutl -inkey private_key.pem -decrypt \ + -pkeyopt rsa_padding_mode:oaep -pkeyopt rsa_oaep_md:sha256 \ + -out recovery_key_decrypted + $ marblerun recover recovery_key_decrypted $MARBLERUN + Successfully verified Coordinator, now uploading key + Secret was processed successfully. Upload the next secret. Remaining secrets: 1 + ``` + + The Coordinator log shows the following: + + ```json + {"level":"info","ts":1674206799.524596,"caller":"clientapi/clientapi.go:234","msg":"Recover called"} + {"level":"info","ts":1674206799.524596,"caller":"clientapi/clientapi.go:253","msg":"Recover: recovery incomplete, more keys needed","remaining":1} + ``` + +3. Bob does the same with his key to complete the recovery procedure: + + ```shell-session + $ marblerun status $MARBLERUN + 1: Coordinator is in recovery mode. Either upload a key to unseal the saved state, or set a new manifest. For more information on how to proceed, consult the documentation. + $ echo "bvPzio4A4SvzeHajsb+dFDpDarErcU9wMR0V9hyHtG2lC4ZfyrYjDBE7wtis3eOPgDaMG/HCt=" > recovery_data + $ base64 -d recovery_data \ + | openssl pkeyutl -inkey private_key.pem -decrypt \ + -pkeyopt rsa_padding_mode:oaep -pkeyopt rsa_oaep_md:sha256 \ + -out recovery_key_decrypted + $ marblerun recover recovery_key_decrypted $MARBLERUN + Successfully verified Coordinator, now uploading key + Recovery successful. + $ marblerun status $MARBLERUN + 3: Coordinator is running correctly and ready to accept marbles. + ``` + + The Coordinator log shows the following: + + ```json + {"level":"info","ts":1674206836.5523517,"caller":"clientapi/clientapi.go:234","msg":"Recover called"} + {"level":"info","ts":1674206836.5563517,"caller":"core/core.go:261","msg":"generating quote"} + {"level":"info","ts":1674206836.6043515,"caller":"clientapi/clientapi.go:281","msg":"Recover successful"} + ``` diff --git a/docs/versioned_docs/version-1.4/workflows/set-manifest.md b/docs/versioned_docs/version-1.4/workflows/set-manifest.md new file mode 100644 index 00000000..924dbe5b --- /dev/null +++ b/docs/versioned_docs/version-1.4/workflows/set-manifest.md @@ -0,0 +1,30 @@ +# Setting a manifest + +After you've [defined a manifest](define-manifest.md), use it to initialize your MarbleRun deployment. + +Set the manifest using the MarbleRun CLI: + +```bash +marblerun manifest set manifest.json $MARBLERUN +``` + +The command first performs [remote attestation on the Coordinator](../features/attestation.md#coordinator-deployment) before uploading the manifest. +If successful, the TLS root certificate of the Coordinator is saved for future connections with the MarbleRun instance. +This ensures you are always talking to the same instance the manifest was uploaded to. + +:::info + +By default the certificate is saved to `$XDG_CONFIG_HOME/marblerun/coordinator-cert.pem`, +or `$HOME/.config/marblerun/coordinator-cert.pem` if `$XDG_CONFIG_HOME` is not set. +Subsequent CLI commands will try loading the certificate from that location. +Use the `--coordinator-cert` flag to choose your own location to save or load the certificate. + +::: + +If the manifest contains a `RecoveryKeys` entry, you will receive a JSON reply including the recovery secrets, encrypted with the public keys supplied in `RecoveryKeys`. + +:::caution + +Keep these values somewhere safe. Without it, you can't [recover the Coordinator](../features/recovery.md). + +::: diff --git a/docs/versioned_docs/version-1.4/workflows/update-manifest.md b/docs/versioned_docs/version-1.4/workflows/update-manifest.md new file mode 100644 index 00000000..a5aae876 --- /dev/null +++ b/docs/versioned_docs/version-1.4/workflows/update-manifest.md @@ -0,0 +1,176 @@ +# Updating a manifest + +When [defining the initial manifest](define-manifest.md), you have to choose whether it should be [partially or fully updatable](../features/manifest.md) by assigning appropriate [roles](define-manifest.md#roles) to eligible users. + +## Package updates + +Updates play an important role in ensuring your software stays secure. To avoid redeploying your application from scratch, MarbleRun allows uploading a separate "update manifest" that increases the minimum `SecurityVersion` of already deployed packages. After such an update is performed, an old version of a defined software package can't be loaded anymore under the current manifest. + +To deploy an update, your user needs to have a [role assigned that contains the `UpdateSecurityVersion` action](define-manifest.md#roles). + +### Defining an update manifest +The update manifest format follows the original manifest's syntax, but it only contains packages with new `SecurityVersion` values. + +For example, the current `Packages` section of your original manifest looks like this: + +```javascript +{ + // ... + "Packages": { + "backend": { + "UniqueID": "6b2822ac2585040d4b9397675d54977a71ef292ab5b3c0a6acceca26074ae585", + "Debug": false + }, + "frontend": { + "SignerID": "43361affedeb75affee9baec7e054a5e14883213e5a121b67d74a0e12e9d2b7a", + "ProductID": 43, + "SecurityVersion": 3, + "Debug": true + } + } + // ... +} +``` + +To update the minimum required version for `frontend`, the complete definition for the update manifest just needs to be: + +```javascript +{ + "Packages": { + "frontend": { + "SecurityVersion": 5 + } + } +} +``` + +Don't define other values except the `SecurityVersion` value for a package, as MarbleRun refuses to accept such an update manifest. + +## Full update + + + +Some deployment scenarios require more flexibility regarding changes to the manifest. To this end, MarbleRun also allows uploading a full manifest. User-defined secrets and secrets of type `symmetric-key` are retained if their definition doesn't change. + +To deploy a new manifest, your user must have a [role assigned that contains the `UpdateManifest` action](define-manifest.md#roles). +If multiple users have such a role assigned, each of them must [acknowledge](#acknowledging-a-multi-party-update) the new manifest. + +## Deploying an update + +Use the CLI to deploy an update, specifying the client certificate and private key of a user with appropriate permissions: + +```bash +marblerun manifest update apply update-manifest.json $MARBLERUN --cert=user-cert.pem --key=user-key.pem +``` + +On success, no message will be returned and your MarbleRun logs should highlight whether the update manifest has been set or further acknowledgements are required. On error, the API endpoint will return an error message. If you receive `unauthorized user`, MarbleRun either received no client certificate over the TLS connection, or you used the wrong certificate. + +## Acknowledging a multi-party update + +All users that have the `UpdateManifest` permission are required to acknowledge a full manifest update. +To this end, they need to upload the same manifest. +This proves that they have knowledge of and agree on this manifest. + +Use the CLI to acknowledge the update: + +```bash +marblerun manifest update acknowledge update-manifest.json $MARBLERUN --cert=user-cert.pem --key=user-key.pem +``` + +See the [CLI reference](../reference/cli.md#marblerun-manifest-update) for further operations like getting the deployed update manifest or canceling the update procedure. + +## Example: Multi-party update + +The following gives an example of a full manifest update with multiple parties. + +Assume the following `Users` and `Roles` were defined in the manifest: + +```javascript + "Users": { + "alice": { + "Certificate": "-----BEGIN CERTIFICATE-----\nMIIFZTCCA02gAwIBAgIUANHwS8RM0PUDl9htA+yWJx9WqucwDQYJKoZIhvcNAQEL\nBQAwQjELMAkGA1UEBhMC=\n-----END CERTIFICATE-----\n", + "Roles": [ + "UpdateManifest" + ] + }, + "bob": { + "Certificate": "-----BEGIN CERTIFICATE-----\nMIIFZTCCA02gAwIBAgIUJvtF7KRsunTmWVtpU9198HUxyLEwDQYJKoZIhvcNAQEL\nBQAwQjELMAkGA1UEBhMC=\n-----END CERTIFICATE-----\n", + "Roles": [ + "UpdateManifest" + ] + } + }, + "Roles": { + "UpdateManifest": { + "ResourceType": "Manifest", + "Actions": ["UpdateManifest"] + } + } +``` + +1. Alice applies an update by uploading an update manifest: + + ```shell-session + $ marblerun manifest update apply new_manifest.json $MARBLERUN --cert alice.crt --key alice-private.pem + Coordinator verified + Successfully verified Coordinator, now uploading manifest + Update manifest set successfully + ``` + + The Coordinator log shows the following: + + ```json + {"level":"info","ts":1674205619.1967707,"caller":"clientapi/clientapi.go:199","msg":"UpdateManifest called"} + {"level":"info","ts":1674205619.2007706,"caller":"clientapi/clientapi.go:282","msg":"UpdateManifest successful. Waiting for acknowledgments to apply the update","missingAcknowledgments":1} + ``` + +2. Bob checks the planned update: + + ```shell-session + $ marblerun manifest update get $MARBLERUN + Successfully verified Coordinator + { + "Marbles": { + ... + } + ``` + +3. Finally, Bob acknowledges the update by providing the updated manifest again. Alice can either distribute the update manifest to Bob via a second channel, or Bob uses the manifest obtained from `marblerun manifest update get`: + + ```shell-session + $ marblerun manifest update acknowledge new_manifest.json $MARBLERUN --cert bob.crt --key bob-private.pem + Coordinator verified + Successfully verified Coordinator + Acknowledgement successful: All users have acknowledged the update manifest. Update successfully applied + ``` + + The Coordinator log shows the following: + + ```json + {"level":"info","ts":1674205860.3970933,"caller":"clientapi/update.go:67","msg":"Received update acknowledgement","user":"bob","missingAcknowledgments":0} + {"level":"info","ts":1674205860.3970933,"caller":"clientapi/update.go:72","msg":"All users have acknowledged the update manifest, applying update"} + ... + {"level":"info","ts":1674205321.4204075,"caller":"clientapi/update.go:176","msg":"An updated manifest overriding the original manifest was set."} + {"level":"info","ts":1674205321.4204075,"caller":"clientapi/update.go:177","msg":"Please restart your Marbles to enforce the update."} + {"level":"info","ts":1674205321.4204075,"caller":"clientapi/update.go:183","msg":"UpdateManifest successful"} + ``` + +4. Alternatively, Alice or Bob may decide to cancel the update procedure instead: + + ```shell-session + $ marblerun manifest update cancel $MARBLERUN --cert bob.crt --key bob-private.pem + Coordinator verified + Loading client certificate + Creating client + Successfully verified Coordinator + Cancellation successful + ``` + + The Coordinator log shows the following: + + ```json + {"level":"info","ts":1674205264.1442728,"caller":"clientapi/update.go:223","msg":"Manifest update canceled","user":"bob"} + ``` + +## Effects of an update +When a manifest has been updated, the Coordinator will generate new certificates which your Marbles will receive upon the next startup. Also, if you are trying to launch Marbles based on packages containing the old `SecurityVersion`, they will refuse to run (unless you are running in SGX Simulation or non-Enclave mode). However, so far currently running Marbles will continue to run and will be able to authenticate each other, as long as they're still running. So if you need to enforce an update, make sure to kill the Marbles on your host and restart them. diff --git a/docs/versioned_docs/version-1.4/workflows/updates.md b/docs/versioned_docs/version-1.4/workflows/updates.md new file mode 100644 index 00000000..1ef32a3f --- /dev/null +++ b/docs/versioned_docs/version-1.4/workflows/updates.md @@ -0,0 +1,56 @@ +# Updating a deployment + +The following gives a walkthrough of typical deployment updates in a Kubernetes cluster and how to handle them with MarbleRun. + +## Updating to a new MarbleRun version + +When updating to a new MarbleRun version, updates to both the control plane and data plane components may be required. + +### Updating the Coordinator + +Updating the Coordinator follows the regular steps for [updating a deployment in Kubernetes](https://kubernetes.io/docs/concepts/workloads/controllers/deployment/#updating-a-deployment). + +```bash +kubectl -n marblerun set image deployment/marblerun-coordinator coordinator=ghcr.io/edgelesssys/marblerun/coordinator:latest --record +``` + +You can also use Helm to upgrade the image. Note that Helm requires you to pass all the flags with `upgrade` that you set during the initial deployment. + +```bash +helm upgrade marblerun edgeless/marblerun \ + -n marblerun \ + --set coordinator.version=latest +``` + +:::caution + +If you've deployed only [one Coordinator instance](../features/recovery.md#single-coordinator) and it gets rescheduled to another node during the updating process, you need to perform the [recovery step](../features/recovery.md). + +::: + +The Marbles won't be affected by the Coordinator update and will continue running. +New Marbles that are started can interact with existing ones. +If the Coordinator version update also affects the data plane or the Marble bootstrapping process, you will need to restart your Marbles. + +```bash +kubectl rollout restart deployment your_deployment_name +``` + +:::info + +Updating the Coordinator image will change its `UniqueID`, but typically not its `SignerID`. + +::: + +### Updating Marbles + +The tight integration of MarbleRun's data plane with your application requires a rebuild of your application and the corresponding containers. +Note that you only need to update if there have been changes affecting the data plane. +The process is similar to a regular update of your application, as is described in the following. + +## Updating your confidential application + +Updating your application is straightforward with MarbleRun. +You can roll out the new version similar to a regular [deployment update](https://kubernetes.io/docs/concepts/workloads/controllers/deployment/#updating-a-deployment). +The Coordinator will manage the attestation and bootstrapping of your new version. +Notably, nothing changes on the client-side of MarbleRun, the version update is transparent and adherent to the manifest in effect. diff --git a/docs/versioned_docs/version-1.4/workflows/verification.md b/docs/versioned_docs/version-1.4/workflows/verification.md new file mode 100644 index 00000000..15bdf81b --- /dev/null +++ b/docs/versioned_docs/version-1.4/workflows/verification.md @@ -0,0 +1,40 @@ +# Verifying a deployment + +The Coordinator provides an API for clients to verify the identity and integrity of the Coordinator itself and the deployed Marbles. + +Specifically, the Coordinator exposes the `/quote` endpoint that returns a quote and a certificate chain consisting of a root CA and an intermediate CA. The root CA is fixed for the lifetime of your deployment, while the intermediate CA changes in case you [update](../workflows/update-manifest.md) the packages specified in your manifest. The Coordinator also makes the effective manifest available via the `/manifest` endpoint. In TLS connections with this endpoint, the Coordinator uses its root CA and intermediate CA. Learn more about this concept in the [Attestation](../features/attestation.md) section. + +## Verifying the quote and the manifest using the CLI + +The `marblerun manifest verify` command uses the two endpoints described above. It first verifies the Coordinator's quote according to a given policy and then checks that the expected `manifest.json` is in effect. + +:::info + +You need to [install and configure a quote provider](../getting-started/installation.md#install-the-marblerun-cli) before you can use the command. + +::: + +The policy includes the Coordinator's `UniqueID` or the tuple `ProductID`, `SecurityVersion`, and `SignerID`. `UniqueID` and `SignerID` are also known as `MRENCLAVE` and `MRSIGNER` in SGX terminology. The policy for a given Coordinator is generated at build time and written to a file named `coordinator-era.json`. This file ships with every release of MarbleRun. You can find the policy file for the latest MarbleRun release at `https://github.com/edgelesssys/marblerun/releases/latest/download/coordinator-era.json` + +The command is used as follows: + +```bash +marblerun manifest verify manifest.json $MARBLERUN +``` + +If successful, the certificates of the root CA and the intermediate CA are saved for future connections. This ensures you are always talking to the same verified instance. + +:::info + +By default, the command will save the Coordinators certificate chain to `$XDG_CONFIG_HOME/marblerun/coordinator-cert.pem`, +or `$HOME/.config/marblerun/coordinator-cert.pem` if `$XDG_CONFIG_HOME` is not set. +Subsequent CLI commands will try loading the certificate from that location. +Use the `--coordinator-cert` flag to choose your own location to save or load the certificate. + +::: + +:::info + +The flag `--era-config` lets you optionally specify a custom policy for the verification of the quote. See the [documentation of the command](../reference/cli.md#marblerun-manifest-verify) for details. + +::: diff --git a/docs/versioned_sidebars/version-1.4-sidebars.json b/docs/versioned_sidebars/version-1.4-sidebars.json new file mode 100644 index 00000000..3eb9e1c3 --- /dev/null +++ b/docs/versioned_sidebars/version-1.4-sidebars.json @@ -0,0 +1,247 @@ +{ + "docs": [ + { + "type": "doc", + "label": "Introduction", + "id": "intro" + }, + { + "type": "category", + "label": "Getting started", + "link": { + "type": "generated-index" + }, + "items": [ + { + "type": "doc", + "label": "Installation", + "id": "getting-started/installation" + }, + { + "type": "doc", + "label": "First steps", + "id": "getting-started/quickstart" + }, + { + "type": "doc", + "label": "Examples", + "id": "getting-started/examples" + } + ] + }, + { + "type": "category", + "label": "Features", + "link": { + "type": "generated-index" + }, + "items": [ + { + "type": "doc", + "label": "Manifest", + "id": "features/manifest" + }, + { + "type": "doc", + "label": "Attestation", + "id": "features/attestation" + }, + { + "type": "doc", + "label": "State and recovery", + "id": "features/recovery" + }, + { + "type": "doc", + "label": "Secrets management", + "id": "features/secrets-management" + }, + { + "type": "doc", + "label": "Transparent TLS", + "id": "features/transparent-TLS" + }, + { + "type": "doc", + "label": "Kubernetes integration", + "id": "features/kubernetes-integration" + }, + { + "type": "doc", + "label": "Supported runtimes", + "id": "features/runtimes" + } + ] + }, + { + "type": "category", + "label": "Deployment", + "link": { + "type": "generated-index" + }, + "items": [ + { + "type": "doc", + "label": "Kubernetes", + "id": "deployment/kubernetes" + }, + { + "type": "doc", + "label": "Standalone", + "id": "deployment/standalone" + }, + { + "type": "category", + "label": "Platforms", + "link": { + "type": "doc", + "id": "deployment/platforms/platforms" + }, + "items": [ + { + "type": "doc", + "label": "Azure", + "id": "deployment/platforms/azure" + }, + { + "type": "doc", + "label": "Alibaba Cloud", + "id": "deployment/platforms/alibaba" + }, + { + "type": "doc", + "label": "On-premises", + "id": "deployment/platforms/on-prem" + } + ] + } + ] + }, + { + "type": "category", + "label": "Workflows", + "link": { + "type": "generated-index" + }, + "items": [ + { + "type": "doc", + "label": "Define a manifest", + "id": "workflows/define-manifest" + }, + { + "type": "doc", + "label": "Set a manifest", + "id": "workflows/set-manifest" + }, + { + "type": "doc", + "label": "Add a service", + "id": "workflows/add-service" + }, + { + "type": "doc", + "label": "Verify a deployment", + "id": "workflows/verification" + }, + { + "type": "doc", + "label": "Monitoring and logging", + "id": "workflows/monitoring" + }, + { + "type": "doc", + "label": "Update a manifest", + "id": "workflows/update-manifest" + }, + { + "type": "doc", + "label": "Manage secrets", + "id": "workflows/managing-secrets" + }, + { + "type": "doc", + "label": "Update a deployment", + "id": "workflows/updates" + }, + { + "type": "doc", + "label": "Recover the Coordinator", + "id": "workflows/recover-coordinator" + } + ] + }, + { + "type": "category", + "label": "Architecture", + "link": { + "type": "generated-index" + }, + "items": [ + { + "type": "doc", + "label": "Concepts", + "id": "architecture/concepts" + }, + { + "type": "doc", + "label": "Coordinator", + "id": "architecture/coordinator" + }, + { + "type": "doc", + "label": "Marbles", + "id": "architecture/marbles" + }, + { + "type": "doc", + "label": "Keys and cryptography", + "id": "architecture/security" + } + ] + }, + { + "type": "category", + "label": "Building Marbles", + "link": { + "type": "generated-index" + }, + "items": [ + { + "type": "doc", + "label": "EGo", + "id": "building-marbles/ego" + }, + { + "type": "doc", + "label": "Gramine", + "id": "building-marbles/gramine" + }, + { + "type": "doc", + "label": "Occlum", + "id": "building-marbles/occlum" + } + ] + }, + { + "type": "category", + "label": "Reference", + "link": { + "type": "generated-index" + }, + "items": [ + { + "type": "doc", + "label": "Coordinator client API", + "id": "reference/coordinator" + }, + { + "type": "doc", + "label": "CLI", + "id": "reference/cli" + } + ] + } + ] +} diff --git a/docs/versions.json b/docs/versions.json index bde1309c..8926c282 100644 --- a/docs/versions.json +++ b/docs/versions.json @@ -1,4 +1,5 @@ [ + "1.4", "1.3", "1.2", "1.1"