From 65ee7e1acccd13dcb515058e71c5f8bfb4281e35 Mon Sep 17 00:00:00 2001 From: Xynnn007 Date: Wed, 24 Apr 2024 17:27:04 +0800 Subject: [PATCH 1/9] KBS: fix session status When an attested session sends a new attest request, the session should not do anything else but only return the old token if it is still valid Close #374 Signed-off-by: Xynnn007 --- kbs/src/api/src/http/attest.rs | 27 +++++++++++++++++++++++---- kbs/src/api/src/session.rs | 4 +++- 2 files changed, 26 insertions(+), 5 deletions(-) diff --git a/kbs/src/api/src/http/attest.rs b/kbs/src/api/src/http/attest.rs index 0484336e74..320123ce8c 100644 --- a/kbs/src/api/src/http/attest.rs +++ b/kbs/src/api/src/http/attest.rs @@ -52,13 +52,31 @@ pub(crate) async fn attest( let session = session.get(); debug!("Session ID {}", session.id()); - let attestation_str = serde_json::to_string_pretty(&attestation.0) - .map_err(|_| Error::AttestationFailed("Failed to serialize Attestation".into()))?; - debug!("Attestation: {attestation_str}"); if session.is_expired() { raise_error!(Error::ExpiredCookie); } + + if let SessionStatus::Attested { token, .. } = session { + debug!( + "Session {} is already attested. Skip attestation and return the old token", + session.id() + ); + let body = serde_json::to_string(&json!({ + "token": token, + })) + .map_err(|e| Error::TokenIssueFailed(format!("Serialize token failed {e}")))?; + + return Ok(HttpResponse::Ok() + .cookie(session.cookie()) + .content_type("application/json") + .body(body)); + } + + let attestation_str = serde_json::to_string_pretty(&attestation.0) + .map_err(|_| Error::AttestationFailed("Failed to serialize Attestation".into()))?; + debug!("Attestation: {attestation_str}"); + (session.request().tee, session.challenge().nonce.to_string()) }; @@ -86,13 +104,14 @@ pub(crate) async fn attest( .await .ok_or(Error::InvalidCookie)?; let session = session.get_mut(); - session.attest(claims); let body = serde_json::to_string(&json!({ "token": token, })) .map_err(|e| Error::TokenIssueFailed(format!("Serialize token failed {e}")))?; + session.attest(claims, token); + Ok(HttpResponse::Ok() .cookie(session.cookie()) .content_type("application/json") diff --git a/kbs/src/api/src/session.rs b/kbs/src/api/src/session.rs index 67943a75db..56f2f80a03 100644 --- a/kbs/src/api/src/session.rs +++ b/kbs/src/api/src/session.rs @@ -38,6 +38,7 @@ pub(crate) enum SessionStatus { Attested { attestation_claims: String, + token: String, id: String, timeout: OffsetDateTime, }, @@ -105,11 +106,12 @@ impl SessionStatus { return *self.timeout() < OffsetDateTime::now_utc(); } - pub fn attest(&mut self, attestation_claims: String) { + pub fn attest(&mut self, attestation_claims: String, token: String) { match self { SessionStatus::Authed { id, timeout, .. } => { *self = SessionStatus::Attested { attestation_claims, + token, id: id.clone(), timeout: *timeout, }; From 52833c7f35244e82b01de9fce93d7bce7d275349 Mon Sep 17 00:00:00 2001 From: Mikko Ylinen Date: Wed, 24 Apr 2024 14:59:10 +0300 Subject: [PATCH 2/9] k8s: add overlay to deploy with a custom Intel DCAP config Signed-off-by: Mikko Ylinen --- kbs/config/kubernetes/README.md | 10 ++++++++++ .../kubernetes/custom_pccs/kustomization.yaml | 17 ++++++++++++++++ .../custom_pccs/set_custom_pccs.yaml | 20 +++++++++++++++++++ .../custom_pccs/sgx_default_qcnl.conf | 1 + 4 files changed, 48 insertions(+) create mode 100644 kbs/config/kubernetes/custom_pccs/kustomization.yaml create mode 100644 kbs/config/kubernetes/custom_pccs/set_custom_pccs.yaml create mode 100644 kbs/config/kubernetes/custom_pccs/sgx_default_qcnl.conf diff --git a/kbs/config/kubernetes/README.md b/kbs/config/kubernetes/README.md index 9fbd1050fe..59bdf6bf63 100644 --- a/kbs/config/kubernetes/README.md +++ b/kbs/config/kubernetes/README.md @@ -58,6 +58,16 @@ Once you deploy the KBS, you can use the services' nodeport and the Kubernetes n echo $(kubectl get nodes -o jsonpath='{.items[0].status.addresses[0].address}'):$(kubectl get svc kbs -n coco-tenant -o jsonpath='{.spec.ports[0].nodePort}') ``` +## Optional: Use custom Intel DCAP configuration + +If you would like to override the default `sgx_default_qcnl.conf` in the KBS/AS images, copy/configure one into `custom_pccs/` directory and deploy using: + +```bash +export DEPLOYMENT_DIR=custom_pccs +``` + +NB: this currently builds on `nodeport` kustomization. + ## Deploy KBS Deploy KBS by running the following command: diff --git a/kbs/config/kubernetes/custom_pccs/kustomization.yaml b/kbs/config/kubernetes/custom_pccs/kustomization.yaml new file mode 100644 index 0000000000..07c08c3cb9 --- /dev/null +++ b/kbs/config/kubernetes/custom_pccs/kustomization.yaml @@ -0,0 +1,17 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: coco-tenant + +resources: +- ../nodeport + +patches: +- path: set_custom_pccs.yaml + target: + kind: Deployment + name: kbs + +configMapGenerator: +- files: + - sgx_default_qcnl.conf + name: dcap-attestation-conf diff --git a/kbs/config/kubernetes/custom_pccs/set_custom_pccs.yaml b/kbs/config/kubernetes/custom_pccs/set_custom_pccs.yaml new file mode 100644 index 0000000000..3b260def1e --- /dev/null +++ b/kbs/config/kubernetes/custom_pccs/set_custom_pccs.yaml @@ -0,0 +1,20 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: kbs +spec: + template: + spec: + containers: + - name: kbs + volumeMounts: + - name: qplconf + mountPath: /etc/sgx_default_qcnl.conf + subPath: sgx_default_qcnl.conf + volumes: + - name: qplconf + configMap: + name: dcap-attestation-conf + items: + - key: sgx_default_qcnl.conf + path: sgx_default_qcnl.conf diff --git a/kbs/config/kubernetes/custom_pccs/sgx_default_qcnl.conf b/kbs/config/kubernetes/custom_pccs/sgx_default_qcnl.conf new file mode 100644 index 0000000000..a00ff81b82 --- /dev/null +++ b/kbs/config/kubernetes/custom_pccs/sgx_default_qcnl.conf @@ -0,0 +1 @@ +{"collateral_service": "https://api.trustedservices.intel.com/sgx/certification/v4/"} From 6adb8383309cbb7279f1d8e1e4620556ac66481e Mon Sep 17 00:00:00 2001 From: Mikko Ylinen Date: Thu, 25 Apr 2024 10:12:35 +0300 Subject: [PATCH 3/9] k8s: document how to use non-release images Signed-off-by: Mikko Ylinen --- kbs/config/kubernetes/README.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/kbs/config/kubernetes/README.md b/kbs/config/kubernetes/README.md index 59bdf6bf63..c8d3b202c9 100644 --- a/kbs/config/kubernetes/README.md +++ b/kbs/config/kubernetes/README.md @@ -44,6 +44,19 @@ kustomize edit add resource ingress.yaml popd ``` +## Optional: Use non-release images + +Sometimes it may be desirable to deploy KBS with an image that is not what is set in the repo (typically +the latest release image). To change the deployment to use a staging build, set the image using `kustomize`: + +```bash +pushd base +kustomize edit set image kbs-container-image=ghcr.io/confidential-containers/staged-images/kbs:65ee7e1acccd13dcb515058e71c5f8bfb4281e35 +popd +``` + +The available image tags can be found in the [CoCo packages listing](https://github.com/orgs/confidential-containers/packages?repo_name=trustee). + ## Optional: Expose KBS using Nodeport If you would like to expose KBS service using Nodeport then export the following environment variable: From dcd04120e278bef5d07839acd3fd6685d9abd580 Mon Sep 17 00:00:00 2001 From: Hyounggyu Choi Date: Tue, 7 May 2024 08:23:22 +0200 Subject: [PATCH 4/9] AS: Make verifier configurable differently for s390x We are encountering an issue on s390x when compiling AS with `all-verifier`. The error message is as follows: ``` error: failed to run custom build command for `tss-esapi-sys v0.5.0` ``` At the moment, it seems that setting `csv-verifier` is the only viable option until the platform-specific verifier (e.g., `se-verifier`) is introduced. Although we can easily configure the verifier using `--features`, this approach lacks flexibility when the crate is selectively called from outside (e.g., kbs) based on `target_arch`. The optimal solution would be to open up room for configuring the verifier at a `dependencies` level rather than a `features` level. This commit aims to remove `all-verifier` from the default feature set and configure it differently for s390x. Signed-off-by: Hyounggyu Choi --- attestation-service/attestation-service/Cargo.toml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/attestation-service/attestation-service/Cargo.toml b/attestation-service/attestation-service/Cargo.toml index d3bda03d33..ec9c53d811 100644 --- a/attestation-service/attestation-service/Cargo.toml +++ b/attestation-service/attestation-service/Cargo.toml @@ -4,7 +4,7 @@ version = "0.1.0" edition = "2021" [features] -default = [ "restful-bin", "rvps-grpc", "rvps-builtin", "all-verifier" ] +default = [ "restful-bin", "rvps-grpc", "rvps-builtin" ] all-verifier = [ "verifier/all-verifier" ] tdx-verifier = [ "verifier/tdx-verifier" ] sgx-verifier = [ "verifier/sgx-verifier" ] @@ -62,7 +62,12 @@ thiserror = { workspace = true, optional = true } tokio.workspace = true tonic = { workspace = true, optional = true } uuid = { version = "1.1.2", features = ["v4"] } -verifier = { path = "../verifier", default-features = false } + +[target.'cfg(not(target_arch = "s390x"))'.dependencies] +verifier = { path = "../verifier", default-features = false, features = ["all-verifier"] } + +[target.'cfg(target_arch = "s390x")'.dependencies] +verifier = { path = "../verifier", default-features = false, features = ["csv-verifier"] } [build-dependencies] shadow-rs.workspace = true From 1195ba531a3d91258b7837e0c7d75895751e45c6 Mon Sep 17 00:00:00 2001 From: Hyounggyu Choi Date: Tue, 7 May 2024 10:12:21 +0200 Subject: [PATCH 5/9] gha: Add actionlint.yaml `.github/actionlint.yaml` is required to make a label `s390x` tolerant against actionlint. Signed-off-by: Hyounggyu Choi --- .github/actionlint.yaml | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 .github/actionlint.yaml diff --git a/.github/actionlint.yaml b/.github/actionlint.yaml new file mode 100644 index 0000000000..afca3bef3c --- /dev/null +++ b/.github/actionlint.yaml @@ -0,0 +1,4 @@ +self-hosted-runner: + # Labels of self-hosted runner in array of string + labels: + - s390x \ No newline at end of file From 3aa6a973190774c625bcabffef53bc376e4586d5 Mon Sep 17 00:00:00 2001 From: Hyounggyu Choi Date: Tue, 7 May 2024 09:33:27 +0200 Subject: [PATCH 6/9] gha: Enable kbs-client binary for s390x This commit introduces a job matrix to allow a s390x self-hosted runner to run alongside the existing x86_64 one. Additionally, two new steps (e.g., {pre,post} action) are introduced to manage the self-hosted runner since it is not provisioned instantly for CI. The make target `cli-static-x86_64-linux` is renamed to `cli-static-linux` for platform independence. Signed-off-by: Hyounggyu Choi --- .../workflows/kbs-client-build-and-push.yaml | 33 ++++++++++++++++--- kbs/Makefile | 14 +++++--- 2 files changed, 38 insertions(+), 9 deletions(-) diff --git a/.github/workflows/kbs-client-build-and-push.yaml b/.github/workflows/kbs-client-build-and-push.yaml index e27aee5160..944b1716fa 100644 --- a/.github/workflows/kbs-client-build-and-push.yaml +++ b/.github/workflows/kbs-client-build-and-push.yaml @@ -7,14 +7,28 @@ on: jobs: build_and_push: + strategy: + fail-fast: false + matrix: + arch: + - x86_64 + - s390x env: RUSTC_VERSION: 1.76.0 - runs-on: ubuntu-22.04 + runs-on: ${{ matrix.arch == 'x86_64' && 'ubuntu-22.04' || 's390x' }} permissions: contents: read packages: write steps: + - name: Take a pre-action for self-hosted runner + run: | + # NOTE: Use file checking instead triggering a step based on a runner type + # to avoid updating the step for each new self-hosted runner. + if [ -f "${HOME}/script/pre_action.sh" ]; then + "${HOME}/script/pre_action.sh" cc-trustee + fi + - name: Check out code uses: actions/checkout@v4 - name: Install rust toolchain @@ -29,14 +43,23 @@ jobs: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - - name: Build a statically linked kbs-client for x86_64 linux + - name: Build a statically linked kbs-client for ${{ matrix.arch }} linux working-directory: kbs run: | - make cli-static-x86_64-linux + make cli-static-linux - name: Push to ghcr.io - working-directory: target/x86_64-unknown-linux-gnu/release + working-directory: target/${{ matrix.arch }}-unknown-linux-gnu/release run: | commit_sha=${{ github.sha }} oras push \ - ghcr.io/confidential-containers/staged-images/kbs-client:sample_only-x86_64-linux-gnu-${commit_sha},latest \ + ghcr.io/confidential-containers/staged-images/kbs-client:sample_only-${{ matrix.arch }}-linux-gnu-${commit_sha},latest-${{ matrix.arch }} \ kbs-client + [ "$(uname -m)" = "x86_64" ] && oras push ghcr.io/confidential-containers/staged-images/kbs-client:latest kbs-client || true + + - name: Take a post-action for self-hosted runner + if: always() + run: | + # Please check out the note in the pre-action step for the reason of using file checking + if [ -f "${HOME}/script/post_action.sh" ]; then + "${HOME}/script/post_action.sh" cc-trustee + fi diff --git a/kbs/Makefile b/kbs/Makefile index 3aa6b36751..f90445f0af 100644 --- a/kbs/Makefile +++ b/kbs/Makefile @@ -2,6 +2,12 @@ AS_TYPE ?= coco-as HTTPS_CRYPTO ?= rustls POLICY_ENGINE ?= +ARCH := $(shell uname -m) +# Check if ARCH is supported, otehrwise return error +ifeq ($(filter $(ARCH),x86_64 s390x),) + $(error "Unsupported architecture: $(ARCH)") +endif + CLI_FEATURES ?= default COCO_AS_INTEGRATION_TYPE ?= builtin @@ -34,12 +40,12 @@ passport-resource-kbs: cli: cargo build -p kbs-client --locked --release --no-default-features --features $(CLI_FEATURES) -.PHONY: cli-static-x86_64-linux -cli-static-x86_64-linux: +.PHONY: cli-static-linux +cli-static-linux: cargo build \ -p kbs-client \ - --target=x86_64-unknown-linux-gnu \ - --config "target.x86_64-unknown-linux-gnu.rustflags = '-C target-feature=+crt-static'" \ + --target=$(ARCH)-unknown-linux-gnu \ + --config "target.$(ARCH)-unknown-linux-gnu.rustflags = '-C target-feature=+crt-static'" \ --locked \ --release \ --no-default-features \ From 992d6da060c5004c87914edfb6c0813ad812d6ca Mon Sep 17 00:00:00 2001 From: Hyounggyu Choi Date: Tue, 7 May 2024 09:50:30 +0200 Subject: [PATCH 7/9] gha: Enable attestation-service images for s390x This commit introduces a job matrix to allow a s390x self-hosted runner to run alongside the existing x86_64 one. Additionally, two new steps (e.g., {pre,post} action) are introduced to manage the self-hosted runner since it is not provisioned instantly for CI. To make the published images support multiple architecture, a new job `publish_multi_arch_image` is also introduced. Signed-off-by: Hyounggyu Choi --- .github/workflows/as-build-and-push.yaml | 74 ++++++++++++++++++++++- attestation-service/Dockerfile.as-grpc | 17 +++--- attestation-service/Dockerfile.as-restful | 17 +++--- 3 files changed, 92 insertions(+), 16 deletions(-) diff --git a/.github/workflows/as-build-and-push.yaml b/.github/workflows/as-build-and-push.yaml index 8a3f474e44..50f1925ec8 100644 --- a/.github/workflows/as-build-and-push.yaml +++ b/.github/workflows/as-build-and-push.yaml @@ -7,10 +7,18 @@ on: jobs: build_and_push: - runs-on: ubuntu-latest + permissions: + packages: write strategy: fail-fast: false matrix: + instance: + - ubuntu-latest + - s390x + tag: + - coco-as-grpc + - coco-as-restful + - rvps include: - docker_file: attestation-service/Dockerfile.as-grpc tag: coco-as-grpc @@ -21,8 +29,17 @@ jobs: - docker_file: attestation-service/rvps/Dockerfile tag: rvps name: RVPS + runs-on: ${{ matrix.instance }} steps: + - name: Take a pre-action for self-hosted runner + run: | + # NOTE: Use file checking instead triggering a step based on a runner type + # to avoid updating the step for each new self-hosted runner. + if [ -f "${HOME}/script/pre_action.sh" ]; then + "${HOME}/script/pre_action.sh" cc-trustee + fi + - name: Checkout code uses: actions/checkout@v4 @@ -39,4 +56,57 @@ jobs: - name: Build ${{ matrix.name }} Container Image run: | commit_sha=${{ github.sha }} - DOCKER_BUILDKIT=1 docker build -t ghcr.io/confidential-containers/staged-images/${{ matrix.tag }}:${commit_sha} -t ghcr.io/confidential-containers/staged-images/${{ matrix.tag }}:latest . -f ${{ matrix.docker_file }} --push + arch=$(uname -m) + DOCKER_BUILDKIT=1 docker build -f ${{ matrix.docker_file }} --push --build-arg ARCH=${arch} \ + -t ghcr.io/confidential-containers/staged-images/${{ matrix.tag }}:${commit_sha}-${arch} \ + -t ghcr.io/confidential-containers/staged-images/${{ matrix.tag }}:latest-${arch} . + + - name: Take a post-action for self-hosted runner + if: always() + run: | + # Please check out the note in the pre-action step for the reason of using file checking + if [ -f "${HOME}/script/post_action.sh" ]; then + "${HOME}/script/post_action.sh" cc-trustee + fi + + publish_multi_arch_image: + needs: build_and_push + permissions: + packages: write + strategy: + fail-fast: false + matrix: + tag: + - coco-as-grpc + - coco-as-restful + - rvps + include: + - tag: coco-as-grpc + name: gRPC CoCo-AS + - tag: coco-as-restful + name: RESTful CoCo-AS + - tag: rvps + name: RVPS + runs-on: ubuntu-latest + steps: + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Login to GHCR Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Publish Multi-arch Image for ${{ matrix.name }} + run: | + commit_sha=${{ github.sha }} + docker manifest create ghcr.io/confidential-containers/staged-images/${{ matrix.tag }}:${commit_sha} \ + --amend ghcr.io/confidential-containers/staged-images/${{ matrix.tag }}:${commit_sha}-s390x \ + --amend ghcr.io/confidential-containers/staged-images/${{ matrix.tag }}:${commit_sha}-x86_64 + docker manifest push ghcr.io/confidential-containers/staged-images/${{ matrix.tag }}:${commit_sha} + docker manifest create ghcr.io/confidential-containers/staged-images/${{ matrix.tag }}:latest \ + --amend ghcr.io/confidential-containers/staged-images/${{ matrix.tag }}:latest-s390x \ + --amend ghcr.io/confidential-containers/staged-images/${{ matrix.tag }}:latest-x86_64 + docker manifest push ghcr.io/confidential-containers/staged-images/${{ matrix.tag }}:latest diff --git a/attestation-service/Dockerfile.as-grpc b/attestation-service/Dockerfile.as-grpc index 403b629389..104be26fe6 100644 --- a/attestation-service/Dockerfile.as-grpc +++ b/attestation-service/Dockerfile.as-grpc @@ -3,13 +3,15 @@ # SPDX-License-Identifier: Apache-2.0 FROM rust:latest as builder +ARG ARCH=x86_64 WORKDIR /usr/src/attestation-service COPY . . # Install golang -RUN wget https://go.dev/dl/go1.20.1.linux-amd64.tar.gz && \ - tar -C /usr/local -xzf go1.20.1.linux-amd64.tar.gz +RUN if [ "${ARCH}" = "x86_64" ]; then GOARCH="amd64"; else GOARCH="${ARCH}"; fi && \ + wget https://go.dev/dl/go1.20.1.linux-${GOARCH}.tar.gz && \ + tar -C /usr/local -xzf go1.20.1.linux-${GOARCH}.tar.gz ENV PATH="/usr/local/go/bin:${PATH}" @@ -17,15 +19,16 @@ ENV PATH="/usr/local/go/bin:${PATH}" RUN apt-get update && apt-get install -y protobuf-compiler clang libtss2-dev # Install TDX Build Dependencies -RUN curl -L https://download.01.org/intel-sgx/sgx_repo/ubuntu/intel-sgx-deb.key | tee intel-sgx-deb.key | apt-key add - && \ +RUN if [ "${ARCH}" = "x86_64" ]; then curl -L https://download.01.org/intel-sgx/sgx_repo/ubuntu/intel-sgx-deb.key | tee intel-sgx-deb.key | apt-key add - && \ echo 'deb [arch=amd64] https://download.01.org/intel-sgx/sgx_repo/ubuntu focal main' | tee /etc/apt/sources.list.d/intel-sgx.list && \ - apt-get update && apt-get install -y libtdx-attest-dev libsgx-dcap-quote-verify-dev + apt-get update && apt-get install -y libtdx-attest-dev libsgx-dcap-quote-verify-dev; fi # Build and Install gRPC attestation-service RUN cargo install --path attestation-service/attestation-service --bin grpc-as --features grpc-bin --locked FROM ubuntu:22.04 +ARG ARCH=x86_64 LABEL org.opencontainers.image.source="https://github.com/confidential-containers/attestation-service" @@ -33,14 +36,14 @@ LABEL org.opencontainers.image.source="https://github.com/confidential-container RUN apt-get update && apt-get install curl gnupg openssl -y && \ rm -rf /var/lib/apt/lists/{apt,dpkg,cache,log} /tmp/* /var/tmp/* -RUN curl -L https://download.01.org/intel-sgx/sgx_repo/ubuntu/intel-sgx-deb.key | tee intel-sgx-deb.key | apt-key add - && \ +RUN if [ "${ARCH}" = "x86_64" ]; then curl -L https://download.01.org/intel-sgx/sgx_repo/ubuntu/intel-sgx-deb.key | tee intel-sgx-deb.key | apt-key add - && \ echo 'deb [arch=amd64] https://download.01.org/intel-sgx/sgx_repo/ubuntu focal main' | tee /etc/apt/sources.list.d/intel-sgx.list && \ apt-get update && \ apt-get install -y libsgx-dcap-default-qpl libsgx-dcap-quote-verify && \ - rm -rf /var/lib/apt/lists/{apt,dpkg,cache,log} /tmp/* /var/tmp/* + rm -rf /var/lib/apt/lists/{apt,dpkg,cache,log} /tmp/* /var/tmp/*; fi # Copy TPM Runtime Dependencies -COPY --from=builder /usr/lib/x86_64-linux-gnu/libtss* /usr/lib/x86_64-linux-gnu +COPY --from=builder /usr/lib/${ARCH}-linux-gnu/libtss* /usr/lib/${ARCH}-linux-gnu COPY --from=builder /usr/local/cargo/bin/grpc-as /usr/local/bin/grpc-as diff --git a/attestation-service/Dockerfile.as-restful b/attestation-service/Dockerfile.as-restful index 322536c051..360b34b438 100644 --- a/attestation-service/Dockerfile.as-restful +++ b/attestation-service/Dockerfile.as-restful @@ -3,13 +3,15 @@ # SPDX-License-Identifier: Apache-2.0 FROM rust:latest as builder +ARG ARCH=x86_64 WORKDIR /usr/src/attestation-service COPY . . # Install golang -RUN wget https://go.dev/dl/go1.20.1.linux-amd64.tar.gz && \ - tar -C /usr/local -xzf go1.20.1.linux-amd64.tar.gz +RUN if [ "${ARCH}" = "x86_64" ]; then GOARCH="amd64"; else GOARCH="${ARCH}"; fi && \ + wget https://go.dev/dl/go1.20.1.linux-${GOARCH}.tar.gz && \ + tar -C /usr/local -xzf go1.20.1.linux-${GOARCH}.tar.gz ENV PATH="/usr/local/go/bin:${PATH}" @@ -17,14 +19,15 @@ ENV PATH="/usr/local/go/bin:${PATH}" RUN apt-get update && apt-get install -y protobuf-compiler clang libtss2-dev # Install TDX Build Dependencies -RUN curl -L https://download.01.org/intel-sgx/sgx_repo/ubuntu/intel-sgx-deb.key | tee intel-sgx-deb.key | apt-key add - && \ +RUN if [ "${ARCH}" = "x86_64" ]; then curl -L https://download.01.org/intel-sgx/sgx_repo/ubuntu/intel-sgx-deb.key | tee intel-sgx-deb.key | apt-key add - && \ echo 'deb [arch=amd64] https://download.01.org/intel-sgx/sgx_repo/ubuntu focal main' | tee /etc/apt/sources.list.d/intel-sgx.list && \ - apt-get update && apt-get install -y libtdx-attest-dev libsgx-dcap-quote-verify-dev + apt-get update && apt-get install -y libtdx-attest-dev libsgx-dcap-quote-verify-dev; fi # Build and Install RESTful attestation-service RUN cargo install --path attestation-service/attestation-service --bin restful-as --features restful-bin --locked FROM ubuntu:22.04 +ARG ARCH=x86_64 LABEL org.opencontainers.image.source="https://github.com/confidential-containers/attestation-service" @@ -32,14 +35,14 @@ LABEL org.opencontainers.image.source="https://github.com/confidential-container RUN apt-get update && apt-get install curl gnupg openssl -y && \ rm -rf /var/lib/apt/lists/{apt,dpkg,cache,log} /tmp/* /var/tmp/* -RUN curl -L https://download.01.org/intel-sgx/sgx_repo/ubuntu/intel-sgx-deb.key | tee intel-sgx-deb.key | apt-key add - && \ +RUN if [ "${ARCH}" = "x86_64" ]; then curl -L https://download.01.org/intel-sgx/sgx_repo/ubuntu/intel-sgx-deb.key | tee intel-sgx-deb.key | apt-key add - && \ echo 'deb [arch=amd64] https://download.01.org/intel-sgx/sgx_repo/ubuntu focal main' | tee /etc/apt/sources.list.d/intel-sgx.list && \ apt-get update && \ apt-get install -y libsgx-dcap-default-qpl libsgx-dcap-quote-verify && \ - rm -rf /var/lib/apt/lists/{apt,dpkg,cache,log} /tmp/* /var/tmp/* + rm -rf /var/lib/apt/lists/{apt,dpkg,cache,log} /tmp/* /var/tmp/*; fi # Copy TPM Runtime Dependencies -COPY --from=builder /usr/lib/x86_64-linux-gnu/libtss* /usr/lib/x86_64-linux-gnu +COPY --from=builder /usr/lib/${ARCH}-linux-gnu/libtss* /usr/lib/${ARCH}-linux-gnu COPY --from=builder /usr/local/cargo/bin/restful-as /usr/local/bin/restful-as From aee742cbdcd051a150e3e25380a07e597d33ec6c Mon Sep 17 00:00:00 2001 From: Hyounggyu Choi Date: Tue, 7 May 2024 09:59:08 +0200 Subject: [PATCH 8/9] gha: Enable KBS images for s390x This commit introduces a job matrix to allow a s390x self-hosted runner to run alongside the existing x86_64 one. Additionally, two new steps (e.g., {pre,post} action) are introduced to manage the self-hosted runner since it is not provisioned instantly for CI. To run the build steps `kbs` and `kbs-grpc-as` in parallel, the build step is generalized into one step and the relevant configuration values are parameterized into the job matrix. Signed-off-by: Hyounggyu Choi --- .github/workflows/kbs-build-and-push.yaml | 79 +++++++++++++++++++++-- kbs/docker/Dockerfile | 34 +++++----- kbs/docker/Dockerfile.coco-as-grpc | 10 +-- 3 files changed, 99 insertions(+), 24 deletions(-) diff --git a/.github/workflows/kbs-build-and-push.yaml b/.github/workflows/kbs-build-and-push.yaml index 7cba6312ea..af2ababc04 100644 --- a/.github/workflows/kbs-build-and-push.yaml +++ b/.github/workflows/kbs-build-and-push.yaml @@ -7,9 +7,37 @@ on: jobs: build_and_push: - runs-on: ubuntu-latest + permissions: + packages: write + strategy: + fail-fast: false + matrix: + instance: + - ubuntu-latest + - s390x + tag: + - kbs + - kbs-grpc-as + include: + - tag: kbs + docker_file: kbs/docker/Dockerfile + https_crypto: openssl + name: build-in AS + - tag: kbs-grpc-as + docker_file: kbs/docker/Dockerfile.coco-as-grpc + https_crypto: rustls + name: gRPC AS + runs-on: ${{ matrix.instance }} steps: + - name: Take a pre-action for self-hosted runner + run: | + # NOTE: Use file checking instead triggering a step based on a runner type + # to avoid updating the step for each new self-hosted runner. + if [ -f "${HOME}/script/pre_action.sh" ]; then + "${HOME}/script/pre_action.sh" cc-trustee + fi + - name: Checkout code uses: actions/checkout@v4 @@ -23,12 +51,53 @@ jobs: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - - name: Build Container Image KBS (built-in AS) + - name: Build Container Image KBS (${{ matrix.name }}) run: | commit_sha=${{ github.sha }} - DOCKER_BUILDKIT=1 docker build -t ghcr.io/confidential-containers/staged-images/kbs:${commit_sha} -t ghcr.io/confidential-containers/staged-images/kbs:latest --build-arg KBS_FEATURES=coco-as-builtin,openssl,resource,opa . -f kbs/docker/Dockerfile --push + arch=$(uname -m) + https_crypto=${{ matrix.https_crypto }} + [ "${arch}" = "s390x" ] && https_crypto=openssl + DOCKER_BUILDKIT=1 docker build -f ${{ matrix.docker_file }} --push \ + -t ghcr.io/confidential-containers/staged-images/${{ matrix.tag }}:${commit_sha}-${arch} \ + -t ghcr.io/confidential-containers/staged-images/${{ matrix.tag }}:latest-${arch} \ + --build-arg ARCH=${arch} --build-arg HTTPS_CRYPTO=${https_crypto} . + + - name: Take a post-action for self-hosted runner + if: always() + run: | + # Please check out the note in the pre-action step for the reason of using file checking + if [ -f "${HOME}/script/post_action.sh" ]; then + "${HOME}/script/post_action.sh" cc-trustee + fi + + publish_multi_arch_image: + needs: build_and_push + strategy: + fail-fast: false + matrix: + image: + - kbs + - kbs-grpc-as + permissions: + packages: write + runs-on: ubuntu-latest + + steps: + - name: Login to GHCR Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} - - name: Build Container Image KBS (gRPC AS) + - name: Publish Multi-Arch ${{ matrix.image }} image run: | commit_sha=${{ github.sha }} - DOCKER_BUILDKIT=1 docker build -t ghcr.io/confidential-containers/staged-images/kbs-grpc-as:${commit_sha} -t ghcr.io/confidential-containers/staged-images/kbs-grpc-as:latest . -f kbs/docker/Dockerfile.coco-as-grpc --push + docker manifest create ghcr.io/confidential-containers/staged-images/${{ matrix.image }}:${commit_sha} \ + --amend ghcr.io/confidential-containers/staged-images/${{ matrix.image }}:${commit_sha}-x86_64 \ + --amend ghcr.io/confidential-containers/staged-images/${{ matrix.image }}:${commit_sha}-s390x + docker manifest push ghcr.io/confidential-containers/staged-images/${{ matrix.image }}:${commit_sha} + docker manifest create ghcr.io/confidential-containers/staged-images/${{ matrix.image }}:latest \ + --amend ghcr.io/confidential-containers/staged-images/${{ matrix.image }}:latest-x86_64 \ + --amend ghcr.io/confidential-containers/staged-images/${{ matrix.image }}:latest-s390x + docker manifest push ghcr.io/confidential-containers/staged-images/${{ matrix.image }}:latest diff --git a/kbs/docker/Dockerfile b/kbs/docker/Dockerfile index 0d2795412f..2367920953 100644 --- a/kbs/docker/Dockerfile +++ b/kbs/docker/Dockerfile @@ -1,4 +1,6 @@ FROM rust:slim as builder +ARG ARCH=x86_64 +ARG HTTPS_CRYPTO=rustls ENV DEBIAN_FRONTEND noninteractive @@ -9,10 +11,10 @@ RUN apt-get update && \ gnupg-agent \ git -RUN curl -fsSL https://download.01.org/intel-sgx/sgx_repo/ubuntu/intel-sgx-deb.key | \ +RUN if [ "${ARCH}" = "x86_64" ]; then curl -fsSL https://download.01.org/intel-sgx/sgx_repo/ubuntu/intel-sgx-deb.key | \ gpg --dearmor --output /usr/share/keyrings/intel-sgx.gpg && \ echo 'deb [arch=amd64 signed-by=/usr/share/keyrings/intel-sgx.gpg] https://download.01.org/intel-sgx/sgx_repo/ubuntu focal main' | \ - tee /etc/apt/sources.list.d/intel-sgx.list && \ + tee /etc/apt/sources.list.d/intel-sgx.list; fi && \ apt-get update && \ apt-get install -y --no-install-recommends \ libclang-dev \ @@ -25,22 +27,24 @@ RUN curl -fsSL https://download.01.org/intel-sgx/sgx_repo/ubuntu/intel-sgx-deb.k wget \ clang \ cmake \ - libtss2-dev \ + libtss2-dev && \ + if [ "${ARCH}" = "x86_64" ]; then apt-get install -y --no-install-recommends \ libsgx-dcap-quote-verify-dev \ - libtdx-attest-dev + libtdx-attest-dev; fi -RUN wget https://go.dev/dl/go1.20.1.linux-amd64.tar.gz -RUN tar -C /usr/local -xzf go1.20.1.linux-amd64.tar.gz +RUN if [ "${ARCH}" = "x86_64" ]; then GOARCH="amd64"; else GOARCH="${ARCH}"; fi && \ + wget https://go.dev/dl/go1.20.1.linux-${GOARCH}.tar.gz && \ + tar -C /usr/local -xzf go1.20.1.linux-${GOARCH}.tar.gz ENV PATH="/usr/local/go/bin:${PATH}" # Build and Install KBS WORKDIR /usr/src/kbs COPY . . -ARG KBS_FEATURES=coco-as-builtin,rustls,resource,opa -RUN cargo install --locked --path kbs/src/kbs --no-default-features --features ${KBS_FEATURES} +RUN cargo install --locked --path kbs/src/kbs --no-default-features --features coco-as-builtin,resource,opa,${HTTPS_CRYPTO} FROM ubuntu:22.04 +ARG ARCH=x86_64 RUN apt-get update && \ apt-get install -y \ @@ -49,14 +53,14 @@ RUN apt-get update && \ gnupg-agent # Install TDX Runtime Dependencies -RUN curl -fsSL https://download.01.org/intel-sgx/sgx_repo/ubuntu/intel-sgx-deb.key | \ - gpg --dearmor --output /usr/share/keyrings/intel-sgx.gpg -RUN echo 'deb [arch=amd64 signed-by=/usr/share/keyrings/intel-sgx.gpg] https://download.01.org/intel-sgx/sgx_repo/ubuntu jammy main' | tee /etc/apt/sources.list.d/intel-sgx.list +RUN if [ "${ARCH}" = "x86_64" ]; then curl -fsSL https://download.01.org/intel-sgx/sgx_repo/ubuntu/intel-sgx-deb.key | \ + gpg --dearmor --output /usr/share/keyrings/intel-sgx.gpg; fi +RUN if [ "${ARCH}" = "x86_64" ]; then echo 'deb [arch=amd64 signed-by=/usr/share/keyrings/intel-sgx.gpg] https://download.01.org/intel-sgx/sgx_repo/ubuntu jammy main' | tee /etc/apt/sources.list.d/intel-sgx.list; fi RUN apt-get update -RUN apt-get install -y --no-install-recommends \ +RUN if [ "${ARCH}" = "x86_64" ]; then apt-get install -y --no-install-recommends \ libsgx-dcap-default-qpl \ libsgx-dcap-quote-verify \ - tpm2-tools + tpm2-tools; fi # Intel PCCS URL Configurations # If you want the AS in KBS to connect to your customized PCCS for Intel TDX/SGX evidence verification, @@ -66,7 +70,7 @@ ENV INTEL_PCCS_URL "https://localhost:8081/sgx/certification/v4/" ENV INTEL_PCCS_USE_SECURE_CERT false # Setup Intel PCCS URL -RUN sed -i "s|\"pccs_url\":.*$|\"pccs_url\":$INTEL_PCCS_URL,|" /etc/sgx_default_qcnl.conf; \ - sed -i "s/\"use_secure_cert\":.*$/\"use_secure_cert\":$INTEL_PCCS_USE_SECURE_CERT,/" /etc/sgx_default_qcnl.conf +RUN if [ "${ARCH}" = "x86_64" ]; then sed -i "s|\"pccs_url\":.*$|\"pccs_url\":$INTEL_PCCS_URL,|" /etc/sgx_default_qcnl.conf; \ + sed -i "s/\"use_secure_cert\":.*$/\"use_secure_cert\":$INTEL_PCCS_USE_SECURE_CERT,/" /etc/sgx_default_qcnl.conf; fi COPY --from=builder /usr/local/cargo/bin/kbs /usr/local/bin/kbs diff --git a/kbs/docker/Dockerfile.coco-as-grpc b/kbs/docker/Dockerfile.coco-as-grpc index 9022d5c9d0..1506741dce 100644 --- a/kbs/docker/Dockerfile.coco-as-grpc +++ b/kbs/docker/Dockerfile.coco-as-grpc @@ -1,17 +1,19 @@ FROM rust:latest as builder +ARG ARCH=x86_64 +ARG HTTPS_CRYPTO=rustls WORKDIR /usr/src/kbs COPY . . RUN apt-get update && apt install -y protobuf-compiler wget git -RUN wget https://go.dev/dl/go1.20.1.linux-amd64.tar.gz -RUN tar -C /usr/local -xzf go1.20.1.linux-amd64.tar.gz +RUN if [ "${ARCH}" = "x86_64" ]; then GOARCH="amd64"; else GOARCH="${ARCH}"; fi && \ +wget https://go.dev/dl/go1.20.1.linux-${GOARCH}.tar.gz && \ +tar -C /usr/local -xzf go1.20.1.linux-${GOARCH}.tar.gz ENV PATH="/usr/local/go/bin:${PATH}" # Build and Install KBS -RUN cargo install --path kbs/src/kbs --no-default-features --features coco-as-grpc,resource,opa,rustls - +RUN cargo install --path kbs/src/kbs --no-default-features --features coco-as-grpc,resource,opa,${HTTPS_CRYPTO} FROM ubuntu:22.04 From 367a0de52011044b31ff55d3d5ebcb3bb8db92b2 Mon Sep 17 00:00:00 2001 From: Hyounggyu Choi Date: Thu, 9 May 2024 05:52:54 +0200 Subject: [PATCH 9/9] Make release helper script publish multi-arch images We need to update the release helper script accordingly to publish multi-arch images. Signed-off-by: Hyounggyu Choi --- hack/release-helper.sh | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/hack/release-helper.sh b/hack/release-helper.sh index 8569c16bed..722fbac0a8 100755 --- a/hack/release-helper.sh +++ b/hack/release-helper.sh @@ -81,9 +81,6 @@ function tag_and_push_packages() { for staged_pkg_name in ${!staged_to_release[@]}; do release_pkg_name=${staged_to_release[${staged_pkg_name}]} - # pull the staged package - docker pull ${ghcr_repo}/${staged_pkg_name}:${release_candidate_sha} - # set tag prefix (if needed) release_tag_prefix= if [[ -v staged_to_release_tag_prefix[${staged_pkg_name}] ]]; then @@ -91,11 +88,23 @@ function tag_and_push_packages() { fi release_tag_full=${release_tag_prefix}${release_tag} - # tag it - docker tag ${ghcr_repo}/${staged_pkg_name}:${release_candidate_sha} ${ghcr_repo}/${release_pkg_name}:${release_tag_full} + for arch in x86_64 s390x; do + # pull the staged package + docker pull ${ghcr_repo}/${staged_pkg_name}:${release_candidate_sha}-${arch} + + # tag it + docker tag ${ghcr_repo}/${staged_pkg_name}:${release_candidate_sha}-${arch} \ + ${ghcr_repo}/${release_pkg_name}:${release_tag_full}-${arch} + + # push it (i.e. release it) + docker push ${ghcr_repo}/${release_pkg_name}:${release_tag_full}-${arch} + done - # push it (i.e. release it) - docker push ${ghcr_repo}/${release_pkg_name}:${release_tag_full} + # Publish the multi-arch manifest + docker manifest create ${ghcr_repo}/${release_pkg_name}:${release_tag_full} \ + --amend ${ghcr_repo}/${release_pkg_name}:${release_tag_full}-x86_64 \ + --amend ${ghcr_repo}/${release_pkg_name}:${release_tag_full}-s390x + docker manifest push ${ghcr_repo}/${release_pkg_name}:${release_tag_full} done }