From 6d90b9b7cae5e4f2ed81c755d68852d0f28f8e67 Mon Sep 17 00:00:00 2001 From: willdavsmith Date: Thu, 8 Aug 2024 19:00:13 -0700 Subject: [PATCH 1/7] wip Signed-off-by: willdavsmith --- .github/workflows/test.yaml | 61 ++++++++++++++++++++++++++++++++++--- 1 file changed, 57 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index fb434133..b07a53b7 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -1,5 +1,11 @@ name: Test Samples (k3d and EKS) +permissions: + id-token: write # Required for requesting the JWT + contents: read # Required for listing the commits + packages: write # Required for uploading the package + checks: write # Required for creating a check run + on: workflow_dispatch: inputs: @@ -24,6 +30,10 @@ on: - cron: "0 12 * * *" env: RUN_IDENTIFIER: samplestest-${{ github.run_id }}-${{ github.run_attempt }} + # Azure workload identity webhook chart version + AZURE_WORKLOAD_IDENTITY_WEBHOOK_VER: "1.3.0" + # Helm version + HELM_VER: "v3.12.0" jobs: test: name: Sample tests @@ -135,6 +145,9 @@ jobs: uses: actions/setup-node@v4 with: node-version: 20 + - uses: azure/setup-helm@v4 + with: + version: ${{ env.HELM_VER }} - name: az CLI login if: steps.gen-id.outputs.RUN_TEST == 'true' && matrix.credential == 'azure' run: | @@ -154,7 +167,43 @@ jobs: run: wget -q -O - https://raw.githubusercontent.com/k3d-io/k3d/main/install.sh | bash - name: Create k3d cluster if: steps.gen-id.outputs.RUN_TEST == 'true' - run: k3d cluster create --agents 2 -p "80:80@loadbalancer" --k3s-arg "--disable=traefik@server:0" --registry-create sampleregistry:51351 + run: | + # Populate the following environment variables for Azure workload identity from secrets. + # AZURE_OIDC_ISSUER_PUBLIC_KEY + # AZURE_OIDC_ISSUER_PRIVATE_KEY + # AZURE_OIDC_ISSUER + eval "export $(echo "${{ secrets.FUNCTEST_AZURE_OIDC_JSON }}" | jq -r 'to_entries | map("\(.key)=\(.value)") | @sh')" + + # Create k3d cluster with OIDC Issuer keys + echo $AZURE_OIDC_ISSUER_PUBLIC_KEY | base64 -d > sa.pub + echo $AZURE_OIDC_ISSUER_PRIVATE_KEY | base64 -d > sa.key + + # Create k3d cluster configuration + cat < k3d-config.yaml + apiVersion: k3d.io/v1alpha4 + kind: Simple + agents: 2 + options: + k3s: + extraArgs: + - --kube-apiserver-arg=service-account-issuer=$AZURE_OIDC_ISSUER + - --kube-apiserver-arg=service-account-key-file=/etc/kubernetes/k3s/pki/sa.pub + - --kube-apiserver-arg=service-account-signing-key-file=/etc/kubernetes/k3s/pki/sa.key + - --kube-controller-manager-arg=service-account-private-key-file=/etc/kubernetes/k3s/pki/sa.key + - --disable=traefik@server:0 + ports: + - port: 80:80@loadbalancer + volumes: + - volume: ./sa.pub:/etc/kubernetes/k3s/pki/sa.pub + - volume: ./sa.key:/etc/kubernetes/k3s/pki/sa.key + registries: + create: + name: sampleregistry + port: 51351 + EOF + + # Create the k3d cluster using the configuration file + k3d cluster create --config ./k3d-config.yaml - name: Build images if: steps.gen-id.outputs.RUN_TEST == 'true' && matrix.images != '' run: | @@ -209,6 +258,10 @@ jobs: run: | helm repo add dapr https://dapr.github.io/helm-charts/ helm install dapr dapr/dapr --version=1.6 --namespace dapr-system --create-namespace --wait + - name: Install azure workload identity webhook chart + run: | + helm repo add azure-workload-identity https://azure.github.io/azure-workload-identity/charts + helm install workload-identity-webhook azure-workload-identity/workload-identity-webhook --namespace radius-default --create-namespace --version ${{ env.AZURE_WORKLOAD_IDENTITY_WEBHOOK_VER }} --set azureTenantID=${{ secrets.AZURE_TEST_TENANTID }} - name: Download rad CLI if: steps.gen-id.outputs.RUN_TEST == 'true' run: | @@ -221,9 +274,9 @@ jobs: if: steps.gen-id.outputs.RUN_TEST == 'true' run: | if [[ "${{ matrix.credential }}" == "aws" ]]; then - rad install kubernetes + rad install kubernetes --set global.azureWorkloadIdentity.enabled=true else - rad install kubernetes --set rp.publicEndpointOverride=localhost + rad install kubernetes --set rp.publicEndpointOverride=localhost,global.azureWorkloadIdentity.enabled=true fi rad group create default rad workspace create kubernetes default --group default @@ -239,7 +292,7 @@ jobs: run: | if [[ "${{ matrix.credential }}" == "azure" ]]; then rad env update default --azure-subscription-id ${{ secrets.AZURE_SUBSCRIPTIONID_TESTS }} --azure-resource-group ${{ steps.gen-id.outputs.TEST_AZURE_RESOURCE_GROUP }} - rad credential register azure sp --client-id ${{ secrets.AZURE_SP_TESTS_APPID }} --client-secret ${{ secrets.AZURE_SP_TESTS_PASSWORD }} --tenant-id ${{ secrets.AZURE_SP_TESTS_TENANTID }} + rad credential register azure wi --client-id ${{ secrets.AZURE_SP_TESTS_APPID }} --tenant-id ${{ secrets.AZURE_SP_TESTS_TENANTID }} fi if [[ "${{ matrix.credential }}" == "aws" ]]; then rad env update default --aws-region ${{ env.AWS_REGION }} --aws-account-id ${{ secrets.AWS_ACCOUNT_ID }} From 20f43cdfe5947d3b95e453c29f6d9cb19b0309af Mon Sep 17 00:00:00 2001 From: willdavsmith Date: Fri, 9 Aug 2024 10:23:38 -0700 Subject: [PATCH 2/7] wip Signed-off-by: willdavsmith --- .github/workflows/test.yaml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index b07a53b7..08ca4d64 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -1,11 +1,5 @@ name: Test Samples (k3d and EKS) -permissions: - id-token: write # Required for requesting the JWT - contents: read # Required for listing the commits - packages: write # Required for uploading the package - checks: write # Required for creating a check run - on: workflow_dispatch: inputs: From f30503c75950609983d72bcb665b2d90cffe6c46 Mon Sep 17 00:00:00 2001 From: willdavsmith Date: Fri, 9 Aug 2024 10:37:38 -0700 Subject: [PATCH 3/7] wip Signed-off-by: willdavsmith --- .github/workflows/test.yaml | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 08ca4d64..0c9760de 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -28,6 +28,8 @@ env: AZURE_WORKLOAD_IDENTITY_WEBHOOK_VER: "1.3.0" # Helm version HELM_VER: "v3.12.0" + # k3d CLI version + K3D_VER: "v5.7.3" jobs: test: name: Sample tests @@ -158,7 +160,7 @@ jobs: aws configure set output json - name: Download k3d if: steps.gen-id.outputs.RUN_TEST == 'true' - run: wget -q -O - https://raw.githubusercontent.com/k3d-io/k3d/main/install.sh | bash + run: wget -q -O - https://raw.githubusercontent.com/k3d-io/k3d/main/install.sh | TAG=${{ env.K3D_VER }} bash - name: Create k3d cluster if: steps.gen-id.outputs.RUN_TEST == 'true' run: | @@ -173,18 +175,19 @@ jobs: echo $AZURE_OIDC_ISSUER_PRIVATE_KEY | base64 -d > sa.key # Create k3d cluster configuration + # https://k3d.io/v5.7.3/usage/configfile/?h=config+file cat < k3d-config.yaml - apiVersion: k3d.io/v1alpha4 + apiVersion: k3d.io/v1alpha5 kind: Simple agents: 2 options: k3s: extraArgs: - - --kube-apiserver-arg=service-account-issuer=$AZURE_OIDC_ISSUER - - --kube-apiserver-arg=service-account-key-file=/etc/kubernetes/k3s/pki/sa.pub - - --kube-apiserver-arg=service-account-signing-key-file=/etc/kubernetes/k3s/pki/sa.key - - --kube-controller-manager-arg=service-account-private-key-file=/etc/kubernetes/k3s/pki/sa.key - - --disable=traefik@server:0 + - arg: "--kube-apiserver-arg=service-account-issuer=$AZURE_OIDC_ISSUER" + - arg: "--kube-apiserver-arg=service-account-key-file=/etc/kubernetes/k3s/pki/sa.pub" + - arg: "--kube-apiserver-arg=service-account-signing-key-file=/etc/kubernetes/k3s/pki/sa.key" + - arg: "--kube-controller-manager-arg=service-account-private-key-file=/etc/kubernetes/k3s/pki/sa.key" + - arg: "--disable=traefik@server:0" ports: - port: 80:80@loadbalancer volumes: @@ -193,7 +196,8 @@ jobs: registries: create: name: sampleregistry - port: 51351 + host: "0.0.0.0" + hostPort: "51351" EOF # Create the k3d cluster using the configuration file From 6fb942f06c617dbffeed9d03077ef408977ce647 Mon Sep 17 00:00:00 2001 From: willdavsmith Date: Fri, 9 Aug 2024 10:41:32 -0700 Subject: [PATCH 4/7] wip Signed-off-by: willdavsmith --- .github/workflows/test.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 0c9760de..eb3a0d3d 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -189,7 +189,9 @@ jobs: - arg: "--kube-controller-manager-arg=service-account-private-key-file=/etc/kubernetes/k3s/pki/sa.key" - arg: "--disable=traefik@server:0" ports: - - port: 80:80@loadbalancer + - port: 80:80 + nodeFilters: + - loadbalancer volumes: - volume: ./sa.pub:/etc/kubernetes/k3s/pki/sa.pub - volume: ./sa.key:/etc/kubernetes/k3s/pki/sa.key From d6979be60ee277ed4202c6453f54aba96860bbe8 Mon Sep 17 00:00:00 2001 From: willdavsmith Date: Fri, 9 Aug 2024 10:44:19 -0700 Subject: [PATCH 5/7] wip Signed-off-by: willdavsmith --- .github/workflows/test.yaml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index eb3a0d3d..0e1ac324 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -184,10 +184,20 @@ jobs: k3s: extraArgs: - arg: "--kube-apiserver-arg=service-account-issuer=$AZURE_OIDC_ISSUER" + nodeFilters: + - server:* - arg: "--kube-apiserver-arg=service-account-key-file=/etc/kubernetes/k3s/pki/sa.pub" + nodeFilters: + - server:* - arg: "--kube-apiserver-arg=service-account-signing-key-file=/etc/kubernetes/k3s/pki/sa.key" + nodeFilters: + - server:* - arg: "--kube-controller-manager-arg=service-account-private-key-file=/etc/kubernetes/k3s/pki/sa.key" + nodeFilters: + - server:* - arg: "--disable=traefik@server:0" + nodeFilters: + - server:* ports: - port: 80:80 nodeFilters: From c594a833200f2862fbdad067451ea778deb4de78 Mon Sep 17 00:00:00 2001 From: willdavsmith Date: Fri, 9 Aug 2024 10:52:03 -0700 Subject: [PATCH 6/7] wip Signed-off-by: willdavsmith --- .github/workflows/test.yaml | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 0e1ac324..cc712291 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -179,32 +179,37 @@ jobs: cat < k3d-config.yaml apiVersion: k3d.io/v1alpha5 kind: Simple + servers: 1 agents: 2 options: k3s: extraArgs: - arg: "--kube-apiserver-arg=service-account-issuer=$AZURE_OIDC_ISSUER" nodeFilters: - - server:* + - server:0 - arg: "--kube-apiserver-arg=service-account-key-file=/etc/kubernetes/k3s/pki/sa.pub" nodeFilters: - - server:* + - server:0 - arg: "--kube-apiserver-arg=service-account-signing-key-file=/etc/kubernetes/k3s/pki/sa.key" nodeFilters: - - server:* + - server:0 - arg: "--kube-controller-manager-arg=service-account-private-key-file=/etc/kubernetes/k3s/pki/sa.key" nodeFilters: - - server:* + - server:0 - arg: "--disable=traefik@server:0" nodeFilters: - - server:* + - server:0 ports: - port: 80:80 nodeFilters: - loadbalancer volumes: - volume: ./sa.pub:/etc/kubernetes/k3s/pki/sa.pub + nodeFilters: + - server:0 - volume: ./sa.key:/etc/kubernetes/k3s/pki/sa.key + nodeFilters: + - server:0 registries: create: name: sampleregistry From f7dd0abe1ef73588b95cd80e936c8b3007901d53 Mon Sep 17 00:00:00 2001 From: willdavsmith Date: Fri, 9 Aug 2024 10:53:17 -0700 Subject: [PATCH 7/7] wip Signed-off-by: willdavsmith --- .github/workflows/test.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index cc712291..a864e5cf 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -204,10 +204,10 @@ jobs: nodeFilters: - loadbalancer volumes: - - volume: ./sa.pub:/etc/kubernetes/k3s/pki/sa.pub + - volume: sa.pub:/etc/kubernetes/k3s/pki/sa.pub nodeFilters: - server:0 - - volume: ./sa.key:/etc/kubernetes/k3s/pki/sa.key + - volume: sa.key:/etc/kubernetes/k3s/pki/sa.key nodeFilters: - server:0 registries: