Merge pull request #5519 from zhzhuang-zju/operator

add ci for karmada operator

.github/workflows/operator.yaml

@@ -0,0 +1,82 @@
+name: Operator
+on:
+  # Run this workflow every time a new commit pushed to upstream/fork repository.
+  # Run workflow on fork repository will help contributors find and resolve issues before sending a PR.
+  push:
+    # Exclude branches created by Dependabot to avoid triggering current workflow
+    # for PRs initiated by Dependabot.
+    branches-ignore:
+      - 'dependabot/**'
+  pull_request:
+# https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#concurrency
+concurrency:
+  group: ${{ github.workflow }}-${{ github.actor }}-${{ github.head_ref || github.run_id }}
+  cancel-in-progress: true
+permissions:
+  contents: read  # for actions/checkout to fetch code
+jobs:
+  test-on-kubernetes-matrix:
+    name: Test on Kubernetes
+    runs-on: ubuntu-22.04
+    strategy:
+      fail-fast: false
+      matrix:
+        # Here support the latest three minor releases of Kubernetes, this can be considered to be roughly
+        # the same as the End of Life of the Kubernetes release: https://kubernetes.io/releases/
+        # Please remember to update the CI Schedule Workflow when we add a new version.
+        k8s: [ v1.29.0, v1.30.0, v1.31.0 ]
+    steps:
+      # Free up disk space on Ubuntu
+      - name: Free Disk Space (Ubuntu)
+        uses: jlumbroso/free-disk-space@main
+        with:
+          # this might remove tools that are actually needed, if set to "true" but frees about 6 GB
+          tool-cache: false
+          # all of these default to true, but feel free to set to "false" if necessary for your workflow
+          android: true
+          dotnet: true
+          haskell: true
+          large-packages: false
+          docker-images: false
+          swap-storage: false
+      - name: checkout code
+        uses: actions/checkout@v4
+        with:
+          # Number of commits to fetch. 0 indicates all history for all branches and tags.
+          # We need to guess version via git tags.
+          fetch-depth: 0
+      - name: install Go
+        uses: actions/setup-go@v5
+        with:
+          go-version-file: go.mod
+      - name: setup operator test environment
+        run: |
+          export CLUSTER_VERSION=kindest/node:${{ matrix.k8s }}
+          hack/local-up-karmada-by-operator.sh
+      - name: run operator test
+        run: |
+          # run a single e2e
+          export KUBECONFIG=${HOME}/.kube/karmada.config
+          kubectl config use-context karmada-apiserver
+          GO111MODULE=on go install github.com/onsi/ginkgo/v2/ginkgo
+          ginkgo -v --race --trace -p  --focus="[BasicPropagation] propagation testing deployment propagation testing"  ./test/e2e/
+      - name: export logs
+        if: always()
+        run: |
+          export ARTIFACTS_PATH=${{ github.workspace }}/karmada-operator-test-logs/${{ matrix.k8s }}/
+          mkdir -p $ARTIFACTS_PATH
+
+          mkdir -p $ARTIFACTS_PATH/karmada-host
+          kind export logs --name=karmada-host $ARTIFACTS_PATH/karmada-host
+      - name: upload logs
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: karmada_operator_test_logs_${{ matrix.k8s }}
+          path: ${{ github.workspace }}/karmada-operator-test-logs/${{ matrix.k8s }}/
+      - name: upload kind logs
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: karmada_kind_log_${{ matrix.k8s }}
+          path: /tmp/karmada/

hack/deploy-karmada-by-operator.sh

@@ -0,0 +1,123 @@
+#!/usr/bin/env bash
+# Copyright 2024 The Karmada Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+set -o errexit
+set -o nounset
+
+# This script deploy karmada instance to any cluster you want via karmada-operator.
+# This script depends on utils in: ${REPO_ROOT}/hack/util.sh
+
+REPO_ROOT=$(dirname "${BASH_SOURCE[0]}")/..
+source ${REPO_ROOT}/hack/util.sh
+
+KARMADA_SYSTEM_NAMESPACE="karmada-system"
+KARMADA_INSTANCE_NAME=${KARMADA_INSTANCE_NAME:-"karmada-demo"}
+KARMADA_INSTANCE_NAMESPACE=${KARMADA_INSTANCE_NAMESPACE:-"test"}
+
+CERT_DIR=${CERT_DIR:-"${HOME}/.karmada"}
+mkdir -p "${CERT_DIR}" &>/dev/null ||  mkdir -p "${CERT_DIR}"
+rm -f "${CERT_DIR}/*" &>/dev/null ||  rm -f "${CERT_DIR}/*"
+
+function usage() {
+  echo "This script deploys karmada instance to a given cluster via karmada-operator."
+  echo "Note: This script is an internal script and is not intended used by end-users."
+  echo "Usage: hack/deploy-karmada-by-operator.sh <KUBECONFIG> <CONTEXT_NAME> <KARMADA_CONTEXT_NAME> <KARMADA_IMAGE_TAG> <ADDON_NEEDED> <CRD_DOWNLOAD_URL>"
+  echo "Example: hack/deploy-karmada-by-operator.sh ~/.kube/members.config member1 karmada-apiserver v1.11.0 true https://github.com/karmada-io/karmada/releases/download/v1.11.0/crds.tar.gz"
+  echo -e "Parameters:\n\tKUBECONFIG\t\tYour cluster's kubeconfig that you want to install to"
+  echo -e "\tCONTEXT_NAME\t\tThe name of context in 'kubeconfig'"
+  echo -e "\tKARMADA_CONTEXT_NAME\t\tThe context name of karmada instance, and different Karmada instances must have unique contexts to avoid being overwritten.'"
+  echo -e "\tKARMADA_IMAGE_TAG\t\tThe tag of image'"
+  echo -e "\tADDON_NEEDED\t\tWhether you need to install addons(KarmadaSearch&karmadaDescheduler), optional, defaults to false."
+  echo -e "\tCRD_DOWNLOAD_URL\t\tThe download url for CRDs, optional.'"
+}
+
+if [[ $# -le 4 ]]; then
+  usage
+  exit 1
+fi
+
+# check config file existence
+HOST_CLUSTER_KUBECONFIG=$1
+if [[ ! -f "${HOST_CLUSTER_KUBECONFIG}" ]]; then
+  echo -e "ERROR: failed to get kubernetes config file: '${HOST_CLUSTER_KUBECONFIG}', not existed.\n"
+  usage
+  exit 1
+fi
+
+# check context existence
+CONTEXT_NAME=$2
+if ! kubectl --kubeconfig="${HOST_CLUSTER_KUBECONFIG}" config get-contexts "${CONTEXT_NAME}" > /dev/null 2>&1;
+then
+  echo -e "ERROR: failed to get context: '${CONTEXT_NAME}' not in ${HOST_CLUSTER_KUBECONFIG}. \n"
+  usage
+  exit 1
+fi
+
+# check for duplicate karmada context name.
+KARMADA_CONTEXT_NAME=$3
+if kubectl --kubeconfig="${HOST_CLUSTER_KUBECONFIG}" config get-contexts "${KARMADA_CONTEXT_NAME}" > /dev/null 2>&1;
+then
+  echo -e "ERROR: context: '${KARMADA_CONTEXT_NAME}' already exists in ${HOST_CLUSTER_KUBECONFIG}. \n"
+  usage
+  exit 1
+fi
+
+TEMP_PATH_BOOTSTRAP=$(mktemp -d)
+trap '{ rm -rf ${TEMP_PATH_BOOTSTRAP}; }' EXIT
+cp -rf "${REPO_ROOT}"/operator/config/samples/karmada-sample.yaml "${TEMP_PATH_BOOTSTRAP}"/karmada-sample-tmp.yaml
+
+if kubectl --kubeconfig="${HOST_CLUSTER_KUBECONFIG}" --context="${CONTEXT_NAME}" get namespace ${KARMADA_INSTANCE_NAMESPACE} > /dev/null 2>&1; then
+  echo "Namespace '${KARMADA_INSTANCE_NAMESPACE}' already exists."
+else
+  echo "Namespace '${KARMADA_INSTANCE_NAMESPACE}' does not exist. Creating now..."
+  kubectl --kubeconfig="${HOST_CLUSTER_KUBECONFIG}" --context="${CONTEXT_NAME}" create ns ${KARMADA_INSTANCE_NAMESPACE}
+fi
+
+# modify `karmada-sample.yaml` based on custom configuration.
+ADDON_NEEDED=${5:-false}
+# if choosing install addons, append karmadaSearch and karmadaDescheduler to 'karmada-sample.yaml'
+if [ ${ADDON_NEEDED} ]; then
+    echo -e '    karmadaDescheduler:\n      imageRepository: docker.io/karmada/karmada-descheduler\n      imageTag: {{image_tag}}\n      replicas: 1' >> "${TEMP_PATH_BOOTSTRAP}"/karmada-sample-tmp.yaml
+    echo -e '    karmadaSearch:\n      imageRepository: docker.io/karmada/karmada-search\n      imageTag: {{image_tag}}\n      replicas: 1' >> "${TEMP_PATH_BOOTSTRAP}"/karmada-sample-tmp.yaml
+fi
+
+IMAGE_TAG=$4
+sed -i'' -e "s/{{image_tag}}/${IMAGE_TAG}/g" "${TEMP_PATH_BOOTSTRAP}"/karmada-sample-tmp.yaml
+sed -i'' -e "s/{{karmada_instance_name}}/${KARMADA_INSTANCE_NAME}/g" "${TEMP_PATH_BOOTSTRAP}"/karmada-sample-tmp.yaml
+sed -i'' -e "s/{{karmada_instance_namespace}}/${KARMADA_INSTANCE_NAMESPACE}/g" "${TEMP_PATH_BOOTSTRAP}"/karmada-sample-tmp.yaml
+
+CRD_DOWNLOAD_URL=${6:-""}
+if [[ -z ${CRD_DOWNLOAD_URL} ]]; then
+	sed -i'' -e "s/{{crd_tarball}}/""/g" "${TEMP_PATH_BOOTSTRAP}"/karmada-sample-tmp.yaml
+else
+	CRD_TAR_BALL="\n    httpSource:\n      url: ${CRD_DOWNLOAD_URL}"
+  awk -v pattern="{{crd_tarball}}" -v replacement="${CRD_TAR_BALL}" '{ gsub(pattern, replacement); print }' "${TEMP_PATH_BOOTSTRAP}"/karmada-sample-tmp.yaml > "${TEMP_PATH_BOOTSTRAP}"/temp && mv "${TEMP_PATH_BOOTSTRAP}"/temp "${TEMP_PATH_BOOTSTRAP}"/karmada-sample-tmp.yaml
+fi
+
+# create and wait for karmada instance to be ready
+kubectl --kubeconfig="${HOST_CLUSTER_KUBECONFIG}" --context="${CONTEXT_NAME}" apply -f "${TEMP_PATH_BOOTSTRAP}"/karmada-sample-tmp.yaml
+kubectl --kubeconfig="${HOST_CLUSTER_KUBECONFIG}" --context="${CONTEXT_NAME}" wait --for=condition=Ready --timeout=1000s karmada ${KARMADA_INSTANCE_NAME} -n ${KARMADA_INSTANCE_NAMESPACE}
+
+# generate kubeconfig for karmada instance
+kubectl --kubeconfig="${HOST_CLUSTER_KUBECONFIG}" --context="${CONTEXT_NAME}" get secret -n ${KARMADA_INSTANCE_NAMESPACE} ${KARMADA_INSTANCE_NAME}-admin-config -o jsonpath={.data.kubeconfig} | base64 -d > ~/.kube/${KARMADA_INSTANCE_NAME}-${KARMADA_INSTANCE_NAMESPACE}-tmp-apiserver.config
+cat ~/.kube/${KARMADA_INSTANCE_NAME}-${KARMADA_INSTANCE_NAMESPACE}-tmp-apiserver.config| grep "certificate-authority-data"| awk '{print $2}'| base64 -d  > ${CERT_DIR}/ca.crt
+cat ~/.kube/${KARMADA_INSTANCE_NAME}-${KARMADA_INSTANCE_NAMESPACE}-tmp-apiserver.config| grep "client-certificate-data"| awk '{print $2}'| base64 -d  > ${CERT_DIR}/karmada.crt
+cat ~/.kube/${KARMADA_INSTANCE_NAME}-${KARMADA_INSTANCE_NAMESPACE}-tmp-apiserver.config| grep "client-key-data"| awk '{print $2}'| base64 -d  > ${CERT_DIR}/karmada.key
+KARMADA_APISERVER=$(cat ~/.kube/${KARMADA_INSTANCE_NAME}-${KARMADA_INSTANCE_NAMESPACE}-tmp-apiserver.config| grep "server:"| awk '{print $2}')
+
+# write karmada api server config to kubeconfig file
+util::append_client_kubeconfig "${HOST_CLUSTER_KUBECONFIG}" "${CERT_DIR}/ca.crt" "${CERT_DIR}/karmada.crt" "${CERT_DIR}/karmada.key" "${KARMADA_APISERVER}" ${KARMADA_CONTEXT_NAME}
+rm ~/.kube/${KARMADA_INSTANCE_NAME}-${KARMADA_INSTANCE_NAMESPACE}-tmp-apiserver.config

hack/deploy-karmada-operator.sh

@@ -0,0 +1,73 @@
+#!/usr/bin/env bash
+# Copyright 2024 The Karmada Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+set -o errexit
+set -o nounset
+
+REPO_ROOT=$(dirname "${BASH_SOURCE[0]}")/..
+source ${REPO_ROOT}/hack/util.sh
+KARMADA_SYSTEM_NAMESPACE="karmada-system"
+
+function usage() {
+  echo "This script will deploy karmada-operator on the specified cluster"
+  echo "Usage: hack/deploy-karmada-operator.sh <KUBECONFIG> <CONTEXT_NAME>"
+  echo "Example: hack/deploy-karmada-operator.sh ~/.kube/config karmada-host"
+}
+
+if [[ $# -ne 2 ]]; then
+  usage
+  exit 1
+fi
+
+# check kube config file existence
+if [[ ! -f "${1}" ]]; then
+  echo -e "ERROR: failed to get kubernetes config file: '${1}', not existed.\n"
+  usage
+  exit 1
+fi
+KUBECONFIG=$1
+
+# check context existence
+if ! kubectl config get-contexts "${2}" --kubeconfig="${KUBECONFIG}" > /dev/null 2>&1;
+then
+  echo -e "ERROR: failed to get context: '${2}' not in ${KUBECONFIG}. \n"
+  usage
+  exit 1
+fi
+CONTEXT_NAME=$2
+
+# make images
+export VERSION="latest"
+export REGISTRY="docker.io/karmada"
+make image-karmada-operator GOOS="linux" --directory=.
+
+# load the karmada-operator images
+kind load docker-image "${REGISTRY}/karmada-controller-manager:${VERSION}" --name="${CONTEXT_NAME}"
+
+# create namespace `karmada-system`
+kubectl --kubeconfig="${KUBECONFIG}" --context="${CONTEXT_NAME}" apply -f "${REPO_ROOT}/artifacts/deploy/namespace.yaml"
+
+# install Karmada operator crds
+kubectl --kubeconfig="${KUBECONFIG}" --context="${CONTEXT_NAME}" apply -f operator/config/crds/
+
+# deploy karmada-operator
+kubectl --kubeconfig="${KUBECONFIG}" --context="${CONTEXT_NAME}" apply -f "${REPO_ROOT}/operator/config/deploy/karmada-operator-clusterrole.yaml"
+kubectl --kubeconfig="${KUBECONFIG}" --context="${CONTEXT_NAME}" apply -f "${REPO_ROOT}/operator/config/deploy/karmada-operator-clusterrolebinding.yaml"
+kubectl --kubeconfig="${KUBECONFIG}" --context="${CONTEXT_NAME}" apply -f "${REPO_ROOT}/operator/config/deploy/karmada-operator-serviceaccount.yaml"
+kubectl --kubeconfig="${KUBECONFIG}" --context="${CONTEXT_NAME}" apply -f "${REPO_ROOT}/operator/config/deploy/karmada-operator-deployment.yaml"
+
+# wait karmada-operator ready
+kubectl --kubeconfig="${KUBECONFIG}" --context="${CONTEXT_NAME}" wait --for=condition=Ready --timeout=30s pods -l karmada-app=karmada-operator -n ${KARMADA_SYSTEM_NAMESPACE}

hack/deploy-karmada.sh

@@ -265,7 +265,7 @@ else
 fi
 
 # write karmada api server config to kubeconfig file
-util::append_client_kubeconfig "${HOST_CLUSTER_KUBECONFIG}" "${ROOT_CA_FILE}" "${CERT_DIR}/client.crt" "${CERT_DIR}/client.key" "${KARMADA_APISERVER_IP}" "${KARMADA_APISERVER_SECURE_PORT}" karmada-apiserver
+util::append_client_kubeconfig "${HOST_CLUSTER_KUBECONFIG}" "${ROOT_CA_FILE}" "${CERT_DIR}/client.crt" "${CERT_DIR}/client.key" "https://${KARMADA_APISERVER_IP}:${KARMADA_APISERVER_SECURE_PORT}" karmada-apiserver
 
 # deploy kube controller manager
 cp "${REPO_ROOT}"/artifacts/deploy/kube-controller-manager.yaml "${TEMP_PATH_APISERVER}"/kube-controller-manager.yaml