From f93532a8e4e10d5e657739db0a05a28313c0d5a2 Mon Sep 17 00:00:00 2001 From: Redouane Kachach Date: Mon, 20 Nov 2023 17:48:26 +0100 Subject: [PATCH 1/3] test: adding support to specify the operator and cluster namepaces adding support to specify operator and cluster namespaces through new command-line arguments: -c for cluster and -o for operator. The script will automatically update the namespace in the corresponding YAML files Signed-off-by: Redouane Kachach --- tests/scripts/create-dev-cluster.sh | 88 +++++++++++++++++++++-------- 1 file changed, 63 insertions(+), 25 deletions(-) diff --git a/tests/scripts/create-dev-cluster.sh b/tests/scripts/create-dev-cluster.sh index fff7195b1443..fe914e0a0353 100755 --- a/tests/scripts/create-dev-cluster.sh +++ b/tests/scripts/create-dev-cluster.sh @@ -1,21 +1,43 @@ #!/usr/bin/env bash - +DEFAULT_NS="rook-ceph" +CLUSTER_FILES="common.yaml operator.yaml cluster-test.yaml cluster-on-pvc-minikube.yaml dashboard-external-http.yaml toolbox.yaml" +MONITORING_FILES="monitoring/prometheus.yaml monitoring/service-monitor.yaml monitoring/exporter-service-monitor.yaml monitoring/prometheus-service.yaml" SCRIPT_ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P) ROOK_EXAMPLES_DIR="${SCRIPT_ROOT}/../../deploy/examples/" init_vars(){ local rook_profile_name=$1 + local rook_cluster_namespace=$2 + local rook_operator_namespace=$3 ROOK_PROFILE_NAME=$rook_profile_name MINIKUBE="minikube --profile $rook_profile_name" KUBECTL="$MINIKUBE kubectl --" + ROOK_CLUSTER_NS=$rook_cluster_namespace + ROOK_OPERATOR_NS=$rook_operator_namespace + echo "Using $ROOK_CLUSTER_NS as cluster namespace.." + echo "Using $ROOK_OPERATOR_NS as operator namespace.." +} + +update_namespaces() { + for file in $CLUSTER_FILES $MONITORING_FILES; do + echo "Updating namespace on $file" + sed -i.bak \ + -e "s/\(.*\):.*# namespace:operator/\1: $ROOK_OPERATOR_NS # namespace:operator/g" \ + -e "s/\(.*\):.*# namespace:cluster/\1: $ROOK_CLUSTER_NS # namespace:cluster/g" \ + -e "s/\(.*serviceaccount\):.*:\(.*\) # serviceaccount:namespace:operator/\1:$ROOK_OPERATOR_NS:\2 # serviceaccount:namespace:operator/g" \ + -e "s/\(.*serviceaccount\):.*:\(.*\) # serviceaccount:namespace:cluster/\1:$ROOK_CLUSTER_NS:\2 # serviceaccount:namespace:cluster/g" \ + -e "s/\(.*\): [-_A-Za-z0-9]*\.\(.*\) # driver:namespace:operator/\1: $ROOK_OPERATOR_NS.\2 # driver:namespace:operator/g" \ + -e "s/\(.*\): [-_A-Za-z0-9]*\.\(.*\) # driver:namespace:cluster/\1: $ROOK_CLUSTER_NS.\2 # driver:namespace:cluster/g" \ + "$file" + done } wait_for_ceph_cluster() { - echo "Waiting for ceph cluster" + echo "Waiting for ceph cluster to enter HEALTH_OK" WAIT_CEPH_CLUSTER_RUNNING=20 - while ! $KUBECTL get cephclusters.ceph.rook.io -n rook-ceph -o jsonpath='{.items[?(@.kind == "CephCluster")].status.ceph.health}' | grep -q "HEALTH_OK"; do - echo "Waiting for Ceph cluster installed" + while ! $KUBECTL get cephclusters.ceph.rook.io -n "$ROOK_CLUSTER_NS" -o jsonpath='{.items[?(@.kind == "CephCluster")].status.ceph.health}' | grep -q "HEALTH_OK"; do + echo "Waiting for Ceph cluster to enter HEALTH_OK" sleep ${WAIT_CEPH_CLUSTER_RUNNING} done echo "Ceph cluster installed and running" @@ -43,16 +65,17 @@ get_minikube_driver() { show_info() { local monitoring_enabled=$1 - DASHBOARD_PASSWORD=$($KUBECTL -n rook-ceph get secret rook-ceph-dashboard-password -o jsonpath="{['data']['password']}" | base64 --decode && echo) - IP_ADDR=$($KUBECTL get po --selector="app=rook-ceph-mgr" -n rook-ceph --output jsonpath='{.items[*].status.hostIP}') - PORT="$($KUBECTL -n rook-ceph -o=jsonpath='{.spec.ports[?(@.name == "dashboard")].nodePort}' get services rook-ceph-mgr-dashboard-external-http)" + DASHBOARD_PASSWORD=$($KUBECTL -n "$ROOK_CLUSTER_NS" get secret rook-ceph-dashboard-password -o jsonpath="{['data']['password']}" | base64 --decode && echo) + IP_ADDR=$($KUBECTL get po --selector="app=rook-ceph-mgr" -n "$ROOK_CLUSTER_NS" --output jsonpath='{.items[*].status.hostIP}') + PORT="$($KUBECTL -n "$ROOK_CLUSTER_NS" -o=jsonpath='{.spec.ports[?(@.name == "dashboard")].nodePort}' get services rook-ceph-mgr-dashboard-external-http)" BASE_URL="http://$IP_ADDR:$PORT" echo "===========================" - echo "Ceph Dashboard: " - echo " IP_ADDRESS: $BASE_URL" - echo " PASSWORD: $DASHBOARD_PASSWORD" + echo "Ceph Dashboard:" + echo " IP_ADDR : $BASE_URL" + echo " USER : admin" + echo " PASSWORD : $DASHBOARD_PASSWORD" if [ "$monitoring_enabled" = true ]; then - PROMETHEUS_API_HOST="http://$(kubectl -n rook-ceph -o jsonpath='{.status.hostIP}' get pod prometheus-rook-prometheus-0):30900" + PROMETHEUS_API_HOST="http://$(kubectl -n "$ROOK_CLUSTER_NS" -o jsonpath='{.status.hostIP}' get pod prometheus-rook-prometheus-0):30900" echo "Prometheus Dashboard: " echo " API_HOST: $PROMETHEUS_API_HOST" fi @@ -87,6 +110,10 @@ setup_minikube_env() { create_rook_cluster() { echo "Creating cluster" + # create operator namespace if it doesn't exist + if ! kubectl get namespace "$ROOK_OPERATOR_NS" &> /dev/null; then + kubectl create namespace "$ROOK_OPERATOR_NS" + fi $KUBECTL apply -f crds.yaml -f common.yaml -f operator.yaml $KUBECTL apply -f cluster-test.yaml -f toolbox.yaml $KUBECTL apply -f dashboard-external-http.yaml @@ -101,20 +128,20 @@ check_examples_dir() { } wait_for_rook_operator() { - echo "Waiting for rook operator" - $KUBECTL rollout status deployment rook-ceph-operator -n rook-ceph --timeout=180s - while ! $KUBECTL get cephclusters.ceph.rook.io -n rook-ceph -o jsonpath='{.items[?(@.kind == "CephCluster")].status.phase}' | grep -q "Ready"; do - echo "Waiting for cluster to be ready..." + echo "Waiting for rook operator..." + $KUBECTL rollout status deployment rook-ceph-operator -n "$ROOK_OPERATOR_NS" --timeout=180s + while ! $KUBECTL get cephclusters.ceph.rook.io -n "$ROOK_CLUSTER_NS" -o jsonpath='{.items[?(@.kind == "CephCluster")].status.phase}' | grep -q "Ready"; do + echo "Waiting for ceph cluster to become ready..." sleep 20 done } enable_rook_orchestrator() { echo "Enabling rook orchestrator" - $KUBECTL rollout status deployment rook-ceph-tools -n rook-ceph --timeout=30s - $KUBECTL -n rook-ceph exec -it deploy/rook-ceph-tools -- ceph mgr module enable rook - $KUBECTL -n rook-ceph exec -it deploy/rook-ceph-tools -- ceph orch set backend rook - $KUBECTL -n rook-ceph exec -it deploy/rook-ceph-tools -- ceph orch status + $KUBECTL rollout status deployment rook-ceph-tools -n "$ROOK_CLUSTER_NS" --timeout=30s + $KUBECTL -n "$ROOK_CLUSTER_NS" exec -it deploy/rook-ceph-tools -- ceph mgr module enable rook + $KUBECTL -n "$ROOK_CLUSTER_NS" exec -it deploy/rook-ceph-tools -- ceph orch set backend rook + $KUBECTL -n "$ROOK_CLUSTER_NS" exec -it deploy/rook-ceph-tools -- ceph orch status } enable_monitoring() { @@ -125,17 +152,20 @@ enable_monitoring() { $KUBECTL apply -f monitoring/exporter-service-monitor.yaml $KUBECTL apply -f monitoring/prometheus.yaml $KUBECTL apply -f monitoring/prometheus-service.yaml - PROMETHEUS_API_HOST="http://$(kubectl -n rook-ceph -o jsonpath='{.status.hostIP}' get pod prometheus-rook-prometheus-0):30900" - $KUBECTL -n rook-ceph exec -it deploy/rook-ceph-tools -- ceph dashboard set-prometheus-api-host "$PROMETHEUS_API_HOST" + PROMETHEUS_API_HOST="http://$(kubectl -n "$ROOK_CLUSTER_NS" -o jsonpath='{.status.hostIP}' get pod prometheus-rook-prometheus-0):30900" + $KUBECTL -n "$ROOK_CLUSTER_NS" exec -it deploy/rook-ceph-tools -- ceph dashboard set-prometheus-api-host "$PROMETHEUS_API_HOST" } show_usage() { echo "" echo " Usage: $(basename "$0") [-r] [-m] [-p ] [-d /path/to/rook-examples/dir]" + echo " -f Force cluster creation by deleting minikube profile" echo " -r Enable rook orchestrator" echo " -m Enable monitoring" echo " -p Specify the minikube profile name" echo " -d value Path to Rook examples directory (i.e github.com/rook/rook/deploy/examples)" + echo " -c " + echo " -o " } invocation_error() { @@ -147,9 +177,7 @@ invocation_error() { #################################################################### ################# MAIN ############################################# - - -while getopts ":hrmfd:p:" opt; do +while getopts ":hrmfd:p:c:o:" opt; do case $opt in h) show_usage @@ -170,6 +198,12 @@ while getopts ":hrmfd:p:" opt; do p) minikube_profile_name="$OPTARG" ;; + c) + rook_cluster_ns="$OPTARG" + ;; + o) + rook_operator_ns="$OPTARG" + ;; \?) invocation_error "Invalid option: -$OPTARG" ;; @@ -183,12 +217,16 @@ echo "Using '$ROOK_EXAMPLES_DIR' as examples directory.." cd "$ROOK_EXAMPLES_DIR" || exit check_examples_dir -init_vars "${minikube_profile_name:-rook}" +init_vars "${minikube_profile_name:-rook}" "${rook_cluster_ns:-$DEFAULT_NS}" "${rook_operator_ns:-$DEFAULT_NS}" if [ -z "$force_minikube" ]; then check_minikube_exists fi +if [ "$rook_cluster_ns" != "$DEFAULT_NS" ] || [ "$rook_operator_ns" != "$DEFAULT_NS" ]; then + update_namespaces +fi + setup_minikube_env create_rook_cluster wait_for_rook_operator From f60cdde460633e41320999fd63f567f410a760ca Mon Sep 17 00:00:00 2001 From: Redouane Kachach Date: Thu, 23 Nov 2023 13:49:10 +0100 Subject: [PATCH 2/3] test: increasing rook-ceph-tools deployment timeout to 90s Signed-off-by: Redouane Kachach --- tests/scripts/create-dev-cluster.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/scripts/create-dev-cluster.sh b/tests/scripts/create-dev-cluster.sh index fe914e0a0353..ba9e468864c4 100755 --- a/tests/scripts/create-dev-cluster.sh +++ b/tests/scripts/create-dev-cluster.sh @@ -138,7 +138,7 @@ wait_for_rook_operator() { enable_rook_orchestrator() { echo "Enabling rook orchestrator" - $KUBECTL rollout status deployment rook-ceph-tools -n "$ROOK_CLUSTER_NS" --timeout=30s + $KUBECTL rollout status deployment rook-ceph-tools -n "$ROOK_CLUSTER_NS" --timeout=90s $KUBECTL -n "$ROOK_CLUSTER_NS" exec -it deploy/rook-ceph-tools -- ceph mgr module enable rook $KUBECTL -n "$ROOK_CLUSTER_NS" exec -it deploy/rook-ceph-tools -- ceph orch set backend rook $KUBECTL -n "$ROOK_CLUSTER_NS" exec -it deploy/rook-ceph-tools -- ceph orch status From 734d1057902a90055a2641d36b37335619da4612 Mon Sep 17 00:00:00 2001 From: Redouane Kachach Date: Thu, 23 Nov 2023 13:48:39 +0100 Subject: [PATCH 3/3] test: applying adding rbac.yaml when enabling monitoring applying adding rbac.yaml when enabling monitoring to avoid permission issues when accessing servicemonitors Signed-off-by: Redouane Kachach --- tests/scripts/create-dev-cluster.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/scripts/create-dev-cluster.sh b/tests/scripts/create-dev-cluster.sh index ba9e468864c4..65463848c83f 100755 --- a/tests/scripts/create-dev-cluster.sh +++ b/tests/scripts/create-dev-cluster.sh @@ -2,7 +2,7 @@ DEFAULT_NS="rook-ceph" CLUSTER_FILES="common.yaml operator.yaml cluster-test.yaml cluster-on-pvc-minikube.yaml dashboard-external-http.yaml toolbox.yaml" -MONITORING_FILES="monitoring/prometheus.yaml monitoring/service-monitor.yaml monitoring/exporter-service-monitor.yaml monitoring/prometheus-service.yaml" +MONITORING_FILES="monitoring/prometheus.yaml monitoring/service-monitor.yaml monitoring/exporter-service-monitor.yaml monitoring/prometheus-service.yaml monitoring/rbac.yaml" SCRIPT_ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P) ROOK_EXAMPLES_DIR="${SCRIPT_ROOT}/../../deploy/examples/" @@ -148,6 +148,7 @@ enable_monitoring() { echo "Enabling monitoring" $KUBECTL apply -f https://raw.githubusercontent.com/coreos/prometheus-operator/v0.40.0/bundle.yaml $KUBECTL wait --for=condition=ready pod -l app.kubernetes.io/name=prometheus-operator --timeout=30s + $KUBECTL apply -f monitoring/rbac.yaml $KUBECTL apply -f monitoring/service-monitor.yaml $KUBECTL apply -f monitoring/exporter-service-monitor.yaml $KUBECTL apply -f monitoring/prometheus.yaml