feat: add io delay chaos meta test scripts (#15769)

* feat: add io delay chaos meta test scripts * feat: add io delay chaos meta test scripts, make clippy happy * z * z --------- Co-authored-by: everpcpc <[email protected]>
databendlabs · Jun 12, 2024 · 498a7f6 · 498a7f6
1 parent fd655db
commit 498a7f6
Show file tree

Hide file tree

Showing 15 changed files with 643 additions and 1 deletion.
diff --git a/.github/actions/artifact_meta_container_failure/action.yml b/.github/actions/artifact_meta_container_failure/action.yml
@@ -0,0 +1,23 @@
+name: "Upload meta containers failure Artifacts"
+description: "Upload meta containers Artifacts"
+inputs:
+  name:
+    description: ""
+    required: true
+runs:
+  using: "composite"
+  steps:
+    - name: pack failure artifacts
+      shell: bash
+      run: |
+        mkdir -p logs
+        for name in "test-databend-meta-0" "test-databend-meta-1" "test-databend-meta-2" "databend-metaverifier"
+        do
+          echo "cat logs of $name"
+          kubectl exec -i $name -n databend -- /cat-logs.sh > logs/$name.log
+        done
+        tar -zcf logs/failure-${{ inputs.name }}.tar.gz logs/*
+    - uses: actions/upload-artifact@v4
+      with:
+        name: ${{ inputs.name }}
+        path: logs/failure-${{ inputs.name }}.tar.gz
diff --git a/.github/actions/io_delay_chaos/action.yml b/.github/actions/io_delay_chaos/action.yml
@@ -0,0 +1,27 @@
+name: "Chaos Test meta service"
+description: "Running stateless tests in management mode"
+inputs:
+  target:
+    description: ""
+    required: true
+    default: "x86_64-unknown-linux-gnu"
+runs:
+  using: "composite"
+  steps:
+    - name: Download artifact
+      uses: ./.github/actions/artifact_download
+      with:
+        sha: ${{ github.sha }}
+        target: ${{ inputs.target }}
+        artifacts: meta,metactl,metaverifier
+
+    - name: Meta Service IO Delay tests
+      shell: bash
+      run: |
+        ./scripts/ci/meta-chaos/ci-io-delay-meta-test.sh
+
+    - name: Upload failure
+      if: failure()
+      uses: ./.github/actions/artifact_meta_container_failure
+      with:
+        name: meta-io-delay-chaos
diff --git a/.github/workflows/meta.yml b/.github/workflows/meta.yml
@@ -0,0 +1,65 @@
+name: Meta
+
+on:
+  pull_request_target:
+    types:
+      - labeled
+    branches:
+      - main
+
+permissions:
+  id-token: write
+  pull-requests: write
+  contents: read
+
+env:
+  BUILD_PROFILE: release
+  RUNNER_PROVIDER: gcp
+
+jobs:
+  info:
+    if: contains(github.event.pull_request.labels.*.name, 'ci-meta-chaos')
+    runs-on: ubuntu-latest
+    outputs:
+      sha: ${{ steps.sha.outputs.sha }}
+      target: ${{ steps.sha.outputs.target }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          ref: "refs/pull/${{ github.event.number }}/merge"
+      - name: Get SHA
+        shell: bash
+        id: sha
+        run: |
+          echo "sha=$(git rev-parse --verify HEAD)" > $GITHUB_OUTPUT
+
+  build:
+    needs: info
+    runs-on: [self-hosted, "${{ matrix.runner }}", Linux, 16c32g, gcp]
+    strategy:
+      matrix:
+        include:
+          - { arch: x86_64, runner: X64 }
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          ref: "refs/pull/${{ github.event.number }}/merge"
+      - name: Build Release
+        uses: ./.github/actions/build_linux
+        timeout-minutes: 60
+        with:
+          sha: ${{ needs.info.outputs.sha }}
+          target: ${{ matrix.arch }}-unknown-linux-gnu
+          artifacts: meta,metactl,metaverifier
+
+  chaos:
+    needs: [info, build]
+    runs-on: [self-hosted, X64, Linux, 8c16g, gcp]
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: "refs/pull/${{ github.event.number }}/merge"
+      - uses: ./.github/actions/io_delay_chaos
+        timeout-minutes: 20
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -122,7 +122,7 @@ jobs:
         with:
           sha: ${{ github.sha }}
           target: ${{ matrix.target }}
-          artifacts: sqllogictests,sqlsmith,metactl,meta,query
+          artifacts: sqllogictests,sqlsmith,query,meta,metactl,metaverifier
       - name: Basic Sqllogic Test
         if: matrix.target != 'aarch64-unknown-linux-musl'
         shell: bash
@@ -627,6 +627,16 @@ jobs:
         with:
           name: test-sqlsmith
 
+  meta_chaos:
+    needs: [create_release, build_default]
+    runs-on: [self-hosted, X64, Linux, 8c16g, gcp]
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ needs.create_release.outputs.sha }}
+      - uses: ./.github/actions/io_delay_chaos
+        timeout-minutes: 20
+
   benchmark:
     needs: [create_release, docker_service, notify]
     uses: ./.github/workflows/reuse.benchmark.yml

diff --git a/docker/meta-chaos/meta.Dockerfile b/docker/meta-chaos/meta.Dockerfile
@@ -0,0 +1,14 @@
+FROM debian:bookworm
+
+ARG TARGETPLATFORM
+ENV TERM=dumb
+ENV DEBIAN_FRONTEND=noninteractive
+RUN apt-get update -y && \
+    apt-get install -y apt-transport-https ca-certificates && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/* && \
+    rm -rf /var/cache/apt/*
+COPY ./distro/$TARGETPLATFORM/databend-meta /databend-meta
+COPY ./distro/$TARGETPLATFORM/databend-metactl /databend-metactl
+COPY ./distro/$TARGETPLATFORM/cat-logs.sh /cat-logs.sh
+ENTRYPOINT ["/databend-meta"]
diff --git a/docker/meta-chaos/verifier.Dockerfile b/docker/meta-chaos/verifier.Dockerfile
@@ -0,0 +1,14 @@
+FROM debian:bookworm
+
+ARG TARGETPLATFORM
+ENV TERM=dumb
+ENV DEBIAN_FRONTEND=noninteractive
+RUN apt-get update -y && \
+    apt-get install -y apt-transport-https ca-certificates curl && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/* && \
+    rm -rf /var/cache/apt/*
+
+COPY ./distro/$TARGETPLATFORM/* /
+
+ENTRYPOINT ["/bin/sh"]
diff --git a/scripts/ci/ci-setup-chaos-meta.sh b/scripts/ci/ci-setup-chaos-meta.sh
@@ -0,0 +1,93 @@
+#!/bin/bash
+# Copyright 2020-2021 The Databend Authors.
+# SPDX-License-Identifier: Apache-2.0.
+
+set -ex
+
+BUILD_PROFILE=${BUILD_PROFILE:-debug}
+
+curl -s https://raw.githubusercontent.com/k3d-io/k3d/main/install.sh | TAG=v5.6.0 bash
+k3d registry create registry.localhost --port 0.0.0.0:5111 -i registry:latest
+k3d cluster create --config ./scripts/ci/meta-chaos/k3d.yaml meta-chaos
+
+echo "127.0.0.1 k3d-registry.localhost" | sudo tee -a /etc/hosts
+
+if kubectl version --client; then
+    echo "kubectl client already installed"
+else
+    echo "install kubectl client"
+    curl -LO "https://dl.k8s.io/release/v1.29.5/bin/linux/amd64/kubectl"
+    chmod +x kubectl
+    sudo mv kubectl /usr/local/bin/
+fi
+
+if helm version; then
+    echo "helm already installed"
+else
+    echo "install helm"
+    curl -fsSL -o get_helm.sh https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3
+    chmod 700 get_helm.sh
+    ./get_helm.sh
+fi
+
+echo "make databend-meta image"
+ls -lh ./target/"${BUILD_PROFILE}"
+mkdir -p temp/distro/amd64
+cp ./target/"${BUILD_PROFILE}"/databend-meta ./temp/distro/amd64
+cp ./target/"${BUILD_PROFILE}"/databend-metactl ./temp/distro/amd64
+cp tests/metaverifier/cat-logs.sh ./temp/distro/amd64
+docker build -t databend-meta:meta-chaos --build-arg TARGETPLATFORM="amd64" -f ./docker/chaos-meta/meta.Dockerfile temp
+docker tag databend-meta:meta-chaos k3d-registry.localhost:5111/databend-meta:meta-chaos
+docker push k3d-registry.localhost:5111/databend-meta:meta-chaos
+
+echo "make databend-metaverifier image"
+rm -rf temp/distro/amd64/*
+cp ./target/"${BUILD_PROFILE}"/databend-metaverifier ./temp/distro/amd64
+cp tests/metaverifier/start-verifier.sh ./temp/distro/amd64
+cp tests/metaverifier/cat-logs.sh ./temp/distro/amd64
+docker build -t databend-metaverifier:meta-chaos --build-arg TARGETPLATFORM="amd64" -f ./docker/chaos-meta/verifier.Dockerfile temp
+docker tag databend-metaverifier:meta-chaos k3d-registry.localhost:5111/databend-metaverifier:meta-chaos
+docker push k3d-registry.localhost:5111/databend-metaverifier:meta-chaos
+
+echo "install chaos mesh on k3d"
+curl -sSL https://mirrors.chaos-mesh.org/v2.6.3/install.sh | bash -s -- --k3s
+
+kubectl get pods -A -o wide
+kubectl get pvc -A
+
+echo "kubectl delete databend-meta pvc"
+kubectl delete pvc --namespace databend data-test-databend-meta-0 data-test-databend-meta-1 data-test-databend-meta-2 --ignore-not-found
+
+helm repo add databend https://charts.databend.rs
+helm install test databend/databend-meta \
+    --namespace databend \
+    --create-namespace \
+    --values scripts/ci/meta-chaos/meta-ha.yaml \
+    --set image.repository=k3d-registry.localhost:5111/databend-meta \
+    --set image.tag=meta-chaos \
+    --wait || true
+
+sleep 10
+echo "check if databend-meta nodes is ready"
+kubectl -n databend wait \
+    --for=condition=ready pod \
+    -l app.kubernetes.io/name=databend-meta \
+    --timeout 120s || true
+
+kubectl get pods -A -o wide
+
+kubectl -n databend exec test-databend-meta-0 -- /databend-metactl --status
+
+echo "create verifier pod.."
+kubectl apply -f scripts/ci/meta-chaos/verifier.yaml
+
+echo "check if databend-metaverifier node is ready"
+kubectl -n databend wait \
+    --for=condition=ready pod \
+    -l app.kubernetes.io/name=databend-metaverifier \
+    --timeout 120s || true
+
+echo "logs databend-metaverifier.."
+kubectl logs databend-metaverifier --namespace databend
+
+kubectl get pods -n databend -o wide
diff --git a/scripts/ci/meta-chaos/ci-io-delay-meta-test.sh b/scripts/ci/meta-chaos/ci-io-delay-meta-test.sh
@@ -0,0 +1,13 @@
+#!/bin/bash
+# Copyright 2020-2021 The Databend Authors.
+# SPDX-License-Identifier: Apache-2.0.
+
+set -e
+
+BUILD_PROFILE=${BUILD_PROFILE:-debug}
+
+echo "setting up meta chaos.."
+./scripts/ci/ci-setup-chaos-meta.sh
+
+HTTP_ADDR="test-databend-meta-0.test-databend-meta.databend.svc.cluster.local:28002,test-databend-meta-1.test-databend-meta.databend.svc.cluster.local:28002,test-databend-meta-2.test-databend-meta.databend.svc.cluster.local:28002"
+python3 tests/metaverifier/chaos-meta.py --mode=io/delay/delay=50ms,percent=10 --namespace=databend --nodes=${HTTP_ADDR} --total=800 --apply_second=5 --recover_second=10
diff --git a/scripts/ci/meta-chaos/k3d.yaml b/scripts/ci/meta-chaos/k3d.yaml
@@ -0,0 +1,26 @@
+apiVersion: k3d.io/v1alpha4
+kind: Simple
+metadata:
+  name: single
+servers: 1
+image: rancher/k3s:v1.29.5-k3s1
+ports:
+  - port: 8888:80
+    nodeFilters:
+      - loadbalancer
+registries:
+  use:
+    - k3d-registry.localhost:5111
+options:
+  k3d:
+    wait: true
+    timeout: "60s"
+    disableLoadbalancer: false
+  k3s:
+    nodeLabels:
+      - label: node.datafuselabs.io/role=warehouse
+        nodeFilters:
+          - server:*
+  kubeconfig:
+    updateDefaultKubeconfig: true
+    switchCurrentContext: true
diff --git a/scripts/ci/meta-chaos/meta-ha.yaml b/scripts/ci/meta-chaos/meta-ha.yaml
@@ -0,0 +1,9 @@
+bootstrap: true
+replicaCount: 3
+persistence:
+  size: 8Gi
+
+config:
+  logDir: /.databend/logs
+  logPrefixFilter: ""
+  logLevel: "DEBUG"
diff --git a/scripts/ci/meta-chaos/verifier.yaml b/scripts/ci/meta-chaos/verifier.yaml
@@ -0,0 +1,26 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  creationTimestamp: null
+  labels:
+    run: databend-metaverifier
+    app.kubernetes.io/name: databend-metaverifier
+  name: databend-metaverifier
+  namespace: databend
+spec:
+  containers:
+    - image: k3d-registry.localhost:5111/databend-metaverifier:meta-chaos
+      imagePullPolicy: Always
+      name: databend-metaverifier
+      resources: {}
+      env:
+        - name: CLIENT
+          value: "10"
+        - name: NUMBER
+          value: "15000"
+        - name: GRPC_ADDRESS
+          value: "test-databend-meta-0.test-databend-meta.databend.svc.cluster.local:9191,test-databend-meta-1.test-databend-meta.databend.svc.cluster.local:9191,test-databend-meta-2.test-databend-meta.databend.svc.cluster.local:9191"
+      command: [ "/start-verifier.sh"]
+  dnsPolicy: ClusterFirst
+  restartPolicy: Always
+status: {}
diff --git a/tests/metaverifier/cat-logs.sh b/tests/metaverifier/cat-logs.sh
@@ -0,0 +1,11 @@
+#!/bin/bash
+
+dir_path="/.databend/logs"
+
+for file in "$dir_path"/*; do
+    if [ -f "$file" ]; then
+        echo "\n=== Contents of $file ==="
+        cat "$file"
+        echo "=== End of $file ===\n"
+    fi
+done