From 498a7f692f3bde1c6dd14b51552a5c8e9db2ad07 Mon Sep 17 00:00:00 2001 From: codedump Date: Wed, 12 Jun 2024 20:50:06 +0800 Subject: [PATCH] feat: add io delay chaos meta test scripts (#15769) * feat: add io delay chaos meta test scripts * feat: add io delay chaos meta test scripts, make clippy happy * z * z --------- Co-authored-by: everpcpc --- .../action.yml | 23 ++ .github/actions/io_delay_chaos/action.yml | 27 ++ .github/workflows/meta.yml | 65 ++++ .github/workflows/release.yml | 12 +- docker/meta-chaos/meta.Dockerfile | 14 + docker/meta-chaos/verifier.Dockerfile | 14 + scripts/ci/ci-setup-chaos-meta.sh | 93 ++++++ .../ci/meta-chaos/ci-io-delay-meta-test.sh | 13 + scripts/ci/meta-chaos/k3d.yaml | 26 ++ scripts/ci/meta-chaos/meta-ha.yaml | 9 + scripts/ci/meta-chaos/verifier.yaml | 26 ++ tests/metaverifier/cat-logs.sh | 11 + tests/metaverifier/chaos-meta.py | 278 ++++++++++++++++++ tests/metaverifier/start-verifier.sh | 15 + .../node-disk-io-delay-template.yaml | 18 ++ 15 files changed, 643 insertions(+), 1 deletion(-) create mode 100644 .github/actions/artifact_meta_container_failure/action.yml create mode 100644 .github/actions/io_delay_chaos/action.yml create mode 100644 .github/workflows/meta.yml create mode 100644 docker/meta-chaos/meta.Dockerfile create mode 100644 docker/meta-chaos/verifier.Dockerfile create mode 100755 scripts/ci/ci-setup-chaos-meta.sh create mode 100755 scripts/ci/meta-chaos/ci-io-delay-meta-test.sh create mode 100644 scripts/ci/meta-chaos/k3d.yaml create mode 100644 scripts/ci/meta-chaos/meta-ha.yaml create mode 100644 scripts/ci/meta-chaos/verifier.yaml create mode 100755 tests/metaverifier/cat-logs.sh create mode 100755 tests/metaverifier/chaos-meta.py create mode 100755 tests/metaverifier/start-verifier.sh create mode 100644 tests/metaverifier/templates/node-disk-io-delay-template.yaml diff --git a/.github/actions/artifact_meta_container_failure/action.yml b/.github/actions/artifact_meta_container_failure/action.yml new file mode 100644 index 000000000000..1de7f0e7ad88 --- /dev/null +++ b/.github/actions/artifact_meta_container_failure/action.yml @@ -0,0 +1,23 @@ +name: "Upload meta containers failure Artifacts" +description: "Upload meta containers Artifacts" +inputs: + name: + description: "" + required: true +runs: + using: "composite" + steps: + - name: pack failure artifacts + shell: bash + run: | + mkdir -p logs + for name in "test-databend-meta-0" "test-databend-meta-1" "test-databend-meta-2" "databend-metaverifier" + do + echo "cat logs of $name" + kubectl exec -i $name -n databend -- /cat-logs.sh > logs/$name.log + done + tar -zcf logs/failure-${{ inputs.name }}.tar.gz logs/* + - uses: actions/upload-artifact@v4 + with: + name: ${{ inputs.name }} + path: logs/failure-${{ inputs.name }}.tar.gz diff --git a/.github/actions/io_delay_chaos/action.yml b/.github/actions/io_delay_chaos/action.yml new file mode 100644 index 000000000000..f54e6d4f5075 --- /dev/null +++ b/.github/actions/io_delay_chaos/action.yml @@ -0,0 +1,27 @@ +name: "Chaos Test meta service" +description: "Running stateless tests in management mode" +inputs: + target: + description: "" + required: true + default: "x86_64-unknown-linux-gnu" +runs: + using: "composite" + steps: + - name: Download artifact + uses: ./.github/actions/artifact_download + with: + sha: ${{ github.sha }} + target: ${{ inputs.target }} + artifacts: meta,metactl,metaverifier + + - name: Meta Service IO Delay tests + shell: bash + run: | + ./scripts/ci/meta-chaos/ci-io-delay-meta-test.sh + + - name: Upload failure + if: failure() + uses: ./.github/actions/artifact_meta_container_failure + with: + name: meta-io-delay-chaos diff --git a/.github/workflows/meta.yml b/.github/workflows/meta.yml new file mode 100644 index 000000000000..dd0039f826de --- /dev/null +++ b/.github/workflows/meta.yml @@ -0,0 +1,65 @@ +name: Meta + +on: + pull_request_target: + types: + - labeled + branches: + - main + +permissions: + id-token: write + pull-requests: write + contents: read + +env: + BUILD_PROFILE: release + RUNNER_PROVIDER: gcp + +jobs: + info: + if: contains(github.event.pull_request.labels.*.name, 'ci-meta-chaos') + runs-on: ubuntu-latest + outputs: + sha: ${{ steps.sha.outputs.sha }} + target: ${{ steps.sha.outputs.target }} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + ref: "refs/pull/${{ github.event.number }}/merge" + - name: Get SHA + shell: bash + id: sha + run: | + echo "sha=$(git rev-parse --verify HEAD)" > $GITHUB_OUTPUT + + build: + needs: info + runs-on: [self-hosted, "${{ matrix.runner }}", Linux, 16c32g, gcp] + strategy: + matrix: + include: + - { arch: x86_64, runner: X64 } + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + ref: "refs/pull/${{ github.event.number }}/merge" + - name: Build Release + uses: ./.github/actions/build_linux + timeout-minutes: 60 + with: + sha: ${{ needs.info.outputs.sha }} + target: ${{ matrix.arch }}-unknown-linux-gnu + artifacts: meta,metactl,metaverifier + + chaos: + needs: [info, build] + runs-on: [self-hosted, X64, Linux, 8c16g, gcp] + steps: + - uses: actions/checkout@v4 + with: + ref: "refs/pull/${{ github.event.number }}/merge" + - uses: ./.github/actions/io_delay_chaos + timeout-minutes: 20 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 0189d0d5f176..d77094164a82 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -122,7 +122,7 @@ jobs: with: sha: ${{ github.sha }} target: ${{ matrix.target }} - artifacts: sqllogictests,sqlsmith,metactl,meta,query + artifacts: sqllogictests,sqlsmith,query,meta,metactl,metaverifier - name: Basic Sqllogic Test if: matrix.target != 'aarch64-unknown-linux-musl' shell: bash @@ -627,6 +627,16 @@ jobs: with: name: test-sqlsmith + meta_chaos: + needs: [create_release, build_default] + runs-on: [self-hosted, X64, Linux, 8c16g, gcp] + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ needs.create_release.outputs.sha }} + - uses: ./.github/actions/io_delay_chaos + timeout-minutes: 20 + benchmark: needs: [create_release, docker_service, notify] uses: ./.github/workflows/reuse.benchmark.yml diff --git a/docker/meta-chaos/meta.Dockerfile b/docker/meta-chaos/meta.Dockerfile new file mode 100644 index 000000000000..b115610c7ffa --- /dev/null +++ b/docker/meta-chaos/meta.Dockerfile @@ -0,0 +1,14 @@ +FROM debian:bookworm + +ARG TARGETPLATFORM +ENV TERM=dumb +ENV DEBIAN_FRONTEND=noninteractive +RUN apt-get update -y && \ + apt-get install -y apt-transport-https ca-certificates && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* && \ + rm -rf /var/cache/apt/* +COPY ./distro/$TARGETPLATFORM/databend-meta /databend-meta +COPY ./distro/$TARGETPLATFORM/databend-metactl /databend-metactl +COPY ./distro/$TARGETPLATFORM/cat-logs.sh /cat-logs.sh +ENTRYPOINT ["/databend-meta"] diff --git a/docker/meta-chaos/verifier.Dockerfile b/docker/meta-chaos/verifier.Dockerfile new file mode 100644 index 000000000000..89439a85fccd --- /dev/null +++ b/docker/meta-chaos/verifier.Dockerfile @@ -0,0 +1,14 @@ +FROM debian:bookworm + +ARG TARGETPLATFORM +ENV TERM=dumb +ENV DEBIAN_FRONTEND=noninteractive +RUN apt-get update -y && \ + apt-get install -y apt-transport-https ca-certificates curl && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* && \ + rm -rf /var/cache/apt/* + +COPY ./distro/$TARGETPLATFORM/* / + +ENTRYPOINT ["/bin/sh"] diff --git a/scripts/ci/ci-setup-chaos-meta.sh b/scripts/ci/ci-setup-chaos-meta.sh new file mode 100755 index 000000000000..38dd7b9dd086 --- /dev/null +++ b/scripts/ci/ci-setup-chaos-meta.sh @@ -0,0 +1,93 @@ +#!/bin/bash +# Copyright 2020-2021 The Databend Authors. +# SPDX-License-Identifier: Apache-2.0. + +set -ex + +BUILD_PROFILE=${BUILD_PROFILE:-debug} + +curl -s https://raw.githubusercontent.com/k3d-io/k3d/main/install.sh | TAG=v5.6.0 bash +k3d registry create registry.localhost --port 0.0.0.0:5111 -i registry:latest +k3d cluster create --config ./scripts/ci/meta-chaos/k3d.yaml meta-chaos + +echo "127.0.0.1 k3d-registry.localhost" | sudo tee -a /etc/hosts + +if kubectl version --client; then + echo "kubectl client already installed" +else + echo "install kubectl client" + curl -LO "https://dl.k8s.io/release/v1.29.5/bin/linux/amd64/kubectl" + chmod +x kubectl + sudo mv kubectl /usr/local/bin/ +fi + +if helm version; then + echo "helm already installed" +else + echo "install helm" + curl -fsSL -o get_helm.sh https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 + chmod 700 get_helm.sh + ./get_helm.sh +fi + +echo "make databend-meta image" +ls -lh ./target/"${BUILD_PROFILE}" +mkdir -p temp/distro/amd64 +cp ./target/"${BUILD_PROFILE}"/databend-meta ./temp/distro/amd64 +cp ./target/"${BUILD_PROFILE}"/databend-metactl ./temp/distro/amd64 +cp tests/metaverifier/cat-logs.sh ./temp/distro/amd64 +docker build -t databend-meta:meta-chaos --build-arg TARGETPLATFORM="amd64" -f ./docker/chaos-meta/meta.Dockerfile temp +docker tag databend-meta:meta-chaos k3d-registry.localhost:5111/databend-meta:meta-chaos +docker push k3d-registry.localhost:5111/databend-meta:meta-chaos + +echo "make databend-metaverifier image" +rm -rf temp/distro/amd64/* +cp ./target/"${BUILD_PROFILE}"/databend-metaverifier ./temp/distro/amd64 +cp tests/metaverifier/start-verifier.sh ./temp/distro/amd64 +cp tests/metaverifier/cat-logs.sh ./temp/distro/amd64 +docker build -t databend-metaverifier:meta-chaos --build-arg TARGETPLATFORM="amd64" -f ./docker/chaos-meta/verifier.Dockerfile temp +docker tag databend-metaverifier:meta-chaos k3d-registry.localhost:5111/databend-metaverifier:meta-chaos +docker push k3d-registry.localhost:5111/databend-metaverifier:meta-chaos + +echo "install chaos mesh on k3d" +curl -sSL https://mirrors.chaos-mesh.org/v2.6.3/install.sh | bash -s -- --k3s + +kubectl get pods -A -o wide +kubectl get pvc -A + +echo "kubectl delete databend-meta pvc" +kubectl delete pvc --namespace databend data-test-databend-meta-0 data-test-databend-meta-1 data-test-databend-meta-2 --ignore-not-found + +helm repo add databend https://charts.databend.rs +helm install test databend/databend-meta \ + --namespace databend \ + --create-namespace \ + --values scripts/ci/meta-chaos/meta-ha.yaml \ + --set image.repository=k3d-registry.localhost:5111/databend-meta \ + --set image.tag=meta-chaos \ + --wait || true + +sleep 10 +echo "check if databend-meta nodes is ready" +kubectl -n databend wait \ + --for=condition=ready pod \ + -l app.kubernetes.io/name=databend-meta \ + --timeout 120s || true + +kubectl get pods -A -o wide + +kubectl -n databend exec test-databend-meta-0 -- /databend-metactl --status + +echo "create verifier pod.." +kubectl apply -f scripts/ci/meta-chaos/verifier.yaml + +echo "check if databend-metaverifier node is ready" +kubectl -n databend wait \ + --for=condition=ready pod \ + -l app.kubernetes.io/name=databend-metaverifier \ + --timeout 120s || true + +echo "logs databend-metaverifier.." +kubectl logs databend-metaverifier --namespace databend + +kubectl get pods -n databend -o wide diff --git a/scripts/ci/meta-chaos/ci-io-delay-meta-test.sh b/scripts/ci/meta-chaos/ci-io-delay-meta-test.sh new file mode 100755 index 000000000000..e6fa0a57ccc5 --- /dev/null +++ b/scripts/ci/meta-chaos/ci-io-delay-meta-test.sh @@ -0,0 +1,13 @@ +#!/bin/bash +# Copyright 2020-2021 The Databend Authors. +# SPDX-License-Identifier: Apache-2.0. + +set -e + +BUILD_PROFILE=${BUILD_PROFILE:-debug} + +echo "setting up meta chaos.." +./scripts/ci/ci-setup-chaos-meta.sh + +HTTP_ADDR="test-databend-meta-0.test-databend-meta.databend.svc.cluster.local:28002,test-databend-meta-1.test-databend-meta.databend.svc.cluster.local:28002,test-databend-meta-2.test-databend-meta.databend.svc.cluster.local:28002" +python3 tests/metaverifier/chaos-meta.py --mode=io/delay/delay=50ms,percent=10 --namespace=databend --nodes=${HTTP_ADDR} --total=800 --apply_second=5 --recover_second=10 diff --git a/scripts/ci/meta-chaos/k3d.yaml b/scripts/ci/meta-chaos/k3d.yaml new file mode 100644 index 000000000000..36d8a49899a7 --- /dev/null +++ b/scripts/ci/meta-chaos/k3d.yaml @@ -0,0 +1,26 @@ +apiVersion: k3d.io/v1alpha4 +kind: Simple +metadata: + name: single +servers: 1 +image: rancher/k3s:v1.29.5-k3s1 +ports: + - port: 8888:80 + nodeFilters: + - loadbalancer +registries: + use: + - k3d-registry.localhost:5111 +options: + k3d: + wait: true + timeout: "60s" + disableLoadbalancer: false + k3s: + nodeLabels: + - label: node.datafuselabs.io/role=warehouse + nodeFilters: + - server:* + kubeconfig: + updateDefaultKubeconfig: true + switchCurrentContext: true diff --git a/scripts/ci/meta-chaos/meta-ha.yaml b/scripts/ci/meta-chaos/meta-ha.yaml new file mode 100644 index 000000000000..467d19e18134 --- /dev/null +++ b/scripts/ci/meta-chaos/meta-ha.yaml @@ -0,0 +1,9 @@ +bootstrap: true +replicaCount: 3 +persistence: + size: 8Gi + +config: + logDir: /.databend/logs + logPrefixFilter: "" + logLevel: "DEBUG" diff --git a/scripts/ci/meta-chaos/verifier.yaml b/scripts/ci/meta-chaos/verifier.yaml new file mode 100644 index 000000000000..58f2e317250e --- /dev/null +++ b/scripts/ci/meta-chaos/verifier.yaml @@ -0,0 +1,26 @@ +apiVersion: v1 +kind: Pod +metadata: + creationTimestamp: null + labels: + run: databend-metaverifier + app.kubernetes.io/name: databend-metaverifier + name: databend-metaverifier + namespace: databend +spec: + containers: + - image: k3d-registry.localhost:5111/databend-metaverifier:meta-chaos + imagePullPolicy: Always + name: databend-metaverifier + resources: {} + env: + - name: CLIENT + value: "10" + - name: NUMBER + value: "15000" + - name: GRPC_ADDRESS + value: "test-databend-meta-0.test-databend-meta.databend.svc.cluster.local:9191,test-databend-meta-1.test-databend-meta.databend.svc.cluster.local:9191,test-databend-meta-2.test-databend-meta.databend.svc.cluster.local:9191" + command: [ "/start-verifier.sh"] + dnsPolicy: ClusterFirst + restartPolicy: Always +status: {} diff --git a/tests/metaverifier/cat-logs.sh b/tests/metaverifier/cat-logs.sh new file mode 100755 index 000000000000..0a1267e39cf6 --- /dev/null +++ b/tests/metaverifier/cat-logs.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +dir_path="/.databend/logs" + +for file in "$dir_path"/*; do + if [ -f "$file" ]; then + echo "\n=== Contents of $file ===" + cat "$file" + echo "=== End of $file ===\n" + fi +done diff --git a/tests/metaverifier/chaos-meta.py b/tests/metaverifier/chaos-meta.py new file mode 100755 index 000000000000..853634d1cbc5 --- /dev/null +++ b/tests/metaverifier/chaos-meta.py @@ -0,0 +1,278 @@ +#!/usr/bin/python3 +import sys +import getopt +import os +import random +import time +import logging + +CHAOS_FILE = "/tmp/chaos.yaml" + +logging.basicConfig( + level=logging.DEBUG, + format="%(asctime)s.%(msecs)06d %(levelname)s %(message)s", + datefmt="%Y-%m-%d,%H:%M:%S", +) + + +class ChaosParams: + def __init__(self): + self.params = {} + self.template = "" + self.template_map = {} + + def to_string(self): + return "" + + def init(self, params): + for k, v in params.items(): + self.params[k] = v + + def generate(self, node): + content = self.template + for template_key, key in self.template_map.items(): + content = content.replace(template_key, self.params[key]) + + content = content.replace("${NODE}", node) + return content + + +class IoDelayParams(ChaosParams): + def __init__(self): + ChaosParams.__init__(self) + # default params + self.params["delay"] = "500ms" + self.params["percent"] = "100" + self.template_map = {"${DELAY}": "delay", "${PENCENT}": "percent"} + self.template = open( + "./tests/metaverifier/templates/node-disk-io-delay-template.yaml", "r" + ).read() + # print('template: ', self.template) + # self.generate('hah') + + def to_string(self): + return ( + "type: IoDelay, params delay: " + + str(self.params["delay"]) + + ", percent:" + + str(self.params["percent"]) + ) + + +class MetaChaos: + def __init__(self, mode, namespace, nodes, total): + self.mode = mode + self.namespace = namespace + self.total = total + self.split_nodes(nodes) + + def split_nodes(self, nodes): + nodes = nodes.split(",") + self.node_port_map = {} + for node in nodes: + ary = node.split(".") + self.node_port_map[ary[0]] = node + + def get_node(self, get_leader): + nodes = [] + cmd = "kubectl exec -i databend-metaverifier -n databend -- " + for node, addr in self.node_port_map.items(): + curl_cmd = cmd + "curl " + addr + "/v1/cluster/status" + content = os.popen(curl_cmd).read() + logging.debug("curl cmd output:" + str(content)) + if get_leader: + if content.find('"state":"Leader"') != -1: + nodes.append(node) + return nodes + else: + if content.find('"state":"Leader"') == -1: + nodes.append(node) + + if len(nodes) == 0: + nodes = list(self.node_port_map.keys()) + logging.debug( + "cannot find nodes, get_leader: " + str(get_leader) + " " + str(nodes) + ) + return nodes + return nodes + + def get_inject_chaos_node(self, node_mode): + # return random.sample(self.get_node(node_mode == "leader"), 1)[0] + return random.sample(list(self.node_port_map.keys()), 1)[0] + + def exec_cat_meta_verifier(self): + cmd = "kubectl exec -i databend-metaverifier -n databend -- cat /tmp/meta-verifier" + content = os.popen(cmd).read().strip() + logging.debug("exec cat meta-verifier: " + str(content)) + + return content + + def wait_verifier(self): + # first start meta-verifier + # make sure meta-verifier has been started + count = 0 + while count < 10: + content = self.exec_cat_meta_verifier() + if content == "START": + logging.debug("databend-metaverifier has started") + return + count += 1 + time.sleep(1) + + logging.error("databend-metaverifier has not started, exit") + sys.exit(-1) + + def is_verifier_end(self): + cmd = "kubectl logs databend-metaverifier -n databend | tail -10" + content = os.popen(cmd).read() + logging.debug( + "kubectl logs databend-metaverifier -n databend:\n" + str(content) + ) + content = self.exec_cat_meta_verifier() + if content == "ERROR": + logging.error("databend-metaverifier return error") + sys.exit(-1) + + return content == "END" + + def run(self, apply_second, recover_second): + logging.info( + "run with apply_second: " + + str(apply_second) + + ", recover_second: " + + str(recover_second) + ) + + mode = self.mode.split("/") + # print("mode: ", mode) + typ, subtype = mode[0], mode[1] + params_vec = mode[2].split(",") + params = {} + for param in params_vec: + param = param.split("=") + params[param[0]] = param[1] + # print("params:", params) + + if typ == "io" and subtype == "delay": + chaos_params = IoDelayParams() + chaos_params.init(params) + self.chaos_params = chaos_params + + logging.info("run with chaos: " + self.chaos_params.to_string()) + self.wait_verifier() + + start = int(time.time()) + node_modes = ["leader", "follower"] + count = 0 + while True: + count += 1 + + # random select node mode, leader or follower + node_mode = random.sample(node_modes, 1)[0] + node = self.get_inject_chaos_node(node_mode) + logging.debug( + "loop " + str(count) + " inject chaos rule to node: " + str(node) + ) + + # generate chaos yaml + chaos_yaml = self.chaos_params.generate(node) + chaos_file = open(CHAOS_FILE, "w") + chaos_file.write(chaos_yaml) + chaos_file.close() + + # apply chaos + cmd = "kubectl apply -f " + CHAOS_FILE + os.system(cmd) + logging.debug("apply chaos..") + + # wait some time + time.sleep(apply_second) + + # delete chaos + cmd = "kubectl delete -f " + CHAOS_FILE + os.system(cmd) + logging.debug("remove chaos..") + + # wait some time + time.sleep(recover_second) + + current = int(time.time()) + diff = current - start + if self.is_verifier_end(): + logging.debug( + "databend-metaverifier has completed, cost:" + str(diff) + "s, exit" + ) + sys.exit(0) + + if diff > self.total: + logging.error( + "databend-metaverifier is not completed in total time, exit -1" + ) + sys.exit(-1) + + +# mode = type/subtype/mode params +# ex: mode = io/delay/delay=300,percent=100 +# nodes = node-pod-name:node-port[,node-pod-name:node-port] +# namespace: databend meta k8s namespace +# total: test total time +# apply_second: apply chaos second +# recover_second: recover from chaos second +def main(argv): + try: + opts, args = getopt.getopt( + argv, + "h", + [ + "mode=", + "namespace=", + "nodes=", + "total=", + "apply_second=", + "recover_second=", + ], + ) + except getopt.GetoptError: + print("chaos-meta.py") + sys.exit(-1) + + mode = "" + namespace = "" + nodes = "" + total = 600 + apply_second = 5 + recover_second = 10 + for opt, arg in opts: + arg = arg.strip() + if opt == "-h": + print("chaos-meta.py --prefix --client") + sys.exit() + elif opt in ("--mode"): + if len(arg) > 0: + mode = arg + elif opt in ("--namespace"): + if len(arg) > 0: + namespace = arg + elif opt in ("--nodes"): + if len(arg) > 0: + nodes = arg + elif opt in ("--total"): + if len(arg) > 0: + total = int(arg) + elif opt in ("--apply_second"): + if len(arg) > 0: + apply_second = int(arg) + elif opt in ("--recover_second"): + if len(arg) > 0: + recover_second = int(arg) + + if len(namespace) == "" or mode == "" or nodes == "": + print("namespace or mode or nodes is empty") + return + + chaos = MetaChaos(mode, namespace, nodes, total) + chaos.run(apply_second, recover_second) + + +if __name__ == "__main__": + main(sys.argv[1:]) diff --git a/tests/metaverifier/start-verifier.sh b/tests/metaverifier/start-verifier.sh new file mode 100755 index 000000000000..3e48b2d9ae62 --- /dev/null +++ b/tests/metaverifier/start-verifier.sh @@ -0,0 +1,15 @@ +#!/bin/bash +# Copyright 2020-2021 The Databend Authors. +# SPDX-License-Identifier: Apache-2.0. + +set -ex + +echo "start databend-metaverifier with params client:${CLIENT}, number:${NUMBER}, grpc-api-address:${GRPC_ADDRESS}" +echo "START" >/tmp/meta-verifier +# wait for chao-meta.py +sleep 3 +/databend-metaverifier --client ${CLIENT} --time 1800 --remove-percent 10 --number ${NUMBER} --grpc-api-address ${GRPC_ADDRESS} || true + +while true; do + sleep 5 +done diff --git a/tests/metaverifier/templates/node-disk-io-delay-template.yaml b/tests/metaverifier/templates/node-disk-io-delay-template.yaml new file mode 100644 index 000000000000..32ac73465cba --- /dev/null +++ b/tests/metaverifier/templates/node-disk-io-delay-template.yaml @@ -0,0 +1,18 @@ +kind: IOChaos +apiVersion: chaos-mesh.org/v1alpha1 +metadata: + namespace: databend + name: meta-node-io-delay + annotations: + experiment.chaos-mesh.org/pause: 'false' +spec: + selector: + namespaces: + - databend + labelSelectors: + statefulset.kubernetes.io/pod-name: ${NODE} + mode: all + action: latency + delay: ${DELAY} + percent: ${PENCENT} + volumePath: /data/databend-meta