From 152f540f13de0b240b579ac6841b1bb773ced6da Mon Sep 17 00:00:00 2001 From: gmhdbjd Date: Thu, 14 Nov 2024 19:53:09 +0800 Subject: [PATCH 1/9] dm: add test --- .github/workflows/dm_upstream_switch.yaml | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/.github/workflows/dm_upstream_switch.yaml b/.github/workflows/dm_upstream_switch.yaml index 718a525113c..66a87462768 100644 --- a/.github/workflows/dm_upstream_switch.yaml +++ b/.github/workflows/dm_upstream_switch.yaml @@ -48,6 +48,8 @@ jobs: - name: Setup containers run: | + sudo curl -L "https://github.com/docker/compose/releases/download/1.29.2/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose + sudo chmod +x /usr/local/bin/docker-compose docker-compose -f ./dm/tests/upstream_switch/docker-compose.yml up -d - name: Run test cases @@ -73,12 +75,14 @@ jobs: path: | ./logs - # send Slack notify if failed. - # NOTE: With the exception of `GITHUB_TOKEN`, secrets are not passed to the runner when a workflow is triggered from a forked repository. - - name: Slack notification + # Send feishu notification if failed. + - name: Feishu notification + continue-on-error: true + uses: foxundermoon/feishu-action@v2 if: ${{ failure() }} - env: - SLACK_WEBHOOK: ${{ secrets.SLACK_NOTIFY }} - uses: Ilshidur/action-slack@2.1.0 with: - args: "upstream-switch job failed, see https://github.com/pingcap/tiflow/actions/runs/{{ GITHUB_RUN_ID }}" + url: ${{ secrets.ENGINE_FEISHU_NOTIFY_URL }} + msg_type: text + content: | + text: | + dm upstream switch job failed, see https://github.com/pingcap/tiflow/actions/runs/${{ github.run_id }} From c27469d5f7c26636e7139270357973ce11189167 Mon Sep 17 00:00:00 2001 From: gmhdbjd Date: Thu, 14 Nov 2024 21:57:18 +0800 Subject: [PATCH 2/9] dm: add test --- .github/workflows/dm_upstream_switch.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/dm_upstream_switch.yaml b/.github/workflows/dm_upstream_switch.yaml index 66a87462768..401d0d4f6d5 100644 --- a/.github/workflows/dm_upstream_switch.yaml +++ b/.github/workflows/dm_upstream_switch.yaml @@ -1,6 +1,9 @@ name: Upstream Database Switch on: + push: + branches: + - master schedule: - cron: '0 17-23 * * *' # run at minute 0 every hour from 01:00 ~ 07:00 UTC+8 workflow_dispatch: From b1dd7744cfa650de756aef6929eac5ad91d29f10 Mon Sep 17 00:00:00 2001 From: gmhdbjd Date: Thu, 14 Nov 2024 21:58:26 +0800 Subject: [PATCH 3/9] dm: add test --- .github/workflows/dm_mariadb_master_down_and_up.yaml | 1 + .github/workflows/dm_upstream_switch.yaml | 3 --- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/dm_mariadb_master_down_and_up.yaml b/.github/workflows/dm_mariadb_master_down_and_up.yaml index a2b8e91aabc..55a89b88989 100644 --- a/.github/workflows/dm_mariadb_master_down_and_up.yaml +++ b/.github/workflows/dm_mariadb_master_down_and_up.yaml @@ -4,6 +4,7 @@ on: push: branches: - test-* + - master schedule: - cron: '0 17-23 * * *' # run at minute 0 every hour from 01:00 ~ 07:00 UTC+8 workflow_dispatch: diff --git a/.github/workflows/dm_upstream_switch.yaml b/.github/workflows/dm_upstream_switch.yaml index 401d0d4f6d5..66a87462768 100644 --- a/.github/workflows/dm_upstream_switch.yaml +++ b/.github/workflows/dm_upstream_switch.yaml @@ -1,9 +1,6 @@ name: Upstream Database Switch on: - push: - branches: - - master schedule: - cron: '0 17-23 * * *' # run at minute 0 every hour from 01:00 ~ 07:00 UTC+8 workflow_dispatch: From 26184d08ccae006af222985e9b347a830535990b Mon Sep 17 00:00:00 2001 From: gmhdbjd Date: Mon, 18 Nov 2024 21:10:28 +0800 Subject: [PATCH 4/9] dm: add test --- dm/pkg/dumpling/utils_test.go | 16 ++++++++++++++-- .../docker-compose.yml | 4 ++-- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/dm/pkg/dumpling/utils_test.go b/dm/pkg/dumpling/utils_test.go index 55bf0610a1c..db26fbd29b3 100644 --- a/dm/pkg/dumpling/utils_test.go +++ b/dm/pkg/dumpling/utils_test.go @@ -246,8 +246,20 @@ Finished dump at: 2020-09-30 12:16:49 } } - noBinlogLoc := `Started dump at: 2020-12-02 17:13:56 -Finished dump at: 2020-12-02 17:13:56 + noBinlogLoc := `Started dump at: 2024-11-18 12:52:52 +SHOW BINARY LOG STATUS: + Log: mysql-bin.000001 + Pos: 16127 + GTID:0-1-7331 + +SHOW SLAVE STATUS: + Connection name: + Host: mariadb_master + Log: mysql-bin.000005 + Pos: 16544 + GTID: + +Finished dump at: 2024-11-18 12:52:52 ` err = os.WriteFile(f.Name(), []byte(noBinlogLoc), 0o644) require.NoError(t, err) diff --git a/dm/tests/mariadb_master_down_and_up/docker-compose.yml b/dm/tests/mariadb_master_down_and_up/docker-compose.yml index e60a02156e4..45b253738d6 100644 --- a/dm/tests/mariadb_master_down_and_up/docker-compose.yml +++ b/dm/tests/mariadb_master_down_and_up/docker-compose.yml @@ -17,7 +17,7 @@ x-node: &default-node services: mariadb_master: <<: *default-node - image: mariadb:10.1 + image: mariadb:10.1.0 command: --log-bin=/var/lib/mysql/mysql-bin --log_slave_updates=ON --server-id=1 --gtid_domain_id=0 --binlog-format=ROW container_name: mariadb_master hostname: mariadb_master @@ -26,7 +26,7 @@ services: mariadb_slave: <<: *default-node - image: mariadb:10.1 + image: mariadb:10.1.0 command: --log-bin=/var/lib/mysql/mysql-bin --log_slave_updates=ON --server-id=2 --gtid_domain_id=0 --binlog-format=ROW container_name: mariadb_slave hostname: mariadb_slave From 7a4d2cb1ef4d9f548514ab08734f2323b3ca2944 Mon Sep 17 00:00:00 2001 From: gmhdbjd Date: Mon, 18 Nov 2024 21:38:35 +0800 Subject: [PATCH 5/9] dm: add test --- dm/tests/mariadb_master_down_and_up/docker-compose.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dm/tests/mariadb_master_down_and_up/docker-compose.yml b/dm/tests/mariadb_master_down_and_up/docker-compose.yml index 45b253738d6..8ff734f3453 100644 --- a/dm/tests/mariadb_master_down_and_up/docker-compose.yml +++ b/dm/tests/mariadb_master_down_and_up/docker-compose.yml @@ -17,7 +17,7 @@ x-node: &default-node services: mariadb_master: <<: *default-node - image: mariadb:10.1.0 + image: mariadb:10.0 command: --log-bin=/var/lib/mysql/mysql-bin --log_slave_updates=ON --server-id=1 --gtid_domain_id=0 --binlog-format=ROW container_name: mariadb_master hostname: mariadb_master @@ -26,7 +26,7 @@ services: mariadb_slave: <<: *default-node - image: mariadb:10.1.0 + image: mariadb:10.0 command: --log-bin=/var/lib/mysql/mysql-bin --log_slave_updates=ON --server-id=2 --gtid_domain_id=0 --binlog-format=ROW container_name: mariadb_slave hostname: mariadb_slave From d5ad1f82b0148f6e968d9fb727600983cd3ac549 Mon Sep 17 00:00:00 2001 From: gmhdbjd Date: Thu, 28 Nov 2024 20:48:29 +0800 Subject: [PATCH 6/9] dm: fix test --- .../workflows/dm_mariadb_master_down_and_up.yaml | 1 - dm/pkg/dumpling/utils_test.go | 16 ++-------------- 2 files changed, 2 insertions(+), 15 deletions(-) diff --git a/.github/workflows/dm_mariadb_master_down_and_up.yaml b/.github/workflows/dm_mariadb_master_down_and_up.yaml index 55a89b88989..a2b8e91aabc 100644 --- a/.github/workflows/dm_mariadb_master_down_and_up.yaml +++ b/.github/workflows/dm_mariadb_master_down_and_up.yaml @@ -4,7 +4,6 @@ on: push: branches: - test-* - - master schedule: - cron: '0 17-23 * * *' # run at minute 0 every hour from 01:00 ~ 07:00 UTC+8 workflow_dispatch: diff --git a/dm/pkg/dumpling/utils_test.go b/dm/pkg/dumpling/utils_test.go index 0cdd1ee278e..7aa373e6b5a 100644 --- a/dm/pkg/dumpling/utils_test.go +++ b/dm/pkg/dumpling/utils_test.go @@ -269,20 +269,8 @@ Finished dump at: 2020-09-30 12:16:49 } } - noBinlogLoc := `Started dump at: 2024-11-18 12:52:52 -SHOW BINARY LOG STATUS: - Log: mysql-bin.000001 - Pos: 16127 - GTID:0-1-7331 - -SHOW SLAVE STATUS: - Connection name: - Host: mariadb_master - Log: mysql-bin.000005 - Pos: 16544 - GTID: - -Finished dump at: 2024-11-18 12:52:52 + noBinlogLoc := `Started dump at: 2020-12-02 17:13:56 +Finished dump at: 2020-12-02 17:13:56 ` err = os.WriteFile(f.Name(), []byte(noBinlogLoc), 0o644) require.NoError(t, err) From daaec12a9f5cd48857d948c5d1863a975f1f4b0f Mon Sep 17 00:00:00 2001 From: gmhdbjd Date: Thu, 28 Nov 2024 20:52:46 +0800 Subject: [PATCH 7/9] dm: fix test --- .github/workflows/dm_mariadb_master_down_and_up.yaml | 12 ++++++++++++ .../mariadb_master_down_and_up/docker-compose.yml | 4 ++-- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/.github/workflows/dm_mariadb_master_down_and_up.yaml b/.github/workflows/dm_mariadb_master_down_and_up.yaml index a2b8e91aabc..5727b6e460f 100644 --- a/.github/workflows/dm_mariadb_master_down_and_up.yaml +++ b/.github/workflows/dm_mariadb_master_down_and_up.yaml @@ -63,3 +63,15 @@ jobs: name: upstream-switch-logs path: | ./logs + + # Send feishu notification if failed. + - name: Feishu notification + continue-on-error: true + uses: foxundermoon/feishu-action@v2 + if: ${{ failure() }} + with: + url: ${{ secrets.ENGINE_FEISHU_NOTIFY_URL }} + msg_type: text + content: | + text: | + mariadb restart job failed, see https://github.com/pingcap/tiflow/actions/runs/${{ github.run_id }} diff --git a/dm/tests/mariadb_master_down_and_up/docker-compose.yml b/dm/tests/mariadb_master_down_and_up/docker-compose.yml index 8ff734f3453..a7210df7e53 100644 --- a/dm/tests/mariadb_master_down_and_up/docker-compose.yml +++ b/dm/tests/mariadb_master_down_and_up/docker-compose.yml @@ -17,7 +17,7 @@ x-node: &default-node services: mariadb_master: <<: *default-node - image: mariadb:10.0 + image: mariadb:11.3 # change to latest after fix https://github.com/pingcap/tiflow/issues/11784 command: --log-bin=/var/lib/mysql/mysql-bin --log_slave_updates=ON --server-id=1 --gtid_domain_id=0 --binlog-format=ROW container_name: mariadb_master hostname: mariadb_master @@ -26,7 +26,7 @@ services: mariadb_slave: <<: *default-node - image: mariadb:10.0 + image: mariadb:11.3 command: --log-bin=/var/lib/mysql/mysql-bin --log_slave_updates=ON --server-id=2 --gtid_domain_id=0 --binlog-format=ROW container_name: mariadb_slave hostname: mariadb_slave From 7b418bbdc7195d5a0eb2a4cd18ad8ba3693d6d2d Mon Sep 17 00:00:00 2001 From: gmhdbjd Date: Tue, 17 Dec 2024 20:34:14 +0800 Subject: [PATCH 8/9] add test --- dm/tests/mariadb_master_down_and_up/case.sh | 1 + dm/tests/mariadb_master_down_and_up/docker-compose.yml | 1 + 2 files changed, 2 insertions(+) diff --git a/dm/tests/mariadb_master_down_and_up/case.sh b/dm/tests/mariadb_master_down_and_up/case.sh index e678d3bcfd2..f3b5a5c5735 100644 --- a/dm/tests/mariadb_master_down_and_up/case.sh +++ b/dm/tests/mariadb_master_down_and_up/case.sh @@ -143,6 +143,7 @@ function test_master_down_and_up() { function run() { wait_mysql 3306 1 wait_mysql 3307 2 + exec_tidb $tidb_port "set global tidb_general_log='on';" test_master_down_and_up no_relay test_master_down_and_up relay } diff --git a/dm/tests/mariadb_master_down_and_up/docker-compose.yml b/dm/tests/mariadb_master_down_and_up/docker-compose.yml index a7210df7e53..07fb357a8ff 100644 --- a/dm/tests/mariadb_master_down_and_up/docker-compose.yml +++ b/dm/tests/mariadb_master_down_and_up/docker-compose.yml @@ -40,3 +40,4 @@ services: - "4000:4000" image: pingcap/tidb:latest restart: always + command: From 111e59f1a749ff539b074e7ad2205b5c61f732b9 Mon Sep 17 00:00:00 2001 From: gmhdbjd Date: Tue, 17 Dec 2024 20:35:18 +0800 Subject: [PATCH 9/9] add test --- dm/tests/mariadb_master_down_and_up/docker-compose.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/dm/tests/mariadb_master_down_and_up/docker-compose.yml b/dm/tests/mariadb_master_down_and_up/docker-compose.yml index 07fb357a8ff..a7210df7e53 100644 --- a/dm/tests/mariadb_master_down_and_up/docker-compose.yml +++ b/dm/tests/mariadb_master_down_and_up/docker-compose.yml @@ -40,4 +40,3 @@ services: - "4000:4000" image: pingcap/tidb:latest restart: always - command: