Skip to content

Commit

Permalink
test: add apiserver integration test (#3674)
Browse files Browse the repository at this point in the history
  • Loading branch information
@lqy222
lqy222 authored Feb 2, 2024
1 parent 7ca5f44 commit b8318b8
Show file tree
Hide file tree
Showing 53 changed files with 1,312 additions and 474 deletions.
6 changes: 0 additions & 6 deletions .github/workflows/integration-test-src.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -104,7 +104,6 @@ jobs:
html_body: test/integration-test/openmldb-test-java/openmldb-sdk-test/target/surefire-reports/html/overview.html
to: ${{ secrets.MAIL_TO }}
from: GitHub Actions
content_type: text/plain
attachments: surefire-reports.tar.gz

java-sdk-cluster-memory-task:
Expand Down Expand Up @@ -148,7 +147,6 @@ jobs:
html_body: file://test/integration-test/openmldb-test-java/openmldb-sdk-test/target/surefire-reports/html/overview.html
to: ${{ secrets.MAIL_TO }}
from: GitHub Actions
content_type: text/plain
attachments: surefire-reports.tar.gz

java-sdk-cluster-memory-1:
Expand Down Expand Up @@ -191,7 +189,6 @@ jobs:
body: OpenMLDB Memory 1 Test Failed
to: ${{ secrets.MAIL_TO }}
from: GitHub Actions
content_type: text/plain
attachments: surefire-reports.tar.gz

java-sdk-cluster-ssd-0:
Expand Down Expand Up @@ -234,7 +231,6 @@ jobs:
body: OpenMLDB SSD Test Failed
to: ${{ secrets.MAIL_TO }}
from: GitHub Actions
content_type: text/plain
attachments: surefire-reports.tar.gz

# java-sdk-cluster-ssd-task:
Expand Down Expand Up @@ -320,7 +316,6 @@ jobs:
body: OpenMLDB HDD Test Failed
to: ${{ secrets.MAIL_TO }}
from: GitHub Actions
content_type: text/plain
attachments: surefire-reports.tar.gz

# java-sdk-cluster-hdd-task:
Expand Down Expand Up @@ -568,5 +563,4 @@ jobs:
html_body: test/integration-test/openmldb-test-java/openmldb-sdk-test/target/surefire-reports/html/overview.html
to: ${{ secrets.MAIL_TO }}
from: GitHub Actions
content_type: text/plain
attachments: surefire-reports.tar.gz
145 changes: 93 additions & 52 deletions .github/workflows/selfhost_intergration.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,9 @@ jobs:
make SQL_JAVASDK_ENABLE=ON && make SQL_JAVASDK_ENABLE=ON install
mv openmldb-linux openmldb-main-linux
tar -zcf openmldb-linux.tar.gz openmldb-main-linux
mv java/openmldb-batch/target/openmldb*-SNAPSHOT.jar openmldb-batch.jar
mv java/openmldb-jdbc/target/openmldb*-SNAPSHOT.jar openmldb-jdbc.jar
mv java/openmldb-native/target/openmldb*-SNAPSHOT.jar openmldb-native.jar
- name: download
if: ${{ env.E_VERSION != 'main' }}
run: |
Expand All @@ -48,9 +51,64 @@ jobs:
uses: actions/upload-artifact@v3
with:
name: openmldb-package
path: openmldb-linux.tar.gz
path: |
openmldb-batch.jar
openmldb-jdbc.jar
openmldb-native.jar
openmldb-linux.tar.gz

apiserver-cluster:
needs: build-openmldb

runs-on: [self-hosted,common-user]
steps:
- uses: actions/checkout@v3
- name: before test
if: ${{ env.ETYPE == 'all' || env.ETYPE == 'java' }}
run: mkdir ${{ github.job }}
- name: download artifact
uses: actions/download-artifact@v3
with:
name: openmldb-package
- name: install openmldb
run: |
tar -zxf openmldb-linux.tar.gz -C ${{ github.job }}/
bash test/steps/format_config.sh $(pwd)/${{ github.job }}/openmldb-${{ env.E_VERSION }}-linux ${{ github.job }} 20001 21000 java
bash ${{ github.job }}/openmldb-${{ env.E_VERSION }}-linux/sbin/deploy-all.sh
bash ${{ github.job }}/openmldb-${{ env.E_VERSION }}-linux/sbin/start-all.sh
- name: test
run: bash test/steps/openmldb-integration-test.sh -j 0.0.0 -d deploy -m apiserver
- name: stop openmldb
if: always()
run: bash ${{ github.job }}/openmldb-${{ env.E_VERSION }}-linux/sbin/stop-all.sh
- name: remove openmldb
if: always()
run: bash ${{ github.job }}/openmldb-${{ env.E_VERSION }}-linux/sbin/clear-all.sh
- name: TEST Results
if: always()
uses: EnricoMi/publish-unit-test-result-action@v1
with:
files: test/integration-test/openmldb-test-java/openmldb-sdk-test/target/surefire-reports/TEST-*.xml
check_name: "SRC apiserver-cluster Report"
comment_title: "SRC apiserver-cluster Report"
- name: tar test report
if: ${{ failure() }}
run: tar -zcvf surefire-reports.tar.gz test/integration-test/openmldb-test-java/openmldb-sdk-test/target/surefire-reports
- name: Send Email
if: ${{ failure() }}
uses: dawidd6/action-send-mail@master
with:
server_address: mail.4paradigm.com
server_port: 995
username: ${{ secrets.MAIL_USERNAME }}
password: ${{ secrets.MAIL_PASSWORD }}
secure: true
subject: OpenMLDB Memory Test
body: OpenMLDB Memory Test Failed
html_body: test/integration-test/openmldb-test-java/openmldb-sdk-test/target/surefire-reports/html/overview.html
to: ${{ secrets.MAIL_TO }}
from: GitHub Actions
attachments: surefire-reports.tar.gz

java-sdk-cluster-memory-0:
needs: build-openmldb
Expand All @@ -72,16 +130,12 @@ jobs:
bash ${{ github.job }}/openmldb-${{ env.E_VERSION }}-linux/sbin/deploy-all.sh
bash ${{ github.job }}/openmldb-${{ env.E_VERSION }}-linux/sbin/start-all.sh
- name: test
run: |
mkdir mvnrepo
export MAVEN_OPTS="-Dmaven.repo.local=$(pwd)/mvnrepo"
echo $MAVEN_OPTS
bash test/steps/openmldb-sdk-test-java-src.sh -c test_cluster.xml -d deploy -l "0" -s "memory"
run: bash test/steps/openmldb-integration-test.sh -j 0.0.0 -c test_cluster.xml -d deploy -l "0" -s "memory"
- name: stop openmldb
if: always()
run: bash ${{ github.job }}/openmldb-${{ env.E_VERSION }}-linux/sbin/stop-all.sh
- name: remove openmldb
if: success()
if: always()
run: bash ${{ github.job }}/openmldb-${{ env.E_VERSION }}-linux/sbin/clear-all.sh
- name: TEST Results
if: always()
Expand All @@ -97,10 +151,11 @@ jobs:
if: ${{ failure() }}
uses: dawidd6/action-send-mail@master
with:
server_address: smtp.partner.outlook.cn
server_port: 587
server_address: mail.4paradigm.com
server_port: 995
username: ${{ secrets.MAIL_USERNAME }}
password: ${{ secrets.MAIL_PASSWORD }}
secure: true
subject: OpenMLDB Memory Test
body: OpenMLDB Memory Test Failed
html_body: test/integration-test/openmldb-test-java/openmldb-sdk-test/target/surefire-reports/html/overview.html
Expand Down Expand Up @@ -129,16 +184,12 @@ jobs:
bash ${{ github.job }}/openmldb-${{ env.E_VERSION }}-linux/sbin/deploy-all.sh
bash ${{ github.job }}/openmldb-${{ env.E_VERSION }}-linux/sbin/start-all.sh
- name: test
run: |
mkdir mvnrepo
export MAVEN_OPTS="-Dmaven.repo.local=$(pwd)/mvnrepo"
echo $MAVEN_OPTS
bash test/steps/openmldb-sdk-test-java-src.sh -c test_cluster.xml -d deploy -l "1,2,3,4,5" -s "memory"
run: bash test/steps/openmldb-integration-test.sh -j 0.0.0 -c test_cluster.xml -d deploy -l "1,2,3,4,5" -s "memory"
- name: stop openmldb
if: always()
run: bash ${{ github.job }}/openmldb-${{ env.E_VERSION }}-linux/sbin/stop-all.sh
- name: remove openmldb
if: success()
if: always()
run: bash ${{ github.job }}/openmldb-${{ env.E_VERSION }}-linux/sbin/clear-all.sh
- name: TEST Results
if: always()
Expand All @@ -154,10 +205,11 @@ jobs:
if: ${{ failure() }}
uses: dawidd6/action-send-mail@master
with:
server_address: smtp.partner.outlook.cn
server_port: 587
server_address: mail.4paradigm.com
server_port: 995
username: ${{ secrets.MAIL_USERNAME }}
password: ${{ secrets.MAIL_PASSWORD }}
secure: true
subject: OpenMLDB Memory Test
body: OpenMLDB Memory Test Failed
html_body: test/integration-test/openmldb-test-java/openmldb-sdk-test/target/surefire-reports/html/overview.html
Expand Down Expand Up @@ -187,16 +239,12 @@ jobs:
bash ${{ github.job }}/openmldb-${{ env.E_VERSION }}-linux/sbin/deploy-all.sh
bash ${{ github.job }}/openmldb-${{ env.E_VERSION }}-linux/sbin/start-all.sh
- name: test
run: |
mkdir mvnrepo
export MAVEN_OPTS="-Dmaven.repo.local=$(pwd)/mvnrepo"
echo $MAVEN_OPTS
bash test/steps/openmldb-sdk-test-java-src.sh -c test_cluster_disk.xml -d deploy -l "0" -s "hdd"
run: bash test/steps/openmldb-integration-test.sh -j 0.0.0 -c test_cluster_disk.xml -d deploy -l "0" -s "hdd"
- name: stop openmldb
if: always()
run: bash ${{ github.job }}/openmldb-${{ env.E_VERSION }}-linux/sbin/stop-all.sh
- name: remove openmldb
if: success()
if: always()
run: bash ${{ github.job }}/openmldb-${{ env.E_VERSION }}-linux/sbin/clear-all.sh
- name: TEST Results
if: always()
Expand All @@ -212,10 +260,11 @@ jobs:
if: ${{ failure() }}
uses: dawidd6/action-send-mail@master
with:
server_address: smtp.partner.outlook.cn
server_port: 587
server_address: mail.4paradigm.com
server_port: 995
username: ${{ secrets.MAIL_USERNAME }}
password: ${{ secrets.MAIL_PASSWORD }}
secure: true
subject: OpenMLDB HDD Test
body: OpenMLDB HDD Test Failed
html_body: test/integration-test/openmldb-test-java/openmldb-sdk-test/target/surefire-reports/html/overview.html
Expand All @@ -240,20 +289,16 @@ jobs:
- name: install openmldb
run: |
tar -zxf openmldb-linux.tar.gz -C ${{ github.job }}/
bash test/steps/format_config.sh $(pwd)/${{ github.job }}/openmldb-${{ env.E_VERSION }}-linux java-sdk-cluster-memory-0 23001 24000 java ssd
bash test/steps/format_config.sh $(pwd)/${{ github.job }}/openmldb-${{ env.E_VERSION }}-linux ${{ github.job }} 23001 24000 java ssd
bash ${{ github.job }}/openmldb-${{ env.E_VERSION }}-linux/sbin/deploy-all.sh
bash ${{ github.job }}/openmldb-${{ env.E_VERSION }}-linux/sbin/start-all.sh
- name: test
run: |
mkdir mvnrepo
export MAVEN_OPTS="-Dmaven.repo.local=$(pwd)/mvnrepo"
echo $MAVEN_OPTS
bash test/steps/openmldb-sdk-test-java-src.sh -c test_cluster_disk.xml -d deploy -l "0" -s "ssd"
run: bash test/steps/openmldb-integration-test.sh -j 0.0.0 -c test_cluster_disk.xml -d deploy -l "0" -s "ssd"
- name: stop openmldb
if: always()
run: bash ${{ github.job }}/openmldb-${{ env.E_VERSION }}-linux/sbin/stop-all.sh
- name: remove openmldb
if: success()
if: always()
run: bash ${{ github.job }}/openmldb-${{ env.E_VERSION }}-linux/sbin/clear-all.sh
- name: TEST Results
if: always()
Expand All @@ -269,10 +314,11 @@ jobs:
if: ${{ failure() }}
uses: dawidd6/action-send-mail@master
with:
server_address: smtp.partner.outlook.cn
server_port: 587
server_address: mail.4paradigm.com
server_port: 995
username: ${{ secrets.MAIL_USERNAME }}
password: ${{ secrets.MAIL_PASSWORD }}
secure: true
subject: OpenMLDB SSD Test
body: OpenMLDB SSD Test Failed
html_body: test/integration-test/openmldb-test-java/openmldb-sdk-test/target/surefire-reports/html/overview.html
Expand All @@ -299,19 +345,15 @@ jobs:
bash test/steps/format_config.sh $(pwd)/${{ github.job }}/openmldb-${{ env.E_VERSION }}-linux ${{ github.job }} 24001 25000 java hadoop
bash ${{ github.job }}/openmldb-${{ env.E_VERSION }}-linux/sbin/deploy-all.sh
bash ${{ github.job }}/openmldb-${{ env.E_VERSION }}-linux/sbin/start-all.sh
bash ${{ github.job }}/openmldb-${{ env.E_VERSION }}-linux/sbin/stop-taskmanager.sh
bash HADOOP_CONF_DIR=/mnt/hdd0/denglong/openmldb_runner_work/hadoop ${{ github.job }}/openmldb-${{ env.E_VERSION }}-linux/sbin/stop-taskmanager.sh
bash ${{ github.job }}/openmldb-${{ env.E_VERSION }}-linux/sbin/stop-taskmanagers.sh
HADOOP_CONF_DIR=/mnt/hdd0/denglong/openmldb_runner_work/hadoop ${{ github.job }}/openmldb-${{ env.E_VERSION }}-linux/sbin/stop-taskmanagers.sh
- name: test
run: |
mkdir mvnrepo
export MAVEN_OPTS="-Dmaven.repo.local=$(pwd)/mvnrepo"
echo $MAVEN_OPTS
bash test/steps/openmldb-sdk-test-java-src.sh -c test_yarn.xml -d deploy -l "0" -s "memory"
run: bash test/steps/openmldb-integration-test.sh -j 0.0.0 -c test_yarn.xml -d deploy -l "0" -s "memory"
- name: stop openmldb
if: always()
run: bash ${{ github.job }}/openmldb-${{ env.E_VERSION }}-linux/sbin/stop-all.sh
- name: remove openmldb
if: success()
if: always()
run: bash ${{ github.job }}/openmldb-${{ env.E_VERSION }}-linux/sbin/clear-all.sh
- name: TEST Results
if: always()
Expand All @@ -327,10 +369,11 @@ jobs:
if: ${{ failure() }}
uses: dawidd6/action-send-mail@master
with:
server_address: smtp.partner.outlook.cn
server_port: 587
server_address: mail.4paradigm.com
server_port: 995
username: ${{ secrets.MAIL_USERNAME }}
password: ${{ secrets.MAIL_PASSWORD }}
secure: true
subject: OpenMLDB yarn Test
body: OpenMLDB yarn Test Failed
html_body: test/integration-test/openmldb-test-java/openmldb-sdk-test/target/surefire-reports/html/overview.html
Expand Down Expand Up @@ -359,15 +402,12 @@ jobs:
bash ${{ github.job }}/openmldb-${{ env.E_VERSION }}-linux/sbin/deploy-all.sh
bash ${{ github.job }}/openmldb-${{ env.E_VERSION }}-linux/sbin/start-all.sh
- name: test
run: |
mkdir mvnrepo
export MAVEN_OPTS="-Dmaven.repo.local=$(pwd)/mvnrepo"
echo $MAVEN_OPTS
run: echo "a"
- name: stop openmldb
if: always()
run: bash ${{ github.job }}/openmldb-${{ env.E_VERSION }}-linux/sbin/stop-all.sh
- name: remove openmldb
if: success()
if: always()
run: bash ${{ github.job }}/openmldb-${{ env.E_VERSION }}-linux/sbin/clear-all.sh
# - name: TEST Results
# if: always()
Expand All @@ -383,8 +423,9 @@ jobs:
# if: ${{ failure() }}
# uses: dawidd6/action-send-mail@master
# with:
# server_address: smtp.partner.outlook.cn
# server_port: 587
# server_address: mail.4paradigm.com
# server_port: 995
# secure: true
# username: ${{ secrets.MAIL_USERNAME }}
# password: ${{ secrets.MAIL_PASSWORD }}
# subject: OpenMLDB yarn Test
Expand Down
Loading

0 comments on commit b8318b8

Please sign in to comment.