From e3beccbde14f9c5d31911b697ce7115cc5b5b3c3 Mon Sep 17 00:00:00 2001 From: Shungang Li Date: Thu, 24 Oct 2024 10:50:29 +0800 Subject: [PATCH 01/28] enh: add more log for last cache retrieve --- source/dnode/vnode/src/tsdb/tsdbCacheRead.c | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/source/dnode/vnode/src/tsdb/tsdbCacheRead.c b/source/dnode/vnode/src/tsdb/tsdbCacheRead.c index d508d759229a..c7626dcf367e 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCacheRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbCacheRead.c @@ -613,6 +613,16 @@ int32_t tsdbRetrieveCacheRows(void* pReader, SSDataBlock* pResBlock, const int32 singleTableLastTs = pColVal->rowKey.ts; } + if (p->colVal.value.type != pColVal->colVal.value.type) { + // check for type/cid mismatch + tsdbError("last cache type mismatch, uid:%" PRIu64 + ", schema-type:%d, slotId:%d, cache-type:%d, cache-col:%d", + uid, p->colVal.value.type, slotIds[k], pColVal->colVal.value.type, pColVal->colVal.cid); + taosArrayClearEx(pRow, tsdbCacheFreeSLastColItem); + code = TSDB_CODE_INVALID_PARA; + goto _end; + } + if (!IS_VAR_DATA_TYPE(pColVal->colVal.value.type)) { p->colVal = pColVal->colVal; } else { From 8984231b9fcb8bd5d4f7fe2fb2a92612f19896a8 Mon Sep 17 00:00:00 2001 From: xiao-77 Date: Mon, 4 Nov 2024 18:48:55 +0800 Subject: [PATCH 02/28] fix vnode cannot restart while wal level =0 --- source/libs/wal/src/walMeta.c | 3 ++ source/libs/wal/src/walWrite.c | 7 +++- source/libs/wal/test/walMetaTest.cpp | 56 ++++++++++++++++++++++++++++ 3 files changed, 64 insertions(+), 2 deletions(-) diff --git a/source/libs/wal/src/walMeta.c b/source/libs/wal/src/walMeta.c index d9666eb02f58..be7f5fb0f5bf 100644 --- a/source/libs/wal/src/walMeta.c +++ b/source/libs/wal/src/walMeta.c @@ -424,6 +424,9 @@ void printFileSet(SArray* fileSet) { int32_t walCheckAndRepairMeta(SWal* pWal) { // load log files, get first/snapshot/last version info + if (pWal->cfg.level == TAOS_WAL_SKIP) { + return TSDB_CODE_SUCCESS; + } int32_t code = 0; const char* logPattern = "^[0-9]+.log$"; const char* idxPattern = "^[0-9]+.idx$"; diff --git a/source/libs/wal/src/walWrite.c b/source/libs/wal/src/walWrite.c index c8c37b11bc56..187988b726fd 100644 --- a/source/libs/wal/src/walWrite.c +++ b/source/libs/wal/src/walWrite.c @@ -294,8 +294,11 @@ int32_t walRollback(SWal *pWal, int64_t ver) { static int32_t walRollImpl(SWal *pWal) { int32_t code = 0, lino = 0; + if (pWal->cfg.level == TAOS_WAL_SKIP && pWal->pIdxFile != NULL && pWal->pLogFile != NULL) { + TAOS_RETURN(TSDB_CODE_SUCCESS); + } if (pWal->pIdxFile != NULL) { - if (pWal->cfg.level != TAOS_WAL_SKIP && (code = taosFsyncFile(pWal->pIdxFile)) != 0) { + if ((code = taosFsyncFile(pWal->pIdxFile)) != 0) { TAOS_CHECK_GOTO(terrno, &lino, _exit); } code = taosCloseFile(&pWal->pIdxFile); @@ -305,7 +308,7 @@ static int32_t walRollImpl(SWal *pWal) { } if (pWal->pLogFile != NULL) { - if (pWal->cfg.level != TAOS_WAL_SKIP && (code = taosFsyncFile(pWal->pLogFile)) != 0) { + if ((code = taosFsyncFile(pWal->pLogFile)) != 0) { TAOS_CHECK_GOTO(terrno, &lino, _exit); } code = taosCloseFile(&pWal->pLogFile); diff --git a/source/libs/wal/test/walMetaTest.cpp b/source/libs/wal/test/walMetaTest.cpp index a0285f13632a..3e6fab116f1b 100644 --- a/source/libs/wal/test/walMetaTest.cpp +++ b/source/libs/wal/test/walMetaTest.cpp @@ -455,3 +455,59 @@ TEST_F(WalRetentionEnv, repairMeta1) { } walCloseReader(pRead); } + +class WalSkipLevel : public ::testing::Test { + protected: + static void SetUpTestCase() { + int code = walInit(NULL); + ASSERT(code == 0); + } + + static void TearDownTestCase() { walCleanUp(); } + + void walResetEnv() { + TearDown(); + taosRemoveDir(pathName); + SetUp(); + } + + void SetUp() override { + SWalCfg cfg; + cfg.rollPeriod = -1; + cfg.segSize = -1; + cfg.committed =-1; + cfg.retentionPeriod = -1; + cfg.retentionSize = 0; + cfg.rollPeriod = 0; + cfg.vgId = 1; + cfg.level = TAOS_WAL_SKIP; + pWal = walOpen(pathName, &cfg); + ASSERT(pWal != NULL); + } + + void TearDown() override { + walClose(pWal); + pWal = NULL; + } + + SWal* pWal = NULL; + const char* pathName = TD_TMP_DIR_PATH "wal_test"; +}; + +TEST_F(WalSkipLevel, restart) { + walResetEnv(); + int code; + + int i; + for (i = 0; i < 100; i++) { + char newStr[100]; + sprintf(newStr, "%s-%d", ranStr, i); + int len = strlen(newStr); + code = walAppendLog(pWal, i, 0, syncMeta, newStr, len); + ASSERT_EQ(code, 0); + } + + TearDown(); + + SetUp(); +} \ No newline at end of file From aa8bc8379192af72636f51df46c9d4211c2ecdac Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Wed, 6 Nov 2024 15:46:21 +0800 Subject: [PATCH 03/28] update fqdn cache --- source/libs/transport/src/transCli.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 43f64924e1ab..3607f153e999 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -1608,6 +1608,8 @@ static int32_t cliDoConn(SCliThrd* pThrd, SCliConn* conn) { ret = uv_tcp_connect(&conn->connReq, (uv_tcp_t*)(conn->stream), (const struct sockaddr*)&addr, cliConnCb); if (ret != 0) { tError("failed connect to %s since %s", conn->dstAddr, uv_err_name(ret)); + if (ret == ) + cliMayUpdateFqdnCache(pThrd->fqdn2ipCache, conn->dstAddr); TAOS_CHECK_GOTO(TSDB_CODE_THIRDPARTY_ERROR, &lino, _exception1); } @@ -1699,6 +1701,7 @@ void cliConnCb(uv_connect_t* req, int status) { if (status != 0) { tDebug("%s conn %p failed to connect to %s since %s", CONN_GET_INST_LABEL(pConn), pConn, pConn->dstAddr, uv_strerror(status)); + cliMayUpdateFqdnCache(pThrd->fqdn2ipCache, pConn->dstAddr); TAOS_UNUSED(transUnrefCliHandle(pConn)); return; } @@ -1850,7 +1853,7 @@ static FORCE_INLINE int32_t cliUpdateFqdnCache(SHashObj* cache, char* fqdn) { size_t len = strlen(fqdn); uint32_t* v = taosHashGet(cache, fqdn, len); if (addr != *v) { - char old[TD_IP_LEN] = {0}, new[TD_IP_LEN] = {0}; + char old[TSDB_FQDN_LEN] = {0}, new[TSDB_FQDN_LEN] = {0}; tinet_ntoa(old, *v); tinet_ntoa(new, addr); tWarn("update ip of fqdn:%s, old: %s, new: %s", fqdn, old, new); @@ -1870,7 +1873,7 @@ static void cliMayUpdateFqdnCache(SHashObj* cache, char* dst) { if (dst[i] == ':') break; } if (i > 0) { - char fqdn[TSDB_FQDN_LEN + 1] = {0}; + char fqdn[TSDB_FQDN_LEN] = {0}; memcpy(fqdn, dst, i); TAOS_UNUSED(cliUpdateFqdnCache(cache, fqdn)); } From 0a9d1d8ff7898b46e9b8729260b1135058f4d793 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Wed, 6 Nov 2024 15:55:42 +0800 Subject: [PATCH 04/28] update fqdn cache --- source/libs/transport/src/transCli.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 3607f153e999..2bc5ddc1a665 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -620,7 +620,7 @@ int32_t cliHandleState_mayCreateAhandle(SCliConn* conn, STransMsgHead* pHead, ST int32_t code = 0; int64_t qId = taosHton64(pHead->qid); if (qId == 0) { - return 0; + return TSDB_CODE_RPC_NO_STATE; } STransCtx* pCtx = taosHashGet(conn->pQTable, &qId, sizeof(qId)); @@ -1608,7 +1608,6 @@ static int32_t cliDoConn(SCliThrd* pThrd, SCliConn* conn) { ret = uv_tcp_connect(&conn->connReq, (uv_tcp_t*)(conn->stream), (const struct sockaddr*)&addr, cliConnCb); if (ret != 0) { tError("failed connect to %s since %s", conn->dstAddr, uv_err_name(ret)); - if (ret == ) cliMayUpdateFqdnCache(pThrd->fqdn2ipCache, conn->dstAddr); TAOS_CHECK_GOTO(TSDB_CODE_THIRDPARTY_ERROR, &lino, _exception1); } From 268576b38646f8b9e26e2ae86ae241910b331fb4 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Wed, 6 Nov 2024 16:15:36 +0800 Subject: [PATCH 05/28] fix double free when meet mem leak --- source/libs/transport/src/transCli.c | 1 + 1 file changed, 1 insertion(+) diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 2bc5ddc1a665..c732def505e2 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -2919,6 +2919,7 @@ bool cliMayRetry(SCliConn* pConn, SCliReq* pReq, STransMsg* pResp) { noDelay = cliResetEpset(pCtx, pResp, false); transFreeMsg(pResp->pCont); } + pResp->pCont = NULL; if (code != TSDB_CODE_RPC_BROKEN_LINK && code != TSDB_CODE_RPC_NETWORK_UNAVAIL && code != TSDB_CODE_SUCCESS) { // save one internal code pCtx->retryCode = code; From 4b4943335e84bdb482506152caff5bc82344245c Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 6 Nov 2024 16:15:40 +0800 Subject: [PATCH 06/28] fix(stream): fix the underlying scan operations for stream, when trying to drop stream tasks. --- source/libs/stream/src/streamMeta.c | 8 ++++++++ source/libs/stream/src/streamTaskSm.c | 4 +++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c index bc7878f7618a..2ce8eefe99f9 100644 --- a/source/libs/stream/src/streamMeta.c +++ b/source/libs/stream/src/streamMeta.c @@ -807,6 +807,14 @@ static int32_t streamTaskSendTransSuccessMsg(SStreamTask* pTask, void* param) { tstrerror(code)); } } + + // let's kill the query procedure within stream, to end it ASAP. + if (pTask->info.taskLevel != TASK_LEVEL__SINK && pTask->exec.pExecutor != NULL) { + code = qKillTask(pTask->exec.pExecutor, TSDB_CODE_SUCCESS); + if (code != TSDB_CODE_SUCCESS) { + stError("s-task:%s failed to kill task related query handle, code:%s", pTask->id.idStr, tstrerror(code)); + } + } return code; } diff --git a/source/libs/stream/src/streamTaskSm.c b/source/libs/stream/src/streamTaskSm.c index c3a2742aa209..f995c4868894 100644 --- a/source/libs/stream/src/streamTaskSm.c +++ b/source/libs/stream/src/streamTaskSm.c @@ -500,7 +500,9 @@ int32_t streamTaskOnHandleEventSuccess(SStreamTaskSM* pSM, EStreamTaskEvent even STaskStateTrans* pTrans = pSM->pActiveTrans; if (pTrans == NULL) { ETaskStatus s = pSM->current.state; - + // when trying to finish current event successfully, another event with high priorities, such as dropping/stop, has + // interrupted this procedure, and changed the status after freeing the activeTrans, resulting in the failure of + // processing of current event. if (s != TASK_STATUS__DROPPING && s != TASK_STATUS__PAUSE && s != TASK_STATUS__STOP && s != TASK_STATUS__UNINIT && s != TASK_STATUS__READY) { stError("s-task:%s invalid task status:%s on handling event:%s success", id, pSM->current.name, From 5cc02a774f685822eced9d50a9e1761001cc201d Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Thu, 7 Nov 2024 11:08:54 +0800 Subject: [PATCH 07/28] test: add smoke test scripts --- .gitignore | 2 +- packaging/smoke_test/conftest.py | 115 ++++++ packaging/smoke_test/debRpmAutoInstall.sh | 15 + packaging/smoke_test/getAndRunInstaller.bat | 57 +++ packaging/smoke_test/getAndRunInstaller.sh | 325 +++++++++++++++ packaging/smoke_test/lib.py | 12 + packaging/smoke_test/main.py | 21 + packaging/smoke_test/pytest_require.txt | 16 + packaging/smoke_test/runCases.bat | 11 + packaging/smoke_test/runCases.sh | 29 ++ packaging/smoke_test/smokeTestClient.py | 251 ++++++++++++ packaging/smoke_test/smokeTestJenkinsFile | 380 ++++++++++++++++++ packaging/smoke_test/start3NodesServer.sh | 67 +++ packaging/smoke_test/test_client.py | 137 +++++++ packaging/smoke_test/test_server.py | 238 +++++++++++ packaging/smoke_test/test_server_unix_case | 10 + packaging/smoke_test/test_server_windows_case | 2 + .../smoke_test/versionCheckAndUninstall.py | 260 ++++++++++++ .../versionCheckAndUninstallforPytest.py | 137 +++++++ tests/system-test/test.py | 5 +- 20 files changed, 2088 insertions(+), 2 deletions(-) create mode 100644 packaging/smoke_test/conftest.py create mode 100755 packaging/smoke_test/debRpmAutoInstall.sh create mode 100644 packaging/smoke_test/getAndRunInstaller.bat create mode 100755 packaging/smoke_test/getAndRunInstaller.sh create mode 100644 packaging/smoke_test/lib.py create mode 100644 packaging/smoke_test/main.py create mode 100644 packaging/smoke_test/pytest_require.txt create mode 100644 packaging/smoke_test/runCases.bat create mode 100644 packaging/smoke_test/runCases.sh create mode 100644 packaging/smoke_test/smokeTestClient.py create mode 100644 packaging/smoke_test/smokeTestJenkinsFile create mode 100644 packaging/smoke_test/start3NodesServer.sh create mode 100644 packaging/smoke_test/test_client.py create mode 100644 packaging/smoke_test/test_server.py create mode 100644 packaging/smoke_test/test_server_unix_case create mode 100644 packaging/smoke_test/test_server_windows_case create mode 100644 packaging/smoke_test/versionCheckAndUninstall.py create mode 100644 packaging/smoke_test/versionCheckAndUninstallforPytest.py diff --git a/.gitignore b/.gitignore index d684262f9dda..7f2382db8707 100644 --- a/.gitignore +++ b/.gitignore @@ -161,4 +161,4 @@ version.h geos_c.h source/libs/parser/src/sql.c include/common/ttokenauto.h - +!packaging/smoke_test/pytest_require.txt diff --git a/packaging/smoke_test/conftest.py b/packaging/smoke_test/conftest.py new file mode 100644 index 000000000000..a5f6ebbbe96a --- /dev/null +++ b/packaging/smoke_test/conftest.py @@ -0,0 +1,115 @@ +# conftest.py +import pytest + + +def pytest_addoption(parser): + parser.addoption( + "--verMode", default="enterprise", help="community or enterprise" + ) + parser.addoption( + "--tVersion", default="3.3.2.6", help="the version of taos" + ) + parser.addoption( + "--baseVersion", default="smoking", help="the path of nas" + ) + parser.addoption( + "--sourcePath", default="nas", help="only support nas currently" + ) + + + + +# Collect the setup and teardown of each test case and their std information +setup_stdout_info = {} +teardown_stdout_info = {} + + +@pytest.hookimpl(hookwrapper=True) +def pytest_runtest_makereport(item, call): + outcome = yield + rep = outcome.get_result() + + # Record the std of setup and teardown + if call.when == 'setup': + for i in rep.sections: + if i[0] == "Captured stdout setup": + if not setup_stdout_info: + setup_stdout_info[item.nodeid] = i[1] + elif call.when == 'teardown': + for i in rep.sections: + if i[0] == "Captured stdout teardown": + teardown_stdout_info[item.nodeid] = i[1] + + +# Insert setup and teardown's std in the summary section +def pytest_html_results_summary(prefix, summary, postfix): + if setup_stdout_info or teardown_stdout_info: + rows = [] + + # Insert setup stdout + if setup_stdout_info: + for nodeid, stdout in setup_stdout_info.items(): + html_content = ''' + + Setup: + + Show Setup + + + + '''.format(stdout.strip()) + + # 如果需要在 Python 脚本中生成 HTML，并使用 JavaScript 控制折叠内容的显示，可以这样做： + + html_script = ''' + + ''' + + # 输出完整的 HTML 代码 + final_html = html_content + html_script + rows.append(final_html) + rows.append("
") + # Insert teardown stdout + if teardown_stdout_info: + for nodeid, stdout in teardown_stdout_info.items(): + html_content = ''' + + Teardown: + + Show Teardown + + + + '''.format(stdout.strip()) + + # 如果需要在 Python 脚本中生成 HTML，并使用 JavaScript 控制折叠内容的显示，可以这样做： + + html_script = ''' + + ''' + + # 输出完整的 HTML 代码 + final_html = html_content + html_script + rows.append(final_html) + + prefix.extend(rows) diff --git a/packaging/smoke_test/debRpmAutoInstall.sh b/packaging/smoke_test/debRpmAutoInstall.sh new file mode 100755 index 000000000000..8fadffe4c68d --- /dev/null +++ b/packaging/smoke_test/debRpmAutoInstall.sh @@ -0,0 +1,15 @@ +#!/usr/bin/expect +set packageName [lindex $argv 0] +set packageSuffix [lindex $argv 1] +set timeout 30 +if { ${packageSuffix} == "deb" } { + spawn dpkg -i ${packageName} +} elseif { ${packageSuffix} == "rpm"} { + spawn rpm -ivh ${packageName} +} +expect "*one:" +send "\r" +expect "*skip:" +send "\r" + +expect eof diff --git a/packaging/smoke_test/getAndRunInstaller.bat b/packaging/smoke_test/getAndRunInstaller.bat new file mode 100644 index 000000000000..08b04a027197 --- /dev/null +++ b/packaging/smoke_test/getAndRunInstaller.bat @@ -0,0 +1,57 @@ +set baseVersion=%1% +set version=%2% +set verMode=%3% +set sType=%4% +echo %fileType% +rem stop services +if EXIST C:\TDengine ( + if EXIST C:\TDengine\stop-all.bat ( + call C:\TDengine\stop-all.bat /silent + echo "***************Stop taos services***************" + ) + if exist C:\TDengine\unins000.exe ( + call C:\TDengine\unins000.exe /silent + echo "***************uninstall TDengine***************" + ) + rd /S /q C:\TDengine +) +if EXIST C:\ProDB ( + if EXIST C:\ProDB\stop-all.bat ( + call C:\ProDB\stop-all.bat /silent + echo "***************Stop taos services***************" + ) + if exist C:\ProDB\unins000.exe ( + call C:\ProDB\unins000.exe /silent + echo "***************uninstall TDengine***************" + ) + rd /S /q C:\ProDB +) +if "%verMode%"=="enterprise" ( + if "%sType%"=="client" ( + set fileType=enterprise-client + ) else ( + set fileType=enterprise + ) +) else ( + set fileType=%sType% +) + +if "%baseVersion%"=="ProDB" ( + echo %fileType% + set installer=ProDB-%fileType%-%version%-Windows-x64.exe +) else ( + echo %fileType% + set installer=TDengine-%fileType%-%version%-Windows-x64.exe +) + +if "%baseVersion%"=="ProDB" ( + echo %installer% + scp root@192.168.1.213:/nas/OEM/ProDB/v%version%/%installer% C:\workspace +) else ( + echo %installer% + scp root@192.168.1.213:/nas/TDengine/%baseVersion%/v%version%/%verMode%/%installer% C:\workspace +) + +echo "***************Finish installer transfer!***************" +C:\workspace\%installer% /silent +echo "***************Finish install!***************" \ No newline at end of file diff --git a/packaging/smoke_test/getAndRunInstaller.sh b/packaging/smoke_test/getAndRunInstaller.sh new file mode 100755 index 000000000000..7defe6394c68 --- /dev/null +++ b/packaging/smoke_test/getAndRunInstaller.sh @@ -0,0 +1,325 @@ +#!/bin/sh + + +function usage() { + echo "$0" + echo -e "\t -f test file type,server/client/tools/" + echo -e "\t -m pacakage version Type,community/enterprise" + echo -e "\t -l package type,lite or not" + echo -e "\t -c operation type,x64/arm64" + echo -e "\t -v pacakage version,3.0.1.7" + echo -e "\t -o pacakage version,3.0.1.7" + echo -e "\t -s source Path,web/nas" + echo -e "\t -t package Type,tar/rpm/deb" + echo -e "\t -h help" +} + + +#parameter +scriptDir=$(dirname $(readlink -f $0)) +version="3.0.1.7" +originversion="smoking" +testFile="server" +verMode="communtity" +sourcePath="nas" +cpuType="x64" +lite="true" +packageType="tar" +subFile="package.tar.gz" +while getopts "m:c:f:l:s:o:t:v:h" opt; do + case $opt in + m) + verMode=$OPTARG + ;; + v) + version=$OPTARG + ;; + f) + testFile=$OPTARG + ;; + l) + lite=$OPTARG + ;; + s) + sourcePath=$OPTARG + ;; + o) + originversion=$OPTARG + ;; + c) + cpuType=$OPTARG + ;; + t) + packageType=$OPTARG + ;; + h) + usage + exit 0 + ;; + ?) + echo "Invalid option: -$OPTARG" + usage + exit 0 + ;; + esac +done + +systemType=`uname` +if [ ${systemType} == "Darwin" ]; then + platform="macOS" +else + platform="Linux" +fi + +echo "testFile:${testFile},verMode:${verMode},lite:${lite},cpuType:${cpuType},packageType:${packageType},version-${version},originversion:${originversion},sourcePath:${sourcePath}" +# Color setting +RED='\033[41;30m' +GREEN='\033[1;32m' +YELLOW='\033[1;33m' +BLUE='\033[1;34m' +GREEN_DARK='\033[0;32m' +YELLOW_DARK='\033[0;33m' +BLUE_DARK='\033[0;34m' +GREEN_UNDERLINE='\033[4;32m' +NC='\033[0m' +if [ "${originversion}" = "ProDB" ]; then + TDengine="ProDB" +else + TDengine="TDengine" +fi +if [[ ${verMode} = "enterprise" ]];then + prePackage="${TDengine}-enterprise" + if [[ ${testFile} = "client" ]];then + prePackage="${TDengine}-enterprise-${testFile}" + fi +elif [ ${verMode} = "community" ];then + prePackage="${TDengine}-${testFile}" +fi +if [ ${lite} = "true" ];then + packageLite="-Lite" +elif [ ${lite} = "false" ];then + packageLite="" +fi +if [[ "$packageType" = "tar" ]] ;then + packageType="tar.gz" +fi + +tdPath="${prePackage}-${version}" + +packageName="${tdPath}-${platform}-${cpuType}${packageLite}.${packageType}" + +if [ "$testFile" == "server" ] ;then + installCmd="install.sh" +elif [ ${testFile} = "client" ];then + installCmd="install_client.sh" +fi + +echo "tdPath:${tdPath},packageName:${packageName}}" +cmdInstall() { +command=$1 +if command -v ${command} ;then + echoColor YD "${command} is already installed" +else + if command -v apt ;then + apt-get install ${command} -y + elif command -v yum ;then + yum -y install ${command} + echoColor YD "you should install ${command} manually" + fi +fi +} + +echoColor() { + color=$1 + command=$2 + if [ ${color} = 'Y' ];then + echo -e "${YELLOW}${command}${NC}" + elif [ ${color} = 'YD' ];then + echo -e "${YELLOW_DARK}${command}${NC}" + elif [ ${color} = 'R' ];then + echo -e "${RED}${command}${NC}" + elif [ ${color} = 'G' ];then + echo -e "${GREEN}${command}${NC}\r\n" + elif [ ${color} = 'B' ];then + echo -e "${BLUE}${command}${NC}" + elif [ ${color} = 'BD' ];then + echo -e "${BLUE_DARK}${command}${NC}" + fi +} + +wgetFile() { + + file=$1 + versionPath=$2 + sourceP=$3 + nasServerIP="192.168.1.213" + if [ "${originversion}" = "ProDB" ]; then + packagePath="/nas/OEM/ProDB/v${versionPath}" + else + packagePath="/nas/TDengine/${originversion}/v${versionPath}/${verMode}" + fi + if [ -f ${file} ];then + echoColor YD "${file} already exists ,it will delete it and download it again " + # rm -rf ${file} + fi + + if [[ ${sourceP} = 'web' ]];then + echoColor BD "====download====:wget https://www.taosdata.com/assets-download/3.0/${file}" + wget https://www.taosdata.com/assets-download/3.0/${file} + elif [[ ${sourceP} = 'nas' ]];then + echoColor BD "====download====:scp root@${nasServerIP}:${packagePath}/${file} ." + scp root@${nasServerIP}:${packagePath}/${file} . + fi +} + +function newPath { + +buildPath=$1 + +if [ ! -d ${buildPath} ] ;then + echoColor BD "mkdir -p ${buildPath}" + mkdir -p ${buildPath} +else + echoColor YD "${buildPath} already exists" +fi + +} + +echoColor G "===== install basesoft =====" +cmdInstall tree +cmdInstall wget +cmdInstall expect + +echoColor G "===== Uninstall all components of TDeingne =====" + +if command -v rmtaos ;then + echoColor YD "uninstall all components of TDeingne:rmtaos" + rmtaos +else + echoColor YD "os doesn't include TDengine" +fi + +if [[ ${packageName} =~ "server" ]] ;then + echoColor BD " pkill -9 taosd " + pkill -9 taosd +fi + +if command -v rmprodb ;then + echoColor YD "uninstall all components of TDeingne:rmprodb" + rmprodb +else + echoColor YD "os doesn't include TDengine" +fi + +if [[ ${packageName} =~ "server" ]] ;then + echoColor BD " pkill -9 prodbd " + pkill -9 prodbd +fi + +echoColor G "===== new workroom path =====" +installPath="/usr/local/src/packageTest" + +if [ ${systemType} == "Darwin" ]; then + installPath="${WORK_DIR}/packageTest" +fi + +newPath ${installPath} + +#if [ -d ${installPath}/${tdPath} ] ;then +# echoColor BD "rm -rf ${installPath}/${tdPath}/*" +# rm -rf ${installPath}/${tdPath}/* +#fi + +echoColor G "===== download installPackage =====" +cd ${installPath} && wgetFile ${packageName} ${version} ${sourcePath} +#cd ${oriInstallPath} && wgetFile ${originPackageName} ${originversion} ${sourcePath} + + +cd ${installPath} +cp -r ${scriptDir}/debRpmAutoInstall.sh . + +packageSuffix=$(echo ${packageName} | awk -F '.' '{print $NF}') + + +if [ ! -f debRpmAutoInstall.sh ];then + echo '#!/usr/bin/expect ' > debRpmAutoInstall.sh + echo 'set packageName [lindex $argv 0]' >> debRpmAutoInstall.sh + echo 'set packageSuffix [lindex $argv 1]' >> debRpmAutoInstall.sh + echo 'set timeout 30 ' >> debRpmAutoInstall.sh + echo 'if { ${packageSuffix} == "deb" } {' >> debRpmAutoInstall.sh + echo ' spawn dpkg -i ${packageName} ' >> debRpmAutoInstall.sh + echo '} elseif { ${packageSuffix} == "rpm"} {' >> debRpmAutoInstall.sh + echo ' spawn rpm -ivh ${packageName}' >> debRpmAutoInstall.sh + echo '}' >> debRpmAutoInstall.sh + echo 'expect "*one:"' >> debRpmAutoInstall.sh + echo 'send "\r"' >> debRpmAutoInstall.sh + echo 'expect "*skip:"' >> debRpmAutoInstall.sh + echo 'send "\r" ' >> debRpmAutoInstall.sh +fi + + +echoColor G "===== install Package =====" + +if [[ ${packageName} =~ "deb" ]];then + cd ${installPath} + dpkg -r taostools + dpkg -r tdengine + if [[ ${packageName} =~ "TDengine" ]];then + echoColor BD "./debRpmAutoInstall.sh ${packageName} ${packageSuffix}" && chmod 755 debRpmAutoInstall.sh && ./debRpmAutoInstall.sh ${packageName} ${packageSuffix} + else + echoColor BD "dpkg -i ${packageName}" && dpkg -i ${packageName} + fi +elif [[ ${packageName} =~ "rpm" ]];then + cd ${installPath} + sudo rpm -e tdengine + sudo rpm -e taostools + if [[ ${packageName} =~ "TDengine" ]];then + echoColor BD "./debRpmAutoInstall.sh ${packageName} ${packageSuffix}" && chmod 755 debRpmAutoInstall.sh && ./debRpmAutoInstall.sh ${packageName} ${packageSuffix} + else + echoColor BD "rpm -ivh ${packageName}" && rpm -ivh ${packageName} + fi +elif [[ ${packageName} =~ "tar" ]];then + echoColor G "===== check installPackage File of tar =====" + + cd ${installPath} + echoColor YD "unzip the new installation package" + echoColor BD "tar -xf ${packageName}" && tar -xf ${packageName} + + cd ${installPath}/${tdPath} && tree -I "driver" > ${installPath}/now_${version}_checkfile + + cd ${installPath} + diff ${installPath}/base_${originversion}_checkfile ${installPath}/now_${version}_checkfile > ${installPath}/diffFile.log + diffNumbers=`cat ${installPath}/diffFile.log |wc -l ` + + if [ ${diffNumbers} != 0 ];then + echoColor R "The number and names of files is different from the previous installation package" + diffLog=`cat ${installPath}/diffFile.log` + echoColor Y "${diffLog}" + exit -1 + else + echoColor G "The number and names of files are the same as previous installation packages" + rm -rf ${installPath}/diffFile.log + fi + echoColor YD "===== install Package of tar =====" + cd ${installPath}/${tdPath} + if [ ${testFile} = "server" ];then + echoColor BD "bash ${installCmd} -e no " + bash ${installCmd} -e no + else + echoColor BD "bash ${installCmd} " + bash ${installCmd} + fi +elif [[ ${packageName} =~ "pkg" ]];then + cd ${installPath} + sudo installer -pkg ${packageName} -target / + echoColor YD "===== install Package successfully! =====" +fi + +#cd ${installPath} +# +#rm -rf ${installPath}/${packageName} +#if [ ${platform} == "Linux" ]; then +# rm -rf ${installPath}/${tdPath}/ +#fi +echoColor YD "===== end of shell file =====" + diff --git a/packaging/smoke_test/lib.py b/packaging/smoke_test/lib.py new file mode 100644 index 000000000000..86c30bf8b121 --- /dev/null +++ b/packaging/smoke_test/lib.py @@ -0,0 +1,12 @@ + +import subprocess + + +def run_cmd(command): + print("CMD:", command) + result = subprocess.run(command, capture_output=True, text=True, shell=True) + print("STDOUT:", result.stdout) + print("STDERR:", result.stderr) + print("Return Code:", result.returncode) + #assert result.returncode == 0 + return result diff --git a/packaging/smoke_test/main.py b/packaging/smoke_test/main.py new file mode 100644 index 000000000000..cb7356f80e80 --- /dev/null +++ b/packaging/smoke_test/main.py @@ -0,0 +1,21 @@ +import pytest + +# python3 -m pytest test_server.py -v --html=/var/www/html/report.html --json-report --json-report-file="/var/www/html/report.json" --timeout=60 + +# pytest.main(["-s", "-v"]) +import pytest + +import subprocess + + +# define cmd function + + + + +def main(): + pytest.main(['--html=report.html']) + + +if __name__ == '__main__': + main() diff --git a/packaging/smoke_test/pytest_require.txt b/packaging/smoke_test/pytest_require.txt new file mode 100644 index 000000000000..eadbe92470dc --- /dev/null +++ b/packaging/smoke_test/pytest_require.txt @@ -0,0 +1,16 @@ +pytest-html +pytest-json-report +pytest-timeout +taospy +numpy +fabric2 +psutil +pandas +toml +distro +requests +pexpect +faker +pyopenssl +taos-ws-py +taospy \ No newline at end of file diff --git a/packaging/smoke_test/runCases.bat b/packaging/smoke_test/runCases.bat new file mode 100644 index 000000000000..922766785caf --- /dev/null +++ b/packaging/smoke_test/runCases.bat @@ -0,0 +1,11 @@ +rm -rf %WIN_TDENGINE_ROOT_DIR%\debug +mkdir %WIN_TDENGINE_ROOT_DIR%\debug +mkdir %WIN_TDENGINE_ROOT_DIR%\debug\build +mkdir %WIN_TDENGINE_ROOT_DIR%\debug\build\bin +xcopy C:\TDengine\taos*.exe %WIN_TDENGINE_ROOT_DIR%\debug\build\bin + +set case_out_file=%cd%\case.out + +cd %WIN_TDENGINE_ROOT_DIR%\tests\system-test +python3 .\test.py -f 0-others\taosShell.py +python3 .\test.py -f 6-cluster\5dnode3mnodeSep1VnodeStopDnodeModifyMeta.py -N 6 -M 3 \ No newline at end of file diff --git a/packaging/smoke_test/runCases.sh b/packaging/smoke_test/runCases.sh new file mode 100644 index 000000000000..4de7a7658bc4 --- /dev/null +++ b/packaging/smoke_test/runCases.sh @@ -0,0 +1,29 @@ +#!/bin/bash +ulimit -c unlimited + +rm -rf ${TDENGINE_ROOT_DIR}/debug +mkdir ${TDENGINE_ROOT_DIR}/debug +mkdir ${TDENGINE_ROOT_DIR}/debug/build +mkdir ${TDENGINE_ROOT_DIR}/debug/build/bin + +systemType=`uname` +if [ ${systemType} == "Darwin" ]; then + cp /usr/local/bin/taos* ${TDENGINE_ROOT_DIR}/debug/build/bin/ +else + cp /usr/bin/taos* ${TDENGINE_ROOT_DIR}/debug/build/bin/ +fi + +case_out_file=`pwd`/case.out +python3 -m pip install -r ${TDENGINE_ROOT_DIR}/tests/requirements.txt >> $case_out_file +python3 -m pip install taos-ws-py taospy >> $case_out_file + +cd ${TDENGINE_ROOT_DIR}/tests/army +python3 ./test.py -f query/query_basic.py -N 3 >> $case_out_file + +cd ${TDENGINE_ROOT_DIR}/tests/system-test +python3 ./test.py -f 1-insert/insert_column_value.py >> $case_out_file +python3 ./test.py -f 2-query/primary_ts_base_5.py >> $case_out_file +python3 ./test.py -f 2-query/case_when.py >> $case_out_file +python3 ./test.py -f 2-query/partition_limit_interval.py >> $case_out_file +python3 ./test.py -f 2-query/join.py >> $case_out_file +python3 ./test.py -f 2-query/fill.py >> $case_out_file diff --git a/packaging/smoke_test/smokeTestClient.py b/packaging/smoke_test/smokeTestClient.py new file mode 100644 index 000000000000..eee9667300fd --- /dev/null +++ b/packaging/smoke_test/smokeTestClient.py @@ -0,0 +1,251 @@ +#!/usr/bin/python +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### +# install pip +# pip install src/connector/python/ + +# -*- coding: utf-8 -*- +import sys, os +import re +import platform +import getopt +import subprocess +# from this import d +import time + +# input for server + +opts, args = getopt.gnu_getopt(sys.argv[1:], 'h:P:v:u', [ + 'host=', 'Port=', 'version=']) +serverHost = "" +serverPort = 0 +version = "" +uninstall = False +for key, value in opts: + if key in ['--help']: + print('A collection of test cases written using Python') + print('-h serverHost') + print('-P serverPort') + print('-v test client version') + print('-u test uninstall process, will uninstall TDengine') + sys.exit(0) + + if key in ['-h']: + serverHost = value + if key in ['-P']: + serverPort = int(value) + if key in ['-v']: + version = value + if key in ['-u']: + uninstall = True +if not serverHost: + print("Please input use -h to specify your server host.") + sys.exit(0) +if not version: + print("No version specified, will not run version check.") +if serverPort == 0: + serverPort = 6030 + print("No server port specified, use default 6030.") + + +system = platform.system() + +arch = platform.machine() + +databaseName = re.sub(r'[^a-zA-Z0-9]', '', subprocess.getoutput("hostname")).lower() +# install taospy +taospy_version = "" +if system == 'Windows': + taospy_version = subprocess.getoutput("pip3 show taospy|findstr Version") +else: + taospy_version = subprocess.getoutput("pip3 show taospy|grep Version| awk -F ':' '{print $2}' ") + +print("taospy version %s " % taospy_version) +if taospy_version == "": + subprocess.getoutput("pip3 install git+https://github.com/taosdata/taos-connector-python.git") + print("install taos python connector") +else: + subprocess.getoutput("pip3 install taospy") + +# prepare data by taosBenchmark +cmd = "taosBenchmark -y -a 3 -n 100 -t 100 -d %s -h %s -P %d &" % (databaseName, serverHost, serverPort) +process_out = subprocess.getoutput(cmd) +print(cmd) +#os.system("taosBenchmark -y -a 3 -n 100 -t 100 -d %s -h %s -P %d" % (databaseName, serverHost, serverPort)) +taosBenchmark_test_result = True +time.sleep(10) +import taos + +conn = taos.connect(host=serverHost, + user="root", + password="taosdata", + database=databaseName, + port=serverPort, + timezone="Asia/Shanghai") # default your host's timezone + +server_version = conn.server_info +print("server_version", server_version) +client_version = conn.client_info +print("client_version", client_version) # 3.0.0.0 + +# Execute a sql and get its result set. It's useful for SELECT statement +result: taos.TaosResult = conn.query("SELECT count(*) from meters") + +data = result.fetch_all() +print(data) +if data[0][0] !=10000: + print(" taosBenchmark work not as expected ") + print("!!!!!!!!!!!Test Result: taosBenchmark test failed! !!!!!!!!!!") + sys.exit(1) +#else: +# print("**********Test Result: taosBenchmark test passed **********") + + +# drop database of test +taos_test_result = False +print("drop database test") +print("run taos -s 'drop database %s;' -h %s -P %d" % (databaseName, serverHost, serverPort)) +taos_cmd_outpur = subprocess.getoutput('taos -s "drop database %s;" -h %s -P %d' % (databaseName, serverHost, serverPort)) +print(taos_cmd_outpur) +if ("Drop OK" in taos_cmd_outpur): + taos_test_result = True + #print("*******Test Result: taos test passed ************") + +version_test_result = False +if version: + print("Client info is: %s"%conn.client_info) + taos_V_output = "" + if system == "Windows": + taos_V_output = subprocess.getoutput("taos -V | findstr version") + else: + taos_V_output = subprocess.getoutput("taos -V | grep version") + + print("taos -V output is: %s" % taos_V_output) + if version in taos_V_output and version in conn.client_info: + version_test_result = True + #print("*******Test Result: Version check passed ************") + +conn.close() +if uninstall: + print("Start to run rmtaos") + leftFile = False + print("Platform: ", system) + + if system == "Linux": + # 创建一个subprocess.Popen对象，并使用stdin和stdout进行交互 + process = subprocess.Popen(['rmtaos'], + stdin=subprocess.PIPE, stdout=subprocess.PIPE, text=True) + # 向子进程发送输入 + process.stdin.write("y\n") + process.stdin.flush() # 确保输入被发送到子进程 + process.stdin.write("I confirm that I would like to delete all data, log and configuration files\n") + process.stdin.flush() # 确保输入被发送到子进程 + # 关闭子进程的stdin，防止它无限期等待更多输入 + process.stdin.close() + # 等待子进程结束 + process.wait() + # 检查目录清除情况 + out = subprocess.getoutput("ls /etc/systemd/system/taos*") + if "No such file or directory" not in out: + print("Uninstall left some files: %s" % out) + leftFile = True + out = subprocess.getoutput("ls /usr/bin/taos*") + if "No such file or directory" not in out: + print("Uninstall left some files: %s" % out) + leftFile = True + out = subprocess.getoutput("ls /usr/local/bin/taos*") + if "No such file or directory" not in out: + print("Uninstall left some files: %s" % out) + leftFile = True + out = subprocess.getoutput("ls /usr/lib/libtaos*") + if "No such file or directory" not in out: + print("Uninstall left some files: %s" % out) + leftFile = True + out = subprocess.getoutput("ls /usr/lib64/libtaos*") + if "No such file or directory" not in out: + print("Uninstall left some files: %s" % out) + leftFile = True + out = subprocess.getoutput("ls /usr/include/taos*") + if "No such file or directory" not in out: + print("Uninstall left some files: %s" % out) + leftFile = True + out = subprocess.getoutput("ls /usr/local/taos") + #print(out) + if "No such file or directory" not in out: + print("Uninstall left some files in /usr/local/taos：%s" % out) + leftFile = True + if not leftFile: + print("*******Test Result: uninstall test passed ************") + + elif system == "Darwin": + # 创建一个subprocess.Popen对象，并使用stdin和stdout进行交互 + process = subprocess.Popen(['sudo', 'rmtaos'], + stdin=subprocess.PIPE, stdout=subprocess.PIPE, text=True) + # 向子进程发送输入 + process.stdin.write("y\n") + process.stdin.flush() # 确保输入被发送到子进程 + process.stdin.write("I confirm that I would like to delete all data, log and configuration files\n") + process.stdin.flush() # 确保输入被发送到子进程 + # 关闭子进程的stdin，防止它无限期等待更多输入 + process.stdin.close() + # 等待子进程结束 + process.wait() + # 检查目录清除情况 + out = subprocess.getoutput("ls /usr/local/bin/taos*") + if "No such file or directory" not in out: + print("Uninstall left some files: %s" % out) + leftFile = True + out = subprocess.getoutput("ls /usr/local/lib/libtaos*") + if "No such file or directory" not in out: + print("Uninstall left some files: %s" % out) + leftFile = True + out = subprocess.getoutput("ls /usr/local/include/taos*") + if "No such file or directory" not in out: + print("Uninstall left some files: %s" % out) + leftFile = True + #out = subprocess.getoutput("ls /usr/local/Cellar/tdengine/") + #print(out) + #if out: + # print("Uninstall left some files: /usr/local/Cellar/tdengine/%s" % out) + # leftFile = True + #if not leftFile: + # print("*******Test Result: uninstall test passed ************") + + elif system == "Windows": + process = subprocess.Popen(['unins000','/silent'], + stdin=subprocess.PIPE, stdout=subprocess.PIPE, text=True) + process.wait() + time.sleep(10) + out = subprocess.getoutput("ls C:\TDengine") + print(out) + if len(out.split("\n")) > 3: + leftFile = True + print("Uninstall left some files: %s" % out) + +if taosBenchmark_test_result: + print("**********Test Result: taosBenchmark test passed! **********") +if taos_test_result: + print("**********Test Result: taos test passed! **********") +else: + print("!!!!!!!!!!!Test Result: taos test failed! !!!!!!!!!!") +if version_test_result: + print("**********Test Result: version test passed! **********") +else: + print("!!!!!!!!!!!Test Result: version test failed! !!!!!!!!!!") +if not leftFile: + print("**********Test Result: uninstall test passed! **********") +else: + print("!!!!!!!!!!!Test Result: uninstall test failed! !!!!!!!!!!") +if taosBenchmark_test_result and taos_test_result and version_test_result and not leftFile: + sys.exit(0) +else: + sys.exit(1) + diff --git a/packaging/smoke_test/smokeTestJenkinsFile b/packaging/smoke_test/smokeTestJenkinsFile new file mode 100644 index 000000000000..464393d85d77 --- /dev/null +++ b/packaging/smoke_test/smokeTestJenkinsFile @@ -0,0 +1,380 @@ +def sync_source(branch_name) { + sh ''' + hostname + ip addr|grep 192|awk '{print $2}'|sed "s/\\/.*//" + echo ''' + branch_name + ''' + ''' + sh ''' + cd ${TDENGINE_ROOT_DIR} + set +e + git reset --hard + git fetch || git fetch + git checkout -f '''+branch_name+''' + git reset --hard origin/'''+branch_name+''' + git log | head -n 20 + git clean -fxd + set -e + ''' + return 1 +} +def sync_source_win() { + bat ''' + hostname + taskkill /f /t /im taosd.exe + ipconfig + set + date /t + time /t + ''' + bat ''' + echo %branch_name% + cd %WIN_TDENGINE_ROOT_DIR% + git reset --hard + git fetch || git fetch + git checkout -f ''' + env.BRANCH_NAME + ''' + git reset --hard origin/''' + env.BRANCH_NAME + ''' + git branch + git restore . + git remote prune origin + git pull || git pull + git log | head -n 20 + git clean -fxd + ''' + return 1 +} +pipeline { + agent none + parameters { + choice( + name: 'sourcePath', + choices: ['nas','web'], + description: 'Choice which way to download the installation pacakge;web is Office Web and nas means taos nas server ' + ) + choice( + name: 'verMode', + choices: ['enterprise','community'], + description: 'Choice which types of package you want do check ' + ) + string ( + name:'version', + defaultValue:'3.3.2.0', + description: 'Release version number,eg: 3.0.0.1' + ) + string ( + name:'baseVersion', + defaultValue:'smoking', + description: 'Tnas root path. eg:smoking, 3.3' + ) + choice ( + name:'mode', + choices: ['server','client'], + description: 'Choose which mode of package you want do run ' + ) + choice ( + name:'smoke_branch', + choices: ['test/3.0/smokeTest','test/main/smokeTest','test/3.1/smokeTest'], + description: 'Choose which mode of package you want do run ' + ) + string ( + name:'runPlatforms', + defaultValue:'server_Linux_x64, server_Linux_arm64, server_Windows_x64, server_Mac_x64', + description: 'run package list hotfix usually run: server: server_Linux_x64, server_Linux_arm64 client: client_Linux_x64, client_Linux_arm64 release usually run: enterprise server: server_Linux_x64, server_Linux_arm64, server_Windows_x64 enterprise client: client_Linux_x64, client_Linux_arm64, client_Windows_x64 community server: server_Linux_x64, server_Linux_arm64, server_Mac_x64, server_Mac_arm64(not supported), server_Linux_x64_lite(not supported) community client: client_Linux_x64, client_Linux_arm64, client_Windows_x64, client_Mac_x64, client_Mac_arm64(not supported), client_Linux_x64_lite(not supported)' + ) + } + environment{ + WORK_DIR = "/var/lib/jenkins/workspace" + TDINTERNAL_ROOT_DIR = '/var/lib/jenkins/workspace/TDinternal' + TDENGINE_ROOT_DIR = '/var/lib/jenkins/workspace/TDinternal/community' + BRANCH_NAME = "${smoke_branch}" + } + stages { + stage ('Start Server for Client Test') { + when { + beforeAgent true + expression { mode == 'client' } + } + agent{label " ubuntu18 "} + steps { + timeout(time: 30, unit: 'MINUTES'){ + sync_source("${BRANCH_NAME}") + withEnv(['JENKINS_NODE_COOKIE=dontkillme']) { + sh ''' + cd ${TDENGINE_ROOT_DIR}/packaging/smokeTest + bash getAndRunInstaller.sh -m ${verMode} -f server -l false -c x64 -v ${version} -o ${baseVersion} -s ${sourcePath} -t tar + bash start3NodesServer.sh + ''' + } + } + } + } + stage ('Run SmokeTest') { + parallel { + stage('server_Linux_x64') { + when { + beforeAgent true + allOf { + expression { mode == 'server' } + expression { runPlatforms.contains('server_Linux_x64') } + } + } + agent{label " ubuntu16 "} + steps { + timeout(time: 30, unit: 'MINUTES'){ + sync_source("${BRANCH_NAME}") + sh ''' + mkdir -p /var/www/html/${baseVersion}/${version}/${verMode}/json + cd ${TDENGINE_ROOT_DIR}/packaging/smokeTest + bash getAndRunInstaller.sh -m ${verMode} -f server -l false -c x64 -v ${version} -o ${baseVersion} -s ${sourcePath} -t tar + python3 -m pytest test_server.py -v --html=/var/www/html/${baseVersion}/${version}/${verMode}/${mode}_linux_x64_report.html --json-report --json-report-file=report.json --timeout=300 --verMode=${verMode} --tVersion=${version} --baseVersion=${baseVersion} --sourcePath=${sourcePath} || true + cp report.json /var/www/html/${baseVersion}/${version}/${verMode}/json/${mode}_linux_x64_report.json + curl "http://192.168.0.176/api/addSmoke?version=${version}&tag=${baseVersion}&type=${verMode}&role=server&build=linux_x64" + ''' + } + } + } + stage('server_Linux_arm64') { + when { + beforeAgent true + allOf { + expression { mode == 'server' } + expression { runPlatforms.contains('server_Linux_arm64') } + } + } + agent{label "worker06_arm64"} + steps { + timeout(time: 60, unit: 'MINUTES'){ + sync_source("${BRANCH_NAME}") + sh ''' + cd ${TDENGINE_ROOT_DIR}/packaging/smokeTest + bash getAndRunInstaller.sh -m ${verMode} -f server -l false -c arm64 -v ${version} -o ${baseVersion} -s ${sourcePath} -t tar + python3 -m pytest test_server.py -v --html=${mode}_linux_arm64_report.html --json-report --json-report-file=report.json --timeout=600 --verMode=${verMode} --tVersion=${version} --baseVersion=${baseVersion} --sourcePath=${sourcePath} || true + scp ${mode}_linux_arm64_report.html root@192.168.0.21:/var/www/html/${baseVersion}/${version}/${verMode}/ + scp report.json root@192.168.0.21:/var/www/html/${baseVersion}/${version}/${verMode}/json/${mode}_linux_arm64_report.json + curl "http://192.168.0.176/api/addSmoke?version=${version}&tag=${baseVersion}&type=${verMode}&role=server&build=linux_arm64" + ''' + } + } + } + stage ('server_Mac_x64') { + when { + beforeAgent true + allOf { + expression { mode == 'server' } + expression { runPlatforms.contains('server_Mac_x64') } + } + } + agent{label " release_Darwin_x64 "} + environment{ + WORK_DIR = "/Users/zwen/jenkins/workspace" + TDINTERNAL_ROOT_DIR = '/Users/zwen/jenkins/workspace/TDinternal' + TDENGINE_ROOT_DIR = '/Users/zwen/jenkins/workspace/TDinternal/community' + } + steps { + timeout(time: 30, unit: 'MINUTES'){ + sync_source("${BRANCH_NAME}") + sh ''' + cd ${TDENGINE_ROOT_DIR}/packaging/smokeTest + bash getAndRunInstaller.sh -m ${verMode} -f server -l false -c x64 -v ${version} -o ${baseVersion} -s ${sourcePath} -t pkg + python3 -m pytest -v -k linux --html=${mode}_Mac_x64_report.html --json-report --json-report-file=report.json --timeout=300 --verMode=${verMode} --tVersion=${version} --baseVersion=${baseVersion} --sourcePath=${sourcePath} || true + scp ${mode}_Mac_x64_report.html root@192.168.0.21:/var/www/html/${baseVersion}/${version}/${verMode}/ + scp report.json root@192.168.0.21:/var/www/html/${baseVersion}/${version}/${verMode}/json/${mode}_Mac_x64_report.json + curl "http://192.168.0.176/api/addSmoke?version=${version}&tag=${baseVersion}&type=${verMode}&role=server&build=Mac_x64" + ''' + } + } + } + stage ('server_Mac_arm64') { + when { + beforeAgent true + allOf { + expression { mode == 'server' } + expression { runPlatforms.contains('server_Mac_arm64') } + } + } + agent{label " release_Darwin_arm64 "} + environment{ + WORK_DIR = "/Users/zwen/jenkins/workspace" + TDINTERNAL_ROOT_DIR = '/Users/zwen/jenkins/workspace/TDinternal' + TDENGINE_ROOT_DIR = '/Users/zwen/jenkins/workspace/TDinternal/community' + } + steps { + timeout(time: 30, unit: 'MINUTES'){ + sync_source("${BRANCH_NAME}") + sh ''' + cd ${TDENGINE_ROOT_DIR}/packaging/smokeTest + bash getAndRunInstaller.sh -m ${verMode} -f server -l false -c arm64 -v ${version} -o ${baseVersion} -s ${sourcePath} -t pkg + python3 -m pytest -v -k linux --html=${mode}_Mac_arm64_report.html --json-report --json-report-file=report.json --timeout=300 --verMode=${verMode} --tVersion=${version} --baseVersion=${baseVersion} --sourcePath=${sourcePath} || true + scp ${mode}_Mac_arm64_report.html root@192.168.0.21:/var/www/html/${baseVersion}/${version}/${verMode}/ + scp report.json root@192.168.0.21:/var/www/html/${baseVersion}/${version}/${verMode}/json/${mode}_Mac_arm64_report.json + curl "http://192.168.0.176/api/addSmoke?version=${version}&tag=${baseVersion}&type=${verMode}&role=server&build=Mac_arm64" + ''' + } + } + } + stage('server_Windows_x64') { + when { + beforeAgent true + allOf { + expression { mode == 'server' } + expression { runPlatforms.contains('server_Windows_x64') } + } + } + agent{label " windows11 "} + environment{ + WIN_WORK_DIR="C:\\workspace" + WIN_TDINTERNAL_ROOT_DIR="C:\\workspace\\TDinternal" + WIN_TDENGINE_ROOT_DIR="C:\\workspace\\TDinternal\\community" + } + steps { + timeout(time: 30, unit: 'MINUTES'){ + sync_source_win() + bat ''' + cd %WIN_TDENGINE_ROOT_DIR%\\packaging\\smokeTest + call getAndRunInstaller.bat %baseVersion% %version% %verMode% server + cd %WIN_TDENGINE_ROOT_DIR%\\packaging\\smokeTest + pip3 install -r pytest_require.txt + python3 -m pytest test_server.py -v --html=%mode%_Windows_x64_report.html --json-report --json-report-file=report.json --timeout=300 --verMode=%verMode% --tVersion=%version% --baseVersion=%baseVersion% --sourcePath=%sourcePath% + scp %mode%_Windows_x64_report.html root@192.168.0.21:/var/www/html/%baseVersion%/%version%/%verMode%/ + scp report.json root@192.168.0.21:/var/www/html/%baseVersion%/%version%/%verMode%/json/%mode%_Windows_x64_report.json + curl "http://192.168.0.176/api/addSmoke?version=%version%&tag=%baseVersion%&type=%verMode%&role=server&build=Windows_x64" + ''' + } + } + } + stage('client_Linux_x64') { + when { + beforeAgent true + allOf { + expression { mode == 'client' } + expression { runPlatforms.contains('client_Linux_x64') } + } + } + agent{label " ubuntu16 "} + steps { + timeout(time: 30, unit: 'MINUTES'){ + sync_source("${BRANCH_NAME}") + sh ''' + mkdir -p /var/www/html/${baseVersion}/${version}/${verMode}/json + cd ${TDENGINE_ROOT_DIR}/packaging/smokeTest + bash getAndRunInstaller.sh -m ${verMode} -f client -l false -c x64 -v ${version} -o ${baseVersion} -s ${sourcePath} -t tar + python3 -m pytest test_client.py -v --html=/var/www/html/${baseVersion}/${version}/${verMode}/${mode}_linux_x64_report.html --json-report --json-report-file=report.json --timeout=300 --verMode=${verMode} --tVersion=${version} --baseVersion=${baseVersion} --sourcePath=${sourcePath} || true + cp report.json /var/www/html/${baseVersion}/${version}/${verMode}/json/${mode}_linux_x64_report.json + curl "http://192.168.0.176/api/addSmoke?version=${version}&tag=${baseVersion}&type=${verMode}&role=client&build=linux_x64" + ''' + } + } + } + stage('client_Linux_arm64') { + when { + beforeAgent true + allOf { + expression { mode == 'client' } + expression { runPlatforms.contains('client_Linux_arm64') } + } + } + agent{label " worker06_arm64 "} + steps { + timeout(time: 30, unit: 'MINUTES'){ + sync_source("${BRANCH_NAME}") + sh ''' + cd ${TDENGINE_ROOT_DIR}/packaging/smokeTest + bash getAndRunInstaller.sh -m ${verMode} -f client -l false -c arm64 -v ${version} -o ${baseVersion} -s ${sourcePath} -t tar + python3 -m pytest test_client.py -v --html=${mode}_linux_arm64_report.html --json-report --json-report-file=report.json --timeout=300 --verMode=${verMode} --tVersion=${version} --baseVersion=${baseVersion} --sourcePath=${sourcePath} || true + scp ${mode}_linux_arm64_report.html root@192.168.0.21:/var/www/html/${baseVersion}/${version}/${verMode}/ + scp report.json root@192.168.0.21:/var/www/html/${baseVersion}/${version}/${verMode}/json/${mode}_linux_arm64_report.json + curl "http://192.168.0.176/api/addSmoke?version=${version}&tag=${baseVersion}&type=${verMode}&role=client&build=linux_arm64" + ''' + } + } + } + stage ('client_Mac_x64') { + when { + beforeAgent true + allOf { + expression { mode == 'client' } + expression { runPlatforms.contains('client_Mac_x64') } + } + } + agent{label " release_Darwin_x64 "} + environment{ + WORK_DIR = "/Users/zwen/jenkins/workspace" + TDINTERNAL_ROOT_DIR = '/Users/zwen/jenkins/workspace/TDinternal' + TDENGINE_ROOT_DIR = '/Users/zwen/jenkins/workspace/TDinternal/community' + } + steps { + timeout(time: 30, unit: 'MINUTES'){ + sync_source("${BRANCH_NAME}") + sh ''' + cd ${TDENGINE_ROOT_DIR}/packaging/smokeTest + bash getAndRunInstaller.sh -m ${verMode} -f client -l false -c x64 -v ${version} -o ${baseVersion} -s ${sourcePath} -t pkg + rm -rf /opt/taos/main/TDinternal/debug/* || true + python3 -m pytest test_client.py -v --html=${mode}_Mac_x64_report.html --json-report --json-report-file=report.json --timeout=300 --verMode=${verMode} --tVersion=${version} --baseVersion=${baseVersion} --sourcePath=${sourcePath} || true + scp ${mode}_Mac_x64_report.html root@192.168.0.21:/var/www/html/${baseVersion}/${version}/${verMode}/ + scp report.json root@192.168.0.21:/var/www/html/${baseVersion}/${version}/${verMode}/json/${mode}_Mac_x64_report.json + curl "http://192.168.0.176/api/addSmoke?version=${version}&tag=${baseVersion}&type=${verMode}&role=client&build=Mac_x64" + ''' + } + } + } + stage ('client_Mac_arm64') { + when { + beforeAgent true + allOf { + expression { mode == 'client' } + expression { runPlatforms.contains('client_Mac_arm64') } + } + } + agent{label " release_Darwin_arm64 "} + environment{ + WORK_DIR = "/Users/zwen/jenkins/workspace" + TDINTERNAL_ROOT_DIR = '/Users/zwen/jenkins/workspace/TDinternal' + TDENGINE_ROOT_DIR = '/Users/zwen/jenkins/workspace/TDinternal/community' + } + steps { + timeout(time: 30, unit: 'MINUTES'){ + sync_source("${BRANCH_NAME}") + sh ''' + cd ${TDENGINE_ROOT_DIR}/packaging/smokeTest + bash getAndRunInstaller.sh -m ${verMode} -f client -l false -c arm64 -v ${version} -o ${baseVersion} -s ${sourcePath} -t pkg + rm -rf /opt/taos/main/TDinternal/debug/* || true + python3 -m pytest test_client.py -v --html=${mode}_Mac_arm64_report.html --json-report --json-report-file=report.json --timeout=300 --verMode=${verMode} --tVersion=${version} --baseVersion=${baseVersion} --sourcePath=${sourcePath} || true + scp ${mode}_Mac_arm64_report.html root@192.168.0.21:/var/www/html/${baseVersion}/${version}/${verMode}/ + scp report.json root@192.168.0.21:/var/www/html/${baseVersion}/${version}/${verMode}/json/${mode}_Mac_arm64_report.json + curl "http://192.168.0.176/api/addSmoke?version=${version}&tag=${baseVersion}&type=${verMode}&role=client&build=Mac_arm64" + ''' + } + } + } + stage('client_Windows_x64') { + when { + beforeAgent true + allOf { + expression { mode == 'client' } + expression { runPlatforms.contains('client_Windows_x64') } + } + } + agent{label " windows71 "} + environment{ + WIN_WORK_DIR="C:\\workspace" + WIN_TDINTERNAL_ROOT_DIR="C:\\workspace\\TDinternal" + WIN_TDENGINE_ROOT_DIR="C:\\workspace\\TDinternal\\community" + } + steps { + timeout(time: 30, unit: 'MINUTES'){ + sync_source_win() + bat ''' + cd %WIN_TDENGINE_ROOT_DIR%\\packaging\\smokeTest + call getAndRunInstaller.bat %baseVersion% %version% %verMode% client + pip3 install -r pytest_require.txt + python3 -m pytest test_client.py -v --html=%mode%_Windows_x64_report.html --json-report --json-report-file=report.json --timeout=300 --verMode=%verMode% --tVersion=%version% --baseVersion=%baseVersion% --sourcePath=%sourcePath% + scp %mode%_Windows_x64_report.html root@192.168.0.21:/var/www/html/%baseVersion%/%version%/%verMode%/ + scp report.json root@192.168.0.21:/var/www/html/%baseVersion%/%version%/%verMode%/json/%mode%_Windows_x64_report.json + curl "http://192.168.0.176/api/addSmoke?version=%version%&tag=%baseVersion%&type=%verMode%&role=client&build=Windows_x64" + ''' + } + } + } + } + } + } +} \ No newline at end of file diff --git a/packaging/smoke_test/start3NodesServer.sh b/packaging/smoke_test/start3NodesServer.sh new file mode 100644 index 000000000000..b446a467efc7 --- /dev/null +++ b/packaging/smoke_test/start3NodesServer.sh @@ -0,0 +1,67 @@ +#!/bin/bash +BUILD_ID=dontKillMe + +#******This script setup 3 nodes env for remote client installer test. Only for Linux ********* + +pwd=`pwd` +hostname=`hostname` +if [ -z $JENKINS_HOME ]; then + workdir="${pwd}/cluster" + echo $workdir +else + workdir="${JENKINS_HOME}/workspace/cluster" + echo $workdir +fi + +name="taos" +if command -v prodb ;then + name="prodb" +fi + +# Stop all taosd processes +for(( i=0; i<3; i++)) +do + pid=$(ps -ef | grep ${name}d | grep -v grep | awk '{print $2}') + if [ -n "$pid" ]; then + ${csudo}kill -9 $pid || : + fi +done + +# Init 3 dnodes workdir and config file +rm -rf ${workdir} +mkdir ${workdir} +mkdir ${workdir}/output +mkdir ${workdir}/dnode1 +mkdir ${workdir}/dnode1/data +mkdir ${workdir}/dnode1/log +mkdir ${workdir}/dnode1/cfg +touch ${workdir}/dnode1/cfg/${name}.cfg +echo -e "firstEp ${hostname}:6031\nsecondEp ${hostname}:6032\nfqdn ${hostname}\nserverPort 6031\nlogDir ${workdir}/dnode1/log\ndataDir ${workdir}/dnode1/data\n" >> ${workdir}/dnode1/cfg/${name}.cfg + +# Start first node +nohup ${name}d -c ${workdir}/dnode1/cfg/${name}.cfg & > /dev/null +sleep 5 + +${name} -P 6031 -s "CREATE DNODE \`${hostname}:6032\`;CREATE DNODE \`${hostname}:6033\`" + +mkdir ${workdir}/dnode2 +mkdir ${workdir}/dnode2/data +mkdir ${workdir}/dnode2/log +mkdir ${workdir}/dnode2/cfg +touch ${workdir}/dnode2/cfg/${name}.cfg +echo -e "firstEp ${hostname}:6031\nsecondEp ${hostname}:6032\nfqdn ${hostname}\nserverPort 6032\nlogDir ${workdir}/dnode2/log\ndataDir ${workdir}/dnode2/data\n" >> ${workdir}/dnode2/cfg/${name}.cfg + +nohup ${name}d -c ${workdir}/dnode2/cfg/${name}.cfg & > /dev/null +sleep 5 + +mkdir ${workdir}/dnode3 +mkdir ${workdir}/dnode3/data +mkdir ${workdir}/dnode3/log +mkdir ${workdir}/dnode3/cfg +touch ${workdir}/dnode3/cfg/${name}.cfg +echo -e "firstEp ${hostname}:6031\nsecondEp ${hostname}:6032\nfqdn ${hostname}\nserverPort 6033\nlogDir ${workdir}/dnode3/log\ndataDir ${workdir}/dnode3/data\n" >> ${workdir}/dnode3/cfg/${name}.cfg + +nohup ${name}d -c ${workdir}/dnode3/cfg/${name}.cfg & > /dev/null +sleep 5 + +${name} -P 6031 -s "CREATE MNODE ON DNODE 2;CREATE MNODE ON DNODE 3;" \ No newline at end of file diff --git a/packaging/smoke_test/test_client.py b/packaging/smoke_test/test_client.py new file mode 100644 index 000000000000..0b1003e3702e --- /dev/null +++ b/packaging/smoke_test/test_client.py @@ -0,0 +1,137 @@ +import pytest +import subprocess +import os +import sys +import platform +import getopt +import re +import time +import taos +from versionCheckAndUninstallforPytest import UninstallTaos + +# python3 smokeTestClient.py -h 192.168.0.22 -P 6031 -v ${version} -u + +OEM = ["ProDB"] + + +@pytest.fixture(scope="module") +def get_config(request): + verMode = request.config.getoption("--verMode") + taosVersion = request.config.getoption("--tVersion") + baseVersion = request.config.getoption("--baseVersion") + sourcePath = request.config.getoption("--sourcePath") + config = { + "verMode": verMode, + "taosVersion": taosVersion, + "baseVersion": baseVersion, + "sourcePath": sourcePath, + "system": platform.system(), + "arch": platform.machine(), + "serverHost": "192.168.0.22", + "serverPort": 6031, + "databaseName": re.sub(r'[^a-zA-Z0-9]', '', subprocess.getoutput("hostname")).lower() + } + return config + + +@pytest.fixture(scope="module") +def setup_module(get_config): + config = get_config + # install taospy + if config["system"] == 'Windows': + taospy_version = subprocess.getoutput("pip3 show taospy|findstr Version") + else: + taospy_version = subprocess.getoutput("pip3 show taospy|grep Version| awk -F ':' '{print $2}' ") + + print("taospy version %s " % taospy_version) + if taospy_version == "": + subprocess.getoutput("pip3 install git+https://github.com/taosdata/taos-connector-python.git") + print("install taos python connector") + else: + subprocess.getoutput("pip3 install taospy") + + +def get_connect(host, port, database=None): + conn = taos.connect(host=host, + user="root", + password="taosdata", + database=database, + port=port, + timezone="Asia/Shanghai") # default your host's timezone + return conn + + +def run_cmd(command): + print("CMD: %s" % command) + result = subprocess.run(command, capture_output=True, text=True, shell=True) + print("STDOUT:", result.stdout) + print("STDERR:", result.stderr) + print("Return Code:", result.returncode) + assert result.returncode == 0 + return result + + +class TestClient: + @pytest.mark.all + def test_basic(self, get_config, setup_module): + config = get_config + name = "taos" + + if config["baseVersion"] in OEM: + name = config["baseVersion"].lower() + if config["baseVersion"] in OEM and config["system"] == 'Windows': + cmd = f'{name} -s "create database {config["databaseName"]};" -h {config["serverHost"]} -P {config["serverPort"]}' + run_cmd(cmd) + cmd = f'{name} -s "CREATE STABLE {config["databaseName"]}.meters (`ts` TIMESTAMP,`current` FLOAT, `phase` FLOAT) TAGS (`groupid` INT, `location` VARCHAR(24));" -h {config["serverHost"]} -P {config["serverPort"]}' + run_cmd(cmd) + else: + cmd = f'{name}Benchmark -y -a 3 -n 100 -t 100 -d {config["databaseName"]} -h {config["serverHost"]} -P {config["serverPort"]} &' + run_cmd(cmd) + # os.system("taosBenchmark -y -a 3 -n 100 -t 100 -d %s -h %s -P %d" % (databaseName, serverHost, serverPort)) + time.sleep(5) + conn = get_connect(config["serverHost"], config["serverPort"], config["databaseName"]) + sql = "SELECT count(*) from meters" + result: taos.TaosResult = conn.query(sql) + data = result.fetch_all() + print("SQL: %s" % sql) + print("Result: %s" % data) + if config["system"] == 'Windows' and config["baseVersion"] in OEM: + pass + elif data[0][0] != 10000: + raise f"{name}Benchmark work not as expected " + # drop database of test + cmd = f'{name} -s "drop database {config["databaseName"]};" -h {config["serverHost"]} -P {config["serverPort"]}' + result = run_cmd(cmd) + assert "Drop OK" in result.stdout + conn.close() + + @pytest.mark.all + def test_version(self, get_config, setup_module): + config = get_config + conn = get_connect(config["serverHost"], config["serverPort"]) + server_version = conn.server_info + print("server_version: ", server_version) + client_version = conn.client_info + print("client_version: ", client_version) + name = "taos" + if config["baseVersion"] in OEM: + name = config["baseVersion"].lower() + if config["system"] == "Windows": + taos_V_output = subprocess.getoutput(f"{name} -V | findstr version") + else: + taos_V_output = subprocess.getoutput(f"{name} -V | grep version") + assert config["taosVersion"] in taos_V_output + assert config["taosVersion"] in client_version + if config["taosVersion"] not in server_version: + print("warning: client version is not same as server version") + conn.close() + + @pytest.mark.all + def test_uninstall(self, get_config, setup_module): + config = get_config + name = "taos" + if config["baseVersion"] in OEM: + name = config["baseVersion"].lower() + subprocess.getoutput("rm /usr/local/bin/taos") + subprocess.getoutput("pkill taosd") + UninstallTaos(config["taosVersion"], config["verMode"], True, name) diff --git a/packaging/smoke_test/test_server.py b/packaging/smoke_test/test_server.py new file mode 100644 index 000000000000..768cd2a96214 --- /dev/null +++ b/packaging/smoke_test/test_server.py @@ -0,0 +1,238 @@ +import pytest +import subprocess +import os +from versionCheckAndUninstallforPytest import UninstallTaos +import platform +import re +import time +import signal + +system = platform.system() +current_path = os.path.abspath(os.path.dirname(__file__)) +if system == 'Windows': + with open(r"%s\test_server_windows_case" % current_path) as f: + cases = f.read().splitlines() +else: + with open("%s/test_server_unix_case" % current_path) as f: + cases = f.read().splitlines() + +OEM = ["ProDB"] + + +@pytest.fixture(scope="module") +def get_config(request): + verMode = request.config.getoption("--verMode") + taosVersion = request.config.getoption("--tVersion") + baseVersion = request.config.getoption("--baseVersion") + sourcePath = request.config.getoption("--sourcePath") + config = { + "verMode": verMode, + "taosVersion": taosVersion, + "baseVersion": baseVersion, + "sourcePath": sourcePath, + "system": platform.system(), + "arch": platform.machine() + } + return config + + +@pytest.fixture(scope="module") +def setup_module(get_config): + def run_cmd(command): + print("CMD:", command) + result = subprocess.run(command, capture_output=True, text=False, shell=False) + print("STDOUT:", result.stdout) + print("STDERR:", result.stderr) + print("Return Code:", result.returncode) + assert result.returncode == 0 + return result + + # setup before module tests + config = get_config + # bash getAndRunInstaller.sh -m ${verMode} -f server -l false -c x64 -v ${version} -o ${baseVersion} -s ${sourcePath} -t tar + # t = "tar" + # if config["system"] == "Darwin": + # t = "pkg" + # cmd = "bash getAndRunInstaller.sh -m %s -f server -l false -c x64 -v %s -o %s -s %s -t %s" % ( + # config["verMode"], config["taosVersion"], config["baseVersion"], config["sourcePath"], t) + # run_cmd(cmd) + if config["system"] == "Windows": + cmd = r"mkdir ..\..\debug\build\bin" + else: + cmd = "mkdir -p ../../debug/build/bin/" + subprocess.getoutput(cmd) + if config["system"] == "Linux": # add tmq_sim + cmd = "cp -rf ../../../debug/build/bin/tmq_sim ../../debug/build/bin/." + subprocess.getoutput(cmd) + if config["system"] == "Darwin": + cmd = "sudo cp -rf /usr/local/bin/taos* ../../debug/build/bin/" + elif config["system"] == "Windows": + cmd = r"xcopy C:\TDengine\taos*.exe ..\..\debug\build\bin /Y" + else: + if config["baseVersion"] in OEM: + cmd = '''sudo find /usr/bin -name 'prodb*' -exec sh -c 'for file; do cp "$file" "../../debug/build/bin/taos${file##/usr/bin/%s}"; done' sh {} +''' % ( + config["baseVersion"].lower()) + else: + cmd = "sudo cp /usr/bin/taos* ../../debug/build/bin/" + run_cmd(cmd) + if config["baseVersion"] in OEM: # mock OEM + cmd = "sed -i 's/taos.cfg/%s.cfg/g' ../../tests/pytest/util/dnodes.py" % config["baseVersion"].lower() + run_cmd(cmd) + cmd = "sed -i 's/taosdlog.0/%sdlog.0/g' ../../tests/pytest/util/dnodes.py" % config["baseVersion"].lower() + run_cmd(cmd) + cmd = "sed -i 's/taos.cfg/%s.cfg/g' ../../tests/army/frame/server/dnode.py" % config["baseVersion"].lower() + run_cmd(cmd) + cmd = "sed -i 's/taosdlog.0/%sdlog.0/g' ../../tests/army/frame/server/dnode.py" % config["baseVersion"].lower() + run_cmd(cmd) + cmd = "ln -s /usr/bin/prodb /usr/local/bin/taos" + subprocess.getoutput(cmd) + + # yield + # + # name = "taos" + # if config["baseVersion"] in OEM: + # name = config["baseVersion"].lower() + # subprocess.getoutput("rm /usr/local/bin/taos") + # subprocess.getoutput("pkill taosd") + # UninstallTaos(config["taosVersion"], config["verMode"], True, name) + + +# use pytest fixture to exec case +@pytest.fixture(params=cases) +def run_command(request): + commands = request.param + if commands.strip().startswith("#"): + pytest.skip("This case has been marked as skipped") + d, command = commands.strip().split(",") + if system == "Windows": + cmd = r"cd %s\..\..\tests\%s && %s" % (current_path, d, command) + else: + cmd = "cd %s/../../tests/%s&&sudo %s" % (current_path, d, command) + print(cmd) + result = subprocess.run(cmd, capture_output=True, text=True, shell=True) + return { + "command": command, + "stdout": result.stdout, + "stderr": result.stderr, + "returncode": result.returncode + } + + +class TestServer: + @pytest.mark.all + def test_taosd_up(self, setup_module): + # start process + if system == 'Windows': + subprocess.getoutput("taskkill /IM taosd.exe /F") + cmd = "..\\..\\debug\\build\\bin\\taosd.exe" + else: + subprocess.getoutput("pkill taosd") + cmd = "../../debug/build/bin/taosd" + process = subprocess.Popen( + [cmd], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True + ) + # monitor output + while True: + line = process.stdout.readline() + if line: + print(line.strip()) + if "succeed to write dnode" in line: + time.sleep(15) + # 发送终止信号 + os.kill(process.pid, signal.SIGTERM) + break + + @pytest.mark.all + def test_execute_cases(self, setup_module, run_command): + # assert the result + if run_command['returncode'] != 0: + print(f"Running command: {run_command['command']}") + print("STDOUT:", run_command['stdout']) + print("STDERR:", run_command['stderr']) + print("Return Code:", run_command['returncode']) + else: + print(f"Running command: {run_command['command']}") + if len(run_command['stdout']) > 1000: + print("STDOUT:", run_command['stdout'][:1000] + "...") + else: + print("STDOUT:", run_command['stdout']) + print("STDERR:", run_command['stderr']) + print("Return Code:", run_command['returncode']) + + assert run_command[ + 'returncode'] == 0, f"Command '{run_command['command']}' failed with return code {run_command['returncode']}" + + @pytest.mark.all + @pytest.mark.check_version + def test_check_version(self, get_config, setup_module): + config = get_config + databaseName = re.sub(r'[^a-zA-Z0-9]', '', subprocess.getoutput("hostname")).lower() + # install taospy + taospy_version = "" + system = config["system"] + version = config["taosVersion"] + verMode = config["verMode"] + if system == 'Windows': + taospy_version = subprocess.getoutput("pip3 show taospy|findstr Version") + else: + taospy_version = subprocess.getoutput("pip3 show taospy|grep Version| awk -F ':' '{print $2}' ") + + print("taospy version %s " % taospy_version) + if taospy_version == "": + subprocess.getoutput("pip3 install git+https://github.com/taosdata/taos-connector-python.git") + print("install taos python connector") + else: + subprocess.getoutput("pip3 install taospy") + + # start taosd server + if system == 'Windows': + cmd = ["C:\\TDengine\\start-all.bat"] + # elif system == 'Linux': + # cmd = "systemctl start taosd".split(' ') + else: + # cmd = "sudo launchctl start com.tdengine.taosd".split(' ') + cmd = "start-all.sh" + process_out = subprocess.Popen(cmd, + stdin=subprocess.PIPE, stdout=subprocess.PIPE, text=True) + print(cmd) + time.sleep(5) + + import taos + conn = taos.connect() + check_list = {} + check_list["server_version"] = conn.server_info + check_list["client_version"] = conn.client_info + # Execute sql get version info + result: taos.TaosResult = conn.query("SELECT server_version()") + check_list["select_server"] = result.fetch_all()[0][0] + result: taos.TaosResult = conn.query("SELECT client_version()") + check_list["select_client"] = result.fetch_all()[0][0] + conn.close() + + binary_files = ["taos", "taosd", "taosadapter", "taoskeeper", "taosBenchmark"] + if verMode.lower() == "enterprise": + binary_files.append("taosx") + if config["baseVersion"] in OEM: + binary_files = [i.replace("taos", config["baseVersion"].lower()) for i in binary_files] + if system == "Windows": + for i in binary_files: + check_list[i] = subprocess.getoutput("%s -V | findstr version" % i) + else: + for i in binary_files: + check_list[i] = subprocess.getoutput("%s -V | grep version | awk -F ' ' '{print $3}'" % i) + for i in check_list: + print("%s version is: %s" % (i, check_list[i])) + assert version in check_list[i] + + @pytest.mark.all + def test_uninstall(self, get_config, setup_module): + config = get_config + name = "taos" + if config["baseVersion"] in OEM: + name = config["baseVersion"].lower() + subprocess.getoutput("rm /usr/local/bin/taos") + subprocess.getoutput("pkill taosd") + UninstallTaos(config["taosVersion"], config["verMode"], True, name) diff --git a/packaging/smoke_test/test_server_unix_case b/packaging/smoke_test/test_server_unix_case new file mode 100644 index 000000000000..1bbde1093265 --- /dev/null +++ b/packaging/smoke_test/test_server_unix_case @@ -0,0 +1,10 @@ +system-test,python3 ./test.py -f 2-query/join.py +system-test,python3 ./test.py -f 1-insert/insert_column_value.py +system-test,python3 ./test.py -f 2-query/primary_ts_base_5.py +system-test,python3 ./test.py -f 2-query/case_when.py +system-test,python3 ./test.py -f 2-query/partition_limit_interval.py +system-test,python3 ./test.py -f 2-query/fill.py +army,python3 ./test.py -f query/query_basic.py -N 3 +system-test,python3 ./test.py -f 7-tmq/basic5.py +system-test,python3 ./test.py -f 8-stream/stream_basic.py +system-test,python3 ./test.py -f 6-cluster/5dnode3mnodeStop.py -N 5 -M 3 \ No newline at end of file diff --git a/packaging/smoke_test/test_server_windows_case b/packaging/smoke_test/test_server_windows_case new file mode 100644 index 000000000000..e64213b1eef7 --- /dev/null +++ b/packaging/smoke_test/test_server_windows_case @@ -0,0 +1,2 @@ +system-test,python3 .\test.py -f 0-others\taosShell.py +system-test,python3 .\test.py -f 6-cluster\5dnode3mnodeSep1VnodeStopDnodeModifyMeta.py -N 6 -M 3 \ No newline at end of file diff --git a/packaging/smoke_test/versionCheckAndUninstall.py b/packaging/smoke_test/versionCheckAndUninstall.py new file mode 100644 index 000000000000..80dea9a15fb0 --- /dev/null +++ b/packaging/smoke_test/versionCheckAndUninstall.py @@ -0,0 +1,260 @@ +#!/usr/bin/python +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### +# install pip +# pip install src/connector/python/ + +# -*- coding: utf-8 -*- +import sys, os +import re +import platform +import getopt +import subprocess +# from this import d +import time + +# input for server + +opts, args = getopt.gnu_getopt(sys.argv[1:], 'v:m:u', ['version=', 'verMode=']) +serverHost = "" +serverPort = 0 +version = "" +uninstall = False +verMode = "" +for key, value in opts: + if key in ['--help']: + print('A collection of test cases written using Python') + print('-v test client version') + print('-u test uninstall process, will uninstall TDengine') + sys.exit(0) + + if key in ['-v']: + version = value + if key in ['-u']: + uninstall = True + if key in ['-m']: + verMode = value +if not version: + print("No version specified, will not run version check.") + + +system = platform.system() +arch = platform.machine() + +databaseName = re.sub(r'[^a-zA-Z0-9]', '', subprocess.getoutput("hostname")).lower() +# install taospy +taospy_version = "" +if system == 'Windows': + taospy_version = subprocess.getoutput("pip3 show taospy|findstr Version") +else: + taospy_version = subprocess.getoutput("pip3 show taospy|grep Version| awk -F ':' '{print $2}' ") + +print("taospy version %s " % taospy_version) +if taospy_version == "": + subprocess.getoutput("pip3 install git+https://github.com/taosdata/taos-connector-python.git") + print("install taos python connector") +else: + subprocess.getoutput("pip3 install taospy") + +# start taosd server +if system == 'Windows': + cmd = ["C:\\TDengine\\start-all.bat"] +elif system == 'Linux': + cmd = "systemctl start taosd".split(' ') +else: + cmd = "sudo launchctl start com.tdengine.taosd".split(' ') +process_out = subprocess.Popen(cmd, + stdin=subprocess.PIPE, stdout=subprocess.PIPE, text=True) +print(cmd) +time.sleep(5) + +#get taosc version info +version_test_result = False +if version: + import taos + conn = taos.connect() + server_version = conn.server_info + print("server_version", server_version) + client_version = conn.client_info + print("client_version", client_version) + # Execute sql get version info + result: taos.TaosResult = conn.query("SELECT server_version()") + select_server = result.fetch_all()[0][0] + print("SELECT server_version():" + select_server) + result: taos.TaosResult = conn.query("SELECT client_version()") + select_client = result.fetch_all()[0][0] + print("SELECT client_version():" + select_client) + conn.close() + + taos_V_output = "" + taosd_V_output = "" + taosadapter_V_output = "" + taoskeeper_V_output = "" + taosx_V_output = "" + taosB_V_output = "" + taosxVersion = False + if system == "Windows": + taos_V_output = subprocess.getoutput("taos -V | findstr version") + taosd_V_output = subprocess.getoutput("taosd -V | findstr version") + taosadapter_V_output = subprocess.getoutput("taosadapter -V | findstr version") + taoskeeper_V_output = subprocess.getoutput("taoskeeper -V | findstr version") + taosB_V_output = subprocess.getoutput("taosBenchmark -V | findstr version") + if verMode == "Enterprise": + taosx_V_output = subprocess.getoutput("taosx -V | findstr version") + else: + taos_V_output = subprocess.getoutput("taos -V | grep version | awk -F ' ' '{print $3}'") + taosd_V_output = subprocess.getoutput("taosd -V | grep version | awk -F ' ' '{print $3}'") + taosadapter_V_output = subprocess.getoutput("taosadapter -V | grep version | awk -F ' ' '{print $3}'") + taoskeeper_V_output = subprocess.getoutput("taoskeeper -V | grep version | awk -F ' ' '{print $3}'") + taosB_V_output = subprocess.getoutput("taosBenchmark -V | grep version | awk -F ' ' '{print $3}'") + if verMode == "Enterprise": + taosx_V_output = subprocess.getoutput("taosx -V | grep version | awk -F ' ' '{print $3}'") + + print("taos -V output is: %s" % taos_V_output) + print("taosd -V output is: %s" % taosd_V_output) + print("taosadapter -V output is: %s" % taosadapter_V_output) + print("taoskeeper -V output is: %s" % taoskeeper_V_output) + print("taosBenchmark -V output is: %s" % taosB_V_output) + if verMode == "Enterprise": + print("taosx -V output is: %s" % taosx_V_output) + taosxVersion = version in taosx_V_output + else: + taosxVersion = True + if (version in client_version + and version in server_version + and version in select_server + and version in select_client + and version in taos_V_output + and version in taosd_V_output + and version in taosadapter_V_output + and version in taoskeeper_V_output + and version in taosB_V_output + and taosxVersion + ): + version_test_result = True +leftFile = False +if uninstall: + print("Start to run rmtaos") + print("Platform: ", system) + # stop taosd server + if system == 'Windows': + cmd = "C:\\TDengine\\stop_all.bat" + elif system == 'Linux': + cmd = "systemctl stop taosd" + else: + cmd = "sudo launchctl stop com.tdengine.taosd" + process_out = subprocess.getoutput(cmd) + print(cmd) + time.sleep(10) + if system == "Linux": + # 创建一个subprocess.Popen对象，并使用stdin和stdout进行交互 + process = subprocess.Popen(['rmtaos'], + stdin=subprocess.PIPE, stdout=subprocess.PIPE, text=True) + # 向子进程发送输入 + process.stdin.write("y\n") + process.stdin.flush() # 确保输入被发送到子进程 + process.stdin.write("I confirm that I would like to delete all data, log and configuration files\n") + process.stdin.flush() # 确保输入被发送到子进程 + # 关闭子进程的stdin，防止它无限期等待更多输入 + process.stdin.close() + # 等待子进程结束 + process.wait() + # 检查目录清除情况 + out = subprocess.getoutput("ls /etc/systemd/system/taos*") + if "No such file or directory" not in out: + print("Uninstall left some files: %s" % out) + leftFile = True + out = subprocess.getoutput("ls /usr/bin/taos*") + if "No such file or directory" not in out: + print("Uninstall left some files: %s" % out) + leftFile = True + out = subprocess.getoutput("ls /usr/local/bin/taos*") + if "No such file or directory" not in out: + print("Uninstall left some files: %s" % out) + leftFile = True + out = subprocess.getoutput("ls /usr/lib/libtaos*") + if "No such file or directory" not in out: + print("Uninstall left some files: %s" % out) + leftFile = True + out = subprocess.getoutput("ls /usr/lib64/libtaos*") + if "No such file or directory" not in out: + print("Uninstall left some files: %s" % out) + leftFile = True + out = subprocess.getoutput("ls /usr/include/taos*") + if "No such file or directory" not in out: + print("Uninstall left some files: %s" % out) + leftFile = True + out = subprocess.getoutput("ls /usr/local/taos") + #print(out) + if "No such file or directory" not in out: + print("Uninstall left some files in /usr/local/taos：%s" % out) + leftFile = True + if not leftFile: + print("*******Test Result: uninstall test passed ************") + + elif system == "Darwin": + # 创建一个subprocess.Popen对象，并使用stdin和stdout进行交互 + process = subprocess.Popen(['sudo', 'rmtaos'], + stdin=subprocess.PIPE, stdout=subprocess.PIPE, text=True) + # 向子进程发送输入 + process.stdin.write("y\n") + process.stdin.flush() # 确保输入被发送到子进程 + process.stdin.write("I confirm that I would like to delete all data, log and configuration files\n") + process.stdin.flush() # 确保输入被发送到子进程 + # 关闭子进程的stdin，防止它无限期等待更多输入 + process.stdin.close() + # 等待子进程结束 + process.wait() + # 检查目录清除情况 + out = subprocess.getoutput("ls /usr/local/bin/taos*") + if "No such file or directory" not in out: + print("Uninstall left some files: %s" % out) + leftFile = True + out = subprocess.getoutput("ls /usr/local/lib/libtaos*") + if "No such file or directory" not in out: + print("Uninstall left some files: %s" % out) + leftFile = True + out = subprocess.getoutput("ls /usr/local/include/taos*") + if "No such file or directory" not in out: + print("Uninstall left some files: %s" % out) + leftFile = True + #out = subprocess.getoutput("ls /usr/local/Cellar/tdengine/") + #print(out) + #if out: + # print("Uninstall left some files: /usr/local/Cellar/tdengine/%s" % out) + # leftFile = True + #if not leftFile: + # print("*******Test Result: uninstall test passed ************") + + elif system == "Windows": + process = subprocess.Popen(['unins000','/silent'], + stdin=subprocess.PIPE, stdout=subprocess.PIPE, text=True) + process.wait() + time.sleep(10) + out = subprocess.getoutput("ls C:\TDengine") + print(out) + if len(out.split("\n")) > 3: + leftFile = True + print("Uninstall left some files: %s" % out) + +if version_test_result: + print("**********Test Result: version test passed! **********") +else: + print("!!!!!!!!!!!Test Result: version test failed! !!!!!!!!!!") +if not leftFile: + print("**********Test Result: uninstall test passed! **********") +else: + print("!!!!!!!!!!!Test Result: uninstall test failed! !!!!!!!!!!") +if version_test_result and not leftFile: + sys.exit(0) +else: + sys.exit(1) + diff --git a/packaging/smoke_test/versionCheckAndUninstallforPytest.py b/packaging/smoke_test/versionCheckAndUninstallforPytest.py new file mode 100644 index 000000000000..5b752195545d --- /dev/null +++ b/packaging/smoke_test/versionCheckAndUninstallforPytest.py @@ -0,0 +1,137 @@ +#!/usr/bin/python +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### +# install pip +# pip install src/connector/python/ + +# -*- coding: utf-8 -*- +import sys, os +import re +import platform +import getopt +import subprocess +# from this import d +import time +from lib import run_cmd + + +# input for server +def UninstallTaos(version, verMode, uninstall, name): + if not version: + raise "No version specified, will not run version check." + + system = platform.system() + arch = platform.machine() + leftFile = False + if uninstall: + print("Start to run rm%s" % name) + print("Platform: ", system) + # stop taosd server + if system == 'Windows': + cmd = "C:\\TDengine\\stop_all.bat" + else: + cmd = "stop_all.sh" + process_out = subprocess.getoutput(cmd) + print(cmd) + time.sleep(5) + print("start to rm%s" % name) + if system == "Linux": + # 启动命令 + process = subprocess.Popen(['rm%s' % name], stdin=subprocess.PIPE, stdout=subprocess.PIPE, + stderr=subprocess.PIPE, text=True) + + # 发送交互输入 + stdout, stderr = process.communicate( + input="y\nI confirm that I would like to delete all data, log and configuration files\n") + + # 打印输出（可选） + print(stdout) + print(stderr) + # 检查目录清除情况 + out = subprocess.getoutput("ls /etc/systemd/system/%s*" % name) + if "No such file or directory" not in out: + print("Uninstall left some files: %s" % out) + leftFile = True + out = subprocess.getoutput("ls /usr/bin/%s*" % name) + if "No such file or directory" not in out: + print("Uninstall left some files: %s" % out) + leftFile = True + out = subprocess.getoutput("ls /usr/local/bin/%s*" % name) + if "No such file or directory" not in out: + print("Uninstall left some files: %s" % out) + leftFile = True + out = subprocess.getoutput("ls /usr/lib/lib%s*" % name) + if "No such file or directory" not in out: + print("Uninstall left some files: %s" % out) + leftFile = True + out = subprocess.getoutput("ls /usr/lib64/lib%s*" % name) + if "No such file or directory" not in out: + print("Uninstall left some files: %s" % out) + leftFile = True + out = subprocess.getoutput("ls /usr/include/%s*" % name) + if "No such file or directory" not in out: + print("Uninstall left some files: %s" % out) + leftFile = True + out = subprocess.getoutput("ls /usr/local/%s" % name) + # print(out) + if "No such file or directory" not in out: + print("Uninstall left some files in /usr/local/%s：%s" % (name, out)) + leftFile = True + if not leftFile: + print("*******Test Result: uninstall test passed ************") + + elif system == "Darwin": + # 创建一个subprocess.Popen对象，并使用stdin和stdout进行交互 + process = subprocess.Popen(['sudo', 'rm%s' % name], + stdin=subprocess.PIPE, stdout=subprocess.PIPE, text=True) + # 向子进程发送输入 + process.stdin.write("y\n") + process.stdin.flush() # 确保输入被发送到子进程 + process.stdin.write("I confirm that I would like to delete all data, log and configuration files\n") + process.stdin.flush() # 确保输入被发送到子进程 + # 关闭子进程的stdin，防止它无限期等待更多输入 + process.stdin.close() + # 等待子进程结束 + process.wait() + # 检查目录清除情况 + out = subprocess.getoutput("ls /usr/local/bin/%s*" % name) + if "No such file or directory" not in out: + print("Uninstall left some files: %s" % out) + leftFile = True + out = subprocess.getoutput("ls /usr/local/lib/lib%s*" % name) + if "No such file or directory" not in out: + print("Uninstall left some files: %s" % out) + leftFile = True + out = subprocess.getoutput("ls /usr/local/include/%s*" % name) + if "No such file or directory" not in out: + print("Uninstall left some files: %s" % out) + leftFile = True + # out = subprocess.getoutput("ls /usr/local/Cellar/tdengine/") + # print(out) + # if out: + # print("Uninstall left some files: /usr/local/Cellar/tdengine/%s" % out) + # leftFile = True + # if not leftFile: + # print("*******Test Result: uninstall test passed ************") + + elif system == "Windows": + process = subprocess.Popen(['unins000', '/silent'], + stdin=subprocess.PIPE, stdout=subprocess.PIPE, text=True) + process.wait() + time.sleep(10) + for file in ["C:\TDengine\\taos.exe", "C:\TDengine\\unins000.exe", "C:\ProDB\prodb.exe", + "C:\ProDB\\unins000.exe"]: + if os.path.exists(file): + leftFile = True + if leftFile: + raise "uninstall %s fail, please check" % name + else: + print("**********Test Result: uninstall test passed! **********") diff --git a/tests/system-test/test.py b/tests/system-test/test.py index fb3357a2b9b5..0d40544be8e2 100644 --- a/tests/system-test/test.py +++ b/tests/system-test/test.py @@ -689,6 +689,9 @@ def runOnPreviousCluster(host, config, fileName): if conn is not None: conn.close() if asan: - #tdDnodes.StopAllSigint() + # tdDnodes.StopAllSigint() tdLog.info("Address sanitizer mode finished") + else: + tdDnodes.stopAll() + tdLog.info("stop all td process finished") sys.exit(0) From 81d321a62f7417048f5f90fc1f6fc7ade726de89 Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Thu, 7 Nov 2024 11:36:59 +0800 Subject: [PATCH 08/28] test:modify somketest dir name --- packaging/{smoke_test => smokeTest}/conftest.py | 0 .../debRpmAutoInstall.sh | 0 .../getAndRunInstaller.bat | 0 .../getAndRunInstaller.sh | 0 packaging/{smoke_test => smokeTest}/lib.py | 0 packaging/{smoke_test => smokeTest}/main.py | 0 packaging/{smoke_test => smokeTest}/runCases.bat | 0 packaging/{smoke_test => smokeTest}/runCases.sh | 0 .../{smoke_test => smokeTest}/smokeTestClient.py | 0 .../smokeTestJenkinsFile | 0 .../start3NodesServer.sh | 0 .../{smoke_test => smokeTest}/test_client.py | 0 .../{smoke_test => smokeTest}/test_server.py | 0 .../test_server_unix_case | 0 .../test_server_windows_case | 0 .../versionCheckAndUninstall.py | 0 .../versionCheckAndUninstallforPytest.py | 0 packaging/smoke_test/pytest_require.txt | 16 ---------------- 18 files changed, 16 deletions(-) rename packaging/{smoke_test => smokeTest}/conftest.py (100%) rename packaging/{smoke_test => smokeTest}/debRpmAutoInstall.sh (100%) rename packaging/{smoke_test => smokeTest}/getAndRunInstaller.bat (100%) rename packaging/{smoke_test => smokeTest}/getAndRunInstaller.sh (100%) rename packaging/{smoke_test => smokeTest}/lib.py (100%) rename packaging/{smoke_test => smokeTest}/main.py (100%) rename packaging/{smoke_test => smokeTest}/runCases.bat (100%) rename packaging/{smoke_test => smokeTest}/runCases.sh (100%) rename packaging/{smoke_test => smokeTest}/smokeTestClient.py (100%) rename packaging/{smoke_test => smokeTest}/smokeTestJenkinsFile (100%) rename packaging/{smoke_test => smokeTest}/start3NodesServer.sh (100%) rename packaging/{smoke_test => smokeTest}/test_client.py (100%) rename packaging/{smoke_test => smokeTest}/test_server.py (100%) rename packaging/{smoke_test => smokeTest}/test_server_unix_case (100%) rename packaging/{smoke_test => smokeTest}/test_server_windows_case (100%) rename packaging/{smoke_test => smokeTest}/versionCheckAndUninstall.py (100%) rename packaging/{smoke_test => smokeTest}/versionCheckAndUninstallforPytest.py (100%) delete mode 100644 packaging/smoke_test/pytest_require.txt diff --git a/packaging/smoke_test/conftest.py b/packaging/smokeTest/conftest.py similarity index 100% rename from packaging/smoke_test/conftest.py rename to packaging/smokeTest/conftest.py diff --git a/packaging/smoke_test/debRpmAutoInstall.sh b/packaging/smokeTest/debRpmAutoInstall.sh similarity index 100% rename from packaging/smoke_test/debRpmAutoInstall.sh rename to packaging/smokeTest/debRpmAutoInstall.sh diff --git a/packaging/smoke_test/getAndRunInstaller.bat b/packaging/smokeTest/getAndRunInstaller.bat similarity index 100% rename from packaging/smoke_test/getAndRunInstaller.bat rename to packaging/smokeTest/getAndRunInstaller.bat diff --git a/packaging/smoke_test/getAndRunInstaller.sh b/packaging/smokeTest/getAndRunInstaller.sh similarity index 100% rename from packaging/smoke_test/getAndRunInstaller.sh rename to packaging/smokeTest/getAndRunInstaller.sh diff --git a/packaging/smoke_test/lib.py b/packaging/smokeTest/lib.py similarity index 100% rename from packaging/smoke_test/lib.py rename to packaging/smokeTest/lib.py diff --git a/packaging/smoke_test/main.py b/packaging/smokeTest/main.py similarity index 100% rename from packaging/smoke_test/main.py rename to packaging/smokeTest/main.py diff --git a/packaging/smoke_test/runCases.bat b/packaging/smokeTest/runCases.bat similarity index 100% rename from packaging/smoke_test/runCases.bat rename to packaging/smokeTest/runCases.bat diff --git a/packaging/smoke_test/runCases.sh b/packaging/smokeTest/runCases.sh similarity index 100% rename from packaging/smoke_test/runCases.sh rename to packaging/smokeTest/runCases.sh diff --git a/packaging/smoke_test/smokeTestClient.py b/packaging/smokeTest/smokeTestClient.py similarity index 100% rename from packaging/smoke_test/smokeTestClient.py rename to packaging/smokeTest/smokeTestClient.py diff --git a/packaging/smoke_test/smokeTestJenkinsFile b/packaging/smokeTest/smokeTestJenkinsFile similarity index 100% rename from packaging/smoke_test/smokeTestJenkinsFile rename to packaging/smokeTest/smokeTestJenkinsFile diff --git a/packaging/smoke_test/start3NodesServer.sh b/packaging/smokeTest/start3NodesServer.sh similarity index 100% rename from packaging/smoke_test/start3NodesServer.sh rename to packaging/smokeTest/start3NodesServer.sh diff --git a/packaging/smoke_test/test_client.py b/packaging/smokeTest/test_client.py similarity index 100% rename from packaging/smoke_test/test_client.py rename to packaging/smokeTest/test_client.py diff --git a/packaging/smoke_test/test_server.py b/packaging/smokeTest/test_server.py similarity index 100% rename from packaging/smoke_test/test_server.py rename to packaging/smokeTest/test_server.py diff --git a/packaging/smoke_test/test_server_unix_case b/packaging/smokeTest/test_server_unix_case similarity index 100% rename from packaging/smoke_test/test_server_unix_case rename to packaging/smokeTest/test_server_unix_case diff --git a/packaging/smoke_test/test_server_windows_case b/packaging/smokeTest/test_server_windows_case similarity index 100% rename from packaging/smoke_test/test_server_windows_case rename to packaging/smokeTest/test_server_windows_case diff --git a/packaging/smoke_test/versionCheckAndUninstall.py b/packaging/smokeTest/versionCheckAndUninstall.py similarity index 100% rename from packaging/smoke_test/versionCheckAndUninstall.py rename to packaging/smokeTest/versionCheckAndUninstall.py diff --git a/packaging/smoke_test/versionCheckAndUninstallforPytest.py b/packaging/smokeTest/versionCheckAndUninstallforPytest.py similarity index 100% rename from packaging/smoke_test/versionCheckAndUninstallforPytest.py rename to packaging/smokeTest/versionCheckAndUninstallforPytest.py diff --git a/packaging/smoke_test/pytest_require.txt b/packaging/smoke_test/pytest_require.txt deleted file mode 100644 index eadbe92470dc..000000000000 --- a/packaging/smoke_test/pytest_require.txt +++ /dev/null @@ -1,16 +0,0 @@ -pytest-html -pytest-json-report -pytest-timeout -taospy -numpy -fabric2 -psutil -pandas -toml -distro -requests -pexpect -faker -pyopenssl -taos-ws-py -taospy \ No newline at end of file From bc095b5409d5875c617780efde62c07cf9daa599 Mon Sep 17 00:00:00 2001 From: Bo Xiao Date: Thu, 7 Nov 2024 14:45:20 +0800 Subject: [PATCH 09/28] Update 02-database.md Add alter_database_option: MINROWS --- docs/zh/14-reference/03-taos-sql/02-database.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/zh/14-reference/03-taos-sql/02-database.md b/docs/zh/14-reference/03-taos-sql/02-database.md index 24eca979522e..91b39976a1a4 100644 --- a/docs/zh/14-reference/03-taos-sql/02-database.md +++ b/docs/zh/14-reference/03-taos-sql/02-database.md @@ -122,6 +122,7 @@ alter_database_option: { | KEEP value | WAL_RETENTION_PERIOD value | WAL_RETENTION_SIZE value + | MINROWS value } ``` From 9473668dd4d7597ff22c506749c7dfefd1e5882a Mon Sep 17 00:00:00 2001 From: haoranchen Date: Thu, 7 Nov 2024 15:46:57 +0800 Subject: [PATCH 10/28] Update test_server.py --- packaging/smokeTest/test_server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packaging/smokeTest/test_server.py b/packaging/smokeTest/test_server.py index 768cd2a96214..36d86357a388 100644 --- a/packaging/smokeTest/test_server.py +++ b/packaging/smokeTest/test_server.py @@ -40,7 +40,7 @@ def get_config(request): def setup_module(get_config): def run_cmd(command): print("CMD:", command) - result = subprocess.run(command, capture_output=True, text=False, shell=False) + result = subprocess.run(command, capture_output=True, text=True, shell=True) print("STDOUT:", result.stdout) print("STDERR:", result.stderr) print("Return Code:", result.returncode) From fa62b95b1fb46b827995c5bc9528c8ccacd296ec Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Thu, 7 Nov 2024 16:20:33 +0800 Subject: [PATCH 11/28] test: add smoke test scripts --- packaging/delete_ref_lock.py | 59 +++++ packaging/smoke_test/assets/style.css | 319 ++++++++++++++++++++++++ packaging/smoke_test/pytest_require.txt | 16 ++ 3 files changed, 394 insertions(+) create mode 100644 packaging/delete_ref_lock.py create mode 100644 packaging/smoke_test/assets/style.css create mode 100644 packaging/smoke_test/pytest_require.txt diff --git a/packaging/delete_ref_lock.py b/packaging/delete_ref_lock.py new file mode 100644 index 000000000000..cf0e4cdd058a --- /dev/null +++ b/packaging/delete_ref_lock.py @@ -0,0 +1,59 @@ +import subprocess +import re + +# 执行 git fetch 命令并捕获输出 +def git_fetch(): + result = subprocess.run(['git', 'fetch'], capture_output=True, text=True) + return result + +# 解析分支名称 +def parse_branch_name_type1(error_output): + # 使用正则表达式匹配 'is at' 前的分支名称 + match = re.search(r"error: cannot lock ref '(refs/remotes/origin/[^']+)': is at", error_output) + if match: + return match.group(1) + return None + +# 解析第二种错误中的分支名称 +def parse_branch_name_type2(error_output): + # 使用正则表达式匹配 'exists' 前的第一个引号内的分支名称 + match = re.search(r"'(refs/remotes/origin/[^']+)' exists;", error_output) + if match: + return match.group(1) + return None + +# 执行 git update-ref -d 命令 +def git_update_ref(branch_name): + if branch_name: + subprocess.run(['git', 'update-ref', '-d', f'{branch_name}'], check=True) + +# 解析错误类型并执行相应的修复操作 +def handle_error(error_output): + # 错误类型1：本地引用的提交ID与远程不一致 + if "is at" in error_output and "but expected" in error_output: + branch_name = parse_branch_name_type1(error_output) + if branch_name: + print(f"Detected error type 1, attempting to delete ref for branch: {branch_name}") + git_update_ref(branch_name) + else: + print("Error parsing branch name for type 1.") + # 错误类型2：尝试创建新的远程引用时，本地已经存在同名的引用 + elif "exists; cannot create" in error_output: + branch_name = parse_branch_name_type2(error_output) + if branch_name: + print(f"Detected error type 2, attempting to delete ref for branch: {branch_name}") + git_update_ref(branch_name) + else: + print("Error parsing branch name for type 2.") + +# 主函数 +def main(): + fetch_result = git_fetch() + if fetch_result.returncode != 0: # 如果 git fetch 命令失败 + error_output = fetch_result.stderr + handle_error(error_output) + else: + print("Git fetch successful.") + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/packaging/smoke_test/assets/style.css b/packaging/smoke_test/assets/style.css new file mode 100644 index 000000000000..c89d42818c65 --- /dev/null +++ b/packaging/smoke_test/assets/style.css @@ -0,0 +1,319 @@ +body { + font-family: Helvetica, Arial, sans-serif; + font-size: 12px; + /* do not increase min-width as some may use split screens */ + min-width: 800px; + color: #999; + } + + h1 { + font-size: 24px; + color: black; + } + + h2 { + font-size: 16px; + color: black; + } + + p { + color: black; + } + + a { + color: #999; + } + + table { + border-collapse: collapse; + } + + /****************************** + * SUMMARY INFORMATION + ******************************/ + #environment td { + padding: 5px; + border: 1px solid #e6e6e6; + vertical-align: top; + } + #environment tr:nth-child(odd) { + background-color: #f6f6f6; + } + #environment ul { + margin: 0; + padding: 0 20px; + } + + /****************************** + * TEST RESULT COLORS + ******************************/ + span.passed, + .passed .col-result { + color: green; + } + + span.skipped, + span.xfailed, + span.rerun, + .skipped .col-result, + .xfailed .col-result, + .rerun .col-result { + color: orange; + } + + span.error, + span.failed, + span.xpassed, + .error .col-result, + .failed .col-result, + .xpassed .col-result { + color: red; + } + + .col-links__extra { + margin-right: 3px; + } + + /****************************** + * RESULTS TABLE + * + * 1. Table Layout + * 2. Extra + * 3. Sorting items + * + ******************************/ + /*------------------ + * 1. Table Layout + *------------------*/ + #results-table { + border: 1px solid #e6e6e6; + color: #999; + font-size: 12px; + width: 100%; + } + #results-table th, + #results-table td { + padding: 5px; + border: 1px solid #e6e6e6; + text-align: left; + } + #results-table th { + font-weight: bold; + } + + /*------------------ + * 2. Extra + *------------------*/ + .logwrapper { + max-height: 230px; + overflow-y: scroll; + background-color: #e6e6e6; + } + .logwrapper.expanded { + max-height: none; + } + .logwrapper.expanded .logexpander:after { + content: "collapse [-]"; + } + .logwrapper .logexpander { + z-index: 1; + position: sticky; + top: 10px; + width: max-content; + border: 1px solid; + border-radius: 3px; + padding: 5px 7px; + margin: 10px 0 10px calc(100% - 80px); + cursor: pointer; + background-color: #e6e6e6; + } + .logwrapper .logexpander:after { + content: "expand [+]"; + } + .logwrapper .logexpander:hover { + color: #000; + border-color: #000; + } + .logwrapper .log { + min-height: 40px; + position: relative; + top: -50px; + height: calc(100% + 50px); + border: 1px solid #e6e6e6; + color: black; + display: block; + font-family: "Courier New", Courier, monospace; + padding: 5px; + padding-right: 80px; + white-space: pre-wrap; + } + + div.media { + border: 1px solid #e6e6e6; + float: right; + height: 240px; + margin: 0 5px; + overflow: hidden; + width: 320px; + } + + .media-container { + display: grid; + grid-template-columns: 25px auto 25px; + align-items: center; + flex: 1 1; + overflow: hidden; + height: 200px; + } + + .media-container--fullscreen { + grid-template-columns: 0px auto 0px; + } + + .media-container__nav--right, + .media-container__nav--left { + text-align: center; + cursor: pointer; + } + + .media-container__viewport { + cursor: pointer; + text-align: center; + height: inherit; + } + .media-container__viewport img, + .media-container__viewport video { + object-fit: cover; + width: 100%; + max-height: 100%; + } + + .media__name, + .media__counter { + display: flex; + flex-direction: row; + justify-content: space-around; + flex: 0 0 25px; + align-items: center; + } + + .collapsible td:not(.col-links) { + cursor: pointer; + } + .collapsible td:not(.col-links):hover::after { + color: #bbb; + font-style: italic; + cursor: pointer; + } + + .col-result { + width: 130px; + } + .col-result:hover::after { + content: " (hide details)"; + } + + .col-result.collapsed:hover::after { + content: " (show details)"; + } + + #environment-header h2:hover::after { + content: " (hide details)"; + color: #bbb; + font-style: italic; + cursor: pointer; + font-size: 12px; + } + + #environment-header.collapsed h2:hover::after { + content: " (show details)"; + color: #bbb; + font-style: italic; + cursor: pointer; + font-size: 12px; + } + + /*------------------ + * 3. Sorting items + *------------------*/ + .sortable { + cursor: pointer; + } + .sortable.desc:after { + content: " "; + position: relative; + left: 5px; + bottom: -12.5px; + border: 10px solid #4caf50; + border-bottom: 0; + border-left-color: transparent; + border-right-color: transparent; + } + .sortable.asc:after { + content: " "; + position: relative; + left: 5px; + bottom: 12.5px; + border: 10px solid #4caf50; + border-top: 0; + border-left-color: transparent; + border-right-color: transparent; + } + + .hidden, .summary__reload__button.hidden { + display: none; + } + + .summary__data { + flex: 0 0 550px; + } + .summary__reload { + flex: 1 1; + display: flex; + justify-content: center; + } + .summary__reload__button { + flex: 0 0 300px; + display: flex; + color: white; + font-weight: bold; + background-color: #4caf50; + text-align: center; + justify-content: center; + align-items: center; + border-radius: 3px; + cursor: pointer; + } + .summary__reload__button:hover { + background-color: #46a049; + } + .summary__spacer { + flex: 0 0 550px; + } + + .controls { + display: flex; + justify-content: space-between; + } + + .filters, + .collapse { + display: flex; + align-items: center; + } + .filters button, + .collapse button { + color: #999; + border: none; + background: none; + cursor: pointer; + text-decoration: underline; + } + .filters button:hover, + .collapse button:hover { + color: #ccc; + } + + .filter__label { + margin-right: 10px; + } \ No newline at end of file diff --git a/packaging/smoke_test/pytest_require.txt b/packaging/smoke_test/pytest_require.txt new file mode 100644 index 000000000000..eadbe92470dc --- /dev/null +++ b/packaging/smoke_test/pytest_require.txt @@ -0,0 +1,16 @@ +pytest-html +pytest-json-report +pytest-timeout +taospy +numpy +fabric2 +psutil +pandas +toml +distro +requests +pexpect +faker +pyopenssl +taos-ws-py +taospy \ No newline at end of file From 446122ad9650258832e1a212232dc961152e7fc2 Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Thu, 7 Nov 2024 16:22:30 +0800 Subject: [PATCH 12/28] test: add smoke test scripts --- packaging/{smoke_test => smokeTest}/assets/style.css | 0 packaging/smoke_test/pytest_require.txt | 3 ++- 2 files changed, 2 insertions(+), 1 deletion(-) rename packaging/{smoke_test => smokeTest}/assets/style.css (100%) diff --git a/packaging/smoke_test/assets/style.css b/packaging/smokeTest/assets/style.css similarity index 100% rename from packaging/smoke_test/assets/style.css rename to packaging/smokeTest/assets/style.css diff --git a/packaging/smoke_test/pytest_require.txt b/packaging/smoke_test/pytest_require.txt index eadbe92470dc..34019c6e8a7d 100644 --- a/packaging/smoke_test/pytest_require.txt +++ b/packaging/smoke_test/pytest_require.txt @@ -13,4 +13,5 @@ pexpect faker pyopenssl taos-ws-py -taospy \ No newline at end of file +taospy +tzlocal \ No newline at end of file From 234846b5436dd00f447f0cfd1ea182253b3eef4e Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Thu, 7 Nov 2024 16:24:54 +0800 Subject: [PATCH 13/28] test: add smoke test scripts --- .gitignore | 2 +- packaging/{smoke_test => smokeTest}/pytest_require.txt | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename packaging/{smoke_test => smokeTest}/pytest_require.txt (100%) diff --git a/.gitignore b/.gitignore index 7f2382db8707..8f461f2b0228 100644 --- a/.gitignore +++ b/.gitignore @@ -161,4 +161,4 @@ version.h geos_c.h source/libs/parser/src/sql.c include/common/ttokenauto.h -!packaging/smoke_test/pytest_require.txt +!packaging/smokeTest/pytest_require.txt diff --git a/packaging/smoke_test/pytest_require.txt b/packaging/smokeTest/pytest_require.txt similarity index 100% rename from packaging/smoke_test/pytest_require.txt rename to packaging/smokeTest/pytest_require.txt From daedcc53fb2b84b5a9acafffb0486cb3b84dd952 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 7 Nov 2024 16:28:47 +0800 Subject: [PATCH 14/28] refactor: return the error code and do some refactor. --- .../libs/executor/src/anomalywindowoperator.c | 17 ++++++++++++----- source/util/src/tanal.c | 3 +++ source/util/src/terror.c | 4 ++-- 3 files changed, 17 insertions(+), 7 deletions(-) diff --git a/source/libs/executor/src/anomalywindowoperator.c b/source/libs/executor/src/anomalywindowoperator.c index d03e527c2ba0..6adfbc6c7731 100644 --- a/source/libs/executor/src/anomalywindowoperator.c +++ b/source/libs/executor/src/anomalywindowoperator.c @@ -55,7 +55,7 @@ typedef struct { static void anomalyDestroyOperatorInfo(void* param); static int32_t anomalyAggregateNext(SOperatorInfo* pOperator, SSDataBlock** ppRes); -static void anomalyAggregateBlocks(SOperatorInfo* pOperator); +static int32_t anomalyAggregateBlocks(SOperatorInfo* pOperator); static int32_t anomalyCacheBlock(SAnomalyWindowOperatorInfo* pInfo, SSDataBlock* pBlock); int32_t createAnomalywindowOperatorInfo(SOperatorInfo* downstream, SPhysiNode* physiNode, SExecTaskInfo* pTaskInfo, @@ -78,6 +78,7 @@ int32_t createAnomalywindowOperatorInfo(SOperatorInfo* downstream, SPhysiNode* p code = TSDB_CODE_ANAL_ALGO_NOT_FOUND; goto _error; } + if (taosAnalGetAlgoUrl(pInfo->algoName, ANAL_ALGO_TYPE_ANOMALY_DETECT, pInfo->algoUrl, sizeof(pInfo->algoUrl)) != 0) { qError("failed to get anomaly_window algorithm url from %s", pInfo->algoName); code = TSDB_CODE_ANAL_ALGO_NOT_LOAD; @@ -198,7 +199,9 @@ static int32_t anomalyAggregateNext(SOperatorInfo* pOperator, SSDataBlock** ppRe QUERY_CHECK_CODE(code, lino, _end); } else { qDebug("group:%" PRId64 ", read finish for new group coming, blocks:%d", pSupp->groupId, numOfBlocks); - anomalyAggregateBlocks(pOperator); + code = anomalyAggregateBlocks(pOperator); + QUERY_CHECK_CODE(code, lino, _end); + pSupp->groupId = pBlock->info.id.groupId; numOfBlocks = 1; pSupp->cachedRows = pBlock->info.rows; @@ -217,7 +220,7 @@ static int32_t anomalyAggregateNext(SOperatorInfo* pOperator, SSDataBlock** ppRe if (numOfBlocks > 0) { qDebug("group:%" PRId64 ", read finish, blocks:%d", pInfo->anomalySup.groupId, numOfBlocks); - anomalyAggregateBlocks(pOperator); + code = anomalyAggregateBlocks(pOperator); } int64_t cost = taosGetTimestampUs() - st; @@ -229,6 +232,7 @@ static int32_t anomalyAggregateNext(SOperatorInfo* pOperator, SSDataBlock** ppRe pTaskInfo->code = code; T_LONG_JMP(pTaskInfo->env, code); } + (*ppRes) = (pBInfo->pRes->info.rows == 0) ? NULL : pBInfo->pRes; return code; } @@ -338,8 +342,8 @@ static int32_t anomalyAnalysisWindow(SOperatorInfo* pOperator) { SAnalBuf analBuf = {.bufType = ANAL_BUF_TYPE_JSON}; char dataBuf[64] = {0}; int32_t code = 0; + int64_t ts = 0; - int64_t ts = 0; // int64_t ts = taosGetTimestampMs(); snprintf(analBuf.fileName, sizeof(analBuf.fileName), "%s/tdengine-anomaly-%" PRId64 "-%" PRId64, tsTempDir, ts, pSupp->groupId); @@ -431,6 +435,7 @@ static int32_t anomalyAnalysisWindow(SOperatorInfo* pOperator) { if (code != 0) { qError("failed to analysis window since %s", tstrerror(code)); } + taosAnalBufDestroy(&analBuf); if (pJson != NULL) tjsonDelete(pJson); return code; @@ -473,7 +478,7 @@ static int32_t anomalyBuildResult(SOperatorInfo* pOperator) { return code; } -static void anomalyAggregateBlocks(SOperatorInfo* pOperator) { +static int32_t anomalyAggregateBlocks(SOperatorInfo* pOperator) { int32_t code = TSDB_CODE_SUCCESS; int32_t lino = 0; SAnomalyWindowOperatorInfo* pInfo = pOperator->info; @@ -623,6 +628,8 @@ static void anomalyAggregateBlocks(SOperatorInfo* pOperator) { pSupp->curWin.ekey = 0; pSupp->curWin.skey = 0; pSupp->curWinIndex = 0; + + return code; } #else diff --git a/source/util/src/tanal.c b/source/util/src/tanal.c index 92eee28ba87a..bb46a48c4298 100644 --- a/source/util/src/tanal.c +++ b/source/util/src/tanal.c @@ -167,6 +167,8 @@ int32_t taosAnalGetAlgoUrl(const char *algoName, EAnalAlgoType type, char *url, char name[TSDB_ANAL_ALGO_KEY_LEN] = {0}; int32_t nameLen = 1 + tsnprintf(name, sizeof(name) - 1, "%d:%s", type, algoName); + char *unused = strntolower(name, name, nameLen); + if (taosThreadMutexLock(&tsAlgos.lock) == 0) { SAnalUrl *pUrl = taosHashAcquire(tsAlgos.hash, name, nameLen); if (pUrl != NULL) { @@ -178,6 +180,7 @@ int32_t taosAnalGetAlgoUrl(const char *algoName, EAnalAlgoType type, char *url, code = terrno; uError("algo:%s, type:%s, url not found", algoName, taosAnalAlgoStr(type)); } + if (taosThreadMutexUnlock(&tsAlgos.lock) != 0) { uError("failed to unlock hash"); return TSDB_CODE_OUT_OF_MEMORY; diff --git a/source/util/src/terror.c b/source/util/src/terror.c index d660edd0b86e..ce209cc7184f 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -360,8 +360,8 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MND_ANODE_TOO_MANY_ALGO, "Anode too many algori TAOS_DEFINE_ERROR(TSDB_CODE_MND_ANODE_TOO_LONG_ALGO_NAME, "Anode too long algorithm name") TAOS_DEFINE_ERROR(TSDB_CODE_MND_ANODE_TOO_MANY_ALGO_TYPE, "Anode too many algorithm type") -TAOS_DEFINE_ERROR(TSDB_CODE_ANAL_URL_RSP_IS_NULL, "Analysis url response is NULL") -TAOS_DEFINE_ERROR(TSDB_CODE_ANAL_URL_CANT_ACCESS, "Analysis url can't access") +TAOS_DEFINE_ERROR(TSDB_CODE_ANAL_URL_RSP_IS_NULL, "Analysis service response is NULL") +TAOS_DEFINE_ERROR(TSDB_CODE_ANAL_URL_CANT_ACCESS, "Analysis service can't access") TAOS_DEFINE_ERROR(TSDB_CODE_ANAL_ALGO_NOT_FOUND, "Analysis algorithm not found") TAOS_DEFINE_ERROR(TSDB_CODE_ANAL_ALGO_NOT_LOAD, "Analysis algorithm not loaded") TAOS_DEFINE_ERROR(TSDB_CODE_ANAL_BUF_INVALID_TYPE, "Analysis invalid buffer type") From a8f7f153d92ce3ddb4dba07af57d7ec45fcb1bad Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 7 Nov 2024 16:44:43 +0800 Subject: [PATCH 15/28] refactor: do some internal refacrtor. --- include/common/{tanal.h => tanalytics.h} | 10 +++---- source/common/src/tmsg.c | 12 ++++---- source/dnode/mgmt/mgmt_dnode/src/dmHandle.c | 2 +- source/dnode/mgmt/mgmt_dnode/src/dmInt.c | 4 +-- source/dnode/mgmt/node_mgmt/src/dmEnv.c | 4 +-- source/dnode/mgmt/node_mgmt/src/dmTransport.c | 2 +- source/dnode/mnode/impl/CMakeLists.txt | 2 +- source/dnode/mnode/impl/src/mndAnode.c | 10 +++---- source/libs/executor/CMakeLists.txt | 2 +- .../libs/executor/src/anomalywindowoperator.c | 4 +-- source/libs/executor/src/forecastoperator.c | 4 +-- source/libs/function/src/builtins.c | 2 +- source/libs/function/src/builtinsimpl.c | 2 +- source/libs/parser/src/parTranslater.c | 2 +- source/util/CMakeLists.txt | 2 +- source/util/src/{tanal.c => tanalytics.c} | 29 +++++++++---------- 16 files changed, 46 insertions(+), 47 deletions(-) rename include/common/{tanal.h => tanalytics.h} (96%) rename source/util/src/{tanal.c => tanalytics.c} (97%) diff --git a/include/common/tanal.h b/include/common/tanalytics.h similarity index 96% rename from include/common/tanal.h rename to include/common/tanalytics.h index 69d110d161d4..85eb963129fb 100644 --- a/include/common/tanal.h +++ b/include/common/tanalytics.h @@ -36,7 +36,7 @@ typedef struct { int32_t anode; int32_t urlLen; char *url; -} SAnalUrl; +} SAnalyticsUrl; typedef enum { ANAL_BUF_TYPE_JSON = 0, @@ -53,18 +53,18 @@ typedef struct { TdFilePtr filePtr; char fileName[TSDB_FILENAME_LEN + 10]; int64_t numOfRows; -} SAnalColBuf; +} SAnalyticsColBuf; typedef struct { EAnalBufType bufType; TdFilePtr filePtr; char fileName[TSDB_FILENAME_LEN]; int32_t numOfCols; - SAnalColBuf *pCols; + SAnalyticsColBuf *pCols; } SAnalBuf; -int32_t taosAnalInit(); -void taosAnalCleanup(); +int32_t taosAnalyticsInit(); +void taosAnalyticsCleanup(); SJson *taosAnalSendReqRetJson(const char *url, EAnalHttpType type, SAnalBuf *pBuf); int32_t taosAnalGetAlgoUrl(const char *algoName, EAnalAlgoType type, char *url, int32_t urlLen); diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 458badc7640e..6d1699b91165 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -40,7 +40,7 @@ #define TD_MSG_RANGE_CODE_ #include "tmsgdef.h" -#include "tanal.h" +#include "tanalytics.h" #include "tcol.h" #include "tlog.h" @@ -2166,7 +2166,7 @@ int32_t tSerializeRetrieveAnalAlgoRsp(void *buf, int32_t bufLen, SRetrieveAnalAl int32_t numOfAlgos = 0; void *pIter = taosHashIterate(pRsp->hash, NULL); while (pIter != NULL) { - SAnalUrl *pUrl = pIter; + SAnalyticsUrl *pUrl = pIter; size_t nameLen = 0; const char *name = taosHashGetKey(pIter, &nameLen); if (nameLen > 0 && nameLen <= TSDB_ANAL_ALGO_KEY_LEN && pUrl->urlLen > 0) { @@ -2181,7 +2181,7 @@ int32_t tSerializeRetrieveAnalAlgoRsp(void *buf, int32_t bufLen, SRetrieveAnalAl pIter = taosHashIterate(pRsp->hash, NULL); while (pIter != NULL) { - SAnalUrl *pUrl = pIter; + SAnalyticsUrl *pUrl = pIter; size_t nameLen = 0; const char *name = taosHashGetKey(pIter, &nameLen); if (nameLen > 0 && pUrl->urlLen > 0) { @@ -2225,7 +2225,7 @@ int32_t tDeserializeRetrieveAnalAlgoRsp(void *buf, int32_t bufLen, SRetrieveAnal int32_t nameLen; int32_t type; char name[TSDB_ANAL_ALGO_KEY_LEN]; - SAnalUrl url = {0}; + SAnalyticsUrl url = {0}; TAOS_CHECK_EXIT(tStartDecode(&decoder)); TAOS_CHECK_EXIT(tDecodeI64(&decoder, &pRsp->ver)); @@ -2245,7 +2245,7 @@ int32_t tDeserializeRetrieveAnalAlgoRsp(void *buf, int32_t bufLen, SRetrieveAnal TAOS_CHECK_EXIT(tDecodeBinaryAlloc(&decoder, (void **)&url.url, NULL) < 0); } - TAOS_CHECK_EXIT(taosHashPut(pRsp->hash, name, nameLen, &url, sizeof(SAnalUrl))); + TAOS_CHECK_EXIT(taosHashPut(pRsp->hash, name, nameLen, &url, sizeof(SAnalyticsUrl))); } tEndDecode(&decoder); @@ -2258,7 +2258,7 @@ int32_t tDeserializeRetrieveAnalAlgoRsp(void *buf, int32_t bufLen, SRetrieveAnal void tFreeRetrieveAnalAlgoRsp(SRetrieveAnalAlgoRsp *pRsp) { void *pIter = taosHashIterate(pRsp->hash, NULL); while (pIter != NULL) { - SAnalUrl *pUrl = (SAnalUrl *)pIter; + SAnalyticsUrl *pUrl = (SAnalyticsUrl *)pIter; taosMemoryFree(pUrl->url); pIter = taosHashIterate(pRsp->hash, pIter); } diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c index d6b792ca74bd..78cc35a62c1d 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c @@ -18,7 +18,7 @@ #include "dmInt.h" #include "monitor.h" #include "systable.h" -#include "tanal.h" +#include "tanalytics.h" #include "tchecksum.h" extern SConfig *tsCfg; diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmInt.c b/source/dnode/mgmt/mgmt_dnode/src/dmInt.c index 04b4e9101ca9..fb7d891c67cd 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmInt.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmInt.c @@ -16,7 +16,7 @@ #define _DEFAULT_SOURCE #include "dmInt.h" #include "libs/function/tudf.h" -#include "tanal.h" +#include "tanalytics.h" static int32_t dmStartMgmt(SDnodeMgmt *pMgmt) { int32_t code = 0; @@ -85,7 +85,7 @@ static int32_t dmOpenMgmt(SMgmtInputOpt *pInput, SMgmtOutputOpt *pOutput) { dError("failed to start udfd since %s", tstrerror(code)); } - if ((code = taosAnalInit()) != 0) { + if ((code = taosAnalyticsInit()) != 0) { dError("failed to init analysis env since %s", tstrerror(code)); } diff --git a/source/dnode/mgmt/node_mgmt/src/dmEnv.c b/source/dnode/mgmt/node_mgmt/src/dmEnv.c index 694cc52d6472..6d4ebe424a58 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmEnv.c +++ b/source/dnode/mgmt/node_mgmt/src/dmEnv.c @@ -21,7 +21,7 @@ #include "tgrant.h" #include "tcompare.h" #include "tcs.h" -#include "tanal.h" +#include "tanalytics.h" // clang-format on #define DM_INIT_AUDIT() \ @@ -209,7 +209,7 @@ void dmCleanup() { dError("failed to close udfc"); } udfStopUdfd(); - taosAnalCleanup(); + taosAnalyticsCleanup(); taosStopCacheRefreshWorker(); (void)dmDiskClose(); DestroyRegexCache(); diff --git a/source/dnode/mgmt/node_mgmt/src/dmTransport.c b/source/dnode/mgmt/node_mgmt/src/dmTransport.c index 5a276de251b9..61543e619e28 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmTransport.c +++ b/source/dnode/mgmt/node_mgmt/src/dmTransport.c @@ -16,7 +16,7 @@ #define _DEFAULT_SOURCE #include "dmMgmt.h" #include "qworker.h" -#include "tanal.h" +#include "tanalytics.h" #include "tversion.h" static inline void dmSendRsp(SRpcMsg *pMsg) { diff --git a/source/dnode/mnode/impl/CMakeLists.txt b/source/dnode/mnode/impl/CMakeLists.txt index 8a390948aebc..ad36d8c8aea9 100644 --- a/source/dnode/mnode/impl/CMakeLists.txt +++ b/source/dnode/mnode/impl/CMakeLists.txt @@ -18,7 +18,7 @@ if(TD_ENTERPRISE) endif() if(${BUILD_WITH_ANALYSIS}) - add_definitions(-DUSE_ANAL) + add_definitions(-DUSE_ANALYTICS) endif() endif() diff --git a/source/dnode/mnode/impl/src/mndAnode.c b/source/dnode/mnode/impl/src/mndAnode.c index 17e3e84c810f..87bfe9f7afa9 100644 --- a/source/dnode/mnode/impl/src/mndAnode.c +++ b/source/dnode/mnode/impl/src/mndAnode.c @@ -21,10 +21,10 @@ #include "mndShow.h" #include "mndTrans.h" #include "mndUser.h" -#include "tanal.h" +#include "tanalytics.h" #include "tjson.h" -#ifdef USE_ANAL +#ifdef USE_ANALYTICS #define TSDB_ANODE_VER_NUMBER 1 #define TSDB_ANODE_RESERVE_SIZE 64 @@ -806,7 +806,7 @@ static int32_t mndProcessAnalAlgoReq(SRpcMsg *pReq) { SSdb *pSdb = pMnode->pSdb; int32_t code = -1; SAnodeObj *pObj = NULL; - SAnalUrl url; + SAnalyticsUrl url; int32_t nameLen; char name[TSDB_ANAL_ALGO_KEY_LEN]; SRetrieveAnalAlgoReq req = {0}; @@ -838,7 +838,7 @@ static int32_t mndProcessAnalAlgoReq(SRpcMsg *pReq) { SAnodeAlgo *algo = taosArrayGet(algos, a); nameLen = 1 + tsnprintf(name, sizeof(name) - 1, "%d:%s", url.type, algo->name); - SAnalUrl *pOldUrl = taosHashAcquire(rsp.hash, name, nameLen); + SAnalyticsUrl *pOldUrl = taosHashAcquire(rsp.hash, name, nameLen); if (pOldUrl == NULL || (pOldUrl != NULL && pOldUrl->anode < url.anode)) { if (pOldUrl != NULL) { taosMemoryFreeClear(pOldUrl->url); @@ -855,7 +855,7 @@ static int32_t mndProcessAnalAlgoReq(SRpcMsg *pReq) { url.urlLen = 1 + tsnprintf(url.url, TSDB_ANAL_ANODE_URL_LEN + TSDB_ANAL_ALGO_TYPE_LEN, "%s/%s", pAnode->url, taosAnalAlgoUrlStr(url.type)); - if (taosHashPut(rsp.hash, name, nameLen, &url, sizeof(SAnalUrl)) != 0) { + if (taosHashPut(rsp.hash, name, nameLen, &url, sizeof(SAnalyticsUrl)) != 0) { taosMemoryFree(url.url); sdbRelease(pSdb, pAnode); goto _OVER; diff --git a/source/libs/executor/CMakeLists.txt b/source/libs/executor/CMakeLists.txt index 014b53837559..9a49076b6b0c 100644 --- a/source/libs/executor/CMakeLists.txt +++ b/source/libs/executor/CMakeLists.txt @@ -7,7 +7,7 @@ if(${TD_DARWIN}) endif(${TD_DARWIN}) if(${BUILD_WITH_ANALYSIS}) - add_definitions(-DUSE_ANAL) + add_definitions(-DUSE_ANALYTICS) endif() target_link_libraries(executor diff --git a/source/libs/executor/src/anomalywindowoperator.c b/source/libs/executor/src/anomalywindowoperator.c index 6adfbc6c7731..94cc5d912956 100644 --- a/source/libs/executor/src/anomalywindowoperator.c +++ b/source/libs/executor/src/anomalywindowoperator.c @@ -19,14 +19,14 @@ #include "functionMgt.h" #include "operator.h" #include "querytask.h" -#include "tanal.h" +#include "tanalytics.h" #include "tcommon.h" #include "tcompare.h" #include "tdatablock.h" #include "tjson.h" #include "ttime.h" -#ifdef USE_ANAL +#ifdef USE_ANALYTICS typedef struct { SArray* blocks; // SSDataBlock* diff --git a/source/libs/executor/src/forecastoperator.c b/source/libs/executor/src/forecastoperator.c index 0afa933ee83a..20dc9e28ba7b 100644 --- a/source/libs/executor/src/forecastoperator.c +++ b/source/libs/executor/src/forecastoperator.c @@ -19,14 +19,14 @@ #include "operator.h" #include "querytask.h" #include "storageapi.h" -#include "tanal.h" +#include "tanalytics.h" #include "tcommon.h" #include "tcompare.h" #include "tdatablock.h" #include "tfill.h" #include "ttime.h" -#ifdef USE_ANAL +#ifdef USE_ANALYTICS typedef struct { char algoName[TSDB_ANAL_ALGO_NAME_LEN]; diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index 552933dcad36..2d68eb9d51c3 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -19,7 +19,7 @@ #include "geomFunc.h" #include "querynodes.h" #include "scalar.h" -#include "tanal.h" +#include "tanalytics.h" #include "taoserror.h" #include "ttime.h" diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index 0aad1501cebb..acdac7cbc398 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -19,7 +19,7 @@ #include "functionResInfoInt.h" #include "query.h" #include "querynodes.h" -#include "tanal.h" +#include "tanalytics.h" #include "tcompare.h" #include "tdatablock.h" #include "tdigest.h" diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 02295b34da8a..99c03c412ce3 100755 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -24,7 +24,7 @@ #include "parUtil.h" #include "scalar.h" #include "systable.h" -#include "tanal.h" +#include "tanalytics.h" #include "tcol.h" #include "tglobal.h" #include "ttime.h" diff --git a/source/util/CMakeLists.txt b/source/util/CMakeLists.txt index 7f5955f3ddb6..2633bb3268d4 100644 --- a/source/util/CMakeLists.txt +++ b/source/util/CMakeLists.txt @@ -18,7 +18,7 @@ else() endif(${ASSERT_NOT_CORE}) if(${BUILD_WITH_ANALYSIS}) - add_definitions(-DUSE_ANAL) + add_definitions(-DUSE_ANALYTICS) endif() target_include_directories( diff --git a/source/util/src/tanal.c b/source/util/src/tanalytics.c similarity index 97% rename from source/util/src/tanal.c rename to source/util/src/tanalytics.c index bb46a48c4298..99d91700a2ea 100644 --- a/source/util/src/tanal.c +++ b/source/util/src/tanalytics.c @@ -14,18 +14,17 @@ */ #define _DEFAULT_SOURCE -#include "tanal.h" -#include "tmsg.h" +#include "tanalytics.h" #include "ttypes.h" #include "tutil.h" -#ifdef USE_ANAL +#ifdef USE_ANALYTICS #include #define ANAL_ALGO_SPLIT "," typedef struct { int64_t ver; - SHashObj *hash; // algoname:algotype -> SAnalUrl + SHashObj *hash; // algoname:algotype -> SAnalyticsUrl TdThreadMutex lock; } SAlgoMgmt; @@ -69,7 +68,7 @@ EAnalAlgoType taosAnalAlgoInt(const char *name) { return ANAL_ALGO_TYPE_END; } -int32_t taosAnalInit() { +int32_t taosAnalyticsInit() { if (curl_global_init(CURL_GLOBAL_ALL) != 0) { uError("failed to init curl"); return -1; @@ -94,14 +93,14 @@ int32_t taosAnalInit() { static void taosAnalFreeHash(SHashObj *hash) { void *pIter = taosHashIterate(hash, NULL); while (pIter != NULL) { - SAnalUrl *pUrl = (SAnalUrl *)pIter; + SAnalyticsUrl *pUrl = (SAnalyticsUrl *)pIter; taosMemoryFree(pUrl->url); pIter = taosHashIterate(hash, pIter); } taosHashCleanup(hash); } -void taosAnalCleanup() { +void taosAnalyticsCleanup() { curl_global_cleanup(); if (taosThreadMutexDestroy(&tsAlgos.lock) != 0) { uError("failed to destroy anal lock"); @@ -170,7 +169,7 @@ int32_t taosAnalGetAlgoUrl(const char *algoName, EAnalAlgoType type, char *url, char *unused = strntolower(name, name, nameLen); if (taosThreadMutexLock(&tsAlgos.lock) == 0) { - SAnalUrl *pUrl = taosHashAcquire(tsAlgos.hash, name, nameLen); + SAnalyticsUrl *pUrl = taosHashAcquire(tsAlgos.hash, name, nameLen); if (pUrl != NULL) { tstrncpy(url, pUrl->url, urlLen); uDebug("algo:%s, type:%s, url:%s", algoName, taosAnalAlgoStr(type), url); @@ -406,7 +405,7 @@ static int32_t tsosAnalJsonBufOpen(SAnalBuf *pBuf, int32_t numOfCols) { return terrno; } - pBuf->pCols = taosMemoryCalloc(numOfCols, sizeof(SAnalColBuf)); + pBuf->pCols = taosMemoryCalloc(numOfCols, sizeof(SAnalyticsColBuf)); if (pBuf->pCols == NULL) return TSDB_CODE_OUT_OF_MEMORY; pBuf->numOfCols = numOfCols; @@ -415,7 +414,7 @@ static int32_t tsosAnalJsonBufOpen(SAnalBuf *pBuf, int32_t numOfCols) { } for (int32_t i = 0; i < numOfCols; ++i) { - SAnalColBuf *pCol = &pBuf->pCols[i]; + SAnalyticsColBuf *pCol = &pBuf->pCols[i]; snprintf(pCol->fileName, sizeof(pCol->fileName), "%s-c%d", pBuf->fileName, i); pCol->filePtr = taosOpenFile(pCol->fileName, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC | TD_FILE_WRITE_THROUGH); @@ -549,7 +548,7 @@ static int32_t taosAnalJsonBufWriteDataEnd(SAnalBuf *pBuf) { if (pBuf->bufType == ANAL_BUF_TYPE_JSON_COL) { for (int32_t i = 0; i < pBuf->numOfCols; ++i) { - SAnalColBuf *pCol = &pBuf->pCols[i]; + SAnalyticsColBuf *pCol = &pBuf->pCols[i]; code = taosFsyncFile(pCol->filePtr); if (code != 0) return code; @@ -591,7 +590,7 @@ int32_t taosAnalJsonBufClose(SAnalBuf *pBuf) { if (pBuf->bufType == ANAL_BUF_TYPE_JSON_COL) { for (int32_t i = 0; i < pBuf->numOfCols; ++i) { - SAnalColBuf *pCol = &pBuf->pCols[i]; + SAnalyticsColBuf *pCol = &pBuf->pCols[i]; if (pCol->filePtr != NULL) { code = taosFsyncFile(pCol->filePtr); if (code != 0) return code; @@ -613,7 +612,7 @@ void taosAnalBufDestroy(SAnalBuf *pBuf) { if (pBuf->bufType == ANAL_BUF_TYPE_JSON_COL) { for (int32_t i = 0; i < pBuf->numOfCols; ++i) { - SAnalColBuf *pCol = &pBuf->pCols[i]; + SAnalyticsColBuf *pCol = &pBuf->pCols[i]; if (pCol->fileName[0] != 0) { if (pCol->filePtr != NULL) (void)taosCloseFile(&pCol->filePtr); if (taosRemoveFile(pCol->fileName) != 0) { @@ -729,8 +728,8 @@ static int32_t taosAnalBufGetCont(SAnalBuf *pBuf, char **ppCont, int64_t *pContL #else -int32_t taosAnalInit() { return 0; } -void taosAnalCleanup() {} +int32_t taosAnalyticsInit() { return 0; } +void taosAnalyticsCleanup() {} SJson *taosAnalSendReqRetJson(const char *url, EAnalHttpType type, SAnalBuf *pBuf) { return NULL; } int32_t taosAnalGetAlgoUrl(const char *algoName, EAnalAlgoType type, char *url, int32_t urlLen) { return 0; } From 73881169c11837750f12509e7cf0201e21abdc10 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 7 Nov 2024 17:51:12 +0800 Subject: [PATCH 16/28] fix: install for taosanode --- packaging/tools/make_install.sh | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/packaging/tools/make_install.sh b/packaging/tools/make_install.sh index 1b8fa2fb702f..0874433e94f2 100755 --- a/packaging/tools/make_install.sh +++ b/packaging/tools/make_install.sh @@ -145,7 +145,14 @@ function kill_taosd() { function install_main_path() { #create install main dir and all sub dir - ${csudo}rm -rf ${install_main_dir} || : + ${csudo}rm -rf ${install_main_dir}/cfg || : + ${csudo}rm -rf ${install_main_dir}/bin || : + ${csudo}rm -rf ${install_main_dir}/driver || : + ${csudo}rm -rf ${install_main_dir}/examples || : + ${csudo}rm -rf ${install_main_dir}/include || : + ${csudo}rm -rf ${install_main_dir}/share || : + ${csudo}rm -rf ${install_main_dir}/log || : + ${csudo}mkdir -p ${install_main_dir} ${csudo}mkdir -p ${install_main_dir}/cfg ${csudo}mkdir -p ${install_main_dir}/bin From 6a601e0459e82b0e1fe499bc904071148587d492 Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao@163.com> Date: Thu, 7 Nov 2024 17:54:01 +0800 Subject: [PATCH 17/28] doc(stream):recover stream --- docs/zh/06-advanced/03-stream.md | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/docs/zh/06-advanced/03-stream.md b/docs/zh/06-advanced/03-stream.md index 7486b3b043a3..199715aba51a 100644 --- a/docs/zh/06-advanced/03-stream.md +++ b/docs/zh/06-advanced/03-stream.md @@ -228,4 +228,30 @@ PAUSE STREAM [IF EXISTS] stream_name; RESUME STREAM [IF EXISTS] [IGNORE UNTREATED] stream_name; ``` -没有指定 IF EXISTS，如果该 stream 不存在，则报错。如果存在，则恢复流计算。指定了 IF EXISTS，如果 stream 不存在，则返回成功。如果存在，则恢复流计算。如果指定 IGNORE UNTREATED，则恢复流计算时，忽略流计算暂停期间写入的数据。 \ No newline at end of file +没有指定 IF EXISTS，如果该 stream 不存在，则报错。如果存在，则恢复流计算。指定了 IF EXISTS，如果 stream 不存在，则返回成功。如果存在，则恢复流计算。如果指定 IGNORE UNTREATED，则恢复流计算时，忽略流计算暂停期间写入的数据。 + +### 流计算升级故障恢复 + +升级导致流计算不兼容，需要删除流计算，然后重新创建流计算。步骤如下： +1.修改taos.cfg，添加disableStream 1 +2.启动taosd +3.启动taos， + +```sql +drop stream xxxx; ---- xxx指stream name +flush database stream_source_db; ----流计算读取数据的超级表所在的database； +flush database stream_dest_db; -----流计算写入数据的超级表所在的database； +``` + +举例： + +```sql +create stream streams1 into test1.streamst as select _wstart, count(a) c1 from test.st interval(1s) ; +drop database streams1; +flush database test; +flush database test1; +``` + +4.关闭taosd +5.修改taos.cfg，去掉disableStream 1，或将disableStream改为 0 +6.启动taosd \ No newline at end of file From 56be3e79d259c65c7b038d8435c03cc8a5414a10 Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao@163.com> Date: Thu, 7 Nov 2024 17:56:10 +0800 Subject: [PATCH 18/28] doc(stream):recover stream --- docs/zh/06-advanced/03-stream.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/zh/06-advanced/03-stream.md b/docs/zh/06-advanced/03-stream.md index 199715aba51a..73756192705c 100644 --- a/docs/zh/06-advanced/03-stream.md +++ b/docs/zh/06-advanced/03-stream.md @@ -232,10 +232,10 @@ RESUME STREAM [IF EXISTS] [IGNORE UNTREATED] stream_name; ### 流计算升级故障恢复 -升级导致流计算不兼容，需要删除流计算，然后重新创建流计算。步骤如下： -1.修改taos.cfg，添加disableStream 1 -2.启动taosd -3.启动taos， +升级 TDengine 后，如果流计算不兼容，需要删除流计算，然后重新创建流计算。步骤如下： +1.修改 taos.cfg，添加disableStream 1 +2.启动 taosd +3.启动 taos， ```sql drop stream xxxx; ---- xxx指stream name @@ -252,6 +252,6 @@ flush database test; flush database test1; ``` -4.关闭taosd -5.修改taos.cfg，去掉disableStream 1，或将disableStream改为 0 -6.启动taosd \ No newline at end of file +4.关闭 taosd +5.修改 taos.cfg，去掉 disableStream 1，或将 disableStream 改为 0 +6.启动 taosd \ No newline at end of file From 7b6824dd9557416f3e57607b758d99491fe5552b Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao@163.com> Date: Thu, 7 Nov 2024 17:57:26 +0800 Subject: [PATCH 19/28] doc(stream):recover stream --- docs/zh/06-advanced/03-stream.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/zh/06-advanced/03-stream.md b/docs/zh/06-advanced/03-stream.md index 73756192705c..8b8ec3b7a402 100644 --- a/docs/zh/06-advanced/03-stream.md +++ b/docs/zh/06-advanced/03-stream.md @@ -233,8 +233,11 @@ RESUME STREAM [IF EXISTS] [IGNORE UNTREATED] stream_name; ### 流计算升级故障恢复 升级 TDengine 后，如果流计算不兼容，需要删除流计算，然后重新创建流计算。步骤如下： + 1.修改 taos.cfg，添加disableStream 1 + 2.启动 taosd + 3.启动 taos， ```sql @@ -253,5 +256,7 @@ flush database test1; ``` 4.关闭 taosd + 5.修改 taos.cfg，去掉 disableStream 1，或将 disableStream 改为 0 + 6.启动 taosd \ No newline at end of file From 875702106548136059f42696643103ba2060c33b Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao@163.com> Date: Thu, 7 Nov 2024 17:58:56 +0800 Subject: [PATCH 20/28] doc(stream):recover stream --- docs/zh/06-advanced/03-stream.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/zh/06-advanced/03-stream.md b/docs/zh/06-advanced/03-stream.md index 8b8ec3b7a402..cc63a54008d7 100644 --- a/docs/zh/06-advanced/03-stream.md +++ b/docs/zh/06-advanced/03-stream.md @@ -236,7 +236,7 @@ RESUME STREAM [IF EXISTS] [IGNORE UNTREATED] stream_name; 1.修改 taos.cfg，添加disableStream 1 -2.启动 taosd +2.重启 taosd 3.启动 taos， From 334f82215e7bcff9035739ffeecdcdd828850350 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 7 Nov 2024 18:06:20 +0800 Subject: [PATCH 21/28] doc: minor changes --- docs/zh/06-advanced/03-stream.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/zh/06-advanced/03-stream.md b/docs/zh/06-advanced/03-stream.md index cc63a54008d7..671a6ec55cd6 100644 --- a/docs/zh/06-advanced/03-stream.md +++ b/docs/zh/06-advanced/03-stream.md @@ -234,16 +234,16 @@ RESUME STREAM [IF EXISTS] [IGNORE UNTREATED] stream_name; 升级 TDengine 后，如果流计算不兼容，需要删除流计算，然后重新创建流计算。步骤如下： -1.修改 taos.cfg，添加disableStream 1 +1.修改 taos.cfg，添加 disableStream 1 2.重启 taosd -3.启动 taos， +3.启动 taos ```sql -drop stream xxxx; ---- xxx指stream name -flush database stream_source_db; ----流计算读取数据的超级表所在的database； -flush database stream_dest_db; -----流计算写入数据的超级表所在的database； +drop stream xxxx; ---- xxx 指stream name +flush database stream_source_db; ---- 流计算读取数据的超级表所在的 database +flush database stream_dest_db; ---- 流计算写入数据的超级表所在的 database ``` 举例： From 9a8a91b619b07b9ec048dbc91738f079791912b3 Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao@163.com> Date: Thu, 7 Nov 2024 18:10:11 +0800 Subject: [PATCH 22/28] doc(stream):recover stream --- docs/zh/06-advanced/03-stream.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/zh/06-advanced/03-stream.md b/docs/zh/06-advanced/03-stream.md index cc63a54008d7..463fda3f471c 100644 --- a/docs/zh/06-advanced/03-stream.md +++ b/docs/zh/06-advanced/03-stream.md @@ -236,7 +236,7 @@ RESUME STREAM [IF EXISTS] [IGNORE UNTREATED] stream_name; 1.修改 taos.cfg，添加disableStream 1 -2.重启 taosd +2.重启 taosd。如果启动失败，需要手动删除文件夹：$dataDir/vnode/vnode*/tq/stream，$dataDir指 TDengine 存储数据的目录，在 $dataDir/vnode/ 目录下会有多个类似vnode1 、vnode2...vnode* 的目录 3.启动 taos， @@ -259,4 +259,4 @@ flush database test1; 5.修改 taos.cfg，去掉 disableStream 1，或将 disableStream 改为 0 -6.启动 taosd \ No newline at end of file +6.启动 taosd From 044645fa135bd169181fbe8042839b7c7880d7bf Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao@163.com> Date: Thu, 7 Nov 2024 18:12:17 +0800 Subject: [PATCH 23/28] doc(stream):recover stream --- docs/zh/06-advanced/03-stream.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/zh/06-advanced/03-stream.md b/docs/zh/06-advanced/03-stream.md index 463fda3f471c..d1b2f31b2fe3 100644 --- a/docs/zh/06-advanced/03-stream.md +++ b/docs/zh/06-advanced/03-stream.md @@ -236,7 +236,7 @@ RESUME STREAM [IF EXISTS] [IGNORE UNTREATED] stream_name; 1.修改 taos.cfg，添加disableStream 1 -2.重启 taosd。如果启动失败，需要手动删除文件夹：$dataDir/vnode/vnode*/tq/stream，$dataDir指 TDengine 存储数据的目录，在 $dataDir/vnode/ 目录下会有多个类似vnode1 、vnode2...vnode* 的目录 +2.重启 taosd。如果启动失败，需要手动删除文件夹：$dataDir/vnode/vnode*/tq/stream，$dataDir指 TDengine 存储数据的目录，在 $dataDir/vnode/ 目录下会有多个类似vnode1 、vnode2...vnode* 的目录，全部需要删除里面的tq/stream目录 3.启动 taos， From ce0bd07de059c650b63ca0850089974d412ae0d4 Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao@163.com> Date: Thu, 7 Nov 2024 18:15:39 +0800 Subject: [PATCH 24/28] doc(stream):recover stream --- docs/zh/06-advanced/03-stream.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/zh/06-advanced/03-stream.md b/docs/zh/06-advanced/03-stream.md index d1b2f31b2fe3..c187adbe1af2 100644 --- a/docs/zh/06-advanced/03-stream.md +++ b/docs/zh/06-advanced/03-stream.md @@ -236,7 +236,7 @@ RESUME STREAM [IF EXISTS] [IGNORE UNTREATED] stream_name; 1.修改 taos.cfg，添加disableStream 1 -2.重启 taosd。如果启动失败，需要手动删除文件夹：$dataDir/vnode/vnode*/tq/stream，$dataDir指 TDengine 存储数据的目录，在 $dataDir/vnode/ 目录下会有多个类似vnode1 、vnode2...vnode* 的目录，全部需要删除里面的tq/stream目录 +2.重启 taosd。如果启动失败，需要手动修改文件夹名字：$dataDir/vnode/vnode*/tq/stream，$dataDir指 TDengine 存储数据的目录，在 $dataDir/vnode/ 目录下会有多个类似vnode1 、vnode2...vnode* 的目录，全部需要修改里面的tq/stream目录的名字，改为vnode1.bk 、vnode2.bk ... vnode*.bk 3.启动 taos， From 1f7c9c93bd8e8870d138867f538daaaa9a05d35f Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao@163.com> Date: Thu, 7 Nov 2024 18:18:03 +0800 Subject: [PATCH 25/28] doc(stream):recover stream --- docs/zh/06-advanced/03-stream.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/zh/06-advanced/03-stream.md b/docs/zh/06-advanced/03-stream.md index 0ebb4b5b3e7e..758fbbd10caf 100644 --- a/docs/zh/06-advanced/03-stream.md +++ b/docs/zh/06-advanced/03-stream.md @@ -259,4 +259,4 @@ flush database test1; 5.修改 taos.cfg，去掉 disableStream 1，或将 disableStream 改为 0 -6.启动 taosd +6.启动 taosd \ No newline at end of file From ba27958c67e97e3998257c59ef267ae4df38292c Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao@163.com> Date: Thu, 7 Nov 2024 18:21:03 +0800 Subject: [PATCH 26/28] doc(stream):recover stream --- docs/zh/06-advanced/03-stream.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/zh/06-advanced/03-stream.md b/docs/zh/06-advanced/03-stream.md index 758fbbd10caf..d22da5c4c830 100644 --- a/docs/zh/06-advanced/03-stream.md +++ b/docs/zh/06-advanced/03-stream.md @@ -236,7 +236,7 @@ RESUME STREAM [IF EXISTS] [IGNORE UNTREATED] stream_name; 1.修改 taos.cfg，添加 disableStream 1 -2.重启 taosd。如果启动失败，需要手动修改文件夹名字：$dataDir/vnode/vnode*/tq/stream，$dataDir指 TDengine 存储数据的目录，在 $dataDir/vnode/ 目录下会有多个类似vnode1 、vnode2...vnode* 的目录，全部需要修改里面的tq/stream目录的名字，改为vnode1.bk 、vnode2.bk ... vnode*.bk +2.重启 taosd。如果启动失败，需要手动修改文件夹名字：$dataDir/vnode/vnode*/tq/stream，$dataDir 指 TDengine 存储数据的目录，在 $dataDir/vnode/ 目录下会有多个类似 vnode1 、vnode2...vnode* 的目录，全部需要修改里面的 tq/stream 目录的名字，改为 vnode1.bk 、vnode2.bk ... vnode*.bk 3.启动 taos From 774ad21b22250a4b0877bd45bb06457f377acae5 Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao@163.com> Date: Thu, 7 Nov 2024 19:03:13 +0800 Subject: [PATCH 27/28] doc(stream):recover stream --- docs/zh/06-advanced/03-stream.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/zh/06-advanced/03-stream.md b/docs/zh/06-advanced/03-stream.md index d22da5c4c830..2f6e515fd71d 100644 --- a/docs/zh/06-advanced/03-stream.md +++ b/docs/zh/06-advanced/03-stream.md @@ -236,7 +236,7 @@ RESUME STREAM [IF EXISTS] [IGNORE UNTREATED] stream_name; 1.修改 taos.cfg，添加 disableStream 1 -2.重启 taosd。如果启动失败，需要手动修改文件夹名字：$dataDir/vnode/vnode*/tq/stream，$dataDir 指 TDengine 存储数据的目录，在 $dataDir/vnode/ 目录下会有多个类似 vnode1 、vnode2...vnode* 的目录，全部需要修改里面的 tq/stream 目录的名字，改为 vnode1.bk 、vnode2.bk ... vnode*.bk +2.重启 taosd。如果启动失败，修改 stream 目录的名称，避免 taosd 启动的时候尝试加载 stream 目录下的流计算数据信息。不使用删除操作避免误操作导致的风险。需要修改的文件夹：$dataDir/vnode/vnode*/tq/stream，$dataDir 指 TDengine 存储数据的目录，在 $dataDir/vnode/ 目录下会有多个类似 vnode1 、vnode2...vnode* 的目录，全部需要修改里面的 tq/stream 目录的名字，改为 stream.bk 3.启动 taos From e64ad3a27d40df6d9bddbdfdece17a9724e434bc Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao@163.com> Date: Thu, 7 Nov 2024 19:04:23 +0800 Subject: [PATCH 28/28] doc(stream):recover stream --- docs/zh/06-advanced/03-stream.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/zh/06-advanced/03-stream.md b/docs/zh/06-advanced/03-stream.md index 2f6e515fd71d..c47831dde370 100644 --- a/docs/zh/06-advanced/03-stream.md +++ b/docs/zh/06-advanced/03-stream.md @@ -236,7 +236,7 @@ RESUME STREAM [IF EXISTS] [IGNORE UNTREATED] stream_name; 1.修改 taos.cfg，添加 disableStream 1 -2.重启 taosd。如果启动失败，修改 stream 目录的名称，避免 taosd 启动的时候尝试加载 stream 目录下的流计算数据信息。不使用删除操作避免误操作导致的风险。需要修改的文件夹：$dataDir/vnode/vnode*/tq/stream，$dataDir 指 TDengine 存储数据的目录，在 $dataDir/vnode/ 目录下会有多个类似 vnode1 、vnode2...vnode* 的目录，全部需要修改里面的 tq/stream 目录的名字，改为 stream.bk +2.重启 taosd。如果启动失败，修改 stream 目录的名称，避免 taosd 启动的时候尝试加载 stream 目录下的流计算数据信息。不使用删除操作避免误操作导致的风险。需要修改的文件夹：$dataDir/vnode/vnode*/tq/stream，$dataDir 指 TDengine 存储数据的目录，在 $dataDir/vnode/ 目录下会有多个类似 vnode1 、vnode2...vnode* 的目录，全部需要修改里面的 tq/stream 目录的名字，改为 tq/stream.bk 3.启动 taos