Skip to content

Commit

Permalink
Merge pull request #23 from F-ca7/feature/v1_4_0
Browse files Browse the repository at this point in the history 
Feature/v1_4_0
  • Loading branch information
@F-ca7
F-ca7 authored Apr 9, 2024
2 parents e8534e0 + 55d5a36 commit 6feabb0
Show file tree
Hide file tree
Showing 66 changed files with 164,698 additions and 112 deletions.
49 changes: 47 additions & 2 deletions batch-tool/docs/usage-details.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,11 +3,11 @@
* [数据库表导出](#数据库表导出)
* [数据库表导入](#数据库表导入)
* [使用yaml配置](#使用yaml配置)
* [TPC-H数据集](#TPC-H数据集)
* [常见问题排查](#常见问题排查)
<!-- TOC -->


# 常见导入导出场景
# 常见使用场景
以下省略 `java -jar batch-tool.jar -h 127.0.0.1 -u polardbx_root -p "123456" -P 8527` 等数据库连接信息配置 ，仅展示命令行功能参数的设置

## 数据库表导出
Expand Down Expand Up @@ -103,6 +103,17 @@ c_custkey|c_name|c_address|c_nationkey|c_phone|c_acctbal|c_mktsegment|c_comment
### 导入Excel文件
`-D sbtest_auto -o import -s , -t "sbtest1" -format XLSX -f "sbtest1_0.xlsx"`

### 导入TPC-H数据集

`-D tpch_auto -o import -benchmark TPCH -scale 100`
> 1. 使用 -scale 指定导入数据集的规模，单位：G
> 2. 可以使用 -t "lineitem;orders" 来指定表进行导入
### 更新TPC-H数据集

`-D tpch_auto -o update -benchmark TPCH -scale 100 -F 3`
> 1. 使用 -F 来指定更新的轮数
## 使用yaml配置
当有很多配置项需要设置时，使用命令行参数会很不方便编辑，此时建议使用yaml格式的配置文件，示例如下：

Expand Down Expand Up @@ -134,6 +145,29 @@ mask: >-
如果配置值包含[yaml特殊字符](https://yaml.org/spec/1.2.2/#53-indicator-characters)的话， 需要用引号括起来。
## TPC-H数据集
### 导入 TPC-H
`-o import -benchmark tpch -scale ${规格}`

其中规格的单位为GB，比如要导入 TPC-H 100G，则输入`-scale 100`。

根据经验来看，当数据库不成为瓶颈的时候，4C16G的客户端配置可以在30分钟内完成 TPC-H 100G 的导入
（使用参数`-pro 1 -con 80 -ringsize 8192 -minConn 81 -maxConn 81 -batchSize 500`）。

### 更新 TPC-H

` -o update -benchmark tpch -scale ${规格} -F ${更新轮数} `

可以通过设置 并发数 和 BatchSize 来对更新性能进行调优。

### 回滚 TPC-H 更新

` -o delete -benchmark tpch -scale ${规格} -F ${回滚轮数} `

回滚轮数需要与之前更新轮数一致，且确保之前的更新已完整执行

# 常见问题排查
1. 报错 **the server time zone value '' is unrecognized**

Expand Down Expand Up @@ -207,3 +241,14 @@ mask: >-
**解决**：Linux系统上，可以通过`locale`命令查看编码；
如果是数据库`character_set_server`变量的问题，BatchTool可以加上`-connParam "useUnicode=true&characterEncoding=utf-8"`

12. 使用`-con`指定了消费者线程数不生效

**原因**：消费者线程数会设置为`-con`和CPU核数的最大值

**解决**：当前可通过`-fcon`参数强制设置消费者线程数；后续可能会修改这个行为

13. 导出视图的相关问题

**原因**：BatchTool 在 v1.4.0 以前的版本，在导出整库时会默认导出所有表和视图的数据，无法控制是否导出视图

**解决**：自v1.4.0开始，可指定参数`-withView true`来导出视图数据
2 changes: 1 addition & 1 deletion batch-tool/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@

<groupId>com.alibaba.polardbx</groupId>
<artifactId>batch-tool</artifactId>
<version>1.3.7</version>
<version>1.4.0</version>

<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
Expand Down
3 changes: 3 additions & 0 deletions batch-tool/src/main/java/BatchTool.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -74,6 +74,9 @@ public void doBatchOp(BaseOperateCommand command, DataSourceConfig dataSourceCon
} finally {
commandExecutor.close();
}
if (commandExecutor.hasFatalException()) {
throw new RuntimeException("Fatal exception occurred during batch operation.");
}
}

private void addHooks() {
Expand Down
2 changes: 1 addition & 1 deletion batch-tool/src/main/java/BatchToolLauncher.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@ public static void main(String[] args) {
handleCmd(commandLine);
} catch (Throwable e) {
// 主线程异常
logger.error(e.getMessage(), e);
logger.error(e.getMessage());
System.exit(1);
}
}
Expand Down
43 changes: 39 additions & 4 deletions batch-tool/src/main/java/cmd/CommandUtil.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -118,6 +118,7 @@
import static cmd.FlagOption.ARG_SHORT_WITH_HEADER;
import static cmd.FlagOption.ARG_SHORT_WITH_LAST_SEP;
import static cmd.FlagOption.ARG_TRIM_RIGHT;
import static cmd.FlagOption.ARG_WITH_VIEW;

/**
* 从命令行输入解析配置
Expand Down Expand Up @@ -239,7 +240,10 @@ private static int getMinConnNum(ConfigResult result) {
if (result.hasOption(ARG_SHORT_MIN_CONN_NUM)) {
return Integer.parseInt(result.getOptionValue(ARG_SHORT_MIN_CONN_NUM));
} else {
return DatasourceConstant.MIN_CONN_NUM;
int pro = getProducerParallelism(result);
int con = getConsumerParallelism(result);
int maxParallelism = Math.max(pro, con);
return Math.min(maxParallelism + 1, DatasourceConstant.MIN_CONN_NUM);
}
}

Expand Down Expand Up @@ -332,23 +336,31 @@ private static BaseOperateCommand parseImportCommand(ConfigResult result) {
}

private static BaseOperateCommand parseDeleteCommand(ConfigResult result) {
requireOnlyOneArg(result, ARG_SHORT_FROM_FILE, ARG_SHORT_DIRECTORY);
requireOnlyOneArg(result, ARG_SHORT_FROM_FILE, ARG_SHORT_DIRECTORY, ARG_SHORT_BENCHMARK);

ProducerExecutionContext producerExecutionContext = new ProducerExecutionContext();
ConsumerExecutionContext consumerExecutionContext = new ConsumerExecutionContext();
configureCommonContext(result, producerExecutionContext, consumerExecutionContext);

if (producerExecutionContext.getBenchmarkMode() == BenchmarkMode.TPCH) {
setUpdateBatchSize(result);
}

consumerExecutionContext.setWhereCondition(getWhereCondition(result));
return new DeleteCommand(getDbName(result), producerExecutionContext, consumerExecutionContext);
}

private static BaseOperateCommand parseUpdateCommand(ConfigResult result) {
requireOnlyOneArg(result, ARG_SHORT_FROM_FILE, ARG_SHORT_DIRECTORY);
requireOnlyOneArg(result, ARG_SHORT_FROM_FILE, ARG_SHORT_DIRECTORY, ARG_SHORT_BENCHMARK);

ProducerExecutionContext producerExecutionContext = new ProducerExecutionContext();
ConsumerExecutionContext consumerExecutionContext = new ConsumerExecutionContext();
configureCommonContext(result, producerExecutionContext, consumerExecutionContext);

if (producerExecutionContext.getBenchmarkMode() == BenchmarkMode.TPCH) {
setUpdateBatchSize(result);
}

consumerExecutionContext.setWhereCondition(getWhereCondition(result));
consumerExecutionContext.setFuncSqlForUpdateEnabled(getFuncEnabled(result));
return new UpdateCommand(getDbName(result), producerExecutionContext, consumerExecutionContext);
Expand Down Expand Up @@ -403,6 +415,10 @@ private static boolean getWithLastSep(ConfigResult result) {
return result.getBooleanFlag(ARG_SHORT_WITH_LAST_SEP);
}

private static boolean getWithView(ConfigResult result) {
return result.getBooleanFlag(ARG_WITH_VIEW);
}

private static boolean getEmptyAsNull(ConfigResult result) {
return result.getBooleanFlag(ARG_EMPTY_AS_NULL);
}
Expand Down Expand Up @@ -439,6 +455,7 @@ private static ExportCommand parseExportCommand(ConfigResult result) {
exportConfig.setParallelism(getProducerParallelism(result));
exportConfig.setQuoteEncloseMode(getQuoteEncloseMode(result));
exportConfig.setWithLastSep(getWithLastSep(result));
exportConfig.setWithView(getWithView(result));
setDir(result, exportConfig);
setFilenamePrefix(result, exportConfig);
setFileNum(result, exportConfig);
Expand Down Expand Up @@ -553,6 +570,12 @@ private static void configureGlobalVar(ConfigResult result) {
setPerfMode(result);
}

private static void setUpdateBatchSize(ConfigResult result) {
if (result.hasOption(ARG_SHORT_BATCH_SIZE)) {
GlobalVar.setTpchUpdateBatchSize(Integer.parseInt(result.getOptionValue(ARG_SHORT_BATCH_SIZE)));
}
}

/**
* 配置生产者
*/
Expand All @@ -571,6 +594,7 @@ private static void configureProducerContext(ConfigResult result,
producerExecutionContext.setParallelism(getProducerParallelism(result));
producerExecutionContext.setReadBlockSizeInMb(getReadBlockSizeInMb(result));
producerExecutionContext.setWithHeader(getWithHeader(result));
producerExecutionContext.setWithView(getWithView(result));
producerExecutionContext.setCompressMode(getCompressMode(result));
producerExecutionContext.setEncryptionConfig(getEncryptionConfig(result));
producerExecutionContext.setFileFormat(getFileFormat(result));
Expand All @@ -579,6 +603,7 @@ private static void configureProducerContext(ConfigResult result,
producerExecutionContext.setQuoteEncloseMode(getQuoteEncloseMode(result));
producerExecutionContext.setTrimRight(getTrimRight(result));
producerExecutionContext.setBenchmarkMode(getBenchmarkMode(result));
producerExecutionContext.setBenchmarkRound(getBenchmarkRound(result));
producerExecutionContext.setScale(getScale(result));

producerExecutionContext.validate();
Expand All @@ -603,6 +628,7 @@ private static void configureConsumerContext(ConfigResult result,
consumerExecutionContext.setTpsLimit(getTpsLimit(result));
consumerExecutionContext.setUseColumns(getUseColumns(result));
consumerExecutionContext.setEmptyStrAsNull(getEmptyAsNull(result));
consumerExecutionContext.setMaxRetry(getMaxErrorCount(result));

consumerExecutionContext.validate();
}
Expand Down Expand Up @@ -671,6 +697,14 @@ private static BenchmarkMode getBenchmarkMode(ConfigResult result) {
}
}

private static int getBenchmarkRound(ConfigResult result) {
if (result.hasOption(ARG_SHORT_FILE_NUM)) {
return Integer.parseInt(result.getOptionValue(ARG_SHORT_FILE_NUM));
} else {
return 0;
}
}

private static int getScale(ConfigResult result) {
if (result.hasOption(ARG_SHORT_SCALE)) {
return Integer.parseInt(result.getOptionValue(ARG_SHORT_SCALE));
Expand Down Expand Up @@ -777,7 +811,8 @@ private static void setGlobalDdlConfig(ConfigResult result) {

private static int getMaxErrorCount(ConfigResult result) {
if (result.hasOption(ARG_SHORT_MAX_ERROR)) {
return Integer.parseInt(result.getOptionValue(ARG_SHORT_MAX_ERROR));
int maxError = Integer.parseInt(result.getOptionValue(ARG_SHORT_MAX_ERROR));
return Math.max(0, maxError);
}
return ConfigConstant.DEFAULT_MAX_ERROR_COUNT;
}
Expand Down
47 changes: 23 additions & 24 deletions batch-tool/src/main/java/cmd/ConfigArgOption.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,26 +17,6 @@
package cmd;

public class ConfigArgOption {
protected final String argShort;
protected final String argLong;
protected final String desc;
protected final String argName;

protected ConfigArgOption(String argShort, String argLong, String desc, String argName) {
this.argShort = argShort;
this.argLong = argLong;
this.desc = desc;
this.argName = argName;
}

private static ConfigArgOption of(String argShort, String argLong, String desc) {
return new ConfigArgOption(argShort, argLong, desc, null);
}

private static ConfigArgOption of(String argShort, String argLong, String desc, String argName) {
return new ConfigArgOption(argShort, argLong, desc, argName);
}

public static final ConfigArgOption ARG_SHORT_HELP =
of("help", "help", "Help message.");
public static final ConfigArgOption ARG_SHORT_VERSION =
Expand Down Expand Up @@ -72,7 +52,7 @@ private static ConfigArgOption of(String argShort, String argLong, String desc,
public static final ConfigArgOption ARG_SHORT_LINE =
of("L", "line", "Max line limit of one single export file.", "line count");
public static final ConfigArgOption ARG_SHORT_FILE_NUM =
of("F", "filenum", "Fixed number of exported files.", "file count");
of("F", "filenum", "Fixed number of exported files, or benchmark round.", "count");
public static final ConfigArgOption ARG_SHORT_HISTORY_FILE =
of("H", "historyFile", "History file name.", "filepath");
public static final ConfigArgOption ARG_SHORT_WHERE =
Expand All @@ -81,13 +61,13 @@ private static ConfigArgOption of(String argShort, String argLong, String desc,
of("dir", "directory", "Directory path including files to import.", "directory path");
public static final ConfigArgOption ARG_SHORT_CHARSET =
of("cs", "charset", "The charset of files.", "charset");
public static final ConfigArgOption ARG_SHORT_PRODUCER =
public static final ConfigArgOption ARG_SHORT_PRODUCER =
of("pro", "producer", "Configure number of producer threads (export / import).", "producer count");
public static final ConfigArgOption ARG_SHORT_CONSUMER =
of("con", "consumer", "Configure number of consumer threads.", "consumer count");
public static final ConfigArgOption ARG_SHORT_FORCE_CONSUMER =
of("fcon", "forceConsumer", "Configure if allow force consumer parallelism.", "parallelism");
public static final ConfigArgOption ARG_SHORT_MAX_CONN_NUM =
public static final ConfigArgOption ARG_SHORT_MAX_CONN_NUM =
of("maxConn", "maxConnection", "Max connection count (druid).", "max connection");
public static final ConfigArgOption ARG_SHORT_MAX_WAIT =
of("maxWait", "connMaxWait", "Max wait time when getting a connection.", "wait time(ms)");
Expand All @@ -103,7 +83,7 @@ private static ConfigArgOption of(String argShort, String argLong, String desc,
of("readsize", "readSize", "Read block size.", "size(MB)");
public static final ConfigArgOption ARG_SHORT_RING_BUFFER_SIZE =
of("ringsize", "ringSize", "Ring buffer size.", "size (power of 2)");
public static final ConfigArgOption ARG_SHORT_QUOTE_ENCLOSE_MODE =
public static final ConfigArgOption ARG_SHORT_QUOTE_ENCLOSE_MODE =
of("quote", "quoteMode",
"The mode of how field values are enclosed by double-quotes when exporting table (default FORCE).",
"AUTO | FORCE | NONE");
Expand Down Expand Up @@ -133,6 +113,25 @@ private static ConfigArgOption of(String argShort, String argLong, String desc,
public static final ConfigArgOption ARG_SHORT_SCALE =
of("scale", "scale", "The size scale benchmark data (GB for tpch).", "size");

protected final String argShort;
protected final String argLong;
protected final String desc;
protected final String argName;
protected ConfigArgOption(String argShort, String argLong, String desc, String argName) {
this.argShort = argShort;
this.argLong = argLong;
this.desc = desc;
this.argName = argName;
}

private static ConfigArgOption of(String argShort, String argLong, String desc) {
return new ConfigArgOption(argShort, argLong, desc, null);
}

private static ConfigArgOption of(String argShort, String argLong, String desc, String argName) {
return new ConfigArgOption(argShort, argLong, desc, argName);
}

public boolean hasArg() {
return argName != null;
}
Expand Down
2 changes: 2 additions & 0 deletions batch-tool/src/main/java/cmd/FlagOption.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -70,4 +70,6 @@ private static FlagOption of(String argShort, String argLong, String desc, Boole
of("dropTableIfExists", "dropTableIfExists",
"Add 'drop table if exists xxx' when exporting DDL (default false).",
false);
public static final FlagOption ARG_WITH_VIEW =
of("withView", "withView", "Export views into files, or (default false).", false);
}
1 change: 0 additions & 1 deletion batch-tool/src/main/java/datasource/DataSourceConfig.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -210,7 +210,6 @@ public String toString() {
", port='" + port + '\'' +
", dbName='" + dbName + '\'' +
", username='" + username + '\'' +
", password='" + password + '\'' +
", maxConnectionNum=" + maxConnectionNum +
", minConnectionNum=" + minConnectionNum +
", loadBalanceEnabled=" + loadBalanceEnabled +
Expand Down
5 changes: 4 additions & 1 deletion batch-tool/src/main/java/exec/BaseExecutor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -89,7 +89,6 @@ protected void checkTableExists(List<String> tableNames) {
throw new RuntimeException(String.format("Table [%s] does not exist", tableName));
}
} catch (SQLException | DatabaseException e) {
e.printStackTrace();
throw new RuntimeException(e.getMessage());
}
}
Expand Down Expand Up @@ -339,6 +338,10 @@ public void close() {

}

public boolean hasFatalException() {
return false;
}

protected String getSchemaName() {
return dataSourceConfig.getDbName();
}
Expand Down
Loading

0 comments on commit 6feabb0

Please sign in to comment.