From bfbb226841016bb00f8ca0606b6023d35aab9ea6 Mon Sep 17 00:00:00 2001 From: F-ca7 <627955292@qq.com> Date: Wed, 7 Sep 2022 16:12:48 +0800 Subject: [PATCH] fix command option and update README --- batch-tool/README.md | 169 +++++++++--------- batch-tool/docs/usage-details.md | 2 +- batch-tool/src/main/java/cmd/CommandUtil.java | 11 +- 3 files changed, 90 insertions(+), 92 deletions(-) diff --git a/batch-tool/README.md b/batch-tool/README.md index 555fe65..1ac4cff 100644 --- a/batch-tool/README.md +++ b/batch-tool/README.md @@ -10,92 +10,87 @@ Batch Tool工具是专为 PolarDB-X数据库提供数据导入导出服务的工 ## 快速上手 常见场景与问题排查可参考文档 [usage-details](docs/usage-details.md)。 ### 参数介绍 -命令行用法： +命令行用法：`java -jar batch-tool.jar --help` ``` - usage: BatchTool [-batchsize ] [-con ] [-cs - ] [-D ] [-dir ] [-f ] [-F - ] [-fcon ] [-func] [-h ] [-H - ] [-header] [-help] [-i] [-in] [-initSqls ] - [-L ] [-lastSep] [-lb] [-local] [-maxConn ] [-maxWait - ] [-minConn ] [-noesc] [-O ] [-o - ] [-OC ] [-p ] [-P ] - [-para] [-param ] [-pre ] [-pro ] - [-quote ] [-readsize ] [-rfonly] [-ringsize - ] [-s ] [-t ] [-tps ] [-u ] [-v] - [-w ] - -batchsize,--batchSize Batch size of emitted - tuples. - -con,--consumer Configure number of - consumer threads. - -cs,--charset Define charset of files. - -D,--database Database to use. - -dir,--dir Directory path including - files to import. - -f,--from Source file(s), separated - by ; . - -F,--filenum Fixed number of exported - files. - -fcon,--force consumer Configure if allow force - consumer parallelism. - -func,--sqlfunc Use sql function to update. - -h,--host Connect to host. - -H,--historyFile Configure of historyfile - name. - -header,--header Whether the header line is - column names. - -help,--help Help message. - -i,--ignoreandresume Flag of insert ignore and - resume breakpoint. - -in,--wherein Using where ... in (...) - -initSqls,--initSqls Connection init sqls. - -L,--line Max line limit of exported - files. - -lastSep,--withLastSep Whether line ends with - separator. - -lb,--loadbalance If using load balance. - -local,--localmerge o local merge sort. - -maxConn,--maxConnection Max connection number - limit. - -maxWait,--connMaxWait Max wait time(ms) when - getting a connection. - -minConn,--minConnection Mim connection number - limit. - -noesc,--noescape Don't escape values. - -O,--orderby asc or desc. - -o,--operation Batch operation type: - export / import / delete / - update. - -OC,--orderCol col1;col2;col3. - -p,--password Password to use when - connecting to server. - -P,--port Port number to use for - connection. - -para,--paraMerge Using parallel merge when - doing order by export. - -param,--connParam Connection params - -pre,--prefix Export file name prefix. - -pro,--producer Configure number of - producer threads (export / - import). - -quote,--quoteMode The mode of how field - values are enclosed by - double-quotes when - exporting table. Default - value is auto. - -readsize,--readSize Read block size in MB. - -rfonly,--rfonly Only read and process file, - no sql execution. - -ringsize,--ringBufferSize Ring buffer size. - -s,--sep Separator between fields - (delimiter). - -t,--table
Target table. - -tps,--tpsLimit Configure of tps limit, - default -1: no limit. - -u,--user User for login. - -v,--version Show version - -w,--where Where condition: col1>99 - AND col2<100 ... + usage: BatchTool [-batchsize ] [-col ] [-comp ] [-con ] + [-config ] [-cs ] [-D ] [-DDL ] [-dir ] [-encrypt ] [-error ] [-f ] [-F ] [-fcon ] [-format ] [-func ] [-h + ] [-H ] [-header ] [-help] [-i ] [-in ] + [-initSqls ] [-key ] [-L ] [-lastSep ] [-lb ] [-local ] [-mask ] [-maxConn ] [-maxWait + ] [-minConn ] [-noEsc ] [-o ] [-O ] + [-OC ] [-p ] [-P ] [-para ] [-param + ] [-perf ] [-pre ] [-pro ] [-quote ] [-readsize ] [-rfonly ] [-ringsize ] [-s + ] [-sharding ] [-t ] [-tps ] [-u + ] [-v] [-w ] + -batchsize,--batchSize Batch size of insert. + -col,--columns Target columns for export. + -comp,--compress Export or import compressed file (default NONE). + -con,--consumer Configure number of consumer threads. + -config,--configFile Use yaml config file. + -cs,--charset The charset of files. + -D,--database Database name. + -DDL,--DDL Export or import with DDL sql mode (default NONE). + -dir,--directory Directory path including files to import. + -encrypt,--encrypt Export or import with encrypted file (default NONE). + -error,--maxError Max error count threshold, program exits when the + limit is exceeded. + -f,--file Source file(s). + -F,--filenum Fixed number of exported files. + -fcon,--forceConsumer Configure if allow force consumer parallelism. + -format,--fileFormat File format (default NONE). + -func,--sqlFunc Use sql function to update (default false). + -h,--host Host of database. + -H,--historyFile History file name. + -header,--header Whether the header line is column names (default + false). + -help,--help Help message. + -i,--ignore Flag of insert ignore and resume breakpoint (default + false). + -in,--whereIn Using where cols in (values). + -initSqls,--initSqls Connection init sqls (druid). + -key,--secretKey Secret key used during encryption. + -L,--line Max line limit of one single export file. + -lastSep,--withLastSep Whether line ends with separator (default false). + -lb,--loadbalance Use jdbc load balance, filling the arg in $host like + 'host1:port1,host2:port2' (default false). + -local,--localMerge Use local merge sort (default false). + -mask,--mask Masking sensitive columns while exporting data. + -maxConn,--maxConnection Max connection count (druid). + -maxWait,--connMaxWait Max wait time when getting a connection. + -minConn,--minConnection Min connection count (druid). + -noEsc,--noEscape Do not escape value for sql (default false). + -o,--operation Batch operation type: export / import / delete / + update. + -O,--orderby Order by type: asc / desc. + -OC,--orderCol Ordered column names. + -p,--password Password of user. + -P,--port Port number of database. + -para,--paraMerge Use parallel merge when doing order by export + (default false). + -param,--connParam Jdbc connection params. + -perf,--perfMode Use performance mode at the sacrifice of compatibility + (default false). + -pre,--prefix Export file name prefix. + -pro,--producer Configure number of producer threads (export / + import). + -quote,--quoteMode The mode of how field values are enclosed by + double-quotes when exporting table (default AUTO). + -readsize,--readSize Read block size. + -rfonly,--readFileOnly Only read and process file, no sql execution (default + false). + -ringsize,--ringSize Ring buffer size. + -s,--sep Separator between fields (delimiter). + -sharding,--sharding Whether enable sharding mode (default value depends on + operation). + -t,--table Target table. + -tps,--tpsLimit Configure of tps limit (default -1: no limit). + -u,--user User for login. + -v,--version Show batch-tool version. + -w,--where Where condition: col1>99 AND col2<100 ... ``` 命令主要分别为两个类别： @@ -110,6 +105,8 @@ Batch Tool工具是专为 PolarDB-X数据库提供数据导入导出服务的工 - 文件数量、文件行数等导出配置 - insert ingore、断点续传等导入配置 - where、order by等sql条件 + - 压缩算法、加密算法、脱敏算法 + - 文件格式：csv、excel、txt等 - 批处理性能参数 - 生产者、消费者并行度设置 - ringBuffer缓冲区、批数量、读取文件块等大小设置 @@ -161,8 +158,8 @@ Batch Tool工具是专为 PolarDB-X数据库提供数据导入导出服务的工 - [x] 对接新分区表 - [ ] 调优实践 - [x] 指定字段（包括顺序）的导入导出 -- [ ] 简单的数据清洗，如trim尾部空格、日期时间格式等 -- [ ] 基于SQL函数的数据清洗，以及AES加解密函数调用 +- [ ] 简单的数据清洗，如：trim尾部空格、日期时间格式等 +- [x] 数据脱敏功能，如：掩码、哈希、加密、取整等 - [ ] 可视化监控 - [x] 错误情况下的断点记录（精确到行/块） - [x] 限流功能 diff --git a/batch-tool/docs/usage-details.md b/batch-tool/docs/usage-details.md index 424d321..05461b2 100644 --- a/batch-tool/docs/usage-details.md +++ b/batch-tool/docs/usage-details.md @@ -151,5 +151,5 @@ mask: >- ``` > ^A 为 \x01 的Caret notation - **解决**：输入`-s $'\x01'` 即可。 + **解决**：输入`-s $'\x01'` 即可。 > 暂时无法处理NULL字符(`\x00`)作为分隔符，可以通过修改源代码解决。 \ No newline at end of file diff --git a/batch-tool/src/main/java/cmd/CommandUtil.java b/batch-tool/src/main/java/cmd/CommandUtil.java index fb8ef80..d9e5f3e 100644 --- a/batch-tool/src/main/java/cmd/CommandUtil.java +++ b/batch-tool/src/main/java/cmd/CommandUtil.java @@ -120,16 +120,17 @@ public class CommandUtil { static { formatter.setWidth(110); - addCommandOptions(); + addCommandOptions(ConfigArgOption.class); + addCommandOptions(FlagOption.class); } - private static void addCommandOptions() { - Field[] fields = ConfigArgOption.class.getFields(); + private static void addCommandOptions(Class clazz) { + Field[] fields = clazz.getFields(); try { for (Field field : fields) { if (Modifier.isStatic(field.getModifiers()) - && field.getType() == ConfigArgOption.class) { - ConfigArgOption option = (ConfigArgOption) field.get(ConfigArgOption.class); + && field.getType() == clazz) { + ConfigArgOption option = (ConfigArgOption) field.get(clazz); addConfigOption(option); } }