diff --git a/batch-tool/docs/usage-details.md b/batch-tool/docs/usage-details.md index a5b0361..ac8baa3 100644 --- a/batch-tool/docs/usage-details.md +++ b/batch-tool/docs/usage-details.md @@ -252,3 +252,15 @@ mask: >- **原因**:BatchTool 在 v1.4.0 以前的版本,在导出整库时会默认导出所有表和视图的数据,无法控制是否导出视图 **解决**:自v1.4.0开始,可指定参数`-withView true`来导出视图数据 + +14. 指定目录导入时报错:`No filename with suffix starts with table name...` 或者 `No filename matches table name...` + + **原因**:BatchTool 为了适配自身整库导出的文件名规则,要求文件名必须以表名开头,且文件名后缀满足一定规则(如分区表的分区下标); + 因此对于包含了自定义文件名导出的目录时,BatchTool 无法对,进而会报错 + + **解决**:自v1.4.1开始,导入时可指定参数`--prefix "${文件名前缀}"`来指定目录里与导入表匹配的文件名前缀 + +15. 导出 PolarDB-X 某张表的指定物理分片 + + **解决**:例如某张表有128个物理分片,想导出第0号分片至第63号分片; + 自v1.4.1开始,可指定参数`-part 0:63`来导出第0号分片至第63号分片 \ No newline at end of file diff --git a/batch-tool/src/main/java/model/config/GlobalVar.java b/batch-tool/src/main/java/model/config/GlobalVar.java index 21fc63d..62c3c1d 100644 --- a/batch-tool/src/main/java/model/config/GlobalVar.java +++ b/batch-tool/src/main/java/model/config/GlobalVar.java @@ -56,6 +56,11 @@ public class GlobalVar { */ public static int TPCH_UPDATE_INSERT_BATCH_NUM = 20; + /** + * max line count for single csv row + */ + public static int MAX_CSV_MULTI_LINE = 500; + public static void setTpchUpdateBatchSize(int batchSize) { if (batchSize >= BaseOrderLineUpdateGenerator.SCALE_BASE) { throw new IllegalArgumentException( diff --git a/batch-tool/src/main/java/worker/common/reader/CsvReader.java b/batch-tool/src/main/java/worker/common/reader/CsvReader.java index e78f477..cb1272f 100644 --- a/batch-tool/src/main/java/worker/common/reader/CsvReader.java +++ b/batch-tool/src/main/java/worker/common/reader/CsvReader.java @@ -24,6 +24,7 @@ import com.opencsv.exceptions.CsvValidationException; import model.ProducerExecutionContext; import model.config.ConfigConstant; +import model.config.GlobalVar; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import util.IOUtil; @@ -54,7 +55,7 @@ public CsvReader(ProducerExecutionContext context, try { this.reader = new CSVReaderBuilder(new InputStreamReader( new FileInputStream(fileList.get(fileIndex).getAbsolutePath()), context.getCharset())) - .withCSVParser(parser).build(); + .withCSVParser(parser).withMultilineLimit(GlobalVar.MAX_CSV_MULTI_LINE).build(); } catch (FileNotFoundException e) { throw new RuntimeException(e.getMessage()); }