From b96554f3c7502d8dbbdb6faaefc40dceeb0e66f6 Mon Sep 17 00:00:00 2001
From: F-ca7 <yanzhi.fyz@alibaba-inc.com>
Date: Fri, 10 May 2024 15:18:14 +0800
Subject: [PATCH] to #56171689 add csv max multiline limit

---
 batch-tool/docs/usage-details.md                     | 12 ++++++++++++
 batch-tool/src/main/java/model/config/GlobalVar.java |  5 +++++
 .../main/java/worker/common/reader/CsvReader.java    |  3 ++-
 3 files changed, 19 insertions(+), 1 deletion(-)

diff --git a/batch-tool/docs/usage-details.md b/batch-tool/docs/usage-details.md
index a5b0361..ac8baa3 100644
--- a/batch-tool/docs/usage-details.md
+++ b/batch-tool/docs/usage-details.md
@@ -252,3 +252,15 @@ mask: >-
    **原因**：BatchTool 在 v1.4.0 以前的版本，在导出整库时会默认导出所有表和视图的数据，无法控制是否导出视图
    
    **解决**：自v1.4.0开始，可指定参数`-withView true`来导出视图数据
+
+14. 指定目录导入时报错：`No filename with suffix starts with table name...` 或者 `No filename matches table name...`
+
+   **原因**：BatchTool 为了适配自身整库导出的文件名规则，要求文件名必须以表名开头，且文件名后缀满足一定规则（如分区表的分区下标）；
+   因此对于包含了自定义文件名导出的目录时，BatchTool 无法对，进而会报错
+
+   **解决**：自v1.4.1开始，导入时可指定参数`--prefix "${文件名前缀}"`来指定目录里与导入表匹配的文件名前缀
+
+15. 导出 PolarDB-X 某张表的指定物理分片
+
+   **解决**：例如某张表有128个物理分片，想导出第0号分片至第63号分片；
+   自v1.4.1开始，可指定参数`-part 0:63`来导出第0号分片至第63号分片
\ No newline at end of file
diff --git a/batch-tool/src/main/java/model/config/GlobalVar.java b/batch-tool/src/main/java/model/config/GlobalVar.java
index 21fc63d..62c3c1d 100644
--- a/batch-tool/src/main/java/model/config/GlobalVar.java
+++ b/batch-tool/src/main/java/model/config/GlobalVar.java
@@ -56,6 +56,11 @@ public class GlobalVar {
      */
     public static int TPCH_UPDATE_INSERT_BATCH_NUM = 20;
 
+    /**
+     * max line count for single csv row
+     */
+    public static int MAX_CSV_MULTI_LINE = 500;
+
     public static void setTpchUpdateBatchSize(int batchSize) {
         if (batchSize >= BaseOrderLineUpdateGenerator.SCALE_BASE) {
             throw new IllegalArgumentException(
diff --git a/batch-tool/src/main/java/worker/common/reader/CsvReader.java b/batch-tool/src/main/java/worker/common/reader/CsvReader.java
index e78f477..cb1272f 100644
--- a/batch-tool/src/main/java/worker/common/reader/CsvReader.java
+++ b/batch-tool/src/main/java/worker/common/reader/CsvReader.java
@@ -24,6 +24,7 @@
 import com.opencsv.exceptions.CsvValidationException;
 import model.ProducerExecutionContext;
 import model.config.ConfigConstant;
+import model.config.GlobalVar;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import util.IOUtil;
@@ -54,7 +55,7 @@ public CsvReader(ProducerExecutionContext context,
         try {
             this.reader = new CSVReaderBuilder(new InputStreamReader(
                 new FileInputStream(fileList.get(fileIndex).getAbsolutePath()), context.getCharset()))
-                .withCSVParser(parser).build();
+                .withCSVParser(parser).withMultilineLimit(GlobalVar.MAX_CSV_MULTI_LINE).build();
         } catch (FileNotFoundException e) {
             throw new RuntimeException(e.getMessage());
         }