to #56171689 support retry when error occurs during importing

polardb · May 24, 2024 · f8115a1 · f8115a1
1 parent b96554f
commit f8115a1
Show file tree

Hide file tree

Showing 3 changed files with 25 additions and 3 deletions.
diff --git a/batch-tool/docs/usage-details.md b/batch-tool/docs/usage-details.md
@@ -263,4 +263,10 @@ mask: >-
 15. 导出 PolarDB-X 某张表的指定物理分片
 
    **解决**：例如某张表有128个物理分片，想导出第0号分片至第63号分片；
-   自v1.4.1开始，可指定参数`-part 0:63`来导出第0号分片至第63号分片
+   自v1.4.1开始，可指定参数`-part 0:63`来导出第0号分片至第63号分片
+
+16. 导入时，由于一些数据库侧的偶发报错，希望能自动重试
+
+   **原因**：BatchTool 默认情况下，导入失败会直接退出，不会自动重试
+
+   **解决**：自v1.4.1开始，可指定参数`-maxError 10`来指定最大错误重试次数为10次，目前暂不支持根据错误码来进行重试
diff --git a/batch-tool/src/main/java/cmd/ConfigArgOption.java b/batch-tool/src/main/java/cmd/ConfigArgOption.java
@@ -100,7 +100,7 @@ public class ConfigArgOption {
     public static final ConfigArgOption ARG_SHORT_FILE_FORMAT =
         of("format", "fileFormat", "File format (default NONE).", "NONE | TXT | CSV | XLS | XLSX");
     public static final ConfigArgOption ARG_SHORT_MAX_ERROR =
-        of("error", "maxError", "Max error count threshold, program exits when the limit is exceeded.",
+        of("maxError", "maxError", "Max error count threshold, program exits when the limit is exceeded.",
             "max error count");
     public static final ConfigArgOption ARG_SHORT_MASK =
         of("mask", "mask", "Masking sensitive columns while exporting data.", "Json format config");

diff --git a/batch-tool/src/main/java/worker/common/BaseDefaultConsumer.java b/batch-tool/src/main/java/worker/common/BaseDefaultConsumer.java
@@ -38,9 +38,11 @@ public abstract class BaseDefaultConsumer extends BaseWorkHandler {
 
     protected int estimateFieldCount = 16;
     protected final SqlStat sqlStat = new SqlStat();
+    protected int maxRetry = 0;
 
     protected void initLocalVars() {
         super.initLocalVars();
+        maxRetry = consumerContext.getMaxRetry();
     }
 
     @Override
@@ -107,7 +109,21 @@ protected void execSql(StringBuilder data) throws SQLException {
             stmt = conn.createStatement();
             sql = getSql(data);
             long startTime = System.nanoTime();
-            stmt.execute(sql);
+            for (int i = 0; i <= maxRetry; i++) {
+                try {
+                    stmt.execute(sql);
+                    break;
+                } catch (SQLException e) {
+                    logger.error("Error executing SQL (retry count: {}): {}",
+                        i, e.getMessage());
+                    // 如果达到最大重试次数，抛出异常
+                    if (i >= maxRetry) {
+                        throw e;
+                    }
+                    // 暂不添加延迟逻辑
+                }
+            }
+
             long endTime = System.nanoTime();
             sqlStat.addTimeNs(endTime - startTime);
         } catch (SQLException e) {