e2pg: expose concurrency and batch_size in config

indexsupply · Nov 7, 2023 · 3eff00d · 3eff00d
1 parent 5f53f46
commit 3eff00d
Showing 1 changed file with 9 additions and 7 deletions.
diff --git a/e2pg/e2pg.go b/e2pg/e2pg.go
@@ -1054,7 +1054,7 @@ func loadTasks(ctx context.Context, pgp *pgxpool.Pool, conf Config) ([]*Task, er
 			WithSourceConfig(sc),
 			WithPG(pgp),
 			WithRange(sc.Start, sc.Stop),
-			WithConcurrency(1, 100),
+			WithConcurrency(max(1, sc.Concurrenty), max(1, sc.BatchSize)),
 			WithDestinations(dests[sc.Name]...),
 		))
 		if len(destsBF[sc.Name]) == 0 {
@@ -1065,7 +1065,7 @@ func loadTasks(ctx context.Context, pgp *pgxpool.Pool, conf Config) ([]*Task, er
 			WithSourceConfig(sc),
 			WithPG(pgp),
 			WithRange(startBF[sc.Name], 0),
-			WithConcurrency(10, 100),
+			WithConcurrency(max(1, sc.Concurrenty), max(1, sc.BatchSize)),
 			WithDestinations(destsBF[sc.Name]...),
 		))
 	}
@@ -1146,11 +1146,13 @@ func SourceConfigs(ctx context.Context, pgp *pgxpool.Pool) ([]SourceConfig, erro
 }
 
 type SourceConfig struct {
-	Name    string `json:"name"`
-	ChainID uint64 `json:"chain_id"`
-	URL     string `json:"url"`
-	Start   uint64 `json:"start"`
-	Stop    uint64 `json:"stop"`
+	Name        string `json:"name"`
+	ChainID     uint64 `json:"chain_id"`
+	URL         string `json:"url"`
+	Start       uint64 `json:"start"`
+	Stop        uint64 `json:"stop"`
+	Concurrenty int    `json:"concurrency"`
+	BatchSize   int    `json:"batch_size"`
 }
 
 type Compiled struct {