Add possibility to remove indices before bulk insertion

db/drop_indices.sql

@@ -0,0 +1,7 @@
+-- Remove indices before bulk insertion
+DROP INDEX IF EXISTS data_timestamp_index,
+                     data_timeseries_index,
+                     nonscalar_data_timestamp_index,
+                     nonscalar_data_timeseries_index,
+                     old_flags_obtime_index,
+                     old_flags_timeseries_index;

db/flags.sql

@@ -11,7 +11,6 @@ CREATE TABLE IF NOT EXISTS flags.kvdata (
     cfailed INT4 NULL,
     CONSTRAINT unique_kvdata_timeseries_obstime UNIQUE (timeseries, obstime)
 );
-
 CREATE INDEX IF NOT EXISTS kvdata_obtime_index ON flags.kvdata (obstime);
 CREATE INDEX IF NOT EXISTS kvdata_timeseries_index ON flags.kvdata USING HASH (timeseries); 
 
@@ -22,8 +21,7 @@ CREATE TABLE IF NOT EXISTS flags.old_databases (
     controlinfo TEXT NULL,
     useinfo TEXT NULL,
     cfailed TEXT NULL ,
-    CONSTRAINT unique_kdvh_timeseries_obstime UNIQUE (timeseries, obstime)
+    CONSTRAINT unique_old_flags_timeseries_obstime UNIQUE (timeseries, obstime)
 );
-
-CREATE INDEX IF NOT EXISTS kdvh_obtime_index ON flags.old_databases (obstime);
-CREATE INDEX IF NOT EXISTS kdvh_timeseries_index ON flags.old_databases USING HASH (timeseries); 
+CREATE INDEX IF NOT EXISTS old_flags_obtime_index ON flags.old_databases (obstime);
+CREATE INDEX IF NOT EXISTS old_flags_timeseries_index ON flags.old_databases USING HASH (timeseries); 

migrations/kdvh/import/main.go

@@ -24,6 +24,7 @@ type Config struct {
 	HasHeader bool     `long:"header" description:"Add this flag if the dumped files have a header row"`
 	Skip      string   `long:"skip" choice:"data" choice:"flags" description:"Skip import of data or flags"`
 	Email     []string `long:"email" delimiter:"," description:"Optional comma separated list of email addresses used to notify if the program crashed"`
+	Reindex   bool     `long:"reindex" description:"Drops PG indices before insertion. Might improve performance"`
 }
 
 func (config *Config) Execute([]string) error {
@@ -45,6 +46,10 @@ func (config *Config) Execute([]string) error {
 	}
 	defer pool.Close()
 
+	if config.Reindex {
+		dropIndices(pool)
+	}
+
 	for _, table := range kdvh.Tables {
 		if len(config.Tables) > 0 && !slices.Contains(config.Tables, table.TableName) {
 			continue
@@ -61,5 +66,40 @@ func (config *Config) Execute([]string) error {
 		ImportTable(table, cache, pool, config)
 	}
 
+	if config.Reindex {
+		createIndices(pool)
+	}
+
 	return nil
 }
+
+func dropIndices(pool *pgxpool.Pool) {
+	fmt.Println("Dropping table indices...")
+
+	file, err := os.ReadFile("../db/drop_indices.sql")
+	if err != nil {
+		panic(err.Error())
+	}
+
+	_, err = pool.Exec(context.Background(), string(file))
+	if err != nil {
+		panic(err.Error())
+	}
+}
+
+func createIndices(pool *pgxpool.Pool) {
+	fmt.Println("Recreating table indices...")
+
+	files := []string{"../db/public.sql", "../db/flags.sql"}
+	for _, filename := range files {
+		file, err := os.ReadFile(filename)
+		if err != nil {
+			panic(err.Error())
+		}
+
+		_, err = pool.Exec(context.Background(), string(file))
+		if err != nil {
+			panic(err.Error())
+		}
+	}
+}