Merge pull request #2376 from gobitfly/BIDS-2258/improve-charts

(BIDS-2258) use chart_series for some consensus-charts aswell
gobitfly · Jul 4, 2023 · 72e0860 · 72e0860
2 parents 964a717 + 86304c9
commit 72e0860
Show file tree

Hide file tree

Showing 6 changed files with 402 additions and 353 deletions.
diff --git a/cmd/statistics/main.go b/cmd/statistics/main.go
@@ -25,20 +25,23 @@ type options struct {
 	statisticsValidatorToggle bool
 	statisticsResetColumns    string
 	statisticsChartToggle     bool
+	statisticsGraffitiToggle  bool
+	concurrencyTotal          uint64
+	concurrencyCl             uint64
 }
 
-var opt *options
+var opt = &options{}
 
 func main() {
-	configPath := flag.String("config", "", "Path to the config file")
-	statisticsDayToExport := flag.Int64("statistics.day", -1, "Day to export statistics (will export the day independent if it has been already exported or not")
-	statisticsDaysToExport := flag.String("statistics.days", "", "Days to export statistics (will export the day independent if it has been already exported or not")
-	// we can use a higher concurrency for cl as the querie is smaller then for the total stats
-	concurrencyTotal := flag.Uint64("concurrency.total", 10, "Concurrency to use when writing total rewards/performance postgres queries ")
-	concurrencyCl := flag.Uint64("concurrency.cl", 50, "Concurrency to use when writing cl postgres queries")
-	statisticsValidatorToggle := flag.Bool("validators.enabled", false, "Toggle exporting validator statistics")
-	statisticsResetColumns := flag.String("validators.reset", "", "validator_stats_status columns to reset. Comma separated. Use 'all' for complete resync.")
-	statisticsChartToggle := flag.Bool("charts.enabled", false, "Toggle exporting chart series")
+	flag.StringVar(&opt.configPath, "config", "", "Path to the config file")
+	flag.Int64Var(&opt.statisticsDayToExport, "statistics.day", -1, "Day to export statistics (will export the day independent if it has been already exported or not")
+	flag.StringVar(&opt.statisticsDaysToExport, "statistics.days", "", "Days to export statistics (will export the day independent if it has been already exported or not")
+	flag.BoolVar(&opt.statisticsValidatorToggle, "validators.enabled", false, "Toggle exporting validator statistics")
+	flag.StringVar(&opt.statisticsResetColumns, "validators.reset", "", "validator_stats_status columns to reset. Comma separated. Use 'all' for complete resync.")
+	flag.BoolVar(&opt.statisticsChartToggle, "charts.enabled", false, "Toggle exporting chart series")
+	flag.BoolVar(&opt.statisticsGraffitiToggle, "graffiti.enabled", false, "Toggle exporting graffiti statistics")
+	flag.Uint64Var(&opt.concurrencyTotal, "concurrency.total", 10, "Concurrency to use when writing total rewards/performance postgres queries")
+	flag.Uint64Var(&opt.concurrencyCl, "concurrency.cl", 50, "Concurrency to use when writing cl postgres queries")
 
 	versionFlag := flag.Bool("version", false, "Show version and exit")
 	flag.Parse()
@@ -48,28 +51,15 @@ func main() {
 		return
 	}
 
-	opt = &options{
-		configPath:                *configPath,
-		statisticsDayToExport:     *statisticsDayToExport,
-		statisticsDaysToExport:    *statisticsDaysToExport,
-		statisticsChartToggle:     *statisticsChartToggle,
-		statisticsResetColumns:    *statisticsResetColumns,
-		statisticsValidatorToggle: *statisticsValidatorToggle,
-	}
-
-	logrus.Printf("version: %v, config file path: %v", version.Version, *configPath)
+	logrus.Printf("version: %v, config file path: %v", version.Version, opt.configPath)
 	cfg := &types.Config{}
-	err := utils.ReadConfig(cfg, *configPath)
+	err := utils.ReadConfig(cfg, opt.configPath)
 
 	if err != nil {
 		logrus.Fatalf("error reading config file: %v", err)
 	}
 	utils.Config = cfg
 
-	if *statisticsChartToggle && utils.Config.Chain.Config.DepositChainID != 1 {
-		logrus.Infof("Execution charts are currently only available for mainnet")
-	}
-
 	db.MustInitDB(&types.DatabaseConfig{
 		Username:     cfg.WriterDatabase.Username,
 		Password:     cfg.WriterDatabase.Password,
@@ -127,8 +117,8 @@ func main() {
 		logrus.Fatalf("No cache provider set. Please set TierdCacheProvider (example redis, bigtable)")
 	}
 
-	if *statisticsDaysToExport != "" {
-		s := strings.Split(*statisticsDaysToExport, "-")
+	if opt.statisticsDaysToExport != "" {
+		s := strings.Split(opt.statisticsDaysToExport, "-")
 		if len(s) < 2 {
 			logrus.Fatalf("invalid arg")
 		}
@@ -141,21 +131,21 @@ func main() {
 			utils.LogFatal(err, "error parsing last day of statisticsDaysToExport flag to uint", 0)
 		}
 
-		if *statisticsValidatorToggle {
+		if opt.statisticsValidatorToggle {
 			logrus.Infof("exporting validator statistics for days %v-%v", firstDay, lastDay)
 			for d := firstDay; d <= lastDay; d++ {
 
 				clearStatsStatusTable(d, opt.statisticsResetColumns)
 
-				err = db.WriteValidatorStatisticsForDay(uint64(d), *concurrencyTotal, *concurrencyCl)
+				err = db.WriteValidatorStatisticsForDay(uint64(d), opt.concurrencyTotal, opt.concurrencyCl)
 				if err != nil {
 					logrus.Errorf("error exporting stats for day %v: %v", d, err)
 					break
 				}
 			}
 		}
 
-		if *statisticsChartToggle && utils.Config.Chain.Config.DepositChainID == 1 {
+		if opt.statisticsChartToggle {
 			logrus.Infof("exporting chart series for days %v-%v", firstDay, lastDay)
 			for d := firstDay; d <= lastDay; d++ {
 				_, err = db.WriterDb.Exec("delete from chart_series_status where day = $1", d)
@@ -171,33 +161,50 @@ func main() {
 			}
 		}
 
+		if opt.statisticsGraffitiToggle {
+			for d := firstDay; d <= lastDay; d++ {
+				err = db.WriteGraffitiStatisticsForDay(int64(d))
+				if err != nil {
+					logrus.Errorf("error exporting graffiti-stats from day %v: %v", opt.statisticsDayToExport, err)
+					break
+				}
+			}
+		}
+
 		return
-	} else if *statisticsDayToExport >= 0 {
+	} else if opt.statisticsDayToExport >= 0 {
 
-		if *statisticsValidatorToggle {
-			clearStatsStatusTable(uint64(*statisticsDayToExport), opt.statisticsResetColumns)
+		if opt.statisticsValidatorToggle {
+			clearStatsStatusTable(uint64(opt.statisticsDayToExport), opt.statisticsResetColumns)
 
-			err = db.WriteValidatorStatisticsForDay(uint64(*statisticsDayToExport), *concurrencyTotal, *concurrencyCl)
+			err = db.WriteValidatorStatisticsForDay(uint64(opt.statisticsDayToExport), opt.concurrencyTotal, opt.concurrencyCl)
 			if err != nil {
-				logrus.Errorf("error exporting stats for day %v: %v", *statisticsDayToExport, err)
+				logrus.Errorf("error exporting stats for day %v: %v", opt.statisticsDayToExport, err)
 			}
 		}
 
-		if *statisticsChartToggle && utils.Config.Chain.Config.DepositChainID == 1 {
-			_, err = db.WriterDb.Exec("delete from chart_series_status where day = $1", *statisticsDayToExport)
+		if opt.statisticsChartToggle {
+			_, err = db.WriterDb.Exec("delete from chart_series_status where day = $1", opt.statisticsDayToExport)
 			if err != nil {
-				logrus.Fatalf("error resetting status for chart series status for day %v: %v", *statisticsDayToExport, err)
+				logrus.Fatalf("error resetting status for chart series status for day %v: %v", opt.statisticsDayToExport, err)
 			}
 
-			err = db.WriteChartSeriesForDay(int64(*statisticsDayToExport))
+			err = db.WriteChartSeriesForDay(int64(opt.statisticsDayToExport))
 			if err != nil {
-				logrus.Errorf("error exporting chart series from day %v: %v", *statisticsDayToExport, err)
+				logrus.Errorf("error exporting chart series from day %v: %v", opt.statisticsDayToExport, err)
+			}
+		}
+
+		if opt.statisticsGraffitiToggle {
+			err = db.WriteGraffitiStatisticsForDay(int64(opt.statisticsDayToExport))
+			if err != nil {
+				logrus.Errorf("error exporting chart series from day %v: %v", opt.statisticsDayToExport, err)
 			}
 		}
 		return
 	}
 
-	go statisticsLoop(*concurrencyTotal, *concurrencyCl)
+	go statisticsLoop(opt.concurrencyTotal, opt.concurrencyCl)
 
 	utils.WaitForCtrlC()
 
@@ -258,20 +265,32 @@ func statisticsLoop(concurrencyTotal uint64, concurrencyCl uint64) {
 			if lastExportedDayChart != 0 {
 				lastExportedDayChart++
 			}
-			if utils.Config.Chain.Config.DepositChainID == 1 {
-				logrus.Infof("Chart statistics: latest epoch is %v, previous day is %v, last exported day is %v", latestEpoch, previousDay, lastExportedDayChart)
-				if lastExportedDayChart <= previousDay || lastExportedDayChart == 0 {
-					for day := lastExportedDayChart; day <= previousDay; day++ {
-						err = db.WriteChartSeriesForDay(int64(day))
-						if err != nil {
-							logrus.Errorf("error exporting chart series from day %v: %v", day, err)
-							break
-						}
+			logrus.Infof("Chart statistics: latest epoch is %v, previous day is %v, last exported day is %v", latestEpoch, previousDay, lastExportedDayChart)
+			if lastExportedDayChart <= previousDay || lastExportedDayChart == 0 {
+				for day := lastExportedDayChart; day <= previousDay; day++ {
+					err = db.WriteChartSeriesForDay(int64(day))
+					if err != nil {
+						logrus.Errorf("error exporting chart series from day %v: %v", day, err)
+						break
 					}
 				}
 			}
 		}
 
+		if opt.statisticsGraffitiToggle {
+			var lastDay int64
+			err := db.WriterDb.Get(&lastDay, "select COALESCE(max(day), 0) from graffiti_stats")
+			if err != nil {
+				logrus.Errorf("error retreiving latest exported day from graffiti_stats: %v", err)
+			} else {
+				nextDay := lastDay + 1
+				err = db.WriteGraffitiStatisticsForDay(nextDay)
+				if err != nil {
+					logrus.Errorf("error exporting graffiti-stats for day %v: %v", nextDay, err)
+				}
+			}
+		}
+
 		services.ReportStatus("statistics", "Running", nil)
 		time.Sleep(time.Minute)
 	}

diff --git a/db/db.go b/db/db.go
@@ -2231,7 +2231,7 @@ func GetBlockNumber(slot uint64) (block uint64, err error) {
 func SaveChartSeriesPoint(date time.Time, indicator string, value any) error {
 	_, err := WriterDb.Exec(`INSERT INTO chart_series (time, indicator, value) VALUES($1, $2, $3) ON CONFLICT (time, indicator) DO UPDATE SET value = EXCLUDED.value`, date, indicator, value)
 	if err != nil {
-		return fmt.Errorf("error calculating NON_FAILED_TX_GAS_USAGE chart_series: %w", err)
+		return fmt.Errorf("error saving chart_series: %v: %w", indicator, err)
 	}
 	return err
 }

diff --git a/db/migrations/20230629144437_add_graffiti_stats_tables.sql b/db/migrations/20230629144437_add_graffiti_stats_tables.sql
@@ -0,0 +1,19 @@
+-- +goose Up
+-- +goose StatementBegin
+SELECT 'up SQL query - add graffiti_stats tables';
+CREATE TABLE IF NOT EXISTS
+    graffiti_stats (
+        day INTEGER NOT NULL,
+        graffiti BYTEA NOT NULL,
+        graffiti_text TEXT NOT NULL,
+        count INTEGER NOT NULL,
+        proposer_count INTEGER NOT NULL,
+        PRIMARY KEY (graffiti, day)
+    );
+-- +goose StatementEnd
+
+-- +goose Down
+-- +goose StatementBegin
+SELECT 'down SQL query - remove graffiti_stats tables';
+DROP TABLE IF EXISTS graffiti_stats;
+-- +goose StatementEnd
diff --git a/db/migrations/20230629144438_add_graffiti_stats_indices.sql b/db/migrations/20230629144438_add_graffiti_stats_indices.sql
@@ -0,0 +1,20 @@
+-- +goose NO TRANSACTION
+-- +goose Up
+SELECT 'up SQL query - add graffiti_stats indices';
+
+-- +goose StatementBegin
+CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_graffiti_stats_day ON graffiti_stats (day);
+-- +goose StatementEnd
+-- +goose StatementBegin
+CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_graffiti_stats_day_graffiti_text ON graffiti_stats USING gin (graffiti_text gin_trgm_ops);
+-- +goose StatementEnd
+
+-- +goose Down
+SELECT 'down SQL query - remove graffiti_stats indices';
+
+-- +goose StatementBegin
+DROP INDEX CONCURRENTLY IF EXISTS idx_graffiti_stats_day;
+-- +goose StatementEnd
+-- +goose StatementBegin
+DROP INDEX CONCURRENTLY IF EXISTS idx_graffiti_stats_day_graffiti_text;
+-- +goose StatementEnd