From e4e70cb5c514c660b294400ffd7c014f5ceb3e5f Mon Sep 17 00:00:00 2001 From: Mahendra Paipuri Date: Sat, 13 Apr 2024 22:50:23 +0200 Subject: [PATCH] feat: Add last_updated_at col in usage table * This can be used to clean up DB by identifying stale projects * This col is not sent in HTTP responses. Can be changed that in future if necessary Signed-off-by: Mahendra Paipuri --- pkg/api/db/db.go | 1 + .../000005_alter_usage_tables.down.sql | 1 + .../migrations/000005_alter_usage_tables.up.sql | 1 + pkg/api/http/server.go | 17 ++++++++++++----- pkg/api/models/models.go | 1 + 5 files changed, 16 insertions(+), 5 deletions(-) create mode 100644 pkg/api/db/migrations/000005_alter_usage_tables.down.sql create mode 100644 pkg/api/db/migrations/000005_alter_usage_tables.up.sql diff --git a/pkg/api/db/db.go b/pkg/api/db/db.go index 6d3fb186..80516e44 100644 --- a/pkg/api/db/db.go +++ b/pkg/api/db/db.go @@ -467,6 +467,7 @@ func (s *statsDB) execStatements(statements map[string]*sql.Stmt, units []models sql.Named(base.UsageDBTableStructFieldColNameMap["NumUnits"], unitIncr), sql.Named(base.UsageDBTableStructFieldColNameMap["Project"], unit.Project), sql.Named(base.UsageDBTableStructFieldColNameMap["Usr"], unit.Usr), + sql.Named(base.UsageDBTableStructFieldColNameMap["LastUpdatedAt"], time.Now().Format(base.DatetimeLayout)), sql.Named(base.UsageDBTableStructFieldColNameMap["TotalCPUBilling"], unit.TotalCPUBilling), sql.Named(base.UsageDBTableStructFieldColNameMap["TotalGPUBilling"], unit.TotalGPUBilling), sql.Named(base.UsageDBTableStructFieldColNameMap["TotalMiscBilling"], unit.TotalMiscBilling), diff --git a/pkg/api/db/migrations/000005_alter_usage_tables.down.sql b/pkg/api/db/migrations/000005_alter_usage_tables.down.sql new file mode 100644 index 00000000..657dc922 --- /dev/null +++ b/pkg/api/db/migrations/000005_alter_usage_tables.down.sql @@ -0,0 +1 @@ +ALTER TABLE usage DROP COLUMN last_updated_at; diff --git a/pkg/api/db/migrations/000005_alter_usage_tables.up.sql b/pkg/api/db/migrations/000005_alter_usage_tables.up.sql new file mode 100644 index 00000000..dd365196 --- /dev/null +++ b/pkg/api/db/migrations/000005_alter_usage_tables.up.sql @@ -0,0 +1 @@ +ALTER TABLE usage ADD COLUMN "last_updated_at" text; diff --git a/pkg/api/http/server.go b/pkg/api/http/server.go index a7ecb2eb..8643aa99 100644 --- a/pkg/api/http/server.go +++ b/pkg/api/http/server.go @@ -67,7 +67,7 @@ type Response struct { } var ( - aggUsageDBCols = make([]string, len(base.UsageDBTableColNames)+1) + aggUsageDBCols = make([]string, len(base.UsageDBTableColNames)) defaultQueryWindow = time.Duration(2 * time.Hour) // Two hours ) @@ -77,17 +77,24 @@ func init() { aggUsageDBCols[0] = "id" // Use SQL aggregate functions in query + j := 0 for i := 0; i < len(base.UsageDBTableColNames); i++ { col := base.UsageDBTableColNames[i] + // Ignore last_updated_at col + if slices.Contains([]string{"last_updated_at"}, col) { + continue + } + if strings.HasPrefix(col, "avg") { - aggUsageDBCols[i+1] = fmt.Sprintf("AVG(%[1]s) AS %[1]s", col) + aggUsageDBCols[j+1] = fmt.Sprintf("AVG(%[1]s) AS %[1]s", col) } else if strings.HasPrefix(col, "total") { - aggUsageDBCols[i+1] = fmt.Sprintf("SUM(%[1]s) AS %[1]s", col) + aggUsageDBCols[j+1] = fmt.Sprintf("SUM(%[1]s) AS %[1]s", col) } else if strings.HasPrefix(col, "num") { - aggUsageDBCols[i+1] = "COUNT(id) AS num_units" + aggUsageDBCols[j+1] = "COUNT(id) AS num_units" } else { - aggUsageDBCols[i+1] = col + aggUsageDBCols[j+1] = col } + j++ } } diff --git a/pkg/api/models/models.go b/pkg/api/models/models.go index dea212ac..06adb10b 100644 --- a/pkg/api/models/models.go +++ b/pkg/api/models/models.go @@ -74,6 +74,7 @@ type Usage struct { NumUnits int64 `json:"num_units" sql:"num_units" sqlitetype:"integer"` // Number of consumed units Project string `json:"project" sql:"project" sqlitetype:"text"` // Account in batch systems, Tenant in Openstack, Namespace in k8s Usr string `json:"usr" sql:"usr" sqlitetype:"text"` // Username + LastUpdatedAt string `json:"-" sql:"last_updated_at" sqlitetype:"text"` // Last updated time. It can be used to clean up DB TotalCPUBilling int64 `json:"total_cpu_billing" sql:"total_cpu_billing" sqlitetype:"integer"` // Total CPU billing for project TotalGPUBilling int64 `json:"total_gpu_billing" sql:"total_gpu_billing" sqlitetype:"integer"` // Total GPU billing for project TotalMiscBilling int64 `json:"total_misc_billing" sql:"total_misc_billing" sqlitetype:"integer"` // Total billing for project that are not in CPU and GPU billing