feat: Add last_updated_at col in usage table

* This can be used to clean up DB by identifying stale projects

* This col is not sent in HTTP responses. Can be changed that in future if necessary

Signed-off-by: Mahendra Paipuri <[email protected]>

pkg/api/db/db.go

@@ -467,6 +467,7 @@ func (s *statsDB) execStatements(statements map[string]*sql.Stmt, units []models
 			sql.Named(base.UsageDBTableStructFieldColNameMap["NumUnits"], unitIncr),
 			sql.Named(base.UsageDBTableStructFieldColNameMap["Project"], unit.Project),
 			sql.Named(base.UsageDBTableStructFieldColNameMap["Usr"], unit.Usr),
+			sql.Named(base.UsageDBTableStructFieldColNameMap["LastUpdatedAt"], time.Now().Format(base.DatetimeLayout)),
 			sql.Named(base.UsageDBTableStructFieldColNameMap["TotalCPUBilling"], unit.TotalCPUBilling),
 			sql.Named(base.UsageDBTableStructFieldColNameMap["TotalGPUBilling"], unit.TotalGPUBilling),
 			sql.Named(base.UsageDBTableStructFieldColNameMap["TotalMiscBilling"], unit.TotalMiscBilling),

pkg/api/db/migrations/000005_alter_usage_tables.down.sql

@@ -0,0 +1 @@
+ALTER TABLE usage DROP COLUMN last_updated_at;

pkg/api/db/migrations/000005_alter_usage_tables.up.sql

@@ -0,0 +1 @@
+ALTER TABLE usage ADD COLUMN "last_updated_at" text;

pkg/api/http/server.go

@@ -67,7 +67,7 @@ type Response struct {
 }
 
 var (
-	aggUsageDBCols     = make([]string, len(base.UsageDBTableColNames)+1)
+	aggUsageDBCols     = make([]string, len(base.UsageDBTableColNames))
 	defaultQueryWindow = time.Duration(2 * time.Hour) // Two hours
 )
 
@@ -77,17 +77,24 @@ func init() {
 	aggUsageDBCols[0] = "id"
 
 	// Use SQL aggregate functions in query
+	j := 0
 	for i := 0; i < len(base.UsageDBTableColNames); i++ {
 		col := base.UsageDBTableColNames[i]
+		// Ignore last_updated_at col
+		if slices.Contains([]string{"last_updated_at"}, col) {
+			continue
+		}
+
 		if strings.HasPrefix(col, "avg") {
-			aggUsageDBCols[i+1] = fmt.Sprintf("AVG(%[1]s) AS %[1]s", col)
+			aggUsageDBCols[j+1] = fmt.Sprintf("AVG(%[1]s) AS %[1]s", col)
 		} else if strings.HasPrefix(col, "total") {
-			aggUsageDBCols[i+1] = fmt.Sprintf("SUM(%[1]s) AS %[1]s", col)
+			aggUsageDBCols[j+1] = fmt.Sprintf("SUM(%[1]s) AS %[1]s", col)
 		} else if strings.HasPrefix(col, "num") {
-			aggUsageDBCols[i+1] = "COUNT(id) AS num_units"
+			aggUsageDBCols[j+1] = "COUNT(id) AS num_units"
 		} else {
-			aggUsageDBCols[i+1] = col
+			aggUsageDBCols[j+1] = col
 		}
+		j++
 	}
 }
 

pkg/api/models/models.go

@@ -74,6 +74,7 @@ type Usage struct {
 	NumUnits            int64     `json:"num_units"                  sql:"num_units"                  sqlitetype:"integer"` // Number of consumed units
 	Project             string    `json:"project"                    sql:"project"                    sqlitetype:"text"`    // Account in batch systems, Tenant in Openstack, Namespace in k8s
 	Usr                 string    `json:"usr"                        sql:"usr"                        sqlitetype:"text"`    // Username
+	LastUpdatedAt       string    `json:"-"                          sql:"last_updated_at"            sqlitetype:"text"`    // Last updated time. It can be used to clean up DB
 	TotalCPUBilling     int64     `json:"total_cpu_billing"          sql:"total_cpu_billing"          sqlitetype:"integer"` // Total CPU billing for project
 	TotalGPUBilling     int64     `json:"total_gpu_billing"          sql:"total_gpu_billing"          sqlitetype:"integer"` // Total GPU billing for project
 	TotalMiscBilling    int64     `json:"total_misc_billing"         sql:"total_misc_billing"         sqlitetype:"integer"` // Total billing for project that are not in CPU and GPU billing