Skip to content

Commit

Permalink
chore: Minor impovements
Browse files Browse the repository at this point in the history 
* Fetch units and users concurrently in go routines

* Use only 2 decimals in metrics to save DB storage

* Use a generic mock TSDB to get response for any uuid

* Use testify to assert tests in db package. We will have to use it for all pkg in future

* Use multiple updaters in e2e tests and update test fixtures

Signed-off-by: Mahendra Paipuri <[email protected]>
  • Loading branch information
@mahendrapaipuri
mahendrapaipuri committed Jul 4, 2024
1 parent b0c1678 commit 9cef141
Show file tree
Hide file tree
Showing 15 changed files with 232 additions and 420 deletions.
302 changes: 81 additions & 221 deletions cmd/mock_tsdb/main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,10 +3,13 @@ package main
import (
"encoding/json"
"fmt"
"hash/fnv"
"log"
"math"
"net/http"
"regexp"
"slices"
"strconv"
"strings"
"time"

Expand All @@ -22,20 +25,24 @@ var (
regexpUUID = regexp.MustCompile("(?:.+?)[^gpu]uuid=[~]{0,1}\"(?P<uuid>[a-zA-Z0-9-|]+)\"(?:.*)")
)

// filterResults returns the filtered results based on uuids slice
func filterResults(uuids []string, allResults []interface{}) []interface{} {
// Return results corresponding to UUIDs
var responseResults []interface{}
for _, result := range allResults {
if m, ok := result.(map[string]interface{})["metric"]; ok {
if uuid, ok := m.(map[string]string)["uuid"]; ok {
if slices.Contains(uuids, uuid) {
responseResults = append(responseResults, result)
}
}
}
// hash returns hash of a given string
func hash(s string) uint32 {
h := fnv.New32a()
h.Write([]byte(s))
return h.Sum32()
}

// lenLoop returns number of digits in an integer
func lenLoop(i uint32) int {
if i == 0 {
return 1
}
count := 0
for i != 0 {
i /= 10
count++
}
return responseResults
return count
}

// QueryHandler handles queries
Expand Down Expand Up @@ -79,246 +86,99 @@ func QueryHandler(w http.ResponseWriter, r *http.Request) {

// log.Println("Query", query, "UUIDs", uuids)

var allResults []interface{}
var results []interface{}
if slices.Contains(
[]string{
"avg_cpu_usage", "avg_cpu_mem_usage", "avg_gpu_usage",
"avg_gpu_mem_usage", "total_cpu_energy_usage_kwh", "total_gpu_energy_usage_kwh",
"total_cpu_emissions_gms", "total_gpu_emissions_gms",
}, query) {
allResults = []interface{}{
map[string]interface{}{
"metric": map[string]string{
"uuid": "1479763",
},
"value": []interface{}{
12345, "14.79",
},
},
map[string]interface{}{
"metric": map[string]string{
"uuid": "1481508",
},
"value": []interface{}{
12345, "14.58",
},
},
map[string]interface{}{
"metric": map[string]string{
"uuid": "147975",
},
"value": []interface{}{
12345, "14.79",
},
},
map[string]interface{}{
"metric": map[string]string{
"uuid": "11508",
},
"value": []interface{}{
12345, "11.50",
},
},
map[string]interface{}{
"metric": map[string]string{
"uuid": "81510",
},
"value": []interface{}{
12345, "81.51",
},
},
// Convert uuid into hash and transform that hash number into float64 between 0 and 100
for _, uuid := range uuids {
h := hash(uuid)
numDigits := lenLoop(h)
value := float64(h) / math.Pow(10, float64(numDigits)-2)
results = append(results,
map[string]interface{}{
"metric": map[string]string{
"uuid": uuid,
},
"value": []interface{}{
12345, strconv.FormatFloat(value, 'f', -1, 64),
},
})
}
} else if slices.Contains(
[]string{
"total_io_read_stats_bytes", "total_io_write_stats_bytes",
}, query) {
allResults = []interface{}{
map[string]interface{}{
"metric": map[string]string{
"uuid": "1479763",
},
"value": []interface{}{
12345, "1479763",
},
},
map[string]interface{}{
"metric": map[string]string{
"uuid": "1481508",
},
"value": []interface{}{
12345, "1481508",
},
},
map[string]interface{}{
"metric": map[string]string{
"uuid": "147975",
},
"value": []interface{}{
12345, "147975",
},
},
map[string]interface{}{
"metric": map[string]string{
"uuid": "11508",
},
"value": []interface{}{
12345, "11508",
},
},
map[string]interface{}{
"metric": map[string]string{
"uuid": "81510",
},
"value": []interface{}{
12345, "81510",
},
},
for _, uuid := range uuids {
h := hash(uuid)
results = append(results,
map[string]interface{}{
"metric": map[string]string{
"uuid": uuid,
},
"value": []interface{}{
12345, strconv.FormatUint(uint64(h), 10),
},
})
}
} else if slices.Contains(
[]string{
"total_io_read_stats_requests", "total_io_write_stats_requests",
}, query) {
allResults = []interface{}{
map[string]interface{}{
"metric": map[string]string{
"uuid": "1479763",
},
"value": []interface{}{
12345, "14797630",
},
},
map[string]interface{}{
"metric": map[string]string{
"uuid": "1481508",
},
"value": []interface{}{
12345, "14815080",
},
},
map[string]interface{}{
"metric": map[string]string{
"uuid": "147975",
},
"value": []interface{}{
12345, "1479750",
},
},
map[string]interface{}{
"metric": map[string]string{
"uuid": "11508",
},
"value": []interface{}{
12345, "115080",
},
},
map[string]interface{}{
"metric": map[string]string{
"uuid": "81510",
},
"value": []interface{}{
12345, "815100",
},
},
for _, uuid := range uuids {
h := hash(uuid)
results = append(results,
map[string]interface{}{
"metric": map[string]string{
"uuid": uuid,
},
"value": []interface{}{
12345, strconv.FormatUint(uint64(h)*10, 10),
},
})
}
} else if slices.Contains(
[]string{
"total_ingress_stats_bytes", "total_outgress_stats_bytes",
}, query) {
allResults = []interface{}{
map[string]interface{}{
"metric": map[string]string{
"uuid": "1479763",
},
"value": []interface{}{
12345, "147976300",
},
},
map[string]interface{}{
"metric": map[string]string{
"uuid": "1481508",
},
"value": []interface{}{
12345, "148150800",
},
},
map[string]interface{}{
"metric": map[string]string{
"uuid": "147975",
},
"value": []interface{}{
12345, "14797500",
},
},
map[string]interface{}{
"metric": map[string]string{
"uuid": "11508",
},
"value": []interface{}{
12345, "1150800",
},
},
map[string]interface{}{
"metric": map[string]string{
"uuid": "81510",
},
"value": []interface{}{
12345, "8151000",
},
},
for _, uuid := range uuids {
h := hash(uuid)
results = append(results,
map[string]interface{}{
"metric": map[string]string{
"uuid": uuid,
},
"value": []interface{}{
12345, strconv.FormatUint(uint64(h)*100, 10),
},
})
}
} else if slices.Contains(
[]string{
"total_ingress_stats_packets", "total_outgress_stats_packets",
}, query) {
allResults = []interface{}{
map[string]interface{}{
"metric": map[string]string{
"uuid": "1479763",
},
"value": []interface{}{
12345, "1479763000",
},
},
map[string]interface{}{
"metric": map[string]string{
"uuid": "1481508",
},
"value": []interface{}{
12345, "1481508000",
},
},
map[string]interface{}{
"metric": map[string]string{
"uuid": "147975",
},
"value": []interface{}{
12345, "147975000",
},
},
map[string]interface{}{
"metric": map[string]string{
"uuid": "11508",
},
"value": []interface{}{
12345, "11508000",
},
},
map[string]interface{}{
"metric": map[string]string{
"uuid": "81510",
},
"value": []interface{}{
12345, "81510000",
},
},
for _, uuid := range uuids {
h := hash(uuid)
results = append(results,
map[string]interface{}{
"metric": map[string]string{
"uuid": uuid,
},
"value": []interface{}{
12345, strconv.FormatUint(uint64(h)*1000, 10),
},
})
}
}
responseResults := filterResults(uuids, allResults)
// responseResults := filterResults(uuids, esults)
response = tsdb.Response{
Status: "success",
Data: map[string]interface{}{
"resultType": "vector",
"result": responseResults,
"result": results,
},
}
if err := json.NewEncoder(w).Encode(&response); err != nil {
Expand Down
1 change: 1 addition & 0 deletions go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ require (
github.com/prometheus/common v0.45.0
github.com/prometheus/exporter-toolkit v0.10.0
github.com/prometheus/procfs v0.12.0
github.com/stretchr/testify v1.9.0
github.com/swaggo/http-swagger/v2 v2.0.2
github.com/swaggo/swag v1.16.3
github.com/zeebo/xxh3 v1.0.2
Expand Down
6 changes: 4 additions & 2 deletions pkg/api/db/db.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -169,7 +169,6 @@ updatetime:
setup:
// Setup manager struct that retrieves unit data
manager, err := c.ResourceManager(c.Logger)
fmt.Printf("%#v\n", manager)
if err != nil {
level.Error(c.Logger).Log("msg", "Resource manager setup failed", "err", err)
return nil, err
Expand Down Expand Up @@ -259,6 +258,9 @@ setup:

// Collect unit stats
func (s *statsDB) Collect() error {
// Measure elapsed time
defer common.TimeTrack(time.Now(), "Data collection", s.logger)

var currentTime = time.Now()

// If duration is less than 1 day do single update
Expand Down Expand Up @@ -452,7 +454,7 @@ func (s *statsDB) execStatements(
clusterProjects []models.ClusterProjects,
) {
// Measure elapsed time
defer common.TimeTrack(time.Now(), "DB insertions", s.logger)
defer common.TimeTrack(time.Now(), "DB insertion", s.logger)

var ignore = 0
var err error
Expand Down
Loading

0 comments on commit 9cef141

Please sign in to comment.