feat: Cache current usage query result

* Current usage query is quite slow due to go-cgo trips for custom funcs. So we cache the query results by rounding the query period to the cache interval.

* As long as query period stays within cache interval, results of cached query will be returned.

* Rate limiting is applied over entire API server which can be configured from config file

Signed-off-by: Mahendra Paipuri <[email protected]>

build/config/ceems_api_server/ceems_api_server.yml

@@ -209,6 +209,21 @@ ceems_api_server:
     #
     max_query: 0s
 
+    # Number of requests allowed in ONE MINUTE per client identified by Real IP address.
+    # Request headers `True-Client-IP`, `X-Real-IP` and `X-Forwarded-For` are looked up
+    # to get the real IP address.
+    #
+    # This is to effectively impose a rate limit for the entire CEEMS server irrespective
+    # of URL path. We advise to set it to a value based on your needs to avoid DoS/DDoS
+    # attacks.
+    #
+    # Rate limiting is done using the Sliding Window Counter pattern inspired by 
+    # CloudFlare https://blog.cloudflare.com/counting-things-a-lot-of-different-things/
+    #
+    # Default value `0` means no rate limiting is applied.
+    #
+    requests_limit: 0
+
     # It will be used to prefix all HTTP endpoints served by CEEMS API server. 
     # For example, if CEEMS API server is served via a reverse proxy. 
     # 

go.mod

@@ -5,10 +5,12 @@ go 1.22.5
 require (
 	github.com/alecthomas/kingpin/v2 v2.4.0
 	github.com/containerd/cgroups/v3 v3.0.4-0.20240117155926-c00d22e55fef
+	github.com/go-chi/httprate v0.10.0
 	github.com/go-kit/log v0.2.1
 	github.com/golang-migrate/migrate/v4 v4.17.0
 	github.com/google/uuid v1.4.0
 	github.com/gorilla/mux v1.8.1
+	github.com/jellydator/ttlcache/v3 v3.2.0
 	github.com/mattn/go-sqlite3 v1.14.18
 	github.com/prometheus/client_golang v1.17.0
 	github.com/prometheus/common v0.45.0
@@ -26,7 +28,7 @@ require (
 	github.com/KyleBanks/depth v1.2.1 // indirect
 	github.com/alecthomas/units v0.0.0-20231202071711-9a357b53e9c9 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
-	github.com/cespare/xxhash/v2 v2.2.0 // indirect
+	github.com/cespare/xxhash/v2 v2.3.0 // indirect
 	github.com/cilium/ebpf v0.11.0 // indirect
 	github.com/coreos/go-systemd/v22 v22.5.0 // indirect
 	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect

go.sum

@@ -6,8 +6,8 @@ github.com/alecthomas/units v0.0.0-20231202071711-9a357b53e9c9 h1:ez/4by2iGztzR4
 github.com/alecthomas/units v0.0.0-20231202071711-9a357b53e9c9/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE=
 github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
 github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
-github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44=
-github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
+github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
+github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/cilium/ebpf v0.11.0 h1:V8gS/bTCCjX9uUnkUFUpPsksM8n1lXBAvHcpiFk1X2Y=
 github.com/cilium/ebpf v0.11.0/go.mod h1:WE7CZAnqOL2RouJ4f1uyNhqr2P4CCvXFIqdRDUgWsVs=
 github.com/containerd/cgroups/v3 v3.0.4-0.20240117155926-c00d22e55fef h1:/l2M/jl0S3eRGtmXsNzezBCZwKP8cZqXolQ9cPqexyY=
@@ -22,6 +22,8 @@ github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4
 github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
 github.com/frankban/quicktest v1.14.5 h1:dfYrrRyLtiqT9GyKXgdh+k4inNeTvmGbuSgZ3lx3GhA=
 github.com/frankban/quicktest v1.14.5/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0=
+github.com/go-chi/httprate v0.10.0 h1:NnHj/C9pNR1mwDX9biXXYmLhEhVqz+PR3MR3kWdHYYY=
+github.com/go-chi/httprate v0.10.0/go.mod h1:IqW+o6o/dkOUqyR9weVur+ob3gpEemmbfSRiBxUwqJU=
 github.com/go-kit/log v0.2.1 h1:MRVx0/zhvdseW+Gza6N9rVzU/IVzaeE1SFI4raAhmBU=
 github.com/go-kit/log v0.2.1/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBjv0=
 github.com/go-logfmt/logfmt v0.6.0 h1:wGYYu3uicYdqXVgoYbvnkrPVXkuLM1p1ifugDMEdRi4=
@@ -55,6 +57,8 @@ github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY
 github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
 github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo=
 github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM=
+github.com/jellydator/ttlcache/v3 v3.2.0 h1:6lqVJ8X3ZaUwvzENqPAobDsXNExfUJd61u++uW8a3LE=
+github.com/jellydator/ttlcache/v3 v3.2.0/go.mod h1:hi7MGFdMAwZna5n2tuvh63DvFLzVKySzCVW6+0gA2n4=
 github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
 github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
 github.com/jpillora/backoff v1.0.0 h1:uvFg412JmmHBHw7iwprIxkPMI+sGQ4kzOWsMeHnm2EA=

internal/common/helpers.go

@@ -4,6 +4,7 @@ package common
 import (
 	"errors"
 	"fmt"
+	"hash/fnv"
 	"math"
 	"net"
 	"net/url"
@@ -19,6 +20,18 @@ import (
 	"gopkg.in/yaml.v3"
 )
 
+// GenerateKey generates a reproducible key from a given URL string
+func GenerateKey(url string) uint64 {
+	hash := fnv.New64a()
+	hash.Write([]byte(url))
+	return hash.Sum64()
+}
+
+// Round returns a value less than or equal to value that is multiple of nearest
+func Round(value int64, nearest int64) int64 {
+	return (value / nearest) * nearest
+}
+
 // TimeTrack tracks execution time of each function
 func TimeTrack(start time.Time, name string, logger log.Logger) {
 	elapsed := time.Since(start)

internal/common/helpers_test.go

@@ -51,6 +51,59 @@ func TestSanitizeFloat(t *testing.T) {
 	}
 }
 
+func TestGenerateKey(t *testing.T) {
+	tests := []struct {
+		name     string
+		input    string
+		expected uint64
+	}{
+		{
+			name:     "Regular URL",
+			input:    "/foo",
+			expected: 0xf9be0e9c9154425e,
+		},
+		{
+			name:     "URL with query params",
+			input:    "/foo?q1=bar&q2=bar2",
+			expected: 0xe71d223b0fec69f4,
+		},
+		{
+			name:     "URL with special characters",
+			input:    "/?^1234567890ßqwertzuiopü+asdfghjklöä#<yxcvbnm",
+			expected: 0x19a8532cae702ffa,
+		},
+	}
+
+	for _, test := range tests {
+		got := GenerateKey(test.input)
+		assert.Equal(t, test.expected, got, test.name)
+	}
+}
+
+func TestRound(t *testing.T) {
+	tests := []struct {
+		name     string
+		input    int64
+		expected int64
+	}{
+		{
+			name:     "Floor",
+			input:    400,
+			expected: 0,
+		},
+		{
+			name:     "Ceil",
+			input:    897,
+			expected: 0,
+		},
+	}
+
+	for _, test := range tests {
+		got := Round(test.input, 900)
+		assert.Equal(t, test.expected, got, test.name)
+	}
+}
+
 func TestGetUuid(t *testing.T) {
 	expected := "d808af89-684c-6f3f-a474-8d22b566dd12"
 	got, err := GetUUIDFromString([]string{"foo", "1234", "bar567"})

pkg/api/cli/cli.go

@@ -220,6 +220,7 @@ func (b *CEEMSServer) Main() error {
 			WebSystemdSocket: *systemdSocket,
 			WebConfigFile:    *webConfigFile,
 			RoutePrefix:      config.Server.Web.RoutePrefix,
+			RequestsLimit:    config.Server.Web.RequestsLimit,
 			MaxQueryPeriod:   config.Server.Web.MaxQueryPeriod,
 		},
 		DB: *dbConfig,

pkg/api/http/docs/docs.go

@@ -565,7 +565,7 @@ const docTemplate = `{
                         "BasicAuth": []
                     }
                 ],
-                "description": "This endpoint will return the usage statistics current user. The\ncurrent user is always identified by the header ` + "`" + `X-Grafana-User` + "`" + ` in\nthe request.\n\nA path parameter ` + "`" + `mode` + "`" + ` is required to return the kind of usage statistics.\nCurrently, two modes of statistics are supported:\n- ` + "`" + `current` + "`" + `: In this mode the usage between two time periods is returned\nbased on ` + "`" + `from` + "`" + ` and ` + "`" + `to` + "`" + ` query parameters.\n- ` + "`" + `global` + "`" + `: In this mode the _total_ usage statistics are returned. For\ninstance, if the retention period of the DB is set to 2 years, usage\nstatistics of last 2 years will be returned.\n\nThe statistics can be limited to certain projects by passing ` + "`" + `project` + "`" + ` query,\nparameter.\n\nIf ` + "`" + `to` + "`" + ` query parameter is not provided, current time will be used. If ` + "`" + `from` + "`" + `\nquery parameter is not used, a default query window of 24 hours will be used.\nIt means if ` + "`" + `to` + "`" + ` is provided, ` + "`" + `from` + "`" + ` will be calculated as ` + "`" + `to` + "`" + ` - 24hrs.\n\nTo limit the number of fields in the response, use ` + "`" + `field` + "`" + ` query parameter. By default, all\nfields will be included in the response if they are _non-empty_.",
+                "description": "This endpoint will return the usage statistics current user. The\ncurrent user is always identified by the header ` + "`" + `X-Grafana-User` + "`" + ` in\nthe request.\n\nA path parameter ` + "`" + `mode` + "`" + ` is required to return the kind of usage statistics.\nCurrently, two modes of statistics are supported:\n- ` + "`" + `current` + "`" + `: In this mode the usage between two time periods is returned\nbased on ` + "`" + `from` + "`" + ` and ` + "`" + `to` + "`" + ` query parameters.\n- ` + "`" + `global` + "`" + `: In this mode the _total_ usage statistics are returned. For\ninstance, if the retention period of the DB is set to 2 years, usage\nstatistics of last 2 years will be returned.\n\nThe statistics can be limited to certain projects by passing ` + "`" + `project` + "`" + ` query,\nparameter.\n\nIf ` + "`" + `to` + "`" + ` query parameter is not provided, current time will be used. If ` + "`" + `from` + "`" + `\nquery parameter is not used, a default query window of 24 hours will be used.\nIt means if ` + "`" + `to` + "`" + ` is provided, ` + "`" + `from` + "`" + ` will be calculated as ` + "`" + `to` + "`" + ` - 24hrs.\n\nTo limit the number of fields in the response, use ` + "`" + `field` + "`" + ` query parameter. By default, all\nfields will be included in the response if they are _non-empty_.\n\nThe ` + "`" + `current` + "`" + ` usage mode can be slow query depending the requested\nwindow interval. This is mostly due to the fact that the CEEMS DB\nuses custom JSON types to store metric data and usage statistics\nneeds to aggregate metrics over these JSON types using custom aggregate\nfunctions which can be slow.\n\nTherefore the query results are cached for 15 min to avoid load on server.\nURL string is used as the cache key. Thus, the query parameters\n` + "`" + `from` + "`" + ` and ` + "`" + `to` + "`" + ` are rounded to the nearest timestamp that are\nmultiple of 900 sec (15 min). The first query will make a DB query and\ncache results and subsequent queries, for a given user and same URL\nquery parameters, will return the same cached result until the cache\nis invalidated after 15 min.",
                 "produces": [
                     "application/json"
                 ],
@@ -664,7 +664,7 @@ const docTemplate = `{
                         "BasicAuth": []
                     }
                 ],
-                "description": "This admin endpoint will return the usage statistics of _queried_ user. The\ncurrent user is always identified by the header ` + "`" + `X-Grafana-User` + "`" + ` in\nthe request.\n\nThe user who is making the request must be in the list of admin users\nconfigured for the server.\n\nA path parameter ` + "`" + `mode` + "`" + ` is required to return the kind of usage statistics.\nCurrently, two modes of statistics are supported:\n- ` + "`" + `current` + "`" + `: In this mode the usage between two time periods is returned\nbased on ` + "`" + `from` + "`" + ` and ` + "`" + `to` + "`" + ` query parameters.\n- ` + "`" + `global` + "`" + `: In this mode the _total_ usage statistics are returned. For\ninstance, if the retention period of the DB is set to 2 years, usage\nstatistics of last 2 years will be returned.\n\nThe statistics can be limited to certain projects by passing ` + "`" + `project` + "`" + ` query,\nparameter.\n\nIf ` + "`" + `to` + "`" + ` query parameter is not provided, current time will be used. If ` + "`" + `from` + "`" + `\nquery parameter is not used, a default query window of 24 hours will be used.\nIt means if ` + "`" + `to` + "`" + ` is provided, ` + "`" + `from` + "`" + ` will be calculated as ` + "`" + `to` + "`" + ` - 24hrs.\n\nTo limit the number of fields in the response, use ` + "`" + `field` + "`" + ` query parameter. By default, all\nfields will be included in the response if they are _non-empty_.",
+                "description": "This admin endpoint will return the usage statistics of _queried_ user. The\ncurrent user is always identified by the header ` + "`" + `X-Grafana-User` + "`" + ` in\nthe request.\n\nThe user who is making the request must be in the list of admin users\nconfigured for the server.\n\nA path parameter ` + "`" + `mode` + "`" + ` is required to return the kind of usage statistics.\nCurrently, two modes of statistics are supported:\n- ` + "`" + `current` + "`" + `: In this mode the usage between two time periods is returned\nbased on ` + "`" + `from` + "`" + ` and ` + "`" + `to` + "`" + ` query parameters.\n- ` + "`" + `global` + "`" + `: In this mode the _total_ usage statistics are returned. For\ninstance, if the retention period of the DB is set to 2 years, usage\nstatistics of last 2 years will be returned.\n\nThe statistics can be limited to certain projects by passing ` + "`" + `project` + "`" + ` query,\nparameter.\n\nIf ` + "`" + `to` + "`" + ` query parameter is not provided, current time will be used. If ` + "`" + `from` + "`" + `\nquery parameter is not used, a default query window of 24 hours will be used.\nIt means if ` + "`" + `to` + "`" + ` is provided, ` + "`" + `from` + "`" + ` will be calculated as ` + "`" + `to` + "`" + ` - 24hrs.\n\nTo limit the number of fields in the response, use ` + "`" + `field` + "`" + ` query parameter. By default, all\nfields will be included in the response if they are _non-empty_.\n\nThe ` + "`" + `current` + "`" + ` usage mode can be slow query depending the requested\nwindow interval. This is mostly due to the fact that the CEEMS DB\nuses custom JSON types to store metric data and usage statistics\nneeds to aggregate metrics over these JSON types using custom aggregate\nfunctions which can be slow.\n\nTherefore the query results are cached for 15 min to avoid load on server.\nURL string is used as the cache key. Thus, the query parameters\n` + "`" + `from` + "`" + ` and ` + "`" + `to` + "`" + ` are rounded to the nearest timestamp that are\nmultiple of 900 sec (15 min). The first query will make a DB query and\ncache results and subsequent queries, for a given user and same URL\nquery parameters, will return the same cached result until the cache\nis invalidated after 15 min.",
                 "produces": [
                     "application/json"
                 ],