Skip to content

Commit

Permalink
[#3550] feat-487: Add cpu and mem utilization percentage for rack pro…
Browse files Browse the repository at this point in the history 
…cesses

This provides backend support for : convox/issues-private#487

aws cloudwatch metrics only provides cpu and mem utilization metrics for ecs services and in this PR we used that for processes cpu and mem current utilization
  • Loading branch information
@nightfury1204
nightfury1204 committed Jul 21, 2022
1 parent 7094424 commit 2db503d
Show file tree
Hide file tree
Showing 8 changed files with 222 additions and 80 deletions.
25 changes: 13 additions & 12 deletions pkg/structs/process.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,18 +8,19 @@ import (
type Process struct {
Id string `json:"id"`

App string `json:"app"`
Command string `json:"command"`
Cpu float64 `json:"cpu"`
Host string `json:"host"`
Image string `json:"image"`
Instance string `json:"instance"`
Memory float64 `json:"memory"`
Name string `json:"name"`
Ports []string `json:"ports"`
Release string `json:"release"`
Started time.Time `json:"started"`
Status string `json:"status"`
App string `json:"app"`
Command string `json:"command"`
Cpu float64 `json:"cpu"`
Host string `json:"host"`
Image string `json:"image"`
Instance string `json:"instance"`
Memory float64 `json:"memory"`
Name string `json:"name"`
Ports []string `json:"ports"`
Release string `json:"release"`
Started time.Time `json:"started"`
Status string `json:"status"`
TaskDefinition string `json:"task_definition"`
}

type Processes []Process
Expand Down
9 changes: 9 additions & 0 deletions provider/aws/aws_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,10 +5,13 @@ import (
"net/http/httptest"
"os"

"github.com/aws/aws-sdk-go/service/cloudwatch"
"github.com/convox/logger"
mockaws "github.com/convox/rack/pkg/mock/aws"
"github.com/convox/rack/pkg/structs"
"github.com/convox/rack/pkg/test/awsutil"
"github.com/convox/rack/provider/aws"
"github.com/stretchr/testify/mock"
)

func init() {
Expand All @@ -34,6 +37,11 @@ func StubAwsProvider(cycles ...awsutil.Cycle) *AwsStub {
os.Setenv("AWS_ACCESS_KEY_ID", "test-access")
os.Setenv("AWS_SECRET_ACCESS_KEY", "test-secret")

cw := &mockaws.CloudWatchAPI{}
output := &cloudwatch.GetMetricDataOutput{MetricDataResults: []*cloudwatch.MetricDataResult{}}

cw.On("GetMetricData", mock.Anything).Return(output, nil)

p := &aws.Provider{
Region: "us-test-1",
Endpoint: s.URL,
Expand All @@ -46,6 +54,7 @@ func StubAwsProvider(cycles ...awsutil.Cycle) *AwsStub {
Rack: "convox",
SettingsBucket: "convox-settings",
SkipCache: true,
CloudWatch: cw,
}

return &AwsStub{p, s}
Expand Down
4 changes: 4 additions & 0 deletions provider/aws/helpers.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1509,3 +1509,7 @@ func (cr *CronJob) LongName() string {
}
return prefix + suffix
}

func serviceMetricsKey(metricType, serviceName string) string {
return fmt.Sprintf("service:%s:utilization:%s", metricType, strings.ReplaceAll(serviceName, "-", "_"))
}
28 changes: 28 additions & 0 deletions provider/aws/metrics.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -307,6 +307,34 @@ func (p *Provider) systemMetricQueries() []metricDataQuerier {
return mdqs
}

func (p *Provider) servicesMetricQueries(names []string) []metricDataQuerier {
mdqs := []metricDataQuerier{}

for _, name := range names {
mdqs = append(mdqs, metricStatistics{
Name: serviceMetricsKey("cpu", name),
Namespace: "AWS/ECS",
Metric: "CPUUtilization",
Dimensions: map[string]string{
"ClusterName": p.Cluster,
"ServiceName": name,
},
Statistics: []string{"Average"},
}, metricStatistics{
Name: serviceMetricsKey("mem", name),
Namespace: "AWS/ECS",
Metric: "MemoryUtilization",
Dimensions: map[string]string{
"ClusterName": p.Cluster,
"ServiceName": name,
},
Statistics: []string{"Average"},
})
}

return mdqs
}

type metricDataQuerier interface {
MetricDataQueries(period int64, suffix string) []*cloudwatch.MetricDataQuery
}
Expand Down
60 changes: 53 additions & 7 deletions provider/aws/processes.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -171,8 +171,53 @@ func (p *Provider) ProcessList(app string, opts structs.ProcessListOptions) (str
ps = pss
}

taskDefMap := map[string]bool{}
for i := range ps {
ps[i].App = app
taskDefMap[ps[i].TaskDefinition] = true
}

services, err := p.clusterServices()
if err != nil {
return nil, err
}

serviceNames := []string{}
taskToServiceMap := map[string]string{}
for _, s := range services {
if s.ServiceName != nil && s.TaskDefinition != nil && taskDefMap[*s.TaskDefinition] {
serviceNames = append(serviceNames, *s.ServiceName)
taskToServiceMap[*s.TaskDefinition] = *s.ServiceName
}
}

mdqs := p.servicesMetricQueries(serviceNames)

ms, err := p.cloudwatchMetrics(mdqs, structs.MetricsOptions{
Start: aws.Time(time.Now().Add(-5 * time.Minute)),
})
if err != nil {
return nil, err
}

mMap := map[string]structs.Metric{}
for _, m := range ms {
mMap[m.Name] = m
}

for i := range ps {
if serviceName, has := taskToServiceMap[ps[i].TaskDefinition]; has {
if m, has := mMap[serviceMetricsKey("mem", serviceName)]; has && len(m.Values) > 0 {
// normally there should be one point but if multiple points are fetched, pick the latest one
// points are sorted in TimestampAscending order
ps[i].Memory = m.Values[len(m.Values)-1].Average
}
if m, has := mMap[serviceMetricsKey("cpu", serviceName)]; has && len(m.Values) > 0 {
// normally there should be one point but if multiple points are fetched, pick the latest one
// points are sorted in TimestampAscending order
ps[i].Cpu = m.Values[len(m.Values)-1].Average
}
}
}

return ps, nil
Expand Down Expand Up @@ -683,13 +728,14 @@ func (p *Provider) fetchProcess(task *ecs.Task, psch chan structs.Process, errch
}

ps := structs.Process{
Id: arnToPid(*task.TaskArn),
Name: *container.Name,
App: coalesces(labels["convox.app"], env["APP"]),
Release: coalesces(labels["convox.release"], env["RELEASE"]),
Image: *cd.Image,
Ports: ports,
Status: taskStatus(*task.LastStatus),
Id: arnToPid(*task.TaskArn),
Name: *container.Name,
App: coalesces(labels["convox.app"], env["APP"]),
Release: coalesces(labels["convox.release"], env["RELEASE"]),
Image: *cd.Image,
Ports: ports,
Status: taskStatus(*task.LastStatus),
TaskDefinition: *task.TaskDefinitionArn,
}

if task.ContainerInstanceArn != nil {
Expand Down
135 changes: 74 additions & 61 deletions provider/aws/processes_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,8 @@ func TestProcessExec(t *testing.T) {
cycleProcessDescribeTaskDefinition1,
cycleProcessDescribeContainerInstances,
cycleProcessDescribeRackInstances,
cycleECSListServices,
cycleECSDescribeServices,
cycleProcessListTasksRunning,
cycleProcessListTasksStopped,
cycleProcessDescribeTasks,
Expand Down Expand Up @@ -88,39 +90,43 @@ func TestProcessList(t *testing.T) {
cycleProcessDescribeTaskDefinition2,
cycleProcessDescribeContainerInstances,
cycleProcessDescribeRackInstances,
cycleECSListServices,
cycleECSDescribeServices,
)
defer provider.Close()

s, err := provider.ProcessList("myapp", structs.ProcessListOptions{})

ps := structs.Processes{
structs.Process{
Id: "5850760f0845",
App: "myapp",
Name: "web",
Release: "R1234",
Command: "",
Host: "10.0.1.244",
Image: "778743527532.dkr.ecr.us-east-1.amazonaws.com/convox-myapp-nkdecwppkq:web.BMPBJLITPZT",
Instance: "i-5bc45dc2",
Ports: []string{},
Cpu: 0,
Memory: 0,
Status: "running",
Id: "5850760f0845",
App: "myapp",
Name: "web",
Release: "R1234",
Command: "",
Host: "10.0.1.244",
Image: "778743527532.dkr.ecr.us-east-1.amazonaws.com/convox-myapp-nkdecwppkq:web.BMPBJLITPZT",
Instance: "i-5bc45dc2",
Ports: []string{},
Cpu: 0,
Memory: 0,
Status: "running",
TaskDefinition: "arn:aws:ecs:us-east-1:778743527532:task-definition/convox-myapp-web:34",
},
structs.Process{
Id: "5850760f0846",
App: "myapp",
Name: "web",
Release: "R1234",
Command: "ls -la 'name space'",
Host: "10.0.1.244",
Image: "778743527532.dkr.ecr.us-east-1.amazonaws.com/convox-myapp-nkdecwppkq:web.BMPBJLITPZT",
Instance: "i-5bc45dc2",
Ports: []string{},
Cpu: 0,
Memory: 0,
Status: "pending",
Id: "5850760f0846",
App: "myapp",
Name: "web",
Release: "R1234",
Command: "ls -la 'name space'",
Host: "10.0.1.244",
Image: "778743527532.dkr.ecr.us-east-1.amazonaws.com/convox-myapp-nkdecwppkq:web.BMPBJLITPZT",
Instance: "i-5bc45dc2",
Ports: []string{},
Cpu: 0,
Memory: 0,
Status: "pending",
TaskDefinition: "arn:aws:ecs:us-east-1:778743527532:task-definition/convox-myapp-web:34",
},
}

Expand All @@ -137,6 +143,8 @@ func TestProcessListEmpty(t *testing.T) {
cycleProcessListTasksByService2Empty,
cycleProcessListTasksByStartedEmpty,
cycleProcessDescribeRackInstances,
cycleECSListServices,
cycleECSDescribeServices,
)
defer provider.Close()

Expand Down Expand Up @@ -164,6 +172,8 @@ func TestProcessListWithBuildCluster(t *testing.T) {
cycleProcessDescribeTaskDefinition2,
cycleProcessDescribeContainerInstancesBuild,
cycleProcessDescribeRackInstances,
cycleECSListServices,
cycleECSDescribeServices,
)
defer provider.Close()

Expand All @@ -173,46 +183,49 @@ func TestProcessListWithBuildCluster(t *testing.T) {

ps := structs.Processes{
structs.Process{
Id: "5850760f0845",
App: "myapp",
Name: "web",
Release: "R1234",
Command: "",
Host: "10.0.1.244",
Image: "778743527532.dkr.ecr.us-east-1.amazonaws.com/convox-myapp-nkdecwppkq:web.BMPBJLITPZT",
Instance: "i-5bc45dc2",
Ports: []string{},
Cpu: 0,
Memory: 0,
Status: "running",
Id: "5850760f0845",
App: "myapp",
Name: "web",
Release: "R1234",
Command: "",
Host: "10.0.1.244",
Image: "778743527532.dkr.ecr.us-east-1.amazonaws.com/convox-myapp-nkdecwppkq:web.BMPBJLITPZT",
Instance: "i-5bc45dc2",
Ports: []string{},
Cpu: 0,
Memory: 0,
Status: "running",
TaskDefinition: "arn:aws:ecs:us-east-1:778743527532:task-definition/convox-myapp-web:34",
},
structs.Process{
Id: "5850760f0846",
App: "myapp",
Name: "web",
Release: "R1234",
Command: "ls -la 'name space'",
Host: "10.0.1.244",
Image: "778743527532.dkr.ecr.us-east-1.amazonaws.com/convox-myapp-nkdecwppkq:web.BMPBJLITPZT",
Instance: "i-5bc45dc2",
Ports: []string{},
Cpu: 0,
Memory: 0,
Status: "running",
Id: "5850760f0846",
App: "myapp",
Name: "web",
Release: "R1234",
Command: "ls -la 'name space'",
Host: "10.0.1.244",
Image: "778743527532.dkr.ecr.us-east-1.amazonaws.com/convox-myapp-nkdecwppkq:web.BMPBJLITPZT",
Instance: "i-5bc45dc2",
Ports: []string{},
Cpu: 0,
Memory: 0,
Status: "running",
TaskDefinition: "arn:aws:ecs:us-east-1:778743527532:task-definition/convox-myapp-web:34",
},
structs.Process{
Id: "5850760f0848",
App: "myapp",
Name: "web",
Release: "R1234",
Command: "",
Host: "10.0.1.244",
Image: "778743527532.dkr.ecr.us-east-1.amazonaws.com/convox-myapp-nkdecwppkq:web.BMPBJLITPZT",
Instance: "i-5bc45dc2",
Ports: []string{},
Cpu: 0,
Memory: 0,
Status: "running",
Id: "5850760f0848",
App: "myapp",
Name: "web",
Release: "R1234",
Command: "",
Host: "10.0.1.244",
Image: "778743527532.dkr.ecr.us-east-1.amazonaws.com/convox-myapp-nkdecwppkq:web.BMPBJLITPZT",
Instance: "i-5bc45dc2",
Ports: []string{},
Cpu: 0,
Memory: 0,
Status: "running",
TaskDefinition: "arn:aws:ecs:us-east-1:778743527532:task-definition/convox-myapp-web:34",
},
}

Expand Down Expand Up @@ -256,7 +269,7 @@ func TestProcessRunDetached(t *testing.T) {
Release: options.String("RVFETUHHKKD"),
})

pse := &structs.Process{Id: "5850760f0845", App: "", Command: "ls -la 'name space'", Cpu: 0, Host: "10.0.1.244", Image: "778743527532.dkr.ecr.us-east-1.amazonaws.com/convox-myapp-nkdecwppkq:web.BMPBJLITPZT", Instance: "i-5bc45dc2", Memory: 0, Name: "web", Ports: []string{}, Release: "R1234", Status: "running"}
pse := &structs.Process{Id: "5850760f0845", App: "", Command: "ls -la 'name space'", Cpu: 0, Host: "10.0.1.244", Image: "778743527532.dkr.ecr.us-east-1.amazonaws.com/convox-myapp-nkdecwppkq:web.BMPBJLITPZT", Instance: "i-5bc45dc2", Memory: 0, Name: "web", Ports: []string{}, Release: "R1234", Status: "running", TaskDefinition: "arn:aws:ecs:us-east-1:778743527532:task-definition/convox-myapp-web:34"}

assert.NoError(t, err)
assert.Equal(t, pse, psa)
Expand Down
Loading

0 comments on commit 2db503d

Please sign in to comment.