From bec2a803f1a9add60b9c9aee9e7f111d0a646dfb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Sevilla?= Date: Wed, 8 Nov 2023 13:22:21 +0100 Subject: [PATCH 1/7] General code quality improvements (#495) * General code quality improvements Signed-off-by: Raul Sevilla * Missing UUID in benchmarkMetadata doc Signed-off-by: Raul Sevilla --------- Signed-off-by: Raul Sevilla --- cmd/kube-burner/ocp.go | 47 ++++++++----------- pkg/measurements/pod_latency.go | 10 ++--- pkg/workloads/helpers.go | 80 +++++++++++---------------------- pkg/workloads/types.go | 69 ++++++++++++++++++++++++++++ pkg/workloads/workloads.go | 15 ------- 5 files changed, 118 insertions(+), 103 deletions(-) create mode 100644 pkg/workloads/types.go delete mode 100644 pkg/workloads/workloads.go diff --git a/cmd/kube-burner/ocp.go b/cmd/kube-burner/ocp.go index 0b8338dd4..6ca9f9ef6 100644 --- a/cmd/kube-burner/ocp.go +++ b/cmd/kube-burner/ocp.go @@ -17,9 +17,7 @@ package main import ( "embed" _ "embed" - "fmt" "os" - "strings" "time" "github.com/cloud-bulldozer/go-commons/indexers" @@ -38,44 +36,35 @@ func openShiftCmd() *cobra.Command { Short: "OpenShift wrapper", Long: `This subcommand is meant to be used against OpenShift clusters and serve as a shortcut to trigger well-known workloads`, } + var workloadConfig workloads.Config var wh workloads.WorkloadHelper - var indexingType indexers.IndexerType - esServer := ocpCmd.PersistentFlags().String("es-server", "", "Elastic Search endpoint") + ocpCmd.PersistentFlags().StringVar(&workloadConfig.EsServer, "es-server", "", "Elastic Search endpoint") + ocpCmd.PersistentFlags().StringVar(&workloadConfig.Esindex, "es-index", "", "Elastic Search index") localIndexing := ocpCmd.PersistentFlags().Bool("local-indexing", false, "Enable local indexing") - esIndex := ocpCmd.PersistentFlags().String("es-index", "", "Elastic Search index") - metricsEndpoint := ocpCmd.PersistentFlags().String("metrics-endpoint", "", "YAML file with a list of metric endpoints") - alerting := ocpCmd.PersistentFlags().Bool("alerting", true, "Enable alerting") - uuid := ocpCmd.PersistentFlags().String("uuid", uid.NewV4().String(), "Benchmark UUID") - timeout := ocpCmd.PersistentFlags().Duration("timeout", 4*time.Hour, "Benchmark timeout") - qps := ocpCmd.PersistentFlags().Int("qps", 20, "QPS") - burst := ocpCmd.PersistentFlags().Int("burst", 20, "Burst") - gc := ocpCmd.PersistentFlags().Bool("gc", true, "Garbage collect created namespaces") - gcMetrics := ocpCmd.PersistentFlags().Bool("gc-metrics", false, "Collect metrics during garbage collection") + ocpCmd.PersistentFlags().StringVar(&workloadConfig.MetricsEndpoint, "metrics-endpoint", "", "YAML file with a list of metric endpoints") + ocpCmd.PersistentFlags().BoolVar(&workloadConfig.Alerting, "alerting", true, "Enable alerting") + ocpCmd.PersistentFlags().StringVar(&workloadConfig.UUID, "uuid", uid.NewV4().String(), "Benchmark UUID") + ocpCmd.PersistentFlags().DurationVar(&workloadConfig.Timeout, "timeout", 4*time.Hour, "Benchmark timeout") + ocpCmd.PersistentFlags().IntVar(&workloadConfig.QPS, "qps", 20, "QPS") + ocpCmd.PersistentFlags().IntVar(&workloadConfig.Burst, "burst", 20, "Burst") + ocpCmd.PersistentFlags().BoolVar(&workloadConfig.Gc, "gc", true, "Garbage collect created namespaces") + ocpCmd.PersistentFlags().BoolVar(&workloadConfig.GcMetrics, "gc-metrics", false, "Collect metrics during garbage collection") userMetadata := ocpCmd.PersistentFlags().String("user-metadata", "", "User provided metadata file, in YAML format") extract := ocpCmd.PersistentFlags().Bool("extract", false, "Extract workload in the current directory") - profileType := ocpCmd.PersistentFlags().String("profile-type", "both", "Metrics profile to use, supported options are: regular, reporting or both") + ocpCmd.PersistentFlags().StringVar(&workloadConfig.ProfileType, "profile-type", "both", "Metrics profile to use, supported options are: regular, reporting or both") + ocpCmd.PersistentFlags().BoolVar(&workloadConfig.Reporting, "reporting", false, "Enable benchmark report indexing") ocpCmd.MarkFlagsRequiredTogether("es-server", "es-index") ocpCmd.MarkFlagsMutuallyExclusive("es-server", "local-indexing") ocpCmd.PersistentPreRun = func(cmd *cobra.Command, args []string) { rootCmd.PersistentPreRun(cmd, args) - if *esServer != "" || *localIndexing { - if *esServer != "" { - indexingType = indexers.ElasticIndexer + if workloadConfig.EsServer != "" || *localIndexing { + if workloadConfig.EsServer != "" { + workloadConfig.Indexer = indexers.ElasticIndexer } else { - indexingType = indexers.LocalIndexer + workloadConfig.Indexer = indexers.LocalIndexer } } - envVars := map[string]string{ - "ES_SERVER": strings.TrimSuffix(*esServer, "/"), - "ES_INDEX": *esIndex, - "QPS": fmt.Sprintf("%d", *qps), - "BURST": fmt.Sprintf("%d", *burst), - "GC": fmt.Sprintf("%v", *gc), - "GC_METRICS": fmt.Sprintf("%v", *gcMetrics), - "INDEXING_TYPE": string(indexingType), - } - wh = workloads.NewWorkloadHelper(envVars, *alerting, *profileType, ocpConfig, *timeout, *metricsEndpoint) - wh.Metadata.UUID = *uuid + wh = workloads.NewWorkloadHelper(workloadConfig, ocpConfig) if *extract { if err := wh.ExtractWorkload(cmd.Name(), workloads.MetricsProfileMap[cmd.Name()]); err != nil { log.Fatal(err) diff --git a/pkg/measurements/pod_latency.go b/pkg/measurements/pod_latency.go index 0969e648a..8f4d62028 100644 --- a/pkg/measurements/pod_latency.go +++ b/pkg/measurements/pod_latency.go @@ -28,7 +28,6 @@ import ( "github.com/cloud-bulldozer/kube-burner/pkg/measurements/types" log "github.com/sirupsen/logrus" corev1 "k8s.io/api/core/v1" - v1 "k8s.io/api/core/v1" "k8s.io/client-go/rest" "k8s.io/client-go/tools/cache" ) @@ -93,7 +92,7 @@ func (p *podLatency) handleCreatePod(obj interface{}) { } func (p *podLatency) handleUpdatePod(obj interface{}) { - pod := obj.(*v1.Pod) + pod := obj.(*corev1.Pod) p.metricLock.Lock() defer p.metricLock.Unlock() if pm, exists := p.metrics[string(pod.UID)]; exists && pm.podReady.IsZero() { @@ -170,7 +169,7 @@ func (p *podLatency) stop() error { } if globalCfg.IndexerConfig.Type != "" { if factory.jobConfig.SkipIndexing { - log.Infof("Skipping pod latency data indexing in job") + log.Infof("Skipping pod latency data indexing in job: %s", factory.jobConfig.Name) } else { log.Infof("Indexing pod latency data for job: %s", factory.jobConfig.Name) p.index() @@ -190,6 +189,7 @@ func (p *podLatency) stop() error { // index sends metrics to the configured indexer func (p *podLatency) index() { + log.Infof("Indexing pod latency data for job: %s", factory.jobConfig.Name) metricMap := map[string][]interface{}{ podLatencyMeasurement: p.normLatencies, podLatencyQuantilesMeasurement: p.latencyQuantiles, @@ -201,7 +201,7 @@ func (p *podLatency) index() { indexingOpts := indexers.IndexingOpts{ MetricName: fmt.Sprintf("%s-%s", metricName, factory.jobConfig.Name), } - log.Debugf("Indexing [%d] documents", len(data)) + log.Debugf("Indexing [%d] documents: %s", len(data), metricName) resp, err := (*factory.indexer).Index(data, indexingOpts) if err != nil { log.Error(err.Error()) @@ -327,7 +327,7 @@ func (p *podLatency) validateConfig() error { // validatePod validates a pod based on job type and returns its timestamp for latency calculation. // It returns a timestamp and a boolean value indicating validation details. -func validatePod(jobType config.JobType, pod *v1.Pod) (time.Time, bool) { +func validatePod(jobType config.JobType, pod *corev1.Pod) (time.Time, bool) { if jobType == config.CreationJob { runid, exists := pod.Labels["kube-burner-runid"] if exists && runid == globalCfg.RUNID { diff --git a/pkg/workloads/helpers.go b/pkg/workloads/helpers.go index 93e09cd19..4e0f80d41 100644 --- a/pkg/workloads/helpers.go +++ b/pkg/workloads/helpers.go @@ -33,7 +33,6 @@ import ( "github.com/cloud-bulldozer/kube-burner/pkg/util" "github.com/cloud-bulldozer/kube-burner/pkg/util/metrics" log "github.com/sirupsen/logrus" - "k8s.io/client-go/rest" "k8s.io/client-go/tools/clientcmd" ) @@ -45,43 +44,10 @@ const ( reportProfile = "metrics-report.yml" ) -type ProfileType string - -const ( - regular ProfileType = "regular" - reporting ProfileType = "reporting" - both ProfileType = "both" -) - -type BenchmarkMetadata struct { - ocpmetadata.ClusterMetadata - UUID string `json:"uuid"` - Benchmark string `json:"benchmark"` - Timestamp time.Time `json:"timestamp"` - EndDate time.Time `json:"endDate"` - Passed bool `json:"passed"` - ExecutionErrors string `json:"executionErrors"` - UserMetadata map[string]interface{} `json:"metadata,omitempty"` -} - -type WorkloadHelper struct { - envVars map[string]string - prometheusURL string - prometheusToken string - MetricsEndpoint string - timeout time.Duration - Metadata BenchmarkMetadata - alerting bool - ocpConfig embed.FS - OcpMetaAgent ocpmetadata.Metadata - profileType string - restConfig *rest.Config -} - var configSpec config.Spec // NewWorkloadHelper initializes workloadHelper -func NewWorkloadHelper(envVars map[string]string, alerting bool, profileType string, ocpConfig embed.FS, timeout time.Duration, metricsEndpoint string) WorkloadHelper { +func NewWorkloadHelper(config Config, ocpConfig embed.FS) WorkloadHelper { var kubeconfig string if os.Getenv("KUBECONFIG") != "" { kubeconfig = os.Getenv("KUBECONFIG") @@ -97,14 +63,10 @@ func NewWorkloadHelper(envVars map[string]string, alerting bool, profileType str log.Fatal(err.Error()) } return WorkloadHelper{ - envVars: envVars, - alerting: alerting, - MetricsEndpoint: metricsEndpoint, - ocpConfig: ocpConfig, - OcpMetaAgent: ocpMetadata, - timeout: timeout, - restConfig: restConfig, - profileType: profileType, + Config: config, + ocpConfig: ocpConfig, + OcpMetaAgent: ocpMetadata, + restConfig: restConfig, } } @@ -114,18 +76,26 @@ var indexer *indexers.Indexer func (wh *WorkloadHelper) SetKubeBurnerFlags() { var err error if wh.MetricsEndpoint == "" { - prometheusURL, prometheusToken, err := wh.OcpMetaAgent.GetPrometheus() + wh.prometheusURL, wh.prometheusToken, err = wh.OcpMetaAgent.GetPrometheus() if err != nil { log.Fatal("Error obtaining Prometheus information: ", err.Error()) } - wh.prometheusURL = prometheusURL - wh.prometheusToken = prometheusToken } - wh.envVars["INGRESS_DOMAIN"], err = wh.OcpMetaAgent.GetDefaultIngressDomain() + ingressDomain, err := wh.OcpMetaAgent.GetDefaultIngressDomain() if err != nil { log.Fatal("Error obtaining default ingress domain: ", err.Error()) } - for k, v := range wh.envVars { + envVars := map[string]string{ + "ES_SERVER": wh.EsServer, + "ES_INDEX": wh.Esindex, + "QPS": fmt.Sprintf("%d", wh.QPS), + "BURST": fmt.Sprintf("%d", wh.Burst), + "INGRESS_DOMAIN": ingressDomain, + "GC": fmt.Sprintf("%v", wh.Gc), + "GC_METRICS": fmt.Sprintf("%v", wh.GcMetrics), + "INDEXING_TYPE": string(wh.Indexer), + } + for k, v := range envVars { os.Setenv(k, v) } } @@ -144,6 +114,7 @@ func (wh *WorkloadHelper) GatherMetadata(userMetadata string) error { } wh.Metadata.UserMetadata = userMetadataContent } + wh.Metadata.UUID = wh.UUID wh.Metadata.Timestamp = time.Now().UTC() return nil } @@ -183,7 +154,7 @@ func (wh *WorkloadHelper) run(workload, metricsProfile string) { log.Fatalf("Error reading configuration file %s: %s", configFile, err) } } - configSpec, err = config.Parse(wh.Metadata.UUID, f) + configSpec, err = config.Parse(wh.UUID, f) if err != nil { log.Fatal(err) } @@ -214,7 +185,7 @@ func (wh *WorkloadHelper) run(workload, metricsProfile string) { Profile: reportProfile, Token: wh.prometheusToken, } - switch ProfileType(wh.profileType) { + switch ProfileType(wh.ProfileType) { case regular: metricsEndpoints = append(metricsEndpoints, regularProfile) case reporting: @@ -226,7 +197,7 @@ func (wh *WorkloadHelper) run(workload, metricsProfile string) { case both: metricsEndpoints = append(metricsEndpoints, regularProfile, reportingProfile) default: - log.Fatalf("Metrics profile type not supported: %v", wh.profileType) + log.Fatalf("Metrics profile type not supported: %v", wh.ProfileType) } } for _, metricsEndpoint := range metricsEndpoints { @@ -243,7 +214,7 @@ func (wh *WorkloadHelper) run(workload, metricsProfile string) { if err != nil { log.Fatal(err) } - if wh.alerting && metricsEndpoint.AlertProfile != "" { + if wh.Alerting && metricsEndpoint.AlertProfile != "" { alertM, err = alerting.NewAlertManager(metricsEndpoint.AlertProfile, wh.Metadata.UUID, indexer, p, embedConfig) if err != nil { log.Fatal(err) @@ -253,7 +224,8 @@ func (wh *WorkloadHelper) run(workload, metricsProfile string) { alertMs = append(alertMs, alertM) alertM = nil } - rc, err = burner.Run(configSpec, prometheusClients, alertMs, indexer, wh.timeout, metadata) + configSpec.GlobalConfig.GCMetrics = wh.GcMetrics + rc, err = burner.Run(configSpec, prometheusClients, alertMs, indexer, wh.Timeout, metadata) if err != nil { wh.Metadata.ExecutionErrors = err.Error() log.Error(err) @@ -262,7 +234,7 @@ func (wh *WorkloadHelper) run(workload, metricsProfile string) { if indexerConfig.Type != "" { IndexMetadata(indexer, wh.Metadata) } - log.Info("๐Ÿ‘‹ Exiting kube-burner ", wh.Metadata.UUID) + log.Info("๐Ÿ‘‹ Exiting kube-burner ", wh.UUID) os.Exit(rc) } diff --git a/pkg/workloads/types.go b/pkg/workloads/types.go new file mode 100644 index 000000000..a2f23a8b7 --- /dev/null +++ b/pkg/workloads/types.go @@ -0,0 +1,69 @@ +package workloads + +import ( + "embed" + "time" + + "github.com/cloud-bulldozer/go-commons/indexers" + ocpmetadata "github.com/cloud-bulldozer/go-commons/ocp-metadata" + "k8s.io/client-go/rest" +) + +type ProfileType string + +var MetricsProfileMap = map[string]string{ + "cluster-density-ms": "metrics-aggregated.yml", + "cluster-density-v2": "metrics-aggregated.yml", + "cluster-density": "metrics-aggregated.yml", + "crd-scale": "metrics-aggregated.yml", + "node-density": "metrics.yml", + "node-density-heavy": "metrics.yml", + "node-density-cni": "metrics.yml", + "networkpolicy-multitenant": "metrics.yml", + "networkpolicy-matchlabels": "metrics.yml", + "networkpolicy-matchexpressions": "metrics.yml", + "pvc-density": "metrics.yml", +} + +const ( + regular ProfileType = "regular" + reporting ProfileType = "reporting" + both ProfileType = "both" +) + +type Config struct { + UUID string + EsServer string + Esindex string + QPS int + Burst int + Gc bool + GcMetrics bool + Indexer indexers.IndexerType + Alerting bool + Reporting bool + Timeout time.Duration + MetricsEndpoint string + ProfileType string +} + +type BenchmarkMetadata struct { + ocpmetadata.ClusterMetadata + UUID string `json:"uuid"` + Benchmark string `json:"benchmark"` + Timestamp time.Time `json:"timestamp"` + EndDate time.Time `json:"endDate"` + Passed bool `json:"passed"` + ExecutionErrors string `json:"executionErrors"` + UserMetadata map[string]interface{} `json:"metadata,omitempty"` +} + +type WorkloadHelper struct { + Config + prometheusURL string + prometheusToken string + Metadata BenchmarkMetadata + ocpConfig embed.FS + OcpMetaAgent ocpmetadata.Metadata + restConfig *rest.Config +} diff --git a/pkg/workloads/workloads.go b/pkg/workloads/workloads.go deleted file mode 100644 index 0ffd5cabc..000000000 --- a/pkg/workloads/workloads.go +++ /dev/null @@ -1,15 +0,0 @@ -package workloads - -var MetricsProfileMap = map[string]string{ - "cluster-density-ms": "metrics-aggregated.yml", - "cluster-density-v2": "metrics-aggregated.yml", - "cluster-density": "metrics-aggregated.yml", - "crd-scale": "metrics-aggregated.yml", - "node-density": "metrics.yml", - "node-density-heavy": "metrics.yml", - "node-density-cni": "metrics.yml", - "networkpolicy-multitenant": "metrics.yml", - "networkpolicy-matchlabels": "metrics.yml", - "networkpolicy-matchexpressions": "metrics.yml", - "pvc-density": "metrics.yml", -} From 17fae8ba38697a226f2aebb7c12620b92853c6aa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Sevilla?= Date: Wed, 8 Nov 2023 16:42:06 +0100 Subject: [PATCH 2/7] Remove duped section from docs (#506) Signed-off-by: Raul Sevilla --- mkdocs.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/mkdocs.yml b/mkdocs.yml index ef0ec2d24..d695e478d 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -14,7 +14,6 @@ nav: - OpenShift Wrapper: ocp.md - Contributing: - contributing/index.md - - Contributing: contributing/index.md - GitHub Workflows: - contributing/pullrequest.md - contributing/release.md From b7ab7ebb8727e745729ea8dc12fe0da8cfb318b4 Mon Sep 17 00:00:00 2001 From: Andrew Collins Date: Thu, 9 Nov 2023 12:14:01 -0500 Subject: [PATCH 3/7] Set more accurate 70m cpu request on cluster-density builds for better scheduling (#508) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Raรบl Sevilla --- cmd/kube-burner/ocp-config/cluster-density-v2/build.yml | 2 +- cmd/kube-burner/ocp-config/cluster-density/build.yml | 4 ++++ examples/workloads/cluster-density/templates/build.yml | 4 ++++ 3 files changed, 9 insertions(+), 1 deletion(-) diff --git a/cmd/kube-burner/ocp-config/cluster-density-v2/build.yml b/cmd/kube-burner/ocp-config/cluster-density-v2/build.yml index 706cea168..e38253b4a 100644 --- a/cmd/kube-burner/ocp-config/cluster-density-v2/build.yml +++ b/cmd/kube-burner/ocp-config/cluster-density-v2/build.yml @@ -6,7 +6,7 @@ metadata: spec: resources: requests: - cpu: 20m + cpu: 70m memory: "10Mi" nodeSelector: node-role.kubernetes.io/worker: "" diff --git a/cmd/kube-burner/ocp-config/cluster-density/build.yml b/cmd/kube-burner/ocp-config/cluster-density/build.yml index 7ea974586..e38253b4a 100644 --- a/cmd/kube-burner/ocp-config/cluster-density/build.yml +++ b/cmd/kube-burner/ocp-config/cluster-density/build.yml @@ -4,6 +4,10 @@ apiVersion: build.openshift.io/v1 metadata: name: {{.JobName}}-{{.Replica}} spec: + resources: + requests: + cpu: 70m + memory: "10Mi" nodeSelector: node-role.kubernetes.io/worker: "" serviceAccount: builder diff --git a/examples/workloads/cluster-density/templates/build.yml b/examples/workloads/cluster-density/templates/build.yml index 4dd5097c2..7d650ffc6 100644 --- a/examples/workloads/cluster-density/templates/build.yml +++ b/examples/workloads/cluster-density/templates/build.yml @@ -4,6 +4,10 @@ apiVersion: build.openshift.io/v1 metadata: name: {{.JobName}}-{{.Replica}} spec: + resources: + requests: + cpu: 70m + memory: "10Mi" nodeSelector: node-role.kubernetes.io/worker: "" serviceAccount: builder From 3b168fda321caa2e74b379b3dc32873944c9231a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Sevilla?= Date: Fri, 10 Nov 2023 10:55:41 +0100 Subject: [PATCH 4/7] Rate limit per clientSet request and not per namespace (#509) Rate limit per request and not per namespace Signed-off-by: Raul Sevilla --- pkg/burner/waiters.go | 114 +++++++++++++++++++++++------------------- 1 file changed, 62 insertions(+), 52 deletions(-) diff --git a/pkg/burner/waiters.go b/pkg/burner/waiters.go index 3dd7909cd..260d07238 100644 --- a/pkg/burner/waiters.go +++ b/pkg/burner/waiters.go @@ -31,50 +31,51 @@ import ( ) func (ex *Executor) waitForObjects(ns string, limiter *rate.Limiter) { - limiter.Wait(context.TODO()) for _, obj := range ex.objects { - if obj.Wait { - if obj.WaitOptions.ForCondition != "" { - if !obj.Namespaced { - ns = "" - } - waitForCondition(obj.gvr, ns, obj.WaitOptions.ForCondition, ex.MaxWaitTimeout) - } else { - switch obj.kind { - case "Deployment": - waitForDeployments(ns, ex.MaxWaitTimeout) - case "ReplicaSet": - waitForRS(ns, ex.MaxWaitTimeout) - case "ReplicationController": - waitForRC(ns, ex.MaxWaitTimeout) - case "StatefulSet": - waitForStatefulSet(ns, ex.MaxWaitTimeout) - case "DaemonSet": - waitForDS(ns, ex.MaxWaitTimeout) - case "Pod": - waitForPod(ns, ex.MaxWaitTimeout) - case "Build", "BuildConfig": - waitForBuild(ns, ex.MaxWaitTimeout, obj.Replicas) - case "VirtualMachine": - waitForVM(ns, ex.MaxWaitTimeout) - case "VirtualMachineInstance": - waitForVMI(ns, ex.MaxWaitTimeout) - case "VirtualMachineInstanceReplicaSet": - waitForVMIRS(ns, ex.MaxWaitTimeout) - case "Job": - waitForJob(ns, ex.MaxWaitTimeout) - case "PersistentVolumeClaim": - waitForPVC(ns, ex.MaxWaitTimeout) - } + if !obj.Wait { + continue + } + if obj.WaitOptions.ForCondition != "" { + if !obj.Namespaced { + ns = "" + } + waitForCondition(obj.gvr, ns, obj.WaitOptions.ForCondition, ex.MaxWaitTimeout, limiter) + } else { + switch obj.kind { + case "Deployment": + waitForDeployments(ns, ex.MaxWaitTimeout, limiter) + case "ReplicaSet": + waitForRS(ns, ex.MaxWaitTimeout, limiter) + case "ReplicationController": + waitForRC(ns, ex.MaxWaitTimeout, limiter) + case "StatefulSet": + waitForStatefulSet(ns, ex.MaxWaitTimeout, limiter) + case "DaemonSet": + waitForDS(ns, ex.MaxWaitTimeout, limiter) + case "Pod": + waitForPod(ns, ex.MaxWaitTimeout, limiter) + case "Build", "BuildConfig": + waitForBuild(ns, ex.MaxWaitTimeout, obj.Replicas, limiter) + case "VirtualMachine": + waitForVM(ns, ex.MaxWaitTimeout, limiter) + case "VirtualMachineInstance": + waitForVMI(ns, ex.MaxWaitTimeout, limiter) + case "VirtualMachineInstanceReplicaSet": + waitForVMIRS(ns, ex.MaxWaitTimeout, limiter) + case "Job": + waitForJob(ns, ex.MaxWaitTimeout, limiter) + case "PersistentVolumeClaim": + waitForPVC(ns, ex.MaxWaitTimeout, limiter) } } } log.Infof("Actions in namespace %v completed", ns) } -func waitForDeployments(ns string, maxWaitTimeout time.Duration) { +func waitForDeployments(ns string, maxWaitTimeout time.Duration, limiter *rate.Limiter) { // TODO handle errors such as timeouts wait.PollUntilContextTimeout(context.TODO(), time.Second, maxWaitTimeout, true, func(ctx context.Context) (done bool, err error) { + limiter.Wait(context.TODO()) deps, err := ClientSet.AppsV1().Deployments(ns).List(context.TODO(), metav1.ListOptions{}) if err != nil { return false, err @@ -89,8 +90,9 @@ func waitForDeployments(ns string, maxWaitTimeout time.Duration) { }) } -func waitForRS(ns string, maxWaitTimeout time.Duration) { +func waitForRS(ns string, maxWaitTimeout time.Duration, limiter *rate.Limiter) { wait.PollUntilContextTimeout(context.TODO(), time.Second, maxWaitTimeout, true, func(ctx context.Context) (done bool, err error) { + limiter.Wait(context.TODO()) rss, err := ClientSet.AppsV1().ReplicaSets(ns).List(context.TODO(), metav1.ListOptions{}) if err != nil { return false, err @@ -105,8 +107,9 @@ func waitForRS(ns string, maxWaitTimeout time.Duration) { }) } -func waitForStatefulSet(ns string, maxWaitTimeout time.Duration) { +func waitForStatefulSet(ns string, maxWaitTimeout time.Duration, limiter *rate.Limiter) { wait.PollUntilContextTimeout(context.TODO(), time.Second, maxWaitTimeout, true, func(ctx context.Context) (done bool, err error) { + limiter.Wait(context.TODO()) stss, err := ClientSet.AppsV1().StatefulSets(ns).List(context.TODO(), metav1.ListOptions{}) if err != nil { return false, err @@ -121,8 +124,9 @@ func waitForStatefulSet(ns string, maxWaitTimeout time.Duration) { }) } -func waitForPVC(ns string, maxWaitTimeout time.Duration) { +func waitForPVC(ns string, maxWaitTimeout time.Duration, limiter *rate.Limiter) { wait.PollUntilContextTimeout(context.TODO(), time.Second, maxWaitTimeout, true, func(ctx context.Context) (done bool, err error) { + limiter.Wait(context.TODO()) pvc, err := ClientSet.CoreV1().PersistentVolumeClaims(ns).List(context.TODO(), metav1.ListOptions{FieldSelector: "status.phase!=Bound"}) if err != nil { return false, err @@ -131,8 +135,9 @@ func waitForPVC(ns string, maxWaitTimeout time.Duration) { }) } -func waitForRC(ns string, maxWaitTimeout time.Duration) { +func waitForRC(ns string, maxWaitTimeout time.Duration, limiter *rate.Limiter) { wait.PollUntilContextTimeout(context.TODO(), time.Second, maxWaitTimeout, true, func(ctx context.Context) (done bool, err error) { + limiter.Wait(context.TODO()) rcs, err := ClientSet.CoreV1().ReplicationControllers(ns).List(context.TODO(), metav1.ListOptions{}) if err != nil { return false, err @@ -147,8 +152,9 @@ func waitForRC(ns string, maxWaitTimeout time.Duration) { }) } -func waitForDS(ns string, maxWaitTimeout time.Duration) { +func waitForDS(ns string, maxWaitTimeout time.Duration, limiter *rate.Limiter) { wait.PollUntilContextTimeout(context.TODO(), time.Second, maxWaitTimeout, true, func(ctx context.Context) (done bool, err error) { + limiter.Wait(context.TODO()) dss, err := ClientSet.AppsV1().DaemonSets(ns).List(context.TODO(), metav1.ListOptions{}) if err != nil { return false, err @@ -163,8 +169,9 @@ func waitForDS(ns string, maxWaitTimeout time.Duration) { }) } -func waitForPod(ns string, maxWaitTimeout time.Duration) { +func waitForPod(ns string, maxWaitTimeout time.Duration, limiter *rate.Limiter) { wait.PollUntilContextTimeout(context.TODO(), time.Second, maxWaitTimeout, true, func(ctx context.Context) (done bool, err error) { + limiter.Wait(context.TODO()) pods, err := ClientSet.CoreV1().Pods(ns).List(context.TODO(), metav1.ListOptions{FieldSelector: "status.phase!=Running"}) if err != nil { return false, err @@ -173,7 +180,7 @@ func waitForPod(ns string, maxWaitTimeout time.Duration) { }) } -func waitForBuild(ns string, maxWaitTimeout time.Duration, expected int) { +func waitForBuild(ns string, maxWaitTimeout time.Duration, expected int, limiter *rate.Limiter) { buildStatus := []string{"New", "Pending", "Running"} var build types.UnstructuredContent gvr := schema.GroupVersionResource{ @@ -182,6 +189,7 @@ func waitForBuild(ns string, maxWaitTimeout time.Duration, expected int) { Resource: types.OpenShiftBuildResource, } wait.PollUntilContextTimeout(context.TODO(), time.Second, maxWaitTimeout, true, func(ctx context.Context) (done bool, err error) { + limiter.Wait(context.TODO()) builds, err := DynamicClient.Resource(gvr).Namespace(ns).List(context.TODO(), metav1.ListOptions{}) if err != nil { return false, err @@ -207,23 +215,24 @@ func waitForBuild(ns string, maxWaitTimeout time.Duration, expected int) { }) } -func waitForJob(ns string, maxWaitTimeout time.Duration) { +func waitForJob(ns string, maxWaitTimeout time.Duration, limiter *rate.Limiter) { gvr := schema.GroupVersionResource{ Group: "batch", Version: "v1", Resource: "jobs", } - verifyCondition(gvr, ns, "Complete", maxWaitTimeout) + verifyCondition(gvr, ns, "Complete", maxWaitTimeout, limiter) } -func waitForCondition(gvr schema.GroupVersionResource, ns, condition string, maxWaitTimeout time.Duration) { - verifyCondition(gvr, ns, condition, maxWaitTimeout) +func waitForCondition(gvr schema.GroupVersionResource, ns, condition string, maxWaitTimeout time.Duration, limiter *rate.Limiter) { + verifyCondition(gvr, ns, condition, maxWaitTimeout, limiter) } -func verifyCondition(gvr schema.GroupVersionResource, ns, condition string, maxWaitTimeout time.Duration) { +func verifyCondition(gvr schema.GroupVersionResource, ns, condition string, maxWaitTimeout time.Duration, limiter *rate.Limiter) { var uObj types.UnstructuredContent wait.PollUntilContextTimeout(context.TODO(), 10*time.Second, maxWaitTimeout, true, func(ctx context.Context) (done bool, err error) { var objs *unstructured.UnstructuredList + limiter.Wait(context.TODO()) if ns != "" { objs, err = DynamicClient.Resource(gvr).Namespace(ns).List(context.TODO(), metav1.ListOptions{}) } else { @@ -256,25 +265,25 @@ func verifyCondition(gvr schema.GroupVersionResource, ns, condition string, maxW }) } -func waitForVM(ns string, maxWaitTimeout time.Duration) { +func waitForVM(ns string, maxWaitTimeout time.Duration, limiter *rate.Limiter) { vmGVR := schema.GroupVersionResource{ Group: types.KubevirtGroup, Version: types.KubevirtAPIVersion, Resource: types.VirtualMachineResource, } - verifyCondition(vmGVR, ns, "Ready", maxWaitTimeout) + verifyCondition(vmGVR, ns, "Ready", maxWaitTimeout, limiter) } -func waitForVMI(ns string, maxWaitTimeout time.Duration) { +func waitForVMI(ns string, maxWaitTimeout time.Duration, limiter *rate.Limiter) { vmiGVR := schema.GroupVersionResource{ Group: types.KubevirtGroup, Version: types.KubevirtAPIVersion, Resource: types.VirtualMachineInstanceResource, } - verifyCondition(vmiGVR, ns, "Ready", maxWaitTimeout) + verifyCondition(vmiGVR, ns, "Ready", maxWaitTimeout, limiter) } -func waitForVMIRS(ns string, maxWaitTimeout time.Duration) { +func waitForVMIRS(ns string, maxWaitTimeout time.Duration, limiter *rate.Limiter) { var rs types.UnstructuredContent vmiGVRRS := schema.GroupVersionResource{ Group: types.KubevirtGroup, @@ -282,6 +291,7 @@ func waitForVMIRS(ns string, maxWaitTimeout time.Duration) { Resource: types.VirtualMachineInstanceReplicaSetResource, } wait.PollUntilContextTimeout(context.TODO(), 10*time.Second, maxWaitTimeout, true, func(ctx context.Context) (done bool, err error) { + limiter.Wait(context.TODO()) objs, err := DynamicClient.Resource(vmiGVRRS).Namespace(ns).List(context.TODO(), metav1.ListOptions{}) if err != nil { log.Debugf("VMIRS error %v", err) From b1350f85b8b9fc1f438cc8d2b419f4cee649deda Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Sevilla?= Date: Fri, 10 Nov 2023 19:55:24 +0100 Subject: [PATCH 5/7] =?UTF-8?q?Pod=20latency=20threshold=20flag=20for=20al?= =?UTF-8?q?l=20cluster-density=20and=20node-density=20wo=E2=80=A6=20(#487)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Pod latency threshold flag for all cluster-density and node-density workloads Signed-off-by: Raul Sevilla --------- Signed-off-by: Raul Sevilla --- .../ocp-config/cluster-density-ms/cluster-density-ms.yml | 4 ++++ .../ocp-config/cluster-density-v2/cluster-density-v2.yml | 4 ++++ .../ocp-config/cluster-density/cluster-density.yml | 4 ++++ .../ocp-config/node-density-cni/node-density-cni.yml | 4 ++++ pkg/workloads/cluster-density.go | 3 +++ pkg/workloads/index.go | 2 +- pkg/workloads/node-density-cni.go | 4 ++++ pkg/workloads/node-density-heavy.go | 2 +- 8 files changed, 25 insertions(+), 2 deletions(-) diff --git a/cmd/kube-burner/ocp-config/cluster-density-ms/cluster-density-ms.yml b/cmd/kube-burner/ocp-config/cluster-density-ms/cluster-density-ms.yml index 13b135f5d..91cd84665 100644 --- a/cmd/kube-burner/ocp-config/cluster-density-ms/cluster-density-ms.yml +++ b/cmd/kube-burner/ocp-config/cluster-density-ms/cluster-density-ms.yml @@ -9,6 +9,10 @@ global: type: {{.INDEXING_TYPE}} measurements: - name: podLatency + thresholds: + - conditionType: Ready + metric: P99 + threshold: {{.POD_READY_THRESHOLD}} jobs: - name: cluster-density-ms namespace: cluster-density-ms diff --git a/cmd/kube-burner/ocp-config/cluster-density-v2/cluster-density-v2.yml b/cmd/kube-burner/ocp-config/cluster-density-v2/cluster-density-v2.yml index dba833189..28ff6743b 100644 --- a/cmd/kube-burner/ocp-config/cluster-density-v2/cluster-density-v2.yml +++ b/cmd/kube-burner/ocp-config/cluster-density-v2/cluster-density-v2.yml @@ -9,6 +9,10 @@ global: type: {{.INDEXING_TYPE}} measurements: - name: podLatency + thresholds: + - conditionType: Ready + metric: P99 + threshold: {{.POD_READY_THRESHOLD}} jobs: - name: cluster-density-v2 namespace: cluster-density-v2 diff --git a/cmd/kube-burner/ocp-config/cluster-density/cluster-density.yml b/cmd/kube-burner/ocp-config/cluster-density/cluster-density.yml index 1dae59fab..b482b8ab6 100644 --- a/cmd/kube-burner/ocp-config/cluster-density/cluster-density.yml +++ b/cmd/kube-burner/ocp-config/cluster-density/cluster-density.yml @@ -9,6 +9,10 @@ global: type: {{.INDEXING_TYPE}} measurements: - name: podLatency + thresholds: + - conditionType: Ready + metric: P99 + threshold: {{.POD_READY_THRESHOLD}} jobs: - name: cluster-density namespace: cluster-density diff --git a/cmd/kube-burner/ocp-config/node-density-cni/node-density-cni.yml b/cmd/kube-burner/ocp-config/node-density-cni/node-density-cni.yml index c0dfa3a6c..4b4aa77d1 100644 --- a/cmd/kube-burner/ocp-config/node-density-cni/node-density-cni.yml +++ b/cmd/kube-burner/ocp-config/node-density-cni/node-density-cni.yml @@ -9,6 +9,10 @@ global: type: {{.INDEXING_TYPE}} measurements: - name: podLatency + thresholds: + - conditionType: Ready + metric: P99 + threshold: {{.POD_READY_THRESHOLD}} jobs: - name: node-density-cni namespace: node-density-cni diff --git a/pkg/workloads/cluster-density.go b/pkg/workloads/cluster-density.go index 09b81f561..59679260e 100644 --- a/pkg/workloads/cluster-density.go +++ b/pkg/workloads/cluster-density.go @@ -28,6 +28,7 @@ func NewClusterDensity(wh *WorkloadHelper, variant string) *cobra.Command { var iterations, churnPercent int var churn bool var churnDelay, churnDuration time.Duration + var podReadyThreshold time.Duration cmd := &cobra.Command{ Use: variant, Short: fmt.Sprintf("Runs %v workload", variant), @@ -41,11 +42,13 @@ func NewClusterDensity(wh *WorkloadHelper, variant string) *cobra.Command { os.Setenv("CHURN_DURATION", fmt.Sprintf("%v", churnDuration)) os.Setenv("CHURN_DELAY", fmt.Sprintf("%v", churnDelay)) os.Setenv("CHURN_PERCENT", fmt.Sprint(churnPercent)) + os.Setenv("POD_READY_THRESHOLD", fmt.Sprintf("%v", podReadyThreshold)) }, Run: func(cmd *cobra.Command, args []string) { wh.run(cmd.Name(), MetricsProfileMap[cmd.Name()]) }, } + cmd.Flags().DurationVar(&podReadyThreshold, "pod-ready-threshold", 2*time.Minute, "Pod ready timeout threshold") cmd.Flags().IntVar(&iterations, "iterations", 0, fmt.Sprintf("%v iterations", variant)) cmd.Flags().BoolVar(&churn, "churn", true, "Enable churning") cmd.Flags().DurationVar(&churnDuration, "churn-duration", 1*time.Hour, "Churn duration") diff --git a/pkg/workloads/index.go b/pkg/workloads/index.go index 7c98b92b4..6487803f1 100644 --- a/pkg/workloads/index.go +++ b/pkg/workloads/index.go @@ -116,7 +116,7 @@ func NewIndex(metricsEndpoint *string, metadata *BenchmarkMetadata, ocpMetaAgent } }, } - cmd.Flags().StringVar(&metricsProfile, "metrics-profile", "metrics.yml", "Metrics profile file") + cmd.Flags().StringVarP(&metricsProfile, "metrics-profile", "m", "metrics.yml", "Metrics profile file") cmd.Flags().StringVar(&metricsDirectory, "metrics-directory", "collected-metrics", "Directory to dump the metrics files in, when using default local indexing") cmd.Flags().DurationVar(&prometheusStep, "step", 30*time.Second, "Prometheus step size") cmd.Flags().Int64Var(&start, "start", time.Now().Unix()-3600, "Epoch start time") diff --git a/pkg/workloads/node-density-cni.go b/pkg/workloads/node-density-cni.go index 529195c5e..7048b06a0 100644 --- a/pkg/workloads/node-density-cni.go +++ b/pkg/workloads/node-density-cni.go @@ -17,6 +17,7 @@ package workloads import ( "fmt" "os" + "time" log "github.com/sirupsen/logrus" @@ -27,6 +28,7 @@ import ( func NewNodeDensityCNI(wh *WorkloadHelper) *cobra.Command { var podsPerNode int var namespacedIterations bool + var podReadyThreshold time.Duration var iterationsPerNamespace int cmd := &cobra.Command{ Use: "node-density-cni", @@ -42,11 +44,13 @@ func NewNodeDensityCNI(wh *WorkloadHelper) *cobra.Command { os.Setenv("JOB_ITERATIONS", fmt.Sprint((totalPods-podCount)/2)) os.Setenv("NAMESPACED_ITERATIONS", fmt.Sprint(namespacedIterations)) os.Setenv("ITERATIONS_PER_NAMESPACE", fmt.Sprint(iterationsPerNamespace)) + os.Setenv("POD_READY_THRESHOLD", fmt.Sprintf("%v", podReadyThreshold)) }, Run: func(cmd *cobra.Command, args []string) { wh.run(cmd.Name(), MetricsProfileMap[cmd.Name()]) }, } + cmd.Flags().DurationVar(&podReadyThreshold, "pod-ready-threshold", 1*time.Minute, "Pod ready timeout threshold") cmd.Flags().IntVar(&podsPerNode, "pods-per-node", 245, "Pods per node") cmd.Flags().BoolVar(&namespacedIterations, "namespaced-iterations", true, "Namespaced iterations") cmd.Flags().IntVar(&iterationsPerNamespace, "iterations-per-namespace", 1000, "Iterations per namespace") diff --git a/pkg/workloads/node-density-heavy.go b/pkg/workloads/node-density-heavy.go index 1f3372077..fba04fba4 100644 --- a/pkg/workloads/node-density-heavy.go +++ b/pkg/workloads/node-density-heavy.go @@ -52,7 +52,7 @@ func NewNodeDensityHeavy(wh *WorkloadHelper) *cobra.Command { wh.run(cmd.Name(), MetricsProfileMap[cmd.Name()]) }, } - cmd.Flags().DurationVar(&podReadyThreshold, "pod-ready-threshold", 1*time.Hour, "Pod ready timeout threshold") + cmd.Flags().DurationVar(&podReadyThreshold, "pod-ready-threshold", 2*time.Minute, "Pod ready timeout threshold") cmd.Flags().DurationVar(&probesPeriod, "probes-period", 10*time.Second, "Perf app readiness/livenes probes period") cmd.Flags().IntVar(&podsPerNode, "pods-per-node", 245, "Pods per node") cmd.Flags().BoolVar(&namespacedIterations, "namespaced-iterations", true, "Namespaced iterations") From c9c175912546b2d9d5c0bc3b625be7ee54ed98bf Mon Sep 17 00:00:00 2001 From: Raul Sevilla Date: Tue, 7 Nov 2023 17:17:34 +0100 Subject: [PATCH 6/7] Foof Signed-off-by: Raul Sevilla --- foo | 1 + 1 file changed, 1 insertion(+) create mode 100644 foo diff --git a/foo b/foo new file mode 100644 index 000000000..257cc5642 --- /dev/null +++ b/foo @@ -0,0 +1 @@ +foo From f62da9ed1058739d809f259e240228d7c4a0648c Mon Sep 17 00:00:00 2001 From: Raul Sevilla Date: Mon, 13 Nov 2023 11:19:17 +0100 Subject: [PATCH 7/7] Hello there Signed-off-by: Raul Sevilla --- foo | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/foo b/foo index 257cc5642..099a1da4d 100644 --- a/foo +++ b/foo @@ -1 +1 @@ -foo +hello there!