From c976f2f1e260cd60178e09d673d87f2c48a9697e Mon Sep 17 00:00:00 2001 From: Jan Hutar Date: Tue, 23 Apr 2024 09:04:11 +0200 Subject: [PATCH 1/7] feat(KONFLUX-2603): Make it possible to create a template from a repo Changes in wait for branch to appear discussed with Pavel Sturc in https://redhat-internal.slack.com/archives/C02FANRBZQD/p1713856264091809 --- cmd/loadTests.go | 101 ++++++++++++++++++++++++++++++++++---- pkg/clients/github/git.go | 20 +++++--- 2 files changed, 105 insertions(+), 16 deletions(-) diff --git a/cmd/loadTests.go b/cmd/loadTests.go index 311c335e5..caef376a0 100644 --- a/cmd/loadTests.go +++ b/cmd/loadTests.go @@ -6,6 +6,7 @@ import ( "flag" "fmt" "os" + "regexp" "runtime" "strconv" "strings" @@ -219,7 +220,9 @@ func (u *UserAppsCompsMap) GetIntegrationTestScenarios(userName, appName string) var ( componentRepoUrl string = "https://github.com/devfile-samples/devfile-sample-code-with-quarkus" - componentsCount int = 1 + componentRepoRevision string = "main" + componentRepoTemplate bool + quayRepo string = "redhat-user-workloads-stage" usernamePrefix string = "testuser" numberOfUsers int testScenarioGitURL string = "https://github.com/konflux-ci/integration-examples.git" @@ -322,7 +325,9 @@ type LogData struct { MachineName string `json:"machineName"` BinaryDetails string `json:"binaryDetails"` ComponentRepoUrl string `json:"componentRepoUrl"` - ComponentsCount int `json:"componentsCount"` + ComponentRepoRevision string `json:"componentRepoRevision"` + ComponentRepoTemplate bool `json:"componentRepoTemplate"` + QuayRepo string `json:"quayRepo"` NumberOfThreads int `json:"threads"` NumberOfUsersPerThread int `json:"usersPerThread"` NumberOfUsers int `json:"totalUsers"` @@ -450,7 +455,9 @@ func ExecuteLoadTest() { func init() { rootCmd.Flags().StringVar(&componentRepoUrl, "component-repo", componentRepoUrl, "the component repo URL to be used") - rootCmd.Flags().IntVar(&componentsCount, "components-count", componentsCount, "number of components to create per application") + rootCmd.Flags().StringVar(&componentRepoRevision, "component-repo-revision", componentRepoRevision, "the component repo revision, git branch") + rootCmd.Flags().BoolVarP(&componentRepoTemplate, "component-repo-template", "e", false, "if you want to use per-user branch based on provided branch for PaC testing") + rootCmd.Flags().StringVar(&quayRepo, "quay-repo", quayRepo, "the target quay repo for PaC templated image pushes") rootCmd.Flags().StringVar(&usernamePrefix, "username", usernamePrefix, "the prefix used for usersignup names") rootCmd.Flags().BoolVarP(&verbose, "verbose", "v", false, "if 'debug' traces should be displayed in the console") rootCmd.Flags().BoolVarP(&stage, "stage", "s", false, "is you want to run the test on stage") @@ -589,7 +596,9 @@ func setup(cmd *cobra.Command, args []string) { MachineName: machineName, BinaryDetails: binaryDetails, ComponentRepoUrl: componentRepoUrl, - ComponentsCount: componentsCount, + ComponentRepoRevision: componentRepoRevision, + ComponentRepoTemplate: componentRepoTemplate, + QuayRepo: quayRepo, NumberOfThreads: threadCount, NumberOfUsersPerThread: numberOfUsers, NumberOfUsers: overallCount, @@ -1469,12 +1478,20 @@ func handleItsFailure(ctx *JourneyContext, applicationName string, err, conditio increaseBar(ctx.ItsBar, itsBarMutex) } -func (h *ConcreteHandlerResources) handleCDQCreation(ctx *JourneyContext, framework *framework.Framework, username, usernamespace, applicationName, cdqName string) (bool, *appstudioApi.ComponentDetectionQuery) { - klog.V(5).Infof("handleCDQCreation start username: %s, usernamespace: %s, applicationName: %s", username, usernamespace, applicationName) - defer klog.V(5).Infof("handleCDQCreation end username: %s, usernamespace: %s, applicationName: %s", username, usernamespace, applicationName) +func (h *ConcreteHandlerResources) handleCDQCreation(ctx *JourneyContext, framework *framework.Framework, username, usernamespace string) (bool, *appstudioApi.ComponentDetectionQuery) { + err, componentRepoRevisionFinal := handleRepoTemplating(ctx, framework, username, usernamespace) + if err != nil { + logError(30, fmt.Sprintf("Unable to template repository for user %s: %v", username, err)) + //FailedCDQCreationsPerThread[ctx.ThreadIndex] += 1 + //MetricsWrapper(MetricsController, metricsConstants.CollectorCDQ, metricsConstants.MetricTypeCounter, metricsConstants.MetricFailedCDQCreationCounter) + increaseBar(ctx.CDQsBar, cdqsBarMutex) + return false, nil + } + ApplicationName := fmt.Sprintf("%s-app", username) + ComponentDetectionQueryName := fmt.Sprintf("%s-cdq", username) startTimeForCDQ := time.Now() - cdq, err := framework.AsKubeDeveloper.HasController.CreateComponentDetectionQueryWithTimeout(cdqName, usernamespace, componentRepoUrl, "", "", "", false, 60*time.Minute) + cdq, err := framework.AsKubeDeveloper.HasController.CreateComponentDetectionQueryWithTimeout(ComponentDetectionQueryName, usernamespace, componentRepoUrl, componentRepoRevisionFinal, "", "", false, 60*time.Minute) cdqCreationTime := time.Since(startTimeForCDQ) if err != nil { @@ -1508,7 +1525,73 @@ func (h *ConcreteHandlerResources) handleCDQCreation(ctx *JourneyContext, framew return h.validateCDQCreation(ctx, framework, cdqName, applicationName, username, usernamespace, cdqCreationTime) } -func (h *ConcreteHandlerResources) validateCDQCreation(ctx *JourneyContext, framework *framework.Framework, cdqName, applicationName, username, usernamespace string, cdqCreationTime time.Duration) (bool, *appstudioApi.ComponentDetectionQuery) { +func handleRepoTemplating(ctx *JourneyContext, framework *framework.Framework, username, usernamespace string) (error, string) { + // Usual case, no repo templating takes place + if ! componentRepoTemplate { + return nil, componentRepoRevision + } + + // PaC testing, let's template repo and return branch name + var branchName string + var componentRepoName string + var err error + var exists bool + + // Parse just repo name out of url + regex := regexp.MustCompile(`/([^/]+)/?$`) + match := regex.FindStringSubmatch(componentRepoUrl) + if match != nil { + componentRepoName = match[1] + } else { + return fmt.Errorf("Failed to parse repo name out of url %s", componentRepoUrl), "" + } + + // Cleanup if it already exists + branchName = username + exists, err = framework.AsKubeAdmin.CommonController.Github.ExistsRef(componentRepoName, branchName) + if err != nil { + return err, "" + } + if exists { + klog.Errorf("Branch %s already exists, deleting it", branchName) + err := framework.AsKubeAdmin.CommonController.Github.DeleteRef(componentRepoName, branchName) + if err != nil { + return err, "" + } + } + + // Create branch + err = framework.AsKubeAdmin.CommonController.Github.CreateRef(componentRepoName, componentRepoRevision, "", branchName) + if err != nil { + return err, "" + } + + // Template files we care about + fileList := []string{".tekton/multi-platform-test-pull-request.yaml", ".tekton/multi-platform-test-push.yaml"} + for _, file := range fileList { + fileResponse, err := framework.AsKubeAdmin.CommonController.Github.GetFile(componentRepoName, file, branchName) + if err != nil { + return err, "" + } + + fileContent, err2 := fileResponse.GetContent() + if err2 != nil { + return err2, "" + } + + fileContentNew := strings.ReplaceAll(fileContent, "NAMESPACE", usernamespace) + fileContentNew = strings.ReplaceAll(fileContentNew, "QUAY_REPO", quayRepo) + + _, err3 := framework.AsKubeAdmin.CommonController.Github.UpdateFile(componentRepoName, file, fileContentNew, branchName, *fileResponse.SHA) + if err3 != nil { + return err3, "" + } + } + + return nil, branchName +} + +func (h *ConcreteHandlerResources) validateCDQ(ctx *JourneyContext, framework *framework.Framework, CDQName, ApplicationName, username, usernamespace string, cdqCreationTime time.Duration) (bool, *appstudioApi.ComponentDetectionQuery) { cdqValidationInterval := time.Second * 20 cdqValidationTimeout := time.Minute * 30 var conditionError error diff --git a/pkg/clients/github/git.go b/pkg/clients/github/git.go index d8cbc9641..ef36f2393 100644 --- a/pkg/clients/github/git.go +++ b/pkg/clients/github/git.go @@ -6,9 +6,8 @@ import ( "strings" "time" - . "github.com/onsi/gomega" - "github.com/google/go-github/v44/github" + "github.com/redhat-appstudio/e2e-tests/pkg/utils" ) func (g *Github) DeleteRef(repository, branchName string) error { @@ -39,12 +38,19 @@ func (g *Github) CreateRef(repository, baseBranchName, sha, newBranchName string if err != nil { return fmt.Errorf("error when creating a new branch '%s' for the repo '%s': %+v", newBranchName, repository, err) } - Eventually(func(gomega Gomega) { + err = utils.WaitUntilWithInterval(func() (done bool, err error) { exist, err := g.ExistsRef(repository, newBranchName) - gomega.Expect((err)).NotTo(HaveOccurred()) - gomega.Expect(exist).To(BeTrue()) - - }, 2*time.Minute, 2*time.Second).Should(Succeed()) //Wait for the branch to actually exist + if err != nil { + return false, err + } + if exist && err == nil { + return exist, err + } + return false, nil + }, 2*time.Second, 2*time.Minute) //Wait for the branch to actually exist + if err != nil { + return fmt.Errorf("Failed waiting for repo being done: %+v", err) + } return nil } From 9d8b8d9ecb9da45ba186aabd8436c8b844aecf4d Mon Sep 17 00:00:00 2001 From: Jan Hutar Date: Wed, 24 Apr 2024 10:03:29 +0200 Subject: [PATCH 2/7] fix: Do not spam output if we are just waiting for a deployment to appear --- cmd/loadTests.go | 1 - 1 file changed, 1 deletion(-) diff --git a/cmd/loadTests.go b/cmd/loadTests.go index caef376a0..7e13fc02b 100644 --- a/cmd/loadTests.go +++ b/cmd/loadTests.go @@ -2210,7 +2210,6 @@ func (h *ConcreteHandlerDeployments) validateDeploymentCreation(ctx *JourneyCont err := k8swait.PollUntilContextTimeout(context.Background(), deploymentCreatedRetryInterval, deploymentCreatedTimeout, false, func(ctx context.Context) (done bool, err error) { _, err = framework.AsKubeDeveloper.CommonController.GetDeployment(componentName, usernamespace) if err != nil { - klog.Infof("Unable getting deployment - %v", err) time.Sleep(time.Millisecond * time.Duration(rand.IntnRange(10, 200))) return false, nil } From 82fab563b578b4919c6c695a940f4478ae65c22f Mon Sep 17 00:00:00 2001 From: Jan Hutar Date: Fri, 5 Apr 2024 13:51:03 +0200 Subject: [PATCH 3/7] feat(KONFLUX-2547): Make it possible to configure BuildPipelineSelector --- cmd/loadTests.go | 132 ++++++++++++++++++ .../load-tests/ci-scripts/collect-results.sh | 4 + tests/load-tests/run-stage.sh | 1 + tests/load-tests/run.sh | 1 + 4 files changed, 138 insertions(+) diff --git a/cmd/loadTests.go b/cmd/loadTests.go index 7e13fc02b..9b07d0521 100644 --- a/cmd/loadTests.go +++ b/cmd/loadTests.go @@ -19,10 +19,12 @@ import ( metricsConstants "github.com/redhat-appstudio-qe/perf-monitoring/api/pkg/constants" "github.com/redhat-appstudio-qe/perf-monitoring/api/pkg/metrics" appstudioApi "github.com/redhat-appstudio/application-api/api/v1alpha1" + buildservice "github.com/redhat-appstudio/build-service/api/v1alpha1" "github.com/redhat-appstudio/e2e-tests/pkg/constants" "github.com/redhat-appstudio/e2e-tests/pkg/framework" "github.com/redhat-appstudio/e2e-tests/pkg/utils" loadtestUtils "github.com/redhat-appstudio/e2e-tests/pkg/utils/loadtests" + "github.com/redhat-appstudio/e2e-tests/pkg/utils/tekton" integrationv1beta1 "github.com/redhat-appstudio/integration-service/api/v1beta1" "github.com/spf13/cobra" pipeline "github.com/tektoncd/pipeline/pkg/apis/pipeline/v1" @@ -33,6 +35,7 @@ import ( k8swait "k8s.io/apimachinery/pkg/util/wait" "k8s.io/klog/v2" "knative.dev/pkg/apis" + rclient "sigs.k8s.io/controller-runtime/pkg/client" ) type UserAppsCompsMap struct { @@ -244,6 +247,7 @@ var ( enableProgressBars bool pushGatewayURI string = "" jobName string = "" + buildPipelineSelectorBundle string = "" ) var ( @@ -479,6 +483,7 @@ func init() { rootCmd.Flags().BoolVar(&enableProgressBars, "enable-progress-bars", false, "if you want to enable progress bars") rootCmd.Flags().StringVar(&pushGatewayURI, "pushgateway-url", pushGatewayURI, "PushGateway url (needs to be set if metrics are enabled)") rootCmd.Flags().StringVar(&jobName, "job-name", jobName, "Job Name to track Metrics (needs to be set if metrics are enabled)") + rootCmd.Flags().StringVar(&buildPipelineSelectorBundle, "build-pipeline-selector-bundle", buildPipelineSelectorBundle, "BuildPipelineSelector bundle to use when testing with build-definition PR") } func logError(errCode int, message string) { @@ -991,6 +996,11 @@ func StageCleanup(journeyContexts []*JourneyContext) { klog.Errorf("while deleting component detection queries for username: %s, got error: %v\n", username, err) } } + + err = deleteAllBuildPipelineSelectors(framework, time.Minute) + if err != nil { + klog.Errorf("while deleting build pipeline selectors for user: %s, got error: %v\n", user.Username, err) + } } } @@ -1034,6 +1044,128 @@ func increaseBar(bar *uiprogress.Bar, mutex *sync.Mutex) { } } +func componentForUser(username string) string { + val, ok := userComponentMap.Load(username) + if ok { + componentName, ok2 := val.(string) + if ok2 { + return componentName + } else { + klog.Errorf("Invalid type of map value: %+v", val) + } + } + return "" +} + +func listAllBuildPipelineSelectors(f *framework.Framework) (*buildservice.BuildPipelineSelectorList, error) { + list := &buildservice.BuildPipelineSelectorList{} + err := f.AsKubeDeveloper.HasController.KubeRest().List(context.Background(), list, &rclient.ListOptions{Namespace: f.UserNamespace}) + klog.V(5).Infof("listAllBuildPipelineSelectors namespace: %s, len: %d, err: %v", f.UserNamespace, len(list.Items), err) + return list, err +} + +func deleteAllBuildPipelineSelectors(f *framework.Framework, timeout time.Duration) error { + klog.V(5).Infof("deleteAllBuildPipelineSelectors start namespace: %s", f.UserNamespace) + defer klog.V(5).Infof("deleteAllBuildPipelineSelectors end") + + list, err := listAllBuildPipelineSelectors(f) + if err != nil { + return fmt.Errorf("error listing build pipeline selectors from %s: %v", f.UserNamespace, err) + } + + for _, bps := range list.Items { + toDelete := bps + err = f.AsKubeDeveloper.HasController.KubeRest().Delete(context.Background(), &toDelete) + if err != nil { + return fmt.Errorf("error deleting build pipeline selector %s from %s: %v", bps.Name, f.UserNamespace, err) + } + } + + return utils.WaitUntil(func() (done bool, err error) { + list, err := listAllBuildPipelineSelectors(f) + if err != nil { + return false, nil + } + return len(list.Items) == 0, nil + }, timeout) +} + +func createBuildPipelineSelector(f *framework.Framework, bundle *string) error { + klog.V(5).Infof("createBuildPipelineSelector start bundle: %s, namespace: %s", *bundle, f.UserNamespace) + defer klog.V(5).Infof("createBuildPipelineSelector end") + + var err error + + err = deleteAllBuildPipelineSelectors(f, time.Minute) + if err != nil { + klog.Errorf("error deleting build pipeline selectors from %s: %v\n", f.UserNamespace, err) + } + + bps := &buildservice.BuildPipelineSelector{ + ObjectMeta: metav1.ObjectMeta{ + Name: "build-pipeline-selector", + Namespace: f.UserNamespace, + }, + Spec: buildservice.BuildPipelineSelectorSpec{Selectors: []buildservice.PipelineSelector{ + { + Name: "all-pipelines", + PipelineRef: *tekton.NewBundleResolverPipelineRef("docker-build", *bundle), + }, + }}, + } + err = f.AsKubeAdmin.CommonController.KubeRest().Create(context.TODO(), bps) + if err != nil { + return fmt.Errorf("error creating build pipeline selector in %s with bundle %s: %v", f.UserNamespace, *bundle, err) + } + + return nil +} + +func frameworkForUser(username string) *framework.Framework { + val, ok := frameworkMap.Load(username) + if ok { + framework, ok2 := val.(*framework.Framework) + if ok2 { + if buildPipelineSelectorBundle != "" { + err := createBuildPipelineSelector(framework, &buildPipelineSelectorBundle) + if err != nil { + klog.Errorf("Error creating build pipeline selector: %v", err) + } + } + return framework + } else { + klog.Errorf("Invalid type of map value: %+v", val) + } + } + return nil +} + +func testScenarioForUser(username string) string { + val, ok := userTestScenarioMap.Load(username) + if ok { + testScenarioName, ok2 := val.(string) + if ok2 { + return testScenarioName + } else { + klog.Errorf("Invalid type of map value: %+v", val) + } + } + return "" +} + +func userComponentPipelineRunForUser(username string) string { + val, ok := userComponentPipelineRunMap.Load(username) + if ok { + componentPipelineRunName, ok2 := val.(string) + if ok2 { + return componentPipelineRunName + } else { + klog.Errorf("Invalid type of map value: %+v", val) + } + } + return "" +} + func tryNewFramework(username string, user loadtestUtils.User, timeout time.Duration) (*framework.Framework, error) { ch := make(chan *framework.Framework) var fw *framework.Framework diff --git a/tests/load-tests/ci-scripts/collect-results.sh b/tests/load-tests/ci-scripts/collect-results.sh index f33538952..8070cfc04 100755 --- a/tests/load-tests/ci-scripts/collect-results.sh +++ b/tests/load-tests/ci-scripts/collect-results.sh @@ -23,6 +23,7 @@ find "$output_dir" -type f -name '*.pprof' -exec cp -vf {} "${ARTIFACT_DIR}" \; pipelineruns_json=$ARTIFACT_DIR/pipelineruns.json taskruns_json=$ARTIFACT_DIR/taskruns.json pods_json=$ARTIFACT_DIR/pods.json +buildpipelineselectors_json=$ARTIFACT_DIR/buildpipelineselectors.json application_timestamps=$ARTIFACT_DIR/applications.appstudio.redhat.com_timestamps application_timestamps_csv=${application_timestamps}.csv @@ -182,6 +183,9 @@ jq -r ".items[] | .spec.nodeName" "$pods_json" | sort | uniq -c | sed -e 's,\s\+ echo "Node;Pods" >"$task_pods_distribution_csv" jq -r '.items[] | select(.metadata.labels."appstudio.openshift.io/application" != null).spec.nodeName' "$pods_json" | sort | uniq -c | sed -e 's,\s\+\([0-9]\+\)\s\+\(.*\),\2;\1,g' >>"$task_pods_distribution_csv" +## Record BuildPipelineSelectors +oc get BuildPipelineSelectors -A -o json >"$buildpipelineselectors_json" + ## Tekton Artifact Performance Analysis tapa_dir=./tapa.git diff --git a/tests/load-tests/run-stage.sh b/tests/load-tests/run-stage.sh index dc7ab1a43..f1bbf8dce 100755 --- a/tests/load-tests/run-stage.sh +++ b/tests/load-tests/run-stage.sh @@ -1,5 +1,6 @@ go run loadtest.go \ --component-repo "${COMPONENT_REPO:-https://github.com/devfile-samples/devfile-sample-code-with-quarkus}" \ + --build-pipeline-selector-bundle "${BUILD_PIPELINE_SELECTOR_BUNDLE:-}" \ --username "$USER_PREFIX" \ --users "${USERS_PER_THREAD:-2}" \ --test-scenario-git-url "${TEST_SCENARIO_GIT_URL:-https://github.com/konflux-ci/integration-examples.git}" \ diff --git a/tests/load-tests/run.sh b/tests/load-tests/run.sh index 90122433a..e9af2523b 100755 --- a/tests/load-tests/run.sh +++ b/tests/load-tests/run.sh @@ -82,6 +82,7 @@ else ## To enable progress bar , add `--enable-progress-bars` in [OPTIONS] go run loadtest.go \ --component-repo "${COMPONENT_REPO:-https://github.com/devfile-samples/devfile-sample-code-with-quarkus}" \ + --build-pipeline-selector-bundle "${BUILD_PIPELINE_SELECTOR_BUNDLE:-}" \ --username "$USER_PREFIX" \ --users "${USERS_PER_THREAD:-50}" \ --test-scenario-git-url "${TEST_SCENARIO_GIT_URL:-https://github.com/konflux-ci/integration-examples.git}" \ From 45bce464451337e0c0756d0e2d2808367c317c23 Mon Sep 17 00:00:00 2001 From: Jan Hutar Date: Mon, 22 Apr 2024 13:16:01 +0200 Subject: [PATCH 4/7] debug: Setup with secrets for Build and App services --- tests/load-tests/ci-scripts/setup-cluster.sh | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/tests/load-tests/ci-scripts/setup-cluster.sh b/tests/load-tests/ci-scripts/setup-cluster.sh index 9f40a48cc..d4e737aca 100755 --- a/tests/load-tests/ci-scripts/setup-cluster.sh +++ b/tests/load-tests/ci-scripts/setup-cluster.sh @@ -58,10 +58,15 @@ if [ "${TEKTON_PERF_ENABLE_CPU_PROFILING:-}" == "true" ] || [ "${TEKTON_PERF_ENA oc patch -n tekton-results cm tekton-results-config-observability --type=json -p='[{"op": "add", "path": "/data/profiling.enable", "value": "true"}]' fi +## Patch Build Service github secret +echo "Patching Build Service github token" +oc patch -n build-service secret pipelines-as-code-secret --type=json -p='[{"op": "add", "path": "/data/password", "value": "'"$(cat /usr/local/ci-secrets/redhat-appstudio-load-test/github-token | tr -d "\n" | base64 -w0)"'"}]' +oc rollout restart deployment -n build-service +oc rollout status deployment -n build-service -w + ## Patch HAS github secret echo "Patching HAS github tokens" -oc patch -n application-service secret has-github-token --type=json -p='[{"op": "add", "path": "/data/tokens", "value": "'"$(base64 -w0 Date: Fri, 26 Apr 2024 13:39:54 +0200 Subject: [PATCH 5/7] feat: Make it possible to set COMPONENT_REPO_REVISION in SCENARIO config --- tests/load-tests/run-max-concurrency.sh | 1 + tests/load-tests/run-stage.sh | 1 + tests/load-tests/run.sh | 1 + 3 files changed, 3 insertions(+) diff --git a/tests/load-tests/run-max-concurrency.sh b/tests/load-tests/run-max-concurrency.sh index 4a58375d6..4ce0adc72 100755 --- a/tests/load-tests/run-max-concurrency.sh +++ b/tests/load-tests/run-max-concurrency.sh @@ -53,6 +53,7 @@ load_test() { rm -rvf "$output_dir/load-test.log" go run loadtest.go \ --component-repo "${COMPONENT_REPO:-https://github.com/nodeshift-starters/devfile-sample.git}" \ + --component-repo-revision "${COMPONENT_REPO_REVISION:-main}" \ --username "$USER_PREFIX-$index" \ --users 1 \ -w="${WAIT_PIPELINES:-true}" \ diff --git a/tests/load-tests/run-stage.sh b/tests/load-tests/run-stage.sh index f1bbf8dce..b088d09cc 100755 --- a/tests/load-tests/run-stage.sh +++ b/tests/load-tests/run-stage.sh @@ -1,5 +1,6 @@ go run loadtest.go \ --component-repo "${COMPONENT_REPO:-https://github.com/devfile-samples/devfile-sample-code-with-quarkus}" \ + --component-repo-revision "${COMPONENT_REPO_REVISION:-main}" \ --build-pipeline-selector-bundle "${BUILD_PIPELINE_SELECTOR_BUNDLE:-}" \ --username "$USER_PREFIX" \ --users "${USERS_PER_THREAD:-2}" \ diff --git a/tests/load-tests/run.sh b/tests/load-tests/run.sh index e9af2523b..be5bf5eb0 100755 --- a/tests/load-tests/run.sh +++ b/tests/load-tests/run.sh @@ -82,6 +82,7 @@ else ## To enable progress bar , add `--enable-progress-bars` in [OPTIONS] go run loadtest.go \ --component-repo "${COMPONENT_REPO:-https://github.com/devfile-samples/devfile-sample-code-with-quarkus}" \ + --component-repo-revision "${COMPONENT_REPO_REVISION:-main}" \ --build-pipeline-selector-bundle "${BUILD_PIPELINE_SELECTOR_BUNDLE:-}" \ --username "$USER_PREFIX" \ --users "${USERS_PER_THREAD:-50}" \ From 04defb36c8e029806e753301a9cf37bd28559de2 Mon Sep 17 00:00:00 2001 From: Naftaly Shprai Date: Sun, 4 Feb 2024 15:40:42 +0200 Subject: [PATCH 6/7] fix(KONFLUX-870): KONFLUX-870-Expose-accurate-resources-metrics Expose accurate resources metrics into load-tests.json (metrics that overcome the time-skew issue) Signed-Off-By: Naftaly Shprai --- cmd/loadTests.go | 67 +++++++++++++++++++++++++----------------------- 1 file changed, 35 insertions(+), 32 deletions(-) diff --git a/cmd/loadTests.go b/cmd/loadTests.go index 9b07d0521..eb95b1ff1 100644 --- a/cmd/loadTests.go +++ b/cmd/loadTests.go @@ -1356,13 +1356,8 @@ func (h *ConcreteHandlerResources) handleApplicationCreation(ctx *JourneyContext return false } - ApplicationCreationTimeSumPerThread[ctx.ThreadIndex] += applicationCreationTime MetricsWrapper(MetricsController, metricsConstants.CollectorApplications, metricsConstants.MetricTypeGuage, metricsConstants.MetricApplicationCreationTimeGauge, applicationCreationTime.Seconds()) - if applicationCreationTime > ApplicationCreationTimeMaxPerThread[ctx.ThreadIndex] { - ApplicationCreationTimeMaxPerThread[ctx.ThreadIndex] = applicationCreationTime - } - - return h.validateApplicationCreation(ctx, framework, applicationName, username, usernamespace, applicationCreationTime) + return h.validateApplicationCreation(ctx, framework, ApplicationName, username, usernamespace, applicationCreationTime) } func isConditionError(condition metav1.Condition) bool { @@ -1484,7 +1479,13 @@ func handleApplicationSuccess(ctx *JourneyContext, username, applicationName str SuccessfulApplicationCreationsPerThread[ctx.ThreadIndex] += 1 MetricsWrapper(MetricsController, metricsConstants.CollectorApplications, metricsConstants.MetricTypeCounter, metricsConstants.MetricSuccessfulApplicationCreationCounter) increaseBar(ctx.ApplicationsBar, applicationsBarMutex) - ctx.userAppsCompsMap.AddApplication(username, applicationName) + + // Transform applicationActualCreationTimeInSeconds from float64 to time.Duration + applicationActualCreationTimeInSecondsToDuration := time.Duration(applicationActualCreationTimeInSeconds * float64(time.Second)) + ApplicationCreationTimeSumPerThread[ctx.ThreadIndex] += applicationActualCreationTimeInSecondsToDuration + if applicationActualCreationTimeInSecondsToDuration > ApplicationCreationTimeMaxPerThread[ctx.ThreadIndex] { + ApplicationCreationTimeMaxPerThread[ctx.ThreadIndex] = applicationActualCreationTimeInSecondsToDuration + } } func handleApplicationFailure(ctx *JourneyContext, applicationName string, username string, err error, conditionError error) { @@ -1519,13 +1520,9 @@ func (h *ConcreteHandlerResources) handleIntegrationTestScenarioCreation(ctx *Jo return false } - ItsCreationTimeSumPerThread[ctx.ThreadIndex] += itsCreationTime + itsName := integrationTestScenario.Name MetricsWrapper(MetricsController, metricsConstants.CollectorIntegrationTestsSC, metricsConstants.MetricTypeGuage, metricsConstants.MetricIntegrationTestSenarioCreationTimeGauge, itsCreationTime.Seconds()) - if itsCreationTime > ItsCreationTimeMaxPerThread[ctx.ThreadIndex] { - ItsCreationTimeMaxPerThread[ctx.ThreadIndex] = itsCreationTime - } - - return h.validateIntegrationTestScenarioCreation(ctx, framework, itsName, applicationName, username, usernamespace, itsCreationTime) + return h.validateIntegrationTestScenario(ctx, framework, itsName, ApplicationName, username, usernamespace, itsCreationTime) } func findTestScenarioByName(scenarios []integrationv1beta1.IntegrationTestScenario, name string) *integrationv1beta1.IntegrationTestScenario { @@ -1595,7 +1592,14 @@ func handleItsSuccess(ctx *JourneyContext, itsName, username string, application SuccessfulItsCreationsPerThread[ctx.ThreadIndex] += 1 MetricsWrapper(MetricsController, metricsConstants.CollectorIntegrationTestsSC, metricsConstants.MetricTypeCounter, metricsConstants.MetricSuccessfulIntegrationTestSenarioCreationCounter) increaseBar(ctx.ItsBar, itsBarMutex) - ctx.userAppsCompsMap.AddIntegrationTestScenario(username, applicationName, itsName) + userTestScenarioMap.Store(username, itsName) + + // Transform itsActualCreationTimeInSeconds from float64 to time.Duration + itsActualCreationTimeInSecondsToDuration := time.Duration(itsActualCreationTimeInSeconds * float64(time.Second)) + ItsCreationTimeSumPerThread[ctx.ThreadIndex] += itsActualCreationTimeInSecondsToDuration + if itsActualCreationTimeInSecondsToDuration > ItsCreationTimeMaxPerThread[ctx.ThreadIndex] { + ItsCreationTimeMaxPerThread[ctx.ThreadIndex] = itsActualCreationTimeInSecondsToDuration + } } func handleItsFailure(ctx *JourneyContext, applicationName string, err, conditionError error) { @@ -1648,13 +1652,8 @@ func (h *ConcreteHandlerResources) handleCDQCreation(ctx *JourneyContext, framew return false, nil } - CDQCreationTimeSumPerThread[ctx.ThreadIndex] += cdqCreationTime MetricsWrapper(MetricsController, metricsConstants.CollectorCDQ, metricsConstants.MetricTypeGuage, metricsConstants.MetricCDQCreationTimeGauge, cdqCreationTime.Seconds()) - if cdqCreationTime > CDQCreationTimeMaxPerThread[ctx.ThreadIndex] { - CDQCreationTimeMaxPerThread[ctx.ThreadIndex] = cdqCreationTime - } - - return h.validateCDQCreation(ctx, framework, cdqName, applicationName, username, usernamespace, cdqCreationTime) + return h.validateCDQ(ctx, framework, ComponentDetectionQueryName, ApplicationName, username, usernamespace, cdqCreationTime) } func handleRepoTemplating(ctx *JourneyContext, framework *framework.Framework, username, usernamespace string) (error, string) { @@ -1776,6 +1775,13 @@ func handleCdqSuccess(ctx *JourneyContext, cdqName string, cdqActualCreationTime SuccessfulCDQCreationsPerThread[ctx.ThreadIndex] += 1 MetricsWrapper(MetricsController, metricsConstants.CollectorCDQ, metricsConstants.MetricTypeCounter, metricsConstants.MetricSuccessfulCDQCreationCounter) increaseBar(ctx.CDQsBar, cdqsBarMutex) + + // Transform cdqActualCreationTimeInSeconds from float64 to time.Duration + cdqActualCreationTimeInSecondsToDuration := time.Duration(cdqActualCreationTimeInSeconds * float64(time.Second)) + CDQCreationTimeSumPerThread[ctx.ThreadIndex] += cdqActualCreationTimeInSecondsToDuration + if cdqActualCreationTimeInSecondsToDuration > CDQCreationTimeMaxPerThread[ctx.ThreadIndex] { + CDQCreationTimeMaxPerThread[ctx.ThreadIndex] = cdqActualCreationTimeInSecondsToDuration + } } func handleCdqFailure(ctx *JourneyContext, applicationName string, err, conditionError error) { @@ -1839,18 +1845,8 @@ func (h *ConcreteHandlerResources) handleComponentCreation(ctx *JourneyContext, increaseBar(ctx.ComponentsBar, componentsBarMutex) return false } - - ComponentCreationTimeSumPerThread[ctx.ThreadIndex] += componentCreationTime + componentName = component.Name MetricsWrapper(MetricsController, metricsConstants.CollectorComponents, metricsConstants.MetricTypeGuage, metricsConstants.MetricComponentCreationTimeGauge, componentCreationTime.Seconds()) - if componentCreationTime > ComponentCreationTimeMaxPerThread[ctx.ThreadIndex] { - ComponentCreationTimeMaxPerThread[ctx.ThreadIndex] = componentCreationTime - } - - validated := h.validateComponent(ctx, framework, component.Name, applicationName, username, usernamespace, componentCreationTime) - if !validated { - logError(30, fmt.Sprintf("Validation of component name (%s) failed with: %v", component.Name, err)) - return false - } } ctx.ChPipelines <- username @@ -1913,7 +1909,14 @@ func handleComponentSuccess(ctx *JourneyContext, username, applicationName, comp SuccessfulComponentCreationsPerThread[ctx.ThreadIndex] += 1 MetricsWrapper(MetricsController, metricsConstants.CollectorComponents, metricsConstants.MetricTypeCounter, metricsConstants.MetricSuccessfulComponentCreationCounter) increaseBar(ctx.ComponentsBar, componentsBarMutex) - ctx.userAppsCompsMap.AddComponent(username, applicationName, componentName) + ctx.ChPipelines <- username + + // Transform componentActualCreationTimeInSeconds from float64 to time.Duration + componentActualCreationTimeInSecondsToDuration := time.Duration(componentActualCreationTimeInSeconds * float64(time.Second)) + ComponentCreationTimeSumPerThread[ctx.ThreadIndex] += componentActualCreationTimeInSecondsToDuration + if componentActualCreationTimeInSecondsToDuration > ComponentCreationTimeMaxPerThread[ctx.ThreadIndex] { + ComponentCreationTimeMaxPerThread[ctx.ThreadIndex] = componentActualCreationTimeInSecondsToDuration + } } func handleComponentFailure(ctx *JourneyContext, applicationName string, err, conditionError error) { From 71646ae919fc38992f160bb2aa3cc37d3379e806 Mon Sep 17 00:00:00 2001 From: Jan Hutar Date: Fri, 3 May 2024 10:37:26 +0200 Subject: [PATCH 7/7] fix: Return back pieces dropped from previous merge --- cmd/loadTests.go | 97 +++++++++++------------------------------------- 1 file changed, 21 insertions(+), 76 deletions(-) diff --git a/cmd/loadTests.go b/cmd/loadTests.go index eb95b1ff1..557d92842 100644 --- a/cmd/loadTests.go +++ b/cmd/loadTests.go @@ -223,6 +223,7 @@ func (u *UserAppsCompsMap) GetIntegrationTestScenarios(userName, appName string) var ( componentRepoUrl string = "https://github.com/devfile-samples/devfile-sample-code-with-quarkus" + componentsCount int = 1 componentRepoRevision string = "main" componentRepoTemplate bool quayRepo string = "redhat-user-workloads-stage" @@ -329,6 +330,7 @@ type LogData struct { MachineName string `json:"machineName"` BinaryDetails string `json:"binaryDetails"` ComponentRepoUrl string `json:"componentRepoUrl"` + ComponentsCount int `json:"componentsCount"` ComponentRepoRevision string `json:"componentRepoRevision"` ComponentRepoTemplate bool `json:"componentRepoTemplate"` QuayRepo string `json:"quayRepo"` @@ -459,6 +461,7 @@ func ExecuteLoadTest() { func init() { rootCmd.Flags().StringVar(&componentRepoUrl, "component-repo", componentRepoUrl, "the component repo URL to be used") + rootCmd.Flags().IntVar(&componentsCount, "components-count", componentsCount, "number of components to create per application") rootCmd.Flags().StringVar(&componentRepoRevision, "component-repo-revision", componentRepoRevision, "the component repo revision, git branch") rootCmd.Flags().BoolVarP(&componentRepoTemplate, "component-repo-template", "e", false, "if you want to use per-user branch based on provided branch for PaC testing") rootCmd.Flags().StringVar(&quayRepo, "quay-repo", quayRepo, "the target quay repo for PaC templated image pushes") @@ -601,6 +604,7 @@ func setup(cmd *cobra.Command, args []string) { MachineName: machineName, BinaryDetails: binaryDetails, ComponentRepoUrl: componentRepoUrl, + ComponentsCount: componentsCount, ComponentRepoRevision: componentRepoRevision, ComponentRepoTemplate: componentRepoTemplate, QuayRepo: quayRepo, @@ -983,10 +987,13 @@ func StageCleanup(journeyContexts []*JourneyContext) { klog.V(5).Infof("StageCleanup start") defer klog.V(5).Infof("StageCleanup end") + var err error + var framework *framework.Framework + for _, journeyCtx := range journeyContexts { for _, username := range journeyCtx.userAppsCompsMap.GetUserNames() { - framework := journeyCtx.userAppsCompsMap.GetUserFramework(username) - err := framework.AsKubeDeveloper.HasController.DeleteAllApplicationsInASpecificNamespace(framework.UserNamespace, 5*time.Minute) + framework = journeyCtx.userAppsCompsMap.GetUserFramework(username) + err = framework.AsKubeDeveloper.HasController.DeleteAllApplicationsInASpecificNamespace(framework.UserNamespace, 5*time.Minute) if err != nil { klog.Errorf("while deleting resources for username: %s, got error: %v\n", username, err) } @@ -995,11 +1002,11 @@ func StageCleanup(journeyContexts []*JourneyContext) { if err != nil { klog.Errorf("while deleting component detection queries for username: %s, got error: %v\n", username, err) } - } - err = deleteAllBuildPipelineSelectors(framework, time.Minute) - if err != nil { - klog.Errorf("while deleting build pipeline selectors for user: %s, got error: %v\n", user.Username, err) + err = deleteAllBuildPipelineSelectors(framework, time.Minute) + if err != nil { + klog.Errorf("while deleting build pipeline selectors for user: %s, got error: %v\n", username, err) + } } } } @@ -1044,19 +1051,6 @@ func increaseBar(bar *uiprogress.Bar, mutex *sync.Mutex) { } } -func componentForUser(username string) string { - val, ok := userComponentMap.Load(username) - if ok { - componentName, ok2 := val.(string) - if ok2 { - return componentName - } else { - klog.Errorf("Invalid type of map value: %+v", val) - } - } - return "" -} - func listAllBuildPipelineSelectors(f *framework.Framework) (*buildservice.BuildPipelineSelectorList, error) { list := &buildservice.BuildPipelineSelectorList{} err := f.AsKubeDeveloper.HasController.KubeRest().List(context.Background(), list, &rclient.ListOptions{Namespace: f.UserNamespace}) @@ -1121,51 +1115,6 @@ func createBuildPipelineSelector(f *framework.Framework, bundle *string) error { return nil } -func frameworkForUser(username string) *framework.Framework { - val, ok := frameworkMap.Load(username) - if ok { - framework, ok2 := val.(*framework.Framework) - if ok2 { - if buildPipelineSelectorBundle != "" { - err := createBuildPipelineSelector(framework, &buildPipelineSelectorBundle) - if err != nil { - klog.Errorf("Error creating build pipeline selector: %v", err) - } - } - return framework - } else { - klog.Errorf("Invalid type of map value: %+v", val) - } - } - return nil -} - -func testScenarioForUser(username string) string { - val, ok := userTestScenarioMap.Load(username) - if ok { - testScenarioName, ok2 := val.(string) - if ok2 { - return testScenarioName - } else { - klog.Errorf("Invalid type of map value: %+v", val) - } - } - return "" -} - -func userComponentPipelineRunForUser(username string) string { - val, ok := userComponentPipelineRunMap.Load(username) - if ok { - componentPipelineRunName, ok2 := val.(string) - if ok2 { - return componentPipelineRunName - } else { - klog.Errorf("Invalid type of map value: %+v", val) - } - } - return "" -} - func tryNewFramework(username string, user loadtestUtils.User, timeout time.Duration) (*framework.Framework, error) { ch := make(chan *framework.Framework) var fw *framework.Framework @@ -1319,8 +1268,7 @@ func (h *ConcreteHandlerResources) Handle(ctx *JourneyContext) { } // Handle Component Detection Query Creation - cdqName := fmt.Sprintf("%s-cdq-%s", username, util.GenerateRandomString(5)) - blnOK, cdq := h.handleCDQCreation(ctx, framework, username, usernamespace, applicationName, cdqName) + blnOK, cdq := h.handleCDQCreation(ctx, framework, username, usernamespace) if !blnOK { // If CDQ creation failed, continue with the next user continue @@ -1357,7 +1305,7 @@ func (h *ConcreteHandlerResources) handleApplicationCreation(ctx *JourneyContext } MetricsWrapper(MetricsController, metricsConstants.CollectorApplications, metricsConstants.MetricTypeGuage, metricsConstants.MetricApplicationCreationTimeGauge, applicationCreationTime.Seconds()) - return h.validateApplicationCreation(ctx, framework, ApplicationName, username, usernamespace, applicationCreationTime) + return h.validateApplicationCreation(ctx, framework, applicationName, username, usernamespace, applicationCreationTime) } func isConditionError(condition metav1.Condition) bool { @@ -1520,9 +1468,8 @@ func (h *ConcreteHandlerResources) handleIntegrationTestScenarioCreation(ctx *Jo return false } - itsName := integrationTestScenario.Name MetricsWrapper(MetricsController, metricsConstants.CollectorIntegrationTestsSC, metricsConstants.MetricTypeGuage, metricsConstants.MetricIntegrationTestSenarioCreationTimeGauge, itsCreationTime.Seconds()) - return h.validateIntegrationTestScenario(ctx, framework, itsName, ApplicationName, username, usernamespace, itsCreationTime) + return h.validateIntegrationTestScenarioCreation(ctx, framework, itsName, applicationName, username, usernamespace, itsCreationTime) } func findTestScenarioByName(scenarios []integrationv1beta1.IntegrationTestScenario, name string) *integrationv1beta1.IntegrationTestScenario { @@ -1592,7 +1539,6 @@ func handleItsSuccess(ctx *JourneyContext, itsName, username string, application SuccessfulItsCreationsPerThread[ctx.ThreadIndex] += 1 MetricsWrapper(MetricsController, metricsConstants.CollectorIntegrationTestsSC, metricsConstants.MetricTypeCounter, metricsConstants.MetricSuccessfulIntegrationTestSenarioCreationCounter) increaseBar(ctx.ItsBar, itsBarMutex) - userTestScenarioMap.Store(username, itsName) // Transform itsActualCreationTimeInSeconds from float64 to time.Duration itsActualCreationTimeInSecondsToDuration := time.Duration(itsActualCreationTimeInSeconds * float64(time.Second)) @@ -1631,14 +1577,14 @@ func (h *ConcreteHandlerResources) handleCDQCreation(ctx *JourneyContext, framew cdqCreationTime := time.Since(startTimeForCDQ) if err != nil { - logError(9, fmt.Sprintf("Unable to create ComponentDetectionQuery %s: %v", cdqName, err)) + logError(9, fmt.Sprintf("Unable to create ComponentDetectionQuery %s: %v", ComponentDetectionQueryName, err)) FailedCDQCreationsPerThread[ctx.ThreadIndex] += 1 MetricsWrapper(MetricsController, metricsConstants.CollectorCDQ, metricsConstants.MetricTypeCounter, metricsConstants.MetricFailedCDQCreationCounter) increaseBar(ctx.CDQsBar, cdqsBarMutex) return false, nil } - if cdq.Name != cdqName { - logError(10, fmt.Sprintf("Actual cdq name (%s) does not match expected (%s): %v", cdq.Name, cdqName, err)) + if cdq.Name != ComponentDetectionQueryName { + logError(10, fmt.Sprintf("Actual cdq name (%s) does not match expected (%s): %v", cdq.Name, ComponentDetectionQueryName, err)) FailedCDQCreationsPerThread[ctx.ThreadIndex] += 1 MetricsWrapper(MetricsController, metricsConstants.CollectorCDQ, metricsConstants.MetricTypeCounter, metricsConstants.MetricFailedCDQCreationCounter) increaseBar(ctx.CDQsBar, cdqsBarMutex) @@ -1722,7 +1668,7 @@ func handleRepoTemplating(ctx *JourneyContext, framework *framework.Framework, u return nil, branchName } -func (h *ConcreteHandlerResources) validateCDQ(ctx *JourneyContext, framework *framework.Framework, CDQName, ApplicationName, username, usernamespace string, cdqCreationTime time.Duration) (bool, *appstudioApi.ComponentDetectionQuery) { +func (h *ConcreteHandlerResources) validateCDQ(ctx *JourneyContext, framework *framework.Framework, cdqName, ApplicationName, username, usernamespace string, cdqCreationTime time.Duration) (bool, *appstudioApi.ComponentDetectionQuery) { cdqValidationInterval := time.Second * 20 cdqValidationTimeout := time.Minute * 30 var conditionError error @@ -1763,7 +1709,7 @@ func (h *ConcreteHandlerResources) validateCDQ(ctx *JourneyContext, framework *f }, cdqValidationInterval, cdqValidationTimeout) if err != nil || conditionError != nil { - handleCdqFailure(ctx, applicationName, err, conditionError) + handleCdqFailure(ctx, ApplicationName, err, conditionError) return false, nil } return true, cdq @@ -1845,7 +1791,6 @@ func (h *ConcreteHandlerResources) handleComponentCreation(ctx *JourneyContext, increaseBar(ctx.ComponentsBar, componentsBarMutex) return false } - componentName = component.Name MetricsWrapper(MetricsController, metricsConstants.CollectorComponents, metricsConstants.MetricTypeGuage, metricsConstants.MetricComponentCreationTimeGauge, componentCreationTime.Seconds()) }