Simplify scenario preparation, create objects concurrently

Now that experiment objects are associated with a single experiment run, we can skip deleting excess objects and simply create
the desired number of objects.
However, creating them sequentially might take a while (limited to a single core's performance).

webhosting-operator/go.mod

@@ -4,6 +4,7 @@ go 1.20
 
 require (
 	github.com/go-logr/logr v1.2.4
+	github.com/hashicorp/go-multierror v1.1.1
 	github.com/onsi/ginkgo/v2 v2.11.0
 	github.com/onsi/gomega v1.27.8
 	github.com/prometheus/client_golang v1.16.0
@@ -46,6 +47,7 @@ require (
 	github.com/google/gofuzz v1.1.0 // indirect
 	github.com/google/pprof v0.0.0-20230705174524-200ffdc848b8 // indirect
 	github.com/google/uuid v1.3.0 // indirect
+	github.com/hashicorp/errwrap v1.0.0 // indirect
 	github.com/imdario/mergo v0.3.12 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/josharian/intern v1.0.0 // indirect

webhosting-operator/go.sum

@@ -72,6 +72,10 @@ github.com/google/pprof v0.0.0-20230705174524-200ffdc848b8 h1:n6vlPhxsA+BW/XsS5+
 github.com/google/pprof v0.0.0-20230705174524-200ffdc848b8/go.mod h1:Jh3hGz2jkYak8qXPD19ryItVnUgpgeqzdkY/D0EaeuA=
 github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
 github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA=
+github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
+github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo=
+github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM=
 github.com/imdario/mergo v0.3.12 h1:b6R2BslTbIEToALKP7LxUvijTsNI9TAe80pLWN2g/HU=
 github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA=
 github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=

webhosting-operator/pkg/experiment/generator/project.go

@@ -22,38 +22,16 @@ import (
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"sigs.k8s.io/controller-runtime/pkg/client"
-
-	"github.com/timebertt/kubernetes-controller-sharding/webhosting-operator/pkg/utils"
 )
 
-// EnsureProjects ensures there are exactly n projects with the given labels.
-// It keeps existing projects to speed up experiment preparation.
-func EnsureProjects(ctx context.Context, c client.Client, n int, opts ...GenerateOption) error {
-	options := (&GenerateOptions{}).ApplyOptions(opts...)
-
-	// delete excess projects
-	namespaceList := &corev1.NamespaceList{}
-	if err := c.List(ctx, namespaceList, client.MatchingLabels(options.Labels)); err != nil {
-		return err
-	}
-
-	for _, namespace := range utils.PickNRandom(namespaceList.Items, len(namespaceList.Items)-n) {
-		if err := c.Delete(ctx, &namespace); err != nil {
-			return err
-		}
-	}
-
-	// create missing projects
-	for i := 0; i < n-len(namespaceList.Items); i++ {
-		if err := CreateProject(ctx, c, options); err != nil {
-			return err
-		}
-	}
-
-	return nil
+// CreateProjects creates n random project namespaces.
+func CreateProjects(ctx context.Context, c client.Client, n int, opts ...GenerateOption) error {
+	return NTimesConcurrently(n, 10, func() error {
+		return CreateProject(ctx, c, opts...)
+	})
 }
 
-// CreateProject creates a random project namespace using the given client and labels.
+// CreateProject creates a random project namespace.
 func CreateProject(ctx context.Context, c client.Client, opts ...GenerateOption) error {
 	options := (&GenerateOptions{}).ApplyOptions(opts...)
 

webhosting-operator/pkg/experiment/generator/theme.go

@@ -31,34 +31,14 @@ var (
 	themeFonts  = []string{"Arial", "Verdana", "Tahoma", "Trebuchet MS", "Times New Roman", "Georgia", "Garamond", "Courier New", "Brush Script MT"}
 )
 
-// EnsureThemes ensures there are exactly n themes with the given labels.
-// It keeps existing themes to speed up experiment preparation.
-func EnsureThemes(ctx context.Context, c client.Client, n int, opts ...GenerateOption) error {
-	options := (&GenerateOptions{}).ApplyOptions(opts...)
-
-	// delete excess themes
-	themeList := &webhostingv1alpha1.ThemeList{}
-	if err := c.List(ctx, themeList, client.MatchingLabels(options.Labels)); err != nil {
-		return err
-	}
-
-	for _, theme := range utils.PickNRandom(themeList.Items, len(themeList.Items)-n) {
-		if err := c.Delete(ctx, &theme); err != nil {
-			return err
-		}
-	}
-
-	// create missing themes
-	for i := 0; i < n-len(themeList.Items); i++ {
-		if err := CreateTheme(ctx, c, options); err != nil {
-			return err
-		}
-	}
-
-	return nil
+// CreateThemes creates n random themes.
+func CreateThemes(ctx context.Context, c client.Client, n int, opts ...GenerateOption) error {
+	return NTimesConcurrently(n, 10, func() error {
+		return CreateTheme(ctx, c, opts...)
+	})
 }
 
-// CreateTheme creates a random theme using the given client and labels.
+// CreateTheme creates a random theme.
 func CreateTheme(ctx context.Context, c client.Client, opts ...GenerateOption) error {
 	options := (&GenerateOptions{}).ApplyOptions(opts...)
 

webhosting-operator/pkg/experiment/generator/utils.go

@@ -19,7 +19,9 @@ package generator
 import (
 	"context"
 	"fmt"
+	"sync"
 
+	"github.com/hashicorp/go-multierror"
 	rbacv1 "k8s.io/api/rbac/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/client-go/util/workqueue"
@@ -58,6 +60,55 @@ func EmitN(n int) source.Source {
 	})
 }
 
+// NTimesConcurrently runs the given action n times. It distributes the work across the given number of concurrent
+// workers.
+func NTimesConcurrently(n, workers int, do func() error) error {
+	var (
+		wg   sync.WaitGroup
+		work = make(chan struct{}, workers)
+		errs = make(chan error, workers)
+	)
+
+	// start workers
+	for i := 0; i < workers; i++ {
+		wg.Add(1)
+
+		go func() {
+			defer wg.Done()
+
+			for range work {
+				errs <- do()
+			}
+		}()
+	}
+
+	// collect all errors
+	var (
+		allErrs  *multierror.Error
+		errsDone = make(chan struct{})
+	)
+	go func() {
+		for err := range errs {
+			allErrs = multierror.Append(allErrs, err)
+		}
+		close(errsDone)
+	}()
+
+	// emit n work items
+	for i := 0; i < n; i++ {
+		work <- struct{}{}
+	}
+	close(work)
+
+	// wait for all workers to process all work items
+	wg.Wait()
+	// signal error worker and wait for it to process all errors
+	close(errs)
+	<-errsDone
+
+	return allErrs.ErrorOrNil()
+}
+
 // CreateClusterScopedOwnerObject creates a new cluster-scoped object that has a single purpose: being used as an owner
 // for multiple objects that should be cleaned up at once. This is useful for cleaning up a lot of objects (of different
 // kinds) at once with a single DELETE call.

webhosting-operator/pkg/experiment/generator/website.go

@@ -28,34 +28,14 @@ import (
 	"github.com/timebertt/kubernetes-controller-sharding/webhosting-operator/pkg/utils"
 )
 
-// EnsureWebsites ensures there are exactly n websites with the given labels.
-// It keeps existing websites to speed up experiment preparation.
-func EnsureWebsites(ctx context.Context, c client.Client, n int, opts ...GenerateOption) error {
-	options := (&GenerateOptions{}).ApplyOptions(opts...)
-
-	// delete excess websites
-	websiteList := &webhostingv1alpha1.WebsiteList{}
-	if err := c.List(ctx, websiteList, client.MatchingLabels(options.Labels)); err != nil {
-		return err
-	}
-
-	for _, theme := range utils.PickNRandom(websiteList.Items, len(websiteList.Items)-n) {
-		if err := c.Delete(ctx, &theme); err != nil {
-			return err
-		}
-	}
-
-	// create missing websites
-	for i := 0; i < n-len(websiteList.Items); i++ {
-		if err := CreateWebsite(ctx, c, options); err != nil {
-			return err
-		}
-	}
-
-	return nil
+// CreateWebsites creates n random websites.
+func CreateWebsites(ctx context.Context, c client.Client, n int, opts ...GenerateOption) error {
+	return NTimesConcurrently(n, 10, func() error {
+		return CreateWebsite(ctx, c, opts...)
+	})
 }
 
-// CreateWebsite creates a random website using the given client and labels.
+// CreateWebsite creates a random website.
 func CreateWebsite(ctx context.Context, c client.Client, opts ...GenerateOption) error {
 	options := (&GenerateOptions{}).ApplyOptions(opts...)
 

webhosting-operator/pkg/experiment/scenario/basic/basic.go

@@ -99,15 +99,13 @@ func (s *scenario) Start(ctx context.Context) error {
 	log.Info("Preparing scenario objects")
 	opts := []generator.GenerateOption{generator.WithLabels(s.labels), generator.WithOwnerReference(ownerRef)}
 
-	// ensure there are exactly 50 themes generated by this scenario
-	log.Info("Ensuring themes")
-	if err := generator.EnsureThemes(ctx, s.Client, 50, opts...); err != nil {
+	log.Info("Preparing themes")
+	if err := generator.CreateThemes(ctx, s.Client, 50, opts...); err != nil {
 		return err
 	}
 
-	// ensure there are exactly 20 projects generated by this scenario
-	log.Info("Ensuring projects")
-	if err := generator.EnsureProjects(ctx, s.Client, 20, opts...); err != nil {
+	log.Info("Preparing projects")
+	if err := generator.CreateProjects(ctx, s.Client, 20, opts...); err != nil {
 		return err
 	}
 

webhosting-operator/pkg/experiment/scenario/reconcile/reconcile.go

@@ -95,21 +95,18 @@ func (s *scenario) Start(ctx context.Context) error {
 	log.Info("Preparing scenario objects")
 	opts := []generator.GenerateOption{generator.WithLabels(s.labels), generator.WithOwnerReference(ownerRef)}
 
-	// ensure there are exactly 50 themes generated by this scenario
-	log.Info("Ensuring themes")
-	if err := generator.EnsureThemes(ctx, s.Client, 50, opts...); err != nil {
+	log.Info("Preparing themes")
+	if err := generator.CreateThemes(ctx, s.Client, 50, opts...); err != nil {
 		return err
 	}
 
-	// ensure there are exactly 20 projects generated by this scenario
-	log.Info("Ensuring projects")
-	if err := generator.EnsureProjects(ctx, s.Client, 20, opts...); err != nil {
+	log.Info("Preparing projects")
+	if err := generator.CreateProjects(ctx, s.Client, 20, opts...); err != nil {
 		return err
 	}
 
-	// ensure there are exactly 10000 websites generated by this scenario
-	log.Info("Ensuring websites")
-	if err := generator.EnsureWebsites(ctx, s.Client, 10000, opts...); err != nil {
+	log.Info("Preparing websites")
+	if err := generator.CreateWebsites(ctx, s.Client, 10000, opts...); err != nil {
 		return err
 	}