Allow environment provider jobs not bound to cluster

api/v1alpha1/environmentrequest_types.go

@@ -45,6 +45,27 @@ type Splitter struct {
 	Tests []Test `json:"tests"`
 }
 
+// EnvironmentRequestJobConfig defines parameters required by
+type EnvironmentRequestJobConfig struct {
+	EiffelMessageBus      RabbitMQ `json:"eiffelMessageBus"`
+	EtosMessageBus        RabbitMQ `json:"etosMessageBus"`
+	EtosApi               string   `json:"etosApi"`
+	EtosEncryptionKey     Var      `json:"etosEncryptionKeySecretRef"`
+	EtosEtcdHost          string   `json:"etosEtcdHost"`
+	EtosEtcdPort          string   `json:"etosEtcdPort"`
+	EtosEventDataTimeout  string   `json:"etosEventDataTimeout"`
+	EtosGraphQlServer     string   `json:"etosGraphQlServer"`
+	EtosRoutingKeyTag     string   `json:"etosRoutingKeyTag"`
+	EtosWaitForTimeout    string   `json:"etosWaitForTimeout"`
+	EtosTestRunnerVersion string   `json:"etosTestRunnerVersion"`
+
+	EnvironmentProviderEventDataTimeout string            `json:"environmentProviderEventDataTimeout"`
+	EnvironmentProviderImage            string            `json:"environmentProviderImage"`
+	EnvironmentProviderImagePullPolicy  corev1.PullPolicy `json:"environmentProviderImagePullPolicy"`
+	EnvironmentProviderServiceAccount   string            `json:"environmentProviderServiceAccount"`
+	EnvironmentProviderTestSuiteTimeout string            `json:"environmentProviderTestSuiteTimeout"`
+}
+
 // EnvironmentRequestSpec defines the desired state of EnvironmentRequest
 type EnvironmentRequestSpec struct {
 	// ID is the ID for the environments generated. Will be generated if nil. The ID is a UUID, any version, and regex matches that.
@@ -61,8 +82,10 @@ type EnvironmentRequestSpec struct {
 	// TODO: Dataset per provider?
 	Dataset *apiextensionsv1.JSON `json:"dataset,omitempty"`
 
-	Providers EnvironmentProviders `json:"providers"`
-	Splitter  Splitter             `json:"splitter"`
+	Providers          EnvironmentProviders        `json:"providers"`
+	Splitter           Splitter                    `json:"splitter"`
+	ServiceAccountName string                      `json:"serviceaccountname,omitempty"`
+	JobConfig          EnvironmentRequestJobConfig `json:"jobConfig,omitempty"`
 }
 
 // EnvironmentRequestStatus defines the observed state of EnvironmentRequest

config/manager/kustomization.yaml

@@ -5,4 +5,4 @@ kind: Kustomization
 images:
 - name: controller
   newName: registry.nordix.org/eiffel/etos-controller
-  newTag: 61d5b687
+  newTag: f5d42ddf

internal/controller/environmentrequest_controller.go

@@ -211,7 +211,10 @@ func (r *EnvironmentRequestReconciler) reconcileEnvironmentProvider(ctx context.
 	}
 	// No environment providers, create environment provider
 	if providers.empty() {
-		environmentProvider := r.environmentProviderJob(environmentrequest)
+		environmentProvider, _err := r.environmentProviderJob(ctx, environmentrequest)
+		if _err != nil {
+			return _err
+		}
 		if err := ctrl.SetControllerReference(environmentrequest, environmentProvider, r.Scheme); err != nil {
 			return err
 		}
@@ -222,20 +225,104 @@ func (r *EnvironmentRequestReconciler) reconcileEnvironmentProvider(ctx context.
 	return nil
 }
 
+func (r EnvironmentRequestReconciler) envVarListFrom(ctx context.Context, environmentrequest *etosv1alpha1.EnvironmentRequest) ([]corev1.EnvVar, error) {
+	etosEncryptionKey, err := environmentrequest.Spec.JobConfig.EtosEncryptionKey.Get(ctx, r.Client, environmentrequest.Namespace)
+	if err != nil {
+		return nil, err
+	}
+	envList := []corev1.EnvVar{
+		{
+			Name:  "REQUEST",
+			Value: environmentrequest.Name,
+		},
+		{
+			Name:  "ETOS_API",
+			Value: environmentrequest.Spec.JobConfig.EtosApi,
+		},
+		{
+			Name:  "ETOS_GRAPHQL_SERVER",
+			Value: environmentrequest.Spec.JobConfig.EtosGraphQlServer,
+		},
+		{
+			Name:  "ETOS_ENCRYPTION_KEY",
+			Value: string(etosEncryptionKey),
+		},
+		{
+			Name:  "ETOS_ETCD_HOST",
+			Value: environmentrequest.Spec.JobConfig.EtosEtcdHost,
+		},
+		{
+			Name:  "ETOS_ETCD_PORT",
+			Value: environmentrequest.Spec.JobConfig.EtosEtcdPort,
+		},
+		{
+			// Optional when environmentrequest is not issued by testrun, i. e. created separately.
+			// When the environment request is issued by a testrun, this variable is propagated
+			// further from environment provider to test runner.
+			Name:  "ETR_VERSION",
+			Value: environmentrequest.Spec.JobConfig.EtosTestRunnerVersion,
+		},
+	}
+
+	var bus etosv1alpha1.RabbitMQ
+	for _, prefix := range []string{"", "ETOS_"} {
+		if prefix == "" {
+			bus = environmentrequest.Spec.JobConfig.EiffelMessageBus
+		} else if prefix == "ETOS_" {
+			bus = environmentrequest.Spec.JobConfig.EtosMessageBus
+		}
+		envVars := []corev1.EnvVar{
+			{
+				Name:  fmt.Sprintf("%sRABBITMQ_HOST", prefix),
+				Value: bus.Host,
+			},
+			{
+				Name:  fmt.Sprintf("%sRABBITMQ_VHOST", prefix),
+				Value: bus.Vhost,
+			},
+			{
+				Name:  fmt.Sprintf("%sRABBITMQ_PORT", prefix),
+				Value: bus.Port,
+			},
+			{
+				Name:  fmt.Sprintf("%sRABBITMQ_SSL", prefix),
+				Value: bus.SSL,
+			},
+			{
+				Name:  fmt.Sprintf("%sRABBITMQ_EXCHANGE", prefix),
+				Value: bus.Exchange,
+			},
+			{
+				Name:  fmt.Sprintf("%sRABBITMQ_USERNAME", prefix),
+				Value: bus.Username,
+			},
+			{
+				Name:  fmt.Sprintf("%sRABBITMQ_PASSWORD", prefix),
+				Value: bus.Password.Value,
+			},
+		}
+		envList = append(envList, envVars...)
+	}
+	return envList, nil
+}
+
 // environmentProviderJob is the job definition for an etos environment provider.
-func (r EnvironmentRequestReconciler) environmentProviderJob(environmentrequest *etosv1alpha1.EnvironmentRequest) *batchv1.Job {
+func (r EnvironmentRequestReconciler) environmentProviderJob(ctx context.Context, environmentrequest *etosv1alpha1.EnvironmentRequest) (*batchv1.Job, error) {
+	logger := log.FromContext(ctx)
 	ttl := int32(300)
 	grace := int64(30)
 	backoff := int32(0)
-	// TODO: Cluster might not be a part of the environment request.
-	cluster := environmentrequest.Labels["etos.eiffel-community.github.io/cluster"]
+	envVarList, err := r.envVarListFrom(ctx, environmentrequest)
+	if err != nil {
+		logger.Error(err, "Failed to create environment variable list for environment provider")
+		return nil, err
+	}
 	return &batchv1.Job{
 		ObjectMeta: metav1.ObjectMeta{
 			Labels: map[string]string{
-				"etos.eiffel-community.github.io/id":      environmentrequest.Spec.Identifier, // TODO: omitempty
-				"etos.eiffel-community.github.io/cluster": cluster,
-				"app.kubernetes.io/name":                  "environment-provider",
-				"app.kubernetes.io/part-of":               "etos",
+				"etos.eiffel-community.github.io/id": environmentrequest.Spec.Identifier, // TODO: omitempty
+				"app.kubernetes.io/name":             "environment-provider",
+				"app.kubernetes.io/part-of":          "etos",
 			},
 			Annotations:  make(map[string]string),
 			GenerateName: "environment-provider-", // unique names to allow multiple environment provider jobs
@@ -249,8 +336,8 @@ func (r EnvironmentRequestReconciler) environmentProviderJob(environmentrequest
 					Name: environmentrequest.Name,
 				},
 				Spec: corev1.PodSpec{
+					ServiceAccountName:            environmentrequest.Spec.ServiceAccountName,
 					TerminationGracePeriodSeconds: &grace,
-					ServiceAccountName:            fmt.Sprintf("%s-provider", cluster),
 					RestartPolicy:                 "Never",
 					Containers: []corev1.Container{
 						{
@@ -267,27 +354,13 @@ func (r EnvironmentRequestReconciler) environmentProviderJob(environmentrequest
 									corev1.ResourceCPU:    resource.MustParse("100m"),
 								},
 							},
-							EnvFrom: []corev1.EnvFromSource{
-								{
-									SecretRef: &corev1.SecretEnvSource{
-										LocalObjectReference: corev1.LocalObjectReference{
-											Name: fmt.Sprintf("%s-environment-provider-cfg", cluster),
-										},
-									},
-								},
-							},
-							Env: []corev1.EnvVar{
-								{
-									Name:  "REQUEST",
-									Value: environmentrequest.Name,
-								},
-							},
+							Env: envVarList,
 						},
 					},
 				},
 			},
 		},
-	}
+	}, nil
 }
 
 // registerOwnerIndexForJob will set an index of the jobs that an environment request owns.

internal/controller/jobs.go

@@ -145,6 +145,7 @@ func terminationLog(ctx context.Context, c client.Reader, job *batchv1.Job, cont
 				return &Result{Conclusion: ConclusionFailed}, errors.New("could not read termination log from pod")
 			}
 			var result Result
+
 			if err := json.Unmarshal([]byte(status.State.Terminated.Message), &result); err != nil {
 				logger.Error(err, "failed to unmarshal termination log to a result struct")
 				return &Result{Conclusion: ConclusionFailed, Description: status.State.Terminated.Message}, nil

internal/controller/testrun_controller.go

@@ -59,6 +59,7 @@ type TestRunReconciler struct {
 	client.Client
 	Scheme *runtime.Scheme
 	Clock
+	Cluster *etosv1alpha1.Cluster
 }
 
 /*
@@ -130,9 +131,19 @@ func (r *TestRunReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ct
 			return ctrl.Result{RequeueAfter: next}, nil
 		}
 		return ctrl.Result{}, nil
+
+	}
+	clusterNamespacedName := types.NamespacedName{
+		Name:      testrun.Spec.Cluster,
+		Namespace: req.NamespacedName.Namespace,
+	}
+	cluster := &etosv1alpha1.Cluster{}
+	if err := r.Get(ctx, clusterNamespacedName, cluster); err != nil {
+		logger.Info("Failed to get cluster resource!")
+		return ctrl.Result{}, nil
 	}
 
-	if err := r.reconcile(ctx, testrun); err != nil {
+	if err := r.reconcile(ctx, cluster, testrun); err != nil {
 		if apierrors.IsConflict(err) {
 			return ctrl.Result{Requeue: true}, nil
 		}
@@ -143,7 +154,7 @@ func (r *TestRunReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ct
 	return ctrl.Result{}, nil
 }
 
-func (r *TestRunReconciler) reconcile(ctx context.Context, testrun *etosv1alpha1.TestRun) error {
+func (r *TestRunReconciler) reconcile(ctx context.Context, cluster *etosv1alpha1.Cluster, testrun *etosv1alpha1.TestRun) error {
 	logger := log.FromContext(ctx)
 
 	// Set initial statuses if not set.
@@ -187,7 +198,7 @@ func (r *TestRunReconciler) reconcile(ctx context.Context, testrun *etosv1alpha1
 	}
 
 	// Create environment request
-	err, exit := r.reconcileEnvironmentRequest(ctx, testrun)
+	err, exit := r.reconcileEnvironmentRequest(ctx, cluster, testrun)
 	if err != nil {
 		return err
 	}
@@ -236,7 +247,7 @@ func (r *TestRunReconciler) complete(ctx context.Context, testrun *etosv1alpha1.
 }
 
 // reconcileEnvironmentRequest will check the status of environment requests, create new ones if necessary.
-func (r *TestRunReconciler) reconcileEnvironmentRequest(ctx context.Context, testrun *etosv1alpha1.TestRun) (error, bool) {
+func (r *TestRunReconciler) reconcileEnvironmentRequest(ctx context.Context, cluster *etosv1alpha1.Cluster, testrun *etosv1alpha1.TestRun) (error, bool) {
 	logger := log.FromContext(ctx)
 	var environmentRequestList etosv1alpha1.EnvironmentRequestList
 	if err := r.List(ctx, &environmentRequestList, client.InNamespace(testrun.Namespace), client.MatchingFields{TestRunOwnerKey: testrun.Name}); err != nil {
@@ -261,7 +272,7 @@ func (r *TestRunReconciler) reconcileEnvironmentRequest(ctx context.Context, tes
 			}
 		}
 		if !found {
-			request := r.environmentRequest(testrun, suite)
+			request := r.environmentRequest(cluster, testrun, suite)
 			if err := ctrl.SetControllerReference(testrun, request, r.Scheme); err != nil {
 				return err, true
 			}
@@ -414,7 +425,18 @@ func (r *TestRunReconciler) checkEnvironment(ctx context.Context, testrun *etosv
 }
 
 // environmentRequest is the definition for an environment request.
-func (r TestRunReconciler) environmentRequest(testrun *etosv1alpha1.TestRun, suite etosv1alpha1.Suite) *etosv1alpha1.EnvironmentRequest {
+func (r TestRunReconciler) environmentRequest(cluster *etosv1alpha1.Cluster, testrun *etosv1alpha1.TestRun, suite etosv1alpha1.Suite) *etosv1alpha1.EnvironmentRequest {
+	eventRepository := cluster.Spec.EventRepository.Host
+	if cluster.Spec.ETOS.Config.ETOSEventRepositoryURL != "" {
+		eventRepository = cluster.Spec.ETOS.Config.ETOSEventRepositoryURL
+	}
+
+	eiffelMessageBus := cluster.Spec.MessageBus.EiffelMessageBus
+	eiffelMessageBus.Host = fmt.Sprintf("%s-%s", cluster.Name, eiffelMessageBus.Host)
+
+	etosMessageBus := cluster.Spec.MessageBus.ETOSMessageBus
+	etosMessageBus.Host = fmt.Sprintf("%s-%s", cluster.Name, etosMessageBus.Host)
+
 	return &etosv1alpha1.EnvironmentRequest{
 		ObjectMeta: metav1.ObjectMeta{
 			Labels: map[string]string{
@@ -451,7 +473,26 @@ func (r TestRunReconciler) environmentRequest(testrun *etosv1alpha1.TestRun, sui
 			Splitter: etosv1alpha1.Splitter{
 				Tests: suite.Tests,
 			},
-			Image: testrun.Spec.EnvironmentProvider.Image,
+			Image:              testrun.Spec.EnvironmentProvider.Image,
+			ServiceAccountName: fmt.Sprintf("%s-provider", testrun.Spec.Cluster),
+			JobConfig: etosv1alpha1.EnvironmentRequestJobConfig{
+				EiffelMessageBus:                    eiffelMessageBus,
+				EtosMessageBus:                      etosMessageBus,
+				EtosApi:                             cluster.Spec.ETOS.Config.ETOSApiURL,
+				EtosEncryptionKey:                   cluster.Spec.ETOS.Config.EncryptionKey,
+				EtosRoutingKeyTag:                   cluster.Spec.ETOS.Config.RoutingKeyTag,
+				EtosGraphQlServer:                   eventRepository,
+				EtosEtcdHost:                        fmt.Sprintf("%s-etcd", cluster.Name),
+				EtosEtcdPort:                        cluster.Spec.Database.Etcd.Port,
+				EtosEventDataTimeout:                cluster.Spec.ETOS.Config.EventDataTimeout,
+				EtosWaitForTimeout:                  cluster.Spec.ETOS.Config.EnvironmentTimeout,
+				EnvironmentProviderEventDataTimeout: cluster.Spec.ETOS.Config.EventDataTimeout,
+				EnvironmentProviderImage:            cluster.Spec.ETOS.EnvironmentProvider.Image.Image,
+				EnvironmentProviderImagePullPolicy:  cluster.Spec.ETOS.EnvironmentProvider.ImagePullPolicy,
+				EnvironmentProviderServiceAccount:   fmt.Sprintf("%s-provider", cluster.Name),
+				EnvironmentProviderTestSuiteTimeout: cluster.Spec.ETOS.Config.TestSuiteTimeout,
+				EtosTestRunnerVersion:               cluster.Spec.ETOS.TestRunner.Version,
+			},
 		},
 	}
 }

internal/etos/etos.go

@@ -282,7 +282,7 @@ func (r *ETOSDeployment) reconcileSecret(ctx context.Context, logger logr.Logger
 // reconcileEnvironmentProviderConfig will reconcile the secret to use as configuration for the ETOS environment provider.
 func (r *ETOSDeployment) reconcileEnvironmentProviderConfig(ctx context.Context, logger logr.Logger, name types.NamespacedName, encryptionKeyName, configmapName string, owner metav1.Object) (*corev1.Secret, error) {
 	name = types.NamespacedName{Name: fmt.Sprintf("%s-environment-provider-cfg", name.Name), Namespace: name.Namespace}
-	target, err := r.environmentProviderConfig(ctx, name, encryptionKeyName, configmapName)
+	target, err := r.environmentProviderConfig(ctx, logger, name, encryptionKeyName, configmapName)
 	if err != nil {
 		return nil, err
 	}
@@ -305,7 +305,7 @@ func (r *ETOSDeployment) reconcileEnvironmentProviderConfig(ctx context.Context,
 }
 
 // config creates a new Secret to be used as configuration for the ETOS API.
-func (r *ETOSDeployment) environmentProviderConfig(ctx context.Context, name types.NamespacedName, encryptionKeyName, configmapName string) (*corev1.Secret, error) {
+func (r *ETOSDeployment) environmentProviderConfig(ctx context.Context, logger logr.Logger, name types.NamespacedName, encryptionKeyName, configmapName string) (*corev1.Secret, error) {
 	eiffel := &corev1.Secret{}
 	if err := r.Get(ctx, types.NamespacedName{Name: r.rabbitmqSecret, Namespace: name.Namespace}, eiffel); err != nil {
 		return nil, err
@@ -327,10 +327,11 @@ func (r *ETOSDeployment) environmentProviderConfig(ctx context.Context, name typ
 	maps.Copy(data, etos.Data)
 	maps.Copy(data, encryption.Data)
 	maps.Copy(data, config.Data)
-	return &corev1.Secret{
+	secret := &corev1.Secret{
 		ObjectMeta: r.meta(name),
 		Data:       data,
-	}, nil
+	}
+	return secret, nil
 }
 
 // ingress creates an ingress resource definition for ETOS.