pgupgrade: Ensure that old pg deployments are terminated before upgra…

…de job is run (PROJQUAY-8092)

apis/quay/v1/quayregistry_types.go

@@ -197,9 +197,10 @@ const (
 	ConditionReasonMigrationsFailed     ConditionReason = "MigrationsFailed"
 	ConditionReasonMigrationsJobMissing ConditionReason = "MigrationsJobMissing"
 
-	ConditionReasonPostgresUpgradeInProgress ConditionReason = "PostgresUpgradeInProgress"
-	ConditionReasonPostgresUpgradeFailed     ConditionReason = "PostgresUpgradeFailed"
-	ConditionReasonPostgresUpgradeJobMissing ConditionReason = "PostgresUpgradeJobMissing"
+	ConditionReasonPostgresUpgradeInitializing ConditionReason = "PostgresUpgradeInitializing"
+	ConditionReasonPostgresUpgradeInProgress   ConditionReason = "PostgresUpgradeInProgress"
+	ConditionReasonPostgresUpgradeFailed       ConditionReason = "PostgresUpgradeFailed"
+	ConditionReasonPostgresUpgradeJobMissing   ConditionReason = "PostgresUpgradeJobMissing"
 
 	ConditionReasonComponentsCreationSuccess             ConditionReason = "ComponentsCreationSuccess"
 	ConditionReasonUpgradeUnsupported                    ConditionReason = "UpgradeUnsupported"

controllers/quay/features.go

@@ -463,6 +463,45 @@ func (r *QuayRegistryReconciler) checkNeedsPostgresUpgradeForComponent(
 	return nil
 }
 
+func (r *QuayRegistryReconciler) checkPostgresUpgradeInitializationStatusForComponent(
+	ctx context.Context, qctx *quaycontext.QuayRegistryContext, quay *v1.QuayRegistry, component v1.ComponentKind,
+) error {
+	componentInfo := map[v1.ComponentKind]struct {
+		deploymentSuffix string
+		upgradeField     *bool
+	}{
+		v1.ComponentClairPostgres: {"clair-postgres", &qctx.ClairPgUpradeInitializing},
+		v1.ComponentPostgres:      {"quay-database", &qctx.PgUpradeInitializing},
+	}
+
+	info, ok := componentInfo[component]
+	if !ok {
+		return fmt.Errorf("invalid component kind: %s", component)
+	}
+
+	deploymentName := fmt.Sprintf("%s-%s", quay.GetName(), info.deploymentSuffix)
+	r.Log.Info(fmt.Sprintf("checking deployment termination status for %s", component))
+
+	postgresDeployment := &appsv1.Deployment{}
+	if err := r.Client.Get(
+		ctx,
+		types.NamespacedName{
+			Name:      deploymentName,
+			Namespace: quay.GetNamespace(),
+		},
+		postgresDeployment,
+	); err != nil {
+		r.Log.Info(fmt.Sprintf("%s deployment not found, skipping", component))
+		return nil
+	}
+
+	// if the deployment is found, we need to set initiliztion to true
+	*info.upgradeField = true
+
+	return nil
+
+}
+
 func extractImageName(imageName string) string {
 	parts := strings.Split(imageName, "@")
 	if len(parts) > 1 {

controllers/quay/quayregistry_controller.go

@@ -536,6 +536,18 @@ func (r *QuayRegistryReconciler) Reconcile(ctx context.Context, req ctrl.Request
 				fmt.Sprintf("error checking for pg upgrade: %s", err),
 			)
 		}
+
+		err = r.checkPostgresUpgradeInitializationStatusForComponent(ctx, quayContext, updatedQuay, v1.ComponentPostgres)
+		if err != nil {
+			return r.reconcileWithCondition(
+				ctx,
+				&quay,
+				v1.ConditionTypeRolloutBlocked,
+				metav1.ConditionTrue,
+				v1.ConditionReasonPostgresUpgradeFailed,
+				fmt.Sprintf("error checking for pg upgrade: %s", err),
+			)
+		}
 	}
 
 	// Populate the QuayContext with whether or not the QuayRegistry needs an upgrade
@@ -551,6 +563,18 @@ func (r *QuayRegistryReconciler) Reconcile(ctx context.Context, req ctrl.Request
 				fmt.Sprintf("error checking for pg upgrade: %s", err),
 			)
 		}
+
+		err = r.checkPostgresUpgradeInitializationStatusForComponent(ctx, quayContext, updatedQuay, v1.ComponentClairPostgres)
+		if err != nil {
+			return r.reconcileWithCondition(
+				ctx,
+				&quay,
+				v1.ConditionTypeRolloutBlocked,
+				metav1.ConditionTrue,
+				v1.ConditionReasonPostgresUpgradeFailed,
+				fmt.Sprintf("error checking for pg upgrade: %s", err),
+			)
+		}
 	}
 
 	if err := r.checkBuildManagerAvailable(quayContext, cbundle); err != nil {
@@ -729,6 +753,21 @@ func (r *QuayRegistryReconciler) Reconcile(ctx context.Context, req ctrl.Request
 		}
 	}
 
+	// If postgres pods are still terminating, requeue
+	if quayContext.ClairPgUpradeInitializing || quayContext.PgUpradeInitializing {
+		if err := r.updateWithCondition(
+			ctx,
+			updatedQuay,
+			v1.ConditionComponentsCreated,
+			metav1.ConditionFalse,
+			v1.ConditionReasonPostgresUpgradeInitializing,
+			"running postgres update initialization",
+		); err != nil {
+			log.Error(err, "failed to update `conditions` of `QuayRegistry`")
+		}
+		return r.Requeue, nil
+	}
+
 	// if postgres needs to be updated, then we need to wait until the upgrade job
 	if quayContext.NeedsPgUpgrade || quayContext.NeedsClairPgUpgrade {
 		if err := r.updateWithCondition(

kustomize/components/clairpgupgrade/base/kustomization.yaml

@@ -4,5 +4,5 @@ resources:
   - ./clair-pg-upgrade.job.yaml
   - ./clair-pg-old.persistentvolumeclaim.yaml
   - ./clair-pg-old.deployment.yaml
-patchesStrategicMerge:
-  - ./clair-pg.deployment.patch.yaml
+components:
+  - ../initialize

kustomize/components/clairpgupgrade/initialize/kustomization.yaml

@@ -0,0 +1,4 @@
+apiVersion: kustomize.config.k8s.io/v1alpha1
+kind: Component
+patchesStrategicMerge:
+  - ./clair-pg.deployment.patch.yaml

kustomize/components/clairpgupgrade/scale-down-clair/kustomization.yaml

@@ -2,5 +2,3 @@ apiVersion: kustomize.config.k8s.io/v1alpha1
 kind: Component
 patchesStrategicMerge:
   - ./clair.deployment.patch.yaml
-components:
-  - "../base"

kustomize/components/pgupgrade/kustomization.yaml → kustomize/components/pgupgrade/base/kustomization.yaml

@@ -7,4 +7,5 @@ resources:
   - ./quay-pg-old.deployment.yaml
 patchesStrategicMerge:
   - ./quay.deployment.patch.yaml
-  - ./quay-pg.deployment.patch.yaml
+components:
+  - ../initialize

kustomize/components/pgupgrade/initialize/kustomization.yaml

@@ -0,0 +1,5 @@
+# Overlay variant for upgrading to current Project Quay release.
+apiVersion: kustomize.config.k8s.io/v1alpha1
+kind: Component
+patchesStrategicMerge:
+  - ./quay-pg.deployment.patch.yaml

pkg/context/context.go

@@ -33,10 +33,14 @@ type QuayRegistryContext struct {
 	SecretKey         string
 
 	// Database
-	DbUri               string
-	DbRootPw            string
-	NeedsPgUpgrade      bool
-	NeedsClairPgUpgrade bool
+	DbUri    string
+	DbRootPw string
+
+	// Upgrade flags
+	NeedsPgUpgrade            bool
+	PgUpradeInitializing      bool
+	NeedsClairPgUpgrade       bool
+	ClairPgUpradeInitializing bool
 
 	// Clair integration
 	SecurityScannerV4PSK string

pkg/kustomize/kustomize.go

@@ -460,11 +460,19 @@ func KustomizationFor(
 	}
 
 	if ctx.NeedsPgUpgrade {
-		componentPaths = append(componentPaths, "../components/pgupgrade")
+		if ctx.PgUpradeInitializing {
+			componentPaths = append(componentPaths, "../components/pgupgrade/initialize")
+		} else {
+			componentPaths = append(componentPaths, "../components/pgupgrade/base")
+		}
 	}
+
 	if ctx.NeedsClairPgUpgrade {
 		if v1.ComponentIsManaged(quay.Spec.Components, v1.ComponentClair) {
 			componentPaths = append(componentPaths, "../components/clairpgupgrade/scale-down-clair")
+		}
+		if ctx.ClairPgUpradeInitializing {
+			componentPaths = append(componentPaths, "../components/clairpgupgrade/initialize")
 		} else {
 			componentPaths = append(componentPaths, "../components/clairpgupgrade/base")
 		}