Merge pull request openshift#4753 from cheesesashimi/zzlotnik/incorpo…

…rate-changes-to-helpers

MCO-1470: Incorporates upstream devex helper changes for OCL testing

Containerfile.helpers-ci

@@ -0,0 +1,11 @@
+FROM registry.ci.openshift.org/ocp/builder:rhel-9-golang-1.22-builder-multi-openshift-4.18 AS builder
+ENV GOCACHE="/go/.cache" \
+    GOMODCACHE="/go/pkg/mod"
+WORKDIR /go/src/github.com/openshift/machine-config-operator
+COPY . .
+RUN --mount=type=cache,target=/go/.cache,z \
+    --mount=type=cache,target=/go/pkg/mod,z \
+    make install-helpers DESTDIR=/helpers
+
+FROM registry.ci.openshift.org/ocp/builder:rhel-9-enterprise-base-multi-openshift-4.18 AS final
+COPY --from=builder /helpers/usr/bin /usr/bin

Makefile

@@ -1,3 +1,6 @@
+E2E_ROOT_DIR = ./test
+E2E_SUITES = $(notdir $(wildcard $(E2E_ROOT_DIR)/e2e*))
+
 MCO_COMPONENTS = daemon controller server operator
 EXTRA_COMPONENTS = apiserver-watcher machine-os-builder
 ALL_COMPONENTS = $(patsubst %,machine-config-%,$(MCO_COMPONENTS)) $(EXTRA_COMPONENTS)
@@ -41,9 +44,13 @@ clean:
 _build-component-%:
 	WHAT_PATH=cmd/$* WHAT=$(basename $*) hack/build-go.sh
 
-# Build the helpers under hack/cmd.
+# Build the helpers under devex/cmd.
 _build-helper-%:
-	WHAT_PATH=hack/cmd/$* WHAT=$(basename $*) hack/build-go.sh
+	WHAT_PATH=devex/cmd/$* WHAT=$(basename $*) hack/build-go.sh
+
+# Verify that an e2e test is valid Golang by doing a trial compilation.
+_verify-e2e-%:
+	go test -c -tags=$(GOTAGS) -o _output/$* ./test/$*/...
 
 # Use podman to build the image.
 image:
@@ -103,16 +110,25 @@ endif
 
 install-tools: install-golangci-lint install-go-junit-report install-setup-envtest
 
-# Run verification steps
-# Example:
-#    make verify
-verify: install-tools
+# Runs golangci-lint
+lint: install-tools
 	./hack/golangci-lint.sh $(GOTAGS)
+
+# Verifies templates.
+verify-templates:
 	hack/verify-templates.sh
+
+# Verifies devex helpers
+verify-helpers:
 	# Conditionally tries to build the helper binaries in CI.
 	hack/verify-helpers.sh
 
-HELPERS_DIR := hack/cmd
+# Runs all verification steps
+# Example:
+#    make verify
+verify: install-tools verify-e2e lint verify-templates verify-helpers
+
+HELPERS_DIR := devex/cmd
 HELPER_BINARIES := $(notdir $(wildcard $(HELPERS_DIR)/*))
 
 .PHONY: helpers
@@ -135,14 +151,22 @@ endef
 # Create a target for each component
 $(foreach C, $(HELPER_BINARIES), $(eval $(call helper_target_template,$(C))))
 
+define verify_e2e_target_template =
+ .PHONY: $(1)
+ $(1): _verify-e2e-$(1)
+endef
+# Create a target for each e2e suite
+$(foreach C, $(E2E_SUITES), $(eval $(call verify_e2e_target_template,$(C))))
+
+
 .PHONY: binaries helpers install
 
 # Build all binaries:
 # Example:
 #    make binaries
 binaries: $(patsubst %,_build-component-%,$(ALL_COMPONENTS))
 
-# Installs the helper binaries from hack/cmd.
+# Installs the helper binaries from devex/cmd.
 install-helpers: helpers
 	for helper in $(HELPER_BINARIES); do \
 		install -D -m 0755 _output/linux/$(GOARCH)/$${helper} $(DESTDIR)$(PREFIX)/bin/$${helper}; \
@@ -157,6 +181,9 @@ Dockerfile.rhel7: Dockerfile Makefile
 	(echo '# THIS FILE IS GENERATED FROM '$<' DO NOT EDIT' && \
 	 sed -e s,org/openshift/release,org/ocp/builder, -e s,/openshift/origin-v4.0:base,/ocp/4.0:base, < $<) > $@.tmp && mv $@.tmp $@
 
+# Validates that all of the e2e test suites are valid Golang by performing a test compilation.
+verify-e2e: $(patsubst %,_verify-e2e-%,$(E2E_SUITES))
+
 # This was copied from https://github.com/openshift/cluster-image-registry-operator
 test-e2e: install-go-junit-report
 	set -o pipefail; go test -tags=$(GOTAGS) -failfast -timeout 170m -v$${WHAT:+ -run="$$WHAT"} ./test/e2e/ ./test/e2e-techpreview-shared/ | ./hack/test-with-junit.sh $(@)

devex/README.md

@@ -0,0 +1,18 @@
+# devex
+
+## Background
+
+This directory contains Golang programs which are most likely to be of use to
+fellow OpenShift developers, especially members of the
+[machine-config-operator](https://github.com/openshift/machine-config-operator)
+team. The helpers found here may be of use to you. They may not. They may
+completely break entirely.
+
+It is worth mentioning that these helpers may get your cluster into a
+difficult-to-recover-from state. So do not use these on a production OpenShift
+cluster.
+
+## Installation
+
+From the repo root, run: `make install-helpers`. Note: You'll periodically have
+to update the helpers based upon the current state of the MCO repository.

hack/cmd/mco-builder/internal/builders/common.go → devex/cmd/mco-builder/internal/builders/common.go

@@ -6,7 +6,7 @@ import (
 	"os/exec"
 	"strings"
 
-	"github.com/openshift/machine-config-operator/hack/internal/pkg/errors"
+	"github.com/openshift/machine-config-operator/devex/internal/pkg/errors"
 	"k8s.io/apimachinery/pkg/util/sets"
 	"k8s.io/klog/v2"
 )

hack/cmd/mco-builder/internal/builders/podman.go → devex/cmd/mco-builder/internal/builders/podman.go

@@ -5,7 +5,7 @@ import (
 	"os"
 	"os/exec"
 
-	"github.com/openshift/machine-config-operator/hack/internal/pkg/errors"
+	"github.com/openshift/machine-config-operator/devex/internal/pkg/errors"
 	"k8s.io/klog/v2"
 )
 

hack/cmd/mco-builder/local.go → devex/cmd/mco-builder/local.go

@@ -9,10 +9,10 @@ import (
 	"path/filepath"
 	"runtime"
 
-	"github.com/openshift/machine-config-operator/hack/cmd/mco-builder/internal/builders"
-	"github.com/openshift/machine-config-operator/hack/internal/pkg/containers"
-	"github.com/openshift/machine-config-operator/hack/internal/pkg/rollout"
-	"github.com/openshift/machine-config-operator/hack/internal/pkg/utils"
+	"github.com/openshift/machine-config-operator/devex/cmd/mco-builder/internal/builders"
+	"github.com/openshift/machine-config-operator/devex/internal/pkg/containers"
+	"github.com/openshift/machine-config-operator/devex/internal/pkg/rollout"
+	"github.com/openshift/machine-config-operator/devex/internal/pkg/utils"
 	ctrlcommon "github.com/openshift/machine-config-operator/pkg/controller/common"
 	"github.com/openshift/machine-config-operator/test/framework"
 	"github.com/spf13/cobra"

hack/cmd/mco-builder/revert.go → devex/cmd/mco-builder/revert.go

@@ -4,7 +4,7 @@ import (
 	"context"
 	"fmt"
 
-	"github.com/openshift/machine-config-operator/hack/internal/pkg/rollout"
+	"github.com/openshift/machine-config-operator/devex/internal/pkg/rollout"
 	ctrlcommon "github.com/openshift/machine-config-operator/pkg/controller/common"
 	"github.com/openshift/machine-config-operator/test/framework"
 	"github.com/spf13/cobra"

hack/cmd/mco-push/replace.go → devex/cmd/mco-push/replace.go

@@ -3,8 +3,8 @@ package main
 import (
 	"fmt"
 
-	"github.com/openshift/machine-config-operator/hack/internal/pkg/containers"
-	"github.com/openshift/machine-config-operator/hack/internal/pkg/rollout"
+	"github.com/openshift/machine-config-operator/devex/internal/pkg/containers"
+	"github.com/openshift/machine-config-operator/devex/internal/pkg/rollout"
 	"github.com/openshift/machine-config-operator/test/framework"
 	"github.com/spf13/cobra"
 	"k8s.io/klog/v2"

hack/cmd/mco-push/restart.go → devex/cmd/mco-push/restart.go

@@ -1,7 +1,7 @@
 package main
 
 import (
-	"github.com/openshift/machine-config-operator/hack/internal/pkg/rollout"
+	"github.com/openshift/machine-config-operator/devex/internal/pkg/rollout"
 	"github.com/openshift/machine-config-operator/test/framework"
 	"github.com/spf13/cobra"
 	"k8s.io/klog/v2"

hack/cmd/mco-push/revert.go → devex/cmd/mco-push/revert.go

@@ -1,7 +1,7 @@
 package main
 
 import (
-	"github.com/openshift/machine-config-operator/hack/internal/pkg/rollout"
+	"github.com/openshift/machine-config-operator/devex/internal/pkg/rollout"
 	"github.com/openshift/machine-config-operator/test/framework"
 	"github.com/spf13/cobra"
 	"k8s.io/klog/v2"

hack/cmd/onclustertesting/build.go → devex/cmd/onclustertesting/build.go

@@ -6,7 +6,7 @@ import (
 	"os/exec"
 	"path/filepath"
 
-	"github.com/openshift/machine-config-operator/hack/internal/pkg/utils"
+	"github.com/openshift/machine-config-operator/devex/internal/pkg/utils"
 	ctrlcommon "github.com/openshift/machine-config-operator/pkg/controller/common"
 	"github.com/openshift/machine-config-operator/test/framework"
 	"golang.org/x/sync/errgroup"

hack/cmd/onclustertesting/ci.go → devex/cmd/onclustertesting/ci.go

@@ -5,8 +5,8 @@ import (
 	"fmt"
 	"time"
 
-	"github.com/openshift/machine-config-operator/hack/internal/pkg/rollout"
-	"github.com/openshift/machine-config-operator/hack/internal/pkg/utils"
+	"github.com/openshift/machine-config-operator/devex/internal/pkg/rollout"
+	"github.com/openshift/machine-config-operator/devex/internal/pkg/utils"
 	"github.com/openshift/machine-config-operator/test/framework"
 	"golang.org/x/sync/errgroup"
 	apierrs "k8s.io/apimachinery/pkg/api/errors"
@@ -156,7 +156,9 @@ func waitForPoolsToComplete(cs *framework.ClientSet, pools []string) error {
 		pool := pool
 
 		eg.Go(func() error {
-			return rollout.WaitForMachineConfigPoolUpdateToComplete(cs, time.Minute*30, pool)
+			// TODO: Figure out why WaitForMachineConfigPoolToComplete() is not
+			// showing control-plane node status when nodes are considered.
+			return rollout.WaitForOnlyMachineConfigPoolToComplete(cs, pool, time.Minute*30)
 		})
 	}
 

hack/cmd/onclustertesting/helpers.go → devex/cmd/onclustertesting/helpers.go

@@ -6,7 +6,7 @@ import (
 	"fmt"
 	"strings"
 
-	"github.com/openshift/machine-config-operator/hack/cmd/onclustertesting/internal/legacycmds"
+	"github.com/openshift/machine-config-operator/devex/cmd/onclustertesting/internal/legacycmds"
 	"github.com/openshift/machine-config-operator/test/framework"
 	"github.com/openshift/machine-config-operator/test/helpers"
 	"golang.org/x/sync/errgroup"

hack/cmd/onclustertesting/internal/legacycmds/clearstatus.go → devex/cmd/onclustertesting/internal/legacycmds/clearstatus.go

@@ -5,7 +5,7 @@ import (
 	"fmt"
 
 	mcfgv1 "github.com/openshift/api/machineconfiguration/v1"
-	"github.com/openshift/machine-config-operator/hack/internal/pkg/utils"
+	"github.com/openshift/machine-config-operator/devex/internal/pkg/utils"
 	"github.com/openshift/machine-config-operator/test/framework"
 	"github.com/spf13/cobra"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"

hack/cmd/onclustertesting/internal/legacycmds/extract.go → devex/cmd/onclustertesting/internal/legacycmds/extract.go

@@ -6,7 +6,7 @@ import (
 	"os"
 	"path/filepath"
 
-	"github.com/openshift/machine-config-operator/hack/internal/pkg/utils"
+	"github.com/openshift/machine-config-operator/devex/internal/pkg/utils"
 	ctrlcommon "github.com/openshift/machine-config-operator/pkg/controller/common"
 	"github.com/openshift/machine-config-operator/test/framework"
 	"github.com/spf13/cobra"

hack/cmd/onclustertesting/internal/legacycmds/machineconfig.go → devex/cmd/onclustertesting/internal/legacycmds/machineconfig.go

@@ -7,7 +7,7 @@ import (
 	"path/filepath"
 
 	"github.com/ghodss/yaml"
-	"github.com/openshift/machine-config-operator/hack/internal/pkg/utils"
+	"github.com/openshift/machine-config-operator/devex/internal/pkg/utils"
 	ctrlcommon "github.com/openshift/machine-config-operator/pkg/controller/common"
 
 	ign3types "github.com/coreos/ignition/v2/config/v3_4/types"

hack/cmd/onclustertesting/internal/legacycmds/setimage.go → devex/cmd/onclustertesting/internal/legacycmds/setimage.go

@@ -5,7 +5,7 @@ import (
 	"fmt"
 
 	mcfgv1 "github.com/openshift/api/machineconfiguration/v1"
-	"github.com/openshift/machine-config-operator/hack/internal/pkg/utils"
+	"github.com/openshift/machine-config-operator/devex/internal/pkg/utils"
 	ctrlcommon "github.com/openshift/machine-config-operator/pkg/controller/common"
 	"github.com/openshift/machine-config-operator/test/framework"
 	"github.com/spf13/cobra"

hack/cmd/onclustertesting/internal/legacycmds/setstatus.go → devex/cmd/onclustertesting/internal/legacycmds/setstatus.go

@@ -4,7 +4,7 @@ import (
 	"context"
 
 	mcfgv1 "github.com/openshift/api/machineconfiguration/v1"
-	"github.com/openshift/machine-config-operator/hack/internal/pkg/utils"
+	"github.com/openshift/machine-config-operator/devex/internal/pkg/utils"
 	"github.com/openshift/machine-config-operator/pkg/apihelpers"
 	"github.com/openshift/machine-config-operator/test/framework"
 	"github.com/spf13/cobra"

hack/cmd/onclustertesting/legacycmds.go → devex/cmd/onclustertesting/legacycmds.go

@@ -3,7 +3,7 @@ package main
 import (
 	"os"
 
-	"github.com/openshift/machine-config-operator/hack/cmd/onclustertesting/internal/legacycmds"
+	"github.com/openshift/machine-config-operator/devex/cmd/onclustertesting/internal/legacycmds"
 	"github.com/spf13/cobra"
 )
 

hack/cmd/onclustertesting/machineosconfigs.go → devex/cmd/onclustertesting/machineosconfigs.go

@@ -8,7 +8,7 @@ import (
 	mcfgv1 "github.com/openshift/api/machineconfiguration/v1"
 	mcfgv1alpha1 "github.com/openshift/api/machineconfiguration/v1alpha1"
 	clientmachineconfigv1alpha1 "github.com/openshift/client-go/machineconfiguration/clientset/versioned/typed/machineconfiguration/v1alpha1"
-	"github.com/openshift/machine-config-operator/hack/internal/pkg/utils"
+	"github.com/openshift/machine-config-operator/devex/internal/pkg/utils"
 	ctrlcommon "github.com/openshift/machine-config-operator/pkg/controller/common"
 	"github.com/openshift/machine-config-operator/test/framework"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"

hack/cmd/onclustertesting/optin.go → devex/cmd/onclustertesting/optin.go

@@ -4,7 +4,7 @@ import (
 	"context"
 	"fmt"
 
-	"github.com/openshift/machine-config-operator/hack/internal/pkg/utils"
+	"github.com/openshift/machine-config-operator/devex/internal/pkg/utils"
 	"github.com/openshift/machine-config-operator/test/framework"
 	"github.com/openshift/machine-config-operator/test/helpers"
 	"github.com/spf13/cobra"

hack/cmd/onclustertesting/optout.go → devex/cmd/onclustertesting/optout.go

@@ -4,7 +4,7 @@ import (
 	"context"
 	"fmt"
 
-	"github.com/openshift/machine-config-operator/hack/internal/pkg/utils"
+	"github.com/openshift/machine-config-operator/devex/internal/pkg/utils"
 	"github.com/openshift/machine-config-operator/test/framework"
 	"github.com/openshift/machine-config-operator/test/helpers"
 	"github.com/spf13/cobra"

hack/cmd/onclustertesting/secrets.go → devex/cmd/onclustertesting/secrets.go

@@ -7,7 +7,7 @@ import (
 	"strings"
 
 	"github.com/ghodss/yaml"
-	"github.com/openshift/machine-config-operator/hack/internal/pkg/utils"
+	"github.com/openshift/machine-config-operator/devex/internal/pkg/utils"
 	ctrlcommon "github.com/openshift/machine-config-operator/pkg/controller/common"
 	"github.com/openshift/machine-config-operator/test/framework"
 	"github.com/openshift/machine-config-operator/test/helpers"

hack/cmd/onclustertesting/setup.go → devex/cmd/onclustertesting/setup.go

@@ -5,7 +5,7 @@ import (
 	"fmt"
 	"time"
 
-	"github.com/openshift/machine-config-operator/hack/internal/pkg/utils"
+	"github.com/openshift/machine-config-operator/devex/internal/pkg/utils"
 	ctrlcommon "github.com/openshift/machine-config-operator/pkg/controller/common"
 	"github.com/openshift/machine-config-operator/test/framework"
 	"github.com/spf13/cobra"

hack/cmd/onclustertesting/teardown.go → devex/cmd/onclustertesting/teardown.go

@@ -3,7 +3,7 @@ package main
 import (
 	"context"
 
-	"github.com/openshift/machine-config-operator/hack/internal/pkg/utils"
+	"github.com/openshift/machine-config-operator/devex/internal/pkg/utils"
 	"github.com/openshift/machine-config-operator/test/framework"
 	"github.com/spf13/cobra"
 	apierrs "k8s.io/apimachinery/pkg/api/errors"

hack/cmd/run-on-all-nodes/main.go → devex/cmd/run-on-all-nodes/main.go

@@ -17,7 +17,7 @@ import (
 
 	"golang.org/x/sync/errgroup"
 
-	"github.com/openshift/machine-config-operator/hack/internal/pkg/utils"
+	"github.com/openshift/machine-config-operator/devex/internal/pkg/utils"
 	"github.com/openshift/machine-config-operator/test/framework"
 	"github.com/spf13/cobra"
 	aggerrs "k8s.io/apimachinery/pkg/util/errors"

hack/cmd/wait-for-mcp/main.go → devex/cmd/wait-for-mcp/main.go

@@ -4,7 +4,7 @@ import (
 	"os"
 	"time"
 
-	"github.com/openshift/machine-config-operator/hack/internal/pkg/rollout"
+	"github.com/openshift/machine-config-operator/devex/internal/pkg/rollout"
 	"github.com/openshift/machine-config-operator/test/framework"
 	"github.com/spf13/cobra"
 	"k8s.io/component-base/cli"