e2e: Add SyncSource.Path (#1406)

- Add Path method that wraps Git & OCI dir & Helm chart name.
- Add SetExpectedSyncPath to update the directory expectation.
- Add SetExpectedGitSource, SetExpectedHelmSource, &
  SetExpectedOCISource to update source expectations and replace
  WithSyncDirectoryMap.
- Move all Git repo test defaults to setupTestCase
- Remove obsolete HelmChartVersionShaFn
- Add kinds.LookupID to replace usace of core.IDof using the schema.
- Add OCIImageID to simplify address/url generation
- Add RootSyncObjectGit and RepoSyncObjectGit to handle setting
  defaults that were in multiple places before.
- Add RootSyncObjectOCI & RepoSyncObjectOCI for building RSyncs with
  OCI sources.
- Add RootSyncObjectHelm & RepoSyncObjectHelm for building RSyncs
  with Helm sources.

e2e/nomostest/config_sync_sources.go

@@ -0,0 +1,139 @@
+// Copyright 2024 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package nomostest
+
+import (
+	"kpt.dev/configsync/e2e/nomostest/gitproviders"
+	"kpt.dev/configsync/e2e/nomostest/registryproviders"
+	"kpt.dev/configsync/e2e/nomostest/syncsource"
+	"kpt.dev/configsync/pkg/api/configsync"
+	"kpt.dev/configsync/pkg/core"
+)
+
+// SetExpectedSyncPath updates the SyncSource for the specified RootSync or
+// RepoSync with the provided Git dir, OCI dir.
+func SetExpectedSyncPath(nt *NT, syncID core.ID, syncPath string) {
+	source, exists := nt.SyncSources[syncID]
+	if !exists {
+		nt.T.Fatalf("Failed to update expectation for %s %s: nt.SyncSources not registered", syncID.Kind, syncID.ObjectKey)
+	}
+	switch tSource := source.(type) {
+	case *syncsource.GitSyncSource:
+		tSource.Directory = syncPath
+	case *syncsource.OCISyncSource:
+		tSource.Directory = syncPath
+	case *syncsource.HelmSyncSource:
+		nt.T.Fatalf("Failed to update expectation for %s %s: Use SetExpectedHelmChart instead", syncID.Kind, syncID.ObjectKey)
+	default:
+		nt.T.Fatalf("Invalid %s source %T: %s", syncID.Kind, source, syncID.Name)
+	}
+}
+
+// SetExpectedGitSource creates, updates, or replaces the SyncSource for the
+// specified RootSync or RepoSync with the provided Git repo.
+func SetExpectedGitSource(nt *NT, syncID core.ID, repo *gitproviders.Repository, syncPath string, sourceFormat configsync.SourceFormat) {
+	source, exists := nt.SyncSources[syncID]
+	if !exists {
+		nt.T.Logf("Creating expectation for %s %s to sync with Git repo (repository: %q, directory: %q, sourceFormat: %q)",
+			syncID.Kind, syncID.ObjectKey, repo.Name, syncPath, sourceFormat)
+		nt.SyncSources[syncID] = &syncsource.GitSyncSource{
+			Repository:   repo,
+			SourceFormat: sourceFormat,
+			Directory:    syncPath,
+		}
+		return
+	}
+	switch tSource := source.(type) {
+	case *syncsource.GitSyncSource:
+		nt.T.Logf("Updating expectation for %s %s to sync with Git repo (repository: %q, directory: %q, sourceFormat: %q)",
+			syncID.Kind, syncID.ObjectKey, repo.Name, syncPath, sourceFormat)
+		tSource.Repository = repo
+		tSource.SourceFormat = sourceFormat
+		tSource.Directory = syncPath
+	case *syncsource.OCISyncSource, *syncsource.HelmSyncSource:
+		nt.T.Logf("Replacing expectation for %s %s to sync with Git repo (repository: %q, directory: %q, sourceFormat: %q), instead of %T",
+			syncID.Kind, syncID.ObjectKey, repo.Name, syncPath, sourceFormat, source)
+		nt.SyncSources[syncID] = &syncsource.GitSyncSource{
+			Repository:   repo,
+			SourceFormat: sourceFormat,
+			Directory:    syncPath,
+		}
+	default:
+		nt.T.Fatalf("Invalid %s source %T: %s", syncID.Kind, source, syncID.Name)
+	}
+}
+
+// SetExpectedOCISource creates, updates, or replaces the SyncSource for the
+// specified RootSync or RepoSync with the provided OCI image.
+// ImageID is used for pulling. ImageDigest is used for validating the result.
+func SetExpectedOCISource(nt *NT, syncID core.ID, imageID registryproviders.OCIImageID, imageDigest, syncPath string) {
+	source, exists := nt.SyncSources[syncID]
+	if !exists {
+		nt.T.Logf("Creating expectation for %s %s to sync with OCI image (image: %q, directory: %q)",
+			syncID.Kind, syncID.ObjectKey, imageID, syncPath)
+		nt.SyncSources[syncID] = &syncsource.OCISyncSource{
+			ImageID:   imageID,
+			Digest:    imageDigest,
+			Directory: syncPath,
+		}
+		return
+	}
+	switch tSource := source.(type) {
+	case *syncsource.OCISyncSource:
+		nt.T.Logf("Updating expectation for %s %s to sync with OCI image (image: %q, directory: %q)",
+			syncID.Kind, syncID.ObjectKey, imageID, syncPath)
+		tSource.ImageID = imageID
+		tSource.Digest = imageDigest
+		tSource.Directory = syncPath
+	case *syncsource.GitSyncSource, *syncsource.HelmSyncSource:
+		nt.T.Logf("Replacing expectation for %s %s to sync with OCI image (image: %q, directory: %q), instead of %T",
+			syncID.Kind, syncID.ObjectKey, imageID, syncPath, source)
+		nt.SyncSources[syncID] = &syncsource.OCISyncSource{
+			ImageID:   imageID,
+			Digest:    imageDigest,
+			Directory: syncPath,
+		}
+	default:
+		nt.T.Fatalf("Invalid %s source %T: %s", syncID.Kind, source, syncID.Name)
+	}
+}
+
+// SetExpectedHelmSource creates, updates, or replaces the SyncSource for the
+// specified RootSync or RepoSync with the provided Helm chart ID.
+func SetExpectedHelmSource(nt *NT, syncID core.ID, chartID registryproviders.HelmChartID) {
+	source, exists := nt.SyncSources[syncID]
+	if !exists {
+		nt.T.Logf("Creating expectation for %s %s to sync with helm chart (id: %q)",
+			syncID.Kind, syncID.ObjectKey, chartID)
+		nt.SyncSources[syncID] = &syncsource.HelmSyncSource{
+			ChartID: chartID,
+		}
+		return
+	}
+	switch tSource := source.(type) {
+	case *syncsource.HelmSyncSource:
+		nt.T.Logf("Updating expectation for %s %s to sync with helm chart (id: %q)",
+			syncID.Kind, syncID.ObjectKey, chartID)
+		tSource.ChartID = chartID
+	case *syncsource.GitSyncSource, *syncsource.OCISyncSource:
+		nt.T.Logf("Replacing expectation for %s %s to sync with helm chart (id: %q), instead of %T",
+			syncID.Kind, syncID.ObjectKey, chartID, source)
+		nt.SyncSources[syncID] = &syncsource.HelmSyncSource{
+			ChartID: chartID,
+		}
+	default:
+		nt.T.Fatalf("Invalid %s source %T: %s", syncID.Kind, source, syncID.Name)
+	}
+}

e2e/nomostest/git-server.go

@@ -34,9 +34,9 @@ import (
 const testGitNamespace = "config-management-system-test"
 const testGitServer = "test-git-server"
 
-const testGitServerImage = testing.TestInfraArtifactRegistry + "/git-server:v1.0.0-68df304c"
+const testGitServerImage = testing.TestInfraArtifactRepositoryAddress + "/git-server:v1.0.0-68df304c"
 const testGitHTTPServer = "http-git-server"
-const testGitHTTPServerImage = testing.TestInfraArtifactRegistry + "/http-git-server:v1.0.0-b3b4984cd"
+const testGitHTTPServerImage = testing.TestInfraArtifactRepositoryAddress + "/http-git-server:v1.0.0-b3b4984cd"
 const safeToEvictAnnotation = "cluster-autoscaler.kubernetes.io/safe-to-evict"
 
 func testGitServerSelector() map[string]string {

e2e/nomostest/new.go

@@ -64,15 +64,7 @@ func newOptStruct(testName, tmpDir string, ntOptions ...ntopts.Opt) *ntopts.New
 		Name:   testName,
 		TmpDir: tmpDir,
 		MultiRepo: ntopts.MultiRepo{
-			SyncSources: syncsource.Set{
-				// All tests default to having a RootSync named root-sync using
-				// a Git repository, unless otherwise specified.
-				DefaultRootSyncID: &syncsource.GitSyncSource{
-					// RootSync root-sync defaults to using hierarchy.
-					// Use ntopts.SyncWithGitSource with ntopts.Unstructured to change.
-					SourceFormat: configsync.SourceFormatHierarchy,
-				},
-			},
+			SyncSources: syncsource.Set{},
 			// Default to 1m to keep tests fast.
 			// To override, use WithReconcileTimeout.
 			ReconcileTimeout: ptr.To(1 * time.Minute),
@@ -425,15 +417,36 @@ func applyAutoPilotKeepAlive(nt *NT) error {
 
 func setupTestCase(nt *NT, opts *ntopts.New) {
 	nt.T.Log("[SETUP] New test case")
+	// Expect default RootSync root-sync
+	if _, found := opts.SyncSources[DefaultRootSyncID]; !found {
+		opts.SyncSources[DefaultRootSyncID] = &syncsource.GitSyncSource{}
+	}
 	// init all repositories. This ensures that:
 	// - local repo is initialized and empty
 	// - remote repo exists (except for LocalProvider, which is done in portForwardGitServer)
 	for id, source := range opts.SyncSources {
 		switch tSource := source.(type) {
 		case *syncsource.GitSyncSource:
-			nt.SyncSources[id] = &syncsource.GitSyncSource{
-				Repository: initRepository(nt, id.Kind, id.ObjectKey, tSource.SourceFormat),
+			if tSource.Repository == nil {
+				// Initialize the Repository
+				tSource.Repository = initRepository(nt, id.Kind, id.ObjectKey, tSource.SourceFormat)
+			}
+			if tSource.Directory == "" {
+				// Set the default Directory
+				tSource.Directory = gitproviders.DefaultSyncDir
+			}
+			if tSource.SourceFormat == "" {
+				// Set the default SourceFormat
+				switch id.Kind {
+				case configsync.RootSyncKind:
+					// RootSyncs use Hierarchy by default
+					tSource.SourceFormat = configsync.SourceFormatHierarchy
+				case configsync.RepoSyncKind:
+					// RepoSync only ever use unstructured
+					tSource.SourceFormat = configsync.SourceFormatUnstructured
+				}
 			}
+			nt.SyncSources[id] = tSource
 		// case *syncsource.HelmSyncSource:
 		// case *syncsource.OCISyncSource:
 		// TODO: setup OCI & Helm RootSyncs

e2e/nomostest/nt.go

@@ -952,13 +952,6 @@ func RemoteRootRepoSha1Fn(nt *NT, nn types.NamespacedName) (string, error) {
 	return commit, nil
 }
 
-// HelmChartVersionShaFn returns the provided chart version as a function.
-func HelmChartVersionShaFn(chartVersion string) Sha1Func {
-	return func(*NT, types.NamespacedName) (string, error) {
-		return chartVersion, nil
-	}
-}
-
 // RemoteNsRepoSha1Fn returns the latest commit from a RepoSync Git spec.
 func RemoteNsRepoSha1Fn(nt *NT, nn types.NamespacedName) (string, error) {
 	rs := &v1beta1.RepoSync{}

e2e/nomostest/ntopts/multi_repo.go

@@ -71,15 +71,6 @@ func Unstructured(source *syncsource.GitSyncSource) {
 func SyncWithGitSource(id core.ID, gitOpts ...GitSourceOption) func(opt *New) {
 	return func(opt *New) {
 		source := &syncsource.GitSyncSource{}
-		// Set the default source options
-		switch id.Kind {
-		case configsync.RootSyncKind:
-			// RootSyncs use Hierarchy by default
-			source.SourceFormat = configsync.SourceFormatHierarchy
-		case configsync.RepoSyncKind:
-			// RepoSync only ever use unstructured
-			source.SourceFormat = configsync.SourceFormatUnstructured
-		}
 		// Modify the default source with user specified options
 		for _, gitOpt := range gitOpts {
 			gitOpt(source)

e2e/nomostest/registryproviders/artifact_registry.go

@@ -76,21 +76,27 @@ func (a *ArtifactRegistryProvider) Teardown() error {
 	return nil
 }
 
-// registryHost returns the domain of the artifact registry
-func (a *ArtifactRegistryProvider) registryHost() string {
+// RegistryRemoteAddress returns the remote address of the registry.
+func (a *ArtifactRegistryProvider) RegistryRemoteAddress() string {
 	return fmt.Sprintf("%s-docker.pkg.dev", a.location)
 }
 
+// RepositoryPath returns the name or path of the repository, without the
+// registry or image details.
+func (a *ArtifactRegistryProvider) RepositoryPath() string {
+	return fmt.Sprintf("%s/%s/%s", a.project, a.repositoryName, a.repositorySuffix)
+}
+
 // repositoryLocalAddress returns the address of the repository in the
 // registry for use by local clients, like the helm or crane clients.
 func (a *ArtifactRegistryProvider) repositoryLocalAddress() (string, error) {
-	return a.repositoryRemoteAddress()
+	return a.repositoryRemoteAddress(), nil
 }
 
 // repositoryRemoteAddress returns the address of the repository in the
 // registry for use by remote clients, like Kubernetes or Config Sync.
-func (a *ArtifactRegistryProvider) repositoryRemoteAddress() (string, error) {
-	return fmt.Sprintf("%s/%s/%s/%s", a.registryHost(), a.project, a.repositoryName, a.repositorySuffix), nil
+func (a *ArtifactRegistryProvider) repositoryRemoteAddress() string {
+	return fmt.Sprintf("%s/%s", a.RegistryRemoteAddress(), a.RepositoryPath())
 }
 
 // ImageLocalAddress returns the address of the image in the registry for
@@ -105,12 +111,8 @@ func (a *ArtifactRegistryProvider) ImageLocalAddress(imageName string) (string,
 
 // ImageRemoteAddress returns the address of the image in the registry for
 // use by remote clients, like Kubernetes or Config Sync.
-func (a *ArtifactRegistryProvider) ImageRemoteAddress(imageName string) (string, error) {
-	repoAddr, err := a.repositoryRemoteAddress()
-	if err != nil {
-		return "", err
-	}
-	return fmt.Sprintf("%s/%s", repoAddr, imageName), nil
+func (a *ArtifactRegistryProvider) ImageRemoteAddress(imageName string) string {
+	return fmt.Sprintf("%s/%s", a.repositoryRemoteAddress(), imageName)
 }
 
 // createRepository uses gcloud to create the repository, if it doesn't exist.
@@ -146,11 +148,7 @@ func (a *ArtifactRegistryProvider) createRepository() error {
 // Helm doesn't provide a way to delete remote package images, so we're using
 // gcloud instead. For helm charts, the image name is the chart name.
 func (a *ArtifactRegistryProvider) deleteImage(imageName, digest string) error {
-	imageAddress, err := a.ImageRemoteAddress(imageName)
-	if err != nil {
-		return err
-	}
-	imageAddress = fmt.Sprintf("%s@%s", imageAddress, digest)
+	imageAddress := fmt.Sprintf("%s@%s", a.ImageRemoteAddress(imageName), digest)
 	if _, err := a.gcloudClient.Gcloud("artifacts", "docker", "images", "delete", imageAddress, "--delete-tags", "--project", a.project); err != nil {
 		return fmt.Errorf("deleting image from registry %s: %w", imageAddress, err)
 	}
@@ -174,7 +172,7 @@ func (a *ArtifactRegistryOCIProvider) Client() CraneClient {
 
 // Login to the registry with the crane client
 func (a *ArtifactRegistryOCIProvider) Login() error {
-	registryHost := a.registryHost()
+	registryHost := a.RegistryRemoteAddress()
 	if err := a.OCIClient.Login(registryHost); err != nil {
 		return fmt.Errorf("OCIClient.Login(%q): %w", registryHost, err)
 	}
@@ -183,7 +181,7 @@ func (a *ArtifactRegistryOCIProvider) Login() error {
 
 // Logout of the registry with the crane client
 func (a *ArtifactRegistryOCIProvider) Logout() error {
-	registryHost := a.registryHost()
+	registryHost := a.RegistryRemoteAddress()
 	if err := a.OCIClient.Logout(registryHost); err != nil {
 		return fmt.Errorf("OCIClient.Logout(%q): %w", registryHost, err)
 	}
@@ -253,7 +251,7 @@ func (a *ArtifactRegistryHelmProvider) Client() HelmClient {
 
 // Login to the registry with the HelmClient
 func (a *ArtifactRegistryHelmProvider) Login() error {
-	registryHost := a.registryHost()
+	registryHost := a.RegistryRemoteAddress()
 	if err := a.HelmClient.Login(registryHost); err != nil {
 		return fmt.Errorf("HelmClient.Login(%q): %w", registryHost, err)
 	}
@@ -262,7 +260,7 @@ func (a *ArtifactRegistryHelmProvider) Login() error {
 
 // Logout of the registry with the HelmClient
 func (a *ArtifactRegistryHelmProvider) Logout() error {
-	registryHost := a.registryHost()
+	registryHost := a.RegistryRemoteAddress()
 	if err := a.HelmClient.Logout(registryHost); err != nil {
 		return fmt.Errorf("HelmClient.Logout(%q): %w", registryHost, err)
 	}
@@ -287,12 +285,8 @@ func (a *ArtifactRegistryHelmProvider) RepositoryLocalURL() (string, error) {
 
 // RepositoryRemoteURL is the repositoryRemoteAddress prepended with oci://
 // For pulling with RSync's `.spec.helm.repo`
-func (a *ArtifactRegistryHelmProvider) RepositoryRemoteURL() (string, error) {
-	repoAddr, err := a.repositoryRemoteAddress()
-	if err != nil {
-		return "", err
-	}
-	return ociURL(repoAddr), nil
+func (a *ArtifactRegistryHelmProvider) RepositoryRemoteURL() string {
+	return ociURL(a.repositoryRemoteAddress())
 }
 
 // PushPackage pushes the local helm chart as an OCI image to the remote registry.