Merge pull request #2189 from husseinferas/images-by-semver

Adding support for semver filter in sync command

cmd/skopeo/sync.go

@@ -12,6 +12,7 @@ import (
 	"regexp"
 	"strings"
 
+	"github.com/Masterminds/semver/v3"
 	commonFlag "github.com/containers/common/pkg/flag"
 	"github.com/containers/common/pkg/retry"
 	"github.com/containers/image/v5/copy"
@@ -71,6 +72,7 @@ type tlsVerifyConfig struct {
 type registrySyncConfig struct {
 	Images           map[string][]string    // Images map images name to slices with the images' references (tags, digests)
 	ImagesByTagRegex map[string]string      `yaml:"images-by-tag-regex"` // Images map images name to regular expression with the images' tags
+	ImagesBySemver   map[string]string      `yaml:"images-by-semver"`    // ImagesBySemver maps a repository to a semver constraint (e.g. '>=3.14') to match images' tags to
 	Credentials      types.DockerAuthConfig // Username and password used to authenticate with the registry
 	TLSVerify        tlsVerifyConfig        `yaml:"tls-verify"` // TLS verification mode (enabled by default)
 	CertDir          string                 `yaml:"cert-dir"`   // Path to the TLS certificates of the registry
@@ -304,6 +306,14 @@ func imagesToCopyFromRegistry(registryName string, cfg registrySyncConfig, sourc
 		serverCtx.DockerAuthConfig = &cfg.Credentials
 	}
 	var repoDescList []repoDescriptor
+
+	if len(cfg.Images) == 0 && len(cfg.ImagesByTagRegex) == 0 && len(cfg.ImagesBySemver) == 0 {
+		logrus.WithFields(logrus.Fields{
+			"registry": registryName,
+		}).Warn("No images specified for registry")
+		return repoDescList, nil
+	}
+
 	for imageName, refs := range cfg.Images {
 		repoLogger := logrus.WithFields(logrus.Fields{
 			"repo":     imageName,
@@ -368,61 +378,144 @@ func imagesToCopyFromRegistry(registryName string, cfg registrySyncConfig, sourc
 			Context:   serverCtx})
 	}
 
-	for imageName, tagRegex := range cfg.ImagesByTagRegex {
-		repoLogger := logrus.WithFields(logrus.Fields{
-			"repo":     imageName,
+	// include repository descriptors for cfg.ImagesByTagRegex
+	{
+		filterCollection, err := tagRegexFilterCollection(cfg.ImagesByTagRegex)
+		if err != nil {
+			logrus.Error(err)
+		} else {
+			additionalRepoDescList := filterSourceReferences(serverCtx, registryName, filterCollection)
+			repoDescList = append(repoDescList, additionalRepoDescList...)
+		}
+	}
+
+	// include repository descriptors for cfg.ImagesBySemver
+	{
+		filterCollection, err := semverFilterCollection(cfg.ImagesBySemver)
+		if err != nil {
+			logrus.Error(err)
+		} else {
+			additionalRepoDescList := filterSourceReferences(serverCtx, registryName, filterCollection)
+			repoDescList = append(repoDescList, additionalRepoDescList...)
+		}
+	}
+
+	return repoDescList, nil
+}
+
+// filterFunc is a function used to limit the initial set of image references
+// using tags, patterns, semver, etc.
+type filterFunc func(*logrus.Entry, types.ImageReference) bool
+
+// filterCollection is a map of repository names to filter functions.
+type filterCollection map[string]filterFunc
+
+// filterSourceReferences lists tags for images specified in the collection and
+// filters them using assigned filter functions.
+// It returns a list of repoDescriptors.
+func filterSourceReferences(sys *types.SystemContext, registryName string, collection filterCollection) []repoDescriptor {
+	var repoDescList []repoDescriptor
+	for repoName, filter := range collection {
+		logger := logrus.WithFields(logrus.Fields{
+			"repo":     repoName,
 			"registry": registryName,
 		})
-		repoRef, err := parseRepositoryReference(fmt.Sprintf("%s/%s", registryName, imageName))
+
+		repoRef, err := parseRepositoryReference(fmt.Sprintf("%s/%s", registryName, repoName))
 		if err != nil {
-			repoLogger.Error("Error parsing repository name, skipping")
+			logger.Error("Error parsing repository name, skipping")
 			logrus.Error(err)
 			continue
 		}
 
-		repoLogger.Info("Processing repo")
+		logger.Info("Processing repo")
 
 		var sourceReferences []types.ImageReference
 
-		tagReg, err := regexp.Compile(tagRegex)
+		logger.Info("Querying registry for image tags")
+		sourceReferences, err = imagesToCopyFromRepo(sys, repoRef)
 		if err != nil {
-			repoLogger.WithFields(logrus.Fields{
-				"regex": tagRegex,
-			}).Error("Error parsing regex, skipping")
+			logger.Error("Error processing repo, skipping")
 			logrus.Error(err)
 			continue
 		}
 
-		repoLogger.Info("Querying registry for image tags")
-		allSourceReferences, err := imagesToCopyFromRepo(serverCtx, repoRef)
-		if err != nil {
-			repoLogger.Error("Error processing repo, skipping")
-			logrus.Error(err)
+		var filteredSourceReferences []types.ImageReference
+		for _, ref := range sourceReferences {
+			if filter(logger, ref) {
+				filteredSourceReferences = append(filteredSourceReferences, ref)
+			}
+		}
+
+		if len(filteredSourceReferences) == 0 {
+			logger.Warnf("No refs to sync found")
 			continue
 		}
 
-		repoLogger.Infof("Start filtering using the regular expression: %v", tagRegex)
-		for _, sReference := range allSourceReferences {
-			tagged, isTagged := sReference.DockerReference().(reference.Tagged)
+		repoDescList = append(repoDescList, repoDescriptor{
+			ImageRefs: filteredSourceReferences,
+			Context:   sys,
+		})
+	}
+	return repoDescList
+}
+
+// tagRegexFilterCollection converts a map of (repository name, tag regex) pairs
+// into a filterCollection, which is a map of (repository name, filter function)
+// pairs.
+func tagRegexFilterCollection(collection map[string]string) (filterCollection, error) {
+	filters := filterCollection{}
+
+	for repoName, tagRegex := range collection {
+		pattern, err := regexp.Compile(tagRegex)
+		if err != nil {
+			return nil, err
+		}
+
+		f := func(logger *logrus.Entry, sourceReference types.ImageReference) bool {
+			tagged, isTagged := sourceReference.DockerReference().(reference.Tagged)
 			if !isTagged {
-				repoLogger.Errorf("Internal error, reference %s does not have a tag, skipping", sReference.DockerReference())
-				continue
-			}
-			if tagReg.MatchString(tagged.Tag()) {
-				sourceReferences = append(sourceReferences, sReference)
+				logger.Errorf("Internal error, reference %s does not have a tag, skipping", sourceReference.DockerReference())
+				return false
 			}
+			return pattern.MatchString(tagged.Tag())
 		}
+		filters[repoName] = f
+	}
 
-		if len(sourceReferences) == 0 {
-			repoLogger.Warnf("No refs to sync found")
-			continue
+	return filters, nil
+}
+
+// semverFilterCollection converts a map of (repository name, array of semver constraints) pairs
+// into a filterCollection, which is a map of (repository name, filter function)
+// pairs.
+func semverFilterCollection(collection map[string]string) (filterCollection, error) {
+	filters := filterCollection{}
+
+	for repoName, constraintString := range collection {
+		constraint, err := semver.NewConstraint(constraintString)
+		if err != nil {
+			return nil, err
 		}
-		repoDescList = append(repoDescList, repoDescriptor{
-			ImageRefs: sourceReferences,
-			Context:   serverCtx})
+
+		f := func(logger *logrus.Entry, sourceReference types.ImageReference) bool {
+			tagged, isTagged := sourceReference.DockerReference().(reference.Tagged)
+			if !isTagged {
+				logger.Errorf("Internal error, reference %s does not have a tag, skipping", sourceReference.DockerReference())
+				return false
+			}
+			tagVersion, err := semver.NewVersion(tagged.Tag())
+			if err != nil {
+				logger.Tracef("Tag %q cannot be parsed as semver, skipping", tagged.Tag())
+				return false
+			}
+			return constraint.Check(tagVersion)
+		}
+
+		filters[repoName] = f
 	}
 
-	return repoDescList, nil
+	return filters, nil
 }
 
 // imagesToCopy retrieves all the images to copy from a specified sync source
@@ -489,13 +582,6 @@ func imagesToCopy(source string, transport string, sourceCtx *types.SystemContex
 			return descriptors, err
 		}
 		for registryName, registryConfig := range cfg {
-			if len(registryConfig.Images) == 0 && len(registryConfig.ImagesByTagRegex) == 0 {
-				logrus.WithFields(logrus.Fields{
-					"registry": registryName,
-				}).Warn("No images specified for registry")
-				continue
-			}
-
 			descs, err := imagesToCopyFromRegistry(registryName, registryConfig, *sourceCtx)
 			if err != nil {
 				return descriptors, fmt.Errorf("Failed to retrieve list of images from registry %q: %w", registryName, err)

docs/skopeo-sync.1.md

@@ -219,6 +219,8 @@ registry.example.com:
             - "sha256:0000000000000000000000000000000011111111111111111111111111111111"
     images-by-tag-regex:
         nginx: ^1\.13\.[12]-alpine-perl$
+    images-by-semver:
+        alpine: ">= 3.12.0"
     credentials:
         username: john
         password: this is a secret
@@ -239,6 +241,14 @@ This will copy the following images:
 - Repository `registry.example.com/redis`: images tagged "1.0" and "2.0" along with image with digest "sha256:0000000000000000000000000000000011111111111111111111111111111111".
 - Repository `registry.example.com/nginx`: images tagged "1.13.1-alpine-perl" and "1.13.2-alpine-perl".
 - Repository `quay.io/coreos/etcd`: images tagged "latest".
+- Repository `registry.example.com/alpine`: all images with tags match the semantic version constraint ">= 3.12.0" ("3.12.0, "3.12.1", ... ,"4.0.0", ...)
+
+The full list of possible semantic version comparisons can be found in the
+upstream library's documentation:
+https://github.com/Masterminds/semver/tree/v3.2.0#basic-comparisons.
+
+Version ordering and precedence is understood as defined here:
+https://semver.org/#spec-item-11.
 
 For the registry `registry.example.com`, the "john"/"this is a secret" credentials are used, with server TLS certificates located at `/home/john/certs`.
 

go.mod

@@ -24,6 +24,7 @@ require (
 require (
 	dario.cat/mergo v1.0.0 // indirect
 	github.com/BurntSushi/toml v1.3.2 // indirect
+	github.com/Masterminds/semver/v3 v3.2.1
 	github.com/Microsoft/go-winio v0.6.1 // indirect
 	github.com/Microsoft/hcsshim v0.12.0-rc.1 // indirect
 	github.com/VividCortex/ewma v1.2.0 // indirect

go.sum

@@ -6,6 +6,8 @@ github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 h1:L/gRVlceqvL25
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/BurntSushi/toml v1.3.2 h1:o7IhLm0Msx3BaB+n3Ag7L8EVlByGnpq14C4YWiu/gL8=
 github.com/BurntSushi/toml v1.3.2/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
+github.com/Masterminds/semver/v3 v3.2.1 h1:RN9w6+7QoMeJVGyfmbcgs28Br8cvmnucEXnY0rYXWg0=
+github.com/Masterminds/semver/v3 v3.2.1/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYrf8m9wsX0PNOMQ=
 github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow=
 github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM=
 github.com/Microsoft/hcsshim v0.12.0-rc.1 h1:Hy+xzYujv7urO5wrgcG58SPMOXNLrj4WCJbySs2XX/A=