Merge pull request #11 from appuio/oprhan-management

Orphaned image deletion

README.md

@@ -10,7 +10,7 @@
 
 ## General
 
-The image cleanup client is used to clean up Docker images in a Docker Registry when they are tagged using git SHA. The cleaning is done either using git commit hashes or tags. Defaults to hashes otherwise ```-t``` flag should be used.
+The image cleanup client is used to clean up container images in an image registry when they are tagged using git SHA. The cleaning is done either using git commit hashes or tags. Defaults to hashes otherwise ```--tag``` flag should be used. The tool also allows to clean orphan image stream tags using ```--orphan``` flag, the orphan image stream tags are images that do not have any Git commit/tag. There are secondary flags which help to norrow the cleaning process, for more information use ```--help```.
 
 This helps to save space because obsolete images are being removed from the registry.
 

cmd/imagestream.go

@@ -2,24 +2,30 @@ package cmd
 
 import (
 	log "github.com/sirupsen/logrus"
+	"time"
+	"regexp"
 
 	"github.com/appuio/image-cleanup/pkg/cleanup"
 	"github.com/appuio/image-cleanup/pkg/git"
 	"github.com/appuio/image-cleanup/pkg/kubernetes"
 	"github.com/appuio/image-cleanup/pkg/openshift"
 	"github.com/spf13/cobra"
+	"github.com/karrick/tparse"
 )
 
 // ImageStreamCleanupOptions is a struct to support the cleanup command
 type ImageStreamCleanupOptions struct {
-	Force       bool
-	CommitLimit int
-	RepoPath    string
-	Keep        int
-	ImageStream string
-	Namespace   string
-	Tag         bool
-	Sorted      string
+	Force       			 bool
+	CommitLimit 			 int
+	RepoPath    			 string
+	Keep        			 int
+	ImageStream 			 string
+	Namespace   			 string
+	Tag         			 bool
+	Sorted      			 string
+	Orphan      			 bool
+	OlderThan    			 string
+	OrphanIncludeRegex       string
 }
 
 // NewImageStreamCleanupCommand creates a cobra command to clean up an imagestream based on commits
@@ -33,12 +39,15 @@ func NewImageStreamCleanupCommand() *cobra.Command {
 		Run:     o.cleanupImageStreamTags,
 	}
 	cmd.Flags().BoolVarP(&o.Force, "force", "f", false, "delete image stream tags")
-	cmd.Flags().IntVarP(&o.CommitLimit, "git-commit-limit", "l", 100, "only look at the first <n> commits to compare with tags or use -1 for all commits")
+	cmd.Flags().IntVarP(&o.CommitLimit, "git-commit-limit", "l", 0, "only look at the first <n> commits to compare with tags or use 0 for all commits")
 	cmd.Flags().StringVarP(&o.RepoPath, "git-repo-path", "p", ".", "absolute path to Git repository (for current dir use: $PWD)")
 	cmd.Flags().StringVarP(&o.Namespace, "namespace", "n", "", "Kubernetes namespace")
 	cmd.Flags().IntVarP(&o.Keep, "keep", "k", 10, "keep most current <n> images")
 	cmd.Flags().BoolVarP(&o.Tag, "tag", "t", false, "use tags instead of commit hashes")
 	cmd.Flags().StringVar(&o.Sorted, "sort", string(git.SortOptionVersion), "sort tags by criteria. Allowed values: [version, alphabetical]")
+	cmd.Flags().BoolVarP(&o.Orphan, "orphan", "o", false, "delete images that do not match any git commit")
+	cmd.Flags().StringVar(&o.OlderThan, "older-than", "", "delete images that are older than the duration. Ex.: [1y2mo3w4d5h6m7s]")
+	cmd.Flags().StringVarP(&o.OrphanIncludeRegex, "orphan-deletion-pattern", "i", "^[a-z0-9]{40}$", "delete images that match the regex, works only with the -o flag, defaults to matching Git SHA commits")
 	return cmd
 }
 
@@ -47,9 +56,11 @@ func (o *ImageStreamCleanupOptions) cleanupImageStreamTags(cmd *cobra.Command, a
 		o.ImageStream = args[0]
 	}
 
-	if o.Tag && !git.IsValidSortValue(o.Sorted) {
-		log.WithField("sort_criteria", o.Sorted).Fatal("Invalid sort criteria")
-	}
+	validateFlagCombinationInput(o)
+
+	orphanIncludeRegex := parseOrphanIncludeRegex(o.OrphanIncludeRegex)
+
+	cutOffDateTime := parseCutOffDateTime(o.OlderThan)
 
 	if len(o.Namespace) == 0 {
 		namespace, err := kubernetes.Namespace()
@@ -96,7 +107,7 @@ func (o *ImageStreamCleanupOptions) cleanupImageStreamTags(cmd *cobra.Command, a
 		}
 	}
 
-	imageStreamTags, err := openshift.GetImageStreamTags(o.Namespace, o.ImageStream)
+	imageStreamObjectTags, err := openshift.GetImageStreamTags(o.Namespace, o.ImageStream)
 	if err != nil {
 		log.WithError(err).
 			WithFields(log.Fields{
@@ -105,12 +116,20 @@ func (o *ImageStreamCleanupOptions) cleanupImageStreamTags(cmd *cobra.Command, a
 			Fatal("Could not retrieve image stream.")
 	}
 
+	imageStreamTags := cleanup.FilterImageTagsByTime(&imageStreamObjectTags, cutOffDateTime)
+
 	var matchOption cleanup.MatchOption
 	if o.Tag {
 		matchOption = cleanup.MatchOptionExact
 	}
 
-	matchingTags := cleanup.GetMatchingTags(&matchValues, &imageStreamTags, matchOption)
+	var matchingTags []string
+	if o.Orphan {
+		matchingTags = cleanup.GetOrphanImageTags(&matchValues, &imageStreamTags, matchOption)
+		matchingTags = cleanup.FilterByRegex(&imageStreamTags, orphanIncludeRegex)
+	} else {
+		matchingTags = cleanup.GetMatchingTags(&matchValues, &imageStreamTags, matchOption)
+	}
 
 	activeImageStreamTags, err := openshift.GetActiveImageStreamTags(o.Namespace, o.ImageStream, imageStreamTags)
 	if err != nil {
@@ -137,3 +156,44 @@ func (o *ImageStreamCleanupOptions) cleanupImageStreamTags(cmd *cobra.Command, a
 		log.Info("--force was not specified. Nothing has been deleted.")
 	}
 }
+
+func validateFlagCombinationInput(o *ImageStreamCleanupOptions) {
+
+	if o.Orphan == false && o.OrphanIncludeRegex != "^[a-z0-9]{40}$" {
+		log.WithFields(log.Fields{"Orphan": o.Orphan, "Regex": o.OrphanIncludeRegex}).
+			Fatal("Missing Orphan flag")
+	}
+
+	if o.Tag && !git.IsValidSortValue(o.Sorted) {
+		log.WithField("sort_criteria", o.Sorted).Fatal("Invalid sort criteria.")
+	}
+
+	if o.CommitLimit !=0 && o.Orphan == true {
+		log.WithFields(log.Fields{"CommitLimit": o.CommitLimit, "Orphan": o.Orphan}).
+			Fatal("Mutually exclusive flags")
+	}
+}
+
+func parseOrphanIncludeRegex(orphanIncludeRegex string) *regexp.Regexp {
+	regexp, err := regexp.Compile(orphanIncludeRegex)
+	if err != nil {
+		log.WithField("orphanIncludeRegex", orphanIncludeRegex).
+			Fatal("Invalid orphan include regex.")
+	}
+
+	return regexp
+}
+
+func parseCutOffDateTime(olderThan string) time.Time {
+	if len(olderThan) > 0 {
+		cutOffDateTime, err := tparse.ParseNow(time.RFC3339, "now-" + olderThan)
+		if err != nil {
+			log.WithError(err).
+				WithField("--older-than", olderThan).
+				Fatal("Could not parse --older-than flag.")
+		}
+		return cutOffDateTime;
+	}
+
+	return time.Now()
+}

go.mod

@@ -9,7 +9,8 @@ require (
 	github.com/heroku/docker-registry-client v0.0.0-20190909225348-afc9e1acc3d5
 	github.com/imdario/mergo v0.3.8 // indirect
 	github.com/json-iterator/go v1.1.8 // indirect
-	github.com/openshift/api v3.9.1-0.20190322043348-8741ff068a47+incompatible // indirect
+	github.com/karrick/tparse v2.4.2+incompatible
+	github.com/openshift/api v3.9.1-0.20190322043348-8741ff068a47+incompatible
 	github.com/openshift/client-go v0.0.0-20180830153425-431ec9a26e50
 	github.com/sirupsen/logrus v1.4.2
 	github.com/spf13/cobra v0.0.5

go.sum

@@ -101,6 +101,8 @@ github.com/jessevdk/go-flags v1.4.0 h1:4IU2WS7AumrZ/40jfhf4QVDMsQwqA7VEHozFRrGAR
 github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
 github.com/json-iterator/go v1.1.8 h1:QiWkFLKq0T7mpzwOTu6BzNDbfTE8OLrYhVKYMLF46Ok=
 github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
+github.com/karrick/tparse v2.4.2+incompatible h1:+cW306qKAzrASC5XieHkgN7/vPaGKIuK62Q7nI7DIRc=
+github.com/karrick/tparse v2.4.2+incompatible/go.mod h1:ASPA+vrIcN1uEW6BZg8vfWbzm69ODPSYZPU6qJyfdK0=
 github.com/kevinburke/ssh_config v0.0.0-20190725054713-01f96b0aa0cd h1:Coekwdh0v2wtGp9Gmz1Ze3eVRAWJMLokvN3QjdzCHLY=
 github.com/kevinburke/ssh_config v0.0.0-20190725054713-01f96b0aa0cd/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM=
 github.com/kisielk/gotool v0.0.0-20161130080628-0de1eaf82fa3/go.mod h1:jxZFDH7ILpTPQTk+E2s+z4CUas9lVNjIuKR4c5/zKgM=
@@ -146,6 +148,7 @@ github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGV
 github.com/onsi/gomega v1.4.2/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
 github.com/opencontainers/go-digest v1.0.0-rc1 h1:WzifXhOVOEOuFYOJAW6aQqW0TooG2iki3E3Ii+WN7gQ=
 github.com/opencontainers/go-digest v1.0.0-rc1/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s=
+github.com/openshift/api v3.9.0+incompatible h1:fJ/KsefYuZAjmrr3+5U9yZIZbTOpVkDDLDLFresAeYs=
 github.com/openshift/api v3.9.1-0.20190322043348-8741ff068a47+incompatible h1:p0ypM7AY7k2VY6ILDPbg3LajGA97hFUt2DGVEQz2Yd4=
 github.com/openshift/api v3.9.1-0.20190322043348-8741ff068a47+incompatible/go.mod h1:dh9o4Fs58gpFXGSYfnVxGR9PnV53I8TW84pQaJDdGiY=
 github.com/openshift/client-go v0.0.0-20180830153425-431ec9a26e50 h1:y59/+XbTbwzEdS2wveRQTZvjkar7sbVjTNnqFBufr74=

pkg/cleanup/cleanup.go

@@ -2,8 +2,11 @@ package cleanup
 
 import (
 	"strings"
+	"regexp"
+	"time"
 
 	log "github.com/sirupsen/logrus"
+	imagev1 "github.com/openshift/api/image/v1"
 )
 
 // MatchOption type defines how the tags should be matched
@@ -62,6 +65,45 @@ func GetInactiveTags(activeTags, tags *[]string) []string {
 	return inactiveTags
 }
 
+// GetOrphanImageTags returns the tags that do not have any git commit match
+func GetOrphanImageTags(gitValues, imageTags *[]string, matchOption MatchOption) []string {
+	orphans := []string{}
+
+	log.WithField("gitValues", gitValues).Debug("Git commits/tags")
+	log.WithField("imageTags", imageTags).Debug("Image stream tags")
+
+	for _, tag := range *imageTags {
+		found := false
+		for _, value := range *gitValues {
+			if match(tag, value, matchOption) {
+				found = true
+				break
+			}
+		}
+		if !found {
+			orphans = append(orphans, tag)
+		}
+	}
+
+	return orphans
+}
+
+// FilterByRegex returns the tags that match the regexp
+func FilterByRegex(imageTags *[]string, regexp *regexp.Regexp) []string {
+	var matchedTags []string
+
+	log.WithField("pattern:", regexp).Debug("Filtering image tags with regex...")
+
+	for _, tag := range *imageTags {
+		imageTagMatched := regexp.MatchString(tag)
+		log.WithField("imageTag:", tag).WithField("match:", imageTagMatched).Debug("Matching image tag")
+		if imageTagMatched {
+			matchedTags = append(matchedTags, tag)
+		}
+	}
+	return matchedTags
+}
+
 // LimitTags returns the tags which should not be kept by removing the first n tags
 func LimitTags(tags *[]string, keep int) []string {
 	if len(*tags) > keep {
@@ -73,6 +115,26 @@ func LimitTags(tags *[]string, keep int) []string {
 	return []string{}
 }
 
+// FilterImageTagsByTime returns the tags which are older than the specified time
+func FilterImageTagsByTime(imageStreamObjectTags *[]imagev1.NamedTagEventList, olderThan time.Time) []string {
+	var imageStreamTags []string
+
+	for _, imageStreamTag := range *imageStreamObjectTags {
+		lastUpdatedDate := imageStreamTag.Items[0].Created.Time
+		for _, tagEvent := range imageStreamTag.Items {
+			if lastUpdatedDate.Before(tagEvent.Created.Time) {
+				lastUpdatedDate = tagEvent.Created.Time
+			}
+		}
+
+		if lastUpdatedDate.Before(olderThan) {
+			imageStreamTags = append(imageStreamTags, imageStreamTag.Tag)
+		}
+	}
+
+	return imageStreamTags
+}
+
 func match(tag, value string, matchOption MatchOption) bool {
 	switch matchOption {
 	case MatchOptionDefault, MatchOptionPrefix: