Merge pull request #5249 from dctrud/issue5095

Option to clean cache items older than n days

CHANGELOG.md

@@ -33,6 +33,9 @@ _The old changelog can be found in the `release-2.6` branch_
     Singularity command line.
   - A new `--env-file` flag allows container environment variables to be set from
     a specified file.
+  - A new `--days` flag for `cache clean` allows removal of items older than a
+    specified number of days. Replaces the `--name` flag which is not generally
+    useful as the cache entries are stored by hash, not a friendly name.
 
 ## Changed defaults / behaviours
   - Environment variables prefixed with `SINGULARITYENV_` always take
@@ -49,6 +52,9 @@ _The old changelog can be found in the `release-2.6` branch_
   - `--fusemount` with the `container:` default directive will foreground the FUSE
      process. Use `container-daemon:` for previous behavior.
 
+## Deprecated / removed commands
+  - Removed `--name` flag for `cache clean`; replaced with `--days`.
+
 ## Bug Fixes
   - Don't try to mount `$HOME` when it is `/` (e.g. `nobody` user).
   - Process `%appinstall` sections in order when building from a definition file.

cmd/internal/cli/cache_clean_linux.go

@@ -22,15 +22,15 @@ import (
 func init() {
 	addCmdInit(func(cmdManager *cmdline.CommandManager) {
 		cmdManager.RegisterFlagForCmd(&cacheCleanTypesFlag, cacheCleanCmd)
-		cmdManager.RegisterFlagForCmd(&cacheCleanNameFlag, cacheCleanCmd)
+		cmdManager.RegisterFlagForCmd(&cacheCleanDaysFlag, cacheCleanCmd)
 		cmdManager.RegisterFlagForCmd(&cacheCleanDryFlag, cacheCleanCmd)
 		cmdManager.RegisterFlagForCmd(&cacheCleanForceFlag, cacheCleanCmd)
 	})
 }
 
 var (
 	cacheCleanTypes []string
-	cacheCleanNames []string
+	cacheCleanDays  int
 	cacheCleanDry   bool
 	cacheCleanForce bool
 
@@ -44,14 +44,14 @@ var (
 		Usage:        "a list of cache types to clean (possible values: library, oci, shub, blob, net, oras, all)",
 	}
 
-	// -N|--name
-	cacheCleanNameFlag = cmdline.Flag{
-		ID:           "cacheCleanNameFlag",
-		Value:        &cacheCleanNames,
-		DefaultValue: []string{},
-		Name:         "name",
-		ShortHand:    "N",
-		Usage:        "specify a container cache to clean (will clear all cache with the same name)",
+	// -D|--days
+	cacheCleanDaysFlag = cmdline.Flag{
+		ID:           "cacheCleanDaysFlag",
+		Value:        &cacheCleanDays,
+		DefaultValue: 0,
+		Name:         "days",
+		ShortHand:    "D",
+		Usage:        "remove all cache entries older than specified number of days",
 	}
 
 	// -n|--dry-run
@@ -107,7 +107,7 @@ func cleanCache() error {
 
 	// create a handle to access the current image cache
 	imgCache := getCacheHandle(cache.Config{})
-	err := singularity.CleanSingularityCache(imgCache, cacheCleanDry, cacheCleanTypes, cacheCleanNames)
+	err := singularity.CleanSingularityCache(imgCache, cacheCleanDry, cacheCleanTypes, cacheCleanDays)
 	if err != nil {
 		return fmt.Errorf("could not clean cache: %v", err)
 	}

docs/content.go

@@ -179,13 +179,13 @@ Enterprise Performance Computing (EPC)`
 	CacheCleanLong  string = `
   This will clean your local cache (stored at $HOME/.singularity/cache if
   SINGULARITY_CACHEDIR is not set). By default the entire cache is cleaned, use
-  --name or --type flags to override this behavior. Note: if you use Singularity
+  --days and --type flags to override this behavior. Note: if you use Singularity
   as root, cache will be stored in '/root/.singularity/.cache', to clean that
-  cache, you will need to run 'cache clean --all' as root, or with 'sudo'.`
+  cache, you will need to run 'cache clean' as root, or with 'sudo'.`
 	CacheCleanExample string = `
   All group commands have their own help output:
 
-  $ singularity help cache clean --name cache_name.sif
+  $ singularity help cache clean --days 30
   $ singularity help cache clean --type=library,oci
   $ singularity cache clean --help`
 

e2e/cache/cache.go

@@ -63,6 +63,22 @@ func (c cacheTests) testNoninteractiveCacheCmds(t *testing.T) {
 			expectedEmptyCache: true,
 			exit:               0,
 		},
+		{
+			name:               "clean force days beyond age",
+			options:            []string{"clean", "--force", "--days", "30"},
+			expectedOutput:     "",
+			needImage:          true,
+			expectedEmptyCache: false,
+			exit:               0,
+		},
+		{
+			name:               "clean force days within age",
+			options:            []string{"clean", "--force", "--days", "0"},
+			expectedOutput:     "",
+			needImage:          true,
+			expectedEmptyCache: true,
+			exit:               0,
+		},
 		{
 			name:           "clean help",
 			options:        []string{"clean", "--help"},
@@ -166,35 +182,35 @@ func (c cacheTests) testInteractiveCacheCmds(t *testing.T) {
 			exit:               0,
 		},
 		{
-			name:               "clean name confirmed",
-			options:            []string{"clean", "--name", imgName},
+			name:               "clean type confirmed",
+			options:            []string{"clean", "--type", "library"},
 			expect:             "Do you want to continue? [N/y]",
 			send:               "y",
 			expectedEmptyCache: true,
 			exit:               0,
 		},
 		{
-			name:               "clean name not confirmed",
-			options:            []string{"clean", "--name", imgName},
+			name:               "clean type not confirmed",
+			options:            []string{"clean", "--type", "library"},
 			expect:             "Do you want to continue? [N/y]",
 			send:               "n",
 			expectedEmptyCache: false,
 			exit:               0,
 		},
 		{
-			name:               "clean type confirmed",
-			options:            []string{"clean", "--type", "library"},
+			name:               "clean days beyond age",
+			options:            []string{"clean", "--days", "30"},
 			expect:             "Do you want to continue? [N/y]",
 			send:               "y",
-			expectedEmptyCache: true,
+			expectedEmptyCache: false,
 			exit:               0,
 		},
 		{
-			name:               "clean type not confirmed",
-			options:            []string{"clean", "--type", "library"},
+			name:               "clean days within age",
+			options:            []string{"clean", "--days", "0"},
 			expect:             "Do you want to continue? [N/y]",
-			send:               "n",
-			expectedEmptyCache: false,
+			send:               "y",
+			expectedEmptyCache: true,
 			exit:               0,
 		},
 	}

internal/app/singularity/cache_clean_linux.go

@@ -19,86 +19,36 @@ var (
 
 // cleanCache cleans the given type of cache cacheType. It will return a
 // error if one occurs.
-func cleanCache(imgCache *cache.Handle, cacheType string, dryRun bool) error {
+func cleanCache(imgCache *cache.Handle, cacheType string, dryRun bool, days int) error {
 	if imgCache == nil {
 		return fmt.Errorf("invalid image cache handle")
 	}
-	return imgCache.CleanCache(cacheType, dryRun)
+	return imgCache.CleanCache(cacheType, dryRun, days)
 }
 
-/*
-func removeCacheEntry(name, cacheType, cacheDir string, op func(string) error) (bool, error) {
-	foundMatch := false
-	done := fmt.Errorf("done")
-	err := filepath.Walk(cacheDir, func(path string, info os.FileInfo, err error) error {
-		if err != nil {
-			sylog.Debugf("Error while walking directory %s for cache %s at %s while looking for entry %s: %+v",
-				cacheDir,
-				cacheType,
-				path,
-				name,
-				err)
-			return err
-		}
-		if !info.IsDir() && info.Name() == name {
-			sylog.Debugf("Removing entry %s from cache %s at %s", name, cacheType, path)
-			if err := op(path); err != nil {
-				return fmt.Errorf("unable to remove entry %s from cache %s at path %s: %v", name, cacheType, path, err)
-			}
-			foundMatch = true
-			return done
-		}
-		return nil
-	})
-	if err == done {
-		err = nil
-	}
-	return foundMatch, err
-}
-*/
-
 // CleanSingularityCache is the main function that drives all these
 // other functions. If force is true, remove the entries, otherwise only
 // provide a summary of what would have been done. If cacheCleanTypes
 // contains something, only clean that type. The special value "all" is
 // interpreted as "all types of entries". If cacheName contains
 // something, clean only cache entries matching that name.
-func CleanSingularityCache(imgCache *cache.Handle, dryRun bool, cacheCleanTypes []string, cacheName []string) error {
+func CleanSingularityCache(imgCache *cache.Handle, dryRun bool, cacheCleanTypes []string, days int) error {
 	if imgCache == nil {
 		return errInvalidCacheHandle
 	}
 
-	/*
-		if len(cacheName) > 0 {
+	// Default is all caches
+	cachesToClean := append(cache.OciCacheTypes, cache.FileCacheTypes...)
 
+	// If specified caches, and we don't have 'all' specified then clean the specified
+	// ones only.
+	if len(cacheCleanTypes) > 0 && !stringInSlice("all", cacheCleanTypes) {
+		cachesToClean = cacheCleanTypes
+	}
 
-			// a name was specified, only clean matching entries
-			for _, name := range cacheName {
-				matches := 0
-				for _, cacheType := range cacheTypes {
-					cacheDir, _ := cacheTypeToDir(imgCache, cacheType)
-					sylog.Debugf("Removing cache type %q with name %q from directory %q ...", cacheType, name, cacheDir)
-					foundMatch, err := removeCacheEntry(name, cacheType, cacheDir, op)
-					if err != nil {
-						return err
-					}
-					if foundMatch {
-						matches++
-					}
-				}
-
-				if matches == 0 {
-					sylog.Warningf("No cache found with given name: %s", name)
-				}
-			}
-			return nil
-		}
-
-	*/
-
-	for _, cacheType := range cache.FileCacheTypes {
+	for _, cacheType := range cachesToClean {
 		sylog.Debugf("Cleaning %s cache...", cacheType)
-		if err := cleanCache(imgCache, cacheType, dryRun); err != nil {
+		if err := cleanCache(imgCache, cacheType, dryRun, days); err != nil {
 			return err
 		}
 	}

internal/pkg/cache/cache.go

@@ -14,6 +14,7 @@ import (
 	"path"
 	"path/filepath"
 	"strconv"
+	"time"
 
 	"github.com/sylabs/singularity/internal/pkg/util/fs"
 	"github.com/sylabs/singularity/pkg/syfs"
@@ -158,7 +159,7 @@ func (h *Handle) GetEntry(cacheType string, hash string) (e *Entry, err error) {
 	return e, nil
 }
 
-func (h *Handle) CleanCache(cacheType string, dryRun bool) (err error) {
+func (h *Handle) CleanCache(cacheType string, dryRun bool, days int) (err error) {
 	dir := h.getCacheTypeDir(cacheType)
 
 	files, err := ioutil.ReadDir(dir)
@@ -169,9 +170,18 @@ func (h *Handle) CleanCache(cacheType string, dryRun bool) (err error) {
 
 	errCount := 0
 	for _, f := range files {
-		sylog.Infof("Removing cacheType cache entry: %s", f.Name())
+
+		if days >= 0 {
+			if time.Since(f.ModTime()) < time.Duration(days*24)*time.Hour {
+				sylog.Debugf("Skipping %s: less that %d days old", f.Name(), days)
+				continue
+			}
+		}
+
+		sylog.Infof("Removing %s cache entry: %s", cacheType, f.Name())
 		if !dryRun {
-			err := os.Remove(path.Join(dir, f.Name()))
+			// We RemoveAll in case the entry is a directory from Singularity <3.6
+			err := os.RemoveAll(path.Join(dir, f.Name()))
 			if err != nil {
 				sylog.Errorf("Could not remove cache entry '%s': %v", f.Name(), err)
 				errCount = errCount + 1
@@ -193,12 +203,13 @@ func (h *Handle) cleanAllCaches() {
 		return
 	}
 
-	for _, ct := range FileCacheTypes {
+	for _, ct := range append(FileCacheTypes, OciCacheTypes...) {
 		dir := h.getCacheTypeDir(ct)
 		if err := os.RemoveAll(dir); err != nil {
 			sylog.Verbosef("unable to clean %s cache, directory %s: %v", ct, dir, err)
 		}
 	}
+
 }
 
 // IsDisabled returns true if the cache is disabled