Cleanup files.PublishedStorage after dropping a publish

deb/publish.go

@@ -17,6 +17,7 @@ import (
 
 	"github.com/aptly-dev/aptly/aptly"
 	"github.com/aptly-dev/aptly/database"
+	"github.com/aptly-dev/aptly/files"
 	"github.com/aptly-dev/aptly/pgp"
 	"github.com/aptly-dev/aptly/utils"
 )
@@ -1632,13 +1633,19 @@ func (collection *PublishedRepoCollection) Remove(publishedStorageProvider aptly
 	collection.list[len(collection.list)-1], collection.list[repoPosition], collection.list =
 		nil, collection.list[len(collection.list)-1], collection.list[:len(collection.list)-1]
 
-	if !skipCleanup && len(cleanComponents) > 0 {
-		err = collection.CleanupPrefixComponentFiles(publishedStorageProvider, repo, cleanComponents, collectionFactory, progress)
-		if err != nil {
-			if !force {
-				return fmt.Errorf("cleanup failed, use -force-drop to override: %s", err)
+	if !skipCleanup {
+		if len(cleanComponents) > 0 {
+			err = collection.CleanupPrefixComponentFiles(publishedStorageProvider, repo, cleanComponents, collectionFactory, progress)
+			if err != nil {
+				if !force {
+					return fmt.Errorf("cleanup failed, use -force-drop to override: %s", err)
+				}
 			}
 		}
+
+		if localStorage, ok := publishedStorageProvider.GetPublishedStorage(storage).(*files.PublishedStorage); ok {
+			localStorage.CleanupPublishTree(repo.Prefix)
+		}
 	}
 
 	batch := collection.db.CreateBatch()

files/public.go

@@ -12,6 +12,7 @@ import (
 
 	"github.com/aptly-dev/aptly/aptly"
 	"github.com/aptly-dev/aptly/utils"
+	"github.com/rs/zerolog/log"
 	"github.com/saracen/walker"
 )
 
@@ -119,6 +120,36 @@ func (storage *PublishedStorage) RemoveDirs(path string, progress aptly.Progress
 	return os.RemoveAll(filepath)
 }
 
+// Cleans up PublishedStorage tree towards the publish storage root up to first non-empty directory
+func (storage *PublishedStorage) CleanupPublishTree(path string) {
+	path = strings.TrimLeft(path, "./")
+	if path == "" {
+		return
+	}
+	/*
+		Say, `path` is "a/b/c/d", let's remove all empty dirs:
+		 storage.rootPath + "a/b/c/d"
+		 storage.rootPath + "a/b/c"
+		 storage.rootPath + "a/b"
+		 ... and so on, up to storage.rootPath, first non-empty directory or any other os.Remove error, whichever comes first
+	*/
+
+	segments := strings.Split(path, "/")
+
+	for segmentsCount := len(segments); segmentsCount > 0; segmentsCount-- {
+		fullPath := filepath.Join(storage.rootPath, strings.Join(segments[:segmentsCount], "/"))
+
+		if err := os.Remove(fullPath); err != nil {
+			// We stop crawling the tree on any os.Remove error.
+			// Among all of the return values there might be 'directory not empty',
+			// which means, we reached non-empty dir, which is okay.
+			// Any other errors are logged for informational reasons.
+			log.Printf("CleanupPublishTree: %s. '%s' not removed\n", err, fullPath)
+			break
+		}
+	}
+}
+
 // LinkFromPool links package file from pool to dist's pool location
 //
 // publishedPrefix is desired prefix for the location in the pool.

system/t06_publish/PublishDrop21Test_gold

@@ -0,0 +1,5 @@
+Removing ${HOME}/.aptly/public/ppa/smira/dists...
+Removing ${HOME}/.aptly/public/ppa/smira/pool...
+{"level":"debug","time":"now","message":"CleanupPublishTree: remove ${HOME}/.aptly/public/ppa: directory not empty. '${HOME}/.aptly/public/ppa' not removed\n"}
+
+Published repository has been removed successfully.

system/t06_publish/drop.py

@@ -1,3 +1,4 @@
+import re
 from lib import BaseTest
 
 
@@ -25,6 +26,7 @@ def check(self):
 class PublishDrop2Test(BaseTest):
     """
     publish drop: under prefix
+    if remaining tree is empty, it is now removed
     """
     requiresGPG2 = True
     fixtureDB = True
@@ -39,9 +41,38 @@ class PublishDrop2Test(BaseTest):
     def check(self):
         super(PublishDrop2Test, self).check()
 
-        self.check_not_exists('public/ppa/smira/dists/')
-        self.check_not_exists('public/ppa/smira/pool/')
-        self.check_exists('public/ppa/smira/')
+        self.check_not_exists('public/ppa/')
+        self.check_exists('public/')
+
+
+class PublishDrop21Test(BaseTest):
+    """
+    publish drop: under prefix
+    if some publishes have common part of the prefix, only empty parts are cleared
+    """
+    requiresGPG2 = True
+    fixtureDB = True
+    fixturePool = True
+    fixtureCmds = [
+        "aptly snapshot create snap1 from mirror gnuplot-maverick",
+        "aptly snapshot create snap2 from mirror gnuplot-maverick",
+        "aptly publish snapshot -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec snap1 ppa/smira",
+        "aptly publish snapshot -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec snap2 ppa/another",
+    ]
+    runCmd = "aptly publish drop maverick ppa/smira"
+    gold_processor = BaseTest.expand_environ
+
+    def outputMatchPrepare(_, s):
+        """
+        remove timestamp from log output
+        """
+        return re.sub(r'"time":".[^\"]+"', '"time":"now"', s, flags=re.MULTILINE)
+
+    def check(self):
+        super(PublishDrop21Test, self).check()
+
+        self.check_not_exists('public/ppa/smira/')
+        self.check_exists('public/ppa/another')
 
 
 class PublishDrop3Test(BaseTest):
@@ -132,6 +163,7 @@ class PublishDrop6Test(BaseTest):
 class PublishDrop7Test(BaseTest):
     """
     publish drop: under prefix with trailing & leading slashes
+    see comment in PublishDrop2Test
     """
     requiresGPG2 = True
     fixtureDB = True
@@ -148,7 +180,8 @@ def check(self):
 
         self.check_not_exists('public/ppa/smira/dists/')
         self.check_not_exists('public/ppa/smira/pool/')
-        self.check_exists('public/ppa/smira/')
+        self.check_not_exists('public/ppa/smira/')
+        self.check_exists('public/')
 
 
 class PublishDrop8Test(BaseTest):