Skip deleting files that we just deleted (#2185)

* Skip deleting files that we just deleted We see this happening with Swift. Because the consistency of swift is eventual, swift sometimes didn't process the deletion of the meta file yet, and so it turns up in the bkt.Iter(). The second deletion then causes a 404 and compaction fails. Signed-off-by: Wim Fournier <[email protected]> * return, as this is a func. Add debug log and comment Signed-off-by: Wim Fournier <[email protected]> * fixing build: wrong parameter name Signed-off-by: Wim Fournier <[email protected]> * fix lint Signed-off-by: Wim Fournier <[email protected]> * Refactor deleteDir into deleteDirRec and add a parameter for a function that allows to keep certain files. Signed-off-by: Wim Fournier <[email protected]> * Fix lint Signed-off-by: Wim Fournier <[email protected]> * implementing suggested fixes Signed-off-by: Wim Fournier <[email protected]>
thanos-io · Mar 2, 2020 · 6ebc9f0 · 6ebc9f0
1 parent 92c4470
commit 6ebc9f0
Showing 1 changed file with 10 additions and 4 deletions.
diff --git a/pkg/block/block.go b/pkg/block/block.go
@@ -145,16 +145,22 @@ func Delete(ctx context.Context, logger log.Logger, bkt objstore.Bucket, id ulid
 		level.Debug(logger).Log("msg", "deleted file", "file", metaFile, "bucket", bkt.Name())
 	}
 
-	return deleteDir(ctx, logger, bkt, id.String())
+	// Delete the bucket, but skip the metaFile as we just deleted that. This is required for eventual object storages (list after write).
+	return deleteDirRec(ctx, logger, bkt, id.String(), func(name string) bool {
+		return name == metaFile
+	})
 }
 
-// deleteDir removes all objects prefixed with dir from the bucket.
+// deleteDirRec removes all objects prefixed with dir from the bucket. It skips objects that return true for the passed keep function.
 // NOTE: For objects removal use `block.Delete` strictly.
-func deleteDir(ctx context.Context, logger log.Logger, bkt objstore.Bucket, dir string) error {
+func deleteDirRec(ctx context.Context, logger log.Logger, bkt objstore.Bucket, dir string, keep func(name string) bool) error {
 	return bkt.Iter(ctx, dir, func(name string) error {
 		// If we hit a directory, call DeleteDir recursively.
 		if strings.HasSuffix(name, objstore.DirDelim) {
-			return deleteDir(ctx, logger, bkt, name)
+			return deleteDirRec(ctx, logger, bkt, name, keep)
+		}
+		if keep(name) {
+			return nil
 		}
 		if err := bkt.Delete(ctx, name); err != nil {
 			return err