Fix removal of wanted files

Previous implementation of `400-clean` build script was expecting a very specific directory structure of `repo/addons`, or otherwise it could be removing things that you actually wanted.

Imagine for instance that 2 doodba projects share the same private addons, so you add them in `odoo/src/private` in the 1st one, and link them in the 2nd one with repos+addons combination, adding something like this to `addons.yaml`:

```yaml
other-doodba/odoo/src/private:
- custom_other_addon
```

Previous implementation of the clean step would be removing the `other-doodba` folder entierly, as it was not following the expected structure.

This new implementation is smarter:

1. Preserve all Odoo source code, outside of its `addons` folder (which is handled as a normal addons repo, to be able to restrict them if wanted).
2. For other folders, check if it can be found inside an activated addons path. If not, remove it.
3. Repeat, recursively, bottom-up.

It should be more flexible and less error-prone. A test that was testing the cleanup has been slightly modified to support this use case.

build.d/400-clean

@@ -4,43 +4,55 @@ import os
 import shutil
 import sys
 
-from doodbalib import addons_config, CORE, CLEAN, logger, PRIVATE, SRC_DIR
+from doodbalib import (
+    addons_config,
+    CLEAN,
+    logger,
+    ODOO_DIR,
+    PRIVATE_DIR,
+    SRC_DIR,
+)
 
 if not CLEAN:
     logger.warning("Not cleaning garbage")
     sys.exit()
 
-addons = set(addons_config(filtered=False))
-repos = {addon[1] for addon in addons} | {CORE, PRIVATE}
-for directory in os.listdir(SRC_DIR):
-    # Special directories must be preserved
-    if directory == "odoo":
-        directory = CORE
+# Get the enabled paths
+repos_addons = {}
+for addon, repo in addons_config(filtered=False):
+    repo_path = os.path.realpath(os.path.join(SRC_DIR, repo))
+    repos_addons.setdefault(repo_path, set())
+    repos_addons[repo_path].add(addon)
+logger.debug("Addon paths enabled: %s", repos_addons)
 
-    # Skip regular files
-    full = os.path.join(SRC_DIR, directory)
-    if not os.path.isdir(full):
+# Traverse src dir and remove anything not explicitly enabled
+for directory, subdirectories, subfiles in os.walk(SRC_DIR):
+    logger.debug("Checking for cleanup directory %s", directory)
+    # Skip main src directory
+    if directory == SRC_DIR:
         continue
-
-    # Remove directories not listed in addons.yaml
-    if directory not in repos:
-        logger.info("Removing directory %s", full)
-        try:
-          shutil.rmtree(full)
-        except OSError:
-          os.unlink(full)
+    # Always skip private/*
+    if directory == PRIVATE_DIR:
+        subdirectories[:] = []
         continue
-
-    # Traverse addons
-    for subdirectory in os.listdir(full):
-        subfull = os.path.join(full, subdirectory)
-        # Skip regular files
-        if not os.path.isdir(subfull):
+    # Inside the odoo dir, skip all but addons dir
+    if directory == ODOO_DIR:
+        subdirectories[:] = ["addons"]
+        continue
+    try:
+        # Get addons enalbed in current directory
+        enabled_addons = repos_addons[directory]
+    except KeyError:
+        # This isn't a repo; is there anything inside to preserve?
+        directory += os.path.sep
+        if any(repo.startswith(directory) for repo in repos_addons):
+            # Then, let's walk in; we'll remove later if needed
             continue
-        # Remove addon if not used
-        if (subdirectory, directory) not in addons:
-            logger.info("Removing subdirectory %s", subfull)
-            try:
-              shutil.rmtree(subfull)
-            except OSError:
-              os.unlink(subfull)
+    else:
+        # This is an addons repo; do not walk into the enabled ones
+        for addon in enabled_addons:
+            subdirectories.remove(addon)
+        continue
+    # Remove every other directory
+    logger.info("Removing directory %s", directory)
+    shutil.rmtree(directory)

lib/doodbalib/__init__.py

@@ -41,6 +41,7 @@
 CORE = "odoo/addons"
 PRIVATE_DIR = os.path.join(SRC_DIR, PRIVATE)
 CORE_DIR = os.path.join(SRC_DIR, CORE)
+ODOO_DIR = os.path.join(SRC_DIR, "odoo")
 ODOO_VERSION = os.environ["ODOO_VERSION"]
 MANIFESTS = ("__manifest__.py", "__openerp__.py")
 if ODOO_VERSION in {"8.0", "9.0"}:

tests/__init__.py

@@ -147,7 +147,7 @@ def test_addons_filtered(self):
                 # Addon from extra repo takes higher priority than core version
                 ("realpath", "auto/addons/product"),
                 ("bash", "-c", 'test "$(realpath auto/addons/product)" == '
-                               '/opt/odoo/custom/src/dummy_repo/product'),
+                 '/opt/odoo/custom/src/other-doodba/odoo/src/private/product'),
                 ("bash", "-c",
                  'test "$(addons list -e)" == dummy_addon,product'),
             )

tests/scaffoldings/dotd/custom/src/addons.yaml

@@ -1,4 +1,4 @@
-dummy_repo:
+other-doodba/odoo/src/private:
   - absent_addon
   - dummy_addon
   - product

tests/scaffoldings/dotd/custom/src/other-doodba/odoo/src/private/product/__manifest__.py

@@ -0,0 +1,3 @@
+{
+    "name": "other-doodba/product",
+}