From b028c64887cd98418d28419fd819a5f7431ce063 Mon Sep 17 00:00:00 2001 From: Jairo Llopis Date: Wed, 10 Oct 2018 10:54:41 +0100 Subject: [PATCH] Fix removal of wanted files Previous implementation of `400-clean` build script was expecting a very specific directory structure of `repo/addons`, or otherwise it could be removing things that you actually wanted. Imagine for instance that 2 doodba projects share the same private addons, so you add them in `odoo/src/private` in the 1st one, and link them in the 2nd one with repos+addons combination, adding something like this to `addons.yaml`: ```yaml other-doodba/odoo/src/private: - custom_other_addon ``` Previous implementation of the clean step would be removing the `other-doodba` folder entierly, as it was not following the expected structure. This new implementation is smarter: 1. Preserve all Odoo source code, outside of its `addons` folder (which is handled as a normal addons repo, to be able to restrict them if wanted). 2. For other folders, check if it can be found inside an activated addons path. If not, remove it. 3. Repeat, recursively, bottom-up. It should be more flexible and less error-prone. A test that was testing the cleanup has been slightly modified to support this use case. --- build.d/400-clean | 74 +++++++++++-------- lib/doodbalib/__init__.py | 1 + tests/__init__.py | 2 +- .../scaffoldings/dotd/custom/src/addons.yaml | 2 +- .../src/dummy_repo/product/__manifest__.py | 3 - .../odoo/src/private}/dummy_addon/__init__.py | 0 .../src/private}/dummy_addon/__manifest__.py | 0 .../odoo/src/private}/product/__init__.py | 0 .../odoo/src/private/product/__manifest__.py | 3 + 9 files changed, 49 insertions(+), 36 deletions(-) delete mode 100644 tests/scaffoldings/dotd/custom/src/dummy_repo/product/__manifest__.py rename tests/scaffoldings/dotd/custom/src/{dummy_repo => other-doodba/odoo/src/private}/dummy_addon/__init__.py (100%) rename tests/scaffoldings/dotd/custom/src/{dummy_repo => other-doodba/odoo/src/private}/dummy_addon/__manifest__.py (100%) rename tests/scaffoldings/dotd/custom/src/{dummy_repo => other-doodba/odoo/src/private}/product/__init__.py (100%) create mode 100644 tests/scaffoldings/dotd/custom/src/other-doodba/odoo/src/private/product/__manifest__.py diff --git a/build.d/400-clean b/build.d/400-clean index d92bfc59..eef3ac28 100755 --- a/build.d/400-clean +++ b/build.d/400-clean @@ -4,43 +4,55 @@ import os import shutil import sys -from doodbalib import addons_config, CORE, CLEAN, logger, PRIVATE, SRC_DIR +from doodbalib import ( + addons_config, + CLEAN, + logger, + ODOO_DIR, + PRIVATE_DIR, + SRC_DIR, +) if not CLEAN: logger.warning("Not cleaning garbage") sys.exit() -addons = set(addons_config(filtered=False)) -repos = {addon[1] for addon in addons} | {CORE, PRIVATE} -for directory in os.listdir(SRC_DIR): - # Special directories must be preserved - if directory == "odoo": - directory = CORE +# Get the enabled paths +repos_addons = {} +for addon, repo in addons_config(filtered=False): + repo_path = os.path.realpath(os.path.join(SRC_DIR, repo)) + repos_addons.setdefault(repo_path, set()) + repos_addons[repo_path].add(addon) +logger.debug("Addon paths enabled: %s", repos_addons) - # Skip regular files - full = os.path.join(SRC_DIR, directory) - if not os.path.isdir(full): +# Traverse src dir and remove anything not explicitly enabled +for directory, subdirectories, subfiles in os.walk(SRC_DIR): + logger.debug("Checking for cleanup directory %s", directory) + # Skip main src directory + if directory == SRC_DIR: continue - - # Remove directories not listed in addons.yaml - if directory not in repos: - logger.info("Removing directory %s", full) - try: - shutil.rmtree(full) - except OSError: - os.unlink(full) + # Always skip private/* + if directory == PRIVATE_DIR: + subdirectories[:] = [] continue - - # Traverse addons - for subdirectory in os.listdir(full): - subfull = os.path.join(full, subdirectory) - # Skip regular files - if not os.path.isdir(subfull): + # Inside the odoo dir, skip all but addons dir + if directory == ODOO_DIR: + subdirectories[:] = ["addons"] + continue + try: + # Get addons enalbed in current directory + enabled_addons = repos_addons[directory] + except KeyError: + # This isn't a repo; is there anything inside to preserve? + directory += os.path.sep + if any(repo.startswith(directory) for repo in repos_addons): + # Then, let's walk in; we'll remove later if needed continue - # Remove addon if not used - if (subdirectory, directory) not in addons: - logger.info("Removing subdirectory %s", subfull) - try: - shutil.rmtree(subfull) - except OSError: - os.unlink(subfull) + else: + # This is an addons repo; do not walk into the enabled ones + for addon in enabled_addons: + subdirectories.remove(addon) + continue + # Remove every other directory + logger.info("Removing directory %s", directory) + shutil.rmtree(directory) diff --git a/lib/doodbalib/__init__.py b/lib/doodbalib/__init__.py index e1c8d0b1..3f3ffa7f 100644 --- a/lib/doodbalib/__init__.py +++ b/lib/doodbalib/__init__.py @@ -41,6 +41,7 @@ CORE = "odoo/addons" PRIVATE_DIR = os.path.join(SRC_DIR, PRIVATE) CORE_DIR = os.path.join(SRC_DIR, CORE) +ODOO_DIR = os.path.join(SRC_DIR, "odoo") ODOO_VERSION = os.environ["ODOO_VERSION"] MANIFESTS = ("__manifest__.py", "__openerp__.py") if ODOO_VERSION in {"8.0", "9.0"}: diff --git a/tests/__init__.py b/tests/__init__.py index f7296e3d..0af3924c 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -147,7 +147,7 @@ def test_addons_filtered(self): # Addon from extra repo takes higher priority than core version ("realpath", "auto/addons/product"), ("bash", "-c", 'test "$(realpath auto/addons/product)" == ' - '/opt/odoo/custom/src/dummy_repo/product'), + '/opt/odoo/custom/src/other-doodba/odoo/src/private/product'), ("bash", "-c", 'test "$(addons list -e)" == dummy_addon,product'), ) diff --git a/tests/scaffoldings/dotd/custom/src/addons.yaml b/tests/scaffoldings/dotd/custom/src/addons.yaml index f4689cb2..41128ef9 100644 --- a/tests/scaffoldings/dotd/custom/src/addons.yaml +++ b/tests/scaffoldings/dotd/custom/src/addons.yaml @@ -1,4 +1,4 @@ -dummy_repo: +other-doodba/odoo/src/private: - absent_addon - dummy_addon - product diff --git a/tests/scaffoldings/dotd/custom/src/dummy_repo/product/__manifest__.py b/tests/scaffoldings/dotd/custom/src/dummy_repo/product/__manifest__.py deleted file mode 100644 index a77ba4fd..00000000 --- a/tests/scaffoldings/dotd/custom/src/dummy_repo/product/__manifest__.py +++ /dev/null @@ -1,3 +0,0 @@ -{ - "name": "dummy_repo/product", -} diff --git a/tests/scaffoldings/dotd/custom/src/dummy_repo/dummy_addon/__init__.py b/tests/scaffoldings/dotd/custom/src/other-doodba/odoo/src/private/dummy_addon/__init__.py similarity index 100% rename from tests/scaffoldings/dotd/custom/src/dummy_repo/dummy_addon/__init__.py rename to tests/scaffoldings/dotd/custom/src/other-doodba/odoo/src/private/dummy_addon/__init__.py diff --git a/tests/scaffoldings/dotd/custom/src/dummy_repo/dummy_addon/__manifest__.py b/tests/scaffoldings/dotd/custom/src/other-doodba/odoo/src/private/dummy_addon/__manifest__.py similarity index 100% rename from tests/scaffoldings/dotd/custom/src/dummy_repo/dummy_addon/__manifest__.py rename to tests/scaffoldings/dotd/custom/src/other-doodba/odoo/src/private/dummy_addon/__manifest__.py diff --git a/tests/scaffoldings/dotd/custom/src/dummy_repo/product/__init__.py b/tests/scaffoldings/dotd/custom/src/other-doodba/odoo/src/private/product/__init__.py similarity index 100% rename from tests/scaffoldings/dotd/custom/src/dummy_repo/product/__init__.py rename to tests/scaffoldings/dotd/custom/src/other-doodba/odoo/src/private/product/__init__.py diff --git a/tests/scaffoldings/dotd/custom/src/other-doodba/odoo/src/private/product/__manifest__.py b/tests/scaffoldings/dotd/custom/src/other-doodba/odoo/src/private/product/__manifest__.py new file mode 100644 index 00000000..3a269390 --- /dev/null +++ b/tests/scaffoldings/dotd/custom/src/other-doodba/odoo/src/private/product/__manifest__.py @@ -0,0 +1,3 @@ +{ + "name": "other-doodba/product", +}