From 219fd2f3106e21d352361df471f96e80e16651df Mon Sep 17 00:00:00 2001 From: Jairo Llopis Date: Tue, 3 Oct 2017 14:31:44 +0200 Subject: [PATCH] Addons autoinstaller and env-based filter (#84) - Allow to define enabled addons by environment variables. - Brand-new `addons-install` script that installs all available addons for the environment. --- Dockerfile | 26 ++--- README.md | 101 ++++++++++-------- bin/addons-install | 62 +++++++++++ build.d/400-clean | 23 ++-- conf.d/10-addons.conf | 2 +- entrypoint.d/40-addons-link | 49 ++++----- lib/odoobaselib/__init__.py | 66 ++++++++++-- tests/__init__.py | 58 +++++++++- .../scaffoldings/dotd/custom/src/addons.yaml | 15 +++ tests/scaffoldings/dotd/docker-compose.yaml | 2 + 10 files changed, 287 insertions(+), 117 deletions(-) create mode 100755 bin/addons-install diff --git a/Dockerfile b/Dockerfile index 09d56946..55a9ba01 100644 --- a/Dockerfile +++ b/Dockerfile @@ -53,40 +53,32 @@ ONBUILD USER odoo ARG PYTHONOPTIMIZE=2 ARG WKHTMLTOPDF_VERSION=0.12.4 ARG WKHTMLTOPDF_CHECKSUM='049b2cdec9a8254f0ef8ac273afaf54f7e25459a273e27189591edc7d7cf29db' -ENV OPENERP_SERVER=/opt/odoo/auto/odoo.conf \ - UNACCENT=true \ - # Git and git-aggregator - GIT_AUTHOR_NAME=docker-odoo \ - EMAIL=https://hub.docker.com/r/tecnativa/odoo \ - DEPTH_DEFAULT=1 \ +ENV DEPTH_DEFAULT=1 \ DEPTH_MERGE=100 \ - # Postgres - WAIT_DB=true \ - # PuDB debugger + EMAIL=https://hub.docker.com/r/tecnativa/odoo \ + GIT_AUTHOR_NAME=docker-odoo \ + LC_ALL=C.UTF-8 \ + OPENERP_SERVER=/opt/odoo/auto/odoo.conf \ + PATH="~/.local/bin:$PATH" \ PUDB_RDB_HOST=0.0.0.0 \ PUDB_RDB_PORT=6899 \ - # WDB debugger + UNACCENT=true \ + WAIT_DB=true \ WDB_NO_BROWSER_AUTO_OPEN=True \ WDB_SOCKET_SERVER=wdb \ WDB_WEB_PORT=1984 \ - WDB_WEB_SERVER=localhost \ - # Other - LC_ALL=C.UTF-8 \ - PATH="~/.local/bin:$PATH" + WDB_WEB_SERVER=localhost # Other requirements and recommendations to run Odoo # See https://github.com/$ODOO_SOURCE/blob/$ODOO_VERSION/debian/control RUN apt-get update \ && apt-get -y upgrade \ && apt-get install -y --no-install-recommends \ - # Odoo direct dependencies python ruby-compass \ - # Odoo indirect dependencies fontconfig libfreetype6 libxml2 libxslt1.1 libjpeg62-turbo zlib1g \ libfreetype6 liblcms2-2 libopenjpeg5 libtiff5 tk tcl libpq5 \ libldap-2.4-2 libsasl2-2 libx11-6 libxext6 libxrender1 \ locales-all zlibc \ - # This image's facilities bzip2 ca-certificates curl gettext-base git nano npm \ openssh-client telnet xz-utils \ && curl https://bootstrap.pypa.io/get-pip.py | python /dev/stdin --no-cache-dir \ diff --git a/README.md b/README.md index ceb3400a..26a8d1b1 100644 --- a/README.md +++ b/README.md @@ -196,58 +196,65 @@ web: - web_responsive ``` -You can bundle [several YAML documents][] if you want to logically group your -addons and some repos are repeated among groups, by separating each document -with `---`: +Advanced features: -```yaml -# Spanish Localization -l10n-spain: - - l10n_es -server-tools: - - date_range ---- -# SEO tools -website: - - website_blog_excertp_img -server-tools: # Here we repeat server-tools, but no problem because it's a - # different document - - html_image_url_extractor - - html_text -``` - -You can add all modules in a repo by using a `*`: - -```yaml -website: - - "*" -``` +- You can bundle [several YAML documents][] if you want to logically group your + addons and some repos are repeated among groups, by separating each document + with `---`. -Important notes: +- Addons under `private` and `odoo/addons` are linked automatically unless you + specify them. -- Do not add repos for the required [`odoo`][] and [`private`][] directories; - those are automatic. +- You can use `ONLY` to supply a dictionary of environment variables and a + list of possible values to enable that document in the matching environments. -- Only addons here are symlinked in [`/opt/odoo/auto/addons`][]. Addons from - the required directories above are added directly in the [`odoo.conf`][] - file. - -- This means that if you have an addon with the same name in any of [`odoo`][], - [`private`][] or [`/opt/odoo/auto/addons`][] directories, this will be the - importance order in which they will be loaded (from most to least important): +- If an addon is found in several places at the same time, it will get linked + according to this priority table: 1. Addons in [`private`][]. - 2. Custom addons listed in [`addons.yaml`][]. - 3. Core Odoo addons from [`./odoo/addons`][`odoo`]. + 2. Addons in other repositories (in case one is matched in several, it will + be random, BEWARE!). Better have no duplicated names if possible. + 3. Core Odoo addons from [`odoo/addons`][`odoo`]. - Although it is better to simply have no name conflicts if possible. +- If an addon is specified but not available at runtime, it will fail silently. -- Any other addon not listed here will not be usable in Odoo (and will be - removed by default, to keep the resulting image thin). +- You can use any wildcards supported by [Python's glob module][glob]. -- If you list 2 addons with the same name, you'll get a build error. +This example shows these advanced features: -- If you use the wildcard (`*`), it must be encapsulated in quotes. +```yaml +# Spanish Localization +l10n-spain: + - l10n_es # Overrides built-in l10n_es under odoo/addons +server-tools: + - "*date*" # All modules that contain "date" in their name + - module_auto_update # Makes `autoupdate` script actually autoupdate addons +web: + - "*" # All web addons +--- +# Different YAML document to separate SEO Tools +website: + - website_blog_excertp_img +server-tools: # Here we repeat server-tools, but no problem because it's a + # different document + - html_image_url_extractor + - html_text +--- +# Enable demo ribbon only for devel and test environments +ONLY: + PGDATABASE: # This environment variable must exist and be in the list + - devel + - test +server-tools: + - web_environment_ribbon +--- +# Enable special authentication methods only in production environment +ONLY: + PGDATABASE: + - prod +server-tools: + - auth_* +``` ##### `/opt/odoo/custom/dependencies/*.txt` @@ -310,6 +317,15 @@ now keep this in mind: ## Bundled tools +### `addons-install` + +A handy CLI tool to automate addon management based on the current environment. +It allows you to install, update, test and/or list private, extra and/or core +addons available to current container, based on current [`addons.yaml`][] +configuration. + +Call `addons-install --help` for usage instructions. + ### [`nano`][] The CLI text editor we all know, just in case you need to inspect some bug in @@ -915,6 +931,7 @@ scaffolding versions is preserved. [builds]: https://hub.docker.com/r/tecnativa/odoo-base/builds/ [docker-socket-proxy]: https://hub.docker.com/r/tecnativa/docker-socket-proxy/ [Fish]: http://fishshell.com/ +[glob]: https://docs.python.org/3/library/glob.html [Let's Encrypt]: https://letsencrypt.org/ [OCA]: https://odoo-community.org/ [OCB]: https://github.com/OCA/OCB diff --git a/bin/addons-install b/bin/addons-install new file mode 100755 index 00000000..58ba7d15 --- /dev/null +++ b/bin/addons-install @@ -0,0 +1,62 @@ +#!/usr/bin/env python +from __future__ import print_function +import sys +from argparse import ArgumentParser +from subprocess import check_call +from odoobaselib import addons_config, CORE, PRIVATE, logging + +# Define CLI options +parser = ArgumentParser(description="Install addons in current environment") +parser.add_argument( + "-c", "--core", action="store_true", + help="Install all Odoo core addons") +parser.add_argument( + "-e", "--extra", action="store_true", + help="Install all extra addons") +parser.add_argument( + "-l", "--list", action="store_true", + help="Only list addons instead of installing them") +parser.add_argument( + "-p", "--private", action="store_true", + help="Install all private addons") +parser.add_argument( + "-s", "--separator", type=str, default=",", + help="String that separates addons when using --list") +parser.add_argument( + "-t", "--test", action="store_true", + help="Run unit tests for these addons, usually combined with --update") +parser.add_argument( + "-u", "--update", action="store_true", + help="Update addons instead of installing them") + +# Check no CLI conflicts +args = parser.parse_args() +if not (args.private or args.core or args.extra): + parser.error("You have to choose an option at least") +if args.list and args.update: + parser.error("Cannot --list and --update together") +if args.separator != "," and not args.list: + parser.error("Cannot use --separator without --list") + +# Generate the matching addons set +addons = set() +for addon, repo in addons_config(): + core_ok = args.core and repo == CORE + extra_ok = args.extra and repo not in {CORE, PRIVATE} + private_ok = args.private and repo == PRIVATE + if private_ok or core_ok or extra_ok: + addons.add(addon) + +# Do the required action +if not addons: + sys.exit("No addons found") +addons = args.separator.join(sorted(addons)) +if args.list: + print(addons) +else: + command = ["odoo", "--stop-after-init"] + if args.test: + command += ["--test-enable", "--workers", "0"] + command += ["--update" if args.update else "--init", addons] + logging.info("Executing %s", " ".join(command)) + check_call(command) diff --git a/build.d/400-clean b/build.d/400-clean index 555a9339..83fd2c3a 100755 --- a/build.d/400-clean +++ b/build.d/400-clean @@ -5,33 +5,30 @@ import os import shutil import sys -import odoobaselib +from odoobaselib import addons_config, CORE, CLEAN, PRIVATE, SRC_DIR -if not odoobaselib.CLEAN: +if not CLEAN: logging.warning("Not cleaning garbage") sys.exit() -config = odoobaselib.addons_config() -for directory in os.listdir(odoobaselib.SRC_DIR): +addons = set(addons_config()) +repos = {addon[1] for addon in addons} | {CORE, PRIVATE} +for directory in os.listdir(SRC_DIR): # Special directories must be preserved - if directory in {"private", "odoo"}: - continue + if directory == "odoo": + directory = CORE # Skip regular files - full = os.path.join(odoobaselib.SRC_DIR, directory) + full = os.path.join(SRC_DIR, directory) if not os.path.isdir(full): continue # Remove directories not listed in addons.yaml - if directory not in config: + if directory not in repos: logging.info("Removing directory %s", full) shutil.rmtree(full) continue - # Skip wildcards - if '*' in config[directory]: - continue - # Traverse addons for subdirectory in os.listdir(full): subfull = os.path.join(full, subdirectory) @@ -39,6 +36,6 @@ for directory in os.listdir(odoobaselib.SRC_DIR): if not os.path.isdir(subfull): continue # Remove addon if not used - if subdirectory not in config[directory]: + if (subdirectory, directory) not in addons: logging.info("Removing subdirectory %s", subfull) shutil.rmtree(subfull) diff --git a/conf.d/10-addons.conf b/conf.d/10-addons.conf index 8a60c790..56393e80 100644 --- a/conf.d/10-addons.conf +++ b/conf.d/10-addons.conf @@ -1,3 +1,3 @@ [options] ; Addons in priority order: private, then other repos, then base Odoo -addons_path = /opt/odoo/custom/src/private,/opt/odoo/auto/addons,/opt/odoo/custom/src/odoo/addons +addons_path = /opt/odoo/auto/addons diff --git a/entrypoint.d/40-addons-link b/entrypoint.d/40-addons-link index c477b425..a7a53a89 100755 --- a/entrypoint.d/40-addons-link +++ b/entrypoint.d/40-addons-link @@ -2,39 +2,26 @@ # -*- coding: utf-8 -*- import logging import os +from glob import iglob -import odoobaselib +from odoobaselib import ( + addons_config, + ADDONS_DIR, + ADDONS_YAML, + SRC_DIR, +) logging.info( "Linking all addons from %s in %s", - odoobaselib.ADDONS_YAML, - odoobaselib.ADDONS_DIR) + ADDONS_YAML, + ADDONS_DIR) -for repo, addons in odoobaselib.addons_config().items(): - while addons: - addon = addons.pop() - if addon == '*': - addon_dir = os.path.join(odoobaselib.SRC_DIR, repo) - try: - modules = next(os.walk(addon_dir))[1] - except StopIteration: - continue - for module in modules: - if not module.startswith('.') and module != 'setup': - addons.append(module) - continue - src = os.path.relpath( - os.path.join(odoobaselib.SRC_DIR, repo, addon), - odoobaselib.ADDONS_DIR) - dst = os.path.join(odoobaselib.ADDONS_DIR, addon) - try: - ok = os.readlink(dst) == src - except OSError: - ok = False - if not ok: - try: - os.remove(dst) - except OSError: - pass - os.symlink(src, dst) - logging.debug("Linked %s in %s", src, dst) +# Remove all links in addons dir +for link in iglob(os.path.join(ADDONS_DIR, "*")): + os.remove(link) +# Add new links +for addon, repo in addons_config(): + src = os.path.relpath(os.path.join(SRC_DIR, repo, addon), ADDONS_DIR) + dst = os.path.join(ADDONS_DIR, addon) + os.symlink(src, dst) + logging.debug("Linked %s in %s", src, dst) diff --git a/lib/odoobaselib/__init__.py b/lib/odoobaselib/__init__.py index da85a0c0..c2504986 100644 --- a/lib/odoobaselib/__init__.py +++ b/lib/odoobaselib/__init__.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- import logging import os +from glob import iglob import yaml @@ -8,17 +9,20 @@ CUSTOM_DIR = "/opt/odoo/custom" SRC_DIR = os.path.join(CUSTOM_DIR, 'src') ADDONS_YAML = os.path.join(SRC_DIR, 'addons') +if os.path.isfile('%s.yaml' % ADDONS_YAML): + ADDONS_YAML = '%s.yaml' % ADDONS_YAML +else: + ADDONS_YAML = '%s.yml' % ADDONS_YAML ADDONS_DIR = "/opt/odoo/auto/addons" CLEAN = os.environ.get("CLEAN") == "true" LOG_LEVELS = ("DEBUG", "INFO", "WARNING", "ERROR") FILE_APT_BUILD = os.path.join( CUSTOM_DIR, 'dependencies', 'apt_build.txt', ) - -if os.path.isfile('%s.yaml' % ADDONS_YAML): - ADDONS_YAML = '%s.yaml' % ADDONS_YAML -else: - ADDONS_YAML = '%s.yml' % ADDONS_YAML +PRIVATE = "private" +CORE = "odoo/addons" +PRIVATE_DIR = os.path.join(SRC_DIR, PRIVATE) +CORE_DIR = os.path.join(SRC_DIR, CORE) # Customize logging for build logging.root.name = "docker-odoo-base" @@ -35,14 +39,58 @@ def addons_config(): - """Load configurations from ``ADDONS_YAML`` into a dict.""" + """Yield addon name and path from ``ADDONS_YAML``.""" config = dict() + private_done = False + core_done = False try: with open(ADDONS_YAML) as addons_file: for doc in yaml.load_all(addons_file): + # Skip sections with ONLY and that don't match + if any(os.environ.get(key) not in values + for key, values in doc.get("ONLY", dict()).items()): + logging.debug("Skipping section with ONLY %s", doc["ONLY"]) + continue + # Flatten all sections in a single dict for repo, addons in doc.items(): - config.setdefault(repo, list()) - config[repo] += addons + if repo == PRIVATE: + private_done = True + elif repo == CORE: + core_done = True + logging.debug("Processing %s repo", repo) + for glob in addons: + logging.debug("Expanding glob %s", glob) + for addon in iglob(os.path.join(SRC_DIR, repo, glob)): + logging.debug("Registering addon %s", addon) + addon = os.path.basename(addon) + config.setdefault(addon, set()) + config[addon].add(repo) except IOError: logging.debug('Could not find addons configuration yml.') - return config + # By default, all private and core addons are enabled + if not private_done: + logging.debug("Auto-adding all private repo addons") + config.update({ + os.path.basename(addon): {PRIVATE} + for addon in iglob(os.path.join(SRC_DIR, PRIVATE, "*")) + }) + if not core_done: + logging.debug("Auto-adding all core repo addons") + config.update({ + os.path.basename(addon): {CORE} + for addon in iglob(os.path.join(SRC_DIR, CORE, "*")) + }) + logging.debug("Resulting configuration: %r", config) + for addon, repos in config.items(): + # Private addons are most important + if PRIVATE in repos: + yield addon, PRIVATE + # Odoo core addons are least important + elif repos == {CORE}: + yield addon, CORE + # Other addons fall in between + elif len(repos) != 1: + logging.error("Addon %s defined in several repos %s", addon, repos) + raise Exception + else: + yield addon, repos.pop() diff --git a/tests/__init__.py b/tests/__init__.py index c08250f2..1ac6e4d3 100755 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -9,7 +9,7 @@ from glob import iglob from itertools import product, starmap -from os import environ +from os import environ, getlogin from os.path import basename, dirname, join from pwd import getpwnam from subprocess import PIPE, Popen @@ -50,6 +50,10 @@ def matrix(odoo=ODOO_VERSIONS, pg=PG_VERSIONS, class ScaffoldingCase(unittest.TestCase): + def setUp(self): + super().setUp() + self.compose_run = ("docker-compose", "run", "--rm", "odoo") + def popen(self, *args, **kwargs): """Shortcut to open a subprocess and ensure it works.""" logging.info("Subtest execution: %s", self._subtest) @@ -83,8 +87,7 @@ def compose_test(self, workdir, sub_env, *commands): for command in commands: with self.subTest(command=command): self.popen( - ("docker-compose", "run", "--rm", "odoo") + - command, + self.compose_run + command, cwd=workdir, env=full_env, ) @@ -95,6 +98,41 @@ def compose_test(self, workdir, sub_env, *commands): env=full_env, ) + def test_addons_filtered(self): + """Test addons filtering with ``ONLY`` keyword in ``addons.yaml``.""" + project_dir = join(SCAFFOLDINGS_DIR, "dotd") + for sub_env in matrix(odoo={"10.0"}): + self.compose_test( + project_dir, + dict(sub_env, DBNAME="prod"), + ("test", "-e", "auto/addons/website"), + ("test", "-e", "auto/addons/dummy_addon"), + ("test", "-e", "auto/addons/private_addon"), + ("bash", "-c", 'test "$(addons-install -lp)" == private_addon'), + ("bash", "-c", 'test "$(addons-install -le)" == dummy_addon'), + ("bash", "-c", 'addons-install -lc | grep ,crm,'), + ) + self.compose_test( + project_dir, + dict(sub_env, DBNAME="limited_private"), + ("test", "-e", "auto/addons/website"), + ("test", "-e", "auto/addons/dummy_addon"), + ("test", "!", "-e", "auto/addons/private_addon"), + ("bash", "-c", 'test -z "$(addons-install -lp)"'), + ("bash", "-c", 'test "$(addons-install -le)" == dummy_addon'), + ("bash", "-c", 'addons-install -lc | grep ,crm,'), + ) + self.compose_test( + project_dir, + dict(sub_env, DBNAME="limited_core"), + ("test", "!", "-e", "auto/addons/website"), + ("test", "-e", "auto/addons/dummy_addon"), + ("test", "!", "-e", "auto/addons/private_addon"), + ("bash", "-c", 'test -z "$(addons-install -lp)"'), + ("bash", "-c", 'test "$(addons-install -le)" == dummy_addon'), + ("bash", "-c", 'test "$(addons-install -lc)" == crm,sale'), + ) + def test_smallest(self): """Tests for the smallest possible environment.""" commands = ( @@ -146,7 +184,7 @@ def test_dotd(self): ("test", "!", "-e", "custom/src/private/dummy_addon"), ("test", "-d", "custom/src/private/private_addon"), ("test", "-f", "custom/src/private/private_addon/__init__.py"), - ("test", "!", "-e", "auto/addons/private_addon"), + ("test", "-e", "auto/addons/private_addon"), # ``odoo`` command works ("odoo", "--version"), # Implicit ``odoo`` command also works @@ -179,6 +217,12 @@ def test_main_scaffolding(self): for sub_env in matrix(odoo={"10.0"}): # Setup the devel environment self.compose_test(tmpdirname, dict(sub_env, **setup_env), ()) + # Travis seems to have a different UID than 1000 + if environ.get("TRAVIS"): + self.popen( + ("sudo", "chown", "1000:1000", + join(tmpdirname, "odoo", "auto", "addons")), + ) # Test all 3 official environments for dcfile in ("devel", "test", "prod"): sub_env["COMPOSE_FILE"] = f"{dcfile}.yaml" @@ -187,6 +231,12 @@ def test_main_scaffolding(self): # ``odoo`` command works ("odoo", "--version"), ) + # Restore owner in Travis so directory can be removed + if environ.get("TRAVIS"): + self.popen( + ("sudo", "chown", "-R", "{0}:{0}".format(getlogin()), + join(tmpdirname, "odoo", "auto", "addons")), + ) if __name__ == "__main__": diff --git a/tests/scaffoldings/dotd/custom/src/addons.yaml b/tests/scaffoldings/dotd/custom/src/addons.yaml index b6ecb5c5..9d859568 100644 --- a/tests/scaffoldings/dotd/custom/src/addons.yaml +++ b/tests/scaffoldings/dotd/custom/src/addons.yaml @@ -1,2 +1,17 @@ dummy_repo: - dummy_addon + +--- +ONLY: + PGDATABASE: + - limited_core +odoo/addons: + - crm + - sale + +--- +ONLY: + PGDATABASE: + - limited_private + - limited_core +private: [] diff --git a/tests/scaffoldings/dotd/docker-compose.yaml b/tests/scaffoldings/dotd/docker-compose.yaml index f12b7393..8dbadb4b 100644 --- a/tests/scaffoldings/dotd/docker-compose.yaml +++ b/tests/scaffoldings/dotd/docker-compose.yaml @@ -6,11 +6,13 @@ services: dockerfile: ${ODOO_MINOR}.Dockerfile args: CONFIG_BUILD: 'false' + LOG_LEVEL: DEBUG tty: true environment: PGUSER: another_odoo PGPASSWORD: anotherodoopassword PGHOST: postgresql + PGDATABASE: ${DBNAME:-prod} depends_on: - postgresql volumes: