snap: Add custom x-nodejs plugin to support ppc64el and s390x

snap/plugins/x-nodejs.yaml

@@ -0,0 +1,8 @@
+options:
+    source:
+        required: true
+    source-type:
+    source-tag:
+    source-branch:
+    nodejs-target:
+        required: true

snap/plugins/x_nodejs.py

@@ -0,0 +1,332 @@
+# -*- Mode:Python; indent-tabs-mode:nil; tab-width:4 -*-
+#
+# Modified by Anthony Fok on 2018-10-01 to add support for ppc64el and s390x
+#
+# Copyright (C) 2015-2017 Canonical Ltd
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 3 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+"""The nodejs plugin is useful for node/npm based parts.
+
+The plugin uses node to install dependencies from `package.json`. It
+also sets up binaries defined in `package.json` into the `PATH`.
+
+This plugin uses the common plugin keywords as well as those for "sources".
+For more information check the 'plugins' topic for the former and the
+'sources' topic for the latter.
+
+Additionally, this plugin uses the following plugin-specific keywords:
+
+    - node-packages:
+      (list)
+      A list of dependencies to fetch using npm.
+    - node-engine:
+      (string)
+      The version of nodejs you want the snap to run on.
+    - npm-run:
+      (list)
+      A list of targets to `npm run`.
+      These targets will be run in order, after `npm install`
+    - npm-flags:
+      (list)
+      A list of flags for npm.
+    - node-package-manager
+      (string; default: npm)
+      The language package manager to use to drive installation
+      of node packages. Can be either `npm` (default) or `yarn`.
+"""
+
+import collections
+import contextlib
+import json
+import logging
+import os
+import shutil
+import subprocess
+import sys
+
+import snapcraft
+from snapcraft import sources
+from snapcraft.file_utils import link_or_copy_tree
+from snapcraft.internal import errors
+
+logger = logging.getLogger(__name__)
+
+_NODEJS_BASE = "node-v{version}-linux-{arch}"
+_NODEJS_VERSION = "8.12.0"
+_NODEJS_TMPL = "https://nodejs.org/dist/v{version}/{base}.tar.gz"
+_NODEJS_ARCHES = {"i386": "x86", "amd64": "x64", "armhf": "armv7l", "arm64": "arm64", "ppc64el": "ppc64le", "s390x": "s390x"}
+_YARN_URL = "https://yarnpkg.com/latest.tar.gz"
+
+
+class NodePlugin(snapcraft.BasePlugin):
+    @classmethod
+    def schema(cls):
+        schema = super().schema()
+
+        schema["properties"]["node-packages"] = {
+            "type": "array",
+            "minitems": 1,
+            "uniqueItems": True,
+            "items": {"type": "string"},
+            "default": [],
+        }
+        schema["properties"]["node-engine"] = {
+            "type": "string",
+            "default": _NODEJS_VERSION,
+        }
+        schema["properties"]["node-package-manager"] = {
+            "type": "string",
+            "default": "npm",
+            "enum": ["npm", "yarn"],
+        }
+        schema["properties"]["npm-run"] = {
+            "type": "array",
+            "minitems": 1,
+            "uniqueItems": False,
+            "items": {"type": "string"},
+            "default": [],
+        }
+        schema["properties"]["npm-flags"] = {
+            "type": "array",
+            "minitems": 1,
+            "uniqueItems": False,
+            "items": {"type": "string"},
+            "default": [],
+        }
+
+        if "required" in schema:
+            del schema["required"]
+
+        return schema
+
+    @classmethod
+    def get_build_properties(cls):
+        # Inform Snapcraft of the properties associated with building. If these
+        # change in the YAML Snapcraft will consider the build step dirty.
+        return ["node-packages", "npm-run", "npm-flags"]
+
+    @classmethod
+    def get_pull_properties(cls):
+        # Inform Snapcraft of the properties associated with pulling. If these
+        # change in the YAML Snapcraft will consider the build step dirty.
+        return ["node-engine", "node-package-manager"]
+
+    @property
+    def _nodejs_tar(self):
+        if self._nodejs_tar_handle is None:
+            self._nodejs_tar_handle = sources.Tar(
+                self._nodejs_release_uri, self._npm_dir
+            )
+        return self._nodejs_tar_handle
+
+    @property
+    def _yarn_tar(self):
+        if self._yarn_tar_handle is None:
+            self._yarn_tar_handle = sources.Tar(_YARN_URL, self._npm_dir)
+        return self._yarn_tar_handle
+
+    def __init__(self, name, options, project):
+        super().__init__(name, options, project)
+        self._source_package_json = os.path.join(
+            os.path.abspath(self.options.source), "package.json"
+        )
+        self._npm_dir = os.path.join(self.partdir, "npm")
+        self._manifest = collections.OrderedDict()
+        self._nodejs_release_uri = get_nodejs_release(
+            self.options.node_engine, self.project.deb_arch
+        )
+        self._nodejs_tar_handle = None
+        self._yarn_tar_handle = None
+
+    def pull(self):
+        super().pull()
+        os.makedirs(self._npm_dir, exist_ok=True)
+        self._nodejs_tar.download()
+        if self.options.node_package_manager == "yarn":
+            self._yarn_tar.download()
+        # do the install in the pull phase to download all dependencies.
+        if self.options.node_package_manager == "npm":
+            self._npm_install(rootdir=self.sourcedir)
+        else:
+            self._yarn_install(rootdir=self.sourcedir)
+
+    def clean_pull(self):
+        super().clean_pull()
+
+        # Remove the npm directory (if any)
+        if os.path.exists(self._npm_dir):
+            shutil.rmtree(self._npm_dir)
+
+    def build(self):
+        super().build()
+        if self.options.node_package_manager == "npm":
+            installed_node_packages = self._npm_install(rootdir=self.builddir)
+            # Copy the content of the symlink to the build directory
+            # LP: #1702661
+            modules_dir = os.path.join(self.installdir, "lib", "node_modules")
+            _copy_symlinked_content(modules_dir)
+        else:
+            installed_node_packages = self._yarn_install(rootdir=self.builddir)
+            lock_file_path = os.path.join(self.sourcedir, "yarn.lock")
+            if os.path.isfile(lock_file_path):
+                with open(lock_file_path) as lock_file:
+                    self._manifest["yarn-lock-contents"] = lock_file.read()
+
+        self._manifest["node-packages"] = [
+            "{}={}".format(name, installed_node_packages[name])
+            for name in installed_node_packages
+        ]
+
+    def _npm_install(self, rootdir):
+        self._nodejs_tar.provision(
+            self.installdir, clean_target=False, keep_tarball=True
+        )
+        npm_cmd = ["npm"] + self.options.npm_flags
+        npm_install = npm_cmd + ["--cache-min=Infinity", "install"]
+        for pkg in self.options.node_packages:
+            self.run(npm_install + ["--global"] + [pkg], cwd=rootdir)
+        if os.path.exists(os.path.join(rootdir, "package.json")):
+            self.run(npm_install, cwd=rootdir)
+            self.run(npm_install + ["--global"], cwd=rootdir)
+        for target in self.options.npm_run:
+            self.run(npm_cmd + ["run", target], cwd=rootdir)
+        return self._get_installed_node_packages("npm", self.installdir)
+
+    def _yarn_install(self, rootdir):
+        self._nodejs_tar.provision(
+            self.installdir, clean_target=False, keep_tarball=True
+        )
+        self._yarn_tar.provision(self._npm_dir, clean_target=False, keep_tarball=True)
+        yarn_cmd = [os.path.join(self._npm_dir, "bin", "yarn")]
+        yarn_cmd.extend(self.options.npm_flags)
+        if "http_proxy" in os.environ:
+            yarn_cmd.extend(["--proxy", os.environ["http_proxy"]])
+        if "https_proxy" in os.environ:
+            yarn_cmd.extend(["--https-proxy", os.environ["https_proxy"]])
+        flags = []
+        if rootdir == self.builddir:
+            yarn_add = yarn_cmd + ["global", "add"]
+            flags.extend(
+                [
+                    "--offline",
+                    "--prod",
+                    "--global-folder",
+                    self.installdir,
+                    "--prefix",
+                    self.installdir,
+                ]
+            )
+        else:
+            yarn_add = yarn_cmd + ["add"]
+        for pkg in self.options.node_packages:
+            self.run(yarn_add + [pkg] + flags, cwd=rootdir)
+
+        # local packages need to be added as if they were remote, we
+        # remove the local package.json so `yarn add` doesn't pollute it.
+        if os.path.exists(self._source_package_json):
+            with contextlib.suppress(FileNotFoundError):
+                os.unlink(os.path.join(rootdir, "package.json"))
+            shutil.copy(
+                self._source_package_json, os.path.join(rootdir, "package.json")
+            )
+            self.run(yarn_add + ["file:{}".format(rootdir)] + flags, cwd=rootdir)
+
+        # npm run would require to bring back package.json
+        if self.options.npm_run and os.path.exists(self._source_package_json):
+            # The current package.json is the yarn prefilled one.
+            with contextlib.suppress(FileNotFoundError):
+                os.unlink(os.path.join(rootdir, "package.json"))
+            os.link(self._source_package_json, os.path.join(rootdir, "package.json"))
+        for target in self.options.npm_run:
+            self.run(
+                yarn_cmd + ["run", target],
+                cwd=rootdir,
+                env=self._build_environment(rootdir),
+            )
+        return self._get_installed_node_packages("npm", self.installdir)
+
+    def _get_installed_node_packages(self, package_manager, cwd):
+        try:
+            output = self.run_output(
+                [package_manager, "ls", "--global", "--json"], cwd=cwd
+            )
+        except subprocess.CalledProcessError as error:
+            # XXX When dependencies have missing dependencies, an error like
+            # this is printed to stderr:
+            # npm ERR! peer dep missing: glob@*, required by [email protected]
+            # retcode is not 0, which raises an exception.
+            output = error.output.decode(sys.getfilesystemencoding()).strip()
+        packages = collections.OrderedDict()
+        dependencies = json.loads(output, object_pairs_hook=collections.OrderedDict)[
+            "dependencies"
+        ]
+        while dependencies:
+            key, value = dependencies.popitem(last=False)
+            # XXX Just as above, dependencies without version are the ones
+            # missing.
+            if "version" in value:
+                packages[key] = value["version"]
+            if "dependencies" in value:
+                dependencies.update(value["dependencies"])
+        return packages
+
+    def get_manifest(self):
+        return self._manifest
+
+    def _build_environment(self, rootdir):
+        env = os.environ.copy()
+        if rootdir.endswith("src"):
+            hidden_path = os.path.join(rootdir, "node_modules", ".bin")
+            if env.get("PATH"):
+                new_path = "{}:{}".format(hidden_path, env.get("PATH"))
+            else:
+                new_path = hidden_path
+            env["PATH"] = new_path
+        return env
+
+
+def _get_nodejs_base(node_engine, machine):
+    if machine not in _NODEJS_ARCHES:
+        raise errors.SnapcraftEnvironmentError(
+            "architecture not supported ({})".format(machine)
+        )
+    return _NODEJS_BASE.format(version=node_engine, arch=_NODEJS_ARCHES[machine])
+
+
+def get_nodejs_release(node_engine, arch):
+    return _NODEJS_TMPL.format(
+        version=node_engine, base=_get_nodejs_base(node_engine, arch)
+    )
+
+
+def _copy_symlinked_content(modules_dir):
+    """Copy symlinked content.
+
+    When running newer versions of npm, symlinks to the local tree are
+    created from the part's installdir to the root of the builddir of the
+    part (this only affects some build configurations in some projects)
+    which is valid when running from the context of the part but invalid
+    as soon as the artifacts migrate across the steps,
+    i.e.; stage and prime.
+
+    If modules_dir does not exist we simply return.
+    """
+    if not os.path.exists(modules_dir):
+        return
+    modules = [os.path.join(modules_dir, d) for d in os.listdir(modules_dir)]
+    symlinks = [l for l in modules if os.path.islink(l)]
+    for link_path in symlinks:
+        link_target = os.path.realpath(link_path)
+        os.unlink(link_path)
+        link_or_copy_tree(link_target, link_path)

snap/snapcraft.yaml

@@ -61,7 +61,7 @@ parts:
       ls -l $SNAPCRAFT_PART_INSTALL/bin/hugo
 
   node:
-    plugin: nodejs
+    plugin: x-nodejs
     node-packages: [postcss-cli]
     filesets:
       node: