Import pycross_wheel_library

This patch imports a few files from jvolkman/rules_pycross at
757033ff8afeb5f7090b1320759f6f03d9c4615c.

I would like to re-use this rule for the `pypi_install` repo rule that
I'm working on. This rule extracts a downloaded wheel and generates an
appropriate `PyInfo` provider for it.

All the non-BUILD files are taken as-is without modification. A
followup patch will make tweaks so that the code can be used from
within rules_python.

References: bazelbuild#1360

third_party/rules_pycross/pycross/private/providers.bzl

@@ -0,0 +1,17 @@
+"""Pycross providers."""
+
+PycrossWheelInfo = provider(
+    doc = "Information about a Python wheel.",
+    fields = {
+        "name_file": "File: A file containing the canonical name of the wheel.",
+        "wheel_file": "File: The wheel file itself.",
+    },
+)
+
+PycrossTargetEnvironmentInfo = provider(
+    doc = "A target environment description.",
+    fields = {
+        "python_compatible_with": "A list of constraints used to select this platform.",
+        "file": "The JSON file containing target environment information.",
+    },
+)

third_party/rules_pycross/pycross/private/tools/BUILD.bazel

@@ -0,0 +1,38 @@
+load("@rules_python//python:defs.bzl", "py_binary")
+
+package(default_visibility = ["//visibility:private"])
+
+py_library(
+    name = "namespace_pkgs",
+    srcs = [
+        "namespace_pkgs.py",
+    ],
+)
+
+py_test(
+    name = "namespace_pkgs_test",
+    size = "small",
+    srcs = [
+        "namespace_pkgs_test.py",
+    ],
+    tags = [
+        "unit",
+        # TODO(philsc): Make this work.
+        "manual",
+    ],
+    deps = [
+        ":namespace_pkgs",
+    ],
+)
+
+py_binary(
+    name = "wheel_installer",
+    srcs = ["wheel_installer.py"],
+    visibility = ["//visibility:public"],
+    deps = [
+        ":namespace_pkgs",
+        # TODO(philsc): Make this work with what's available in rules_python.
+        #"@rules_pycross_pypi_deps_absl_py//:pkg",
+        #"@rules_pycross_pypi_deps_installer//:pkg",
+    ],
+)

third_party/rules_pycross/pycross/private/tools/namespace_pkgs.py

@@ -0,0 +1,109 @@
+"""Utility functions to discover python package types"""
+import os
+import textwrap
+from pathlib import Path  # supported in >= 3.4
+from typing import List
+from typing import Optional
+from typing import Set
+
+
+def implicit_namespace_packages(
+    directory: str, ignored_dirnames: Optional[List[str]] = None
+) -> Set[Path]:
+    """Discovers namespace packages implemented using the 'native namespace packages' method.
+
+    AKA 'implicit namespace packages', which has been supported since Python 3.3.
+    See: https://packaging.python.org/guides/packaging-namespace-packages/#native-namespace-packages
+
+    Args:
+        directory: The root directory to recursively find packages in.
+        ignored_dirnames: A list of directories to exclude from the search
+
+    Returns:
+        The set of directories found under root to be packages using the native namespace method.
+    """
+    namespace_pkg_dirs: Set[Path] = set()
+    standard_pkg_dirs: Set[Path] = set()
+    directory_path = Path(directory)
+    ignored_dirname_paths: List[Path] = [Path(p) for p in ignored_dirnames or ()]
+    # Traverse bottom-up because a directory can be a namespace pkg because its child contains module files.
+    for dirpath, dirnames, filenames in map(
+        lambda t: (Path(t[0]), *t[1:]), os.walk(directory_path, topdown=False)
+    ):
+        if "__init__.py" in filenames:
+            standard_pkg_dirs.add(dirpath)
+            continue
+        elif ignored_dirname_paths:
+            is_ignored_dir = dirpath in ignored_dirname_paths
+            child_of_ignored_dir = any(
+                d in dirpath.parents for d in ignored_dirname_paths
+            )
+            if is_ignored_dir or child_of_ignored_dir:
+                continue
+
+        dir_includes_py_modules = _includes_python_modules(filenames)
+        parent_of_namespace_pkg = any(
+            Path(dirpath, d) in namespace_pkg_dirs for d in dirnames
+        )
+        parent_of_standard_pkg = any(
+            Path(dirpath, d) in standard_pkg_dirs for d in dirnames
+        )
+        parent_of_pkg = parent_of_namespace_pkg or parent_of_standard_pkg
+        if (
+            (dir_includes_py_modules or parent_of_pkg)
+            and
+            # The root of the directory should never be an implicit namespace
+            dirpath != directory_path
+        ):
+            namespace_pkg_dirs.add(dirpath)
+    return namespace_pkg_dirs
+
+
+def add_pkgutil_style_namespace_pkg_init(dir_path: Path) -> None:
+    """Adds 'pkgutil-style namespace packages' init file to the given directory
+
+    See: https://packaging.python.org/guides/packaging-namespace-packages/#pkgutil-style-namespace-packages
+
+    Args:
+        dir_path: The directory to create an __init__.py for.
+
+    Raises:
+        ValueError: If the directory already contains an __init__.py file
+    """
+    ns_pkg_init_filepath = os.path.join(dir_path, "__init__.py")
+
+    if os.path.isfile(ns_pkg_init_filepath):
+        raise ValueError("%s already contains an __init__.py file." % dir_path)
+
+    with open(ns_pkg_init_filepath, "w") as ns_pkg_init_f:
+        # See https://packaging.python.org/guides/packaging-namespace-packages/#pkgutil-style-namespace-packages
+        ns_pkg_init_f.write(
+            textwrap.dedent(
+                """\
+                # __path__ manipulation added by bazelbuild/rules_python to support namespace pkgs.
+                __path__ = __import__('pkgutil').extend_path(__path__, __name__)
+                """
+            )
+        )
+
+
+def _includes_python_modules(files: List[str]) -> bool:
+    """
+    In order to only transform directories that Python actually considers namespace pkgs
+    we need to detect if a directory includes Python modules.
+
+    Which files are loadable as modules is extension based, and the particular set of extensions
+    varies by platform.
+
+    See:
+    1. https://github.com/python/cpython/blob/7d9d25dbedfffce61fc76bc7ccbfa9ae901bf56f/Lib/importlib/machinery.py#L19
+    2. PEP 420 -- Implicit Namespace Packages, Specification - https://www.python.org/dev/peps/pep-0420/#specification
+    3. dynload_shlib.c and dynload_win.c in python/cpython.
+    """
+    module_suffixes = {
+        ".py",  # Source modules
+        ".pyc",  # Compiled bytecode modules
+        ".so",  # Unix extension modules
+        ".pyd",  # https://docs.python.org/3/faq/windows.html#is-a-pyd-file-the-same-as-a-dll
+    }
+    return any(Path(f).suffix in module_suffixes for f in files)

third_party/rules_pycross/pycross/private/tools/namespace_pkgs_test.py

@@ -0,0 +1,179 @@
+import os
+import pathlib
+import shutil
+import tempfile
+import unittest
+from typing import Optional
+from typing import Set
+
+from pycross.private.tools import namespace_pkgs
+
+
+class TempDir:
+    def __init__(self) -> None:
+        self.dir = tempfile.mkdtemp()
+
+    def root(self) -> str:
+        return self.dir
+
+    def add_dir(self, rel_path: str) -> None:
+        d = pathlib.Path(self.dir, rel_path)
+        d.mkdir(parents=True)
+
+    def add_file(self, rel_path: str, contents: Optional[str] = None) -> None:
+        f = pathlib.Path(self.dir, rel_path)
+        f.parent.mkdir(parents=True, exist_ok=True)
+        if contents:
+            with open(str(f), "w") as writeable_f:
+                writeable_f.write(contents)
+        else:
+            f.touch()
+
+    def remove(self) -> None:
+        shutil.rmtree(self.dir)
+
+
+class TestImplicitNamespacePackages(unittest.TestCase):
+    def assertPathsEqual(self, actual: Set[pathlib.Path], expected: Set[str]) -> None:
+        self.assertEqual(actual, {pathlib.Path(p) for p in expected})
+
+    def test_in_current_directory(self) -> None:
+        directory = TempDir()
+        directory.add_file("foo/bar/biz.py")
+        directory.add_file("foo/bee/boo.py")
+        directory.add_file("foo/buu/__init__.py")
+        directory.add_file("foo/buu/bii.py")
+        cwd = os.getcwd()
+        os.chdir(directory.root())
+        expected = {
+            "foo",
+            "foo/bar",
+            "foo/bee",
+        }
+        try:
+            actual = namespace_pkgs.implicit_namespace_packages(".")
+            self.assertPathsEqual(actual, expected)
+        finally:
+            os.chdir(cwd)
+            directory.remove()
+
+    def test_finds_correct_namespace_packages(self) -> None:
+        directory = TempDir()
+        directory.add_file("foo/bar/biz.py")
+        directory.add_file("foo/bee/boo.py")
+        directory.add_file("foo/buu/__init__.py")
+        directory.add_file("foo/buu/bii.py")
+
+        expected = {
+            directory.root() + "/foo",
+            directory.root() + "/foo/bar",
+            directory.root() + "/foo/bee",
+        }
+        actual = namespace_pkgs.implicit_namespace_packages(directory.root())
+        self.assertPathsEqual(actual, expected)
+
+    def test_ignores_empty_directories(self) -> None:
+        directory = TempDir()
+        directory.add_file("foo/bar/biz.py")
+        directory.add_dir("foo/cat")
+
+        expected = {
+            directory.root() + "/foo",
+            directory.root() + "/foo/bar",
+        }
+        actual = namespace_pkgs.implicit_namespace_packages(directory.root())
+        self.assertPathsEqual(actual, expected)
+
+    def test_empty_case(self) -> None:
+        directory = TempDir()
+        directory.add_file("foo/__init__.py")
+        directory.add_file("foo/bar/__init__.py")
+        directory.add_file("foo/bar/biz.py")
+
+        actual = namespace_pkgs.implicit_namespace_packages(directory.root())
+        self.assertEqual(actual, set())
+
+    def test_ignores_non_module_files_in_directories(self) -> None:
+        directory = TempDir()
+        directory.add_file("foo/__init__.pyi")
+        directory.add_file("foo/py.typed")
+
+        actual = namespace_pkgs.implicit_namespace_packages(directory.root())
+        self.assertEqual(actual, set())
+
+    def test_parent_child_relationship_of_namespace_pkgs(self):
+        directory = TempDir()
+        directory.add_file("foo/bar/biff/my_module.py")
+        directory.add_file("foo/bar/biff/another_module.py")
+
+        expected = {
+            directory.root() + "/foo",
+            directory.root() + "/foo/bar",
+            directory.root() + "/foo/bar/biff",
+        }
+        actual = namespace_pkgs.implicit_namespace_packages(directory.root())
+        self.assertPathsEqual(actual, expected)
+
+    def test_parent_child_relationship_of_namespace_and_standard_pkgs(self):
+        directory = TempDir()
+        directory.add_file("foo/bar/biff/__init__.py")
+        directory.add_file("foo/bar/biff/another_module.py")
+
+        expected = {
+            directory.root() + "/foo",
+            directory.root() + "/foo/bar",
+        }
+        actual = namespace_pkgs.implicit_namespace_packages(directory.root())
+        self.assertPathsEqual(actual, expected)
+
+    def test_parent_child_relationship_of_namespace_and_nested_standard_pkgs(self):
+        directory = TempDir()
+        directory.add_file("foo/bar/__init__.py")
+        directory.add_file("foo/bar/biff/another_module.py")
+        directory.add_file("foo/bar/biff/__init__.py")
+        directory.add_file("foo/bar/boof/big_module.py")
+        directory.add_file("foo/bar/boof/__init__.py")
+        directory.add_file("fim/in_a_ns_pkg.py")
+
+        expected = {
+            directory.root() + "/foo",
+            directory.root() + "/fim",
+        }
+        actual = namespace_pkgs.implicit_namespace_packages(directory.root())
+        self.assertPathsEqual(actual, expected)
+
+    def test_recognized_all_nonstandard_module_types(self):
+        directory = TempDir()
+        directory.add_file("ayy/my_module.pyc")
+        directory.add_file("bee/ccc/dee/eee.so")
+        directory.add_file("eff/jee/aych.pyd")
+
+        expected = {
+            directory.root() + "/ayy",
+            directory.root() + "/bee",
+            directory.root() + "/bee/ccc",
+            directory.root() + "/bee/ccc/dee",
+            directory.root() + "/eff",
+            directory.root() + "/eff/jee",
+        }
+        actual = namespace_pkgs.implicit_namespace_packages(directory.root())
+        self.assertPathsEqual(actual, expected)
+
+    def test_skips_ignored_directories(self):
+        directory = TempDir()
+        directory.add_file("foo/boo/my_module.py")
+        directory.add_file("foo/bar/another_module.py")
+
+        expected = {
+            directory.root() + "/foo",
+            directory.root() + "/foo/bar",
+        }
+        actual = namespace_pkgs.implicit_namespace_packages(
+            directory.root(),
+            ignored_dirnames=[directory.root() + "/foo/boo"],
+        )
+        self.assertPathsEqual(actual, expected)
+
+
+if __name__ == "__main__":
+    unittest.main()