amazonlinux support (#1728)

* Allow testing docker images

Signed-off-by: Bernat Gabor <[email protected]>

* amazonlinux support

Signed-off-by: Bernat Gabor <[email protected]>

.gitignore

@@ -26,3 +26,6 @@ venv*
 .python-version
 
 *wheel-store*
+
+Dockerfile
+.dockerignore

docs/changelog/1719.bugfix.rst

@@ -0,0 +1,3 @@
+Support Python 2 implementations that require the landmark files and ``site.py`` to be in platform standard library
+instead of the standard library path of the virtual environment (notably some RHEL ones, such as the Docker
+image ``amazonlinux:1``) - by :user:`gaborbernat`.

docs/changelog/1728.bugfix.rst

@@ -0,0 +1 @@
+Automatically create the application data folder if it does not exists - by :user:`gaborbernat`.

src/virtualenv/create/describe.py

@@ -20,6 +20,7 @@ def __init__(self, dest, interpreter):
         self.interpreter = interpreter
         self.dest = dest
         self._stdlib = None
+        self._stdlib_platform = None
         self._system_stdlib = None
         self._conf_vars = None
 
@@ -49,6 +50,12 @@ def stdlib(self):
             self._stdlib = Path(self.interpreter.sysconfig_path("stdlib", config_var=self._config_vars))
         return self._stdlib
 
+    @property
+    def stdlib_platform(self):
+        if self._stdlib_platform is None:
+            self._stdlib_platform = Path(self.interpreter.sysconfig_path("platstdlib", config_var=self._config_vars))
+        return self._stdlib_platform
+
     @property
     def _config_vars(self):
         if self._conf_vars is None:

src/virtualenv/create/via_global_ref/builtin/cpython/cpython2.py

@@ -66,8 +66,8 @@ def sources(cls, interpreter):
         for src in super(CPython2Posix, cls).sources(interpreter):
             yield src
         # landmark for exec_prefix
-        name = "lib-dynload"
-        yield PathRefToDest(interpreter.stdlib_path(name), dest=cls.to_stdlib)
+        exec_marker_file, to_path, _ = cls.from_stdlib(cls.mappings(interpreter), "lib-dynload")
+        yield PathRefToDest(exec_marker_file, dest=to_path)
 
 
 class CPython2Windows(CPython2, CPythonWindows):

src/virtualenv/create/via_global_ref/builtin/cpython/mac_os.py

@@ -77,8 +77,9 @@ def current_mach_o_image_path(self):
     def sources(cls, interpreter):
         for src in super(CPython2macOsFramework, cls).sources(interpreter):
             yield src
-        name = "lib-dynload"  # landmark for exec_prefix
-        yield PathRefToDest(interpreter.stdlib_path(name), dest=cls.to_stdlib)
+        # landmark for exec_prefix
+        exec_marker_file, to_path, _ = cls.from_stdlib(cls.mappings(interpreter), "lib-dynload")
+        yield PathRefToDest(exec_marker_file, dest=to_path)
 
     @property
     def reload_code(self):

src/virtualenv/create/via_global_ref/builtin/python2/python2.py

@@ -24,8 +24,15 @@ def create(self):
         """Perform operations needed to make the created environment work on Python 2"""
         super(Python2, self).create()
         # install a patched site-package, the default Python 2 site.py is not smart enough to understand pyvenv.cfg,
-        # so we inject a small shim that can do this
-        site_py = self.stdlib / "site.py"
+        # so we inject a small shim that can do this, the location of this depends where it's on host
+        sys_std_plat = Path(self.interpreter.system_stdlib_platform)
+        site_py_in = (
+            self.stdlib_platform
+            if ((sys_std_plat / "site.py").exists() or (sys_std_plat / "site.pyc").exists())
+            else self.stdlib
+        )
+        site_py = site_py_in / "site.py"
+
         custom_site = get_custom_site()
         if IS_ZIPAPP:
             custom_site_text = read_from_zipapp(custom_site)
@@ -55,33 +62,45 @@ def skip_rewrite(self):
     def sources(cls, interpreter):
         for src in super(Python2, cls).sources(interpreter):
             yield src
-        # install files needed to run site.py
+        # install files needed to run site.py, either from stdlib or stdlib_platform, at least pyc, but both if exists
+        # if neither exists return the module file to trigger failure
+        mappings, needs_py_module = (
+            cls.mappings(interpreter),
+            cls.needs_stdlib_py_module(),
+        )
         for req in cls.modules():
-
-            # the compiled path is optional, but refer to it if exists
-            module_compiled_path = interpreter.stdlib_path("{}.pyc".format(req))
-            has_compile = module_compiled_path.exists()
-            if has_compile:
-                yield PathRefToDest(module_compiled_path, dest=cls.to_stdlib)
-
-            # stdlib module src may be missing if the interpreter allows it by falling back to the compiled
-            module_path = interpreter.stdlib_path("{}.py".format(req))
-            add_py_module = cls.needs_stdlib_py_module()
-            if add_py_module is False:
-                if module_path.exists():  # if present add it
-                    add_py_module = True
-                else:
-                    add_py_module = not has_compile  # otherwise only add it if the pyc is not present
-            if add_py_module:
-                yield PathRefToDest(module_path, dest=cls.to_stdlib)
+            module_file, to_module, module_exists = cls.from_stdlib(mappings, "{}.py".format(req))
+            compiled_file, to_compiled, compiled_exists = cls.from_stdlib(mappings, "{}.pyc".format(req))
+            if needs_py_module or module_exists or not compiled_exists:
+                yield PathRefToDest(module_file, dest=to_module)
+            if compiled_exists:
+                yield PathRefToDest(compiled_file, dest=to_compiled)
+
+    @staticmethod
+    def from_stdlib(mappings, name):
+        for from_std, to_std in mappings:
+            src = from_std / name
+            if src.exists():
+                return src, to_std, True
+        return mappings[0] / name, mappings[1], False
 
     @classmethod
-    def needs_stdlib_py_module(cls):
-        raise NotImplementedError
+    def mappings(cls, interpreter):
+        mappings = [(Path(interpreter.system_stdlib_platform), cls.to_stdlib_platform)]
+        if interpreter.system_stdlib_platform != interpreter.system_stdlib:
+            mappings.append((Path(interpreter.system_stdlib), cls.to_stdlib),)
+        return mappings
 
     def to_stdlib(self, src):
         return self.stdlib / src.name
 
+    def to_stdlib_platform(self, src):
+        return self.stdlib_platform / src.name
+
+    @classmethod
+    def needs_stdlib_py_module(cls):
+        raise NotImplementedError
+
     @classmethod
     def modules(cls):
         return []

src/virtualenv/discovery/py_info.py

@@ -91,7 +91,6 @@ def abs_path(v):
         self.system_stdlib_platform = self.sysconfig_path("platstdlib", confs)
         self.max_size = getattr(sys, "maxsize", getattr(sys, "maxint", None))
         self._creators = None
-        self._stdlib_paths = None
 
     def _fast_get_system_executable(self):
         """Try to get the system executable by just looking at properties"""
@@ -278,7 +277,7 @@ def _to_json(self):
         return json.dumps(self._to_dict(), indent=2)
 
     def _to_dict(self):
-        data = {var: (getattr(self, var) if var not in ("_creators", "_stdlib_paths") else None) for var in vars(self)}
+        data = {var: (getattr(self, var) if var not in ("_creators",) else None) for var in vars(self)}
         # noinspection PyProtectedMember
         data["version_info"] = data["version_info"]._asdict()  # namedtuple to dictionary
         return data
@@ -453,19 +452,6 @@ def _possible_base(self):
                 if upper != base:
                     yield upper
 
-    def stdlib_path(self, name):
-        if self._stdlib_paths is None:
-            from collections import OrderedDict
-            from virtualenv.util.path import Path
-
-            pat = OrderedDict((Path(i), None) for i in (self.system_stdlib, self.system_stdlib_platform))
-            self._stdlib_paths = list(pat.keys())
-        for path in self._stdlib_paths:
-            std_path = path / name
-            if std_path.exists():
-                return std_path
-        return self._stdlib_paths[0] / name
-
 
 if __name__ == "__main__":
     # dump a JSON representation of the current python

src/virtualenv/discovery/py_spec.py

@@ -69,8 +69,6 @@ def _int_or_none(val):
         return cls(string_spec, impl, major, minor, micro, arch, path)
 
     def generate_names(self):
-        if self.implementation is None:
-            return
         impls = OrderedDict()
         if self.implementation:
             # first consider implementation as it is

src/virtualenv/run/app_data.py

@@ -48,7 +48,7 @@ def _check_folder(folder):
         folder = os.path.abspath(folder)
         if not os.path.exists(folder):
             try:
-                os.mkdir(folder)
+                os.makedirs(folder)
                 logging.debug("created app data folder %s", folder)
             except OSError as exception:
                 logging.info("could not create app data folder %s due to %r", folder, exception)

tests/unit/create/test_creator.py

@@ -20,6 +20,7 @@
 
 from virtualenv.__main__ import run, run_with_catch
 from virtualenv.create.creator import DEBUG_SCRIPT, Creator, get_env_debug_info
+from virtualenv.create.via_global_ref.builtin.python2.python2 import Python2
 from virtualenv.discovery.builtin import get_interpreter
 from virtualenv.discovery.py_info import PythonInfo
 from virtualenv.info import IS_PYPY, IS_WIN, PY2, PY3, fs_is_case_sensitive
@@ -463,24 +464,18 @@ def _get_sys_path(flag=None):
 def test_pyc_only(tmp_path, mocker, session_app_data):
     """Ensure that creation can succeed if os.pyc exists (even if os.py has been deleted)"""
     interpreter = PythonInfo.from_exe(sys.executable, session_app_data)
-    host_pyc = interpreter.stdlib_path("os.pyc")
-    if not host_pyc.exists():
+    host_pyc, _, host_pyc_exists = Python2.from_stdlib(Python2.mappings(interpreter), "os.pyc")
+    if not host_pyc_exists:
         pytest.skip("missing system os.pyc at {}".format(host_pyc))
-    previous = interpreter.stdlib_path
+    previous = Python2.from_stdlib
 
-    def stdlib_path(name):
-        path = previous(name)
+    def from_stdlib(mappings, name):
+        path, to, exists = previous(mappings, name)
         if name.endswith(".py"):
+            exists = False
+        return path, to, exists
 
-            class _Path(type(path)):
-                @staticmethod
-                def exists():
-                    return False
-
-            return _Path(path)
-        return path
-
-    mocker.patch.object(interpreter, "stdlib_path", side_effect=stdlib_path)
+    mocker.patch.object(Python2, "from_stdlib", side_effect=from_stdlib)
 
     result = cli_run([ensure_text(str(tmp_path)), "--without-pip", "--activators", ""])