Python 3.8 support on Android (#2044)

* Did basic changes for Python3.8 support

* Added python 3.8.1 patch to git

* Added infrastructure for recipe-version-dependent patching

* Added --fix-cortex-a8 removal patch for py3.8.1

* Fully set up Python3 patches to work with both 3.7.1 and 3.8.1

* Fixed bugs in is_version_{lt,gt} patching helpers

* Replaced func call with pre-existing variable reference

* Added some blank lines to make pep8 happy

pythonforandroid/bootstraps/sdl2/build/src/main/java/org/kivy/android/PythonUtil.java

@@ -44,6 +44,7 @@ protected static ArrayList<String> getLibraries(File libsDir) {
         libsList.add("python3.5m");
         libsList.add("python3.6m");
         libsList.add("python3.7m");
+        libsList.add("python3.8m");
         libsList.add("main");
         return libsList;
     }
@@ -64,7 +65,7 @@ public static void loadLibraries(File filesDir, File libsDir) {
                 // load, and it has failed, give a more
                 // general error
                 Log.v(TAG, "Library loading error: " + e.getMessage());
-                if (lib.startsWith("python3.7") && !foundPython) {
+                if (lib.startsWith("python3.8") && !foundPython) {
                     throw new java.lang.RuntimeException("Could not load any libpythonXXX.so");
                 } else if (lib.startsWith("python")) {
                     continue;
@@ -81,7 +82,7 @@ public static void loadLibraries(File filesDir, File libsDir) {
         } catch(UnsatisfiedLinkError e) {
             Log.v(TAG, "Failed to load _io.so or unicodedata.so...but that's okay.");
         }
-        
+
         try {
             // System.loadLibrary("ctypes");
             System.load(filesDirPath + "/lib/python2.7/lib-dynload/_ctypes.so");
@@ -92,4 +93,3 @@ public static void loadLibraries(File filesDir, File libsDir) {
         Log.v(TAG, "Loaded everything!");
     }
 }
-

pythonforandroid/patching.py

@@ -1,4 +1,5 @@
 from os import uname
+from distutils.version import LooseVersion
 
 
 def check_all(*callables):
@@ -69,3 +70,20 @@ def is_ndk(ndk):
     def is_x(recipe, **kwargs):
         return recipe.ctx.ndk == ndk
     return is_x
+
+
+def is_version_gt(version):
+    def is_x(recipe, **kwargs):
+        return LooseVersion(recipe.version) > version
+
+
+def is_version_lt(version):
+    def is_x(recipe, **kwargs):
+        return LooseVersion(recipe.version) < version
+    return is_x
+
+
+def version_starts_with(version):
+    def is_x(recipe, **kwargs):
+        return recipe.version.startswith(version)
+    return is_x

pythonforandroid/python.py

@@ -438,10 +438,19 @@ def build_arch(self, arch):
                 if not exists('config.status'):
                     shprint(sh.Command(join(recipe_build_dir, 'configure')))
 
-            # Create the Setup file. This copying from Setup.dist
-            # seems to be the normal and expected procedure.
-            shprint(sh.cp, join('Modules', 'Setup.dist'),
-                    join(build_dir, 'Modules', 'Setup'))
+            # Create the Setup file. This copying from Setup.dist is
+            # the normal and expected procedure before Python 3.8, but
+            # after this the file with default options is already named "Setup"
+            setup_dist_location = join('Modules', 'Setup.dist')
+            if exists(setup_dist_location):
+                shprint(sh.cp, setup_dist_location,
+                        join(build_dir, 'Modules', 'Setup'))
+            else:
+                # Check the expected file does exist
+                setup_location = join('Modules', 'Setup')
+                if not exists(setup_location):
+                    raise BuildInterruptingException(
+                        "Could not find Setup.dist or Setup in Python build")
 
             shprint(sh.make, '-j', str(cpu_count()), '-C', build_dir)
 

pythonforandroid/recipes/hostpython3/__init__.py

@@ -9,7 +9,7 @@ class Hostpython3Recipe(HostPythonRecipe):
         Refactored into  the new class
         :class:`~pythonforandroid.python.HostPythonRecipe`
     '''
-    version = '3.7.1'
+    version = '3.8.1'
     name = 'hostpython3'
     conflicts = ['hostpython2']
 

pythonforandroid/recipes/python3/__init__.py

@@ -1,6 +1,7 @@
 import sh
 from pythonforandroid.python import GuestPythonRecipe
 from pythonforandroid.recipe import Recipe
+from pythonforandroid.patching import version_starts_with
 
 
 class Python3Recipe(GuestPythonRecipe):
@@ -18,15 +19,24 @@ class Python3Recipe(GuestPythonRecipe):
         :class:`~pythonforandroid.python.GuestPythonRecipe`
     '''
 
-    version = '3.7.1'
+    version = '3.8.1'
     url = 'https://www.python.org/ftp/python/{version}/Python-{version}.tgz'
     name = 'python3'
 
-    patches = ['patches/fix-ctypes-util-find-library.patch',
-               'patches/fix-zlib-version.patch']
+    patches = [
+        # Python 3.7.1
+        ('patches/py3.7.1_fix-ctypes-util-find-library.patch', version_starts_with("3.7")),
+        ('patches/py3.7.1_fix-zlib-version.patch', version_starts_with("3.7")),
+
+        # Python 3.8.1
+        ('patches/py3.8.1.patch', version_starts_with("3.8"))
+    ]
 
     if sh.which('lld') is not None:
-        patches = patches + ["patches/remove-fix-cortex-a8.patch"]
+        patches = patches + [
+            ("patches/py3.7.1_remove-fix-cortex-a8.patch", version_starts_with("3.7")),
+            ("patches/py3.8.1_remove-fix-cortex-a8.patch", version_starts_with("3.8"))
+        ]
 
     depends = ['hostpython3', 'sqlite3', 'openssl', 'libffi']
     conflicts = ['python2']

pythonforandroid/recipes/python3/patches/py3.8.1.patch

@@ -0,0 +1,42 @@
+diff --git a/Lib/ctypes/util.py b/Lib/ctypes/util.py
+index 97973bc..053c231 100644
+--- a/Lib/ctypes/util.py
++++ b/Lib/ctypes/util.py
+@@ -67,6 +67,13 @@ if os.name == "nt":
+                 return fname
+         return None
+
++# This patch overrides the find_library to look in the right places on
++# Android
++if True:
++    from android._ctypes_library_finder import find_library as _find_lib
++    def find_library(name):
++        return _find_lib(name)
++
+ elif os.name == "posix" and sys.platform == "darwin":
+     from ctypes.macholib.dyld import dyld_find as _dyld_find
+     def find_library(name):
+diff --git a/configure b/configure
+index 0914e24..dd00812 100755
+--- a/configure
++++ b/configure
+@@ -18673,4 +18673,3 @@ if test "$Py_OPT" = 'false' -a "$Py_DEBUG" != 'true'; then
+     echo "" >&6
+     echo "" >&6
+ fi
+-
+diff --git a/setup.py b/setup.py
+index 20d7f35..af15cc2 100644
+--- a/setup.py
++++ b/setup.py
+@@ -1501,7 +1501,9 @@ class PyBuildExt(build_ext):
+         if zlib_inc is not None:
+             zlib_h = zlib_inc[0] + '/zlib.h'
+             version = '"0.0.0"'
+-            version_req = '"1.1.3"'
++            # version_req = '"1.1.3"'
++            version_req = '"{}"'.format(
++                os.environ.get('ZLIB_VERSION', '1.1.3'))
+             if MACOS and is_macosx_sdk_path(zlib_h):
+                 zlib_h = os.path.join(macosx_sdk_root(), zlib_h[1:])
+             with open(zlib_h) as fp:

pythonforandroid/recipes/python3/patches/py3.8.1_fix_cortex_a8.patch

@@ -0,0 +1,18 @@
+diff --git a/configure b/configure
+index 0914e24..7517168 100755
+--- a/configure
++++ b/configure
+@@ -5642,7 +5642,7 @@ $as_echo_n "checking for the Android arm ABI... " >&6; }
+ $as_echo "$_arm_arch" >&6; }
+   if test "$_arm_arch" = 7; then
+     BASECFLAGS="${BASECFLAGS} -mfloat-abi=softfp -mfpu=vfpv3-d16"
+-    LDFLAGS="${LDFLAGS} -march=armv7-a -Wl,--fix-cortex-a8"
++    LDFLAGS="${LDFLAGS} -march=armv7-a"
+   fi
+ else
+   { $as_echo "$as_me:${as_lineno-$LINENO}: result: not Android" >&5
+@@ -18673,4 +18673,3 @@ if test "$Py_OPT" = 'false' -a "$Py_DEBUG" != 'true'; then
+     echo "" >&6
+     echo "" >&6
+ fi
+-