[WIP] Refactor python recipes and update python2's to version 2.7.15

.travis.yml

@@ -18,9 +18,10 @@ env:
     - ANDROID_SDK_HOME=/opt/android/android-sdk
     - ANDROID_NDK_HOME=/opt/android/android-ndk
   matrix:
-    - COMMAND='. venv/bin/activate && cd testapps/ && python setup_testapp_python3.py apk --sdk-dir $ANDROID_SDK_HOME --ndk-dir $ANDROID_NDK_HOME --requirements sdl2,pyjnius,kivy,python3'
+    - COMMAND='. venv/bin/activate && cd testapps/ && python setup_testapp_python3.py apk --sdk-dir $ANDROID_SDK_HOME --ndk-dir $ANDROID_NDK_HOME --requirements sdl2,pyjnius,kivy,python3,numpy'
     # overrides requirements to skip `peewee` pure python module, see:
     # https://github.com/kivy/python-for-android/issues/1263#issuecomment-390421054
+    - COMMAND='. venv/bin/activate && cd testapps/ && python setup_testapp_python3_sqlite_openssl.py apk --sdk-dir $ANDROID_SDK_HOME --ndk-dir $ANDROID_NDK_HOME --requirements libffi,sdl2,pyjnius,kivy,python3,openssl,requests,sqlite3,setuptools'
     - COMMAND='. venv/bin/activate && cd testapps/ && python setup_testapp_python2_sqlite_openssl.py apk --sdk-dir $ANDROID_SDK_HOME --ndk-dir $ANDROID_NDK_HOME --requirements sdl2,pyjnius,kivy,python2,openssl,requests,sqlite3,setuptools'
     - COMMAND='. venv/bin/activate && cd testapps/ && python setup_testapp_python2.py apk --sdk-dir $ANDROID_SDK_HOME --ndk-dir $ANDROID_NDK_HOME --bootstrap sdl2 --requirements python2,numpy'
     # builds only the recipes that moved

Dockerfile.py2

@@ -26,7 +26,7 @@ RUN apt -y update -qq \
 
 
 ENV ANDROID_NDK_HOME="${ANDROID_HOME}/android-ndk"
-ENV ANDROID_NDK_VERSION="16b"
+ENV ANDROID_NDK_VERSION="17c"
 ENV ANDROID_NDK_HOME_V="${ANDROID_NDK_HOME}-r${ANDROID_NDK_VERSION}"
 
 # get the latest version from https://developer.android.com/ndk/downloads/index.html
@@ -104,7 +104,7 @@ RUN dpkg --add-architecture i386 \
 # specific recipes dependencies (e.g. libffi requires autoreconf binary)
 RUN apt -y update -qq \
     && apt -y install -qq --no-install-recommends \
-        autoconf automake cmake gettext libltdl-dev libtool pkg-config \
+        libffi-dev autoconf automake cmake gettext libltdl-dev libtool pkg-config \
     && apt -y autoremove \
     && apt -y clean
 

Dockerfile.py3

@@ -26,7 +26,7 @@ RUN apt -y update -qq \
 
 
 ENV ANDROID_NDK_HOME="${ANDROID_HOME}/android-ndk"
-ENV ANDROID_NDK_VERSION="16b"
+ENV ANDROID_NDK_VERSION="17c"
 ENV ANDROID_NDK_HOME_V="${ANDROID_NDK_HOME}-r${ANDROID_NDK_VERSION}"
 
 # get the latest version from https://developer.android.com/ndk/downloads/index.html
@@ -104,7 +104,7 @@ RUN dpkg --add-architecture i386 \
 # specific recipes dependencies (e.g. libffi requires autoreconf binary)
 RUN apt -y update -qq \
     && apt -y install -qq --no-install-recommends \
-        autoconf automake cmake gettext libltdl-dev libtool pkg-config \
+        libffi-dev autoconf automake cmake gettext libltdl-dev libtool pkg-config \
     && apt -y autoremove \
     && apt -y clean
 

doc/source/quickstart.rst

@@ -93,11 +93,34 @@ named ``tools``, and you will need to run extra commands to install
 the SDK packages needed. 
 
 For Android NDK, note that modern releases will only work on a 64-bit
-operating system. If you are using a 32-bit distribution (or hardware),
-the latest useable NDK version is r10e, which can be downloaded here:
+operating system. The minimal, and recommended, NDK version to use is r17c:
+
+ - `Linux (64-bits) <https://dl.google.com/android/repository/android-ndk-r17c-linux-x86_64.zip>`_
+ - `Macos X (64-bits) <https://dl.google.com/android/repository/android-ndk-r17c-darwin-x86_64.zip>`_
+ - Windows users should create a virtual machine with an GNU Linux os
+   installed, and then you can follow the described instructions from within
+   your virtual machine.
+
+.. note::
+    Versions below r17 may cause troubles while compiling some recipes. Our
+    tests had been done against ndk versions r17b and r17c. Later versions of
+    the android ndk still are not proved to work with python-for-android, and
+    may work or not...so...better to stick to the recommended version (r17c).
+
+If you are using a 32-bit distribution (or hardware),
+the latest usable NDK version is r10e, which can be downloaded here:
 
 - `Legacy 32-bit Linux NDK r10e <http://dl.google.com/android/ndk/android-ndk-r10e-linux-x86.bin>`_
 
+.. warning::
+    **32-bit distributions**
+
+    Since the python2 recipe updated to version 2.7.15, the build system has
+    been changed and you should use an old release of python-for-android, which
+    contains the legacy python recipe (v2.7.2). The last python-for-android
+    release with the legacy version of python is version
+    `0.6.0 <https://github.com/kivy/python-for-android/archive/0.6.0.zip>`_.
+
 First, install a platform to target (you can also replace ``27`` with
 a different platform number, this will be used again later)::
 
@@ -113,8 +136,8 @@ Then, you can edit your ``~/.bashrc`` or other favorite shell to include new env
 
     # Adjust the paths!
     export ANDROIDSDK="$HOME/Documents/android-sdk-27"
-    export ANDROIDNDK="$HOME/Documents/android-ndk-r10e"
-    export ANDROIDAPI="27"  # Target API version of your application
+    export ANDROIDNDK="$HOME/Documents/android-ndk-r17c"
+    export ANDROIDAPI="26"  # Target API version of your application
     export NDKAPI="19"  # Minimum supported API version of your application
     export ANDROIDNDKVER="r10e"  # Version of the NDK you installed
 

pythonforandroid/archs.py

@@ -30,13 +30,24 @@ def include_dirs(self):
                 d.format(arch=self))
             for d in self.ctx.include_dirs]
 
-    def get_env(self, with_flags_in_cc=True):
+    def get_env(self, with_flags_in_cc=True, clang=False):
         env = {}
 
-        env['CFLAGS'] = ' '.join([
-            '-DANDROID', '-mandroid', '-fomit-frame-pointer'
-            ' -D__ANDROID_API__={}'.format(self.ctx.ndk_api),
-            ])
+        cflags = [
+            '-DANDROID',
+            '-fomit-frame-pointer',
+            '-D__ANDROID_API__={}'.format(self.ctx.ndk_api)]
+        if not clang:
+            cflags += ['-mandroid']
+        else:
+            cflags += ['-target armv7-none-linux-androideabi']
+            android_host = 'arm-linux-androideabi'
+            platform_dir = join(self.ctx.ndk_dir, 'platforms', 'android-{}'.format(self.ctx.ndk_api), 'arch-arm')
+            toolchain = '{android_host}-4.9'.format(android_host=android_host)
+            toolchain = join(self.ctx.ndk_dir, 'toolchains', toolchain, 'prebuilt', 'linux-x86_64')
+            cflags += ['-gcc-toolchain {}'.format(toolchain)]
+
+        env['CFLAGS'] = ' '.join(cflags)
         env['LDFLAGS'] = ' '
 
         sysroot = join(self.ctx._ndk_dir, 'sysroot')
@@ -82,8 +93,19 @@ def get_env(self, with_flags_in_cc=True):
             env['NDK_CCACHE'] = self.ctx.ccache
             env.update({k: v for k, v in environ.items() if k.startswith('CCACHE_')})
 
-        cc = find_executable('{command_prefix}-gcc'.format(
-            command_prefix=command_prefix), path=environ['PATH'])
+        if clang:
+            clang_path = join(
+                self.ctx.ndk_dir, 'toolchains', 'llvm', 'prebuilt',
+                'linux-x86_64', 'bin')
+            environ['PATH'] = '{clang_path}:{path}'.format(
+                clang_path=clang_path, path=environ['PATH'])
+            exe = join(clang_path, 'clang')
+            execxx = join(clang_path, 'clang++')
+        else:
+            exe = '{command_prefix}-gcc'.format(command_prefix=command_prefix)
+            execxx = '{command_prefix}-g++'.format(command_prefix=command_prefix)
+
+        cc = find_executable(exe, path=environ['PATH'])
         if cc is None:
             print('Searching path are: {!r}'.format(environ['PATH']))
             raise BuildInterruptingException(
@@ -93,20 +115,20 @@ def get_env(self, with_flags_in_cc=True):
                 'installed. Exiting.')
 
         if with_flags_in_cc:
-            env['CC'] = '{ccache}{command_prefix}-gcc {cflags}'.format(
-                command_prefix=command_prefix,
+            env['CC'] = '{ccache}{exe} {cflags}'.format(
+                exe=exe,
                 ccache=ccache,
                 cflags=env['CFLAGS'])
-            env['CXX'] = '{ccache}{command_prefix}-g++ {cxxflags}'.format(
-                command_prefix=command_prefix,
+            env['CXX'] = '{ccache}{execxx} {cxxflags}'.format(
+                execxx=execxx,
                 ccache=ccache,
                 cxxflags=env['CXXFLAGS'])
         else:
-            env['CC'] = '{ccache}{command_prefix}-gcc'.format(
-                command_prefix=command_prefix,
+            env['CC'] = '{ccache}{exe}'.format(
+                exe=exe,
                 ccache=ccache)
-            env['CXX'] = '{ccache}{command_prefix}-g++'.format(
-                command_prefix=command_prefix,
+            env['CXX'] = '{ccache}{execxx}'.format(
+                execxx=execxx,
                 ccache=ccache)
 
         env['AR'] = '{}-ar'.format(command_prefix)
@@ -123,12 +145,13 @@ def get_env(self, with_flags_in_cc=True):
         env['READELF'] = '{}-readelf'.format(command_prefix)
         env['NM'] = '{}-nm'.format(command_prefix)
 
-        hostpython_recipe = Recipe.get_recipe('hostpython2', self.ctx)
-
-        # This hardcodes python version 2.7, needs fixing
+        hostpython_recipe = Recipe.get_recipe(
+            'host' + self.ctx.python_recipe.name, self.ctx)
         env['BUILDLIB_PATH'] = join(
             hostpython_recipe.get_build_dir(self.arch),
-            'build', 'lib.linux-{}-2.7'.format(uname()[-1]))
+            'build', 'lib.linux-{}-{}'.format(
+                uname()[-1], self.ctx.python_recipe.major_minor_version_string)
+        )
 
         env['PATH'] = environ['PATH']
 
@@ -151,8 +174,8 @@ class ArchARM(Arch):
 class ArchARMv7_a(ArchARM):
     arch = 'armeabi-v7a'
 
-    def get_env(self, with_flags_in_cc=True):
-        env = super(ArchARMv7_a, self).get_env(with_flags_in_cc)
+    def get_env(self, with_flags_in_cc=True, clang=False):
+        env = super(ArchARMv7_a, self).get_env(with_flags_in_cc, clang=clang)
         env['CFLAGS'] = (env['CFLAGS'] +
                          (' -march=armv7-a -mfloat-abi=softfp '
                           '-mfpu=vfp -mthumb'))
@@ -166,8 +189,8 @@ class Archx86(Arch):
     command_prefix = 'i686-linux-android'
     platform_dir = 'arch-x86'
 
-    def get_env(self, with_flags_in_cc=True):
-        env = super(Archx86, self).get_env(with_flags_in_cc)
+    def get_env(self, with_flags_in_cc=True, clang=False):
+        env = super(Archx86, self).get_env(with_flags_in_cc, clang=clang)
         env['CFLAGS'] = (env['CFLAGS'] +
                          ' -march=i686 -mtune=intel -mssse3 -mfpmath=sse -m32')
         env['CXXFLAGS'] = env['CFLAGS']
@@ -180,8 +203,8 @@ class Archx86_64(Arch):
     command_prefix = 'x86_64-linux-android'
     platform_dir = 'arch-x86_64'
 
-    def get_env(self, with_flags_in_cc=True):
-        env = super(Archx86_64, self).get_env(with_flags_in_cc)
+    def get_env(self, with_flags_in_cc=True, clang=False):
+        env = super(Archx86_64, self).get_env(with_flags_in_cc, clang=clang)
         env['CFLAGS'] = (env['CFLAGS'] +
                          ' -march=x86-64 -msse4.2 -mpopcnt -m64 -mtune=intel')
         env['CXXFLAGS'] = env['CFLAGS']
@@ -194,8 +217,8 @@ class ArchAarch_64(Arch):
     command_prefix = 'aarch64-linux-android'
     platform_dir = 'arch-arm64'
 
-    def get_env(self, with_flags_in_cc=True):
-        env = super(ArchAarch_64, self).get_env(with_flags_in_cc)
+    def get_env(self, with_flags_in_cc=True, clang=False):
+        env = super(ArchAarch_64, self).get_env(with_flags_in_cc, clang=clang)
         incpath = ' -I' + join(dirname(__file__), 'includes', 'arm64-v8a')
         env['EXTRA_CFLAGS'] = incpath
         env['CFLAGS'] += incpath

pythonforandroid/bootstrap.py

@@ -269,14 +269,11 @@ def strip_libraries(self, arch):
             return
         strip = sh.Command(strip)
 
-        if self.ctx.python_recipe.name == 'python2':
-            filens = shprint(sh.find, join(self.dist_dir, 'private'),
-                             join(self.dist_dir, 'libs'),
-                             '-iname', '*.so', _env=env).stdout.decode('utf-8')
-        else:
-            filens = shprint(sh.find, join(self.dist_dir, '_python_bundle', '_python_bundle', 'modules'),
-                             join(self.dist_dir, 'libs'),
-                             '-iname', '*.so', _env=env).stdout.decode('utf-8')
+        filens = shprint(sh.find, join(self.dist_dir, '_python_bundle',
+                                       '_python_bundle', 'modules'),
+                         join(self.dist_dir, 'libs'),
+                         '-iname', '*.so', _env=env).stdout.decode('utf-8')
+
         logger.info('Stripping libraries in private dir')
         for filen in filens.split('\n'):
             try:

pythonforandroid/bootstraps/common/build/build.py

@@ -46,7 +46,8 @@ def get_bootstrap_name():
 # Try to find a host version of Python that matches our ARM version.
 PYTHON = join(curdir, 'python-install', 'bin', 'python.host')
 if not exists(PYTHON):
-    print('Could not find hostpython, will not compile to .pyo (this is normal with python3)')
+    print('Could not find hostpython, will not compile to .pyo '
+          '(this is normal with python2 and python3)')
     PYTHON = None
 
 BLACKLIST_PATTERNS = [
@@ -151,7 +152,6 @@ def make_python_zip():
 
     if not exists('private'):
         print('No compiled python is present to zip, skipping.')
-        print('this should only be the case if you are using the CrystaX python')
         return
 
     global python_files