PYCBC-628: Produce ManyLinux builds

Only build 3.5-9

Explicit version list

Use regex to validate extant Pythons
More detailed info on LCB_VERSION found
Do auditwheel

Change-Id: I3e84b00bb05942ca9449f188e06782fc5fb4af04

.travis.yml

@@ -1,37 +1,39 @@
-language: python
-sudo: false
-
-python:
-  - "2.7"
-  - "3.3"
-  - "3.4"
-  - "3.5"
-  - "3.6"
-
-addons:
-  apt:
-    sources:
-      - couchbase-trusty
-    packages:
-      - libcouchbase-dev
-      - libcouchbase2-libevent
+notifications:
+  email: [email protected]
+matrix:
+  include:
+# disabled for now as dep issues
+#  - sudo: required
+#    services:
+#    - docker
+#    env: DOCKER_IMAGE=quay.io/pypa/manylinux1_x86_64 PLAT=manylinux1_x86_64
+#  - sudo: required
+#    services:
+#    - docker
+#    env: DOCKER_IMAGE=quay.io/pypa/manylinux2010_x86_64 PLAT=manylinux2010_x86_64
+  - sudo: required
+    services:
+    - docker
+    env: DOCKER_IMAGE=quay.io/pypa/manylinux2014_x86_64 PLAT=manylinux2014_x86_64
 
 install:
-  - pip -q install testresources==0.2.7
-  - pip -q install -r dev_requirements.txt
-  - pip -q install gevent || echo "Couldn't install gevent"
-  - pip -q install twisted || echo "Couldn't install Twisted"
-  - git describe --tags --long --always
-  - python setup.py build_ext --inplace
-  - python setup.py install
-
+- docker pull $DOCKER_IMAGE
 script:
-  - python -m couchbase.bucket
-  - cp .tests.ini.travis tests.ini
-  - nosetests -v couchbase_tests.test_sync
-  - nosetests -v acouchbase
-  - nosetests -v txcouchbase || echo "Twisted tests failed"
-
-notifications:
-  email:
-    - [email protected]
+- ls -alrt
+- chmod ugo+x travis/build-wheels.sh
+- docker run --rm -e PLAT=$PLAT -v `pwd`:/io $DOCKER_IMAGE $PRE_CMD /io/travis/build-wheels.sh
+- echo `ls wheelhouse/`
+- echo `ls /io/wheelhouse`
+- echo `ls .`
+- echo `ls /`
+- echo `ls /io`
+deploy:
+  provider: releases
+  api_key:
+    secure: YnTiCX6s9Kobprg6wpJq1LIwG1OvF7CWOtGWrG0pd22w529JXKSy0vepDUR2xy+PTECZoypt543XzwXJYRHBSx0RrUJ84UmeYHg2crrozJPtVdcPzrmktYE2TvoS+2GJpxAkrw7qzBIlXspobriCtyXS8vXVQLKUrii4tulMU55Aq/oAQWRtVyn4R0jKaiukNkJpOaI+kOeB7AEd0V2lGIhAtj1l0om9M/3O38spoGURUwTMx0qWY3xTPNrqCAuUJjW2G2dPgu57rkDcgcS2s2Q+1ukQMWyAIZObmlxlvOgxbVlff4f5Dl58AdzdFtpoxbOaXELDFWxtt0GLzWgAEDWif59aYzbhNA4U3fz2SgiRVF7bLkm6GZOOyNXoBME3XPpKcrqkqIZdAUbcKu2SOFEocfa3me9YtpXrBbCaf28SYxiYIToofbNcibCovnzWvPD0lqpRhQdHwxmym8csVGO0vwJ+fh12ekqBeCT1W36Y/YqBsuVkN1VOO9kPKwXBDL0skRu8apJDkCzEAvHqMOiz/Px1p4Tqv9ThcNSyXQT9EkQvil/zifmpBBQ34qXEcqpaPZdV2ZnBXm78L/MVpe7cm9GuXFN3YfRuVkpISxhEQ5VxF9BZsr0SIfMr3YzsDXFBlQBs+4YFrWqMoJlLveLu5jEAaX4u07D+etdu8oM=
+  file_glob: true
+  file: wheelhouse/*
+  skip_cleanup: true
+  on:
+    repo: griels/couchbase-python-client-ng
+    all_branches: true

CMakeLists.txt

@@ -5,30 +5,33 @@ set(THIRDPARTY_LCB_ROOT libcouchbase_src-prefix)
 set(LCBCXX_ROOT libcouchbase-cxx-prefix/src/libcouchbase-cxx)
 set(LCB_ROOT ${THIRDPARTY_LCB_ROOT}/src/libcouchbase_src)
 set(PYCBC_CMAKE_CPYTHON_WRAPPER 1)
-include(FindPythonInterp)
-include(FindPythonLibs)
+cmake_policy(SET CMP0054 NEW)
+if(NOT "${HYBRID_BUILD}")
+    set(ENABLE_HYBRID_BUILD TRUE)
+    include(FindPythonInterp)
+    include(FindPythonLibs)
 
-if (PYTHON_LIBDIR)
-    set(PYTHON_LIBDIR_HINT HINTS ${PYTHON_LIBDIR})
-else(PYTHON_LIBDIR)
-    set(PYTHON_LIBDIR_HINT "")
-endif()
+    if (PYTHON_LIBDIR)
+        set(PYTHON_LIBDIR_HINT HINTS ${PYTHON_LIBDIR})
+    else(PYTHON_LIBDIR)
+        set(PYTHON_LIBDIR_HINT "")
+    endif()
 
-if (PYTHON_VERSION_EXACT)
-    find_package(PythonInterp ${PYTHON_VERSION_EXACT} EXACT REQUIRED)
-    find_package(PythonLibs ${PYTHON_VERSION_EXACT} EXACT  REQUIRED)
-else(PYTHON_VERSION_EXACT)
-    find_package(PythonInterp REQUIRED)
-    find_package(PythonLibs ${PYTHON_LIBDIR_HINT} REQUIRED)
-endif()
+    if (PYTHON_VERSION_EXACT)
+        find_package(PythonInterp ${PYTHON_VERSION_EXACT} EXACT REQUIRED)
+        find_package(PythonLibs ${PYTHON_VERSION_EXACT} EXACT  REQUIRED)
+    else(PYTHON_VERSION_EXACT)
+        find_package(PythonInterp REQUIRED)
+        find_package(PythonLibs ${PYTHON_LIBDIR_HINT} REQUIRED)
+    endif()
 
-cmake_policy(SET CMP0054 NEW)
-if(WIN32)
-    set(PYCBC_C_MOD_NAME "_libcouchbase.${PYTHONLIBS_VERSION_STRING}")
-    set(PYCBC_C_MOD_SUFFIX ".pyd")
-else()
-    set(PYCBC_C_MOD_NAME "_libcouchbase")
-    set(PYCBC_C_MOD_SUFFIX ".so")
+    if(WIN32)
+        set(PYCBC_C_MOD_NAME "_libcouchbase.${PYTHONLIBS_VERSION_STRING}")
+        set(PYCBC_C_MOD_SUFFIX ".pyd")
+    else()
+        set(PYCBC_C_MOD_NAME "_libcouchbase")
+        set(PYCBC_C_MOD_SUFFIX ".so")
+    endif()
 endif()
 if (FORCE_DEBUG OR (CMAKE_BUILD_TYPE MATCHES DEBUG))
     set(RELEASE_TYPE Debug)
@@ -37,7 +40,7 @@ else()
 endif()
 
 set(PYCBC_C_MOD_NAME "_dummy")
-
+set(PYCBC_C_MOD_SUFFIX ".so")
 macro(use_cxx11)
 if (CMAKE_VERSION VERSION_LESS "3.1")
 if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU")

couchbase_version.py

@@ -41,6 +41,9 @@ def __init__(self, rawtext):
 
         # Split up the X.Y.Z
         match = RE_XYZ.match(vinfo)
+        if not match:
+            print("Could not parse tag {}".format(vinfo))
+            raise MalformedGitTag("Could not parse tag {}".format(vinfo))
         (self.ver_maj, self.ver_min, self.ver_patch, self.ver_extra) =\
             match.groups()
 

src/lcb_v4_wrapper.h

@@ -17,6 +17,15 @@
 #ifndef COUCHBASE_PYTHON_CLIENT_LCB_V4_WRAPPER_H
 #define COUCHBASE_PYTHON_CLIENT_LCB_V4_WRAPPER_H
 #include <libcouchbase/couchbase.h>
+#include "../build/lcb_min_version.h"
+#define STR_HELPER(x) #x
+#define STR(x) STR_HELPER(x)
+#pragma message "Got LCB_VERSION " STR(LCB_VERSION)
+#if LCB_VERSION && LCB_VERSION < LCB_MIN_VERSION
+#pragma message "Couchbase Python SDK requires libcouchbase " LCB_MIN_VERSION_TEXT " or greater but found" STR(LCB_VERSION)
+#error "Please upgrade libcouchbase accordingly"
+#endif
+
 #include <libcouchbase/utils.h>
 #include <libcouchbase/crypto.h>
 #ifdef PYCBC_BACKPORT_CRYPTO

src/pycbc.h

@@ -89,11 +89,6 @@ Mostly the V4 API changes
 #endif
 
 #include <libcouchbase/crypto.h>
-#include "../build/lcb_min_version.h"
-#if LCB_VERSION < LCB_MIN_VERSION
-#pragma message "Couchbase Python SDK requires libcouchbase " LCB_MIN_VERSION_TEXT " or greater"
-#error "Please upgrade libcouchbase accordingly"
-#endif
 
 #include <pythread.h>
 #include "mresdict.h"

travis/build-wheels.sh

@@ -0,0 +1,60 @@
+#!/bin/bash
+set -e -x
+
+# Install a system package required by our library
+yum install -y cmake
+yum list installed
+yum install -y python36-devel
+LCB_VER_NUM=3.0.0
+LCB_VER=libcouchbase-3.0.0_centos7_x86_64
+curl -O https://packages.couchbase.com/clients/c/${LCB_VER}.tar
+tar xf ${LCB_VER}.tar
+cd ${LCB_VER}
+yum install -y libcouchbase3{-tools,-libevent,}-3.0.0*.rpm libcouchbase-devel-*.rpm
+
+ls -alr .
+
+echo `ls -alr /io`
+pushd .
+
+# Compile wheels
+PY_BASE="/opt/python"
+PY_VALID=".*3[6].*"
+#export CFLAGS="-static-libstdc++ ${CFLAGS}"
+cd ${PY_BASE}
+for VERSION in */; do
+    PYBIN="${VERSION}bin"
+    echo "Testing ${VERSION}"
+    if [[ ${VERSION} =~ $PY_VALID ]]
+    then
+      echo "${VERSION} matches ${PY_VALID}"
+      if [ -d "${PYBIN}" ]
+      then
+        echo "Building for ${VERSION} at ${PYBIN}"
+        ${PYBIN}/pip wheel /io/ -w /io/wheelhouse/
+      else
+        echo "${PYBIN} does not exist"
+      fi
+    else
+      echo "${VERSION} does not match ${PY_VALID}"
+    fi
+done
+
+popd
+
+# Bundle external shared libraries into the wheels
+for whl in /io/wheelhouse/*.whl; do
+    auditwheel repair "$whl" --plat $PLAT -w /io/wheelhouse/
+done
+
+# Install packages and test
+
+#for PYBIN in /opt/python/*/bin/; do
+#    "${PYBIN}/pip" install -r /io/dev_requirements.txt
+#    "${PYBIN}/pip" install . --no-index -f /io/wheelhouse
+#    (cd "$HOME"; "${PYBIN}/nosetests" pymanylinuxdemo)
+#done
+
+echo `ls /io/wheelhouse`
+echo `ls /wheelhouse`
+echo `ls wheelhouse`