diff --git a/.circleci/config.yml b/.circleci/config.yml index acbb7668fd..5eb22bf8b8 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -85,8 +85,6 @@ jobs: name: Installing python dependencies command: | python -m pip install --user --upgrade pip setuptools setuptools-scm wheel - python -m pip install --no-cache-dir --user -r bindings/py/packaging/requirements.txt --verbose || exit - # Build Release with python - run: diff --git a/.github/workflows/arm64-build.yml b/.github/workflows/arm64-build.yml new file mode 100644 index 0000000000..802d5e3668 --- /dev/null +++ b/.github/workflows/arm64-build.yml @@ -0,0 +1,32 @@ +name: ARM64 build + +on: + # run every day of the week at 06:00 + schedule: + - cron: 0 6 * * * + +jobs: + build-arm64-docker: + name: Build for ARM64 on Docker + runs-on: ubuntu-18.04 + steps: + - name: Install docker + run: | + # bug in ubuntu, conflicts with docker.io + sudo apt-get update + sudo apt-get remove --purge -y moby-engine moby-cli + sudo apt-get install -y qemu-system docker.io + + - name: Setup ARM64 build env + run: | + uname -a + sudo docker run --rm --privileged multiarch/qemu-user-static:register --reset + uname -a + + - uses: actions/checkout@v1 + + - name: ARM64 build + run: | + sudo docker build -t htm-arm64-docker --build-arg arch=arm64 . + uname -a + diff --git a/.github/workflows/htmcore.yml b/.github/workflows/htmcore.yml new file mode 100644 index 0000000000..10ffd01eeb --- /dev/null +++ b/.github/workflows/htmcore.yml @@ -0,0 +1,127 @@ +name: htm.core build + +on: + push: + branches: + - 'v*.*.*' # vX.Y.Z release tag + # run on pull_request events that target the master branch + pull_request: + branches: + - master + # run every day of the week at 02:00 + schedule: + - cron: 0 2 * * * + +jobs: + build: + name: Building on ${{ matrix.os }} + runs-on: ${{ matrix.os }} + strategy: + #max-parallel: 4 + matrix: + python-version: [3.7] + os: [ubuntu-18.04, windows-2019, macOS-latest] + + steps: + - uses: actions/checkout@v1 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v1 + with: + python-version: ${{ matrix.python-version }} + + - name: Versions + run: | + python --version + cmake --version + c++ --version + + - name: Install dependencies (gcc-8) + if: matrix.os == 'ubuntu-18.04' + env: + CC: gcc-8 + CXX: g++-8 + run: | + sudo apt-get update + sudo apt-get -y install gcc-8 g++-8 + python -m pip install --upgrade pip setuptools wheel + python -m pip install -r requirements.txt + python setup.py configure + + - name: Install dependencies + if: matrix.os != 'ubuntu-18.04' + run: | + python -m pip install --upgrade pip setuptools wheel + python -m pip install -r requirements.txt + python setup.py configure + + - name: build htmcore with setup.py + run: python setup.py install --user + + - name: C++ & Python Tests + run: python setup.py test + + - name: Memory leaks check (valgrind) + if: matrix.os == 'ubuntu-18.04' + run: | + sudo apt-get -y install valgrind + LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:${PWD}/build/Release/lib valgrind --show-leak-kinds=definite,indirect,possible,reachable --track-origins=yes --num-callers=40 --error-exitcode=3 ./build/Release/bin/benchmark_hotgym 5 || exit 1 + + - name: Release (make package) + if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags') + run: | + python setup.py bdist_wheel + cd build/scripts + cmake --build . --config Release --target install # aka make install ,but multiplatform + cmake --build . --config Release --target package # make package + + - name: Release (deploy) + if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags') + # from https://github.com/marketplace/actions/gh-release + uses: softprops/action-gh-release@v1 + with: + files: | + build/scripts/htm_core-v* + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: pre-PyPI + if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags') + #copy dist data to /dist, where PyPI Action expects it + run: | + cp -a build/Release/distr/dist . + ls dist + + - name: Publish to PyPI + if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags') + uses: pypa/gh-action-pypi-publish@master + with: + user: __token__ + password: ${{ secrets.pypi_password }} + repository_url: https://test.pypi.org/legacy/ #TODO rm for real pypi + + + + build-debug: + name: Build and test in Debug mode + #currently cannot run on Linux & Debug due to a bug in YAML parser: issue #218 + runs-on: macOS-latest + steps: + - uses: actions/checkout@v1 + + - name: Install dependencies (Debug) + run: | + echo "built type: ${CMAKE_BUILD_TYPE}" + mkdir -p build/scripts + cd build/scripts + cmake ../.. -DCMAKE_BUILD_TYPE=Debug + + - name: Debug build + run: | + cd build/scripts + make -j2 && make install + + - name: C++ Tests + run: | + cd build/scripts + ../Debug/bin/unit_tests diff --git a/.travis.yml b/.travis.yml index b96a187a32..227c5ef201 100644 --- a/.travis.yml +++ b/.travis.yml @@ -65,13 +65,8 @@ install: script: - # Some tests (e.g., helloregion) expect this to be the current directory and - # this also matches current instructions in htm.core/README.md - # unit tests - - "cd $TRAVIS_BUILD_DIR/build/Release/bin" - - "${TRAVIS_BUILD_DIR}/build/Release/bin/unit_tests --gtest_output=xml:${TRAVIS_BUILD_DIR}/build/artifacts/unit_tests_report.xml" - # run python bindings tests - - echo "Python bindings tests" + # run c++ & python bindings tests + - echo "C++ & Python tests" - cd ${TRAVIS_BUILD_DIR} - "python setup.py test" # run memory leak tests with valgrind diff --git a/Dockerfile b/Dockerfile index 430ba31ede..3eac96ec49 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,36 +6,39 @@ ARG arch=amd64 # Multiarch Debian 10 Buster (amd64, arm64, etc). # https://hub.docker.com/r/multiarch/debian-debootstrap -FROM multiarch/debian-debootstrap:$arch-buster +FROM multiarch/debian-debootstrap:${arch}-buster RUN apt-get update -RUN apt-get install -y \ +RUN apt-get install -y --no-install-suggests \ cmake \ - g++ \ + g++-8 \ git-core \ libyaml-dev \ - python \ - python-dev \ - python-numpy \ - python-pip + python3-minimal \ + python3-dev \ + python3-numpy \ + python3-pip \ + python3-venv ADD . /usr/local/src/htm.core WORKDIR /usr/local/src/htm.core +# Setup py env +#RUN python3 -m venv pyenv && . pyenv/bin/activate +RUN pip3 install --upgrade setuptools pip wheel +#RUN export PYTHONPATH=$PYTHONPATH:/usr/local/lib/python3.7/dist-packages + # Install -RUN pip install --upgrade setuptools -RUN pip install wheel -RUN pip install \ +RUN pip3 install \ # Explicitly specify --cache-dir, --build, and --no-clean so that build # artifacts may be extracted from the container later. Final built python # packages can be found in /usr/local/src/htm.core/bindings/py/dist # --cache-dir /usr/local/src/htm.core/pip-cache \ # --build /usr/local/src/htm.core/pip-build \ # --no-clean \ - -r bindings/py/packaging/requirements.txt -RUN python setup.py install + -r requirements.txt +RUN python3 setup.py install --force # Test -RUN ./build/Release/bin/unit_tests -RUN python setup.py test +RUN python3 setup.py test diff --git a/README.md b/README.md index d0a40c22ac..f4f0a7eb99 100644 --- a/README.md +++ b/README.md @@ -64,7 +64,7 @@ Fork or download the HTM-Community htm.core repository from https://github.com/h ``` cd to-repository-root python -m pip install --user --upgrade pip setuptools setuptools-scm wheel -python -m pip install --no-cache-dir --user -r bindings/py/packaging/requirements.txt +python -m pip install --no-cache-dir --user -r requirements.txt ``` Be sure you are running the right version of python. Check it with the following command: @@ -275,7 +275,7 @@ distribution packages as listed and rename them as indicated. Copy these to ### There are two sets of Unit Tests: * C++ Unit tests -- to run: `./build/Release/bin/unit_tests` - * Python Unit tests -- to run: `python setup.py test` + * Python Unit tests -- to run: `python setup.py test` (runs also the C++ tests above) # Examples diff --git a/appveyor.yml b/appveyor.yml index 5c4c66eb1c..017ca31bca 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -99,7 +99,6 @@ build_script: - echo Binding version = %BINDINGS_VERSION% - python -m pip install --user --upgrade pip setuptools setuptools-scm wheel || exit - - python -m pip install --no-cache-dir --user -r bindings/py/packaging/requirements.txt || exit # we need to find the previous git tags so a shallow clone is not enough - git fetch --depth=200 @@ -110,13 +109,7 @@ build_script: after_build: - # Run unit_tests (C++) - - cd %HTM_CORE% - - mkdir %ARTIFACTS_DIR% - - cd %HTM_CORE%\build\Release\bin - - unit_tests.exe --gtest_output=xml:%ARTIFACTS_DIR%\unit_tests_report.xml - - # Run python tests + # Run C++ & python tests - cd %HTM_CORE% - python setup.py test @@ -155,4 +148,4 @@ on_success: } - \ No newline at end of file + diff --git a/bindings/py/packaging/setup.py b/bindings/py/packaging/setup.py index 5a55801066..51fa54bb75 100644 --- a/bindings/py/packaging/setup.py +++ b/bindings/py/packaging/setup.py @@ -46,9 +46,11 @@ # +# bindings cannot be compiled in Debug mode, unless a special python library also in +# Debug is used, which is quite unlikely. So for any CMAKE_BUILD_TYPE setting, override +# to Release mode. build_type = 'Release' - PY_BINDINGS = os.path.dirname(os.path.realpath(__file__)) REPO_DIR = os.path.abspath(os.path.join(PY_BINDINGS, os.pardir, os.pardir, os.pardir)) DISTR_DIR = os.path.join(REPO_DIR, "build", build_type, "distr") @@ -94,6 +96,29 @@ def run(self): +class ConfigureCommand(Command): + """Setup C++ dependencies and call cmake for configuration""" + + description = "Setup C++ dependencies and call cmake for configuration." + user_options = [] + + def initialize_options(self): + pass + + def finalize_options(self): + pass + + def run(self): + platform = getPlatformInfo() + if platform == DARWIN_PLATFORM and not "ARCHFLAGS" in os.environ: + os.system("export ARCHFLAGS=\"-arch x86_64\"") + + # Run CMake if extension files are missing. + # CMake also copies all py files into place in DISTR_DIR + configure(platform, build_type) + + + def fixPath(path): """ Ensures paths are correct for linux and windows @@ -106,12 +131,12 @@ def fixPath(path): -def findRequirements(platform): +def findRequirements(platform, fileName="requirements.txt"): """ Read the requirements.txt file and parse into requirements for setup's install_requirements option. """ - requirementsPath = fixPath(os.path.join(PY_BINDINGS, "requirements.txt")) + requirementsPath = fixPath(os.path.join(REPO_DIR, fileName)) return [ line.strip() for line in open(requirementsPath).readlines() @@ -138,6 +163,12 @@ def finalize_options(self): def run_tests(self): import pytest cwd = os.getcwd() + errno = 0 + # run c++ tests (from python) + cpp_tests = os.path.join(REPO_DIR, "build", "Release", "bin", "unit_tests") + subprocess.check_call([cpp_tests]) + os.chdir(cwd) + # run python bindings tests (in /bindings/py/tests/) try: os.chdir(os.path.join(REPO_DIR, "bindings", "py", "tests")) @@ -232,6 +263,29 @@ def generateExtensions(platform, build_type): Note: for Windows it will force a X64 build. """ cwd = os.getcwd() + scriptsDir = os.path.join(REPO_DIR, "build", "scripts") + try: + if not os.path.isdir(scriptsDir): + os.makedirs(scriptsDir) + os.chdir(scriptsDir) + + # cmake ../.. + configure(platform, build_type) + + # build: make && make install + if platform != "windows": #TODO since cmake 3.12 "-j4" is directly supported (=crossplatform), for now -- passes other options to make + subprocess.check_call(["cmake", "--build", ".", "--target", "install", "--config", build_type, "--", "-j", "4"]) + else: + subprocess.check_call(["cmake", "--build", ".", "--target", "install", "--config", build_type]) + finally: + os.chdir(cwd) + + +def configure(platform, build_type): + """ + Setup C++ dependencies and call cmake for configuration. + """ + cwd = os.getcwd() print("Python version: {}\n".format(sys.version)) from sys import version_info @@ -293,8 +347,6 @@ def generateExtensions(platform, build_type): else: subprocess.check_call(["cmake", "-G", generator, BUILD_TYPE, PY_VER, REPO_DIR]) - # Now do `make install` - subprocess.check_call(["cmake", "--build", ".", "--target", "install", "--config", build_type]) finally: os.chdir(cwd) @@ -304,12 +356,14 @@ def generateExtensions(platform, build_type): platform = getPlatformInfo() if platform == DARWIN_PLATFORM and not "ARCHFLAGS" in os.environ: - raise Exception("To build HTM.Core bindings in OS X, you must " - "`export ARCHFLAGS=\"-arch x86_64\"`.") + os.system("export ARCHFLAGS=\"-arch x86_64\"") # Run CMake if extension files are missing. # CMake also copies all py files into place in DISTR_DIR - getExtensionFiles(platform, build_type) + if len(sys.argv) > 1 and sys.argv[1] == "configure": + configure(platform, build_type) + else: #full build + getExtensionFiles(platform, build_type) with open(os.path.join(REPO_DIR, "README.md"), "r") as fh: long_description = fh.read() @@ -317,9 +371,9 @@ def generateExtensions(platform, build_type): """ set the default directory to the distr, and package it. """ + print("\nbindings/py/setup.py: Setup htm.core Python module in " + DISTR_DIR+ "\n") os.chdir(DISTR_DIR) - print("\nbindings/py/setup.py: Setup htm.core Python module in " + DISTR_DIR+ "\n") setup( # See https://docs.python.org/2/distutils/apiref.html for descriptions of arguments. # https://docs.python.org/2/distutils/setupscript.html @@ -340,11 +394,18 @@ def generateExtensions(platform, build_type): "htm.bindings": ["*.so", "*.pyd"], "htm.examples": ["*.csv"], }, - extras_require = {}, + #install extras by `pip install htm.core[examples]` + extras_require={'scikit-image>0.15.0':'examples', + 'sklearn':'examples', + 'matplotlib':'examples', + 'PIL':'examples', + 'scipy':'examples' + }, zip_safe=False, cmdclass={ "clean": CleanCommand, "test": TestCommand, + "configure": ConfigureCommand, }, author="Numenta & HTM Community", author_email="help@numenta.org", diff --git a/external/bootstrap.cmake b/external/bootstrap.cmake index 3beba401b5..f30cb3c9d3 100644 --- a/external/bootstrap.cmake +++ b/external/bootstrap.cmake @@ -31,12 +31,12 @@ FILE(MAKE_DIRECTORY ${REPOSITORY_DIR}/build/ThirdParty) execute_process(COMMAND ${CMAKE_COMMAND} -G ${CMAKE_GENERATOR} - -D CMAKE_INSTALL_PREFIX=. + -D CMAKE_INSTALL_PREFIX=. -D NEEDS_BOOST:BOOL=${NEEDS_BOOST} -D BINDING_BUILD:STRING=${BINDING_BUILD} - -D CMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} - -D REPOSITORY_DIR=${REPOSITORY_DIR} - ../../external + -D CMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} + -D REPOSITORY_DIR=${REPOSITORY_DIR} + ../../external WORKING_DIRECTORY ${REPOSITORY_DIR}/build/ThirdParty RESULT_VARIABLE result # OUTPUT_QUIET ### Disable this to debug external configuration diff --git a/bindings/py/packaging/requirements.txt b/requirements.txt similarity index 54% rename from bindings/py/packaging/requirements.txt rename to requirements.txt index c79806c3c4..c5c715ff85 100644 --- a/bindings/py/packaging/requirements.txt +++ b/requirements.txt @@ -2,10 +2,10 @@ ## for python bindings (in /bindings/py/) numpy>=1.15 pytest==4.6.5 #4.6.x series is last to support python2, once py2 dropped, we can switch to 5.x -pytest-cov>=2.5.0 ## for python code (in /py/) hexy>=1.4.3 # for grid cell encoder mock>=1.0.1 # for anomaly likelihood test -prettytable>=0.7.2 # For htm.advanced metric class mixins -scikit-image>=0.15.0 # For thalamus example -sklearn # for mnist.py example, dataset downloads from openML.org +prettytable>=0.7.2 # for monitor-mixin in htm.advanced (+its tests) +## optional dependencies, such as for visualizations, running examples +# should be placed in setup.py section extras_require. Install those by +# pip install htm.core[examples] and/or htm.core[monitor-mixin] diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 3a790369f1..02ee776d8a 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -439,6 +439,10 @@ install(DIRECTORY "${CMAKE_INSTALL_PREFIX}/distr/dist/" DESTINATION py FILES_MATCHING PATTERN "*.whl") +# copy requirements.txt into dist/ +install(FILES ${REPOSITORY_DIR}/requirements.txt + DESTINATION distr/dist) + ############### PACKAGE ########################### # diff --git a/src/examples/hotgym/HelloSPTP.cpp b/src/examples/hotgym/HelloSPTP.cpp index 7edea90645..9b7cc90b55 100644 --- a/src/examples/hotgym/HelloSPTP.cpp +++ b/src/examples/hotgym/HelloSPTP.cpp @@ -237,8 +237,8 @@ EPOCHS = 2; // make test faster in Debug const size_t timeTotal = (size_t)floor(tAll.getElapsed()); cout << "Total elapsed time = " << timeTotal << " seconds" << endl; if(EPOCHS >= 100) { //show only relevant values, ie don't run in valgrind (ndebug, epochs=5) run -#ifndef _MSC_VER - const size_t CI_avg_time = (size_t)floor(20*Timer::getSpeed()); //sec +#ifdef NTA_OS_LINUX + const size_t CI_avg_time = (size_t)floor(99*Timer::getSpeed()); //sec //FIXME the CI speed broken for docker linux NTA_CHECK(timeTotal <= CI_avg_time) << //we'll see how stable the time result in CI is, if usable "HelloSPTP test slower than expected! (" << timeTotal << ",should be "<< CI_avg_time << "), speed coef.= " << Timer::getSpeed(); #endif diff --git a/src/test/unit/engine/CppRegionTest.cpp b/src/test/unit/engine/CppRegionTest.cpp index 0450746f4b..c57c42af34 100644 --- a/src/test/unit/engine/CppRegionTest.cpp +++ b/src/test/unit/engine/CppRegionTest.cpp @@ -281,7 +281,6 @@ TEST(CppRegionTest, RegionSerialization) { Region r2(&n); r2.load(ss); EXPECT_EQ(*r1.get(), r2); - }