From 7ed3e8e714700afa39c8ede9b225b775179182d2 Mon Sep 17 00:00:00 2001 From: breznak Date: Tue, 24 Sep 2019 12:34:31 +0000 Subject: [PATCH 01/67] CI: try Github Actions for CI GH Actions initial setup, building on all platforms --- .github/workflows/htmcore.yml | 52 +++++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) create mode 100644 .github/workflows/htmcore.yml diff --git a/.github/workflows/htmcore.yml b/.github/workflows/htmcore.yml new file mode 100644 index 0000000000..2a00843a4e --- /dev/null +++ b/.github/workflows/htmcore.yml @@ -0,0 +1,52 @@ +name: htm.core build + +on: + push: + branches: + - v*\.*\.*\.* # vX.Y.Z release tag + # run on pull_request events that target the master branch + pull_request: + branches: + - master + # run every day of the week from Monday - Friday at 02:00 + schedule: + - cron: 0 2 * * 1-5 + +jobs: + build: + name: Building on ${{ matrix.os }} + runs-on: ${{ matrix.os }} + strategy: + #max-parallel: 4 + matrix: + python-version: [3.7] + os: [ubuntu-18.04, windows-2019, macOS-latest] + + steps: + - uses: actions/checkout@v1 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v1 + with: + python-version: ${{ matrix.python-version }} + - name: Config + run: | + $CC --version + $CXX --version + python --version + cmake --version + - name: Install dependencies + run: | + python -m pip install --upgrade pip setuptools wheel + pip install -r bindings/py/packaging/requirements.txt #TODO move to top level requirements.txt + - name: build htmcore with setup.py + run: | + python setup.py install --user + - name: C++ Tests + run: make tests_all + - name: Python Tests + run: python setup.py test + #TODO run valgrind + #TODO build ARM64 on docker + #TODO 1 Debug build + #TODO GH Releases + #TODO PYPI deploy From 6e9433ed65305a89e3821510b7783a1b0643c7c8 Mon Sep 17 00:00:00 2001 From: Marek Otahal Date: Tue, 24 Sep 2019 14:47:47 +0200 Subject: [PATCH 02/67] GH Actions: CC not set as env variable --- .github/workflows/htmcore.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/htmcore.yml b/.github/workflows/htmcore.yml index 2a00843a4e..96469075d7 100644 --- a/.github/workflows/htmcore.yml +++ b/.github/workflows/htmcore.yml @@ -30,8 +30,6 @@ jobs: python-version: ${{ matrix.python-version }} - name: Config run: | - $CC --version - $CXX --version python --version cmake --version - name: Install dependencies From 385966dd803be649b76ebcfa69f9e286fbdd2cd8 Mon Sep 17 00:00:00 2001 From: Marek Otahal Date: Tue, 24 Sep 2019 14:52:58 +0200 Subject: [PATCH 03/67] GH Actions: fix pip requirements for Windows Win does not handle #TODO at end of pip install -r requirements.txt #TODO --- .github/workflows/htmcore.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/htmcore.yml b/.github/workflows/htmcore.yml index 96469075d7..4347d10bf9 100644 --- a/.github/workflows/htmcore.yml +++ b/.github/workflows/htmcore.yml @@ -35,7 +35,8 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip setuptools wheel - pip install -r bindings/py/packaging/requirements.txt #TODO move to top level requirements.txt + pip install -r bindings/py/packaging/requirements.txt + #TODO move to top level requirements.txt - name: build htmcore with setup.py run: | python setup.py install --user From cbe73a497b5df03d015d86b9a9178034411e4465 Mon Sep 17 00:00:00 2001 From: Marek Otahal Date: Tue, 24 Sep 2019 15:05:05 +0200 Subject: [PATCH 04/67] GH Actions: export ARCH_FLAGS which is needed for OSX --- .github/workflows/htmcore.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/htmcore.yml b/.github/workflows/htmcore.yml index 4347d10bf9..aa37f6fdea 100644 --- a/.github/workflows/htmcore.yml +++ b/.github/workflows/htmcore.yml @@ -38,7 +38,11 @@ jobs: pip install -r bindings/py/packaging/requirements.txt #TODO move to top level requirements.txt - name: build htmcore with setup.py + env: + ARCHFLAGS: "-arch x86_64" + #TODO set ARCH_FLAGS (for OSX) in setup.py run: | + echo $ARCHFLAGS python setup.py install --user - name: C++ Tests run: make tests_all From c100cfe52b7ab4fbfbf3326ae37659b4a41d6836 Mon Sep 17 00:00:00 2001 From: Marek Otahal Date: Tue, 24 Sep 2019 15:23:20 +0200 Subject: [PATCH 05/67] setup.py: set ARCHFLAGS for OSX as required --- .github/workflows/htmcore.yml | 8 +++----- bindings/py/packaging/setup.py | 3 +-- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/.github/workflows/htmcore.yml b/.github/workflows/htmcore.yml index aa37f6fdea..02879732ed 100644 --- a/.github/workflows/htmcore.yml +++ b/.github/workflows/htmcore.yml @@ -36,20 +36,18 @@ jobs: run: | python -m pip install --upgrade pip setuptools wheel pip install -r bindings/py/packaging/requirements.txt - #TODO move to top level requirements.txt - name: build htmcore with setup.py - env: - ARCHFLAGS: "-arch x86_64" - #TODO set ARCH_FLAGS (for OSX) in setup.py run: | - echo $ARCHFLAGS python setup.py install --user - name: C++ Tests run: make tests_all - name: Python Tests run: python setup.py test + #TODO run valgrind #TODO build ARM64 on docker #TODO 1 Debug build #TODO GH Releases #TODO PYPI deploy + #TODO move to top level requirements.txt + #TODO set ARCH_FLAGS (for OSX) in setup.py diff --git a/bindings/py/packaging/setup.py b/bindings/py/packaging/setup.py index 5a55801066..0036a59dc8 100644 --- a/bindings/py/packaging/setup.py +++ b/bindings/py/packaging/setup.py @@ -304,8 +304,7 @@ def generateExtensions(platform, build_type): platform = getPlatformInfo() if platform == DARWIN_PLATFORM and not "ARCHFLAGS" in os.environ: - raise Exception("To build HTM.Core bindings in OS X, you must " - "`export ARCHFLAGS=\"-arch x86_64\"`.") + os.system("export ARCHFLAGS=\"-arch x86_64\"") # Run CMake if extension files are missing. # CMake also copies all py files into place in DISTR_DIR From 46514b2ee1ec461074f389e8c9946979614678d0 Mon Sep 17 00:00:00 2001 From: Marek Otahal Date: Tue, 24 Sep 2019 15:26:18 +0200 Subject: [PATCH 06/67] setup.py: move requirements to top-level dir as it's the common location and easier to find for new developers. --- .circleci/config.yml | 2 -- .github/workflows/htmcore.yml | 4 +--- README.md | 2 +- appveyor.yml | 3 +-- bindings/py/packaging/setup.py | 2 +- bindings/py/packaging/requirements.txt => requirements.txt | 0 6 files changed, 4 insertions(+), 9 deletions(-) rename bindings/py/packaging/requirements.txt => requirements.txt (100%) diff --git a/.circleci/config.yml b/.circleci/config.yml index acbb7668fd..5eb22bf8b8 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -85,8 +85,6 @@ jobs: name: Installing python dependencies command: | python -m pip install --user --upgrade pip setuptools setuptools-scm wheel - python -m pip install --no-cache-dir --user -r bindings/py/packaging/requirements.txt --verbose || exit - # Build Release with python - run: diff --git a/.github/workflows/htmcore.yml b/.github/workflows/htmcore.yml index 02879732ed..51f5ac4fc1 100644 --- a/.github/workflows/htmcore.yml +++ b/.github/workflows/htmcore.yml @@ -35,7 +35,7 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip setuptools wheel - pip install -r bindings/py/packaging/requirements.txt + pip install -r requirements.txt - name: build htmcore with setup.py run: | python setup.py install --user @@ -49,5 +49,3 @@ jobs: #TODO 1 Debug build #TODO GH Releases #TODO PYPI deploy - #TODO move to top level requirements.txt - #TODO set ARCH_FLAGS (for OSX) in setup.py diff --git a/README.md b/README.md index d0a40c22ac..238a734f6b 100644 --- a/README.md +++ b/README.md @@ -64,7 +64,7 @@ Fork or download the HTM-Community htm.core repository from https://github.com/h ``` cd to-repository-root python -m pip install --user --upgrade pip setuptools setuptools-scm wheel -python -m pip install --no-cache-dir --user -r bindings/py/packaging/requirements.txt +python -m pip install --no-cache-dir --user -r requirements.txt ``` Be sure you are running the right version of python. Check it with the following command: diff --git a/appveyor.yml b/appveyor.yml index 5c4c66eb1c..055d381f44 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -99,7 +99,6 @@ build_script: - echo Binding version = %BINDINGS_VERSION% - python -m pip install --user --upgrade pip setuptools setuptools-scm wheel || exit - - python -m pip install --no-cache-dir --user -r bindings/py/packaging/requirements.txt || exit # we need to find the previous git tags so a shallow clone is not enough - git fetch --depth=200 @@ -155,4 +154,4 @@ on_success: } - \ No newline at end of file + diff --git a/bindings/py/packaging/setup.py b/bindings/py/packaging/setup.py index 0036a59dc8..68ca515eda 100644 --- a/bindings/py/packaging/setup.py +++ b/bindings/py/packaging/setup.py @@ -111,7 +111,7 @@ def findRequirements(platform): Read the requirements.txt file and parse into requirements for setup's install_requirements option. """ - requirementsPath = fixPath(os.path.join(PY_BINDINGS, "requirements.txt")) + requirementsPath = fixPath(os.path.join(REPO_DIR, "requirements.txt")) return [ line.strip() for line in open(requirementsPath).readlines() diff --git a/bindings/py/packaging/requirements.txt b/requirements.txt similarity index 100% rename from bindings/py/packaging/requirements.txt rename to requirements.txt From 1e1ee2ac1e0d4f02a2d19d2fdc03e347f99cbfb3 Mon Sep 17 00:00:00 2001 From: Marek Otahal Date: Tue, 24 Sep 2019 16:00:38 +0200 Subject: [PATCH 07/67] GH Actions: fix running c++ tests --- .github/workflows/htmcore.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/htmcore.yml b/.github/workflows/htmcore.yml index 51f5ac4fc1..4cfc1427a9 100644 --- a/.github/workflows/htmcore.yml +++ b/.github/workflows/htmcore.yml @@ -40,7 +40,7 @@ jobs: run: | python setup.py install --user - name: C++ Tests - run: make tests_all + run: ./build/Release/bin/unit_tests - name: Python Tests run: python setup.py test From 9851bd96d1854a122614a4e2f0a78be9c8778766 Mon Sep 17 00:00:00 2001 From: Marek Otahal Date: Tue, 24 Sep 2019 16:21:09 +0200 Subject: [PATCH 08/67] setup.py: python runs also the c++ tests python setup.py test runs both c++ and python tests --- .github/workflows/htmcore.yml | 4 +--- .travis.yml | 9 ++------- README.md | 2 +- appveyor.yml | 8 +------- bindings/py/packaging/setup.py | 10 ++++++++++ 5 files changed, 15 insertions(+), 18 deletions(-) diff --git a/.github/workflows/htmcore.yml b/.github/workflows/htmcore.yml index 4cfc1427a9..eda3b0c243 100644 --- a/.github/workflows/htmcore.yml +++ b/.github/workflows/htmcore.yml @@ -39,9 +39,7 @@ jobs: - name: build htmcore with setup.py run: | python setup.py install --user - - name: C++ Tests - run: ./build/Release/bin/unit_tests - - name: Python Tests + - name: C++ & Python Tests run: python setup.py test #TODO run valgrind diff --git a/.travis.yml b/.travis.yml index b96a187a32..227c5ef201 100644 --- a/.travis.yml +++ b/.travis.yml @@ -65,13 +65,8 @@ install: script: - # Some tests (e.g., helloregion) expect this to be the current directory and - # this also matches current instructions in htm.core/README.md - # unit tests - - "cd $TRAVIS_BUILD_DIR/build/Release/bin" - - "${TRAVIS_BUILD_DIR}/build/Release/bin/unit_tests --gtest_output=xml:${TRAVIS_BUILD_DIR}/build/artifacts/unit_tests_report.xml" - # run python bindings tests - - echo "Python bindings tests" + # run c++ & python bindings tests + - echo "C++ & Python tests" - cd ${TRAVIS_BUILD_DIR} - "python setup.py test" # run memory leak tests with valgrind diff --git a/README.md b/README.md index 238a734f6b..f4f0a7eb99 100644 --- a/README.md +++ b/README.md @@ -275,7 +275,7 @@ distribution packages as listed and rename them as indicated. Copy these to ### There are two sets of Unit Tests: * C++ Unit tests -- to run: `./build/Release/bin/unit_tests` - * Python Unit tests -- to run: `python setup.py test` + * Python Unit tests -- to run: `python setup.py test` (runs also the C++ tests above) # Examples diff --git a/appveyor.yml b/appveyor.yml index 055d381f44..017ca31bca 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -109,13 +109,7 @@ build_script: after_build: - # Run unit_tests (C++) - - cd %HTM_CORE% - - mkdir %ARTIFACTS_DIR% - - cd %HTM_CORE%\build\Release\bin - - unit_tests.exe --gtest_output=xml:%ARTIFACTS_DIR%\unit_tests_report.xml - - # Run python tests + # Run C++ & python tests - cd %HTM_CORE% - python setup.py test diff --git a/bindings/py/packaging/setup.py b/bindings/py/packaging/setup.py index 68ca515eda..d689fef008 100644 --- a/bindings/py/packaging/setup.py +++ b/bindings/py/packaging/setup.py @@ -138,6 +138,16 @@ def finalize_options(self): def run_tests(self): import pytest cwd = os.getcwd() + errno = 0 + # run c++ tests (from python) + try: + cpp_tests = os.path.join(REPO_DIR, "build", "Release", "bin", "unit_tests") + subprocess.check_call([cpp_tests]) + finally: + os.chdir(cwd) + if errno != 0: + sys.exit(errno) + # run python bindings tests (in /bindings/py/tests/) try: os.chdir(os.path.join(REPO_DIR, "bindings", "py", "tests")) From b311854d8ee27b2b579425012ccc7c80489be9fd Mon Sep 17 00:00:00 2001 From: Marek Otahal Date: Tue, 24 Sep 2019 17:11:57 +0200 Subject: [PATCH 09/67] Hotgym: bump CI time for GH Actions --- src/examples/hotgym/HelloSPTP.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/examples/hotgym/HelloSPTP.cpp b/src/examples/hotgym/HelloSPTP.cpp index 7edea90645..ac2f996531 100644 --- a/src/examples/hotgym/HelloSPTP.cpp +++ b/src/examples/hotgym/HelloSPTP.cpp @@ -238,7 +238,7 @@ EPOCHS = 2; // make test faster in Debug cout << "Total elapsed time = " << timeTotal << " seconds" << endl; if(EPOCHS >= 100) { //show only relevant values, ie don't run in valgrind (ndebug, epochs=5) run #ifndef _MSC_VER - const size_t CI_avg_time = (size_t)floor(20*Timer::getSpeed()); //sec + const size_t CI_avg_time = (size_t)floor(26*Timer::getSpeed()); //sec NTA_CHECK(timeTotal <= CI_avg_time) << //we'll see how stable the time result in CI is, if usable "HelloSPTP test slower than expected! (" << timeTotal << ",should be "<< CI_avg_time << "), speed coef.= " << Timer::getSpeed(); #endif From 839dff88b54ed514a7298ec8c9465751c807027b Mon Sep 17 00:00:00 2001 From: Marek Otahal Date: Tue, 24 Sep 2019 17:38:06 +0200 Subject: [PATCH 10/67] GH Actions: run valgrind memcheck on Linux --- .github/workflows/htmcore.yml | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/.github/workflows/htmcore.yml b/.github/workflows/htmcore.yml index eda3b0c243..5bb1309c97 100644 --- a/.github/workflows/htmcore.yml +++ b/.github/workflows/htmcore.yml @@ -20,7 +20,7 @@ jobs: #max-parallel: 4 matrix: python-version: [3.7] - os: [ubuntu-18.04, windows-2019, macOS-latest] + os: [ubuntu-18.04, windows-2019, macOS-latest, ubuntu-latest] steps: - uses: actions/checkout@v1 @@ -41,8 +41,16 @@ jobs: python setup.py install --user - name: C++ & Python Tests run: python setup.py test + - name: Memory leaks check (valgrind) + if: success() && startsWith(${{ matrix.os }}, 'ubuntu-latest') + with: + LD_LIBRARY_PATH: $LD_LIBRARY_PATH:${PWD}/build/Release/lib + run: | + apt install valgrind + valgrind --leak-check=full --show-leak-kinds=all --error-exitcode=3 ./build/Release/bin/benchmark_hotgym 5 || exit 1 + + - #TODO run valgrind #TODO build ARM64 on docker #TODO 1 Debug build #TODO GH Releases From f3ca3d9c134ea639817761678ef881aa424fd2e2 Mon Sep 17 00:00:00 2001 From: Marek Otahal Date: Tue, 24 Sep 2019 17:43:06 +0200 Subject: [PATCH 11/67] Hotgym: check for time only on Linux as in CI, other platforms run slow. --- src/examples/hotgym/HelloSPTP.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/examples/hotgym/HelloSPTP.cpp b/src/examples/hotgym/HelloSPTP.cpp index ac2f996531..141b1226e8 100644 --- a/src/examples/hotgym/HelloSPTP.cpp +++ b/src/examples/hotgym/HelloSPTP.cpp @@ -237,7 +237,7 @@ EPOCHS = 2; // make test faster in Debug const size_t timeTotal = (size_t)floor(tAll.getElapsed()); cout << "Total elapsed time = " << timeTotal << " seconds" << endl; if(EPOCHS >= 100) { //show only relevant values, ie don't run in valgrind (ndebug, epochs=5) run -#ifndef _MSC_VER +#ifdef NTA_OS_LINUX const size_t CI_avg_time = (size_t)floor(26*Timer::getSpeed()); //sec NTA_CHECK(timeTotal <= CI_avg_time) << //we'll see how stable the time result in CI is, if usable "HelloSPTP test slower than expected! (" << timeTotal << ",should be "<< CI_avg_time << "), speed coef.= " << Timer::getSpeed(); From 7c8b5940c2a127357b546204c9430a216f1bd43d Mon Sep 17 00:00:00 2001 From: Marek Otahal Date: Tue, 24 Sep 2019 17:49:25 +0200 Subject: [PATCH 12/67] fix syntax for if for CI valgrind --- .github/workflows/htmcore.yml | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/.github/workflows/htmcore.yml b/.github/workflows/htmcore.yml index 5bb1309c97..af720e1c11 100644 --- a/.github/workflows/htmcore.yml +++ b/.github/workflows/htmcore.yml @@ -42,12 +42,10 @@ jobs: - name: C++ & Python Tests run: python setup.py test - name: Memory leaks check (valgrind) - if: success() && startsWith(${{ matrix.os }}, 'ubuntu-latest') - with: - LD_LIBRARY_PATH: $LD_LIBRARY_PATH:${PWD}/build/Release/lib - run: | + if: success() && matrix.os == 'ubuntu-latest' + run: apt install valgrind - valgrind --leak-check=full --show-leak-kinds=all --error-exitcode=3 ./build/Release/bin/benchmark_hotgym 5 || exit 1 + LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:${PWD}/build/Release/lib valgrind --leak-check=full --show-leak-kinds=all --error-exitcode=3 ./build/Release/bin/benchmark_hotgym 5 || exit 1 From 4cf508713e039aba3e7023b7c829ba27d1550429 Mon Sep 17 00:00:00 2001 From: Marek Otahal Date: Tue, 24 Sep 2019 18:09:47 +0200 Subject: [PATCH 13/67] CI Actions: run one Debug build uses ubuntu-latest as platform --- .github/workflows/htmcore.yml | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/.github/workflows/htmcore.yml b/.github/workflows/htmcore.yml index af720e1c11..54ad553f91 100644 --- a/.github/workflows/htmcore.yml +++ b/.github/workflows/htmcore.yml @@ -21,6 +21,7 @@ jobs: matrix: python-version: [3.7] os: [ubuntu-18.04, windows-2019, macOS-latest, ubuntu-latest] + # ubuntu-latest is used for valgrind and Debug build steps: - uses: actions/checkout@v1 @@ -28,19 +29,32 @@ jobs: uses: actions/setup-python@v1 with: python-version: ${{ matrix.python-version }} + - name: Config run: | python --version cmake --version + - name: Install dependencies run: | python -m pip install --upgrade pip setuptools wheel pip install -r requirements.txt + - name: build htmcore with setup.py + if: matrix.os != 'ubuntu-latest' + run: | + python setup.py install --user + + - name: Debug build htmcore with setup.py + if: matrix.os == 'ubuntu-latest' + env: + CMAKE_BUILD_TYPE: Debug run: | - python setup.py install --user + python setup.py install --user + - name: C++ & Python Tests run: python setup.py test + - name: Memory leaks check (valgrind) if: success() && matrix.os == 'ubuntu-latest' run: @@ -48,8 +62,6 @@ jobs: LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:${PWD}/build/Release/lib valgrind --leak-check=full --show-leak-kinds=all --error-exitcode=3 ./build/Release/bin/benchmark_hotgym 5 || exit 1 - #TODO build ARM64 on docker - #TODO 1 Debug build #TODO GH Releases #TODO PYPI deploy From ae3ec9708d9e5b847b5c571511afae95f6c4130d Mon Sep 17 00:00:00 2001 From: Marek Otahal Date: Tue, 24 Sep 2019 21:41:28 +0200 Subject: [PATCH 14/67] fix valgrind commandline --- .github/workflows/htmcore.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/htmcore.yml b/.github/workflows/htmcore.yml index 54ad553f91..259d4ddb60 100644 --- a/.github/workflows/htmcore.yml +++ b/.github/workflows/htmcore.yml @@ -50,6 +50,7 @@ jobs: env: CMAKE_BUILD_TYPE: Debug run: | + echo "built type: ${CMAKE_BUILD_TYPE}" python setup.py install --user - name: C++ & Python Tests @@ -59,7 +60,7 @@ jobs: if: success() && matrix.os == 'ubuntu-latest' run: apt install valgrind - LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:${PWD}/build/Release/lib valgrind --leak-check=full --show-leak-kinds=all --error-exitcode=3 ./build/Release/bin/benchmark_hotgym 5 || exit 1 + LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:${PWD}/build/Release/lib valgrind --leak-check=full --show-leak-kinds=definite,indirect,possible,reachable --track-origins=yes --num-callers=40 --error-exitcode=3 ./build/Release/bin/benchmark_hotgym 5 || exit 1 #TODO build ARM64 on docker From 0a87372d6daef0aaaa03e46b3eafc7fbd99607a4 Mon Sep 17 00:00:00 2001 From: Marek Otahal Date: Wed, 25 Sep 2019 02:12:10 +0200 Subject: [PATCH 15/67] Docker: update to python3 --- Dockerfile | 31 +++++++++++++++++-------------- src/examples/hotgym/HelloSPTP.cpp | 2 +- 2 files changed, 18 insertions(+), 15 deletions(-) diff --git a/Dockerfile b/Dockerfile index 430ba31ede..3eac96ec49 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,36 +6,39 @@ ARG arch=amd64 # Multiarch Debian 10 Buster (amd64, arm64, etc). # https://hub.docker.com/r/multiarch/debian-debootstrap -FROM multiarch/debian-debootstrap:$arch-buster +FROM multiarch/debian-debootstrap:${arch}-buster RUN apt-get update -RUN apt-get install -y \ +RUN apt-get install -y --no-install-suggests \ cmake \ - g++ \ + g++-8 \ git-core \ libyaml-dev \ - python \ - python-dev \ - python-numpy \ - python-pip + python3-minimal \ + python3-dev \ + python3-numpy \ + python3-pip \ + python3-venv ADD . /usr/local/src/htm.core WORKDIR /usr/local/src/htm.core +# Setup py env +#RUN python3 -m venv pyenv && . pyenv/bin/activate +RUN pip3 install --upgrade setuptools pip wheel +#RUN export PYTHONPATH=$PYTHONPATH:/usr/local/lib/python3.7/dist-packages + # Install -RUN pip install --upgrade setuptools -RUN pip install wheel -RUN pip install \ +RUN pip3 install \ # Explicitly specify --cache-dir, --build, and --no-clean so that build # artifacts may be extracted from the container later. Final built python # packages can be found in /usr/local/src/htm.core/bindings/py/dist # --cache-dir /usr/local/src/htm.core/pip-cache \ # --build /usr/local/src/htm.core/pip-build \ # --no-clean \ - -r bindings/py/packaging/requirements.txt -RUN python setup.py install + -r requirements.txt +RUN python3 setup.py install --force # Test -RUN ./build/Release/bin/unit_tests -RUN python setup.py test +RUN python3 setup.py test diff --git a/src/examples/hotgym/HelloSPTP.cpp b/src/examples/hotgym/HelloSPTP.cpp index 141b1226e8..9b7cc90b55 100644 --- a/src/examples/hotgym/HelloSPTP.cpp +++ b/src/examples/hotgym/HelloSPTP.cpp @@ -238,7 +238,7 @@ EPOCHS = 2; // make test faster in Debug cout << "Total elapsed time = " << timeTotal << " seconds" << endl; if(EPOCHS >= 100) { //show only relevant values, ie don't run in valgrind (ndebug, epochs=5) run #ifdef NTA_OS_LINUX - const size_t CI_avg_time = (size_t)floor(26*Timer::getSpeed()); //sec + const size_t CI_avg_time = (size_t)floor(99*Timer::getSpeed()); //sec //FIXME the CI speed broken for docker linux NTA_CHECK(timeTotal <= CI_avg_time) << //we'll see how stable the time result in CI is, if usable "HelloSPTP test slower than expected! (" << timeTotal << ",should be "<< CI_avg_time << "), speed coef.= " << Timer::getSpeed(); #endif From 7ae9903c274f0d783ce0296fb958a1534a43659e Mon Sep 17 00:00:00 2001 From: Marek Otahal Date: Wed, 25 Sep 2019 03:05:02 +0200 Subject: [PATCH 16/67] GH Actions: docker ARM64 build WIP --- .github/workflows/htmcore.yml | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/.github/workflows/htmcore.yml b/.github/workflows/htmcore.yml index 259d4ddb60..76cd415287 100644 --- a/.github/workflows/htmcore.yml +++ b/.github/workflows/htmcore.yml @@ -20,10 +20,19 @@ jobs: #max-parallel: 4 matrix: python-version: [3.7] - os: [ubuntu-18.04, windows-2019, macOS-latest, ubuntu-latest] + os: [ubuntu-18.04, windows-2019, macOS-latest, ubuntu-latest, ubuntu-16.04] # ubuntu-latest is used for valgrind and Debug build + # ubuntu-16.04 is used for docker build for ARM64 steps: + - name: Docker image for ARM + if: matrix.os == 'ubuntu-16.04' # ubuntu->QEMU->docker->ARM64 build + run: | + apt-get install -y qemu-system docker.io + docker run --rm --privileged multiarch/qemu-user-static:register --reset + uname -a + docker build -t htm-arm64-docker --build-arg arch=arm64 . + - uses: actions/checkout@v1 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v1 From 750db071d2b619651842b681e812e2b893cd5a01 Mon Sep 17 00:00:00 2001 From: Marek Otahal Date: Wed, 25 Sep 2019 03:09:58 +0200 Subject: [PATCH 17/67] Actions: fix order --- .github/workflows/htmcore.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/htmcore.yml b/.github/workflows/htmcore.yml index 76cd415287..d6be083e69 100644 --- a/.github/workflows/htmcore.yml +++ b/.github/workflows/htmcore.yml @@ -25,6 +25,8 @@ jobs: # ubuntu-16.04 is used for docker build for ARM64 steps: + - uses: actions/checkout@v1 + - name: Docker image for ARM if: matrix.os == 'ubuntu-16.04' # ubuntu->QEMU->docker->ARM64 build run: | @@ -33,7 +35,6 @@ jobs: uname -a docker build -t htm-arm64-docker --build-arg arch=arm64 . - - uses: actions/checkout@v1 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v1 with: From 7102d56a84c09f250019907305a00cdd31799974 Mon Sep 17 00:00:00 2001 From: Marek Otahal Date: Wed, 25 Sep 2019 03:12:43 +0200 Subject: [PATCH 18/67] Revert "Actions: fix order" This reverts commit 750db071d2b619651842b681e812e2b893cd5a01. --- .github/workflows/htmcore.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/htmcore.yml b/.github/workflows/htmcore.yml index d6be083e69..76cd415287 100644 --- a/.github/workflows/htmcore.yml +++ b/.github/workflows/htmcore.yml @@ -25,8 +25,6 @@ jobs: # ubuntu-16.04 is used for docker build for ARM64 steps: - - uses: actions/checkout@v1 - - name: Docker image for ARM if: matrix.os == 'ubuntu-16.04' # ubuntu->QEMU->docker->ARM64 build run: | @@ -35,6 +33,7 @@ jobs: uname -a docker build -t htm-arm64-docker --build-arg arch=arm64 . + - uses: actions/checkout@v1 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v1 with: From 2fbd707515ba47a4741a944796f97b68860fe63a Mon Sep 17 00:00:00 2001 From: Marek Otahal Date: Wed, 25 Sep 2019 03:13:21 +0200 Subject: [PATCH 19/67] Docker: try sudo --- .github/workflows/htmcore.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/htmcore.yml b/.github/workflows/htmcore.yml index 76cd415287..cec103da76 100644 --- a/.github/workflows/htmcore.yml +++ b/.github/workflows/htmcore.yml @@ -28,10 +28,10 @@ jobs: - name: Docker image for ARM if: matrix.os == 'ubuntu-16.04' # ubuntu->QEMU->docker->ARM64 build run: | - apt-get install -y qemu-system docker.io - docker run --rm --privileged multiarch/qemu-user-static:register --reset + sudo apt-get install -y qemu-system docker.io + sudo docker run --rm --privileged multiarch/qemu-user-static:register --reset uname -a - docker build -t htm-arm64-docker --build-arg arch=arm64 . + sudo docker build -t htm-arm64-docker --build-arg arch=arm64 . - uses: actions/checkout@v1 - name: Set up Python ${{ matrix.python-version }} From 63f01c652582824e886a347322fb344b680e95b4 Mon Sep 17 00:00:00 2001 From: Marek Otahal Date: Wed, 25 Sep 2019 03:21:18 +0200 Subject: [PATCH 20/67] GH Actions: try get docker running --- .github/workflows/htmcore.yml | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/.github/workflows/htmcore.yml b/.github/workflows/htmcore.yml index cec103da76..9d248bfe51 100644 --- a/.github/workflows/htmcore.yml +++ b/.github/workflows/htmcore.yml @@ -21,19 +21,25 @@ jobs: matrix: python-version: [3.7] os: [ubuntu-18.04, windows-2019, macOS-latest, ubuntu-latest, ubuntu-16.04] - # ubuntu-latest is used for valgrind and Debug build - # ubuntu-16.04 is used for docker build for ARM64 + # ubuntu-16.04 is used for valgrind and Debug build + # ubuntu-latest is used for docker build for ARM64 steps: - name: Docker image for ARM - if: matrix.os == 'ubuntu-16.04' # ubuntu->QEMU->docker->ARM64 build + if: matrix.os == 'ubuntu-latest' # ubuntu->QEMU->docker->ARM64 build run: | + sudo apt-get remove --purge -y moby-engine moby-cli sudo apt-get install -y qemu-system docker.io sudo docker run --rm --privileged multiarch/qemu-user-static:register --reset uname -a - sudo docker build -t htm-arm64-docker --build-arg arch=arm64 . - uses: actions/checkout@v1 + + - name: Docker ARM64 build + if: matrix.os == 'ubuntu-latest' + run: | + sudo docker build -t htm-arm64-docker --build-arg arch=arm64 . + - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v1 with: @@ -50,12 +56,12 @@ jobs: pip install -r requirements.txt - name: build htmcore with setup.py - if: matrix.os != 'ubuntu-latest' + if: matrix.os != 'ubuntu-16.04' run: | python setup.py install --user - name: Debug build htmcore with setup.py - if: matrix.os == 'ubuntu-latest' + if: matrix.os == 'ubuntu-16.04' env: CMAKE_BUILD_TYPE: Debug run: | From 687951390452888a3156b166db82814a1486da73 Mon Sep 17 00:00:00 2001 From: Marek Otahal Date: Wed, 25 Sep 2019 04:01:57 +0200 Subject: [PATCH 21/67] try docker on ubuntu 16.04 --- .github/workflows/htmcore.yml | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/.github/workflows/htmcore.yml b/.github/workflows/htmcore.yml index 9d248bfe51..4c8621767f 100644 --- a/.github/workflows/htmcore.yml +++ b/.github/workflows/htmcore.yml @@ -21,24 +21,29 @@ jobs: matrix: python-version: [3.7] os: [ubuntu-18.04, windows-2019, macOS-latest, ubuntu-latest, ubuntu-16.04] - # ubuntu-16.04 is used for valgrind and Debug build - # ubuntu-latest is used for docker build for ARM64 + # ubuntu-latest is used for valgrind and Debug build + # ubuntu-16.04 is used for docker build for ARM64 steps: - - name: Docker image for ARM - if: matrix.os == 'ubuntu-latest' # ubuntu->QEMU->docker->ARM64 build + - name: Docker ARM64 build 1/2 - image + if: matrix.os == 'ubuntu-16.04' # ubuntu->QEMU->docker->ARM64 build run: | - sudo apt-get remove --purge -y moby-engine moby-cli + # bug in ubuntu, conflicts with docker.io + sudo apt-get update + sudo apt-get remove --purge -y moby-engine moby-cli sudo apt-get install -y qemu-system docker.io + uname -a sudo docker run --rm --privileged multiarch/qemu-user-static:register --reset uname -a - uses: actions/checkout@v1 - - name: Docker ARM64 build - if: matrix.os == 'ubuntu-latest' + - name: Docker ARM64 build 2/2 - build + if: matrix.os == 'ubuntu-16.04' run: | sudo docker build -t htm-arm64-docker --build-arg arch=arm64 . + uname -a + exit 0 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v1 @@ -56,12 +61,12 @@ jobs: pip install -r requirements.txt - name: build htmcore with setup.py - if: matrix.os != 'ubuntu-16.04' + if: matrix.os != 'ubuntu-latest' run: | python setup.py install --user - name: Debug build htmcore with setup.py - if: matrix.os == 'ubuntu-16.04' + if: matrix.os == 'ubuntu-latest' env: CMAKE_BUILD_TYPE: Debug run: | From f717aaaa7692a78f9987c0c647bcfd90699da511 Mon Sep 17 00:00:00 2001 From: Marek Otahal Date: Wed, 25 Sep 2019 05:13:03 +0200 Subject: [PATCH 22/67] GH Actions: ARM as a separate job --- .github/workflows/htmcore.yml | 45 +++++++++++++++++------------------ 1 file changed, 22 insertions(+), 23 deletions(-) diff --git a/.github/workflows/htmcore.yml b/.github/workflows/htmcore.yml index 4c8621767f..2bbaa0b35c 100644 --- a/.github/workflows/htmcore.yml +++ b/.github/workflows/htmcore.yml @@ -20,31 +20,12 @@ jobs: #max-parallel: 4 matrix: python-version: [3.7] - os: [ubuntu-18.04, windows-2019, macOS-latest, ubuntu-latest, ubuntu-16.04] + os: [ubuntu-18.04, windows-2019, macOS-latest, ubuntu-latest] # ubuntu-latest is used for valgrind and Debug build - # ubuntu-16.04 is used for docker build for ARM64 steps: - - name: Docker ARM64 build 1/2 - image - if: matrix.os == 'ubuntu-16.04' # ubuntu->QEMU->docker->ARM64 build - run: | - # bug in ubuntu, conflicts with docker.io - sudo apt-get update - sudo apt-get remove --purge -y moby-engine moby-cli - sudo apt-get install -y qemu-system docker.io - uname -a - sudo docker run --rm --privileged multiarch/qemu-user-static:register --reset - uname -a - - uses: actions/checkout@v1 - - name: Docker ARM64 build 2/2 - build - if: matrix.os == 'ubuntu-16.04' - run: | - sudo docker build -t htm-arm64-docker --build-arg arch=arm64 . - uname -a - exit 0 - - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v1 with: @@ -79,10 +60,28 @@ jobs: - name: Memory leaks check (valgrind) if: success() && matrix.os == 'ubuntu-latest' run: - apt install valgrind - LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:${PWD}/build/Release/lib valgrind --leak-check=full --show-leak-kinds=definite,indirect,possible,reachable --track-origins=yes --num-callers=40 --error-exitcode=3 ./build/Release/bin/benchmark_hotgym 5 || exit 1 + sudo apt-get -y install valgrind + LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:${PWD}/build/Release/lib valgrind --show-leak-kinds=definite,indirect,possible,reachable --track-origins=yes --num-callers=40 --error-exitcode=3 ./build/Release/bin/benchmark_hotgym 5 || exit 1 + build-arm64-docker: + name: Build for ARM64 on Docker + runs_on: ubuntu-18.04 + steps: + - name: Docker ARM64 image + run: | + # bug in ubuntu, conflicts with docker.io + sudo apt-get update + sudo apt-get remove --purge -y moby-engine moby-cli + sudo apt-get install -y qemu-system docker.io + uname -a + sudo docker run --rm --privileged multiarch/qemu-user-static:register --reset + uname -a + + - uses: actions/checkout@v1 + - name: ARM64 build + run: | + sudo docker build -t htm-arm64-docker --build-arg arch=arm64 . + uname -a - #TODO build ARM64 on docker #TODO GH Releases #TODO PYPI deploy From 0a2536d8213fc4fcc1f89adf48056d65d3c99615 Mon Sep 17 00:00:00 2001 From: Marek Otahal Date: Wed, 25 Sep 2019 05:19:42 +0200 Subject: [PATCH 23/67] setup.py: honor CMAKE_BUILD_TYPE if set, use it for build_type in setup.py --- bindings/py/packaging/setup.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bindings/py/packaging/setup.py b/bindings/py/packaging/setup.py index d689fef008..39e53a0da6 100644 --- a/bindings/py/packaging/setup.py +++ b/bindings/py/packaging/setup.py @@ -47,7 +47,8 @@ build_type = 'Release' - +if os.environ['CMAKE_BUILD_TYPE'] == 'Debug': + build_type = 'Debug' PY_BINDINGS = os.path.dirname(os.path.realpath(__file__)) REPO_DIR = os.path.abspath(os.path.join(PY_BINDINGS, os.pardir, os.pardir, os.pardir)) From d663d19b482c2943a3cbb9d169cc6d05cda1b96b Mon Sep 17 00:00:00 2001 From: Marek Otahal Date: Wed, 25 Sep 2019 05:22:18 +0200 Subject: [PATCH 24/67] typo --- .github/workflows/htmcore.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/htmcore.yml b/.github/workflows/htmcore.yml index 2bbaa0b35c..846b71af34 100644 --- a/.github/workflows/htmcore.yml +++ b/.github/workflows/htmcore.yml @@ -65,7 +65,7 @@ jobs: build-arm64-docker: name: Build for ARM64 on Docker - runs_on: ubuntu-18.04 + runs-on: ubuntu-18.04 steps: - name: Docker ARM64 image run: | From 530343c03c424f66a407cfdaf5ff5c91885d9a63 Mon Sep 17 00:00:00 2001 From: Marek Otahal Date: Wed, 25 Sep 2019 05:27:02 +0200 Subject: [PATCH 25/67] setup: dont fail if var not set --- bindings/py/packaging/setup.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/bindings/py/packaging/setup.py b/bindings/py/packaging/setup.py index 39e53a0da6..1fca88cbb6 100644 --- a/bindings/py/packaging/setup.py +++ b/bindings/py/packaging/setup.py @@ -47,8 +47,11 @@ build_type = 'Release' -if os.environ['CMAKE_BUILD_TYPE'] == 'Debug': - build_type = 'Debug' +try: + if os.environ['CMAKE_BUILD_TYPE'] == 'Debug': + build_type = 'Debug' +except: + build_type = 'Release' PY_BINDINGS = os.path.dirname(os.path.realpath(__file__)) REPO_DIR = os.path.abspath(os.path.join(PY_BINDINGS, os.pardir, os.pardir, os.pardir)) From e725466d6ce7b5048dd702b14b0e86ecc6eacfba Mon Sep 17 00:00:00 2001 From: Marek Otahal Date: Wed, 25 Sep 2019 11:29:54 +0200 Subject: [PATCH 26/67] GH Actions: set gcc-8 for ubuntu 18.04 --- .github/workflows/htmcore.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/htmcore.yml b/.github/workflows/htmcore.yml index 846b71af34..639872698d 100644 --- a/.github/workflows/htmcore.yml +++ b/.github/workflows/htmcore.yml @@ -31,10 +31,18 @@ jobs: with: python-version: ${{ matrix.python-version }} + -name: Set up C++ compilers + if: matrix.os == 'ubuntu-18.04' + run: | + sudo apt-get update + sudo apt-get -y install gcc-8 g++-8 + sudo apt-get remove -y gcc-7 g++-7 + - name: Config run: | python --version cmake --version + g++ --version - name: Install dependencies run: | From d19f0c8a04404672ac7efe5f2e0ea1fd8925e067 Mon Sep 17 00:00:00 2001 From: Marek Otahal Date: Wed, 25 Sep 2019 12:55:20 +0200 Subject: [PATCH 27/67] setup.py: split requirements to install, extras extras are installed only optionally. This is needed for ARM build where some deps are not found on PYPI. --- .github/workflows/htmcore.yml | 7 ++----- bindings/py/packaging/setup.py | 17 ++++++++++++----- requirements.txt | 7 +++---- 3 files changed, 17 insertions(+), 14 deletions(-) diff --git a/.github/workflows/htmcore.yml b/.github/workflows/htmcore.yml index 639872698d..b02d5ae68e 100644 --- a/.github/workflows/htmcore.yml +++ b/.github/workflows/htmcore.yml @@ -52,15 +52,12 @@ jobs: - name: build htmcore with setup.py if: matrix.os != 'ubuntu-latest' run: | - python setup.py install --user + python setup.py install --user --force - name: Debug build htmcore with setup.py if: matrix.os == 'ubuntu-latest' - env: - CMAKE_BUILD_TYPE: Debug run: | - echo "built type: ${CMAKE_BUILD_TYPE}" - python setup.py install --user + CMAKE_BUILD_TYPE=Debug python setup.py install --user --force - name: C++ & Python Tests run: python setup.py test diff --git a/bindings/py/packaging/setup.py b/bindings/py/packaging/setup.py index 1fca88cbb6..ce148cae7d 100644 --- a/bindings/py/packaging/setup.py +++ b/bindings/py/packaging/setup.py @@ -110,12 +110,12 @@ def fixPath(path): -def findRequirements(platform): +def findRequirements(platform, fileName="requirements.txt"): """ Read the requirements.txt file and parse into requirements for setup's install_requirements option. """ - requirementsPath = fixPath(os.path.join(REPO_DIR, "requirements.txt")) + requirementsPath = fixPath(os.path.join(REPO_DIR, fileName)) return [ line.strip() for line in open(requirementsPath).readlines() @@ -145,7 +145,7 @@ def run_tests(self): errno = 0 # run c++ tests (from python) try: - cpp_tests = os.path.join(REPO_DIR, "build", "Release", "bin", "unit_tests") + cpp_tests = os.path.join(REPO_DIR, "build", build_dir, "bin", "unit_tests") subprocess.check_call([cpp_tests]) finally: os.chdir(cwd) @@ -330,9 +330,9 @@ def generateExtensions(platform, build_type): """ set the default directory to the distr, and package it. """ + print("\nbindings/py/setup.py: Setup htm.core Python module in " + DISTR_DIR+ "\n") os.chdir(DISTR_DIR) - print("\nbindings/py/setup.py: Setup htm.core Python module in " + DISTR_DIR+ "\n") setup( # See https://docs.python.org/2/distutils/apiref.html for descriptions of arguments. # https://docs.python.org/2/distutils/setupscript.html @@ -353,7 +353,14 @@ def generateExtensions(platform, build_type): "htm.bindings": ["*.so", "*.pyd"], "htm.examples": ["*.csv"], }, - extras_require = {}, + #install extras by `pip install htm.core[examples]` + extras_require={'prettytable>=0.7.2':'monitor-mixin', + 'scikit-image>0.15.0':'examples', + 'sklearn':'examples', + 'matplotlib':'examples', + 'PIL':'examples', + 'scipy':'examples' + }, zip_safe=False, cmdclass={ "clean": CleanCommand, diff --git a/requirements.txt b/requirements.txt index c79806c3c4..ae93b0cc07 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,10 +2,9 @@ ## for python bindings (in /bindings/py/) numpy>=1.15 pytest==4.6.5 #4.6.x series is last to support python2, once py2 dropped, we can switch to 5.x -pytest-cov>=2.5.0 ## for python code (in /py/) hexy>=1.4.3 # for grid cell encoder mock>=1.0.1 # for anomaly likelihood test -prettytable>=0.7.2 # For htm.advanced metric class mixins -scikit-image>=0.15.0 # For thalamus example -sklearn # for mnist.py example, dataset downloads from openML.org +## optional dependencies, such as for visualizations, running examples +# should be placed in setup.py section extras_require. Install those by +# pip install htm.core[examples] and/or htm.core[monitor-mixin] From 9aa6877a8bdefc89b247078ca99c1d7fe648248e Mon Sep 17 00:00:00 2001 From: Marek Otahal Date: Wed, 25 Sep 2019 13:08:45 +0200 Subject: [PATCH 28/67] fixing syntax --- .github/workflows/htmcore.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/htmcore.yml b/.github/workflows/htmcore.yml index b02d5ae68e..639872698d 100644 --- a/.github/workflows/htmcore.yml +++ b/.github/workflows/htmcore.yml @@ -52,12 +52,15 @@ jobs: - name: build htmcore with setup.py if: matrix.os != 'ubuntu-latest' run: | - python setup.py install --user --force + python setup.py install --user - name: Debug build htmcore with setup.py if: matrix.os == 'ubuntu-latest' + env: + CMAKE_BUILD_TYPE: Debug run: | - CMAKE_BUILD_TYPE=Debug python setup.py install --user --force + echo "built type: ${CMAKE_BUILD_TYPE}" + python setup.py install --user - name: C++ & Python Tests run: python setup.py test From 97c84816abd1ab9b1ad04b51523ec979df5241ba Mon Sep 17 00:00:00 2001 From: Marek Otahal Date: Wed, 25 Sep 2019 13:52:34 +0200 Subject: [PATCH 29/67] GH Actions: syntax, ++ cannot be in YAML, as in c++ --- .github/workflows/htmcore.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/htmcore.yml b/.github/workflows/htmcore.yml index 639872698d..785a55bd10 100644 --- a/.github/workflows/htmcore.yml +++ b/.github/workflows/htmcore.yml @@ -31,12 +31,12 @@ jobs: with: python-version: ${{ matrix.python-version }} - -name: Set up C++ compilers - if: matrix.os == 'ubuntu-18.04' - run: | - sudo apt-get update - sudo apt-get -y install gcc-8 g++-8 - sudo apt-get remove -y gcc-7 g++-7 + - name: Set up Cpp compilers + if: matrix.os == 'ubuntu-18.04' + run: | + sudo apt-get update + sudo apt-get -y install gcc-8 g++-8 + sudo apt-get remove -y gcc-7 g++-7 - name: Config run: | From 95c100ac1555c0428ac1101320af2560fe2c5b8f Mon Sep 17 00:00:00 2001 From: Marek Otahal Date: Wed, 25 Sep 2019 14:01:14 +0200 Subject: [PATCH 30/67] GH Actions: set links for gcc to gcc-8 --- .github/workflows/htmcore.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/htmcore.yml b/.github/workflows/htmcore.yml index 785a55bd10..440b9fb048 100644 --- a/.github/workflows/htmcore.yml +++ b/.github/workflows/htmcore.yml @@ -37,6 +37,8 @@ jobs: sudo apt-get update sudo apt-get -y install gcc-8 g++-8 sudo apt-get remove -y gcc-7 g++-7 + sudo ln -s `which g++-8` /usr/local/bin/g++ + sudo ln -s `which gcc-8` /usr/local/bin/gcc - name: Config run: | From 440b713b451b9993ec5f38039c52a2b422cd6d21 Mon Sep 17 00:00:00 2001 From: Marek Otahal Date: Wed, 25 Sep 2019 15:56:40 +0200 Subject: [PATCH 31/67] fix Debug build, cannot use setup.py --- .github/workflows/htmcore.yml | 46 ++++++++++++++++++++++++++++++---- bindings/py/packaging/setup.py | 8 +++--- 2 files changed, 44 insertions(+), 10 deletions(-) diff --git a/.github/workflows/htmcore.yml b/.github/workflows/htmcore.yml index 440b9fb048..83603ee0dd 100644 --- a/.github/workflows/htmcore.yml +++ b/.github/workflows/htmcore.yml @@ -20,8 +20,7 @@ jobs: #max-parallel: 4 matrix: python-version: [3.7] - os: [ubuntu-18.04, windows-2019, macOS-latest, ubuntu-latest] - # ubuntu-latest is used for valgrind and Debug build + os: [ubuntu-18.04, windows-2019, macOS-latest] steps: - uses: actions/checkout@v1 @@ -52,9 +51,7 @@ jobs: pip install -r requirements.txt - name: build htmcore with setup.py - if: matrix.os != 'ubuntu-latest' - run: | - python setup.py install --user + run: python setup.py install --user - name: Debug build htmcore with setup.py if: matrix.os == 'ubuntu-latest' @@ -93,5 +90,44 @@ jobs: sudo docker build -t htm-arm64-docker --build-arg arch=arm64 . uname -a + build-debug: + name: Build and test in Debug mode + runs-on: ubuntu-18.04 + steps: + - uses: actions/checkout@v1 + + - name: Set up Cpp compilers + if: matrix.os == 'ubuntu-18.04' + run: | + sudo apt-get update + sudo apt-get -y install gcc-8 g++-8 + sudo apt-get remove -y gcc-7 g++-7 + sudo ln -s `which g++-8` /usr/local/bin/g++ + sudo ln -s `which gcc-8` /usr/local/bin/gcc + + - name: Config + run: | + cmake --version + g++ --version + + - name: Debug build htmcore with cmake + run: | + echo "built type: ${CMAKE_BUILD_TYPE}" + mkdir -p build/scripts + cd build/scripts + cmake ../.. -DCMAKE_BUILD_TYPE=Debug + make -j4 && make install + + - name: C++ Tests + run: | + cd build/scripts + ../Debug/bin/unit_tests + + - name: Memory leaks check (valgrind) + run: + sudo apt-get -y install valgrind + LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:${PWD}/build/Release/lib valgrind --show-leak-kinds=definite,indirect,possible,reachable --track-origins=yes --num-callers=40 --error-exitcode=3 ./build/Release/bin/benchmark_hotgym 5 || exit 1 + + #TODO GH Releases #TODO PYPI deploy diff --git a/bindings/py/packaging/setup.py b/bindings/py/packaging/setup.py index ce148cae7d..d09b4c522d 100644 --- a/bindings/py/packaging/setup.py +++ b/bindings/py/packaging/setup.py @@ -46,12 +46,10 @@ # +# bindings cannot be compiled in Debug mode, unless a special python library also in +# Debug is used, which is quite unlikely. So for any CMAKE_BUILD_TYPE setting, override +# to Release mode. build_type = 'Release' -try: - if os.environ['CMAKE_BUILD_TYPE'] == 'Debug': - build_type = 'Debug' -except: - build_type = 'Release' PY_BINDINGS = os.path.dirname(os.path.realpath(__file__)) REPO_DIR = os.path.abspath(os.path.join(PY_BINDINGS, os.pardir, os.pardir, os.pardir)) From ccd968743254eac2b2dad0542a4839e78e399a2b Mon Sep 17 00:00:00 2001 From: Marek Otahal Date: Wed, 25 Sep 2019 16:54:09 +0200 Subject: [PATCH 32/67] GH Actions: fix Debug build, see if gcc-8 is fixed --- .github/workflows/htmcore.yml | 23 ++++------------------- 1 file changed, 4 insertions(+), 19 deletions(-) diff --git a/.github/workflows/htmcore.yml b/.github/workflows/htmcore.yml index 83603ee0dd..5e9d1b1122 100644 --- a/.github/workflows/htmcore.yml +++ b/.github/workflows/htmcore.yml @@ -36,8 +36,7 @@ jobs: sudo apt-get update sudo apt-get -y install gcc-8 g++-8 sudo apt-get remove -y gcc-7 g++-7 - sudo ln -s `which g++-8` /usr/local/bin/g++ - sudo ln -s `which gcc-8` /usr/local/bin/gcc + sudo update-alternatives /usr/bin/gcc gcc `which gcc-8` --slave /usr/bin/g++ g++ `which g++-8` - name: Config run: | @@ -53,22 +52,9 @@ jobs: - name: build htmcore with setup.py run: python setup.py install --user - - name: Debug build htmcore with setup.py - if: matrix.os == 'ubuntu-latest' - env: - CMAKE_BUILD_TYPE: Debug - run: | - echo "built type: ${CMAKE_BUILD_TYPE}" - python setup.py install --user - - name: C++ & Python Tests run: python setup.py test - - name: Memory leaks check (valgrind) - if: success() && matrix.os == 'ubuntu-latest' - run: - sudo apt-get -y install valgrind - LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:${PWD}/build/Release/lib valgrind --show-leak-kinds=definite,indirect,possible,reachable --track-origins=yes --num-callers=40 --error-exitcode=3 ./build/Release/bin/benchmark_hotgym 5 || exit 1 build-arm64-docker: name: Build for ARM64 on Docker @@ -97,13 +83,9 @@ jobs: - uses: actions/checkout@v1 - name: Set up Cpp compilers - if: matrix.os == 'ubuntu-18.04' run: | sudo apt-get update sudo apt-get -y install gcc-8 g++-8 - sudo apt-get remove -y gcc-7 g++-7 - sudo ln -s `which g++-8` /usr/local/bin/g++ - sudo ln -s `which gcc-8` /usr/local/bin/gcc - name: Config run: | @@ -111,6 +93,9 @@ jobs: g++ --version - name: Debug build htmcore with cmake + env: + CC: gcc-8 + CXX: g++-8 run: | echo "built type: ${CMAKE_BUILD_TYPE}" mkdir -p build/scripts From 43628c43c5e15f1f8bdcdc01214eb2ee88fc8334 Mon Sep 17 00:00:00 2001 From: Marek Otahal Date: Wed, 25 Sep 2019 17:08:15 +0200 Subject: [PATCH 33/67] fix update-alternatives --- .github/workflows/htmcore.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/htmcore.yml b/.github/workflows/htmcore.yml index 5e9d1b1122..3b0f3901f8 100644 --- a/.github/workflows/htmcore.yml +++ b/.github/workflows/htmcore.yml @@ -36,13 +36,16 @@ jobs: sudo apt-get update sudo apt-get -y install gcc-8 g++-8 sudo apt-get remove -y gcc-7 g++-7 - sudo update-alternatives /usr/bin/gcc gcc `which gcc-8` --slave /usr/bin/g++ g++ `which g++-8` + sudo update-alternatives --install /usr/bin/gcc gcc `which gcc-8` 9999 --slave /usr/bin/g++ g++ `which g++-8` + sudo update-alternatives --install /usr/bin/cc cc gcc 9999 --slave /usr/bin/c++ c++ g++ - name: Config run: | python --version cmake --version g++ --version + sudo update-alternatives --list cc + sudo update-alternatives --list gcc - name: Install dependencies run: | From 6b1ad364cd37eaad407ce3d4266af6f8268c3525 Mon Sep 17 00:00:00 2001 From: Marek Otahal Date: Wed, 25 Sep 2019 17:52:44 +0200 Subject: [PATCH 34/67] requirements: prettytable required for tests if monitor mixin --- bindings/py/packaging/setup.py | 3 +-- requirements.txt | 1 + 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/bindings/py/packaging/setup.py b/bindings/py/packaging/setup.py index d09b4c522d..c57be3ea92 100644 --- a/bindings/py/packaging/setup.py +++ b/bindings/py/packaging/setup.py @@ -352,8 +352,7 @@ def generateExtensions(platform, build_type): "htm.examples": ["*.csv"], }, #install extras by `pip install htm.core[examples]` - extras_require={'prettytable>=0.7.2':'monitor-mixin', - 'scikit-image>0.15.0':'examples', + extras_require={'scikit-image>0.15.0':'examples', 'sklearn':'examples', 'matplotlib':'examples', 'PIL':'examples', diff --git a/requirements.txt b/requirements.txt index ae93b0cc07..c5c715ff85 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,6 +5,7 @@ pytest==4.6.5 #4.6.x series is last to support python2, once py2 dropped, we can ## for python code (in /py/) hexy>=1.4.3 # for grid cell encoder mock>=1.0.1 # for anomaly likelihood test +prettytable>=0.7.2 # for monitor-mixin in htm.advanced (+its tests) ## optional dependencies, such as for visualizations, running examples # should be placed in setup.py section extras_require. Install those by # pip install htm.core[examples] and/or htm.core[monitor-mixin] From b00e53719b3e0b6121d6ee51229090c87426f9a0 Mon Sep 17 00:00:00 2001 From: Marek Otahal Date: Wed, 25 Sep 2019 17:54:29 +0200 Subject: [PATCH 35/67] fix configure gcc cc --- .github/workflows/htmcore.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/htmcore.yml b/.github/workflows/htmcore.yml index 3b0f3901f8..b03272ef82 100644 --- a/.github/workflows/htmcore.yml +++ b/.github/workflows/htmcore.yml @@ -43,9 +43,7 @@ jobs: run: | python --version cmake --version - g++ --version - sudo update-alternatives --list cc - sudo update-alternatives --list gcc + c++ --version - name: Install dependencies run: | From a934eda1d7817866e48334129bbf1cc6b1467cd3 Mon Sep 17 00:00:00 2001 From: Marek Otahal Date: Wed, 25 Sep 2019 17:57:57 +0200 Subject: [PATCH 36/67] update alt --- .github/workflows/htmcore.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/htmcore.yml b/.github/workflows/htmcore.yml index b03272ef82..769076f35d 100644 --- a/.github/workflows/htmcore.yml +++ b/.github/workflows/htmcore.yml @@ -37,7 +37,7 @@ jobs: sudo apt-get -y install gcc-8 g++-8 sudo apt-get remove -y gcc-7 g++-7 sudo update-alternatives --install /usr/bin/gcc gcc `which gcc-8` 9999 --slave /usr/bin/g++ g++ `which g++-8` - sudo update-alternatives --install /usr/bin/cc cc gcc 9999 --slave /usr/bin/c++ c++ g++ + sudo update-alternatives --install /usr/bin/cc cc `which gcc` 9999 --slave /usr/bin/c++ c++ `which g++` - name: Config run: | From 12448e032e5dc2b99a5369c4d84c60c10bc1b095 Mon Sep 17 00:00:00 2001 From: Marek Otahal Date: Wed, 25 Sep 2019 18:06:26 +0200 Subject: [PATCH 37/67] gcc-8 try 999 --- .github/workflows/htmcore.yml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/workflows/htmcore.yml b/.github/workflows/htmcore.yml index 769076f35d..2a2b042fce 100644 --- a/.github/workflows/htmcore.yml +++ b/.github/workflows/htmcore.yml @@ -35,9 +35,8 @@ jobs: run: | sudo apt-get update sudo apt-get -y install gcc-8 g++-8 - sudo apt-get remove -y gcc-7 g++-7 - sudo update-alternatives --install /usr/bin/gcc gcc `which gcc-8` 9999 --slave /usr/bin/g++ g++ `which g++-8` - sudo update-alternatives --install /usr/bin/cc cc `which gcc` 9999 --slave /usr/bin/c++ c++ `which g++` + echo "export CC=gcc-8" >> ~/.bashrc + echo "export CXX=g++-8" >> ~/.bashrc - name: Config run: | @@ -45,6 +44,10 @@ jobs: cmake --version c++ --version + - name: gcc-8 check + if: matrix.os == 'ubuntu-18.04' + run: echo "CC= $CC" + - name: Install dependencies run: | python -m pip install --upgrade pip setuptools wheel From 08c8cb93562d10623e5226b15912456acad625c3 Mon Sep 17 00:00:00 2001 From: Marek Otahal Date: Wed, 25 Sep 2019 18:09:53 +0200 Subject: [PATCH 38/67] Revert "gcc-8 try 999" This reverts commit 12448e032e5dc2b99a5369c4d84c60c10bc1b095. --- .github/workflows/htmcore.yml | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/.github/workflows/htmcore.yml b/.github/workflows/htmcore.yml index 2a2b042fce..769076f35d 100644 --- a/.github/workflows/htmcore.yml +++ b/.github/workflows/htmcore.yml @@ -35,8 +35,9 @@ jobs: run: | sudo apt-get update sudo apt-get -y install gcc-8 g++-8 - echo "export CC=gcc-8" >> ~/.bashrc - echo "export CXX=g++-8" >> ~/.bashrc + sudo apt-get remove -y gcc-7 g++-7 + sudo update-alternatives --install /usr/bin/gcc gcc `which gcc-8` 9999 --slave /usr/bin/g++ g++ `which g++-8` + sudo update-alternatives --install /usr/bin/cc cc `which gcc` 9999 --slave /usr/bin/c++ c++ `which g++` - name: Config run: | @@ -44,10 +45,6 @@ jobs: cmake --version c++ --version - - name: gcc-8 check - if: matrix.os == 'ubuntu-18.04' - run: echo "CC= $CC" - - name: Install dependencies run: | python -m pip install --upgrade pip setuptools wheel From fde3bb883ce478c001d493a3c95506afbec4c978 Mon Sep 17 00:00:00 2001 From: Marek Otahal Date: Wed, 25 Sep 2019 18:17:53 +0200 Subject: [PATCH 39/67] try w/o requirements --- .github/workflows/htmcore.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/htmcore.yml b/.github/workflows/htmcore.yml index 769076f35d..fed724570b 100644 --- a/.github/workflows/htmcore.yml +++ b/.github/workflows/htmcore.yml @@ -35,9 +35,8 @@ jobs: run: | sudo apt-get update sudo apt-get -y install gcc-8 g++-8 - sudo apt-get remove -y gcc-7 g++-7 - sudo update-alternatives --install /usr/bin/gcc gcc `which gcc-8` 9999 --slave /usr/bin/g++ g++ `which g++-8` - sudo update-alternatives --install /usr/bin/cc cc `which gcc` 9999 --slave /usr/bin/c++ c++ `which g++` + sudo update-alternatives --install /usr/bin/g++ g++ `which g++-8` 9999 --slave /usr/bin/gcc gcc `which gcc-8` + sudo update-alternatives --install /usr/bin/c++ c++ `which g++-8` 9999 --slave /usr/bin/cc cc `which gcc-8` - name: Config run: | @@ -48,7 +47,6 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip setuptools wheel - pip install -r requirements.txt - name: build htmcore with setup.py run: python setup.py install --user From cb0651614739be8cc53dcc0ef1d2148e7c07fa7e Mon Sep 17 00:00:00 2001 From: Marek Otahal Date: Wed, 25 Sep 2019 18:29:36 +0200 Subject: [PATCH 40/67] tale of master and slave --- .github/workflows/htmcore.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/htmcore.yml b/.github/workflows/htmcore.yml index fed724570b..ab1ac4b949 100644 --- a/.github/workflows/htmcore.yml +++ b/.github/workflows/htmcore.yml @@ -36,6 +36,8 @@ jobs: sudo apt-get update sudo apt-get -y install gcc-8 g++-8 sudo update-alternatives --install /usr/bin/g++ g++ `which g++-8` 9999 --slave /usr/bin/gcc gcc `which gcc-8` + sudo update-alternatives --remove cc /usr/bin/cc + sudo update-alternatives --remove c++ /usr/bin/c++ sudo update-alternatives --install /usr/bin/c++ c++ `which g++-8` 9999 --slave /usr/bin/cc cc `which gcc-8` - name: Config From 9170cd844be3c2c50af70661bf8a4cb8eda87a07 Mon Sep 17 00:00:00 2001 From: Marek Otahal Date: Wed, 25 Sep 2019 18:33:32 +0200 Subject: [PATCH 41/67] make more slaves --- .github/workflows/htmcore.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/htmcore.yml b/.github/workflows/htmcore.yml index ab1ac4b949..c6e613d56d 100644 --- a/.github/workflows/htmcore.yml +++ b/.github/workflows/htmcore.yml @@ -35,6 +35,8 @@ jobs: run: | sudo apt-get update sudo apt-get -y install gcc-8 g++-8 + sudo update-alternatives --remove gcc /usr/bin/gcc + sudo update-alternatives --remove g++ /usr/bin/g++ sudo update-alternatives --install /usr/bin/g++ g++ `which g++-8` 9999 --slave /usr/bin/gcc gcc `which gcc-8` sudo update-alternatives --remove cc /usr/bin/cc sudo update-alternatives --remove c++ /usr/bin/c++ From 5e7a574fbef535b8f2045fcdc977f0808e55b18b Mon Sep 17 00:00:00 2001 From: Marek Otahal Date: Wed, 25 Sep 2019 18:55:36 +0200 Subject: [PATCH 42/67] setup.py: honor CC, CXX env variables --- .github/workflows/htmcore.yml | 14 ++++++++------ bindings/py/packaging/setup.py | 15 ++++++++++++--- 2 files changed, 20 insertions(+), 9 deletions(-) diff --git a/.github/workflows/htmcore.yml b/.github/workflows/htmcore.yml index c6e613d56d..5559bd11dd 100644 --- a/.github/workflows/htmcore.yml +++ b/.github/workflows/htmcore.yml @@ -35,12 +35,6 @@ jobs: run: | sudo apt-get update sudo apt-get -y install gcc-8 g++-8 - sudo update-alternatives --remove gcc /usr/bin/gcc - sudo update-alternatives --remove g++ /usr/bin/g++ - sudo update-alternatives --install /usr/bin/g++ g++ `which g++-8` 9999 --slave /usr/bin/gcc gcc `which gcc-8` - sudo update-alternatives --remove cc /usr/bin/cc - sudo update-alternatives --remove c++ /usr/bin/c++ - sudo update-alternatives --install /usr/bin/c++ c++ `which g++-8` 9999 --slave /usr/bin/cc cc `which gcc-8` - name: Config run: | @@ -52,7 +46,15 @@ jobs: run: | python -m pip install --upgrade pip setuptools wheel + - name: build htmcore with setup.py (gcc-8) + if: matrix.os == 'ubuntu-18.04' + env: + CC: gcc-8 + CXX: g++-8 + run: python setup.py install --user + - name: build htmcore with setup.py + if: matrix.os != 'ubuntu-18.04' run: python setup.py install --user - name: C++ & Python Tests diff --git a/bindings/py/packaging/setup.py b/bindings/py/packaging/setup.py index c57be3ea92..95a38ce9fa 100644 --- a/bindings/py/packaging/setup.py +++ b/bindings/py/packaging/setup.py @@ -59,7 +59,14 @@ UNIX_PLATFORMS = [LINUX_PLATFORM, DARWIN_PLATFORM] WINDOWS_PLATFORMS = ["windows"] - +# set CC, CXX if exist +cc="" +cxx="" +try: + cc=os.environ['CC'] + cxx=os.environ['CXX'] +except: + pass def getExtensionVersion(): """ @@ -258,6 +265,8 @@ def generateExtensions(platform, build_type): BUILD_TYPE = "-DCMAKE_BUILD_TYPE="+build_type + CXX = "-DCMAKE_CXX_COMPILER="+cxx + CC = "-DCMAKE_C_COMPILER="+cc scriptsDir = os.path.join(REPO_DIR, "build", "scripts") @@ -288,7 +297,7 @@ def generateExtensions(platform, build_type): # we can skip this step, the cache is already setup and we have the right binding specified. else: # For Linux and OSx we can let CMake figure it out. - subprocess.check_call(["cmake",BUILD_TYPE , PY_VER, REPO_DIR]) + subprocess.check_call(["cmake",BUILD_TYPE , CC, CXX, PY_VER, REPO_DIR]) else: # The generator is specified. @@ -303,7 +312,7 @@ def generateExtensions(platform, build_type): else: raise Exception('Did not find Visual Studio for generator "'+generator+ '".') else: - subprocess.check_call(["cmake", "-G", generator, BUILD_TYPE, PY_VER, REPO_DIR]) + subprocess.check_call(["cmake", "-G", generator, BUILD_TYPE, CC, CXX, PY_VER, REPO_DIR]) # Now do `make install` subprocess.check_call(["cmake", "--build", ".", "--target", "install", "--config", build_type]) From 6e6f29ea3462200508dcb7002b69b713dc399b79 Mon Sep 17 00:00:00 2001 From: Marek Otahal Date: Wed, 25 Sep 2019 19:54:31 +0200 Subject: [PATCH 43/67] Revert "setup.py: honor CC, CXX env variables" This reverts commit 5e7a574fbef535b8f2045fcdc977f0808e55b18b. --- .github/workflows/htmcore.yml | 14 ++++++-------- bindings/py/packaging/setup.py | 15 +++------------ 2 files changed, 9 insertions(+), 20 deletions(-) diff --git a/.github/workflows/htmcore.yml b/.github/workflows/htmcore.yml index 5559bd11dd..c6e613d56d 100644 --- a/.github/workflows/htmcore.yml +++ b/.github/workflows/htmcore.yml @@ -35,6 +35,12 @@ jobs: run: | sudo apt-get update sudo apt-get -y install gcc-8 g++-8 + sudo update-alternatives --remove gcc /usr/bin/gcc + sudo update-alternatives --remove g++ /usr/bin/g++ + sudo update-alternatives --install /usr/bin/g++ g++ `which g++-8` 9999 --slave /usr/bin/gcc gcc `which gcc-8` + sudo update-alternatives --remove cc /usr/bin/cc + sudo update-alternatives --remove c++ /usr/bin/c++ + sudo update-alternatives --install /usr/bin/c++ c++ `which g++-8` 9999 --slave /usr/bin/cc cc `which gcc-8` - name: Config run: | @@ -46,15 +52,7 @@ jobs: run: | python -m pip install --upgrade pip setuptools wheel - - name: build htmcore with setup.py (gcc-8) - if: matrix.os == 'ubuntu-18.04' - env: - CC: gcc-8 - CXX: g++-8 - run: python setup.py install --user - - name: build htmcore with setup.py - if: matrix.os != 'ubuntu-18.04' run: python setup.py install --user - name: C++ & Python Tests diff --git a/bindings/py/packaging/setup.py b/bindings/py/packaging/setup.py index 95a38ce9fa..c57be3ea92 100644 --- a/bindings/py/packaging/setup.py +++ b/bindings/py/packaging/setup.py @@ -59,14 +59,7 @@ UNIX_PLATFORMS = [LINUX_PLATFORM, DARWIN_PLATFORM] WINDOWS_PLATFORMS = ["windows"] -# set CC, CXX if exist -cc="" -cxx="" -try: - cc=os.environ['CC'] - cxx=os.environ['CXX'] -except: - pass + def getExtensionVersion(): """ @@ -265,8 +258,6 @@ def generateExtensions(platform, build_type): BUILD_TYPE = "-DCMAKE_BUILD_TYPE="+build_type - CXX = "-DCMAKE_CXX_COMPILER="+cxx - CC = "-DCMAKE_C_COMPILER="+cc scriptsDir = os.path.join(REPO_DIR, "build", "scripts") @@ -297,7 +288,7 @@ def generateExtensions(platform, build_type): # we can skip this step, the cache is already setup and we have the right binding specified. else: # For Linux and OSx we can let CMake figure it out. - subprocess.check_call(["cmake",BUILD_TYPE , CC, CXX, PY_VER, REPO_DIR]) + subprocess.check_call(["cmake",BUILD_TYPE , PY_VER, REPO_DIR]) else: # The generator is specified. @@ -312,7 +303,7 @@ def generateExtensions(platform, build_type): else: raise Exception('Did not find Visual Studio for generator "'+generator+ '".') else: - subprocess.check_call(["cmake", "-G", generator, BUILD_TYPE, CC, CXX, PY_VER, REPO_DIR]) + subprocess.check_call(["cmake", "-G", generator, BUILD_TYPE, PY_VER, REPO_DIR]) # Now do `make install` subprocess.check_call(["cmake", "--build", ".", "--target", "install", "--config", build_type]) From 4cb136be7388a11c971a767b8c15f86d73762e89 Mon Sep 17 00:00:00 2001 From: Marek Otahal Date: Wed, 25 Sep 2019 19:58:23 +0200 Subject: [PATCH 44/67] whitespace --- external/bootstrap.cmake | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/external/bootstrap.cmake b/external/bootstrap.cmake index 3beba401b5..f30cb3c9d3 100644 --- a/external/bootstrap.cmake +++ b/external/bootstrap.cmake @@ -31,12 +31,12 @@ FILE(MAKE_DIRECTORY ${REPOSITORY_DIR}/build/ThirdParty) execute_process(COMMAND ${CMAKE_COMMAND} -G ${CMAKE_GENERATOR} - -D CMAKE_INSTALL_PREFIX=. + -D CMAKE_INSTALL_PREFIX=. -D NEEDS_BOOST:BOOL=${NEEDS_BOOST} -D BINDING_BUILD:STRING=${BINDING_BUILD} - -D CMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} - -D REPOSITORY_DIR=${REPOSITORY_DIR} - ../../external + -D CMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} + -D REPOSITORY_DIR=${REPOSITORY_DIR} + ../../external WORKING_DIRECTORY ${REPOSITORY_DIR}/build/ThirdParty RESULT_VARIABLE result # OUTPUT_QUIET ### Disable this to debug external configuration From f541f9522242194b7c59d7064eedaa1eb2b501cc Mon Sep 17 00:00:00 2001 From: Marek Otahal Date: Wed, 25 Sep 2019 20:11:32 +0200 Subject: [PATCH 45/67] gcc-8 config --- .github/workflows/htmcore.yml | 35 ++++++++++++++++++++++------------- 1 file changed, 22 insertions(+), 13 deletions(-) diff --git a/.github/workflows/htmcore.yml b/.github/workflows/htmcore.yml index c6e613d56d..44e90eb89d 100644 --- a/.github/workflows/htmcore.yml +++ b/.github/workflows/htmcore.yml @@ -30,27 +30,36 @@ jobs: with: python-version: ${{ matrix.python-version }} - - name: Set up Cpp compilers - if: matrix.os == 'ubuntu-18.04' - run: | - sudo apt-get update - sudo apt-get -y install gcc-8 g++-8 - sudo update-alternatives --remove gcc /usr/bin/gcc - sudo update-alternatives --remove g++ /usr/bin/g++ - sudo update-alternatives --install /usr/bin/g++ g++ `which g++-8` 9999 --slave /usr/bin/gcc gcc `which gcc-8` - sudo update-alternatives --remove cc /usr/bin/cc - sudo update-alternatives --remove c++ /usr/bin/c++ - sudo update-alternatives --install /usr/bin/c++ c++ `which g++-8` 9999 --slave /usr/bin/cc cc `which gcc-8` - - - name: Config + - name: Versions run: | python --version cmake --version c++ --version + - name: Install dependencies (gcc-8) + if: matrix.os == 'ubuntu-18.04' + env: + CC: gcc-8 + CXX: g++-8 + run: | + sudo apt-get update + sudo apt-get -y install gcc-8 g++-8 + mkdir -p build/scripts + cd build/scripts + cmake ../.. + python -m pip install --upgrade pip setuptools wheel + python -m pip install -r ../../requirements.txt + - name: Install dependencies + if: matrix.os != 'ubuntu-18.04' run: | + mkdir -p build/scripts + cd build/scripts + cmake ../.. python -m pip install --upgrade pip setuptools wheel + cd .. + cd .. #dummy, but to avoid ../.. which is not multi-platform (Windows) + python -m pip install -r requirements.txt - name: build htmcore with setup.py run: python setup.py install --user From b95e9be459db87b92ed6b15a1e28409a527a41dc Mon Sep 17 00:00:00 2001 From: Marek Otahal Date: Wed, 25 Sep 2019 21:00:24 +0200 Subject: [PATCH 46/67] use GITHUB_WORKSPACE for multiplatform path --- .github/workflows/htmcore.yml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/htmcore.yml b/.github/workflows/htmcore.yml index 44e90eb89d..1123926583 100644 --- a/.github/workflows/htmcore.yml +++ b/.github/workflows/htmcore.yml @@ -53,13 +53,14 @@ jobs: - name: Install dependencies if: matrix.os != 'ubuntu-18.04' run: | + pwd + ls -l + ls ${{ env.GITHUB_WORKSPACE }} mkdir -p build/scripts cd build/scripts - cmake ../.. + cmake ${{ env.GITHUB_WORKSPACE }} python -m pip install --upgrade pip setuptools wheel - cd .. - cd .. #dummy, but to avoid ../.. which is not multi-platform (Windows) - python -m pip install -r requirements.txt + python -m pip install -r ${{env.GITHUB_WORKSPACE}}/requirements.txt - name: build htmcore with setup.py run: python setup.py install --user From d03e2aba36612227e7d19ad77cea0ce1fa9af22c Mon Sep 17 00:00:00 2001 From: Marek Otahal Date: Wed, 25 Sep 2019 21:16:59 +0200 Subject: [PATCH 47/67] debug --- .github/workflows/htmcore.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/htmcore.yml b/.github/workflows/htmcore.yml index 1123926583..cc7cd7b70c 100644 --- a/.github/workflows/htmcore.yml +++ b/.github/workflows/htmcore.yml @@ -55,12 +55,10 @@ jobs: run: | pwd ls -l - ls ${{ env.GITHUB_WORKSPACE }} mkdir -p build/scripts cd build/scripts - cmake ${{ env.GITHUB_WORKSPACE }} + cmake ../.. python -m pip install --upgrade pip setuptools wheel - python -m pip install -r ${{env.GITHUB_WORKSPACE}}/requirements.txt - name: build htmcore with setup.py run: python setup.py install --user From 68092acfda4e3271320b6a8e8b717ea4ba16e6dd Mon Sep 17 00:00:00 2001 From: Marek Otahal Date: Wed, 25 Sep 2019 22:26:01 +0200 Subject: [PATCH 48/67] fix calling cpp tests from python setup --- bindings/py/packaging/setup.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/bindings/py/packaging/setup.py b/bindings/py/packaging/setup.py index c57be3ea92..e56a27f516 100644 --- a/bindings/py/packaging/setup.py +++ b/bindings/py/packaging/setup.py @@ -142,13 +142,9 @@ def run_tests(self): cwd = os.getcwd() errno = 0 # run c++ tests (from python) - try: - cpp_tests = os.path.join(REPO_DIR, "build", build_dir, "bin", "unit_tests") - subprocess.check_call([cpp_tests]) - finally: - os.chdir(cwd) - if errno != 0: - sys.exit(errno) + cpp_tests = os.path.join(REPO_DIR, "build", "Release", "bin", "unit_tests") + subprocess.check_call([cpp_tests]) + os.chdir(cwd) # run python bindings tests (in /bindings/py/tests/) try: From 18f099f41f058e2ac59a4cae5b97f9956764e775 Mon Sep 17 00:00:00 2001 From: Marek Otahal Date: Wed, 25 Sep 2019 23:30:56 +0200 Subject: [PATCH 49/67] add GH Release code --- .github/workflows/htmcore.yml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/.github/workflows/htmcore.yml b/.github/workflows/htmcore.yml index cc7cd7b70c..d4cbff0920 100644 --- a/.github/workflows/htmcore.yml +++ b/.github/workflows/htmcore.yml @@ -65,6 +65,22 @@ jobs: - name: C++ & Python Tests run: python setup.py test + + - name: Release (make package) + run: | + python setup.py bdist_wheel + cd build/scripts + make package + + - name: Release (deploy) + # from https://github.com/marketplace/actions/gh-release + uses: softprops/action-gh-release@v1 + if: startsWith(github.ref, 'refs/tags/') + with: + files: | + build/scripts/htm_core-v* + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} build-arm64-docker: From 149e964a0fb7c8c8794934cbc7b87d4389297706 Mon Sep 17 00:00:00 2001 From: Marek Otahal Date: Thu, 26 Sep 2019 00:13:22 +0200 Subject: [PATCH 50/67] GH Actions: add PYPI publishing publish python package to PYPI Currently Testing only --- .github/workflows/htmcore.yml | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/.github/workflows/htmcore.yml b/.github/workflows/htmcore.yml index d4cbff0920..afef6ec538 100644 --- a/.github/workflows/htmcore.yml +++ b/.github/workflows/htmcore.yml @@ -69,6 +69,7 @@ jobs: - name: Release (make package) run: | python setup.py bdist_wheel + make install cd build/scripts make package @@ -81,7 +82,20 @@ jobs: build/scripts/htm_core-v* env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - + + - name: pre-PyPI + run: | + cp -a build/Release/distr/dist . #copy dist data to /dist, where PyPI Action expects it + ls dist + + - name: Publish to PyPI + if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags') + uses: pypa/gh-action-pypi-publish@master + with: + user: __token__ + password: ${{ secrets.pypi_password }} + repository_url: https://test.pypi.org/legacy/ #TODO rm for real pypi + build-arm64-docker: name: Build for ARM64 on Docker @@ -103,6 +117,7 @@ jobs: sudo docker build -t htm-arm64-docker --build-arg arch=arm64 . uname -a + build-debug: name: Build and test in Debug mode runs-on: ubuntu-18.04 @@ -140,6 +155,3 @@ jobs: sudo apt-get -y install valgrind LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:${PWD}/build/Release/lib valgrind --show-leak-kinds=definite,indirect,possible,reachable --track-origins=yes --num-callers=40 --error-exitcode=3 ./build/Release/bin/benchmark_hotgym 5 || exit 1 - - #TODO GH Releases - #TODO PYPI deploy From d40c636ee5af4155f73151ec665e725929813e57 Mon Sep 17 00:00:00 2001 From: Marek Otahal Date: Thu, 26 Sep 2019 00:17:59 +0200 Subject: [PATCH 51/67] GH Actions: replace make with multiplatform cmake --- .github/workflows/htmcore.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/htmcore.yml b/.github/workflows/htmcore.yml index afef6ec538..671322593e 100644 --- a/.github/workflows/htmcore.yml +++ b/.github/workflows/htmcore.yml @@ -69,9 +69,9 @@ jobs: - name: Release (make package) run: | python setup.py bdist_wheel - make install cd build/scripts - make package + cmake --build . --config Release --target install # aka make install ,but multiplatform + cmake --build . --config Release --target package # make package - name: Release (deploy) # from https://github.com/marketplace/actions/gh-release From 2159d9dcc0061c5d3f3d7276b417821fae9636bd Mon Sep 17 00:00:00 2001 From: Marek Otahal Date: Thu, 26 Sep 2019 07:20:21 +0200 Subject: [PATCH 52/67] GH Actions: fix version tag filter --- .github/workflows/htmcore.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/htmcore.yml b/.github/workflows/htmcore.yml index 671322593e..8af7e6843a 100644 --- a/.github/workflows/htmcore.yml +++ b/.github/workflows/htmcore.yml @@ -3,14 +3,14 @@ name: htm.core build on: push: branches: - - v*\.*\.*\.* # vX.Y.Z release tag + - 'v*.*.*' # vX.Y.Z release tag # run on pull_request events that target the master branch pull_request: branches: - master - # run every day of the week from Monday - Friday at 02:00 + # run every day of the week at 02:00 schedule: - - cron: 0 2 * * 1-5 + - cron: 0 2 * * * jobs: build: From 47b5af93f10c361b8af2d3711ae7b479273a745c Mon Sep 17 00:00:00 2001 From: Marek Otahal Date: Thu, 26 Sep 2019 07:24:53 +0200 Subject: [PATCH 53/67] GH Actions: configure installs requirements.txt had problem with ../../requirements.txt on Windows, so now call it first (=from ./requirements.txt) --- .github/workflows/htmcore.yml | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/.github/workflows/htmcore.yml b/.github/workflows/htmcore.yml index 8af7e6843a..2bc4607527 100644 --- a/.github/workflows/htmcore.yml +++ b/.github/workflows/htmcore.yml @@ -44,21 +44,20 @@ jobs: run: | sudo apt-get update sudo apt-get -y install gcc-8 g++-8 + python -m pip install --upgrade pip setuptools wheel + python -m pip install -r requirements.txt mkdir -p build/scripts cd build/scripts cmake ../.. - python -m pip install --upgrade pip setuptools wheel - python -m pip install -r ../../requirements.txt - name: Install dependencies if: matrix.os != 'ubuntu-18.04' run: | - pwd - ls -l + python -m pip install --upgrade pip setuptools wheel + python -m pip install -r requirements.txt mkdir -p build/scripts cd build/scripts cmake ../.. - python -m pip install --upgrade pip setuptools wheel - name: build htmcore with setup.py run: python setup.py install --user From 3f7a8bcee30b5cc26603d906209e6774d6f8d020 Mon Sep 17 00:00:00 2001 From: Marek Otahal Date: Thu, 26 Sep 2019 07:29:19 +0200 Subject: [PATCH 54/67] GH Actions: gate "make package" to run on tagged only as it's quite time consuming, and needed only for release --- .github/workflows/htmcore.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/htmcore.yml b/.github/workflows/htmcore.yml index 2bc4607527..31eae23351 100644 --- a/.github/workflows/htmcore.yml +++ b/.github/workflows/htmcore.yml @@ -66,6 +66,7 @@ jobs: run: python setup.py test - name: Release (make package) + if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags') run: | python setup.py bdist_wheel cd build/scripts @@ -73,9 +74,9 @@ jobs: cmake --build . --config Release --target package # make package - name: Release (deploy) + if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags') # from https://github.com/marketplace/actions/gh-release uses: softprops/action-gh-release@v1 - if: startsWith(github.ref, 'refs/tags/') with: files: | build/scripts/htm_core-v* @@ -83,6 +84,7 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: pre-PyPI + if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags') run: | cp -a build/Release/distr/dist . #copy dist data to /dist, where PyPI Action expects it ls dist From 9b333172e176101871f0c301b346e14197ba38ee Mon Sep 17 00:00:00 2001 From: Marek Otahal Date: Thu, 26 Sep 2019 07:31:30 +0200 Subject: [PATCH 55/67] GH Actions: fix comment interpreted during pre-pypi on Windows --- .github/workflows/htmcore.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/htmcore.yml b/.github/workflows/htmcore.yml index 31eae23351..c99ad1f3a6 100644 --- a/.github/workflows/htmcore.yml +++ b/.github/workflows/htmcore.yml @@ -85,8 +85,9 @@ jobs: - name: pre-PyPI if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags') + #copy dist data to /dist, where PyPI Action expects it run: | - cp -a build/Release/distr/dist . #copy dist data to /dist, where PyPI Action expects it + cp -a build/Release/distr/dist . ls dist - name: Publish to PyPI From d09a87d15d8778d92ac1565d8f32432d0724490c Mon Sep 17 00:00:00 2001 From: Marek Otahal Date: Thu, 26 Sep 2019 07:40:00 +0200 Subject: [PATCH 56/67] GH Actions: separate ARM64 build into scheduled build as ARM build takes too long (4hrs, due to docker emulation), make it build only nightly (not on each PR) --- .github/workflows/arm64-build.yml | 32 +++++++++++++++++++++++++++++++ .github/workflows/htmcore.yml | 20 ------------------- 2 files changed, 32 insertions(+), 20 deletions(-) create mode 100644 .github/workflows/arm64-build.yml diff --git a/.github/workflows/arm64-build.yml b/.github/workflows/arm64-build.yml new file mode 100644 index 0000000000..802d5e3668 --- /dev/null +++ b/.github/workflows/arm64-build.yml @@ -0,0 +1,32 @@ +name: ARM64 build + +on: + # run every day of the week at 06:00 + schedule: + - cron: 0 6 * * * + +jobs: + build-arm64-docker: + name: Build for ARM64 on Docker + runs-on: ubuntu-18.04 + steps: + - name: Install docker + run: | + # bug in ubuntu, conflicts with docker.io + sudo apt-get update + sudo apt-get remove --purge -y moby-engine moby-cli + sudo apt-get install -y qemu-system docker.io + + - name: Setup ARM64 build env + run: | + uname -a + sudo docker run --rm --privileged multiarch/qemu-user-static:register --reset + uname -a + + - uses: actions/checkout@v1 + + - name: ARM64 build + run: | + sudo docker build -t htm-arm64-docker --build-arg arch=arm64 . + uname -a + diff --git a/.github/workflows/htmcore.yml b/.github/workflows/htmcore.yml index c99ad1f3a6..d0187bc589 100644 --- a/.github/workflows/htmcore.yml +++ b/.github/workflows/htmcore.yml @@ -99,26 +99,6 @@ jobs: repository_url: https://test.pypi.org/legacy/ #TODO rm for real pypi - build-arm64-docker: - name: Build for ARM64 on Docker - runs-on: ubuntu-18.04 - steps: - - name: Docker ARM64 image - run: | - # bug in ubuntu, conflicts with docker.io - sudo apt-get update - sudo apt-get remove --purge -y moby-engine moby-cli - sudo apt-get install -y qemu-system docker.io - uname -a - sudo docker run --rm --privileged multiarch/qemu-user-static:register --reset - uname -a - - - uses: actions/checkout@v1 - - name: ARM64 build - run: | - sudo docker build -t htm-arm64-docker --build-arg arch=arm64 . - uname -a - build-debug: name: Build and test in Debug mode From 4c28afebee02e40d81b87a2e5c1a7cb04ee7b362 Mon Sep 17 00:00:00 2001 From: Marek Otahal Date: Thu, 26 Sep 2019 08:04:50 +0200 Subject: [PATCH 57/67] requirements.txt included in dist/ copy it there during install, needed for pypi --- src/CMakeLists.txt | 4 ++++ src/test/unit/engine/CppRegionTest.cpp | 1 - 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 3a790369f1..02ee776d8a 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -439,6 +439,10 @@ install(DIRECTORY "${CMAKE_INSTALL_PREFIX}/distr/dist/" DESTINATION py FILES_MATCHING PATTERN "*.whl") +# copy requirements.txt into dist/ +install(FILES ${REPOSITORY_DIR}/requirements.txt + DESTINATION distr/dist) + ############### PACKAGE ########################### # diff --git a/src/test/unit/engine/CppRegionTest.cpp b/src/test/unit/engine/CppRegionTest.cpp index 0450746f4b..c57c42af34 100644 --- a/src/test/unit/engine/CppRegionTest.cpp +++ b/src/test/unit/engine/CppRegionTest.cpp @@ -281,7 +281,6 @@ TEST(CppRegionTest, RegionSerialization) { Region r2(&n); r2.load(ss); EXPECT_EQ(*r1.get(), r2); - } From 5a8f08e7e1695293543a450075710b3f3756720b Mon Sep 17 00:00:00 2001 From: Marek Otahal Date: Thu, 26 Sep 2019 08:15:12 +0200 Subject: [PATCH 58/67] debug Windows --- .github/workflows/htmcore.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/htmcore.yml b/.github/workflows/htmcore.yml index d0187bc589..8b42ed932b 100644 --- a/.github/workflows/htmcore.yml +++ b/.github/workflows/htmcore.yml @@ -55,8 +55,11 @@ jobs: run: | python -m pip install --upgrade pip setuptools wheel python -m pip install -r requirements.txt - mkdir -p build/scripts - cd build/scripts + mkdir build + cd build + mkdir scripts + cd scripts + pwd cmake ../.. - name: build htmcore with setup.py From 5803bd5bdfeba0394f1f44f6b9396d95793c8ee4 Mon Sep 17 00:00:00 2001 From: Marek Otahal Date: Thu, 26 Sep 2019 09:36:10 +0200 Subject: [PATCH 59/67] setup.py: add configure command can call python setup.py configure which installs c++ deps and runs cmake --- .github/workflows/htmcore.yml | 11 ++----- bindings/py/packaging/setup.py | 52 ++++++++++++++++++++++++++++++++-- 2 files changed, 51 insertions(+), 12 deletions(-) diff --git a/.github/workflows/htmcore.yml b/.github/workflows/htmcore.yml index 8b42ed932b..4c8af854d7 100644 --- a/.github/workflows/htmcore.yml +++ b/.github/workflows/htmcore.yml @@ -46,21 +46,14 @@ jobs: sudo apt-get -y install gcc-8 g++-8 python -m pip install --upgrade pip setuptools wheel python -m pip install -r requirements.txt - mkdir -p build/scripts - cd build/scripts - cmake ../.. + python setup.py configure - name: Install dependencies if: matrix.os != 'ubuntu-18.04' run: | python -m pip install --upgrade pip setuptools wheel python -m pip install -r requirements.txt - mkdir build - cd build - mkdir scripts - cd scripts - pwd - cmake ../.. + python setup.py configure - name: build htmcore with setup.py run: python setup.py install --user diff --git a/bindings/py/packaging/setup.py b/bindings/py/packaging/setup.py index e56a27f516..9419a36930 100644 --- a/bindings/py/packaging/setup.py +++ b/bindings/py/packaging/setup.py @@ -96,6 +96,29 @@ def run(self): +class ConfigureCommand(Command): + """Setup C++ dependencies and call cmake for configuration""" + + description = "Setup C++ dependencies and call cmake for configuration." + user_options = [] + + def initialize_options(self): + pass + + def finalize_options(self): + pass + + def run(self): + platform = getPlatformInfo() + if platform == DARWIN_PLATFORM and not "ARCHFLAGS" in os.environ: + os.system("export ARCHFLAGS=\"-arch x86_64\"") + + # Run CMake if extension files are missing. + # CMake also copies all py files into place in DISTR_DIR + configure(platform, build_type) + + + def fixPath(path): """ Ensures paths are correct for linux and windows @@ -240,6 +263,27 @@ def generateExtensions(platform, build_type): Note: for Windows it will force a X64 build. """ cwd = os.getcwd() + scriptsDir = os.path.join(REPO_DIR, "build", "scripts") + try: + if not os.path.isdir(scriptsDir): + os.makedirs(scriptsDir) + os.chdir(scriptsDir) + + # cmake ../.. + configure(platform, build_type) + + # build: make && make install + subprocess.check_call(["cmake", "--build", ".", "--target", "install", "--config", build_type]) + + finally: + os.chdir(cwd) + + +def configure(platform, build_type): + """ + Setup C++ dependencies and call cmake for configuration. + """ + cwd = os.getcwd() print("Python version: {}\n".format(sys.version)) from sys import version_info @@ -301,8 +345,6 @@ def generateExtensions(platform, build_type): else: subprocess.check_call(["cmake", "-G", generator, BUILD_TYPE, PY_VER, REPO_DIR]) - # Now do `make install` - subprocess.check_call(["cmake", "--build", ".", "--target", "install", "--config", build_type]) finally: os.chdir(cwd) @@ -316,7 +358,10 @@ def generateExtensions(platform, build_type): # Run CMake if extension files are missing. # CMake also copies all py files into place in DISTR_DIR - getExtensionFiles(platform, build_type) + if len(sys.argv) > 1 and sys.argv[1] == "configure": + configure(platform, build_type) + else: #full build + getExtensionFiles(platform, build_type) with open(os.path.join(REPO_DIR, "README.md"), "r") as fh: long_description = fh.read() @@ -358,6 +403,7 @@ def generateExtensions(platform, build_type): cmdclass={ "clean": CleanCommand, "test": TestCommand, + "configure": ConfigureCommand, }, author="Numenta & HTM Community", author_email="help@numenta.org", From 17b9d5ddc61cc1ec3a04cc8c322b3953104a6656 Mon Sep 17 00:00:00 2001 From: Marek Otahal Date: Thu, 26 Sep 2019 09:51:46 +0200 Subject: [PATCH 60/67] fix broken YAML test in Debug on Linux known issue in 3rd party dependency. --- src/test/unit/engine/CppRegionTest.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/test/unit/engine/CppRegionTest.cpp b/src/test/unit/engine/CppRegionTest.cpp index c57c42af34..d20b6e0fe1 100644 --- a/src/test/unit/engine/CppRegionTest.cpp +++ b/src/test/unit/engine/CppRegionTest.cpp @@ -50,7 +50,7 @@ static bool verbose = false; #define VERBOSE if(verbose) std::cerr << "[ ]" - +#if !defined(NDEBUG) && defined(NTA_OS_LINUX) //TODO YAML libyaml-cpp failing on Linux in Debug mode, https://github.com/htm-community/htm.core/issues/218 TEST(CppRegionTest, testCppLinkingFanIn) { Network net; Real64 *buffer1; @@ -283,5 +283,5 @@ TEST(CppRegionTest, RegionSerialization) { EXPECT_EQ(*r1.get(), r2); } - +#endif } //ns From 551358062ff5a2535ae9873ebb85410096187ccf Mon Sep 17 00:00:00 2001 From: Marek Otahal Date: Thu, 26 Sep 2019 10:03:26 +0200 Subject: [PATCH 61/67] GH Actions: Debug build cleanup --- .github/workflows/htmcore.yml | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/.github/workflows/htmcore.yml b/.github/workflows/htmcore.yml index 4c8af854d7..346239d594 100644 --- a/.github/workflows/htmcore.yml +++ b/.github/workflows/htmcore.yml @@ -107,12 +107,7 @@ jobs: sudo apt-get update sudo apt-get -y install gcc-8 g++-8 - - name: Config - run: | - cmake --version - g++ --version - - - name: Debug build htmcore with cmake + - name: Install dependencies (Debug) env: CC: gcc-8 CXX: g++-8 @@ -121,7 +116,10 @@ jobs: mkdir -p build/scripts cd build/scripts cmake ../.. -DCMAKE_BUILD_TYPE=Debug - make -j4 && make install + + - name: Debug build + run: | + make -j2 && make install - name: C++ Tests run: | From 341b09ad114d2d47a17668a46b6722874492ae42 Mon Sep 17 00:00:00 2001 From: Marek Otahal Date: Thu, 26 Sep 2019 10:20:19 +0200 Subject: [PATCH 62/67] Revert "fix broken YAML test in Debug on Linux" This reverts commit 17b9d5ddc61cc1ec3a04cc8c322b3953104a6656. --- src/test/unit/engine/CppRegionTest.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/test/unit/engine/CppRegionTest.cpp b/src/test/unit/engine/CppRegionTest.cpp index d20b6e0fe1..c57c42af34 100644 --- a/src/test/unit/engine/CppRegionTest.cpp +++ b/src/test/unit/engine/CppRegionTest.cpp @@ -50,7 +50,7 @@ static bool verbose = false; #define VERBOSE if(verbose) std::cerr << "[ ]" -#if !defined(NDEBUG) && defined(NTA_OS_LINUX) //TODO YAML libyaml-cpp failing on Linux in Debug mode, https://github.com/htm-community/htm.core/issues/218 + TEST(CppRegionTest, testCppLinkingFanIn) { Network net; Real64 *buffer1; @@ -283,5 +283,5 @@ TEST(CppRegionTest, RegionSerialization) { EXPECT_EQ(*r1.get(), r2); } -#endif + } //ns From 86250e70dcde1fe149b343276de812c342126deb Mon Sep 17 00:00:00 2001 From: Marek Otahal Date: Thu, 26 Sep 2019 10:25:10 +0200 Subject: [PATCH 63/67] GH Actions: run Debug build on OSX because Debug & Linux is broken due to YAML parser bug: issue #218 --- .github/workflows/htmcore.yml | 23 ++++++++--------------- 1 file changed, 8 insertions(+), 15 deletions(-) diff --git a/.github/workflows/htmcore.yml b/.github/workflows/htmcore.yml index 346239d594..3fb0f5dcbe 100644 --- a/.github/workflows/htmcore.yml +++ b/.github/workflows/htmcore.yml @@ -61,6 +61,12 @@ jobs: - name: C++ & Python Tests run: python setup.py test + - name: Memory leaks check (valgrind) + if: matrix.os == 'ubuntu-18.04' + run: | + sudo apt-get -y install valgrind + LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:${PWD}/build/Release/lib valgrind --show-leak-kinds=definite,indirect,possible,reachable --track-origins=yes --num-callers=40 --error-exitcode=3 ./build/Release/bin/benchmark_hotgym 5 || exit 1 + - name: Release (make package) if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags') run: | @@ -98,19 +104,12 @@ jobs: build-debug: name: Build and test in Debug mode - runs-on: ubuntu-18.04 + #currently cannot run on Linux & Debug due to a bug in YAML parser: issue #218 + runs-on: macOS-latest steps: - uses: actions/checkout@v1 - - name: Set up Cpp compilers - run: | - sudo apt-get update - sudo apt-get -y install gcc-8 g++-8 - - name: Install dependencies (Debug) - env: - CC: gcc-8 - CXX: g++-8 run: | echo "built type: ${CMAKE_BUILD_TYPE}" mkdir -p build/scripts @@ -125,9 +124,3 @@ jobs: run: | cd build/scripts ../Debug/bin/unit_tests - - - name: Memory leaks check (valgrind) - run: - sudo apt-get -y install valgrind - LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:${PWD}/build/Release/lib valgrind --show-leak-kinds=definite,indirect,possible,reachable --track-origins=yes --num-callers=40 --error-exitcode=3 ./build/Release/bin/benchmark_hotgym 5 || exit 1 - From 39058eac556f56ee68070904c5d3cb408d633e58 Mon Sep 17 00:00:00 2001 From: Marek Otahal Date: Thu, 26 Sep 2019 10:44:18 +0200 Subject: [PATCH 64/67] fix debug build --- .github/workflows/htmcore.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/htmcore.yml b/.github/workflows/htmcore.yml index 3fb0f5dcbe..10ffd01eeb 100644 --- a/.github/workflows/htmcore.yml +++ b/.github/workflows/htmcore.yml @@ -118,6 +118,7 @@ jobs: - name: Debug build run: | + cd build/scripts make -j2 && make install - name: C++ Tests From 7166ffdb6c8dfedd019103205fd50fffd710c788 Mon Sep 17 00:00:00 2001 From: Marek Otahal Date: Thu, 26 Sep 2019 11:28:35 +0200 Subject: [PATCH 65/67] setup.py: parallel builds with -j4 uses hack by passing -- -j 4 to cmake's make, Since cmake 3.12 crossplatform -j4 is supported, but cmake 3.12 is quite new to require as a dependency (eg not in ubuntu 18.04 LTS) --- bindings/py/packaging/setup.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/bindings/py/packaging/setup.py b/bindings/py/packaging/setup.py index 9419a36930..51fa54bb75 100644 --- a/bindings/py/packaging/setup.py +++ b/bindings/py/packaging/setup.py @@ -273,8 +273,10 @@ def generateExtensions(platform, build_type): configure(platform, build_type) # build: make && make install - subprocess.check_call(["cmake", "--build", ".", "--target", "install", "--config", build_type]) - + if platform != "windows": #TODO since cmake 3.12 "-j4" is directly supported (=crossplatform), for now -- passes other options to make + subprocess.check_call(["cmake", "--build", ".", "--target", "install", "--config", build_type, "--", "-j", "4"]) + else: + subprocess.check_call(["cmake", "--build", ".", "--target", "install", "--config", build_type]) finally: os.chdir(cwd) From 2b0abc5ab488a968802e8c47ccf6752b6c190c35 Mon Sep 17 00:00:00 2001 From: Marek Otahal Date: Thu, 26 Sep 2019 12:01:04 +0200 Subject: [PATCH 66/67] MSVC: build with /MT instead of /MD for faster builds --- CommonCompilerConfig.cmake | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CommonCompilerConfig.cmake b/CommonCompilerConfig.cmake index 2a22e26550..98cbaf037a 100644 --- a/CommonCompilerConfig.cmake +++ b/CommonCompilerConfig.cmake @@ -171,8 +171,8 @@ if(MSVC) # Release Only: /O2 /Oi /Gy /MD # Debug Only: /Od /Zi /sdl /RTC1 /MDd set(INTERNAL_CXX_FLAGS /permissive- /W3 /Gm- /EHsc /FC /nologo /Zc:__cplusplus - $<$:/O2 /Oi /Gy /GL /MD> - $<$:/Ob0 /Od /Zi /sdl /RTC1 /MDd>) + $<$:/O2 /Oi /Gy /GL /MT> + $<$:/Ob0 /Od /Zi /sdl /RTC1 /MTd>) #linker flags if("${BITNESS}" STREQUAL "32") set(machine "-MACHINE:X86") From 257267f4ac9efbe606417e4215c10118fa201987 Mon Sep 17 00:00:00 2001 From: Marek Otahal Date: Thu, 26 Sep 2019 12:25:45 +0200 Subject: [PATCH 67/67] Revert "MSVC: build with /MT instead of /MD for faster builds" This reverts commit 2b0abc5ab488a968802e8c47ccf6752b6c190c35. --- CommonCompilerConfig.cmake | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CommonCompilerConfig.cmake b/CommonCompilerConfig.cmake index 98cbaf037a..2a22e26550 100644 --- a/CommonCompilerConfig.cmake +++ b/CommonCompilerConfig.cmake @@ -171,8 +171,8 @@ if(MSVC) # Release Only: /O2 /Oi /Gy /MD # Debug Only: /Od /Zi /sdl /RTC1 /MDd set(INTERNAL_CXX_FLAGS /permissive- /W3 /Gm- /EHsc /FC /nologo /Zc:__cplusplus - $<$:/O2 /Oi /Gy /GL /MT> - $<$:/Ob0 /Od /Zi /sdl /RTC1 /MTd>) + $<$:/O2 /Oi /Gy /GL /MD> + $<$:/Ob0 /Od /Zi /sdl /RTC1 /MDd>) #linker flags if("${BITNESS}" STREQUAL "32") set(machine "-MACHINE:X86")