From 041c45baaa895db0c62ab8c2d4d2e60a50c0ed3a Mon Sep 17 00:00:00 2001 From: Jinzhe Zeng Date: Fri, 7 Aug 2020 20:00:30 -0400 Subject: [PATCH 01/12] test on tensorflow 2.3 --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index dd4b0eed9e..f72c76fb79 100644 --- a/.travis.yml +++ b/.travis.yml @@ -66,12 +66,12 @@ jobs: env: - CC=gcc-5 - CXX=g++-5 - - TENSORFLOW_VERSION=2.1 + - TENSORFLOW_VERSION=2.3 - python: 3.7 env: - CC=gcc-8 - CXX=g++-8 - - TENSORFLOW_VERSION=2.1 + - TENSORFLOW_VERSION=2.3 - stage: build whls services: docker env: From 9f5332699d493effcc817845a9fe1c7c31a859f7 Mon Sep 17 00:00:00 2001 From: Jinzhe Zeng Date: Sat, 17 Oct 2020 19:01:13 -0400 Subject: [PATCH 02/12] bump tf version to 2.3; add tf to requirement if it's not detected; don't add cmake when it's installed --- .travis.yml | 10 +++------- README.md | 4 ++-- setup.py | 29 +++++++++++++++++++---------- 3 files changed, 24 insertions(+), 19 deletions(-) diff --git a/.travis.yml b/.travis.yml index f72c76fb79..a9c42a696e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -77,22 +77,18 @@ jobs: env: - TWINE_USERNAME=__token__ - CIBW_BUILD="cp36-* cp37-*" - - CIBW_BEFORE_BUILD="pip install tensorflow && sed -i 's/libresolv.so.2\"/libresolv.so.2\", \"libtensorflow_framework.so.2\"/g' \$(find / -name policy.json)" + - CIBW_BEFORE_BUILD="sed -i 's/libresolv.so.2\"/libresolv.so.2\", \"libtensorflow_framework.so.2\"/g' \$(find / -name policy.json)" - CIBW_SKIP="*-win32 *-manylinux_i686" - CC=gcc-7 - CXX=g++-7 - - TENSORFLOW_VERSION=2.1 + - TENSORFLOW_VERSION=2.3 install: - - python -m pip install twine cibuildwheel==1.1.0 scikit-build setuptools_scm + - python -m pip install twine cibuildwheel==1.6.3 scikit-build setuptools_scm script: - python -m cibuildwheel --output-dir wheelhouse - python setup.py sdist after_success: - if [[ $TRAVIS_TAG ]]; then python -m twine upload wheelhouse/*; python -m twine upload dist/*.tar.gz; fi -before_install: - #- pip install --upgrade pip - - pip install --upgrade setuptools - - pip install tensorflow==$TENSORFLOW_VERSION install: - pip install --verbose .[test] script: diff --git a/README.md b/README.md index 5862ed40b5..0d33c8bd12 100644 --- a/README.md +++ b/README.md @@ -133,7 +133,7 @@ We follow the virtual environment approach to install the tensorflow's Python in virtualenv -p python3 $tensorflow_venv source $tensorflow_venv/bin/activate pip install --upgrade pip -pip install --upgrade tensorflow==2.1.0 +pip install --upgrade tensorflow==2.3.0 ``` It is notice that everytime a new shell is started and one wants to use `DeePMD-kit`, the virtual environment should be activated by ```bash @@ -149,7 +149,7 @@ virtualenv -p python3.7 $tensorflow_venv ``` If one does not need the GPU support of deepmd-kit and is concerned about package size, the CPU-only version of tensorflow should be installed by ```bash -pip install --upgrade tensorflow-cpu==2.1.0 +pip install --upgrade tensorflow-cpu==2.3.0 ``` To verify the installation, run ```bash diff --git a/setup.py b/setup.py index 117ccec2a2..ca5a589f3c 100644 --- a/setup.py +++ b/setup.py @@ -4,7 +4,7 @@ from setuptools_scm import get_version from packaging.version import LegacyVersion from os import path, makedirs -import os, imp, sys, platform, sysconfig +import os, importlib, sys, platform, sysconfig readme_file = path.join(path.dirname(path.abspath(__file__)), 'README.md') @@ -15,19 +15,28 @@ with open(readme_file) as f: readme = f.read() -try: - tf_install_dir = imp.find_module('tensorflow')[1] -except ImportError: - site_packages_path = path.join(path.dirname(path.__file__), 'site-packages') - tf_install_dir = imp.find_module('tensorflow', [site_packages_path])[1] +install_requires=['numpy', 'scipy', 'pyyaml'] +setup_requires=['setuptools_scm', 'scikit-build'] +def find_tf(path): + return importlib.machinery.FileFinder(path).find_spec("tensorflow") -install_requires=['numpy', 'scipy', 'pyyaml'] -setup_requires=['setuptools_scm', 'scikit-build', 'cmake'] +tf_spec = importlib.util.find_spec("tensorflow") +if tf_spec: + tf_install_dir = tf_spec.submodule_search_locations[1] +else: + site_packages_path = path.join(path.dirname(path.__file__), 'site-packages') + tf_spec = importlib.machinery.FileFinder(path).find_spec("tensorflow") + if tf_spec: + tf_install_dir = tf_spec.submodule_search_locations[1] + else: + tf_version = os.environ.get('TENSORFLOW_VERSION', '2.3') + setup_requires.append("tensorflow==" + tf_version) + install_requires.append("tensorflow==" + tf_version) -# add cmake as a build requirement if cmake>3.0 is not installed +# add cmake as a build requirement if cmake>3.7 is not installed try: - if LegacyVersion(get_cmake_version()) < LegacyVersion("3.0"): + if LegacyVersion(get_cmake_version()) < LegacyVersion("3.7"): setup_requires.append('cmake') except SKBuildError: setup_requires.append('cmake') From d77ffd3c29dc34f463644209550eed87d218b9a3 Mon Sep 17 00:00:00 2001 From: Jinzhe Zeng Date: Sat, 17 Oct 2020 19:05:50 -0400 Subject: [PATCH 03/12] remove useless code --- setup.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/setup.py b/setup.py index ca5a589f3c..d599ad4adc 100644 --- a/setup.py +++ b/setup.py @@ -18,9 +18,6 @@ install_requires=['numpy', 'scipy', 'pyyaml'] setup_requires=['setuptools_scm', 'scikit-build'] -def find_tf(path): - return importlib.machinery.FileFinder(path).find_spec("tensorflow") - tf_spec = importlib.util.find_spec("tensorflow") if tf_spec: tf_install_dir = tf_spec.submodule_search_locations[1] From d8a4b77dbdae1c824dd2eb44c839871e8a4a73ef Mon Sep 17 00:00:00 2001 From: Jinzhe Zeng Date: Sat, 17 Oct 2020 22:55:26 -0400 Subject: [PATCH 04/12] fix bug --- .travis.yml | 2 ++ setup.py | 4 +++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index a9c42a696e..380f151b39 100644 --- a/.travis.yml +++ b/.travis.yml @@ -89,6 +89,8 @@ jobs: - python setup.py sdist after_success: - if [[ $TRAVIS_TAG ]]; then python -m twine upload wheelhouse/*; python -m twine upload dist/*.tar.gz; fi +before_install: + - pip install tensorflow==$TENSORFLOW_VERSION install: - pip install --verbose .[test] script: diff --git a/setup.py b/setup.py index ca5a589f3c..0015070266 100644 --- a/setup.py +++ b/setup.py @@ -21,6 +21,7 @@ def find_tf(path): return importlib.machinery.FileFinder(path).find_spec("tensorflow") +extras_require = dict() tf_spec = importlib.util.find_spec("tensorflow") if tf_spec: tf_install_dir = tf_spec.submodule_search_locations[1] @@ -32,7 +33,7 @@ def find_tf(path): else: tf_version = os.environ.get('TENSORFLOW_VERSION', '2.3') setup_requires.append("tensorflow==" + tf_version) - install_requires.append("tensorflow==" + tf_version) + extras_require = {"cpu": "tensorflow_cpu==" + tf_version, "gpu": "tensorflow==" + tf_version} # add cmake as a build requirement if cmake>3.7 is not installed try: @@ -74,6 +75,7 @@ def find_tf(path): cmake_minimum_required_version='3.0', extras_require={ 'test': ['dpdata>=0.1.9'], + **extras_require, }, entry_points={ 'console_scripts': ['dp = deepmd.main:main'] From f0da8521f667b8e056bca4fe6759e85f97e9f1b5 Mon Sep 17 00:00:00 2001 From: Jinzhe Zeng Date: Sat, 17 Oct 2020 23:57:40 -0400 Subject: [PATCH 05/12] bugfix --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 150130d38d..9b6c7cca2f 100644 --- a/setup.py +++ b/setup.py @@ -24,7 +24,7 @@ tf_install_dir = tf_spec.submodule_search_locations[1] else: site_packages_path = path.join(path.dirname(path.__file__), 'site-packages') - tf_spec = importlib.machinery.FileFinder(path).find_spec("tensorflow") + tf_spec = importlib.machinery.FileFinder(site_packages_path).find_spec("tensorflow") if tf_spec: tf_install_dir = tf_spec.submodule_search_locations[1] else: From e760f6c1293d0a4957792ad8072631de6177d2ba Mon Sep 17 00:00:00 2001 From: Jinzhe Zeng Date: Sun, 18 Oct 2020 09:29:56 -0400 Subject: [PATCH 06/12] bugfix --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 9b6c7cca2f..10000aee8f 100644 --- a/setup.py +++ b/setup.py @@ -21,12 +21,12 @@ extras_require = dict() tf_spec = importlib.util.find_spec("tensorflow") if tf_spec: - tf_install_dir = tf_spec.submodule_search_locations[1] + tf_install_dir = tf_spec.submodule_search_locations[0] else: site_packages_path = path.join(path.dirname(path.__file__), 'site-packages') tf_spec = importlib.machinery.FileFinder(site_packages_path).find_spec("tensorflow") if tf_spec: - tf_install_dir = tf_spec.submodule_search_locations[1] + tf_install_dir = tf_spec.submodule_search_locations[0] else: tf_version = os.environ.get('TENSORFLOW_VERSION', '2.3') setup_requires.append("tensorflow==" + tf_version) From 99492e38f6a07524a268eb83b23e12528f93baf5 Mon Sep 17 00:00:00 2001 From: Jinzhe Zeng Date: Sun, 18 Oct 2020 11:15:16 -0400 Subject: [PATCH 07/12] add tf_install_dir --- setup.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 10000aee8f..6f9adc35e7 100644 --- a/setup.py +++ b/setup.py @@ -4,7 +4,9 @@ from setuptools_scm import get_version from packaging.version import LegacyVersion from os import path, makedirs -import os, importlib, sys, platform, sysconfig +import os, importlib +import pkg_resources +from distutils.util import get_platform readme_file = path.join(path.dirname(path.abspath(__file__)), 'README.md') @@ -31,6 +33,10 @@ tf_version = os.environ.get('TENSORFLOW_VERSION', '2.3') setup_requires.append("tensorflow==" + tf_version) extras_require = {"cpu": "tensorflow_cpu==" + tf_version, "gpu": "tensorflow==" + tf_version} + tf_install_dir = path.join(path.dirname(path.abspath(__file__)), '.egg', + pkg_resources.Distribution(project_name="tensorflow", version=tf_version, + platform=get_platform()).egg_name(), + 'tensorflow') # add cmake as a build requirement if cmake>3.7 is not installed try: From fccd45b542d5768951b1098c120e485c71ace040 Mon Sep 17 00:00:00 2001 From: Jinzhe Zeng Date: Sun, 18 Oct 2020 17:47:37 -0400 Subject: [PATCH 08/12] remove before_install --- .travis.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 380f151b39..a9c42a696e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -89,8 +89,6 @@ jobs: - python setup.py sdist after_success: - if [[ $TRAVIS_TAG ]]; then python -m twine upload wheelhouse/*; python -m twine upload dist/*.tar.gz; fi -before_install: - - pip install tensorflow==$TENSORFLOW_VERSION install: - pip install --verbose .[test] script: From 609a3238783e2e93283591946505e53b1db58c43 Mon Sep 17 00:00:00 2001 From: Jinzhe Zeng Date: Sun, 18 Oct 2020 18:09:33 -0400 Subject: [PATCH 09/12] remove verbose --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index a9c42a696e..f6b4c0d137 100644 --- a/.travis.yml +++ b/.travis.yml @@ -90,6 +90,6 @@ jobs: after_success: - if [[ $TRAVIS_TAG ]]; then python -m twine upload wheelhouse/*; python -m twine upload dist/*.tar.gz; fi install: - - pip install --verbose .[test] + - pip install .[test] script: - cd source/tests && python -m unittest From 7dc6cef72410890c8c636bbdb4b2781b63659116 Mon Sep 17 00:00:00 2001 From: Jinzhe Zeng Date: Sun, 18 Oct 2020 20:52:30 -0400 Subject: [PATCH 10/12] install tf --- .travis.yml | 2 +- setup.py | 7 +++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index f6b4c0d137..b434af6819 100644 --- a/.travis.yml +++ b/.travis.yml @@ -90,6 +90,6 @@ jobs: after_success: - if [[ $TRAVIS_TAG ]]; then python -m twine upload wheelhouse/*; python -m twine upload dist/*.tar.gz; fi install: - - pip install .[test] + - pip install .[cpu,test] script: - cd source/tests && python -m unittest diff --git a/setup.py b/setup.py index 6f9adc35e7..4ff61dfd66 100644 --- a/setup.py +++ b/setup.py @@ -20,7 +20,7 @@ install_requires=['numpy', 'scipy', 'pyyaml'] setup_requires=['setuptools_scm', 'scikit-build'] -extras_require = dict() +extras_require = {"cpu": "", "gpu": ""} tf_spec = importlib.util.find_spec("tensorflow") if tf_spec: tf_install_dir = tf_spec.submodule_search_locations[0] @@ -32,7 +32,10 @@ else: tf_version = os.environ.get('TENSORFLOW_VERSION', '2.3') setup_requires.append("tensorflow==" + tf_version) - extras_require = {"cpu": "tensorflow_cpu==" + tf_version, "gpu": "tensorflow==" + tf_version} + if LegacyVersion(tf_version) < LegacyVersion("1.15") or (LegacyVersion(tf_version) >= LegacyVersion("2.0") and LegacyVersion(tf_version) < LegacyVersion("2.1")): + extras_require = {"cpu": "tensorflow==" + tf_version, "gpu": "tensorflow_gpu==" + tf_version} + else: + extras_require = {"cpu": "tensorflow_cpu==" + tf_version, "gpu": "tensorflow==" + tf_version} tf_install_dir = path.join(path.dirname(path.abspath(__file__)), '.egg', pkg_resources.Distribution(project_name="tensorflow", version=tf_version, platform=get_platform()).egg_name(), From 7928151af0ae4917ef3b6dd28015d3186495db47 Mon Sep 17 00:00:00 2001 From: Jinzhe Zeng Date: Sun, 18 Oct 2020 20:59:35 -0400 Subject: [PATCH 11/12] try to fix it --- setup.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/setup.py b/setup.py index 4ff61dfd66..b21a42223e 100644 --- a/setup.py +++ b/setup.py @@ -20,7 +20,7 @@ install_requires=['numpy', 'scipy', 'pyyaml'] setup_requires=['setuptools_scm', 'scikit-build'] -extras_require = {"cpu": "", "gpu": ""} +extras_require = {"cpu": [], "gpu": []} tf_spec = importlib.util.find_spec("tensorflow") if tf_spec: tf_install_dir = tf_spec.submodule_search_locations[0] @@ -33,9 +33,9 @@ tf_version = os.environ.get('TENSORFLOW_VERSION', '2.3') setup_requires.append("tensorflow==" + tf_version) if LegacyVersion(tf_version) < LegacyVersion("1.15") or (LegacyVersion(tf_version) >= LegacyVersion("2.0") and LegacyVersion(tf_version) < LegacyVersion("2.1")): - extras_require = {"cpu": "tensorflow==" + tf_version, "gpu": "tensorflow_gpu==" + tf_version} + extras_require = {"cpu": ["tensorflow==" + tf_version], "gpu": ["tensorflow-gpu==" + tf_version]} else: - extras_require = {"cpu": "tensorflow_cpu==" + tf_version, "gpu": "tensorflow==" + tf_version} + extras_require = {"cpu": ["tensorflow-cpu==" + tf_version], "gpu": ["tensorflow==" + tf_version]} tf_install_dir = path.join(path.dirname(path.abspath(__file__)), '.egg', pkg_resources.Distribution(project_name="tensorflow", version=tf_version, platform=get_platform()).egg_name(), From 7d98beb2d4dc68ca92f2c70d9505b68035af0e2e Mon Sep 17 00:00:00 2001 From: Jinzhe Zeng Date: Sun, 18 Oct 2020 21:07:14 -0400 Subject: [PATCH 12/12] set extras_require --- setup.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/setup.py b/setup.py index b21a42223e..db86971589 100644 --- a/setup.py +++ b/setup.py @@ -20,7 +20,11 @@ install_requires=['numpy', 'scipy', 'pyyaml'] setup_requires=['setuptools_scm', 'scikit-build'] -extras_require = {"cpu": [], "gpu": []} +tf_version = os.environ.get('TENSORFLOW_VERSION', '2.3') +if LegacyVersion(tf_version) < LegacyVersion("1.15") or (LegacyVersion(tf_version) >= LegacyVersion("2.0") and LegacyVersion(tf_version) < LegacyVersion("2.1")): + extras_require = {"cpu": ["tensorflow==" + tf_version], "gpu": ["tensorflow-gpu==" + tf_version]} +else: + extras_require = {"cpu": ["tensorflow-cpu==" + tf_version], "gpu": ["tensorflow==" + tf_version]} tf_spec = importlib.util.find_spec("tensorflow") if tf_spec: tf_install_dir = tf_spec.submodule_search_locations[0] @@ -30,12 +34,7 @@ if tf_spec: tf_install_dir = tf_spec.submodule_search_locations[0] else: - tf_version = os.environ.get('TENSORFLOW_VERSION', '2.3') setup_requires.append("tensorflow==" + tf_version) - if LegacyVersion(tf_version) < LegacyVersion("1.15") or (LegacyVersion(tf_version) >= LegacyVersion("2.0") and LegacyVersion(tf_version) < LegacyVersion("2.1")): - extras_require = {"cpu": ["tensorflow==" + tf_version], "gpu": ["tensorflow-gpu==" + tf_version]} - else: - extras_require = {"cpu": ["tensorflow-cpu==" + tf_version], "gpu": ["tensorflow==" + tf_version]} tf_install_dir = path.join(path.dirname(path.abspath(__file__)), '.egg', pkg_resources.Distribution(project_name="tensorflow", version=tf_version, platform=get_platform()).egg_name(),