alinelena · Flamefire · Jan 6, 2023
diff --git a/easybuild/easyconfigs/t/TensorFlow/TensorFlow-2.9.1-foss-2022a.eb b/easybuild/easyconfigs/t/TensorFlow/TensorFlow-2.9.1-foss-2022a.eb
@@ -22,8 +22,9 @@ dependencies = [
     ('Python', '3.10.4'),
     ('h5py', '3.7.0'),
     ('cURL', '7.83.0'),
+    ('dill', '0.3.6'),
     ('double-conversion', '3.2.0'),
-    ('flatbuffers', '1.12.1'),
+    ('flatbuffers', '2.0.0'),
     ('giflib', '5.2.1'),
     ('hwloc', '2.7.1'),
     ('ICU', '71.1'),
@@ -34,11 +35,10 @@ dependencies = [
     ('nsync', '1.25.0'),
     ('SQLite', '3.38.3'),
     ('protobuf-python', '3.19.4'),
-    ('flatbuffers-python', '1.12.1'),
+    ('flatbuffers-python', '2.0'),
     ('libpng', '1.6.37'),
     ('snappy', '1.1.9'),
     ('zlib', '1.2.12'),
-    ('networkx', '2.8.4'),  # required for pythran
 ]
 
 use_pip = True
@@ -53,24 +53,20 @@ exts_list = [
     ('termcolor', '1.1.0', {
         'checksums': ['1d6d69ce66211143803fbc56652b41d73b4a400a2891d7bf7a1cdf4c02de613b'],
     }),
-    ('tensorflow-io-gcs-filesystem', '0.26.0', {
-        'source_tmpl': 'tensorflow_io_gcs_filesystem-0.26.0-cp310-cp310-manylinux_2_12_x86_64.manylinux2010_x86_64.whl',
-        'checksums': ['5457eeef1f0f5f294225808b2290a251a2e4639ec66db9d32aa4ae62e807d7e8'],
-    }),
     ('tensorflow-estimator', '2.9.0', {
-        'source_tmpl': 'tensorflow_estimator-2.9.0-py2.py3-none-any.whl',
+        'source_tmpl': 'tensorflow_estimator-%(version)s-py2.py3-none-any.whl',
         'checksums': ['e9762bb302f51bc1eb2f35d19f0190a6a2d809d754d5def788c4328fe3746744'],
     }),
     ('Werkzeug', '2.2.2', {
         'checksums': ['7ea2d48322cc7c0f8b3a215ed73eabd7b5d75d0b50e31ab006286ccff9e00b8f'],
     }),
     ('tensorboard-plugin-wit', '1.8.1', {
-        'source_tmpl': 'tensorboard_plugin_wit-1.8.1-py3-none-any.whl',
+        'source_tmpl': 'tensorboard_plugin_wit-%(version)s-py3-none-any.whl',
         'checksums': ['ff26bdd583d155aa951ee3b152b3d0cffae8005dc697f72b44a8e8c2a77a8cbe'],
     }),
     ('tensorboard-data-server', '0.6.1', {
-        'source_tmpl': 'tensorboard_data_server-0.6.1-py3-none-manylinux2010_x86_64.whl',
-        'checksums': ['d8237580755e58eff68d1f3abefb5b1e39ae5c8b127cc40920f9c4fb33f4b98a'],
+        'source_tmpl': 'tensorboard_data_server-%(version)s-py3-none-any.whl',
+        'checksums': ['809fe9887682d35c1f7d1f54f0f40f98bb1f771b14265b453ca051e2ce58fca7'],
     }),
     ('Markdown', '3.4.1', {
         'checksums': ['3b809086bb6efad416156e00a0da66fe47618a5d6918dd688f53f40c8e4cfeff'],
@@ -107,23 +103,19 @@ exts_list = [
         'checksums': ['f568809938c49abbda89826223c992b630afd23c638160ad7840cfe347710d97'],
     }),
     ('tensorboard', version, {
-        'source_tmpl': 'tensorboard-2.9.1-py3-none-any.whl',
+        'source_tmpl': SOURCE_PY3_WHL,
         'checksums': ['baa727f791776f9e5841d347127720ceed4bbd59c36b40604b95fb2ae6029276'],
     }),
     ('opt-einsum', '3.3.0', {
-        'source_tmpl': 'opt_einsum-3.3.0-py3-none-any.whl',
-        'checksums': ['2455e59e3947d3c275477df7f5205b30635e266fe6dc300e3d9f9646bfcea147'],
-    }),
-    ('libclang', '14.0.6', {
-        'modulename': 'clang',
-        'checksums': ['9052a8284d8846984f6fa826b1d7460a66d3b23a486d782633b42b6e3b418789'],
+        'source_tmpl': 'opt_einsum-%(version)s.tar.gz',
+        'checksums': ['59f6475f77bbc37dcf7cd748519c0ec60722e91e63ca114e68821c0c54a46549'],
     }),
     ('Keras-Preprocessing', '1.1.2', {
-        'source_tmpl': 'Keras_Preprocessing-1.1.2-py2.py3-none-any.whl',
-        'checksums': ['7b82029b130ff61cc99b55f3bd27427df4838576838c5b2f65940e4fcec99a7b'],
+        'source_tmpl': 'Keras_Preprocessing-%(version)s.tar.gz',
+        'checksums': ['add82567c50c8bc648c14195bf544a5ce7c1f76761536956c3d2978970179ef3'],
     }),
     ('keras', '2.9.0', {
-        'source_tmpl': 'keras-2.9.0-py2.py3-none-any.whl',
+        'source_tmpl': SOURCE_WHL,
         'checksums': ['55911256f89cfc9343c9fbe4b61ec45a2d33d89729cbe1ab9dcacf8b07b8b6ab'],
     }),
     ('google-pasta', '0.2.0', {
@@ -133,21 +125,29 @@ exts_list = [
     ('astunparse', '1.6.3', {
         'checksums': ['5ad93a8456f0d084c3456d059fd9a92cce667963232cbf763eac3bc5b7940872'],
     }),
-    ('gast', '0.4.0', {
-        'checksums': ['40feb7b8b8434785585ab224d1568b857edb18297e5a3047f1ba012bc83b42c1'],
+    ('portpicker', '1.5.2', {
+        'checksums': ['c55683ad725f5c00a41bc7db0225223e8be024b1fa564d039ed3390e4fd48fb3'],
+    }),
+    # System dependencies
+    ('tblib', '1.7.0', {
+        'checksums': ['059bd77306ea7b419d4f76016aef6d7027cc8a0785579b5aad198803435f882c'],
     }),
-    ('pythran', '0.9.11', {
-        'checksums': ['a317f91e2aade9f6550dc3bf40b5caeb45b7e012daf27e2b3e4ad928edb01667'],
+    ('astor', '0.8.1', {
+        'checksums': ['6a6effda93f4e1ce9f618779b2dd1d9d84f1e32812c23a29b3fff6fd7f63fa5e'],
     }),
-    ('beniget', '0.3.0', {
-        'checksums': ['062c893be9cdf87c3144fb15041cce4d81c67107c1591952cd45fdce789a0ff1'],
+    # Optional profile plugin + dependency
+    ('gviz-api', '1.10.0', {
+        'source_tmpl': 'gviz_api-%(version)s.tar.gz',
+        'checksums': ['846692dd8cc73224fc31b18e41589bd934e1cc05090c6576af4b4b26c2e71b90'],
     }),
-    ('portpicker', '1.5.2', {
-        'checksums': ['c55683ad725f5c00a41bc7db0225223e8be024b1fa564d039ed3390e4fd48fb3'],
+    ('tensorboard-plugin-profile', '2.8.0', {
+        'source_tmpl': 'tensorboard_plugin_profile-%(version)s.tar.gz',
+        'checksums': ['58d65c979951e7628ca3b46c798e10e5813026a3ecfbacc319cf88ebe766134a'],
     }),
     (name, version, {
+        'source_tmpl': 'v%(version)s.tar.gz',
+        'source_urls': ['https://github.com/tensorflow/tensorflow/archive/'],
         'patches': [
-            'TensorFlow-2.1.0_fix-cuda-build.patch',
             'TensorFlow-2.4.0_add-ldl.patch',
             'TensorFlow-2.4.0_dont-use-var-lock.patch',
             'TensorFlow-2.5.0_add-support-for-large-core-systems.patch',
@@ -156,9 +156,40 @@ exts_list = [
             'TensorFlow-2.5.0_fix-arm-vector-intrinsics.patch',
             'TensorFlow-2.5.0_fix-crash-on-shutdown.patch',
             'TensorFlow-2.7.1_fix_cpu_count.patch',
+            'TensorFlow-2.8.4_exclude-xnnpack-on-ppc.patch',
+            'TensorFlow-2.8.4_fix-PPC-JIT.patch',
+            'TensorFlow-2.8.4_resolve-gcc-symlinks.patch',
+            'TensorFlow-2.9.1_remove-duplicate-gpu-tests.patch',
+            'TensorFlow-2.9.1_remove-libclang-and-io-gcs-deps.patch',
+        ],
+        'checksums': [
+            {'v2.9.1.tar.gz': '6eaf86ead73e23988fe192da1db68f4d3828bcdd0f3a9dc195935e339c95dbdc'},
+            {'TensorFlow-2.4.0_add-ldl.patch': '917ee7282e782e48673596d8917c3207e60e0851bb9acf230a2a439b067af2e3'},
+            {'TensorFlow-2.4.0_dont-use-var-lock.patch':
+             'b14f2493fd2edf79abd1c4f2dde6c98a3e7d5cb9c25ab9386df874d5f072d6b5'},
+            {'TensorFlow-2.5.0_add-support-for-large-core-systems.patch':
+             '915f3477d6407fafd48269fe1e684a05ce361d9b9b85e58686682df87760f636'},
+            {'TensorFlow-2.5.0_disable-avx512-extensions.patch':
+             '3655ce24c97569ac9738c07cac85347ba6f5c815ada95b19b606ffa46d4dda03'},
+            {'TensorFlow-2.5.0-fix-alias-violation-in-absl.patch':
+             '12454fda3330fb45cd380377e283f04488b40e0b8ae7378e786ddf731a581f75'},
+            {'TensorFlow-2.5.0_fix-arm-vector-intrinsics.patch':
+             '6abfadc0f67ff3b510d70430843201cb46d7bd65db045ec9b482af70e0c8c0c8'},
+            {'TensorFlow-2.5.0_fix-crash-on-shutdown.patch':
+             '578c7493221ebd3dc25ca43d63a72cbb28fdf4112b1e2baa7390f25781bd78fd'},
+            {'TensorFlow-2.7.1_fix_cpu_count.patch':
+             '5427a4cff0afc2fe5b24776ae9ca3616c56a79c1fde0025b37bec24837bb0698'},
+            {'TensorFlow-2.8.4_exclude-xnnpack-on-ppc.patch':
+             'ebd404ac56cc4ca662483f1f591e62e11749361be57a7ba5f4b2f0d03e829884'},
+            {'TensorFlow-2.8.4_fix-PPC-JIT.patch':
+             '27d28293105b4dd0a25f58346c68b672f57215756f14a97c442d0e3317e93a2b'},
+            {'TensorFlow-2.8.4_resolve-gcc-symlinks.patch':
+             '43ce9acc6bffff68a31d2263d0064d272999b2e0a2c6546690287cd1c9c90f04'},
+            {'TensorFlow-2.9.1_remove-duplicate-gpu-tests.patch':
+             '6fe50faab28387c622c68dc3fc0cbfb2a51000cd750c1a82f8420b54fcd2509f'},
+            {'TensorFlow-2.9.1_remove-libclang-and-io-gcs-deps.patch':
+             '13a987ce8e20957d9a618493f6a0e8f67e0f9c8b11596f27bcd7322a8e62113d'},
         ],
-        'source_tmpl': 'v%(version)s.tar.gz',
-        'source_urls': ['https://github.com/tensorflow/tensorflow/archive/'],
         'test_script': 'TensorFlow-2.x_mnist-test.py',
         'test_tag_filters_cpu': '-gpu,-tpu,-no_cuda_on_cpu_tap,-no_pip,-no_oss,-oss_serial,-benchmark-test,-v1only',
         'test_tag_filters_gpu': 'gpu,-no_gpu,-nogpu,-gpu_cupti,-no_cuda11,-no_pip,-no_oss,-oss_serial,-benchmark-test,-v1only',
@@ -184,24 +215,6 @@ exts_list = [
         'testopts': "--test_timeout=3600 --test_size_filters=small",
         'testopts_gpu': "--test_timeout=3600 --test_size_filters=small --run_under=//tensorflow/tools/ci_build/gpu_build:parallel_gpu_execute",
         'with_xla': True,
-        'checksums': [
-            '6eaf86ead73e23988fe192da1db68f4d3828bcdd0f3a9dc195935e339c95dbdc',  # v2.9.1.tar.gz
-            '78c20aeaa7784b8ceb46238a81e8c2461137d28e0b576deeba8357d23fbe1f5a',  # TensorFlow-2.1.0_fix-cuda-build.patch
-            '917ee7282e782e48673596d8917c3207e60e0851bb9acf230a2a439b067af2e3',  # TensorFlow-2.4.0_add-ldl.patch
-            # TensorFlow-2.4.0_dont-use-var-lock.patch
-            'b14f2493fd2edf79abd1c4f2dde6c98a3e7d5cb9c25ab9386df874d5f072d6b5',
-            # TensorFlow-2.5.0_add-support-for-large-core-systems.patch
-            '915f3477d6407fafd48269fe1e684a05ce361d9b9b85e58686682df87760f636',
-            # TensorFlow-2.5.0_disable-avx512-extensions.patch
-            '3655ce24c97569ac9738c07cac85347ba6f5c815ada95b19b606ffa46d4dda03',
-            # TensorFlow-2.5.0-fix-alias-violation-in-absl.patch
-            '12454fda3330fb45cd380377e283f04488b40e0b8ae7378e786ddf731a581f75',
-            # TensorFlow-2.5.0_fix-arm-vector-intrinsics.patch
-            '6abfadc0f67ff3b510d70430843201cb46d7bd65db045ec9b482af70e0c8c0c8',
-            # TensorFlow-2.5.0_fix-crash-on-shutdown.patch
-            '578c7493221ebd3dc25ca43d63a72cbb28fdf4112b1e2baa7390f25781bd78fd',
-            '5427a4cff0afc2fe5b24776ae9ca3616c56a79c1fde0025b37bec24837bb0698',  # TensorFlow-2.7.1_fix_cpu_count.patch
-        ],
     }),
 ]
 

diff --git a/easybuild/easyconfigs/t/TensorFlow/TensorFlow-2.9.1_remove-duplicate-gpu-tests.patch b/easybuild/easyconfigs/t/TensorFlow/TensorFlow-2.9.1_remove-duplicate-gpu-tests.patch
@@ -0,0 +1,55 @@
+TensorFlow adds some GPU tests twice increasing the runtime of the test suite.
+This filters out the test part meant for CPU.
+
+See https://github.com/tensorflow/tensorflow/issues/47081
+From https://github.com/tensorflow/tensorflow/pull/59129
+
+Author: Alexander Grund (TU Dresden)
+---
+ tensorflow/tensorflow.bzl | 33 +++++++++++++++++----------------
+ 1 file changed, 17 insertions(+), 16 deletions(-)
+
+diff --git a/tensorflow/tensorflow.bzl b/tensorflow/tensorflow.bzl
+index 649c8e22dcc95..d3fc0e3221497 100644
+--- a/tensorflow/tensorflow.bzl
++++ b/tensorflow/tensorflow.bzl
+@@ -1461,22 +1461,23 @@ def tf_gpu_cc_test(
+         linkopts = [],
+         **kwargs):
+     targets = []
+-    tf_cc_test(
+-        name = name,
+-        size = size,
+-        srcs = srcs,
+-        args = args,
+-        data = data,
+-        extra_copts = extra_copts + if_cuda(["-DNV_CUDNN_DISABLE_EXCEPTION"]),
+-        kernels = kernels,
+-        linkopts = linkopts,
+-        linkstatic = linkstatic,
+-        suffix = "_cpu",
+-        tags = tags,
+-        deps = deps,
+-        **kwargs
+-    )
+-    targets.append(name + "_cpu")
++    if 'gpu' not in tags:
++        tf_cc_test(
++            name = name,
++            size = size,
++            srcs = srcs,
++            args = args,
++            data = data,
++            extra_copts = extra_copts + if_cuda(["-DNV_CUDNN_DISABLE_EXCEPTION"]),
++            kernels = kernels,
++            linkopts = linkopts,
++            linkstatic = linkstatic,
++            suffix = "_cpu",
++            tags = tags,
++            deps = deps,
++            **kwargs
++        )
++        targets.append(name + "_cpu")
+     tf_cc_test(
+         name = name,
+         size = size,
diff --git a/easybuild/easyconfigs/t/TensorFlow/TensorFlow-2.9.1_remove-libclang-and-io-gcs-deps.patch b/easybuild/easyconfigs/t/TensorFlow/TensorFlow-2.9.1_remove-libclang-and-io-gcs-deps.patch
@@ -0,0 +1,35 @@
+This combines two patches from TensorFlow 2.7.1.
+
+tensorflow-io-gcs-filesystem is not available as a binary for all architectures and
+building it requires TensorFlow to be installed, i.e. there is a cyclic dependency.
+As it is not actually required (but optional) remove it from `REQUIRED_PACKAGES`.
+See https://github.com/tensorflow/tensorflow/issues/56636
+
+libclang was introduced in
+https://github.com/tensorflow/tensorflow/commit/c211472000ff57bac7fcec9b0465cf73b37bf135
+> This is in preparation to open-source TF's TFRT backend.
+> TFRT generates code using libclang python bindings as part of the build.
+Hence it is not currently used and as it is not (easily) available for all architectures
+simply remove it.
+
+Patch added by Simon Branford (University of Birmingham)
+Updated by Alexander Grund (TU Dresden)
+
+--- a/tensorflow/tools/pip_package/setup.py
++++ b/tensorflow/tools/pip_package/setup.py
+@@ -83,7 +83,6 @@ REQUIRED_PACKAGES = [
+     'google_pasta >= 0.1.1',
+     'h5py >= 2.9.0',
+     'keras_preprocessing >= 1.1.1',  # 1.1.0 needs tensorflow==1.7
+-    'libclang >= 13.0.0',
+     'numpy >= 1.20',
+     'opt_einsum >= 2.3.2',
+     'packaging',
+@@ -100,7 +99,6 @@ REQUIRED_PACKAGES = [
+     'termcolor >= 1.1.0',
+     'typing_extensions >= 3.6.6',
+     'wrapt >= 1.11.0',
+-    'tensorflow-io-gcs-filesystem >= 0.23.1',
+     # grpcio does not build correctly on big-endian machines due to lack of
+     # BoringSSL support.
+     # See https://github.com/tensorflow/tensorflow/issues/17882.