From e5f6914bd2e64060d625237aba10b949aace8f62 Mon Sep 17 00:00:00 2001 From: Glenn Johnson Date: Fri, 18 Feb 2022 14:10:00 -0600 Subject: py-tensorflow: add versions 2.5.0 and 2.6.0 (#27138) * py-tensorflow: add versions 2.5.0 and 2.6.0 - add version 2.5.0 - add version 2.6.0 - add patches for newer protobuf - set constraints * Remove import os. left over from testing * Remove unused patch file * Update var/spack/repos/builtin/packages/py-tensorflow/package.py Co-authored-by: Adam J. Stewart * Update var/spack/repos/builtin/packages/py-tensorflow/package.py Co-authored-by: Adam J. Stewart * Update var/spack/repos/builtin/packages/py-tensorflow/package.py Co-authored-by: Adam J. Stewart * Update var/spack/repos/builtin/packages/py-tensorflow/package.py Co-authored-by: Adam J. Stewart * Update var/spack/repos/builtin/packages/py-tensorflow/package.py Co-authored-by: Adam J. Stewart * Update var/spack/repos/builtin/packages/py-tensorflow/package.py Co-authored-by: Adam J. Stewart * Add py-clang dependency * Adjust py-clang constraint * Build tensorflow with tensorboard - tensorflow - added 2.6.1 and 2.6.2 versions - tensorboard - have bazel use number of jobs set by spack - add versions and constraints - new package: py-tensorboard-data-server - use wheel for py-tensorboard-plugin-wit This package can not build with newer versions of bazel that are needed for newer versions of py-tensorboard. * Update var/spack/repos/builtin/packages/py-clang/package.py Co-authored-by: Adam J. Stewart * Remove empty line at end of file * Fix import sorting * Adjust python dependencies on py-clang * Add version 2.7.0 of pt-tensorflow and py-tensorboard * Adjust bazel constraints * bazel-4 support begins with py-tensorflow-2.7.0 * Adjust dependencies * Loosen cuda constraint on versions > 2.5 Tensorflow-2.5 and above can use cuda up to version 11.4. * Add constraints to patch The 0008-Fix-protobuf-errors-when-using-system-protobuf.patch patch should only apply to versions 2.5 and above. * Adjust constraints - versions 2.4 and below need protobuf-3.12 and below - versions 2.4 and above can use up to cuda-11.4 - versions 2.2 and below can not use cudnn-8 - the null_linker_bin patch should only be applied to versions 2.5 and above. * Update var/spack/repos/builtin/packages/py-tensorflow/package.py Co-authored-by: Adam J. Stewart * Update var/spack/repos/builtin/packages/py-tensorflow/package.py Co-authored-by: Adam J. Stewart * Fix py-grpcio dependency for version 2.7 Also, make sure py-h5py mpi specs are consistent. * Add llvm as run dependency. * Fix python spec for py-tensorboard * Fix py-google-auth spec for py-tensorboard * Do not override the pip spec for tensorboard-plugin-wit * Converted py-tensorboard-plugin-wit to wheel only package * Fix bazel dependency spec in tensorflow * Adjust pip masks - allow tensorboard to be specified in pip constraints - mask tensorflow-estimator * Remove blank line at end of file * Adjust pip constraints in setup.py Also, adjust constraint on a patch that is fixed in 2.7 * Fix flake8 error Adjust formatting for consistency. * Get bazel dep right * Fix old cudnn dependency, caught in audit test * Adjust the regex to ensure proper line is changed * Add py-libclang package - Stripped the py-clang package down to just version 5 - added comments to indicate the purpose of py-clang and that py-libclang should be preferred - set dependencies accordingly in py-tensorflow * Remove cap on py-h5py dependency for v2.7 * Add TODO entries for tensorflow-io-gcs-filesystem * Edit some comments * Add phases and select python in PATH for tensorboard-data-server * py-libclang - remove py-wheel dependency - remove raw string notation in filter_file * py-tensorboard-data-server - remove py-wheel dep - remove py-pip dep - use python from package class * py-tensorboard-plugin-wit - switch to PythonPackage - add version 1.8.1 - remove unneeded code * Add comment as to why a wheel is need for tensorboard-plugin-wit * remove which pip from tensorboard-data-server * Fix dependency specs in tensorboard * tweak dependencies for tensorflow * fix python constraint * Use llvm libs property * py-tensorboard-data-server - merge build into install - use std_pip_args * remove py-clang dependency * remove my edits to py-tensorboard-plugin-wit Co-authored-by: Adam J. Stewart --- .../repos/builtin/packages/py-libclang/package.py | 49 ++++ .../packages/py-tensorboard-data-server/package.py | 45 ++++ .../builtin/packages/py-tensorboard/package.py | 31 ++- .../packages/py-tensorboard/vz_projector.patch | 11 + .../builtin/packages/py-tensorboard/webapp.patch | 11 + ...rotobuf-errors-when-using-system-protobuf.patch | 29 +++ .../packages/py-tensorflow/example_parsing.patch | 11 + .../py-tensorflow/null_linker_bin_path.patch | 15 ++ .../builtin/packages/py-tensorflow/package.py | 254 +++++++++++++++------ 9 files changed, 373 insertions(+), 83 deletions(-) create mode 100644 var/spack/repos/builtin/packages/py-libclang/package.py create mode 100644 var/spack/repos/builtin/packages/py-tensorboard-data-server/package.py create mode 100644 var/spack/repos/builtin/packages/py-tensorboard/vz_projector.patch create mode 100644 var/spack/repos/builtin/packages/py-tensorboard/webapp.patch create mode 100644 var/spack/repos/builtin/packages/py-tensorflow/0008-Fix-protobuf-errors-when-using-system-protobuf.patch create mode 100644 var/spack/repos/builtin/packages/py-tensorflow/example_parsing.patch create mode 100644 var/spack/repos/builtin/packages/py-tensorflow/null_linker_bin_path.patch (limited to 'var') diff --git a/var/spack/repos/builtin/packages/py-libclang/package.py b/var/spack/repos/builtin/packages/py-libclang/package.py new file mode 100644 index 0000000000..f3ce59e490 --- /dev/null +++ b/var/spack/repos/builtin/packages/py-libclang/package.py @@ -0,0 +1,49 @@ +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +from spack import * + + +class PyLibclang(PythonPackage): + """The repository contains code that taken from the LLVM project, to make + it easier to install clang's python bindings.""" + + homepage = "https://github.com/sighingnow/libclang" + + url = "https://github.com/sighingnow/libclang/archive/refs/tags/llvm-11.1.0.tar.gz" + + version( + "11.1.0", + sha256="0b53b3c237725e193c4d2bbbe096f1a1da0f0e5cd528f2892e4dfed3c8fe9506", + ) + version( + "11.0.1", + sha256="739ae984a4a4043ae4d3b4db74597a36a8e46b6f0cbd139c7d2703faf40c5390", + ) + version( + "11.0.0", + sha256="aec204414ea412e4d4e041b0bf48123881338ac723bbcfa948f2a1b92a2428b5", + ) + version( + "10.0.1", + sha256="c15d8f97c4d0f3d4501e8b2625b343569fd92690afebe6260a2c64463d713995", + ) + version( + "9.0.1", + sha256="fc84e7bf3b0eb4f11c496d6603f111e3d8cda97094d6c9c512361371f1b76f1c", + ) + + depends_on("python@2.7:2.8,3.3:", type=("build", "run")) + depends_on("py-setuptools", type="build") + + for ver in ["9", "10", "11"]: + depends_on("llvm+clang@" + ver, when="@" + ver, type="build") + + def patch(self): + filter_file( + "source_dir = './native/'", + "source_dir = '{0}'".format(self.spec["llvm"].libs.directories[0]), + "setup.py", string=True, + ) diff --git a/var/spack/repos/builtin/packages/py-tensorboard-data-server/package.py b/var/spack/repos/builtin/packages/py-tensorboard-data-server/package.py new file mode 100644 index 0000000000..1409be6087 --- /dev/null +++ b/var/spack/repos/builtin/packages/py-tensorboard-data-server/package.py @@ -0,0 +1,45 @@ +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +import glob + +from spack import * + + +class PyTensorboardDataServer(PythonPackage): + """Fast data loading for TensorBoard""" + + homepage = "https://github.com/tensorflow/tensorboard/tree/master/tensorboard/data/server" + git = "https://github.com/tensorflow/tensorboard" + + version('0.6.1', commit='6acf0be88b5727e546dd64a8b9b12d790601d561') + + depends_on('python@3.6:', type=('build', 'run')) + depends_on('py-setuptools', type='build') + depends_on('rust', type='build') + + def setup_build_environment(self, env): + env.set('CARGO_HOME', self.stage.source_path) + + def install(self, spec, prefix): + with working_dir(join_path('tensorboard', 'data', 'server')): + cargo = which('cargo') + cargo('build', '--release') + + with working_dir(join_path('tensorboard', 'data', 'server', + 'pip_package')): + python('build.py', + '--out-dir={0}'.format(self.stage.source_path), + '--server-binary={0}'.format(join_path(self.stage.source_path, + 'tensorboard', + 'data', + 'server', + 'target', + 'release', + 'rustboard'))) + + wheel = glob.glob('*.whl')[0] + args = std_pip_args + ['--prefix=' + prefix, wheel] + pip(*args) diff --git a/var/spack/repos/builtin/packages/py-tensorboard/package.py b/var/spack/repos/builtin/packages/py-tensorboard/package.py index 337a64f41c..e761016d49 100644 --- a/var/spack/repos/builtin/packages/py-tensorboard/package.py +++ b/var/spack/repos/builtin/packages/py-tensorboard/package.py @@ -16,15 +16,20 @@ class PyTensorboard(Package): maintainers = ['aweits'] + version('2.7.0', sha256='5632812bb9450e5741083b5b7826244ffdb732fb5bce970d526c81874a3e7fb2') + version('2.6.0', sha256='3d1e0a05828b25c1c28bd90c73d981a0a65c6a5550510bc7983d03ab915e6503') + version('2.5.0', sha256='58c9e0c31062821ab1c02845c3b7902da92574ef7192d701b1828dacbe4ee610') version('2.4.1', sha256='736dc204aa292d221f5871077e60994a9a9ea8e33b841f0d754d510fe6cc7635') version('2.4.0', sha256='28a30794c1c797357b2086477394b59afa0b18ca48592ca3c0627f7f10536373') version('2.3.0', sha256='947a58702c2841eb4559637dbf8639633f79de9a0f422be9737f3563a1725440') version('2.2.0', sha256='d0dfbf0e4b3b5ebbc3fafa6d281d4b9aa5478eac6bac3330652ab6674278ab77') - depends_on('python@2.7:2.8,3.2:', type=('build', 'run')) + depends_on('python@2.7:2.8,3.2:', type=('build', 'run'), when='@:2.5') + depends_on('python@3.6:', type=('build', 'run'), when='@2.6:') depends_on('bazel@2.1.0:', type='build', when='@2.2.0:') + depends_on('bazel@3.7.0:', type='build', when='@2.5.0:') depends_on('py-pip', type='build') - depends_on('py-wheel', type='build') + depends_on('py-wheel@0.26:', type='build') depends_on('py-setuptools@41.0.0:', type=('build', 'run')) depends_on('py-absl-py@0.4:', type=('build', 'run')) depends_on('py-markdown@2.6.8:', type=('build', 'run')) @@ -32,22 +37,31 @@ class PyTensorboard(Package): depends_on('py-futures@3.1.1:', type=('build', 'run'), when='^python@:2') depends_on('py-grpcio@1.24.3:', type=('build', 'run'), when='@2.3:') depends_on('py-grpcio@1.23.3:', type=('build', 'run'), when='@2.2') - depends_on('py-google-auth@1.6.3:1', type=('build', 'run')) + depends_on('py-google-auth@1.6.3:1', type=('build', 'run'), when='@:2.6') + depends_on('py-google-auth@1.6.3:2', type=('build', 'run'), when='@2.7:') depends_on('py-numpy@1.12.0:', type=('build', 'run')) depends_on('py-protobuf@3.6.0:', type=('build', 'run')) - depends_on('py-six@1.10.0:', type=('build', 'run')) + depends_on('py-six@1.10.0:', type=('build', 'run'), when='@:2.4') depends_on('py-werkzeug@0.11.15:', type=('build', 'run')) - depends_on('py-wheel', type=('build', 'run')) - depends_on('py-wheel@0.26:', type=('build', 'run'), when='@0.6: ^python@3:') depends_on('py-google-auth-oauthlib@0.4.1:0.4', type=('build', 'run')) - depends_on('py-tensorboard-plugin-wit@1.6.0:', type=('build', 'run'), when='@2.2.0:') + + depends_on('py-tensorboard-plugin-wit@1.6.0:', type=('build', 'run')) + + depends_on('py-tensorboard-data-server@0.6', type=('build', 'run'), when='@2.5:') extends('python') - patch('tboard_shellenv.patch') + patch('tboard_shellenv.patch', when='@:2.4') + patch('webapp.patch', when='@2.7:') + patch('vz_projector.patch', when='@2.7:') phases = ['configure', 'build', 'install'] + # Version 2.6.0 and above do not build in parallel + @property + def parallel(self): + return self.spec.version < Version('2.6.0') + def patch(self): filter_file('build --define=angular_ivy_enabled=True', 'build --define=angular_ivy_enabled=True\n' @@ -100,6 +114,7 @@ class PyTensorboard(Package): '--verbose_failures', '--spawn_strategy=local', '--subcommands=pretty_print', + '--jobs={0}'.format(make_jobs), '//tensorboard/pip_package') def install(self, spec, prefix): diff --git a/var/spack/repos/builtin/packages/py-tensorboard/vz_projector.patch b/var/spack/repos/builtin/packages/py-tensorboard/vz_projector.patch new file mode 100644 index 0000000000..5eb50968fb --- /dev/null +++ b/var/spack/repos/builtin/packages/py-tensorboard/vz_projector.patch @@ -0,0 +1,11 @@ +diff -ru a/tensorboard/plugins/projector/vz_projector/BUILD b/tensorboard/plugins/projector/vz_projector/BUILD +--- a/tensorboard/plugins/projector/vz_projector/BUILD 2021-10-13 11:04:47.000000000 -0500 ++++ b/tensorboard/plugins/projector/vz_projector/BUILD 2021-12-02 14:27:59.275004056 -0600 +@@ -64,6 +64,7 @@ + "@npm//@polymer/decorators", + "@npm//@polymer/polymer", + "@npm//numeric", ++ "@npm//@types/three", + "@npm//three", + "@npm//umap-js", + ], diff --git a/var/spack/repos/builtin/packages/py-tensorboard/webapp.patch b/var/spack/repos/builtin/packages/py-tensorboard/webapp.patch new file mode 100644 index 0000000000..e3bd2ada8e --- /dev/null +++ b/var/spack/repos/builtin/packages/py-tensorboard/webapp.patch @@ -0,0 +1,11 @@ +diff -ru a/tensorboard/webapp/runs/views/runs_table/regex_edit_dialog_component.ts b/tensorboard/webapp/runs/views/runs_table/regex_edit_dialog_component.ts +--- a/tensorboard/webapp/runs/views/runs_table/regex_edit_dialog_component.ts 2021-10-13 11:04:47.000000000 -0500 ++++ b/tensorboard/webapp/runs/views/runs_table/regex_edit_dialog_component.ts 2021-12-01 20:39:45.512831971 -0600 +@@ -79,6 +79,6 @@ + + handleFocusOut() { + clearTimeout(this.timeOutId); +- this.timeOutId = setTimeout(this.resetFocus.bind(this), 0); ++ this.timeOutId = window.setTimeout(this.resetFocus.bind(this), 0); + } + } diff --git a/var/spack/repos/builtin/packages/py-tensorflow/0008-Fix-protobuf-errors-when-using-system-protobuf.patch b/var/spack/repos/builtin/packages/py-tensorflow/0008-Fix-protobuf-errors-when-using-system-protobuf.patch new file mode 100644 index 0000000000..201129fe25 --- /dev/null +++ b/var/spack/repos/builtin/packages/py-tensorflow/0008-Fix-protobuf-errors-when-using-system-protobuf.patch @@ -0,0 +1,29 @@ +From 2ea8d31a2a75de75f838b4650e1531c346dfa6fe Mon Sep 17 00:00:00 2001 +From: sclarkson +Date: Thu, 12 Aug 2021 03:23:28 -0700 +Subject: [PATCH 8/8] Fix protobuf errors when using system protobuf + +When tensorflow and python protobuf use the same instance of +libprotobuf, pywrap_tensorflow must be imported before anything +else that would import protobuf definitions. +--- + tensorflow/python/__init__.py | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/tensorflow/python/__init__.py b/tensorflow/python/__init__.py +index 6efba380ca0..38d1eb55027 100644 +--- a/tensorflow/python/__init__.py ++++ b/tensorflow/python/__init__.py +@@ -37,8 +37,8 @@ import traceback + # go/tf-wildcard-import + # pylint: disable=wildcard-import,g-bad-import-order,g-import-not-at-top + +-from tensorflow.python.eager import context + from tensorflow.python import pywrap_tensorflow as _pywrap_tensorflow ++from tensorflow.python.eager import context + + # pylint: enable=wildcard-import + +-- +2.32.0 + diff --git a/var/spack/repos/builtin/packages/py-tensorflow/example_parsing.patch b/var/spack/repos/builtin/packages/py-tensorflow/example_parsing.patch new file mode 100644 index 0000000000..b16fd6e388 --- /dev/null +++ b/var/spack/repos/builtin/packages/py-tensorflow/example_parsing.patch @@ -0,0 +1,11 @@ +--- a/tensorflow/core/kernels/example_parsing_ops.cc ++++ b/tensorflow/core/kernels/example_parsing_ops.cc +@@ -1218,7 +1218,7 @@ class DecodeJSONExampleOp : public OpKernel { + resolver_.get(), "type.googleapis.com/tensorflow.Example", &in, &out); + OP_REQUIRES(ctx, status.ok(), + errors::InvalidArgument("Error while parsing JSON: ", +- string(status.error_message()))); ++ string(status.message()))); + } + } + diff --git a/var/spack/repos/builtin/packages/py-tensorflow/null_linker_bin_path.patch b/var/spack/repos/builtin/packages/py-tensorflow/null_linker_bin_path.patch new file mode 100644 index 0000000000..439120037e --- /dev/null +++ b/var/spack/repos/builtin/packages/py-tensorflow/null_linker_bin_path.patch @@ -0,0 +1,15 @@ +diff -ru a/third_party/gpus/cuda_configure.bzl b/third_party/gpus/cuda_configure.bzl +--- a/third_party/gpus/cuda_configure.bzl 2021-05-12 13:26:41.000000000 +0000 ++++ b/third_party/gpus/cuda_configure.bzl 2021-10-28 21:38:06.949271099 +0000 +@@ -1205,10 +1205,7 @@ + # TODO: when bazel stops adding '-B/usr/bin' by default, remove this + # flag from the CROSSTOOL completely (see + # https://github.com/bazelbuild/bazel/issues/5634) +- if should_download_clang: +- cuda_defines["%{linker_bin_path}"] = "" +- else: +- cuda_defines["%{linker_bin_path}"] = host_compiler_prefix ++ cuda_defines["%{linker_bin_path}"] = "" + + cuda_defines["%{extra_no_canonical_prefixes_flags}"] = "" + cuda_defines["%{unfiltered_compile_flags}"] = "" diff --git a/var/spack/repos/builtin/packages/py-tensorflow/package.py b/var/spack/repos/builtin/packages/py-tensorflow/package.py index 5853b4bb06..bd5079c982 100644 --- a/var/spack/repos/builtin/packages/py-tensorflow/package.py +++ b/var/spack/repos/builtin/packages/py-tensorflow/package.py @@ -17,6 +17,11 @@ class PyTensorflow(Package, CudaPackage): maintainers = ['adamjstewart', 'aweits'] import_modules = ['tensorflow'] + version('2.7.0', sha256='bb124905c7fdacd81e7c842b287c169bbf377d29c74c9dacc04f96c9793747bb') + version('2.6.2', sha256='e68c1d346fc3d529653530ca346b2c62f5b31bd4fcca7ffc9c65bb39ab2f6ed3') + version('2.6.1', sha256='8e457f617bc2eb43de2a51900e7922b60a8107e2524b2576438f1acccee1d043') + version('2.6.0', sha256='41b32eeaddcbc02b0583660bcf508469550e4cd0f86b22d2abe72dfebeacde0f') + version('2.5.0', sha256='233875ea27fc357f6b714b2a0de5f6ff124b50c1ee9b3b41f9e726e9e677b86c') version('2.4.1', sha256='f681331f8fc0800883761c7709d13cda11942d4ad5ff9f44ad855e9dc78387e0') version('2.4.0', sha256='26c833b7e1873936379e810a39d14700281125257ddda8cd822c89111db6f6ae') version('2.3.2', sha256='21a703d2e68cd0677f6f9ce329198c24fd8203125599d791af9f1de61aadf31f') @@ -111,7 +116,8 @@ class PyTensorflow(Package, CudaPackage): # Need to investigate further. # See _TF_MIN_BAZEL_VERSION and _TF_MAX_BAZEL_VERSION in configure.py - depends_on('bazel@3.1.0:3', type='build', when='@2.3:') + depends_on('bazel@3.7.2:4.99.0', type='build', when='@2.7:') + depends_on('bazel@3.1.0:3.99.0', type='build', when='@2.3:2.6') depends_on('bazel@2.0.0', type='build', when='@2.2.0:2.2') depends_on('bazel@0.27.1:0.29.1', type='build', when='@2.1.0:2.1') depends_on('bazel@0.24.1:0.26.1', type='build', when='@1.15:2.0') @@ -140,38 +146,50 @@ class PyTensorflow(Package, CudaPackage): depends_on('py-future', type='build', when='^python@:2') # Listed under REQUIRED_PACKAGES in tensorflow/tools/pip_package/setup.py - depends_on('py-absl-py@0.10:0', type=('build', 'run'), when='@2.4.0:') + depends_on('py-absl-py@0.4:', type=('build', 'run'), when='@2.7:') + depends_on('py-absl-py@0.10:0', type=('build', 'run'), when='@2.4.0:2.6') depends_on('py-absl-py@0.7.0:', type=('build', 'run'), when='@1.12.1,1.14:2.3') depends_on('py-absl-py@0.1.6:', type=('build', 'run'), when='@1.5:1.11') - depends_on('py-astunparse@1.6.3:1.6', type=('build', 'run'), when='@2.4.0:') + depends_on('py-astunparse@1.6.0:', type=('build', 'run'), when='@2.7:') + depends_on('py-astunparse@1.6.3:1.6', type=('build', 'run'), when='@2.4.0:2.6') depends_on('py-astunparse@1.6.3', type=('build', 'run'), when='@2.2:2.3') depends_on('py-astor@0.6.0:', type=('build', 'run'), when='@1.6:2.1') depends_on('py-backports-weakref@1.0:', type=('build', 'run'), when='@1.3: ^python@:3.3') depends_on('py-backports-weakref@1.0rc1', type=('build', 'run'), when='@1.2.0:1.2.1') + depends_on('py-libclang@9.0.1:', type=('build', 'run'), when='@2.7:') depends_on('py-enum34@1.1.6:', type=('build', 'run'), when='@1.5: ^python@:3.3') depends_on('py-enum34@1.1.6:', type=('build', 'run'), when='@1.4.0:1.4.1') - depends_on('py-gast@0.3.3', type=('build', 'run'), when='@2.2:') + depends_on('py-gast@0.2.1:0.4', type=('build', 'run'), when='@2.7:') + depends_on('py-gast@0.4.0', type=('build', 'run'), when='@2.5:2.6') + depends_on('py-gast@0.3.3', type=('build', 'run'), when='@2.2:2.4') depends_on('py-gast@0.2.2', type=('build', 'run'), when='@1.15:2.1') depends_on('py-gast@0.2.0:', type=('build', 'run'), when='@1.6:1.14') - depends_on('py-google-pasta@0.2:0', type=('build', 'run'), when='@2.4.0:') + depends_on('py-google-pasta@0.1.1:', type=('build', 'run'), when='@2.7:') + depends_on('py-google-pasta@0.2:0', type=('build', 'run'), when='@2.4.0:2.6') depends_on('py-google-pasta@0.1.8:', type=('build', 'run'), when='@2.1:2.3') depends_on('py-google-pasta@0.1.6:', type=('build', 'run'), when='@1.14:2.0') depends_on('py-google-pasta@0.1.2:', type=('build', 'run'), when='@1.12.1') # propagate the mpi variant setting for h5py/hdf5 to avoid unexpected crashes - depends_on('py-h5py@2.10.0:2.10+mpi', type=('build', 'run'), when='@2.2:+mpi') - depends_on('py-h5py@2.10.0:2.10~mpi', type=('build', 'run'), when='@2.2:~mpi') + depends_on('py-h5py@2.9:+mpi', type=('build', 'run'), when='@2.7:+mpi') + depends_on('py-h5py@2.9:~mpi', type=('build', 'run'), when='@2.7:~mpi') + depends_on('py-h5py@3.1+mpi', type=('build', 'run'), when='@2.5:2.6+mpi') + depends_on('py-h5py@3.1~mpi', type=('build', 'run'), when='@2.5:2.6~mpi') + depends_on('py-h5py@2.10+mpi', type=('build', 'run'), when='@2.2:2.4+mpi') + depends_on('py-h5py@2.10~mpi', type=('build', 'run'), when='@2.2:2.4~mpi') depends_on('hdf5+mpi', type='build', when='@2.2:+mpi') depends_on('hdf5~mpi', type='build', when='@2.2:~mpi') depends_on('py-keras-applications@1.0.8:', type=('build', 'run'), when='@1.15:2.1') depends_on('py-keras-applications@1.0.6:', type=('build', 'run'), when='@1.12:1.14') depends_on('py-keras-applications@1.0.5:', type=('build', 'run'), when='@1.11.0:1.11') - depends_on('py-keras-preprocessing@1.1.2:1.1', type=('build', 'run'), when='@2.4:') + depends_on('py-keras-preprocessing@1.1.1:', type=('build', 'run'), when='@2.7:') + depends_on('py-keras-preprocessing@1.1.2:1.1', type=('build', 'run'), when='@2.4:2.6') depends_on('py-keras-preprocessing@1.1.1:1', type=('build', 'run'), when='@2.3.0:2.3') depends_on('py-keras-preprocessing@1.1.0:', type=('build', 'run'), when='@2.1:2.2') depends_on('py-keras-preprocessing@1.0.5:', type=('build', 'run'), when='@1.12:2.0') depends_on('py-keras-preprocessing@1.0.3:', type=('build', 'run'), when='@1.11.0:1.11') # https://github.com/tensorflow/tensorflow/issues/40688 - depends_on('py-numpy@1.19.2:1.19', type=('build', 'run'), when='@2.4.0:') + depends_on('py-numpy@1.14.5:', type=('build', 'run'), when='@2.7:') + depends_on('py-numpy@1.19.2:1.19', type=('build', 'run'), when='@2.4:2.6') depends_on('py-numpy@1.16.0:1.18', type=('build', 'run'), when='@1.13.2,1.15:2.3') depends_on('py-numpy@1.14.5:1.18', type=('build', 'run'), when='@1.12.1,1.14.0') depends_on('py-numpy@1.13.3:1.14.5', type=('build', 'run'), when='@1.10.0:1.10.1') @@ -181,8 +199,8 @@ class PyTensorflow(Package, CudaPackage): depends_on('py-numpy@1.10.1:', type=('build', 'run'), when='@0.7.1:0.7 platform=darwin') depends_on('py-numpy@1.8.2:', type=('build', 'run'), when='@0.6:0.10') depends_on('py-numpy@1.9.2:', type=('build', 'run'), when='@0.5.0') - depends_on('py-opt-einsum@3.3.0:3.3', type=('build', 'run'), when='@2.4.0:') - depends_on('py-opt-einsum@2.3.2:', type=('build', 'run'), when='@1.15:2.3') + depends_on('py-opt-einsum@3.3.0:3.3', type=('build', 'run'), when='@2.4.0:2.6') + depends_on('py-opt-einsum@2.3.2:', type=('build', 'run'), when='@1.15:2.3,2.7:') depends_on('py-protobuf@3.9.2:', type=('build', 'run'), when='@2.3:') depends_on('py-protobuf@3.8.0:', type=('build', 'run'), when='@2.1:2.2') depends_on('py-protobuf@3.6.1:', type=('build', 'run'), when='@1.12:2.0') @@ -195,30 +213,43 @@ class PyTensorflow(Package, CudaPackage): depends_on('py-protobuf@3.0.0', type=('build', 'run'), when='@0.11.0') depends_on('py-protobuf@3.0.0b2', type=('build', 'run'), when='@0.7.1:0.10') depends_on('py-protobuf@3.0.0a3', type=('build', 'run'), when='@0.6:0.7.0') - depends_on('protobuf') - depends_on('flatbuffers+python@1.12.0:1.12', type=('build', 'run'), when='@2.4.0:') + depends_on('protobuf@:3.12', when='@:2.4') + depends_on('protobuf@:3.17') + depends_on('flatbuffers+python@1.12:2', type=('build', 'run'), when='@2.7:') + depends_on('flatbuffers+python@1.12', type=('build', 'run'), when='@2.4:2.6') + # tensorboard - # tensorflow-estimator - depends_on('py-termcolor@1.1.0:1.1', type=('build', 'run'), when='@2.4.0:') - depends_on('py-termcolor@1.1.0:', type=('build', 'run'), when='@1.6:2.3') - depends_on('py-wrapt@1.12.1:1.12', type=('build', 'run'), when='@2.4.0:') + depends_on('py-tensorboard@2.5.0:2.5', type=('build', 'run'), when='@2.5') + depends_on('py-tensorboard@2.6.0:2.6', type=('build', 'run'), when='@2.6') + depends_on('py-tensorboard@2.7.0:2.7', type=('build', 'run'), when='@2.7') + + depends_on('py-termcolor@1.1.0:1.1', type=('build', 'run'), when='@2.4:2.6') + depends_on('py-termcolor@1.1.0:', type=('build', 'run'), when='@1.6:2.3,2.7:') + depends_on('py-wrapt@1.11.0:', type=('build', 'run'), when='@2.7:') + depends_on('py-wrapt@1.12.1:1.12', type=('build', 'run'), when='@2.4:2.6') depends_on('py-wrapt@1.11.1:', type=('build', 'run'), when='@1.12.1,1.14:2.3') + depends_on('py-wheel@0.32:0', type=('build', 'run'), when='@2.7:') + depends_on('py-wheel@0.35:0', type=('build', 'run'), when='@2.4:2.6 ^python@3:') depends_on('py-wheel', type=('build', 'run'), when='@0.6:2.3') depends_on('py-wheel@0.26:', type=('build', 'run'), when='@0.6:2.3 ^python@3:') - depends_on('py-wheel@0.35:0', type=('build', 'run'), when='@2.4.0: ^python@3:') depends_on('py-mock@2.0.0:', type=('build', 'run'), when='@0.10: ^python@:2') depends_on('py-functools32@3.2.3:', type=('build', 'run'), when='@1.15: ^python@:2') - depends_on('py-six@1.15.0:1.15', type=('build', 'run'), when='@2.4.0:') - depends_on('py-six@1.12.0:', type=('build', 'run'), when='@2.1:2.3') + depends_on('py-six@1.15.0:1.15', type=('build', 'run'), when='@2.4:2.6') + depends_on('py-six@1.12.0:', type=('build', 'run'), when='@2.1:2.3,2.7:') depends_on('py-six@1.10.0:', type=('build', 'run'), when='@:2.0') depends_on('py-scipy@1.2.2', type=('build', 'run'), when='@2.1.0:2.1.1,2.2.0,2.3.0 ^python@:2') depends_on('py-scipy@1.4.1', type=('build', 'run'), when='@2.1.0:2.1.1,2.2.0,2.3.0 ^python@3:') depends_on('py-grpcio@1.8.6:', type=('build', 'run'), when='@1.6:1.7') - depends_on('py-typing-extensions@3.7.4:3.7', type=('build', 'run'), when='@2.4.0:') + depends_on('py-typing-extensions@3.6.6:', type=('build', 'run'), when='@2.7:') + depends_on('py-typing-extensions@3.7.4:3.7', type=('build', 'run'), when='@2.4.0:2.6') + if sys.byteorder == 'little': # Only builds correctly on little-endian machines depends_on('py-grpcio@1.8.6:', type=('build', 'run'), when='@1.8:2.3') - depends_on('py-grpcio@1.32.0:1.32', type=('build', 'run'), when='@2.4:') + depends_on('py-grpcio@1.32.0:1.32', type=('build', 'run'), when='@2.4.0:2.4') + depends_on('py-grpcio@1.34.0:1.34', type=('build', 'run'), when='@2.5.0:2.5') + depends_on('py-grpcio@1.37.0:1', type=('build', 'run'), when='@2.6.0:2.6') + depends_on('py-grpcio@1.24.3:1', type=('build', 'run'), when='@2.7.0:') # TODO: add packages for some of these dependencies depends_on('mkl', when='+mkl') @@ -226,14 +257,19 @@ class PyTensorflow(Package, CudaPackage): # depends_on('computecpp', when='+opencl+computecpp') # depends_on('trisycl', when='+opencl~computepp') depends_on('cuda@:10.2', when='+cuda @:2.3') - depends_on('cuda@:11.1', when='+cuda @2.4.0:') + depends_on('cuda@:11.4', when='+cuda @2.4:') depends_on('cudnn', when='+cuda') + depends_on('cudnn@:6', when='@0.5:0.6 +cuda') + depends_on('cudnn@:7', when='@0.7:2.2 +cuda') # depends_on('tensorrt', when='+tensorrt') depends_on('nccl', when='+nccl') depends_on('mpi', when='+mpi') # depends_on('android-ndk@10:18', when='+android') # depends_on('android-sdk', when='+android') + # TODO: add support for tensorflow-io-gcs-filesystem + # depends_on('tensorflow-io-gcs-filesystem + # Check configure and configure.py to see when these variants are supported conflicts('+mkl', when='@:1.0') conflicts('+mkl', when='platform=darwin', msg='Darwin is not yet supported') @@ -302,6 +338,20 @@ class PyTensorflow(Package, CudaPackage): patch('contrib_cloud_1.4.patch', when='@1.4:1.8') patch('contrib_cloud_1.1.patch', when='@1.1:1.3') + # needed for protobuf-3.16 and greater + patch('example_parsing.patch', when='^protobuf@3.16:') + + # allow linker to be found in PATH + # https://github.com/tensorflow/tensorflow/issues/39263 + patch('null_linker_bin_path.patch', when='@2.5:') + + # Reset import order to that of 2.4. Part of + # https://bugs.gentoo.org/800824#c3 From the patch: + # When tensorflow and python protobuf use the same instance of libprotobuf, + # pywrap_tensorflow must be imported before anything else that would import + # protobuf definitions. + patch('0008-Fix-protobuf-errors-when-using-system-protobuf.patch', when='@2.5:2.6') + phases = ['configure', 'build', 'install'] # https://www.tensorflow.org/install/source @@ -555,9 +605,10 @@ class PyTensorflow(Package, CudaPackage): env.set('TEST_TMPDIR', tmp_path) env.set('TF_SYSTEM_LIBS', 'com_google_protobuf') - # NOTE: INCLUDEDIR is not just relevant to protobuf - # see third_party/systemlibs/jsoncpp.BUILD - env.set('INCLUDEDIR', spec['protobuf'].prefix.include) + if spec.satisfies('@:2.3'): + # NOTE: INCLUDEDIR is not just relevant to protobuf + # see third_party/systemlibs/jsoncpp.BUILD + env.set('INCLUDEDIR', spec['protobuf'].prefix.include) def patch(self): if self.spec.satisfies('@2.3.0:'): @@ -565,20 +616,31 @@ class PyTensorflow(Package, CudaPackage): 'deps = protodeps,', 'tensorflow/core/platform/default/build_config.bzl', string=True) - if self.spec.satisfies('@2.4.0:'): + if self.spec.satisfies('@2.4.0:2.5'): text = ''' def protobuf_deps(): pass ''' with open('third_party/systemlibs/protobuf_deps.bzl', 'w') as f: f.write(text) + + if self.spec.satisfies('@2.5.0'): + file_to_patch = 'tensorflow/workspace2.bzl' + else: + file_to_patch = 'tensorflow/workspace.bzl' + filter_file( '"//third_party/systemlibs:protobuf.bzl": "protobuf.bzl",', '"//third_party/systemlibs:protobuf.bzl": "protobuf.bzl",\n' '"//third_party/systemlibs:protobuf_deps.bzl": "protobuf_deps.bzl",', # noqa: E501 - 'tensorflow/workspace.bzl', + file_to_patch, string=True) + # Set protobuf path + filter_file(r'(^build:linux --define=PROTOBUF_INCLUDE_PATH=).*', + r'\1{0}'.format(self.spec['protobuf'].prefix.include), + '.bazelrc') + def configure(self, spec, prefix): # NOTE: configure script is interactive. If you set the appropriate # environment variables, this interactivity is skipped. If you don't, @@ -593,72 +655,114 @@ def protobuf_deps(): # make sure xla is actually turned off if spec.satisfies('~xla'): - filter_file(r'--define with_xla_support=true', - r'--define with_xla_support=false', - '.tf_configure.bazelrc') + filter_file( + r'--define with_xla_support=true', + r'--define with_xla_support=false', + '.tf_configure.bazelrc') if spec.satisfies('@1.5.0: ~android'): # env variable is somehow ignored -> brute force # TODO: find a better solution - filter_file(r'if workspace_has_any_android_rule\(\)', - r'if True', - 'configure.py') + filter_file( + r'if workspace_has_any_android_rule\(\)', + r'if True', + 'configure.py') # version dependent fixes if spec.satisfies('@1.3.0:1.5.0'): # checksum for protobuf that bazel downloads (@github) changed - filter_file(r'sha256 = "6d43b9d223ce09e5d4ce8b0060cb8a7513577a35a64c7e3dad10f0703bf3ad93"', - r'sha256 = "e5fdeee6b28cf6c38d61243adff06628baa434a22b5ebb7432d2a7fbabbdb13d"', - 'tensorflow/workspace.bzl') + filter_file( + r'sha256 = "6d43b9d223ce09e5d4ce8b0060cb8a7513577a35a64c7e3dad10f0703bf3ad93"', + r'sha256 = "e5fdeee6b28cf6c38d61243adff06628baa434a22b5ebb7432d2a7fbabbdb13d"', + 'tensorflow/workspace.bzl') + # starting with tensorflow 1.3, tensorboard becomes a dependency - # (...but is not really needed? Tensorboard should depend on - # tensorflow, not the other way!) # -> remove from list of required packages - filter_file(r"'tensorflow-tensorboard", - r"#'tensorflow-tensorboard", - 'tensorflow/tools/pip_package/setup.py') + filter_file( + r"'tensorflow-tensorboard", + r"#'tensorflow-tensorboard", + 'tensorflow/tools/pip_package/setup.py') + if spec.satisfies('@1.5.0: ~gcp'): # google cloud support seems to be installed on default, leading # to boringssl error manually set the flag to false to avoid # installing gcp support # https://github.com/tensorflow/tensorflow/issues/20677#issuecomment-404634519 - filter_file(r'--define with_gcp_support=true', - r'--define with_gcp_support=false', - '.tf_configure.bazelrc') - if spec.satisfies('@1.6.0:'): + filter_file( + r'--define with_gcp_support=true', + r'--define with_gcp_support=false', + '.tf_configure.bazelrc') + + if spec.satisfies('@1.6.0:2.1'): # tensorboard name changed - filter_file(r"'tensorboard >=", - r"#'tensorboard >=", - 'tensorflow/tools/pip_package/setup.py') + # there are no corresponding versions of these in spack + filter_file( + r"(^\s*)'tensorboard (>=|~=)", + r"\1#'tensorboard \2", + 'tensorflow/tools/pip_package/setup.py') + if spec.satisfies('@1.8.0: ~opencl'): # 1.8.0 and 1.9.0 aborts with numpy import error during python_api # generation somehow the wrong PYTHONPATH is used... # set --distinct_host_configuration=false as a workaround # https://github.com/tensorflow/tensorflow/issues/22395#issuecomment-431229451 - filter_file('build --action_env TF_NEED_OPENCL_SYCL="0"', - 'build --action_env TF_NEED_OPENCL_SYCL="0"\n' - 'build --distinct_host_configuration=false\n' - 'build --action_env PYTHONPATH="{0}"'.format( - env['PYTHONPATH']), - '.tf_configure.bazelrc') - if spec.satisfies('@1.13.1'): + with open('.tf_configure.bazelrc', mode='a') as f: + f.write('build --distinct_host_configuration=false\n') + f.write('build --action_env PYTHONPATH="{0}"\n'.format( + env['PYTHONPATH'])) + + if spec.satisfies('@1.13.1:'): # tensorflow_estimator is an API for tensorflow # tensorflow-estimator imports tensorflow during build, so # tensorflow has to be set up first - filter_file(r"'tensorflow_estimator >=", - r"#'tensorflow_estimator >=", - 'tensorflow/tools/pip_package/setup.py') + filter_file( + r"(^\s*)'tensorflow_estimator (>=|~=)", + r"\1#'tensorflow_estimator \2", + 'tensorflow/tools/pip_package/setup.py') + + if spec.satisfies('@2.5'): + filter_file( + r"(^\s*)'keras-nightly (>=|~=)", + r"\1#'keras-nightly \2", + 'tensorflow/tools/pip_package/setup.py') + + if spec.satisfies('@2.6:'): + filter_file( + r"(^\s*)'keras (>=|~=)", + r"\1#'keras \2", + 'tensorflow/tools/pip_package/setup.py') + + if spec.satisfies('@2.6'): + filter_file( + r"(^\s*)'clang (>=|~=)", + r"\1#'clang \2", + 'tensorflow/tools/pip_package/setup.py') + + # TODO: add support for tensorflow-io-gcs-filesystem + if spec.satisfies('@2.7:'): + filter_file( + r"(^\s*)'tensorflow-io-gcs-filesystem (>=|~=)", + r"\1#'tensorflow-io-gcs-filesystem \2", + 'tensorflow/tools/pip_package/setup.py') + if spec.satisfies('@2.0.0:'): # now it depends on the nightly versions... - filter_file(r"'tf-estimator-nightly >=", - r"#'tf-estimator-nightly >=", - 'tensorflow/tools/pip_package/setup.py') - filter_file(r"REQUIRED_PACKAGES\[i\] = 'tb-nightly >=", - r"pass #REQUIRED_PACKAGES\[i\] = 'tb-nightly >=", - 'tensorflow/tools/pip_package/setup.py') - filter_file(r"'tb-nightly >=", - r"#'tb-nightly >=", - 'tensorflow/tools/pip_package/setup.py') + filter_file( + r"REQUIRED_PACKAGES\[i\] = 'tb-nightly (>=|~=)", + r"pass #REQUIRED_PACKAGES[i] = 'tb-nightly \1", + 'tensorflow/tools/pip_package/setup.py') + filter_file( + r"REQUIRED_PACKAGES\[i\] = 'tensorflow-estimator-2.0-preview", + r"pass #REQUIRED_PACKAGES[i] = 'tensorflow-estimator-2.0-preview", + 'tensorflow/tools/pip_package/setup.py') + filter_file( + r"REQUIRED_PACKAGES\[i\] = 'tf-estimator-nightly (>=|~=)", + r"pass #REQUIRED_PACKAGES[i] = 'tf-estimator-nightly \1", + 'tensorflow/tools/pip_package/setup.py') + filter_file( + r"REQUIRED_PACKAGES\[i\] = 'keras-nightly (>=|~=)", + r"pass #REQUIRED_PACKAGES[i] = 'keras-nightly \1", + 'tensorflow/tools/pip_package/setup.py') if spec.satisfies('@1.13.1 +nccl'): filter_file( @@ -674,23 +778,23 @@ def protobuf_deps(): # see tensorflow issue #31187 on github if spec.satisfies('@2.0.0:2.0'): - filter_file(r'\#define RUY_DONOTUSEDIRECTLY_AVX512 1', - '#define RUY_DONOTUSEDIRECTLY_AVX512 0', - 'tensorflow/lite/experimental/ruy/platform.h') + filter_file( + r'\#define RUY_DONOTUSEDIRECTLY_AVX512 1', + '#define RUY_DONOTUSEDIRECTLY_AVX512 0', + 'tensorflow/lite/experimental/ruy/platform.h') if spec.satisfies('+cuda'): libs = spec['cuda'].libs.directories libs.extend(spec['cudnn'].libs.directories) if '+nccl' in spec: libs.extend(spec['nccl'].libs.directories) + if '+tensorrt' in spec: libs.extend(spec['tensorrt'].libs.directories) slibs = ':'.join(libs) - filter_file('build --action_env TF_NEED_OPENCL_SYCL="0"', - 'build --action_env TF_NEED_OPENCL_SYCL="0"\n' - 'build --action_env LD_LIBRARY_PATH="' + slibs + '"', - '.tf_configure.bazelrc') + with open('.tf_configure.bazelrc', mode='a') as f: + f.write('build --action_env LD_LIBRARY_PATH="' + slibs + '"') filter_file('build:opt --copt=-march=native', '', '.tf_configure.bazelrc') -- cgit v1.2.3-60-g2f50