From e20c05fcdfc7a9ad995956bfee269e123860288f Mon Sep 17 00:00:00 2001 From: Massimiliano Culpo Date: Fri, 6 Oct 2023 10:24:21 +0200 Subject: Make "minimal" the default duplicate strategy (#39621) * Allow branching out of the "generic build" unification set For cases like the one in https://github.com/spack/spack/pull/39661 we need to relax rules on unification sets. The issue is that, right now, nodes in the "generic build" unification set are unified together with their build dependencies. This was done out of caution to avoid the risk of circular dependencies, which would ultimately cause a very slow solve. For build-tools like Cython, however, the build dependencies is masked by a long chain of "build, run" dependencies that belong in the "generic build" unification space. To allow splitting on cases like this, we relax the rule disallowing branching out of the "generic build" unification set. * Fix issue with pure build virtual dependencies Pure build virtual dependencies were not accounted properly in the list of possible virtuals. This caused some facts connecting virtuals to the corresponding providers to not be emitted, and in the end lead to unsat problems. * Fixed a few issues in packages py-gevent: restore dependency on py-cython@3 jsoncpp: fix typo in build dependency ecp-data-vis-sdk: update spack.yaml and cmake recipe py-statsmodels: add v0.13.5 * Make dependency on "blt" of type "build" --- etc/spack/defaults/concretizer.yaml | 2 +- lib/spack/docs/build_settings.rst | 167 +- lib/spack/docs/images/shapely_duplicates.svg | 2784 ++++++++++++++++++++ lib/spack/spack/solver/asp.py | 1 + lib/spack/spack/solver/concretize.lp | 19 +- lib/spack/spack/solver/counter.py | 15 +- lib/spack/spack/spec.py | 11 +- lib/spack/spack/test/concretize.py | 28 +- lib/spack/spack/test/data/config/concretizer.yaml | 2 +- .../cloud_pipelines/stacks/data-vis-sdk/spack.yaml | 16 +- var/spack/repos/builtin/packages/camp/package.py | 2 +- var/spack/repos/builtin/packages/cmake/package.py | 2 +- .../repos/builtin/packages/jsoncpp/package.py | 2 +- var/spack/repos/builtin/packages/mgard/package.py | 2 +- .../repos/builtin/packages/py-gevent/package.py | 3 +- .../builtin/packages/py-statsmodels/package.py | 12 +- var/spack/repos/builtin/packages/raja/package.py | 2 +- var/spack/repos/builtin/packages/umpire/package.py | 2 +- .../duplicates.test/packages/pkg-config/package.py | 16 + .../packages/virtual-build/package.py | 16 + 20 files changed, 2984 insertions(+), 120 deletions(-) create mode 100644 lib/spack/docs/images/shapely_duplicates.svg create mode 100644 var/spack/repos/duplicates.test/packages/pkg-config/package.py create mode 100644 var/spack/repos/duplicates.test/packages/virtual-build/package.py diff --git a/etc/spack/defaults/concretizer.yaml b/etc/spack/defaults/concretizer.yaml index 598bb8c349..edefa552ce 100644 --- a/etc/spack/defaults/concretizer.yaml +++ b/etc/spack/defaults/concretizer.yaml @@ -41,4 +41,4 @@ concretizer: # "none": allows a single node for any package in the DAG. # "minimal": allows the duplication of 'build-tools' nodes only (e.g. py-setuptools, cmake etc.) # "full" (experimental): allows separation of the entire build-tool stack (e.g. the entire "cmake" subDAG) - strategy: none \ No newline at end of file + strategy: minimal diff --git a/lib/spack/docs/build_settings.rst b/lib/spack/docs/build_settings.rst index 0bbd27e8c3..402b33f6a2 100644 --- a/lib/spack/docs/build_settings.rst +++ b/lib/spack/docs/build_settings.rst @@ -3,6 +3,103 @@ SPDX-License-Identifier: (Apache-2.0 OR MIT) + +.. _concretizer-options: + +========================================== +Concretization Settings (concretizer.yaml) +========================================== + +The ``concretizer.yaml`` configuration file allows to customize aspects of the +algorithm used to select the dependencies you install. The default configuration +is the following: + +.. literalinclude:: _spack_root/etc/spack/defaults/concretizer.yaml + :language: yaml + +-------------------------------- +Reuse already installed packages +-------------------------------- + +The ``reuse`` attribute controls whether Spack will prefer to use installed packages (``true``), or +whether it will do a "fresh" installation and prefer the latest settings from +``package.py`` files and ``packages.yaml`` (``false``). +You can use: + +.. code-block:: console + + % spack install --reuse + +to enable reuse for a single installation, and you can use: + +.. code-block:: console + + spack install --fresh + +to do a fresh install if ``reuse`` is enabled by default. +``reuse: true`` is the default. + +------------------------------------------ +Selection of the target microarchitectures +------------------------------------------ + +The options under the ``targets`` attribute control which targets are considered during a solve. +Currently the options in this section are only configurable from the ``concretizer.yaml`` file +and there are no corresponding command line arguments to enable them for a single solve. + +The ``granularity`` option can take two possible values: ``microarchitectures`` and ``generic``. +If set to: + +.. code-block:: yaml + + concretizer: + targets: + granularity: microarchitectures + +Spack will consider all the microarchitectures known to ``archspec`` to label nodes for +compatibility. If instead the option is set to: + +.. code-block:: yaml + + concretizer: + targets: + granularity: generic + +Spack will consider only generic microarchitectures. For instance, when running on an +Haswell node, Spack will consider ``haswell`` as the best target in the former case and +``x86_64_v3`` as the best target in the latter case. + +The ``host_compatible`` option is a Boolean option that determines whether or not the +microarchitectures considered during the solve are constrained to be compatible with the +host Spack is currently running on. For instance, if this option is set to ``true``, a +user cannot concretize for ``target=icelake`` while running on an Haswell node. + +--------------- +Duplicate nodes +--------------- + +The ``duplicates`` attribute controls whether the DAG can contain multiple configurations of +the same package. This is mainly relevant for build dependencies, which may have their version +pinned by some nodes, and thus be required at different versions by different nodes in the same +DAG. + +The ``strategy`` option controls how the solver deals with duplicates. If the value is ``none``, +then a single configuration per package is allowed in the DAG. This means, for instance, that only +a single ``cmake`` or a single ``py-setuptools`` version is allowed. The result would be a slightly +faster concretization, at the expense of making a few specs unsolvable. + +If the value is ``minimal`` Spack will allow packages tagged as ``build-tools`` to have duplicates. +This allows, for instance, to concretize specs whose nodes require different, and incompatible, ranges +of some build tool. For instance, in the figure below the latest `py-shapely` requires a newer `py-setuptools`, +while `py-numpy` still needs an older version: + +.. figure:: images/shapely_duplicates.svg + :scale: 70 % + :align: center + +Up to Spack v0.20 ``duplicates:strategy:none`` was the default (and only) behavior. From Spack v0.21 the +default behavior is ``duplicates:strategy:minimal``. + .. _build-settings: ================================ @@ -232,76 +329,6 @@ Specific limitations include: then Spack will not add a new external entry (``spack config blame packages`` can help locate all external entries). -.. _concretizer-options: - ----------------------- -Concretizer options ----------------------- - -``packages.yaml`` gives the concretizer preferences for specific packages, -but you can also use ``concretizer.yaml`` to customize aspects of the -algorithm it uses to select the dependencies you install: - -.. literalinclude:: _spack_root/etc/spack/defaults/concretizer.yaml - :language: yaml - -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Reuse already installed packages -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -The ``reuse`` attribute controls whether Spack will prefer to use installed packages (``true``), or -whether it will do a "fresh" installation and prefer the latest settings from -``package.py`` files and ``packages.yaml`` (``false``). -You can use: - -.. code-block:: console - - % spack install --reuse - -to enable reuse for a single installation, and you can use: - -.. code-block:: console - - spack install --fresh - -to do a fresh install if ``reuse`` is enabled by default. -``reuse: true`` is the default. - -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Selection of the target microarchitectures -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -The options under the ``targets`` attribute control which targets are considered during a solve. -Currently the options in this section are only configurable from the ``concretizer.yaml`` file -and there are no corresponding command line arguments to enable them for a single solve. - -The ``granularity`` option can take two possible values: ``microarchitectures`` and ``generic``. -If set to: - -.. code-block:: yaml - - concretizer: - targets: - granularity: microarchitectures - -Spack will consider all the microarchitectures known to ``archspec`` to label nodes for -compatibility. If instead the option is set to: - -.. code-block:: yaml - - concretizer: - targets: - granularity: generic - -Spack will consider only generic microarchitectures. For instance, when running on an -Haswell node, Spack will consider ``haswell`` as the best target in the former case and -``x86_64_v3`` as the best target in the latter case. - -The ``host_compatible`` option is a Boolean option that determines whether or not the -microarchitectures considered during the solve are constrained to be compatible with the -host Spack is currently running on. For instance, if this option is set to ``true``, a -user cannot concretize for ``target=icelake`` while running on an Haswell node. - .. _package-requirements: -------------------- diff --git a/lib/spack/docs/images/shapely_duplicates.svg b/lib/spack/docs/images/shapely_duplicates.svg new file mode 100644 index 0000000000..912f03b2e5 --- /dev/null +++ b/lib/spack/docs/images/shapely_duplicates.svg @@ -0,0 +1,2784 @@ + + + + + + + + + + image/svg+xml + + + + + + + + G + + + + bqm4trdmbbqhrthe6flwnxp57cfbbser + + nghttp2@1.52.0/bqm4trd + + + + hsp7usvecwby6o6kszujxywbux5f5qc4 + + pkgconf@1.9.5/hsp7usv + + + + bqm4trdmbbqhrthe6flwnxp57cfbbser->hsp7usvecwby6o6kszujxywbux5f5qc4 + + + + + + s3mykqnlex5ygursynhv4cfu4p4jcp5c + + diffutils@3.9/s3mykqn + + + + gpd7yevon44acblslmgorfsxufgk3nhz + + libiconv@1.17/gpd7yev + + + + s3mykqnlex5ygursynhv4cfu4p4jcp5c->gpd7yevon44acblslmgorfsxufgk3nhz + + + + + + + al63766ivhemwb3bxsklvqmhdptf34fn + + geos@3.12.0/al63766 + + + + ma6zn6mykr7xe226v2hvu4ye7jltnddb + + cmake@3.27.4/ma6zn6m + + + + al63766ivhemwb3bxsklvqmhdptf34fn->ma6zn6mykr7xe226v2hvu4ye7jltnddb + + + + + + revhbmcsddofjb7jt3fql7fawtxjihvc + + ninja@1.11.1/revhbmc + + + + al63766ivhemwb3bxsklvqmhdptf34fn->revhbmcsddofjb7jt3fql7fawtxjihvc + + + + + + 7vxac6cvfyqggxsvd7votisi72rdfvoh + + ca-certificates-mozilla@2023-05-30/7vxac6c + + + + wuse2zg2p4ujfbsks4znlwyqumsa476w + + py-cython@0.29.36/wuse2zg + + + + 7ftqkn35sy5bmqv3wui3ap3gubqyu4f4 + + python@3.11.4/7ftqkn3 + + + + wuse2zg2p4ujfbsks4znlwyqumsa476w->7ftqkn35sy5bmqv3wui3ap3gubqyu4f4 + + + + + + + 33tg442mk3uy52ocdgd7uxbusdtkozlq + + py-pip@23.1.2/33tg442 + + + + wuse2zg2p4ujfbsks4znlwyqumsa476w->33tg442mk3uy52ocdgd7uxbusdtkozlq + + + + + + isprdjk4hdva3owdr6bgzavgaqzyjwyj + + py-wheel@0.37.1/isprdjk + + + + wuse2zg2p4ujfbsks4znlwyqumsa476w->isprdjk4hdva3owdr6bgzavgaqzyjwyj + + + + + + esl2253adih4qsbluhmzdtsxfrws4fnt + + py-setuptools@59.4.0/esl2253 + + + + wuse2zg2p4ujfbsks4znlwyqumsa476w->esl2253adih4qsbluhmzdtsxfrws4fnt + + + + + + + e3xjka5zk6vtoen2oexuzxyorp6um5rv + + openssl@3.1.3/e3xjka5 + + + + e3xjka5zk6vtoen2oexuzxyorp6um5rv->7vxac6cvfyqggxsvd7votisi72rdfvoh + + + + + + ez3cm4rogbx7at45wfi6gquti6fbo3zz + + zlib-ng@2.1.3/ez3cm4r + + + + e3xjka5zk6vtoen2oexuzxyorp6um5rv->ez3cm4rogbx7at45wfi6gquti6fbo3zz + + + + + + + 7bvgd7zcvk3hglqgbqczma5h4urvrdjb + + perl@5.38.0/7bvgd7z + + + + e3xjka5zk6vtoen2oexuzxyorp6um5rv->7bvgd7zcvk3hglqgbqczma5h4urvrdjb + + + + + + ys6bcgmvdayitnod74ppxvzbn75e7227 + + py-shapely@2.0.1/ys6bcgm + + + + ys6bcgmvdayitnod74ppxvzbn75e7227->al63766ivhemwb3bxsklvqmhdptf34fn + + + + + + + ys6bcgmvdayitnod74ppxvzbn75e7227->wuse2zg2p4ujfbsks4znlwyqumsa476w + + + + + + ys6bcgmvdayitnod74ppxvzbn75e7227->7ftqkn35sy5bmqv3wui3ap3gubqyu4f4 + + + + + + + ys6bcgmvdayitnod74ppxvzbn75e7227->33tg442mk3uy52ocdgd7uxbusdtkozlq + + + + + + ca3noh6upxuh3hdx2lnrsdvw7blgcj5p + + py-numpy@1.25.2/ca3noh6 + + + + ys6bcgmvdayitnod74ppxvzbn75e7227->ca3noh6upxuh3hdx2lnrsdvw7blgcj5p + + + + + + + + ys6bcgmvdayitnod74ppxvzbn75e7227->isprdjk4hdva3owdr6bgzavgaqzyjwyj + + + + + + 2ok2ozl5i2qphhfsbxkdtq3iezemvpsv + + py-setuptools@68.0.0/2ok2ozl + + + + ys6bcgmvdayitnod74ppxvzbn75e7227->2ok2ozl5i2qphhfsbxkdtq3iezemvpsv + + + + + + aoucvoqqeft4hsw3poydbf4mvong4nry + + ncurses@6.4/aoucvoq + + + + aoucvoqqeft4hsw3poydbf4mvong4nry->hsp7usvecwby6o6kszujxywbux5f5qc4 + + + + + + 7aawlyt3hu24znvpgwedu2s3jmz46dkn + + xz@5.4.1/7aawlyt + + + + 7ftqkn35sy5bmqv3wui3ap3gubqyu4f4->e3xjka5zk6vtoen2oexuzxyorp6um5rv + + + + + + + 7ftqkn35sy5bmqv3wui3ap3gubqyu4f4->aoucvoqqeft4hsw3poydbf4mvong4nry + + + + + + + 7ftqkn35sy5bmqv3wui3ap3gubqyu4f4->7aawlyt3hu24znvpgwedu2s3jmz46dkn + + + + + + + ygkrrpeszr4j377qqtqqecmwt27pm2ho + + expat@2.5.0/ygkrrpe + + + + 7ftqkn35sy5bmqv3wui3ap3gubqyu4f4->ygkrrpeszr4j377qqtqqecmwt27pm2ho + + + + + + + qlqyzklm3yyv6tkqgnj4tzoy7g72ejyu + + sqlite@3.42.0/qlqyzkl + + + + 7ftqkn35sy5bmqv3wui3ap3gubqyu4f4->qlqyzklm3yyv6tkqgnj4tzoy7g72ejyu + + + + + + + ihtvssgtl7yz2wj7wdla4hsi7nqfny42 + + util-linux-uuid@2.38.1/ihtvssg + + + + 7ftqkn35sy5bmqv3wui3ap3gubqyu4f4->ihtvssgtl7yz2wj7wdla4hsi7nqfny42 + + + + + + + bvcsrijbs7lp5jvlyooahoxc3zfapwfp + + gdbm@1.23/bvcsrij + + + + 7ftqkn35sy5bmqv3wui3ap3gubqyu4f4->bvcsrijbs7lp5jvlyooahoxc3zfapwfp + + + + + + + 3o2rmrxpwkmmetxmzvba6sizei5womzv + + gettext@0.21.1/3o2rmrx + + + + 7ftqkn35sy5bmqv3wui3ap3gubqyu4f4->3o2rmrxpwkmmetxmzvba6sizei5womzv + + + + + + + bcjm3vxlgrjgewpdakhpfea3y2kzcspe + + bzip2@1.0.8/bcjm3vx + + + + 7ftqkn35sy5bmqv3wui3ap3gubqyu4f4->bcjm3vxlgrjgewpdakhpfea3y2kzcspe + + + + + + + 7pjirtey2xqww2bbkil3yj3mtmasruaw + + readline@8.2/7pjirte + + + + 7ftqkn35sy5bmqv3wui3ap3gubqyu4f4->7pjirtey2xqww2bbkil3yj3mtmasruaw + + + + + + + d24pqmu7ayswej2jfwwcgnw26t4gatgv + + libxcrypt@4.4.35/d24pqmu + + + + 7ftqkn35sy5bmqv3wui3ap3gubqyu4f4->d24pqmu7ayswej2jfwwcgnw26t4gatgv + + + + + + + 7ftqkn35sy5bmqv3wui3ap3gubqyu4f4->hsp7usvecwby6o6kszujxywbux5f5qc4 + + + + + + rei73bcylffduxjtuwt5sbibc2cbvuyt + + libffi@3.4.4/rei73bc + + + + 7ftqkn35sy5bmqv3wui3ap3gubqyu4f4->rei73bcylffduxjtuwt5sbibc2cbvuyt + + + + + + + 7ftqkn35sy5bmqv3wui3ap3gubqyu4f4->ez3cm4rogbx7at45wfi6gquti6fbo3zz + + + + + + + mof23fyk5qdmzll42yrtvvjeafzn45rl + + libbsd@0.11.7/mof23fy + + + + ygkrrpeszr4j377qqtqqecmwt27pm2ho->mof23fyk5qdmzll42yrtvvjeafzn45rl + + + + + + + pbpdelsw4pyldezsnide5zcc4ym5rrzg + + re2c@2.2/pbpdels + + + + qlqyzklm3yyv6tkqgnj4tzoy7g72ejyu->7pjirtey2xqww2bbkil3yj3mtmasruaw + + + + + + + qlqyzklm3yyv6tkqgnj4tzoy7g72ejyu->ez3cm4rogbx7at45wfi6gquti6fbo3zz + + + + + + + 33tg442mk3uy52ocdgd7uxbusdtkozlq->7ftqkn35sy5bmqv3wui3ap3gubqyu4f4 + + + + + + + ihtvssgtl7yz2wj7wdla4hsi7nqfny42->hsp7usvecwby6o6kszujxywbux5f5qc4 + + + + + + ca3noh6upxuh3hdx2lnrsdvw7blgcj5p->wuse2zg2p4ujfbsks4znlwyqumsa476w + + + + + + ca3noh6upxuh3hdx2lnrsdvw7blgcj5p->7ftqkn35sy5bmqv3wui3ap3gubqyu4f4 + + + + + + + + ca3noh6upxuh3hdx2lnrsdvw7blgcj5p->33tg442mk3uy52ocdgd7uxbusdtkozlq + + + + + + ca3noh6upxuh3hdx2lnrsdvw7blgcj5p->isprdjk4hdva3owdr6bgzavgaqzyjwyj + + + + + + ca3noh6upxuh3hdx2lnrsdvw7blgcj5p->esl2253adih4qsbluhmzdtsxfrws4fnt + + + + + + + buscwcl7gy7xqmrsmtewcustpjoa3jy6 + + openblas@0.3.24/buscwcl + + + + ca3noh6upxuh3hdx2lnrsdvw7blgcj5p->buscwcl7gy7xqmrsmtewcustpjoa3jy6 + + + + + + + bvcsrijbs7lp5jvlyooahoxc3zfapwfp->7pjirtey2xqww2bbkil3yj3mtmasruaw + + + + + + + isprdjk4hdva3owdr6bgzavgaqzyjwyj->7ftqkn35sy5bmqv3wui3ap3gubqyu4f4 + + + + + + + isprdjk4hdva3owdr6bgzavgaqzyjwyj->33tg442mk3uy52ocdgd7uxbusdtkozlq + + + + + + 3o2rmrxpwkmmetxmzvba6sizei5womzv->aoucvoqqeft4hsw3poydbf4mvong4nry + + + + + + + 3o2rmrxpwkmmetxmzvba6sizei5womzv->7aawlyt3hu24znvpgwedu2s3jmz46dkn + + + + + + + + 3o2rmrxpwkmmetxmzvba6sizei5womzv->gpd7yevon44acblslmgorfsxufgk3nhz + + + + + + + 3o2rmrxpwkmmetxmzvba6sizei5womzv->bcjm3vxlgrjgewpdakhpfea3y2kzcspe + + + + + + + jofugpdt2lki4tvw3xa56pxz4kzmjb33 + + tar@1.34/jofugpd + + + + 3o2rmrxpwkmmetxmzvba6sizei5womzv->jofugpdt2lki4tvw3xa56pxz4kzmjb33 + + + + + + + yry2pcjkl2hcfeexfi2yvnar2lyplbyg + + libxml2@2.10.3/yry2pcj + + + + 3o2rmrxpwkmmetxmzvba6sizei5womzv->yry2pcjkl2hcfeexfi2yvnar2lyplbyg + + + + + + + 2ok2ozl5i2qphhfsbxkdtq3iezemvpsv->7ftqkn35sy5bmqv3wui3ap3gubqyu4f4 + + + + + + + 2ok2ozl5i2qphhfsbxkdtq3iezemvpsv->33tg442mk3uy52ocdgd7uxbusdtkozlq + + + + + + ma7u5unvr5auweq7clkgz75hca33j6eb + + pigz@2.7/ma7u5un + + + + ma7u5unvr5auweq7clkgz75hca33j6eb->ez3cm4rogbx7at45wfi6gquti6fbo3zz + + + + + + + ma6zn6mykr7xe226v2hvu4ye7jltnddb->aoucvoqqeft4hsw3poydbf4mvong4nry + + + + + + + omxtm4xh3xbta4le4ehihd26gi3qn2hc + + curl@8.1.2/omxtm4x + + + + ma6zn6mykr7xe226v2hvu4ye7jltnddb->omxtm4xh3xbta4le4ehihd26gi3qn2hc + + + + + + + ma6zn6mykr7xe226v2hvu4ye7jltnddb->ez3cm4rogbx7at45wfi6gquti6fbo3zz + + + + + + + bcjm3vxlgrjgewpdakhpfea3y2kzcspe->s3mykqnlex5ygursynhv4cfu4p4jcp5c + + + + + + 7pjirtey2xqww2bbkil3yj3mtmasruaw->aoucvoqqeft4hsw3poydbf4mvong4nry + + + + + + + esl2253adih4qsbluhmzdtsxfrws4fnt->7ftqkn35sy5bmqv3wui3ap3gubqyu4f4 + + + + + + + esl2253adih4qsbluhmzdtsxfrws4fnt->33tg442mk3uy52ocdgd7uxbusdtkozlq + + + + + + r3mipc2ezzxleb6g3yjy2rgio44tpsnr + + libmd@1.0.4/r3mipc2 + + + + tolbgopadusf5fpqzmhm7qfsnhpluyvv + + zstd@1.5.5/tolbgop + + + + omxtm4xh3xbta4le4ehihd26gi3qn2hc->bqm4trdmbbqhrthe6flwnxp57cfbbser + + + + + + + omxtm4xh3xbta4le4ehihd26gi3qn2hc->e3xjka5zk6vtoen2oexuzxyorp6um5rv + + + + + + + omxtm4xh3xbta4le4ehihd26gi3qn2hc->hsp7usvecwby6o6kszujxywbux5f5qc4 + + + + + + omxtm4xh3xbta4le4ehihd26gi3qn2hc->ez3cm4rogbx7at45wfi6gquti6fbo3zz + + + + + + + revhbmcsddofjb7jt3fql7fawtxjihvc->7ftqkn35sy5bmqv3wui3ap3gubqyu4f4 + + + + + + revhbmcsddofjb7jt3fql7fawtxjihvc->pbpdelsw4pyldezsnide5zcc4ym5rrzg + + + + + + d24pqmu7ayswej2jfwwcgnw26t4gatgv->7bvgd7zcvk3hglqgbqczma5h4urvrdjb + + + + + + jofugpdt2lki4tvw3xa56pxz4kzmjb33->7aawlyt3hu24znvpgwedu2s3jmz46dkn + + + + + + jofugpdt2lki4tvw3xa56pxz4kzmjb33->gpd7yevon44acblslmgorfsxufgk3nhz + + + + + + + jofugpdt2lki4tvw3xa56pxz4kzmjb33->ma7u5unvr5auweq7clkgz75hca33j6eb + + + + + + jofugpdt2lki4tvw3xa56pxz4kzmjb33->bcjm3vxlgrjgewpdakhpfea3y2kzcspe + + + + + + jofugpdt2lki4tvw3xa56pxz4kzmjb33->tolbgopadusf5fpqzmhm7qfsnhpluyvv + + + + + + buscwcl7gy7xqmrsmtewcustpjoa3jy6->7bvgd7zcvk3hglqgbqczma5h4urvrdjb + + + + + + yry2pcjkl2hcfeexfi2yvnar2lyplbyg->7aawlyt3hu24znvpgwedu2s3jmz46dkn + + + + + + + yry2pcjkl2hcfeexfi2yvnar2lyplbyg->gpd7yevon44acblslmgorfsxufgk3nhz + + + + + + + yry2pcjkl2hcfeexfi2yvnar2lyplbyg->hsp7usvecwby6o6kszujxywbux5f5qc4 + + + + + + yry2pcjkl2hcfeexfi2yvnar2lyplbyg->ez3cm4rogbx7at45wfi6gquti6fbo3zz + + + + + + + lcvuenzomq3fdqabnz22ih3kpt4g2nyd + + berkeley-db@18.1.40/lcvuenz + + + + 7bvgd7zcvk3hglqgbqczma5h4urvrdjb->bvcsrijbs7lp5jvlyooahoxc3zfapwfp + + + + + + + 7bvgd7zcvk3hglqgbqczma5h4urvrdjb->bcjm3vxlgrjgewpdakhpfea3y2kzcspe + + + + + + + 7bvgd7zcvk3hglqgbqczma5h4urvrdjb->ez3cm4rogbx7at45wfi6gquti6fbo3zz + + + + + + + 7bvgd7zcvk3hglqgbqczma5h4urvrdjb->lcvuenzomq3fdqabnz22ih3kpt4g2nyd + + + + + + + mof23fyk5qdmzll42yrtvvjeafzn45rl->r3mipc2ezzxleb6g3yjy2rgio44tpsnr + + + + + + diff --git a/lib/spack/spack/solver/asp.py b/lib/spack/spack/solver/asp.py index 1afe31f162..ce0ae2ca7f 100644 --- a/lib/spack/spack/solver/asp.py +++ b/lib/spack/spack/solver/asp.py @@ -2595,6 +2595,7 @@ class SpecBuilder: r"^node_compiler$", r"^package_hash$", r"^root$", + r"^variant_default_value_from_cli$", r"^virtual_node$", r"^virtual_root$", ] diff --git a/lib/spack/spack/solver/concretize.lp b/lib/spack/spack/solver/concretize.lp index 7a41a9e834..fd3fb94f49 100644 --- a/lib/spack/spack/solver/concretize.lp +++ b/lib/spack/spack/solver/concretize.lp @@ -20,7 +20,7 @@ % Integrity constraints on DAG nodes :- attr("root", PackageNode), not attr("node", PackageNode). -:- attr("version", PackageNode), not attr("node", PackageNode). +:- attr("version", PackageNode, _), not attr("node", PackageNode), not attr("virtual_node", PackageNode). :- attr("node_version_satisfies", PackageNode), not attr("node", PackageNode). :- attr("hash", PackageNode, _), not attr("node", PackageNode). :- attr("node_platform", PackageNode, _), not attr("node", PackageNode). @@ -58,7 +58,6 @@ unification_set(SetID, ChildNode) :- attr("depends_on", ParentNode, ChildNode, T unification_set(("build", node(X, Child)), node(X, Child)) :- attr("depends_on", ParentNode, node(X, Child), Type), Type == "build", - SetID != "generic_build", multiple_unification_sets(Child), unification_set(SetID, ParentNode). @@ -68,18 +67,18 @@ unification_set("generic_build", node(X, Child)) not multiple_unification_sets(Child), unification_set(_, ParentNode). -% Any dependency of type "build" in a unification set that is in the leaf unification set, -% stays in that unification set -unification_set(SetID, ChildNode) - :- attr("depends_on", ParentNode, ChildNode, Type), - Type == "build", - SetID == "generic_build", - unification_set(SetID, ParentNode). - unification_set(SetID, VirtualNode) :- provider(PackageNode, VirtualNode), unification_set(SetID, PackageNode). +% Do not allow split dependencies, for now. This ensures that we don't construct graphs where e.g. +% a python extension depends on setuptools@63.4 as a run dependency, but uses e.g. setuptools@68 +% as a build dependency. +% +% We'll need to relax the rule before we get to actual cross-compilation +:- depends_on(ParentNode, node(X, Dependency)), depends_on(ParentNode, node(Y, Dependency)), X < Y. + + #defined multiple_unification_sets/1. %---- diff --git a/lib/spack/spack/solver/counter.py b/lib/spack/spack/solver/counter.py index b238f60d8c..28883817df 100644 --- a/lib/spack/spack/solver/counter.py +++ b/lib/spack/spack/solver/counter.py @@ -5,6 +5,8 @@ import collections from typing import List, Set +from llnl.util import lang + import spack.deptypes as dt import spack.package_base import spack.repo @@ -95,8 +97,17 @@ class MinimalDuplicatesCounter(NoDuplicatesCounter): ) self._link_run_virtuals.update(self._possible_virtuals) for x in self._link_run: - current = spack.repo.PATH.get_pkg_class(x).dependencies_of_type(dt.BUILD) - self._direct_build.update(current) + build_dependencies = spack.repo.PATH.get_pkg_class(x).dependencies_of_type(dt.BUILD) + virtuals, reals = lang.stable_partition( + build_dependencies, spack.repo.PATH.is_virtual_safe + ) + + self._possible_virtuals.update(virtuals) + for virtual_dep in virtuals: + providers = spack.repo.PATH.providers_for(virtual_dep) + self._direct_build.update(str(x) for x in providers) + + self._direct_build.update(reals) self._total_build = set( spack.package_base.possible_dependencies( diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index 85a638b602..8300e4d8a7 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -1604,13 +1604,20 @@ class Spec: try: dspec = next(dspec for dspec in orig if depflag == dspec.depflag) except StopIteration: - raise DuplicateDependencyError("Cannot depend on '%s' twice" % spec) + current_deps = ", ".join( + dt.flag_to_chars(x.depflag) + " " + x.spec.short_spec for x in orig + ) + raise DuplicateDependencyError( + f"{self.short_spec} cannot depend on '{spec.short_spec}' multiple times.\n" + f"\tRequired: {dt.flag_to_chars(depflag)}\n" + f"\tDependency: {current_deps}" + ) try: dspec.spec.constrain(spec) except spack.error.UnsatisfiableSpecError: raise DuplicateDependencyError( - "Cannot depend on incompatible specs '%s' and '%s'" % (dspec.spec, spec) + f"Cannot depend on incompatible specs '{dspec.spec}' and '{spec}'" ) def add_dependency_edge( diff --git a/lib/spack/spack/test/concretize.py b/lib/spack/spack/test/concretize.py index 63c6699de2..53f9c64d5e 100644 --- a/lib/spack/spack/test/concretize.py +++ b/lib/spack/spack/test/concretize.py @@ -2121,12 +2121,9 @@ def duplicates_test_repository(): @pytest.mark.usefixtures("mutable_config", "duplicates_test_repository") +@pytest.mark.only_clingo("Not supported by the original concretizer") class TestConcretizeSeparately: @pytest.mark.parametrize("strategy", ["minimal", "full"]) - @pytest.mark.skipif( - os.environ.get("SPACK_TEST_SOLVER") == "original", - reason="Not supported by the original concretizer", - ) def test_two_gmake(self, strategy): """Tests that we can concretize a spec with nodes using the same build dependency pinned at different versions. @@ -2151,10 +2148,6 @@ class TestConcretizeSeparately: assert len(pinned_gmake) == 1 and pinned_gmake[0].satisfies("@=3.0") @pytest.mark.parametrize("strategy", ["minimal", "full"]) - @pytest.mark.skipif( - os.environ.get("SPACK_TEST_SOLVER") == "original", - reason="Not supported by the original concretizer", - ) def test_two_setuptools(self, strategy): """Tests that we can concretize separate build dependencies, when we are dealing with extensions. @@ -2191,10 +2184,6 @@ class TestConcretizeSeparately: gmake = s["python"].dependencies(name="gmake", deptype="build") assert len(gmake) == 1 and gmake[0].satisfies("@=3.0") - @pytest.mark.skipif( - os.environ.get("SPACK_TEST_SOLVER") == "original", - reason="Not supported by the original concretizer", - ) def test_solution_without_cycles(self): """Tests that when we concretize a spec with cycles, a fallback kicks in to recompute a solution without cycles. @@ -2207,6 +2196,21 @@ class TestConcretizeSeparately: assert s["cycle-a"].satisfies("~cycle") assert s["cycle-b"].satisfies("+cycle") + @pytest.mark.parametrize("strategy", ["minimal", "full"]) + def test_pure_build_virtual_dependency(self, strategy): + """Tests that we can concretize a pure build virtual dependency, and ensures that + pure build virtual dependencies are accounted in the list of possible virtual + dependencies. + + virtual-build@1.0 + | [type=build, virtual=pkgconfig] + pkg-config@1.0 + """ + spack.config.CONFIG.set("concretizer:duplicates:strategy", strategy) + + s = Spec("virtual-build").concretized() + assert s["pkgconfig"].name == "pkg-config" + @pytest.mark.parametrize( "v_str,v_opts,checksummed", diff --git a/lib/spack/spack/test/data/config/concretizer.yaml b/lib/spack/spack/test/data/config/concretizer.yaml index ecf121a917..0dd810163d 100644 --- a/lib/spack/spack/test/data/config/concretizer.yaml +++ b/lib/spack/spack/test/data/config/concretizer.yaml @@ -4,4 +4,4 @@ concretizer: granularity: microarchitectures host_compatible: false duplicates: - strategy: none + strategy: minimal diff --git a/share/spack/gitlab/cloud_pipelines/stacks/data-vis-sdk/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/data-vis-sdk/spack.yaml index b4534f0814..9963f4b777 100644 --- a/share/spack/gitlab/cloud_pipelines/stacks/data-vis-sdk/spack.yaml +++ b/share/spack/gitlab/cloud_pipelines/stacks/data-vis-sdk/spack.yaml @@ -4,22 +4,16 @@ spack: cmake: variants: ~ownlibs ecp-data-vis-sdk: - require: - - one_of: - - +ascent +adios2 +cinema +darshan +faodel +hdf5 +pnetcdf +sensei +sz +unifyfs - +veloc +vtkm +zfp - - one_of: - - +paraview ~visit - - ~paraview +visit + require: "+ascent +adios2 +cinema +darshan +faodel +hdf5 +pnetcdf +sensei +sz +unifyfs +veloc +vtkm +zfp" hdf5: require: - one_of: ['@1.14', '@1.12'] mesa: - require: +glx +osmesa +opengl ~opengles +llvm + require: "+glx +osmesa +opengl ~opengles +llvm" libosmesa: - require: mesa +osmesa + require: "mesa +osmesa" libglx: - require: mesa +glx + require: "mesa +glx" ospray: require: '@2.8.0 +denoiser +mpi' llvm: @@ -57,9 +51,11 @@ spack: # Test ParaView and VisIt builds with different GL backends - matrix: - [$sdk_base_spec] + - ["+paraview ~visit"] - [$^paraview_specs] - matrix: - [$sdk_base_spec] + - ["~paraview +visit"] - [$^visit_specs] mirrors: {mirror: s3://spack-binaries/develop/data-vis-sdk} diff --git a/var/spack/repos/builtin/packages/camp/package.py b/var/spack/repos/builtin/packages/camp/package.py index 8d7b4f62b1..a1bdc830d3 100644 --- a/var/spack/repos/builtin/packages/camp/package.py +++ b/var/spack/repos/builtin/packages/camp/package.py @@ -53,7 +53,7 @@ class Camp(CMakePackage, CudaPackage, ROCmPackage): depends_on("cub", when="+cuda") - depends_on("blt") + depends_on("blt", type="build") conflicts("^blt@:0.3.6", when="+rocm") diff --git a/var/spack/repos/builtin/packages/cmake/package.py b/var/spack/repos/builtin/packages/cmake/package.py index c4c61aee82..796eb4c397 100644 --- a/var/spack/repos/builtin/packages/cmake/package.py +++ b/var/spack/repos/builtin/packages/cmake/package.py @@ -239,7 +239,7 @@ class Cmake(Package): depends_on("libuv@1.10.0:1.10", when="@3.11.0:3.11") depends_on("libuv@1.10.0:", when="@3.12.0:") depends_on("rhash", when="@3.8.0:") - depends_on("jsoncpp", when="@3.2:") + depends_on("jsoncpp build_system=meson", when="@3.2:") depends_on("ncurses", when="+ncurses") diff --git a/var/spack/repos/builtin/packages/jsoncpp/package.py b/var/spack/repos/builtin/packages/jsoncpp/package.py index 55c2f22a02..bd20e2b22e 100644 --- a/var/spack/repos/builtin/packages/jsoncpp/package.py +++ b/var/spack/repos/builtin/packages/jsoncpp/package.py @@ -47,7 +47,7 @@ class Jsoncpp(CMakePackage, MesonPackage): with when("build_system=cmake"): depends_on("cmake@3.1:", type="build") - depends_on("cmake@1.9:", when="@1.9:", type="build") + depends_on("cmake@3.9:", when="@1.9:", type="build") with when("build_system=meson"): depends_on("meson@0.49.0:", type="build") diff --git a/var/spack/repos/builtin/packages/mgard/package.py b/var/spack/repos/builtin/packages/mgard/package.py index ca9f2f46e4..b58f4c0bba 100644 --- a/var/spack/repos/builtin/packages/mgard/package.py +++ b/var/spack/repos/builtin/packages/mgard/package.py @@ -50,7 +50,7 @@ class Mgard(CMakePackage, CudaPackage): depends_on("libarchive", when="@2021-11-12:") depends_on("tclap", when="@2021-11-12") depends_on("yaml-cpp", when="@2021-11-12:") - depends_on("cmake@3.19:") + depends_on("cmake@3.19:", type="build") depends_on("nvcomp@2.2.0:", when="@2022-11-18:+cuda") depends_on("nvcomp@2.0.2", when="@:2021-11-12+cuda") conflicts("cuda_arch=none", when="+cuda") diff --git a/var/spack/repos/builtin/packages/py-gevent/package.py b/var/spack/repos/builtin/packages/py-gevent/package.py index b684829f54..c41b815ef2 100644 --- a/var/spack/repos/builtin/packages/py-gevent/package.py +++ b/var/spack/repos/builtin/packages/py-gevent/package.py @@ -24,8 +24,7 @@ class PyGevent(PythonPackage): depends_on("py-setuptools@40.8:", when="@20.5.1:", type=("build", "run")) depends_on("py-setuptools@40.8:", when="@1.5:", type="build") depends_on("py-setuptools@24.2:", when="@:1.4", type="build") - # TODO: relax this until we support separate concretization of build deps by default - # depends_on("py-cython@3:", when="@20.5.1:", type="build") + depends_on("py-cython@3:", when="@20.5.1:", type="build") depends_on("py-cython@0.29.14:", when="@1.5:", type="build") depends_on("py-cffi@1.12.3:", type=("build", "run")) depends_on("py-greenlet@3:", when="@23.7: ^python@3.12:", type=("build", "run")) diff --git a/var/spack/repos/builtin/packages/py-statsmodels/package.py b/var/spack/repos/builtin/packages/py-statsmodels/package.py index c99901d9ae..36968986a3 100644 --- a/var/spack/repos/builtin/packages/py-statsmodels/package.py +++ b/var/spack/repos/builtin/packages/py-statsmodels/package.py @@ -14,6 +14,7 @@ class PyStatsmodels(PythonPackage): homepage = "https://www.statsmodels.org" pypi = "statsmodels/statsmodels-0.8.0.tar.gz" + version("0.13.5", sha256="593526acae1c0fda0ea6c48439f67c3943094c542fe769f8b90fe9e6c6cc4871") version("0.13.2", sha256="77dc292c9939c036a476f1770f9d08976b05437daa229928da73231147cde7d4") version("0.13.1", sha256="006ec8d896d238873af8178d5475203844f2c391194ed8d42ddac37f5ff77a69") version("0.13.0", sha256="f2efc02011b7240a9e851acd76ab81150a07d35c97021cb0517887539a328f8a") @@ -25,12 +26,15 @@ class PyStatsmodels(PythonPackage): depends_on("python@2.7:2.8,3.4:", when="@0.10.1:", type=("build", "link", "run")) depends_on("python@3.6:", when="@0.12.1:", type=("build", "link", "run")) - # according to https://www.statsmodels.org/dev/install.html earlier versions - # might work. - depends_on("py-setuptools@0.6c5:", type="build") - depends_on("py-cython@0.29:", type="build") + # according to https://www.statsmodels.org/dev/install.html earlier versions might work. + depends_on("py-setuptools", type="build") + depends_on("py-setuptools@59.2.0:", type="build", when="@0.13.5:") + + # https://github.com/statsmodels/statsmodels/blob/01b19d7d111b29c183f620ff0a949ef6391ff8ee/pyproject.toml + depends_on("py-cython@0", type="build") depends_on("py-cython@0.29.14:", type="build", when="@0.12.0:") depends_on("py-cython@0.29.22:", type="build", when="@0.13.0:") + depends_on("py-cython@0.29.32:", type="build", when="@0.13.5:") # patsy@0.5.1 works around a Python change # https://github.com/statsmodels/statsmodels/issues/5343 and diff --git a/var/spack/repos/builtin/packages/raja/package.py b/var/spack/repos/builtin/packages/raja/package.py index 88505b3a81..cc1ede76be 100644 --- a/var/spack/repos/builtin/packages/raja/package.py +++ b/var/spack/repos/builtin/packages/raja/package.py @@ -137,7 +137,7 @@ class Raja(CachedCMakePackage, CudaPackage, ROCmPackage): depends_on("cmake@:3.20", when="@:2022.03+rocm", type="build") depends_on("cmake@3.23:", when="@2022.10:+rocm", type="build") - depends_on("cmake@3.14:", when="@2022.03.0:") + depends_on("cmake@3.14:", when="@2022.03.0:", type="build") depends_on("llvm-openmp", when="+openmp %apple-clang") diff --git a/var/spack/repos/builtin/packages/umpire/package.py b/var/spack/repos/builtin/packages/umpire/package.py index 51f0c034b2..c64bfdf256 100644 --- a/var/spack/repos/builtin/packages/umpire/package.py +++ b/var/spack/repos/builtin/packages/umpire/package.py @@ -168,7 +168,7 @@ class Umpire(CachedCMakePackage, CudaPackage, ROCmPackage): depends_on("cmake@3.8:", type="build") depends_on("cmake@3.9:", when="+cuda", type="build") - depends_on("cmake@3.14:", when="@2022.03.0:") + depends_on("cmake@3.14:", when="@2022.03.0:", type="build") depends_on("blt@0.5.2:", type="build", when="@2022.10.0:") depends_on("blt@0.5.0:", type="build", when="@2022.03.0:") diff --git a/var/spack/repos/duplicates.test/packages/pkg-config/package.py b/var/spack/repos/duplicates.test/packages/pkg-config/package.py new file mode 100644 index 0000000000..eb7b84b88f --- /dev/null +++ b/var/spack/repos/duplicates.test/packages/pkg-config/package.py @@ -0,0 +1,16 @@ +# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) +from spack.package import * + + +class PkgConfig(Package): + """A package providing a virtual, which is frequently used as a pure build dependency.""" + + homepage = "http://www.example.com" + url = "http://www.example.com/tdep-1.0.tar.gz" + + version("1.0.0", md5="0123456789abcdef0123456789abcdef") + + provides("pkgconfig") diff --git a/var/spack/repos/duplicates.test/packages/virtual-build/package.py b/var/spack/repos/duplicates.test/packages/virtual-build/package.py new file mode 100644 index 0000000000..17fc60955d --- /dev/null +++ b/var/spack/repos/duplicates.test/packages/virtual-build/package.py @@ -0,0 +1,16 @@ +# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) +from spack.package import * + + +class VirtualBuild(Package): + """A package that has a pure build virtual dependency""" + + homepage = "http://www.example.com" + url = "http://www.example.com/tdep-1.0.tar.gz" + + version("1.0.0", md5="0123456789abcdef0123456789abcdef") + + depends_on("pkgconfig", type="build") -- cgit v1.2.3-60-g2f50