summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--lib/spack/spack/cmd/__init__.py43
-rw-r--r--lib/spack/spack/cmd/clean.py3
-rw-r--r--lib/spack/spack/cmd/patch.py3
-rw-r--r--lib/spack/spack/cmd/stage.py2
-rw-r--r--lib/spack/spack/concretize.py140
-rw-r--r--lib/spack/spack/environment/environment.py149
-rw-r--r--lib/spack/spack/solver/asp.py2
-rw-r--r--lib/spack/spack/test/cmd/install.py2
-rw-r--r--lib/spack/spack/test/concretize.py18
9 files changed, 237 insertions, 125 deletions
diff --git a/lib/spack/spack/cmd/__init__.py b/lib/spack/spack/cmd/__init__.py
index c481e93131..031b29f952 100644
--- a/lib/spack/spack/cmd/__init__.py
+++ b/lib/spack/spack/cmd/__init__.py
@@ -173,10 +173,29 @@ def parse_specs(
arg_string = " ".join([quote_kvp(arg) for arg in args])
specs = spack.parser.parse(arg_string)
- for spec in specs:
- if concretize:
- spec.concretize(tests=tests)
- return specs
+ if not concretize:
+ return specs
+
+ to_concretize = [(s, None) for s in specs]
+ return _concretize_spec_pairs(to_concretize, tests=tests)
+
+
+def _concretize_spec_pairs(to_concretize, tests=False):
+ """Helper method that concretizes abstract specs from a list of abstract,concrete pairs.
+
+ Any spec with a concrete spec associated with it will concretize to that spec. Any spec
+ with ``None`` for its concrete spec will be newly concretized. This method respects unification
+ rules from config."""
+ unify = spack.config.get("concretizer:unify", False)
+
+ concretize_method = spack.concretize.concretize_separately # unify: false
+ if unify is True:
+ concretize_method = spack.concretize.concretize_together
+ elif unify == "when_possible":
+ concretize_method = spack.concretize.concretize_together_when_possible
+
+ concretized = concretize_method(*to_concretize, tests=tests)
+ return [concrete for _, concrete in concretized]
def matching_spec_from_env(spec):
@@ -192,6 +211,22 @@ def matching_spec_from_env(spec):
return spec.concretized()
+def matching_specs_from_env(specs):
+ """
+ Same as ``matching_spec_from_env`` but respects spec unification rules.
+
+ For each spec, if there is a matching spec in the environment it is used. If no
+ matching spec is found, this will return the given spec but concretized in the
+ context of the active environment and other given specs, with unification rules applied.
+ """
+ env = ev.active_environment()
+ spec_pairs = [(spec, env.matching_spec(spec) if env else None) for spec in specs]
+ additional_concrete_specs = (
+ [(concrete, concrete) for _, concrete in env.concretized_specs()] if env else []
+ )
+ return _concretize_spec_pairs(spec_pairs + additional_concrete_specs)[: len(spec_pairs)]
+
+
def disambiguate_spec(spec, env, local=False, installed=True, first=False):
"""Given a spec, figure out which installed package it refers to.
diff --git a/lib/spack/spack/cmd/clean.py b/lib/spack/spack/cmd/clean.py
index 0b8fb6d6bb..59d650fd12 100644
--- a/lib/spack/spack/cmd/clean.py
+++ b/lib/spack/spack/cmd/clean.py
@@ -105,7 +105,8 @@ def clean(parser, args):
# Then do the cleaning falling through the cases
if args.specs:
specs = spack.cmd.parse_specs(args.specs, concretize=False)
- specs = list(spack.cmd.matching_spec_from_env(x) for x in specs)
+ specs = spack.cmd.matching_specs_from_env(specs)
+
for spec in specs:
msg = "Cleaning build stage [{0}]"
tty.msg(msg.format(spec.short_spec))
diff --git a/lib/spack/spack/cmd/patch.py b/lib/spack/spack/cmd/patch.py
index 885ff2f746..dbd18f7948 100644
--- a/lib/spack/spack/cmd/patch.py
+++ b/lib/spack/spack/cmd/patch.py
@@ -33,8 +33,9 @@ def patch(parser, args):
spack.config.set("config:checksum", False, scope="command_line")
specs = spack.cmd.parse_specs(args.specs, concretize=False)
+ specs = spack.cmd.matching_specs_from_env(specs)
for spec in specs:
- _patch(spack.cmd.matching_spec_from_env(spec).package)
+ _patch(spec.package)
def _patch_env(env: ev.Environment):
diff --git a/lib/spack/spack/cmd/stage.py b/lib/spack/spack/cmd/stage.py
index af5fa412ea..20da92926f 100644
--- a/lib/spack/spack/cmd/stage.py
+++ b/lib/spack/spack/cmd/stage.py
@@ -47,8 +47,8 @@ def stage(parser, args):
if len(specs) > 1 and custom_path:
tty.die("`--path` requires a single spec, but multiple were provided")
+ specs = spack.cmd.matching_specs_from_env(specs)
for spec in specs:
- spec = spack.cmd.matching_spec_from_env(spec)
pkg = spec.package
if custom_path:
diff --git a/lib/spack/spack/concretize.py b/lib/spack/spack/concretize.py
index 387c7f2de2..fabfdbb523 100644
--- a/lib/spack/spack/concretize.py
+++ b/lib/spack/spack/concretize.py
@@ -5,11 +5,17 @@
"""
(DEPRECATED) Used to contain the code for the original concretizer
"""
+import sys
+import time
from contextlib import contextmanager
from itertools import chain
+from typing import Tuple
+
+import llnl.util.tty as tty
import spack.config
import spack.error
+from spack.spec import Spec
CHECK_COMPILER_EXISTENCE = True
@@ -83,6 +89,140 @@ def concretize_specs_together(*abstract_specs, **kwargs):
return [s.copy() for s in result.specs]
+def concretize_together(*spec_list, **kwargs):
+ """Given a number of specs as input, tries to concretize them together.
+
+ Args:
+ tests (bool or list or set): False to run no tests, True to test
+ all packages, or a list of package names to run tests for some
+ *spec_list: list of tuples to concretize. First entry is abstract spec, second entry is
+ already concrete spec or None if not yet concretized
+
+ Returns:
+ List of tuples of abstract and concretized specs
+ """
+ to_concretize = [concrete if concrete else abstract for abstract, concrete in spec_list]
+ abstract_specs = [abstract for abstract, _ in spec_list]
+ concrete_specs = concretize_specs_together(*to_concretize, **kwargs)
+ return list(zip(abstract_specs, concrete_specs))
+
+
+def concretize_together_when_possible(*spec_list, **kwargs):
+ """Given a number of specs as input, tries to concretize them together to the extent possible.
+
+ See documentation for ``unify: when_possible`` concretization for the precise definition of
+ "to the extent possible".
+
+ Args:
+ tests (bool or list or set): False to run no tests, True to test
+ all packages, or a list of package names to run tests for some
+ *spec_list: list of tuples to concretize. First entry is abstract spec, second entry is
+ already concrete spec or None if not yet concretized
+
+ Returns:
+ List of tuples of abstract and concretized specs
+ """
+ to_concretize = [concrete if concrete else abstract for abstract, concrete in spec_list]
+ old_concrete_to_abstract = {
+ concrete: abstract for (abstract, concrete) in spec_list if concrete
+ }
+
+ result_by_user_spec = {}
+ solver = spack.solver.asp.Solver()
+ allow_deprecated = spack.config.get("config:deprecated", False)
+ for result in solver.solve_in_rounds(
+ to_concretize, tests=kwargs.get("tests", False), allow_deprecated=allow_deprecated
+ ):
+ result_by_user_spec.update(result.specs_by_input)
+
+ # If the "abstract" spec is a concrete spec from the previous concretization
+ # translate it back to an abstract spec. Otherwise, keep the abstract spec
+ return [
+ (old_concrete_to_abstract.get(abstract, abstract), concrete)
+ for abstract, concrete in sorted(result_by_user_spec.items())
+ ]
+
+
+def concretize_separately(*spec_list, **kwargs):
+ """Given a number of specs as input, tries to concretize them together.
+
+ Args:
+ tests (bool or list or set): False to run no tests, True to test
+ all packages, or a list of package names to run tests for some
+ *spec_list: list of tuples to concretize. First entry is abstract spec, second entry is
+ already concrete spec or None if not yet concretized
+
+ Returns:
+ List of tuples of abstract and concretized specs
+ """
+ tests = kwargs.get("tests", False)
+ to_concretize = [abstract for abstract, concrete in spec_list if not concrete]
+ args = [
+ (i, str(abstract), tests)
+ for i, abstract in enumerate(to_concretize)
+ if not abstract.concrete
+ ]
+ ret = [(i, abstract) for i, abstract in enumerate(to_concretize) if abstract.concrete]
+ # Ensure we don't try to bootstrap clingo in parallel
+ with spack.bootstrap.ensure_bootstrap_configuration():
+ spack.bootstrap.ensure_clingo_importable_or_raise()
+
+ # Ensure all the indexes have been built or updated, since
+ # otherwise the processes in the pool may timeout on waiting
+ # for a write lock. We do this indirectly by retrieving the
+ # provider index, which should in turn trigger the update of
+ # all the indexes if there's any need for that.
+ _ = spack.repo.PATH.provider_index
+
+ # Ensure we have compilers in compilers.yaml to avoid that
+ # processes try to write the config file in parallel
+ _ = spack.compilers.all_compilers_config(spack.config.CONFIG)
+
+ # Early return if there is nothing to do
+ if len(args) == 0:
+ # Still have to combine the things that were passed in as abstract with the things
+ # that were passed in as pairs
+ return [(abstract, concrete) for abstract, (_, concrete) in zip(to_concretize, ret)] + [
+ (abstract, concrete) for abstract, concrete in spec_list if concrete
+ ]
+
+ # Solve the environment in parallel on Linux
+ # TODO: support parallel concretization on macOS and Windows
+ num_procs = min(len(args), spack.config.determine_number_of_jobs(parallel=True))
+
+ for j, (i, concrete, duration) in enumerate(
+ spack.util.parallel.imap_unordered(
+ spack.concretize._concretize_task,
+ args,
+ processes=num_procs,
+ debug=tty.is_debug(),
+ maxtaskperchild=1,
+ )
+ ):
+ ret.append((i, concrete))
+ percentage = (j + 1) / len(args) * 100
+ tty.verbose(
+ f"{duration:6.1f}s [{percentage:3.0f}%] {concrete.cformat('{hash:7}')} "
+ f"{to_concretize[i].colored_str}"
+ )
+ sys.stdout.flush()
+
+ # Add specs in original order
+ ret.sort(key=lambda x: x[0])
+
+ return [(abstract, concrete) for abstract, (_, concrete) in zip(to_concretize, ret)] + [
+ (abstract, concrete) for abstract, concrete in spec_list if concrete
+ ]
+
+
+def _concretize_task(packed_arguments) -> Tuple[int, Spec, float]:
+ index, spec_str, tests = packed_arguments
+ with tty.SuppressOutput(msg_enabled=False):
+ start = time.time()
+ spec = Spec(spec_str).concretized(tests=tests)
+ return index, spec, time.time() - start
+
+
class UnavailableCompilerVersionError(spack.error.SpackError):
"""Raised when there is no available compiler that satisfies a
compiler spec."""
diff --git a/lib/spack/spack/environment/environment.py b/lib/spack/spack/environment/environment.py
index 81a2223c49..b61332a0ab 100644
--- a/lib/spack/spack/environment/environment.py
+++ b/lib/spack/spack/environment/environment.py
@@ -11,12 +11,10 @@ import pathlib
import re
import shutil
import stat
-import sys
-import time
import urllib.parse
import urllib.request
import warnings
-from typing import Any, Dict, Iterable, List, Optional, Set, Tuple, Union
+from typing import Any, Dict, Iterable, List, Optional, Tuple, Union
import llnl.util.filesystem as fs
import llnl.util.tty as tty
@@ -57,6 +55,8 @@ from spack.spec import Spec
from spack.spec_list import SpecList
from spack.util.path import substitute_path_variables
+SpecPair = Tuple[spack.spec.Spec, spack.spec.Spec]
+
#: environment variable used to indicate the active environment
spack_env_var = "SPACK_ENV"
@@ -1510,7 +1510,7 @@ class Environment:
def _get_specs_to_concretize(
self,
- ) -> Tuple[Set[spack.spec.Spec], Set[spack.spec.Spec], List[spack.spec.Spec]]:
+ ) -> Tuple[List[spack.spec.Spec], List[spack.spec.Spec], List[SpecPair]]:
"""Compute specs to concretize for unify:true and unify:when_possible.
This includes new user specs and any already concretized specs.
@@ -1520,19 +1520,17 @@ class Environment:
"""
# Exit early if the set of concretized specs is the set of user specs
- new_user_specs = set(self.user_specs) - set(self.concretized_user_specs)
- kept_user_specs = set(self.user_specs) & set(self.concretized_user_specs)
- kept_user_specs |= set(self.included_user_specs)
+ new_user_specs = list(set(self.user_specs) - set(self.concretized_user_specs))
+ kept_user_specs = list(set(self.user_specs) & set(self.concretized_user_specs))
+ kept_user_specs += self.included_user_specs
if not new_user_specs:
return new_user_specs, kept_user_specs, []
- concrete_specs_to_keep = [
- concrete
+ specs_to_concretize = [(s, None) for s in new_user_specs] + [
+ (abstract, concrete)
for abstract, concrete in self.concretized_specs()
if abstract in kept_user_specs
]
-
- specs_to_concretize = list(new_user_specs) + concrete_specs_to_keep
return new_user_specs, kept_user_specs, specs_to_concretize
def _concretize_together_where_possible(
@@ -1546,39 +1544,26 @@ class Environment:
if not new_user_specs:
return []
- old_concrete_to_abstract = {
- concrete: abstract for (abstract, concrete) in self.concretized_specs()
- }
-
self.concretized_user_specs = []
self.concretized_order = []
self.specs_by_hash = {}
- result_by_user_spec = {}
- solver = spack.solver.asp.Solver()
- allow_deprecated = spack.config.get("config:deprecated", False)
- for result in solver.solve_in_rounds(
- specs_to_concretize, tests=tests, allow_deprecated=allow_deprecated
- ):
- result_by_user_spec.update(result.specs_by_input)
-
- result = []
- for abstract, concrete in sorted(result_by_user_spec.items()):
- # If the "abstract" spec is a concrete spec from the previous concretization
- # translate it back to an abstract spec. Otherwise, keep the abstract spec
- abstract = old_concrete_to_abstract.get(abstract, abstract)
- if abstract in new_user_specs:
- result.append((abstract, concrete))
-
- # Only add to the environment if it's from this environment (not just included)
+ ret = []
+ result = spack.concretize.concretize_together_when_possible(
+ *specs_to_concretize, tests=tests
+ )
+ for abstract, concrete in result:
+ # Only add to the environment if it's from this environment (not included in)
if abstract in self.user_specs:
self._add_concrete_spec(abstract, concrete)
- return result
+ # Return only the new specs
+ if abstract in new_user_specs:
+ ret.append((abstract, concrete))
- def _concretize_together(
- self, tests: bool = False
- ) -> List[Tuple[spack.spec.Spec, spack.spec.Spec]]:
+ return ret
+
+ def _concretize_together(self, tests: bool = False) -> List[SpecPair]:
"""Concretization strategy that concretizes all the specs
in the same DAG.
"""
@@ -1592,7 +1577,7 @@ class Environment:
self.specs_by_hash = {}
try:
- concrete_specs: List[spack.spec.Spec] = spack.concretize.concretize_specs_together(
+ concretized_specs: List[SpecPair] = spack.concretize.concretize_together(
*specs_to_concretize, tests=tests
)
except spack.error.UnsatisfiableSpecError as e:
@@ -1611,16 +1596,13 @@ class Environment:
)
raise
- # set() | set() does not preserve ordering, even though sets are ordered
- ordered_user_specs = list(new_user_specs) + list(kept_user_specs)
- concretized_specs = [x for x in zip(ordered_user_specs, concrete_specs)]
for abstract, concrete in concretized_specs:
# Don't add if it's just included
if abstract in self.user_specs:
self._add_concrete_spec(abstract, concrete)
- # zip truncates the longer list, which is exactly what we want here
- return list(zip(new_user_specs, concrete_specs))
+ # Return the portion of the return value that is new
+ return concretized_specs[: len(new_user_specs)]
def _concretize_separately(self, tests=False):
"""Concretization strategy that concretizes separately one
@@ -1642,71 +1624,16 @@ class Environment:
concrete = old_specs_by_hash[h]
self._add_concrete_spec(s, concrete, new=False)
- # Concretize any new user specs that we haven't concretized yet
- args, root_specs, i = [], [], 0
- for uspec in self.user_specs:
- if uspec not in old_concretized_user_specs:
- root_specs.append(uspec)
- args.append((i, str(uspec), tests))
- i += 1
-
- # Ensure we don't try to bootstrap clingo in parallel
- with spack.bootstrap.ensure_bootstrap_configuration():
- spack.bootstrap.ensure_clingo_importable_or_raise()
-
- # Ensure all the indexes have been built or updated, since
- # otherwise the processes in the pool may timeout on waiting
- # for a write lock. We do this indirectly by retrieving the
- # provider index, which should in turn trigger the update of
- # all the indexes if there's any need for that.
- _ = spack.repo.PATH.provider_index
-
- # Ensure we have compilers in compilers.yaml to avoid that
- # processes try to write the config file in parallel
- _ = spack.compilers.all_compilers_config(spack.config.CONFIG)
-
- # Early return if there is nothing to do
- if len(args) == 0:
- return []
-
- # Solve the environment in parallel on Linux
- start = time.time()
- num_procs = min(len(args), spack.config.determine_number_of_jobs(parallel=True))
-
- # TODO: support parallel concretization on macOS and Windows
- msg = "Starting concretization"
- if sys.platform not in ("darwin", "win32") and num_procs > 1:
- msg += f" pool with {num_procs} processes"
- tty.msg(msg)
-
- batch = []
- for j, (i, concrete, duration) in enumerate(
- spack.util.parallel.imap_unordered(
- _concretize_task,
- args,
- processes=num_procs,
- debug=tty.is_debug(),
- maxtaskperchild=1,
- )
- ):
- batch.append((i, concrete))
- percentage = (j + 1) / len(args) * 100
- tty.verbose(
- f"{duration:6.1f}s [{percentage:3.0f}%] {concrete.cformat('{hash:7}')} "
- f"{root_specs[i].colored_str}"
- )
- sys.stdout.flush()
+ to_concretize = [
+ (root, None) for root in self.user_specs if root not in old_concretized_user_specs
+ ]
+ concretized_specs = spack.concretize.concretize_separately(*to_concretize, tests=tests)
- # Add specs in original order
- batch.sort(key=lambda x: x[0])
- by_hash = {} # for attaching information on test dependencies
- for root, (_, concrete) in zip(root_specs, batch):
- self._add_concrete_spec(root, concrete)
+ by_hash = {}
+ for abstract, concrete in concretized_specs:
+ self._add_concrete_spec(abstract, concrete)
by_hash[concrete.dag_hash()] = concrete
- finish = time.time()
- tty.msg(f"Environment concretized in {finish - start:.2f} seconds")
-
# Unify the specs objects, so we get correct references to all parents
self._read_lockfile_dict(self._to_lockfile_dict())
@@ -1726,11 +1653,7 @@ class Environment:
test_dependency.copy(), depflag=dt.TEST, virtuals=current_edge.virtuals
)
- results = [
- (abstract, self.specs_by_hash[h])
- for abstract, h in zip(self.concretized_user_specs, self.concretized_order)
- ]
- return results
+ return concretized_specs
@property
def default_view(self):
@@ -2537,14 +2460,6 @@ def display_specs(specs):
print(tree_string)
-def _concretize_task(packed_arguments) -> Tuple[int, Spec, float]:
- index, spec_str, tests = packed_arguments
- with tty.SuppressOutput(msg_enabled=False):
- start = time.time()
- spec = Spec(spec_str).concretized(tests=tests)
- return index, spec, time.time() - start
-
-
def make_repo_path(root):
"""Make a RepoPath from the repo subdirectories in an environment."""
path = spack.repo.RepoPath(cache=spack.caches.MISC_CACHE)
diff --git a/lib/spack/spack/solver/asp.py b/lib/spack/spack/solver/asp.py
index 97fbd03e8f..af2b8a70c3 100644
--- a/lib/spack/spack/solver/asp.py
+++ b/lib/spack/spack/solver/asp.py
@@ -515,6 +515,8 @@ class Result:
best = min(self.answers)
opt, _, answer = best
for input_spec in self.abstract_specs:
+ # The specs must be unified to get here, so it is safe to associate any satisfying spec
+ # with the input. Multiple inputs may be matched to the same concrete spec
node = SpecBuilder.make_node(pkg=input_spec.name)
if input_spec.virtual:
providers = [
diff --git a/lib/spack/spack/test/cmd/install.py b/lib/spack/spack/test/cmd/install.py
index 13721b2a0d..445f376b1b 100644
--- a/lib/spack/spack/test/cmd/install.py
+++ b/lib/spack/spack/test/cmd/install.py
@@ -906,7 +906,7 @@ def test_cdash_configure_warning(tmpdir, mock_fetch, install_mockery, capfd):
specfile = "./spec.json"
with open(specfile, "w") as f:
f.write(spec.to_json())
-
+ print(spec.to_json())
install("--log-file=cdash_reports", "--log-format=cdash", specfile)
# Verify Configure.xml exists with expected contents.
report_dir = tmpdir.join("cdash_reports")
diff --git a/lib/spack/spack/test/concretize.py b/lib/spack/spack/test/concretize.py
index 553d8fd642..4d9940ea9b 100644
--- a/lib/spack/spack/test/concretize.py
+++ b/lib/spack/spack/test/concretize.py
@@ -14,6 +14,7 @@ import archspec.cpu
import llnl.util.lang
import spack.binary_distribution
+import spack.cmd
import spack.compiler
import spack.compilers
import spack.concretize
@@ -3106,3 +3107,20 @@ def test_reuse_prefers_standard_over_git_versions(
test_spec = spack.spec.Spec("git-ref-package@2").concretized()
assert git_spec.dag_hash() != test_spec.dag_hash()
assert standard_spec.dag_hash() == test_spec.dag_hash()
+
+
+@pytest.mark.parametrize("unify", [True, "when_possible", False])
+def test_spec_unification(unify, mutable_config, mock_packages):
+ spack.config.set("concretizer:unify", unify)
+ a = "pkg-a"
+ a_restricted = "pkg-a^pkg-b foo=baz"
+ b = "pkg-b foo=none"
+
+ unrestricted = spack.cmd.parse_specs([a, b], concretize=True)
+ a_concrete_unrestricted = [s for s in unrestricted if s.name == "pkg-a"][0]
+ b_concrete_unrestricted = [s for s in unrestricted if s.name == "pkg-b"][0]
+ assert (a_concrete_unrestricted["pkg-b"] == b_concrete_unrestricted) == (unify is not False)
+
+ maybe_fails = pytest.raises if unify is True else llnl.util.lang.nullcontext
+ with maybe_fails(spack.solver.asp.UnsatisfiableSpecError):
+ _ = spack.cmd.parse_specs([a_restricted, b], concretize=True)
generated by cgit v1.2.3-70-g09d2 (git 2.46.0) at 2025-03-13 22:10:42 +0000