diff options
| -rw-r--r-- | lib/spack/spack/cmd/mirror.py | 46 | ||||
| -rw-r--r-- | lib/spack/spack/environment.py | 25 | ||||
| -rw-r--r-- | lib/spack/spack/test/cmd/mirror.py | 50 | ||||
| -rwxr-xr-x | share/spack/spack-completion.bash | 2 |
4 files changed, 99 insertions, 24 deletions
diff --git a/lib/spack/spack/cmd/mirror.py b/lib/spack/spack/cmd/mirror.py index 1473550a56..2d338204d3 100644 --- a/lib/spack/spack/cmd/mirror.py +++ b/lib/spack/spack/cmd/mirror.py @@ -46,6 +46,15 @@ def setup_parser(subparser): create_parser.add_argument( '-f', '--file', help="file with specs of packages to put in mirror") create_parser.add_argument( + '--exclude-file', + help="specs which Spack should not try to add to a mirror" + " (listed in a file, one per line)") + create_parser.add_argument( + '--exclude-specs', + help="specs which Spack should not try to add to a mirror" + " (specified on command line)") + + create_parser.add_argument( '--skip-unstable-versions', action='store_true', help="don't cache versions unless they identify a stable (unchanging)" " source code") @@ -232,9 +241,7 @@ def _read_specs_from_file(filename): return specs -def mirror_create(args): - """Create a directory to be used as a spack mirror, and fill it with - package archives.""" +def _determine_specs_to_mirror(args): if args.specs and args.all: raise SpackError("Cannot specify specs on command line if you" " chose to mirror all specs with '--all'") @@ -264,6 +271,7 @@ def mirror_create(args): tty.die("Cannot pass specs on the command line with --file.") specs = _read_specs_from_file(args.file) + env_specs = None if not specs: # If nothing is passed, use environment or all if no active env if not args.all: @@ -273,12 +281,9 @@ def mirror_create(args): env = ev.get_env(args, 'mirror') if env: - mirror_specs = env.specs_by_hash.values() + env_specs = env.all_specs() else: specs = [Spec(n) for n in spack.repo.all_package_names()] - mirror_specs = spack.mirror.get_all_versions(specs) - mirror_specs.sort( - key=lambda s: (s.name, s.version)) else: # If the user asked for dependencies, traverse spec DAG get them. if args.dependencies: @@ -297,11 +302,38 @@ def mirror_create(args): msg = 'Skipping {0} as it is an external spec.' tty.msg(msg.format(spec.cshort_spec)) + if env_specs: + if args.versions_per_spec: + tty.warn("Ignoring '--versions-per-spec' for mirroring specs" + " in environment.") + mirror_specs = env_specs + else: if num_versions == 'all': mirror_specs = spack.mirror.get_all_versions(specs) else: mirror_specs = spack.mirror.get_matching_versions( specs, num_versions=num_versions) + mirror_specs.sort( + key=lambda s: (s.name, s.version)) + + exclude_specs = [] + if args.exclude_file: + exclude_specs.extend(_read_specs_from_file(args.exclude_file)) + if args.exclude_specs: + exclude_specs.extend( + spack.cmd.parse_specs(str(args.exclude_specs).split())) + if exclude_specs: + mirror_specs = list( + x for x in mirror_specs + if not any(x.satisfies(y, strict=True) for y in exclude_specs)) + + return mirror_specs + + +def mirror_create(args): + """Create a directory to be used as a spack mirror, and fill it with + package archives.""" + mirror_specs = _determine_specs_to_mirror(args) mirror = spack.mirror.Mirror( args.directory or spack.config.get('config:source_cache')) diff --git a/lib/spack/spack/environment.py b/lib/spack/spack/environment.py index 8d3b1438e1..00af2df2d9 100644 --- a/lib/spack/spack/environment.py +++ b/lib/spack/spack/environment.py @@ -1232,26 +1232,19 @@ class Environment(object): self._install(spec, **kwargs) - def all_specs_by_hash(self): - """Map of hashes to spec for all specs in this environment.""" - # Note this uses dag-hashes calculated without build deps as keys, - # whereas the environment tracks specs based on dag-hashes calculated - # with all dependencies. This function should not be used by an - # Environment object for management of its own data structures - hashes = {} - for h in self.concretized_order: - specs = self.specs_by_hash[h].traverse(deptype=('link', 'run')) - for spec in specs: - hashes[spec.dag_hash()] = spec - return hashes - def all_specs(self): """Return all specs, even those a user spec would shadow.""" - return sorted(self.all_specs_by_hash().values()) + all_specs = set() + for h in self.concretized_order: + all_specs.update(self.specs_by_hash[h].traverse()) + + return sorted(all_specs) def all_hashes(self): - """Return all specs, even those a user spec would shadow.""" - return list(self.all_specs_by_hash().keys()) + """Return hashes of all specs. + + Note these hashes exclude build dependencies.""" + return list(set(s.dag_hash() for s in self.all_specs())) def roots(self): """Specs explicitly requested by the user *in this environment*. diff --git a/lib/spack/spack/test/cmd/mirror.py b/lib/spack/spack/test/cmd/mirror.py index 4bb4fad224..f6fe0b24dd 100644 --- a/lib/spack/spack/test/cmd/mirror.py +++ b/lib/spack/spack/test/cmd/mirror.py @@ -89,6 +89,56 @@ def test_mirror_skip_unstable(tmpdir_factory, mock_packages, config, set(['trivial-pkg-with-valid-hash'])) +class MockMirrorArgs(object): + def __init__(self, specs=None, all=False, file=None, + versions_per_spec=None, dependencies=False, + exclude_file=None, exclude_specs=None): + self.specs = specs or [] + self.all = all + self.file = file + self.versions_per_spec = versions_per_spec + self.dependencies = dependencies + self.exclude_file = exclude_file + self.exclude_specs = exclude_specs + + +def test_exclude_specs(mock_packages): + args = MockMirrorArgs( + specs=['mpich'], + versions_per_spec='all', + exclude_specs="mpich@3.0.1:3.0.2 mpich@1.0") + + mirror_specs = spack.cmd.mirror._determine_specs_to_mirror(args) + expected_include = set(spack.spec.Spec(x) for x in + ['mpich@3.0.3', 'mpich@3.0.4', 'mpich@3.0']) + expected_exclude = set(spack.spec.Spec(x) for x in + ['mpich@3.0.1', 'mpich@3.0.2', 'mpich@1.0']) + assert expected_include <= set(mirror_specs) + assert (not expected_exclude & set(mirror_specs)) + + +def test_exclude_file(mock_packages, tmpdir): + exclude_path = os.path.join(str(tmpdir), 'test-exclude.txt') + with open(exclude_path, 'w') as exclude_file: + exclude_file.write("""\ +mpich@3.0.1:3.0.2 +mpich@1.0 +""") + + args = MockMirrorArgs( + specs=['mpich'], + versions_per_spec='all', + exclude_file=exclude_path) + + mirror_specs = spack.cmd.mirror._determine_specs_to_mirror(args) + expected_include = set(spack.spec.Spec(x) for x in + ['mpich@3.0.3', 'mpich@3.0.4', 'mpich@3.0']) + expected_exclude = set(spack.spec.Spec(x) for x in + ['mpich@3.0.1', 'mpich@3.0.2', 'mpich@1.0']) + assert expected_include <= set(mirror_specs) + assert (not expected_exclude & set(mirror_specs)) + + def test_mirror_crud(tmp_scope, capsys): with capsys.disabled(): mirror('add', '--scope', tmp_scope, 'mirror', 'http://spack.io') diff --git a/share/spack/spack-completion.bash b/share/spack/spack-completion.bash index 6801862bee..fa1e79c9c7 100755 --- a/share/spack/spack-completion.bash +++ b/share/spack/spack-completion.bash @@ -1042,7 +1042,7 @@ _spack_mirror() { _spack_mirror_create() { if $list_options then - SPACK_COMPREPLY="-h --help -d --directory -a --all -f --file --skip-unstable-versions -D --dependencies -n --versions-per-spec" + SPACK_COMPREPLY="-h --help -d --directory -a --all -f --file --exclude-file --exclude-specs --skip-unstable-versions -D --dependencies -n --versions-per-spec" else _all_packages fi |