From f696f02a46f711600267ef171f94bd93cdd73a99 Mon Sep 17 00:00:00 2001 From: Greg Becker Date: Tue, 1 Nov 2022 15:25:55 -0700 Subject: Unit tests: make unit tests work for aarch64 machines (#33625) Currently, many tests hardcode to older versions of gcc for comparisons of concretization among compiler versions. Those versions are too old to concretize for `aarch64`-family targets, which leads to failing tests on `aarch64`. This PR fixes those tests by updating the compiler versions used for testing. Currently, many tests hardcode the expected architecture result in concretization to the `x86_64` family of architectures. This PR generalizes the tests that can be generalized, to cover multiple architecture families. For those that test specific relationships among `x86_64`-family targets, it ensures that concretization uses the `x86_64`-family targets in those cases. Currently, many tests rely on the fact that `AutotoolsPackage` imposes no dependencies on the inheriting package. That is not true on `aarch64`-family architectures. This PR ensures that the fact `AutotoolsPackage` on `aarch64` pulls in a dependency on `gnuconfig` is ignored when testing for the appropriate relationships among dependencies Additionally, 5 tests currently prompt the user for input when `gpg` is available in the user's path. This PR fixes that issue. And 7 tests fail currently when the user has a yubikey available. This PR fixes the incorrect gpg argument causing those issues. --- lib/spack/spack/compilers/__init__.py | 4 + lib/spack/spack/concretize.py | 2 +- lib/spack/spack/environment/environment.py | 2 +- lib/spack/spack/hooks/sbang.py | 4 +- lib/spack/spack/platforms/test.py | 11 +- lib/spack/spack/spec.py | 2 +- lib/spack/spack/test/architecture.py | 7 +- lib/spack/spack/test/build_systems.py | 3 +- lib/spack/spack/test/cmd/ci.py | 67 ++++++--- lib/spack/spack/test/cmd/config.py | 6 +- lib/spack/spack/test/cmd/find.py | 2 +- lib/spack/spack/test/cmd/install.py | 21 ++- lib/spack/spack/test/cmd/style.py | 10 +- lib/spack/spack/test/concretize.py | 63 ++++---- lib/spack/spack/test/concretize_preferences.py | 6 +- lib/spack/spack/test/conftest.py | 9 +- lib/spack/spack/test/data/config/compilers.yaml | 167 ++++++++++++++++++++- lib/spack/spack/test/data/config/packages.yaml | 12 +- lib/spack/spack/test/installer.py | 10 +- lib/spack/spack/test/modules/lmod.py | 2 +- lib/spack/spack/test/sbang.py | 7 +- lib/spack/spack/test/spec_semantics.py | 2 +- lib/spack/spack/test/spec_syntax.py | 8 +- lib/spack/spack/util/gpg.py | 4 +- .../repos/builtin.mock/packages/bowtie/package.py | 2 + .../packages/impossible-concretization/package.py | 1 + .../builtin.mock/packages/openblas/package.py | 4 + 27 files changed, 347 insertions(+), 91 deletions(-) diff --git a/lib/spack/spack/compilers/__init__.py b/lib/spack/spack/compilers/__init__.py index 1d61f37df7..05ffaf9f6c 100644 --- a/lib/spack/spack/compilers/__init__.py +++ b/lib/spack/spack/compilers/__init__.py @@ -346,6 +346,10 @@ def compilers_for_arch(arch_spec, scope=None): return list(get_compilers(config, arch_spec=arch_spec)) +def compiler_specs_for_arch(arch_spec, scope=None): + return [c.spec for c in compilers_for_arch(arch_spec, scope)] + + class CacheReference(object): """This acts as a hashable reference to any object (regardless of whether the object itself is hashable) and also prevents the object from being diff --git a/lib/spack/spack/concretize.py b/lib/spack/spack/concretize.py index 3fbc7e8bff..31066de026 100644 --- a/lib/spack/spack/concretize.py +++ b/lib/spack/spack/concretize.py @@ -734,7 +734,7 @@ def concretize_specs_together(*abstract_specs, **kwargs): Returns: List of concretized specs """ - if spack.config.get("config:concretizer") == "original": + if spack.config.get("config:concretizer", "clingo") == "original": return _concretize_specs_together_original(*abstract_specs, **kwargs) return _concretize_specs_together_new(*abstract_specs, **kwargs) diff --git a/lib/spack/spack/environment/environment.py b/lib/spack/spack/environment/environment.py index 843cd5ff9c..1353b76b56 100644 --- a/lib/spack/spack/environment/environment.py +++ b/lib/spack/spack/environment/environment.py @@ -1376,7 +1376,7 @@ class Environment(object): arguments.append((uspec_constraints, tests)) # Ensure we don't try to bootstrap clingo in parallel - if spack.config.get("config:concretizer") == "clingo": + if spack.config.get("config:concretizer", "clingo") == "clingo": with spack.bootstrap.ensure_bootstrap_configuration(): spack.bootstrap.ensure_clingo_importable_or_raise() diff --git a/lib/spack/spack/hooks/sbang.py b/lib/spack/spack/hooks/sbang.py index 8731ef0c07..5d2de2b865 100644 --- a/lib/spack/spack/hooks/sbang.py +++ b/lib/spack/spack/hooks/sbang.py @@ -205,7 +205,9 @@ def install_sbang(): fs.set_install_permissions(sbang_bin_dir) # set group on sbang_bin_dir if not already set (only if set in configuration) - if group_name and grp.getgrgid(os.stat(sbang_bin_dir).st_gid).gr_name != group_name: + # TODO: after we drop python2 support, use shutil.chown to avoid gid lookups that + # can fail for remote groups + if group_name and os.stat(sbang_bin_dir).st_gid != grp.getgrnam(group_name).gr_gid: os.chown(sbang_bin_dir, os.stat(sbang_bin_dir).st_uid, grp.getgrnam(group_name).gr_gid) # copy over the fresh copy of `sbang` diff --git a/lib/spack/spack/platforms/test.py b/lib/spack/spack/platforms/test.py index d181f2e926..26fe943394 100644 --- a/lib/spack/spack/platforms/test.py +++ b/lib/spack/spack/platforms/test.py @@ -16,9 +16,14 @@ class Test(Platform): if platform.system().lower() == "darwin": binary_formats = ["macho"] - front_end = "x86_64" - back_end = "core2" - default = "core2" + if platform.machine() == "arm64": + front_end = "aarch64" + back_end = "m1" + default = "m1" + else: + front_end = "x86_64" + back_end = "core2" + default = "core2" front_os = "redhat6" back_os = "debian6" diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index 41bb7b57ee..10f6dfb8e9 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -2945,7 +2945,7 @@ class Spec(object): if a list of names activate them for the packages in the list, if True activate 'test' dependencies for all packages. """ - if spack.config.get("config:concretizer") == "clingo": + if spack.config.get("config:concretizer", "clingo") == "clingo": self._new_concretize(tests) else: self._old_concretize(tests) diff --git a/lib/spack/spack/test/architecture.py b/lib/spack/spack/test/architecture.py index c357c650e8..746be0c8be 100644 --- a/lib/spack/spack/test/architecture.py +++ b/lib/spack/spack/test/architecture.py @@ -140,7 +140,7 @@ def test_optimization_flags(compiler_spec, target_name, expected_flags, config): (spack.spec.CompilerSpec("gcc@9.2.0"), None, "haswell", "-march=haswell -mtune=haswell"), # Check that custom string versions are accepted ( - spack.spec.CompilerSpec("gcc@foo"), + spack.spec.CompilerSpec("gcc@10foo"), "9.2.0", "icelake", "-march=icelake-client -mtune=icelake-client", @@ -196,7 +196,10 @@ def test_satisfy_strict_constraint_when_not_concrete(architecture_tuple, constra ], ) @pytest.mark.usefixtures("mock_packages", "config") -def test_concretize_target_ranges(root_target_range, dep_target_range, result): +def test_concretize_target_ranges(root_target_range, dep_target_range, result, monkeypatch): + # Monkeypatch so that all concretization is done as if the machine is core2 + monkeypatch.setattr(spack.platforms.test.Test, "default", "core2") + # use foobar=bar to make the problem simpler for the old concretizer # the new concretizer should not need that help if spack.config.get("config:concretizer") == "original": diff --git a/lib/spack/spack/test/build_systems.py b/lib/spack/spack/test/build_systems.py index c4255eef49..03a920a28e 100644 --- a/lib/spack/spack/test/build_systems.py +++ b/lib/spack/spack/test/build_systems.py @@ -204,11 +204,12 @@ class TestAutotoolsPackage(object): assert "gnuconfig version of config.guess" not in f.read() @pytest.mark.disable_clean_stage_check - def test_autotools_gnuconfig_replacement_no_gnuconfig(self, mutable_database): + def test_autotools_gnuconfig_replacement_no_gnuconfig(self, mutable_database, monkeypatch): """ Tests whether a useful error message is shown when patch_config_files is enabled, but gnuconfig is not listed as a direct build dependency. """ + monkeypatch.setattr(spack.platforms.test.Test, "default", "x86_64") s = Spec("autotools-config-replacement +patch_config_files ~gnuconfig") s.concretize() diff --git a/lib/spack/spack/test/cmd/ci.py b/lib/spack/spack/test/cmd/ci.py index 117dec0ed4..5529defee5 100644 --- a/lib/spack/spack/test/cmd/ci.py +++ b/lib/spack/spack/test/cmd/ci.py @@ -190,6 +190,12 @@ spack: tags: - donotcare image: donotcare + - match: + - arch=test-debian6-m1 + runner-attributes: + tags: + - donotcare + image: donotcare service-job-attributes: image: donotcare tags: [donotcare] @@ -270,10 +276,10 @@ def test_ci_generate_bootstrap_gcc( spack: definitions: - bootstrap: - - gcc@3.0 - - gcc@2.0 + - gcc@9.5 + - gcc@9.0 specs: - - dyninst%gcc@3.0 + - dyninst%gcc@9.5 mirrors: some-mirror: https://my.fake.mirror gitlab-ci: @@ -286,6 +292,11 @@ spack: runner-attributes: tags: - donotcare + - match: + - arch=test-debian6-aarch64 + runner-attributes: + tags: + - donotcare """ ) @@ -338,9 +349,9 @@ def test_ci_generate_bootstrap_artifacts_buildcache( spack: definitions: - bootstrap: - - gcc@3.0 + - gcc@9.5 specs: - - dyninst%gcc@3.0 + - dyninst%gcc@9.5 mirrors: some-mirror: https://my.fake.mirror gitlab-ci: @@ -353,6 +364,11 @@ spack: runner-attributes: tags: - donotcare + - match: + - arch=test-debian6-aarch64 + runner-attributes: + tags: + - donotcare enable-artifacts-buildcache: True """ ) @@ -1524,12 +1540,12 @@ def test_ci_generate_with_workarounds( """\ spack: specs: - - callpath%gcc@3.0 + - callpath%gcc@9.5 mirrors: some-mirror: https://my.fake.mirror gitlab-ci: mappings: - - match: ['%gcc@3.0'] + - match: ['%gcc@9.5'] runner-attributes: tags: - donotcare @@ -1639,28 +1655,28 @@ def test_ci_generate_bootstrap_prune_dag( mirror_url = "file://{0}".format(mirror_dir.strpath) # Install a compiler, because we want to put it in a buildcache - install_cmd("gcc@10.1.0%gcc@4.5.0") + install_cmd("gcc@12.2.0%gcc@10.2.1") # Put installed compiler in the buildcache - buildcache_cmd("create", "-u", "-a", "-f", "-d", mirror_dir.strpath, "gcc@10.1.0%gcc@4.5.0") + buildcache_cmd("create", "-u", "-a", "-f", "-d", mirror_dir.strpath, "gcc@12.2.0%gcc@10.2.1") # Now uninstall the compiler - uninstall_cmd("-y", "gcc@10.1.0%gcc@4.5.0") + uninstall_cmd("-y", "gcc@12.2.0%gcc@10.2.1") monkeypatch.setattr(spack.concretize.Concretizer, "check_for_compiler_existence", False) spack.config.set("config:install_missing_compilers", True) - assert CompilerSpec("gcc@10.1.0") not in compilers.all_compiler_specs() + assert CompilerSpec("gcc@12.2.0") not in compilers.all_compiler_specs() # Configure the mirror where we put that buildcache w/ the compiler mirror_cmd("add", "test-mirror", mirror_url) - install_cmd("--no-check-signature", "a%gcc@10.1.0") + install_cmd("--no-check-signature", "b%gcc@12.2.0") # Put spec built with installed compiler in the buildcache - buildcache_cmd("create", "-u", "-a", "-f", "-d", mirror_dir.strpath, "a%gcc@10.1.0") + buildcache_cmd("create", "-u", "-a", "-f", "-d", mirror_dir.strpath, "b%gcc@12.2.0") # Now uninstall the spec - uninstall_cmd("-y", "a%gcc@10.1.0") + uninstall_cmd("-y", "b%gcc@12.2.0") filename = str(tmpdir.join("spack.yaml")) with open(filename, "w") as f: @@ -1669,9 +1685,9 @@ def test_ci_generate_bootstrap_prune_dag( spack: definitions: - bootstrap: - - gcc@10.1.0%gcc@4.5.0 + - gcc@12.2.0%gcc@10.2.1 specs: - - a%gcc@10.1.0 + - b%gcc@12.2.0 mirrors: atestm: {0} gitlab-ci: @@ -1689,6 +1705,16 @@ spack: runner-attributes: tags: - meh + - match: + - arch=test-debian6-aarch64 + runner-attributes: + tags: + - donotcare + - match: + - arch=test-debian6-m1 + runner-attributes: + tags: + - meh """.format( mirror_url ) @@ -1746,10 +1772,6 @@ spack: "(specs) b": [ "(bootstrap) gcc", ], - "(specs) a": [ - "(bootstrap) gcc", - "(specs) b", - ], } _validate_needs_graph(new_yaml_contents, needs_graph, False) @@ -2152,7 +2174,10 @@ spack: ci_cmd("generate", "--output-file", pipeline_path, "--artifacts-root", artifacts_root) - job_name = ci.get_job_name("specs", False, job_spec, "test-debian6-core2", None) + target_name = spack.platforms.test.Test.default + job_name = ci.get_job_name( + "specs", False, job_spec, "test-debian6-%s" % target_name, None + ) repro_file = os.path.join(working_dir.strpath, "repro.json") repro_details = { diff --git a/lib/spack/spack/test/cmd/config.py b/lib/spack/spack/test/cmd/config.py index cdf9bc00a0..e5ad82cf43 100644 --- a/lib/spack/spack/test/cmd/config.py +++ b/lib/spack/spack/test/cmd/config.py @@ -634,13 +634,13 @@ def test_config_prefer_upstream( # Make sure only the non-default variants are set. assert packages["boost"] == { - "compiler": ["gcc@4.5.0"], + "compiler": ["gcc@10.2.1"], "variants": "+debug +graph", "version": ["1.63.0"], } - assert packages["dependency-install"] == {"compiler": ["gcc@4.5.0"], "version": ["2.0"]} + assert packages["dependency-install"] == {"compiler": ["gcc@10.2.1"], "version": ["2.0"]} # Ensure that neither variant gets listed for hdf5, since they conflict - assert packages["hdf5"] == {"compiler": ["gcc@4.5.0"], "version": ["2.3"]} + assert packages["hdf5"] == {"compiler": ["gcc@10.2.1"], "version": ["2.3"]} # Make sure a message about the conflicting hdf5's was given. assert "- hdf5" in output diff --git a/lib/spack/spack/test/cmd/find.py b/lib/spack/spack/test/cmd/find.py index b313a36446..be83541096 100644 --- a/lib/spack/spack/test/cmd/find.py +++ b/lib/spack/spack/test/cmd/find.py @@ -322,7 +322,7 @@ def test_find_very_long(database, config): @pytest.mark.db def test_find_show_compiler(database, config): output = find("--no-groups", "--show-full-compiler", "mpileaks") - assert "mpileaks@2.3%gcc@4.5.0" in output + assert "mpileaks@2.3%gcc@10.2.1" in output @pytest.mark.db diff --git a/lib/spack/spack/test/cmd/install.py b/lib/spack/spack/test/cmd/install.py index 41de6565a1..843e539eea 100644 --- a/lib/spack/spack/test/cmd/install.py +++ b/lib/spack/spack/test/cmd/install.py @@ -935,7 +935,16 @@ def test_cdash_configure_warning(tmpdir, mock_fetch, install_mockery, capfd): with capfd.disabled(): with tmpdir.as_cwd(): # Test would fail if install raised an error. - install("--log-file=cdash_reports", "--log-format=cdash", "configure-warning") + + # Ensure that even on non-x86_64 architectures, there are no + # dependencies installed + spec = spack.spec.Spec("configure-warning").concretized() + spec.clear_dependencies() + specfile = "./spec.json" + with open(specfile, "w") as f: + f.write(spec.to_json()) + + install("--log-file=cdash_reports", "--log-format=cdash", specfile) # Verify Configure.xml exists with expected contents. report_dir = tmpdir.join("cdash_reports") assert report_dir in tmpdir.listdir() @@ -955,10 +964,10 @@ def test_compiler_bootstrap( ): monkeypatch.setattr(spack.concretize.Concretizer, "check_for_compiler_existence", False) spack.config.set("config:install_missing_compilers", True) - assert CompilerSpec("gcc@2.0") not in compilers.all_compiler_specs() + assert CompilerSpec("gcc@12.0") not in compilers.all_compiler_specs() # Test succeeds if it does not raise an error - install("a%gcc@2.0") + install("a%gcc@12.0") def test_compiler_bootstrap_from_binary_mirror( @@ -1013,11 +1022,11 @@ def test_compiler_bootstrap_already_installed( monkeypatch.setattr(spack.concretize.Concretizer, "check_for_compiler_existence", False) spack.config.set("config:install_missing_compilers", True) - assert CompilerSpec("gcc@2.0") not in compilers.all_compiler_specs() + assert CompilerSpec("gcc@12.0") not in compilers.all_compiler_specs() # Test succeeds if it does not raise an error - install("gcc@2.0") - install("a%gcc@2.0") + install("gcc@12.0") + install("a%gcc@12.0") def test_install_fails_no_args(tmpdir): diff --git a/lib/spack/spack/test/cmd/style.py b/lib/spack/spack/test/cmd/style.py index 179973af52..fd727d088e 100644 --- a/lib/spack/spack/test/cmd/style.py +++ b/lib/spack/spack/test/cmd/style.py @@ -92,14 +92,14 @@ def test_changed_files_from_git_rev_base(tmpdir, capfd): git("checkout", "-b", "main") git("config", "user.name", "test user") git("config", "user.email", "test@user.com") - git("commit", "--allow-empty", "-m", "initial commit") + git("commit", "--no-gpg-sign", "--allow-empty", "-m", "initial commit") tmpdir.ensure("bin/spack") assert changed_files(base="HEAD") == ["bin/spack"] assert changed_files(base="main") == ["bin/spack"] git("add", "bin/spack") - git("commit", "-m", "v1") + git("commit", "--no-gpg-sign", "-m", "v1") assert changed_files(base="HEAD") == [] assert changed_files(base="HEAD~") == ["bin/spack"] @@ -113,7 +113,7 @@ def test_changed_no_base(tmpdir, capfd): git("config", "user.name", "test user") git("config", "user.email", "test@user.com") git("add", ".") - git("commit", "-m", "initial commit") + git("commit", "--no-gpg-sign", "-m", "initial commit") with pytest.raises(SystemExit): changed_files(base="foobar") @@ -198,7 +198,7 @@ def external_style_root(flake8_package_with_errors, tmpdir): git("config", "user.name", "test user") git("config", "user.email", "test@user.com") git("add", ".") - git("commit", "-m", "initial commit") + git("commit", "--no-gpg-sign", "-m", "initial commit") git("branch", "-m", "develop") git("checkout", "-b", "feature") @@ -210,7 +210,7 @@ def external_style_root(flake8_package_with_errors, tmpdir): # add the buggy file on the feature branch with tmpdir.as_cwd(): git("add", str(py_file)) - git("commit", "-m", "add new file") + git("commit", "--no-gpg-sign", "-m", "add new file") yield tmpdir, py_file diff --git a/lib/spack/spack/test/concretize.py b/lib/spack/spack/test/concretize.py index 9f380975ed..bf0110f3e6 100644 --- a/lib/spack/spack/test/concretize.py +++ b/lib/spack/spack/test/concretize.py @@ -313,8 +313,8 @@ class TestConcretize(object): def test_different_compilers_get_different_flags(self): client = Spec( - "cmake-client %gcc@4.7.2 platform=test os=fe target=fe" - + " ^cmake %clang@3.5 platform=test os=fe target=fe" + "cmake-client %gcc@11.1.0 platform=test os=fe target=fe" + + " ^cmake %clang@12.2.0 platform=test os=fe target=fe" ) client.concretize() cmake = client["cmake"] @@ -328,7 +328,7 @@ class TestConcretize(object): UnavailableCompilerVersionError if the architecture is concretized incorrectly. """ - spec = Spec("cmake-client %gcc@4.7.2 os=fe ^ cmake") + spec = Spec("cmake-client %gcc@11.1.0 os=fe ^ cmake") spec.concretize() assert spec["cmake"].architecture == spec.architecture @@ -452,7 +452,7 @@ class TestConcretize(object): spec.normalize() spec.concretize() - @pytest.mark.parametrize("compiler_str", ["clang", "gcc", "gcc@4.5.0", "clang@:3.3.0"]) + @pytest.mark.parametrize("compiler_str", ["clang", "gcc", "gcc@10.2.1", "clang@:12.0.0"]) def test_compiler_inheritance(self, compiler_str): spec_str = "mpileaks %{0}".format(compiler_str) spec = Spec(spec_str).concretized() @@ -695,15 +695,15 @@ class TestConcretize(object): @pytest.mark.regression("8735,14730") def test_compiler_version_matches_any_entry_in_compilers_yaml(self): # Ensure that a concrete compiler with different compiler version - # doesn't match (here it's 4.5 vs. 4.5.0) + # doesn't match (here it's 10.2 vs. 10.2.1) with pytest.raises(spack.concretize.UnavailableCompilerVersionError): - s = Spec("mpileaks %gcc@4.5") + s = Spec("mpileaks %gcc@10.2") s.concretize() # An abstract compiler with a version list could resolve to 4.5.0 - s = Spec("mpileaks %gcc@4.5:") + s = Spec("mpileaks %gcc@10.2:") s.concretize() - assert str(s.compiler.version) == "4.5.0" + assert str(s.compiler.version) == "10.2.1" def test_concretize_anonymous(self): with pytest.raises(spack.error.SpackError): @@ -720,11 +720,11 @@ class TestConcretize(object): "spec_str,expected_str", [ # Unconstrained versions select default compiler (gcc@4.5.0) - ("bowtie@1.3.0", "%gcc@4.5.0"), + ("bowtie@1.4.0", "%gcc@10.2.1"), # Version with conflicts and no valid gcc select another compiler - ("bowtie@1.2.2", "%clang@3.3"), + ("bowtie@1.3.0", "%clang@12.0.0"), # If a higher gcc is available still prefer that - ("bowtie@1.2.2 os=redhat6", "%gcc@4.7.2"), + ("bowtie@1.2.2 os=redhat6", "%gcc@11.1.0"), ], ) def test_compiler_conflicts_in_package_py(self, spec_str, expected_str): @@ -1047,11 +1047,11 @@ class TestConcretize(object): # that doesn't allow newer versions with gcc@4.4.0. Check # that an old version of openblas is selected, rather than # a different compiler for just that node. - spec_str = "simple-inheritance+openblas %gcc@4.4.0 os=redhat6" + spec_str = "simple-inheritance+openblas %gcc@10.1.0 os=redhat6" s = Spec(spec_str).concretized() - assert "openblas@0.2.13" in s - assert s["openblas"].satisfies("%gcc@4.4.0") + assert "openblas@0.2.15" in s + assert s["openblas"].satisfies("%gcc@10.1.0") @pytest.mark.regression("19981") def test_target_ranges_in_conflicts(self): @@ -1080,8 +1080,8 @@ class TestConcretize(object): if spack.config.get("config:concretizer") == "original": pytest.xfail("Known failure of the original concretizer") - s = Spec("a %gcc@foo os=redhat6").concretized() - assert "%gcc@foo" in s + s = Spec("a %gcc@10foo os=redhat6").concretized() + assert "%gcc@10foo" in s def test_all_patches_applied(self): uuidpatch = ( @@ -1273,8 +1273,8 @@ class TestConcretize(object): ("mpileaks", "os=debian6"), # To trigger the bug in 22871 we need to have the same compiler # spec available on both operating systems - ("mpileaks%gcc@4.5.0 platform=test os=debian6", "os=debian6"), - ("mpileaks%gcc@4.5.0 platform=test os=redhat6", "os=redhat6"), + ("mpileaks%gcc@10.2.1 platform=test os=debian6", "os=debian6"), + ("mpileaks%gcc@10.2.1 platform=test os=redhat6", "os=redhat6"), ], ) def test_os_selection_when_multiple_choices_are_possible(self, spec_str, expected_os): @@ -1286,7 +1286,7 @@ class TestConcretize(object): @pytest.mark.regression("22718") @pytest.mark.parametrize( "spec_str,expected_compiler", - [("mpileaks", "%gcc@4.5.0"), ("mpileaks ^mpich%clang@3.3", "%clang@3.3")], + [("mpileaks", "%gcc@10.2.1"), ("mpileaks ^mpich%clang@12.0.0", "%clang@12.0.0")], ) def test_compiler_is_unique(self, spec_str, expected_compiler): s = Spec(spec_str).concretized() @@ -1462,10 +1462,12 @@ class TestConcretize(object): # The test architecture uses core2 as the default target. Check that when # we configure Spack for "generic" granularity we concretize for x86_64 + default_target = spack.platforms.test.Test.default + generic_target = archspec.cpu.TARGETS[default_target].generic.name s = Spec("python") - assert s.concretized().satisfies("target=core2") + assert s.concretized().satisfies("target=%s" % default_target) with spack.config.override("concretizer:targets", {"granularity": "generic"}): - assert s.concretized().satisfies("target=x86_64") + assert s.concretized().satisfies("target=%s" % generic_target) def test_host_compatible_concretization(self): if spack.config.get("config:concretizer") == "original": @@ -1692,12 +1694,19 @@ class TestConcretize(object): # version_declared("b","0.9",1,"package_py"). # version_declared("b","1.0",2,"installed"). # version_declared("b","0.9",3,"installed"). - for criterion in [ - (1, None, "number of packages to build (vs. reuse)"), + # + # Depending on the target, it may also use gnuconfig + result_spec = result.specs[0] + num_specs = len(list(result_spec.traverse())) + + criteria = [ + (num_specs - 1, None, "number of packages to build (vs. reuse)"), (2, 0, "version badness"), - ]: + ] + + for criterion in criteria: assert criterion in result.criteria - assert result.specs[0].satisfies("^b@1.0") + assert result_spec.satisfies("^b@1.0") @pytest.mark.regression("31169") def test_not_reusing_incompatible_os_or_compiler(self): @@ -1717,8 +1726,8 @@ class TestConcretize(object): setup = spack.solver.asp.SpackSolverSetup() result, _, _ = solver.driver.solve(setup, [root_spec], reuse=reusable_specs) concrete_spec = result.specs[0] - assert concrete_spec.satisfies("%gcc@4.5.0") - assert concrete_spec.satisfies("os=debian6") + assert concrete_spec.satisfies("%{}".format(s.compiler)) + assert concrete_spec.satisfies("os={}".format(s.architecture.os)) def test_git_hash_assigned_version_is_preferred(self): hash = "a" * 40 diff --git a/lib/spack/spack/test/concretize_preferences.py b/lib/spack/spack/test/concretize_preferences.py index a6e568bd11..7bae90cb2c 100644 --- a/lib/spack/spack/test/concretize_preferences.py +++ b/lib/spack/spack/test/concretize_preferences.py @@ -8,6 +8,8 @@ import stat import pytest +import archspec + import spack.config import spack.package_prefs import spack.repo @@ -109,7 +111,9 @@ class TestConcretizePreferences(object): pytest.skip("Fixing the parser broke this test for the original concretizer.") # Need to make sure the test uses an available compiler - compiler_list = spack.compilers.all_compiler_specs() + arch = spack.spec.ArchSpec(("test", "redhat6", archspec.cpu.host().name)) + + compiler_list = spack.compilers.compiler_specs_for_arch(arch) assert compiler_list # Try the first available compiler diff --git a/lib/spack/spack/test/conftest.py b/lib/spack/spack/test/conftest.py index 1f3c17916c..2f04e1d4fd 100644 --- a/lib/spack/spack/test/conftest.py +++ b/lib/spack/spack/test/conftest.py @@ -128,7 +128,14 @@ def mock_git_version_info(tmpdir, override_git_repos_cache_path): def commit(message): global commit_counter - git("commit", "--date", "2020-01-%02d 12:0:00 +0300" % commit_counter, "-am", message) + git( + "commit", + "--no-gpg-sign", + "--date", + "2020-01-%02d 12:0:00 +0300" % commit_counter, + "-am", + message, + ) commit_counter += 1 with working_dir(repo_path): diff --git a/lib/spack/spack/test/data/config/compilers.yaml b/lib/spack/spack/test/data/config/compilers.yaml index e0b0464976..6f36c13b59 100644 --- a/lib/spack/spack/test/data/config/compilers.yaml +++ b/lib/spack/spack/test/data/config/compilers.yaml @@ -130,6 +130,7 @@ compilers: f77: /path/to/gfortran440 fc: /path/to/gfortran440 modules: 'None' + target: x86_64 - compiler: spec: clang@3.5 operating_system: redhat6 @@ -167,7 +168,7 @@ compilers: modules: 'None' target: x86_64 - compiler: - spec: gcc@foo + spec: gcc@10foo operating_system: redhat6 paths: cc: /path/to/gcc @@ -186,3 +187,167 @@ compilers: fc: /path/to/gfortran modules: 'None' target: x86_64 +- compiler: + spec: clang@12.0.0 + operating_system: {0.name}{0.version} + paths: + cc: /path/to/clang + cxx: /path/to/clang++ + f77: None + fc: None + modules: 'None' + target: aarch64 +- compiler: + spec: gcc@10.2.1 + operating_system: {0.name}{0.version} + paths: + cc: /path/to/gcc + cxx: /path/to/g++ + f77: None + fc: None + modules: 'None' + target: aarch64 +- compiler: + spec: clang@12.0.0 + operating_system: redhat6 + paths: + cc: /path/to/clang + cxx: /path/to/clang++ + f77: None + fc: None + modules: 'None' + target: aarch64 +- compiler: + spec: gcc@10.2.1 + operating_system: redhat6 + paths: + cc: /path/to/gcc + cxx: /path/to/g++ + f77: None + fc: None + modules: 'None' + target: aarch64 +- compiler: + spec: gcc@10.1.0 + operating_system: redhat6 + paths: + cc: /path/to/gcc + cxx: /path/to/g++ + f77: None + fc: None + modules: 'None' + target: aarch64 +- compiler: + spec: gcc@11.1.0 + operating_system: redhat6 + paths: + cc: /path/to/gcc + cxx: /path/to/g++ + f77: None + fc: None + flags: + cflags: -O0 -g + cxxflags: -O0 -g + fflags: -O0 -g + modules: 'None' + target: aarch64 +- compiler: + spec: clang@12.2.0 + operating_system: redhat6 + paths: + cc: /path/to/clang35 + cxx: /path/to/clang++35 + f77: None + fc: None + flags: + cflags: -O3 + cxxflags: -O3 + modules: 'None' + target: aarch64 +- compiler: + spec: gcc@10foo + operating_system: redhat6 + paths: + cc: /path/to/gcc + cxx: /path/to/g++ + f77: /path/to/gfortran + fc: /path/to/gfortran + modules: 'None' + target: aarch64 +- compiler: + spec: clang@12.0.0 + operating_system: {0.name}{0.version} + paths: + cc: /path/to/clang + cxx: /path/to/clang++ + f77: None + fc: None + modules: 'None' + target: x86_64 +- compiler: + spec: gcc@10.2.1 + operating_system: {0.name}{0.version} + paths: + cc: /path/to/gcc + cxx: /path/to/g++ + f77: None + fc: None + modules: 'None' + target: x86_64 +- compiler: + spec: clang@12.0.0 + operating_system: redhat6 + paths: + cc: /path/to/clang + cxx: /path/to/clang++ + f77: None + fc: None + modules: 'None' + target: x86_64 +- compiler: + spec: gcc@10.2.1 + operating_system: redhat6 + paths: + cc: /path/to/gcc + cxx: /path/to/g++ + f77: None + fc: None + modules: 'None' + target: x86_64 +- compiler: + spec: gcc@10.1.0 + operating_system: redhat6 + paths: + cc: /path/to/gcc + cxx: /path/to/g++ + f77: None + fc: None + modules: 'None' + target: x86_64 +- compiler: + spec: gcc@11.1.0 + operating_system: redhat6 + paths: + cc: /path/to/gcc + cxx: /path/to/g++ + f77: None + fc: None + flags: + cflags: -O0 -g + cxxflags: -O0 -g + fflags: -O0 -g + modules: 'None' + target: x86_64 +- compiler: + spec: clang@12.2.0 + operating_system: redhat6 + paths: + cc: /path/to/clang35 + cxx: /path/to/clang++35 + f77: None + fc: None + flags: + cflags: -O3 + cxxflags: -O3 + modules: 'None' + target: x86_64 diff --git a/lib/spack/spack/test/data/config/packages.yaml b/lib/spack/spack/test/data/config/packages.yaml index 7d5e06a74c..c6ff731a7e 100644 --- a/lib/spack/spack/test/data/config/packages.yaml +++ b/lib/spack/spack/test/data/config/packages.yaml @@ -7,18 +7,18 @@ packages: externaltool: buildable: False externals: - - spec: externaltool@1.0%gcc@4.5.0 + - spec: externaltool@1.0%gcc@10.2.1 prefix: /path/to/external_tool - - spec: externaltool@0.9%gcc@4.5.0 + - spec: externaltool@0.9%gcc@10.2.1 prefix: /usr - - spec: externaltool@0_8%gcc@4.5.0 + - spec: externaltool@0_8%gcc@10.2.1 prefix: /usr externalvirtual: buildable: False externals: - - spec: externalvirtual@2.0%clang@3.3 + - spec: externalvirtual@2.0%clang@12.0.0 prefix: /path/to/external_virtual_clang - - spec: externalvirtual@1.0%gcc@4.5.0 + - spec: externalvirtual@1.0%gcc@10.2.1 prefix: /path/to/external_virtual_gcc externalmodule: buildable: False @@ -49,4 +49,4 @@ packages: - spec: external-non-default-variant@3.8.7~foo~bar prefix: /usr version-test-dependency-preferred: - version: ['5.2.5'] \ No newline at end of file + version: ['5.2.5'] diff --git a/lib/spack/spack/test/installer.py b/lib/spack/spack/test/installer.py index 7f608d6c89..363d92fe1f 100644 --- a/lib/spack/spack/test/installer.py +++ b/lib/spack/spack/test/installer.py @@ -632,7 +632,10 @@ def test_check_deps_status_external(install_mockery, monkeypatch): # Mock the known dependent, b, as external so assumed to be installed monkeypatch.setattr(spack.spec.Spec, "external", True) installer._check_deps_status(request) - assert list(installer.installed)[0].startswith("b") + + # exotic architectures will add dependencies on gnuconfig, which we want to ignore + installed = [x for x in installer.installed if not x.startswith("gnuconfig")] + assert installed[0].startswith("b") def test_check_deps_status_upstream(install_mockery, monkeypatch): @@ -643,7 +646,10 @@ def test_check_deps_status_upstream(install_mockery, monkeypatch): # Mock the known dependent, b, as installed upstream monkeypatch.setattr(spack.spec.Spec, "installed_upstream", True) installer._check_deps_status(request) - assert list(installer.installed)[0].startswith("b") + + # exotic architectures will add dependencies on gnuconfig, which we want to ignore + installed = [x for x in installer.installed if not x.startswith("gnuconfig")] + assert installed[0].startswith("b") def test_add_bootstrap_compilers(install_mockery, monkeypatch): diff --git a/lib/spack/spack/test/modules/lmod.py b/lib/spack/spack/test/modules/lmod.py index 58c013ef86..d24ec8ca0d 100644 --- a/lib/spack/spack/test/modules/lmod.py +++ b/lib/spack/spack/test/modules/lmod.py @@ -24,7 +24,7 @@ writer_cls = spack.modules.lmod.LmodModulefileWriter pytestmark = pytest.mark.skipif(sys.platform == "win32", reason="does not run on windows") -@pytest.fixture(params=["clang@3.3", "gcc@4.5.0"]) +@pytest.fixture(params=["clang@12.0.0", "gcc@10.2.1"]) def compiler(request): return request.param diff --git a/lib/spack/spack/test/sbang.py b/lib/spack/spack/test/sbang.py index ebc9a85347..18a408a549 100644 --- a/lib/spack/spack/test/sbang.py +++ b/lib/spack/spack/test/sbang.py @@ -7,6 +7,7 @@ Test that Spack's shebang filtering works correctly. """ import filecmp +import getpass import os import shutil import stat @@ -268,6 +269,10 @@ def test_shebang_handles_non_writable_files(script_dir, sbang_line): @pytest.fixture(scope="function") def configure_group_perms(): + # On systems with remote groups, the primary user group may be remote + # and grp does not act on remote groups. + # To ensure we find a group we can operate on, we get take the first group + # listed which has the current user as a member. conf = syaml.load_config( """\ all: @@ -276,7 +281,7 @@ all: write: group group: {0} """.format( - grp.getgrgid(os.getegid()).gr_name + [g.gr_name for g in grp.getgrall() if getpass.getuser() in g.gr_mem][0] ) ) spack.config.set("packages", conf, scope="user") diff --git a/lib/spack/spack/test/spec_semantics.py b/lib/spack/spack/test/spec_semantics.py index 7100cfdb5f..711e901416 100644 --- a/lib/spack/spack/test/spec_semantics.py +++ b/lib/spack/spack/test/spec_semantics.py @@ -841,7 +841,7 @@ class TestSpecSematics(object): # Spack was assembling flags in a manner that could result in # different orderings for repeated concretizations of the same # spec and config - spec_str = "libelf %gcc@4.7.2 os=redhat6" + spec_str = "libelf %gcc@11.1.0 os=redhat6" for _ in range(25): s = Spec(spec_str).concretized() assert all( diff --git a/lib/spack/spack/test/spec_syntax.py b/lib/spack/spack/test/spec_syntax.py index d73e226d11..ab560bed08 100644 --- a/lib/spack/spack/test/spec_syntax.py +++ b/lib/spack/spack/test/spec_syntax.py @@ -277,13 +277,17 @@ class TestSpecSyntax(object): "x ^y@1,2:3,4%intel@1,2,3,4+a~b+c~d+e~f", "x ^y~f+e~d+c~b+a@4,2:3,1%intel@4,3,2,1" ) + default_target = spack.platforms.test.Test.default self.check_parse( - "x arch=test-redhat6-None" " ^y arch=test-None-core2" " ^z arch=linux-None-None", + "x arch=test-redhat6-None" + + (" ^y arch=test-None-%s" % default_target) + + " ^z arch=linux-None-None", "x os=fe " "^y target=be " "^z platform=linux", ) self.check_parse( - "x arch=test-debian6-core2" " ^y arch=test-debian6-core2", + ("x arch=test-debian6-%s" % default_target) + + (" ^y arch=test-debian6-%s" % default_target), "x os=default_os target=default_target" " ^y os=default_os target=default_target", ) diff --git a/lib/spack/spack/util/gpg.py b/lib/spack/spack/util/gpg.py index 3f0d74f4b7..ffee274c3f 100644 --- a/lib/spack/spack/util/gpg.py +++ b/lib/spack/spack/util/gpg.py @@ -239,7 +239,7 @@ def trust(keyfile): keys = _get_unimported_public_keys(output) # Import them - GPG("--import", keyfile) + GPG("--batch", "--import", keyfile) # Set trust to ultimate key_to_fpr = dict(public_keys_to_fingerprint()) @@ -285,7 +285,7 @@ def sign(key, file, output, clearsign=False): signature, if False creates a detached signature """ signopt = "--clearsign" if clearsign else "--detach-sign" - GPG(signopt, "--armor", "--default-key", key, "--output", output, file) + GPG(signopt, "--armor", "--local-user", key, "--output", output, file) @_autoinit diff --git a/var/spack/repos/builtin.mock/packages/bowtie/package.py b/var/spack/repos/builtin.mock/packages/bowtie/package.py index 24d7312b74..9bef211172 100644 --- a/var/spack/repos/builtin.mock/packages/bowtie/package.py +++ b/var/spack/repos/builtin.mock/packages/bowtie/package.py @@ -11,8 +11,10 @@ class Bowtie(Package): homepage = "http://www.example.org" url = "http://bowtie-1.2.2.tar.bz2" + version("1.4.0", "1c837ecd990bb022d07e7aab32b09847") version("1.3.0", "1c837ecd990bb022d07e7aab32b09847") version("1.2.2", "1c837ecd990bb022d07e7aab32b09847") version("1.2.0", "1c837ecd990bb022d07e7aab32b09847") conflicts("%gcc@:4.5.0", when="@1.2.2") + conflicts("%gcc@:10.2.1", when="@:1.3.0") diff --git a/var/spack/repos/builtin.mock/packages/impossible-concretization/package.py b/var/spack/repos/builtin.mock/packages/impossible-concretization/package.py index c0f2ce8641..b47e97f362 100644 --- a/var/spack/repos/builtin.mock/packages/impossible-concretization/package.py +++ b/var/spack/repos/builtin.mock/packages/impossible-concretization/package.py @@ -16,3 +16,4 @@ class ImpossibleConcretization(Package): version(1.0, "0123456789abcdef0123456789abcdef") conflicts("target=x86_64:") + conflicts("target=aarch64:") diff --git a/var/spack/repos/builtin.mock/packages/openblas/package.py b/var/spack/repos/builtin.mock/packages/openblas/package.py index 5b8637aca2..1e948da8f2 100644 --- a/var/spack/repos/builtin.mock/packages/openblas/package.py +++ b/var/spack/repos/builtin.mock/packages/openblas/package.py @@ -12,6 +12,7 @@ class Openblas(Package): homepage = "http://www.openblas.net" url = "http://github.com/xianyi/OpenBLAS/archive/v0.2.15.tar.gz" + version("0.2.16", "b1190f3d3471685f17cfd1ec1d252ac9") version("0.2.15", "b1190f3d3471685f17cfd1ec1d252ac9") version("0.2.14", "b1190f3d3471685f17cfd1ec1d252ac9") version("0.2.13", "b1190f3d3471685f17cfd1ec1d252ac9") @@ -19,4 +20,7 @@ class Openblas(Package): # See #20019 for this conflict conflicts("%gcc@:4.4", when="@0.2.14:") + # To ensure test works with newer gcc versions + conflicts("%gcc@:10.1", when="@0.2.16:") + provides("blas") -- cgit v1.2.3-70-g09d2