diff options
Diffstat (limited to 'lib')
-rw-r--r-- | lib/spack/docs/tutorial_advanced_packaging.rst | 4 | ||||
-rw-r--r-- | lib/spack/docs/tutorial_packaging.rst | 9 | ||||
-rw-r--r-- | lib/spack/spack/build_environment.py | 2 | ||||
-rw-r--r-- | lib/spack/spack/directory_layout.py | 10 | ||||
-rw-r--r-- | lib/spack/spack/package.py | 86 | ||||
-rw-r--r-- | lib/spack/spack/test/cmd/env.py | 86 | ||||
-rw-r--r-- | lib/spack/spack/test/cmd/install.py | 2 | ||||
-rw-r--r-- | lib/spack/spack/test/directory_layout.py | 23 | ||||
-rw-r--r-- | lib/spack/spack/test/install.py | 98 |
9 files changed, 228 insertions, 92 deletions
diff --git a/lib/spack/docs/tutorial_advanced_packaging.rst b/lib/spack/docs/tutorial_advanced_packaging.rst index e8466002e1..bd90f724a7 100644 --- a/lib/spack/docs/tutorial_advanced_packaging.rst +++ b/lib/spack/docs/tutorial_advanced_packaging.rst @@ -347,7 +347,7 @@ we'll notice that this time the installation won't complete: 11 options.extend([ See build log for details: - /usr/local/var/spack/stage/arpack-ng-3.5.0-bloz7cqirpdxj33pg7uj32zs5likz2un/arpack-ng-3.5.0/spack-build.out + /usr/local/var/spack/stage/arpack-ng-3.5.0-bloz7cqirpdxj33pg7uj32zs5likz2un/arpack-ng-3.5.0/spack-build-out.txt Unlike ``openblas`` which provides a library named ``libopenblas.so``, ``netlib-lapack`` provides ``liblapack.so``, so it needs to implement @@ -459,7 +459,7 @@ Let's look at an example and try to install ``netcdf ^mpich``: 56 config_args.append('--enable-pnetcdf') See build log for details: - /usr/local/var/spack/stage/netcdf-4.4.1.1-gk2xxhbqijnrdwicawawcll4t3c7dvoj/netcdf-4.4.1.1/spack-build.out + /usr/local/var/spack/stage/netcdf-4.4.1.1-gk2xxhbqijnrdwicawawcll4t3c7dvoj/netcdf-4.4.1.1/spack-build-out.txt We can see from the error that ``netcdf`` needs to know how to link the *high-level interface* of ``hdf5``, and thus passes the extra parameter ``hl`` after the request to retrieve it. diff --git a/lib/spack/docs/tutorial_packaging.rst b/lib/spack/docs/tutorial_packaging.rst index e85401da05..f1eed6ef63 100644 --- a/lib/spack/docs/tutorial_packaging.rst +++ b/lib/spack/docs/tutorial_packaging.rst @@ -123,7 +123,7 @@ to build this package: >> 3 make: *** No targets specified and no makefile found. Stop. See build log for details: - /home/ubuntu/packaging/spack/var/spack/stage/mpileaks-1.0-sv75n3u5ev6mljwcezisz3slooozbbxu/mpileaks-1.0/spack-build.out + /home/ubuntu/packaging/spack/var/spack/stage/mpileaks-1.0-sv75n3u5ev6mljwcezisz3slooozbbxu/mpileaks-1.0/spack-build-out.txt This obviously didn't work; we need to fill in the package-specific information. Specifically, Spack didn't try to build any of mpileaks' @@ -256,7 +256,7 @@ Now when we try to install this package a lot more happens: >> 3 make: *** No targets specified and no makefile found. Stop. See build log for details: - /home/ubuntu/packaging/spack/var/spack/stage/mpileaks-1.0-csoikctsalli4cdkkdk377gprkc472rb/mpileaks-1.0/spack-build.out + /home/ubuntu/packaging/spack/var/spack/stage/mpileaks-1.0-csoikctsalli4cdkkdk377gprkc472rb/mpileaks-1.0/spack-build-out.txt Note that this command may take a while to run and produce more output if you don't have an MPI already installed or configured in Spack. @@ -319,14 +319,15 @@ If we re-run we still get errors: >> 31 configure: error: unable to locate adept-utils installation See build log for details: - /home/ubuntu/packaging/spack/var/spack/stage/mpileaks-1.0-csoikctsalli4cdkkdk377gprkc472rb/mpileaks-1.0/spack-build.out + /home/ubuntu/packaging/spack/var/spack/stage/mpileaks-1.0-csoikctsalli4cdkkdk377gprkc472rb/mpileaks-1.0/spack-build-out.txt Again, the problem may be obvious. But let's pretend we're not all intelligent developers and use this opportunity spend some time debugging. We have a few options that can tell us about what's going wrong: -As per the error message, Spack has given us a ``spack-build.out`` debug log: +As per the error message, Spack has given us a ``spack-build-out.txt`` debug +log: .. code-block:: console diff --git a/lib/spack/spack/build_environment.py b/lib/spack/spack/build_environment.py index 1abc9ebeb7..3f4329731a 100644 --- a/lib/spack/spack/build_environment.py +++ b/lib/spack/spack/build_environment.py @@ -993,7 +993,7 @@ class ChildError(InstallError): if self.build_log and os.path.exists(self.build_log): out.write('See build log for details:\n') - out.write(' %s' % self.build_log) + out.write(' %s\n' % self.build_log) return out.getvalue() diff --git a/lib/spack/spack/directory_layout.py b/lib/spack/spack/directory_layout.py index 60bc18a5b8..5ba8ac144c 100644 --- a/lib/spack/spack/directory_layout.py +++ b/lib/spack/spack/directory_layout.py @@ -193,9 +193,7 @@ class YamlDirectoryLayout(DirectoryLayout): self.metadata_dir = '.spack' self.spec_file_name = 'spec.yaml' self.extension_file_name = 'extensions.yaml' - self.build_log_name = 'build.txt' # build log. - self.build_env_name = 'build.env' # build environment - self.packages_dir = 'repos' # archive of package.py files + self.packages_dir = 'repos' # archive of package.py files @property def hidden_file_paths(self): @@ -242,12 +240,6 @@ class YamlDirectoryLayout(DirectoryLayout): def metadata_path(self, spec): return os.path.join(spec.prefix, self.metadata_dir) - def build_log_path(self, spec): - return os.path.join(self.metadata_path(spec), self.build_log_name) - - def build_env_path(self, spec): - return os.path.join(self.metadata_path(spec), self.build_env_name) - def build_packages_path(self, spec): return os.path.join(self.metadata_path(spec), self.packages_dir) diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index b4e3537e92..e24e3fba14 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -63,6 +63,13 @@ from spack.version import Version from spack.package_prefs import get_package_dir_permissions, get_package_group +# Filename for the Spack build/install log. +_spack_build_logfile = 'spack-build-out.txt' + +# Filename for the Spack build/install environment file. +_spack_build_envfile = 'spack-build-env.txt' + + class InstallPhase(object): """Manages a single phase of the installation. @@ -432,8 +439,9 @@ class PackageBase(with_metaclass(PackageMeta, PackageViewMixin, object)): #: List of glob expressions. Each expression must either be #: absolute or relative to the package source path. - #: Matching artifacts found at the end of the build process will - #: be copied in the same directory tree as build.env and build.txt. + #: Matching artifacts found at the end of the build process will be + #: copied in the same directory tree as _spack_build_logfile and + #: _spack_build_envfile. archive_files = [] # @@ -782,11 +790,55 @@ class PackageBase(with_metaclass(PackageMeta, PackageViewMixin, object)): @property def env_path(self): - return os.path.join(self.stage.path, 'spack-build.env') + """Return the build environment file path associated with staging.""" + # Backward compatibility: Return the name of an existing log path; + # otherwise, return the current install env path name. + old_filename = os.path.join(self.stage.path, 'spack-build.env') + if os.path.exists(old_filename): + return old_filename + else: + return os.path.join(self.stage.path, _spack_build_envfile) + + @property + def install_env_path(self): + """ + Return the build environment file path on successful installation. + """ + install_path = spack.store.layout.metadata_path(self.spec) + + # Backward compatibility: Return the name of an existing log path; + # otherwise, return the current install env path name. + old_filename = os.path.join(install_path, 'build.env') + if os.path.exists(old_filename): + return old_filename + else: + return os.path.join(install_path, _spack_build_envfile) @property def log_path(self): - return os.path.join(self.stage.path, 'spack-build.txt') + """Return the build log file path associated with staging.""" + # Backward compatibility: Return the name of an existing log path. + for filename in ['spack-build.out', 'spack-build.txt']: + old_log = os.path.join(self.stage.path, filename) + if os.path.exists(old_log): + return old_log + + # Otherwise, return the current log path name. + return os.path.join(self.stage.path, _spack_build_logfile) + + @property + def install_log_path(self): + """Return the build log file path on successful installation.""" + install_path = spack.store.layout.metadata_path(self.spec) + + # Backward compatibility: Return the name of an existing install log. + for filename in ['build.out', 'build.txt']: + old_log = os.path.join(install_path, filename) + if os.path.exists(old_log): + return old_log + + # Otherwise, return the current install log path name. + return os.path.join(install_path, _spack_build_logfile) def _make_fetcher(self): # Construct a composite fetcher that always contains at least @@ -1394,7 +1446,6 @@ class PackageBase(with_metaclass(PackageMeta, PackageViewMixin, object)): ) def do_install(self, **kwargs): - """Called by commands to install a package and its dependencies. Package implementations should override install() to describe @@ -1617,6 +1668,9 @@ class PackageBase(with_metaclass(PackageMeta, PackageViewMixin, object)): if mode != perms: os.chmod(self.prefix, perms) + # Ensure the metadata path exists as well + mkdirp(spack.store.layout.metadata_path(self.spec), mode=perms) + # Fork a child to do the actual installation # we preserve verbosity settings across installs. PackageBase._verbose = spack.build_environment.fork( @@ -1724,24 +1778,24 @@ class PackageBase(with_metaclass(PackageMeta, PackageViewMixin, object)): .format(self.last_phase, self.name)) def log(self): - # Copy provenance into the install directory on success - log_install_path = spack.store.layout.build_log_path(self.spec) - env_install_path = spack.store.layout.build_env_path(self.spec) + """Copy provenance into the install directory on success.""" packages_dir = spack.store.layout.build_packages_path(self.spec) # Remove first if we're overwriting another build # (can happen with spack setup) try: - # log_install_path and env_install_path are inside this + # log and env install paths are inside this shutil.rmtree(packages_dir) except Exception as e: # FIXME : this potentially catches too many things... tty.debug(e) # Archive the whole stdout + stderr for the package - install(self.log_path, log_install_path) + install(self.log_path, self.install_log_path) + # Archive the environment used for the build - install(self.env_path, env_install_path) + install(self.env_path, self.install_env_path) + # Finally, archive files that are specific to each package with working_dir(self.stage.path): errors = StringIO() @@ -1816,10 +1870,12 @@ class PackageBase(with_metaclass(PackageMeta, PackageViewMixin, object)): @property def build_log_path(self): - if self.installed: - return spack.store.layout.build_log_path(self.spec) - else: - return self.log_path + """ + Return the expected (or current) build log file path. The path points + to the staging build file until the software is successfully installed, + when it points to the file in the installation directory. + """ + return self.install_log_path if self.installed else self.log_path @classmethod def inject_flags(cls, name, flags): diff --git a/lib/spack/spack/test/cmd/env.py b/lib/spack/spack/test/cmd/env.py index 0da9377196..e8aa38375e 100644 --- a/lib/spack/spack/test/cmd/env.py +++ b/lib/spack/spack/test/cmd/env.py @@ -31,6 +31,19 @@ uninstall = SpackCommand('uninstall') find = SpackCommand('find') +def check_mpileaks_install(viewdir): + """Check that the expected install directories exist.""" + assert os.path.exists(str(viewdir.join('.spack', 'mpileaks'))) + # Check that dependencies got in too + assert os.path.exists(str(viewdir.join('.spack', 'libdwarf'))) + + +def check_viewdir_removal(viewdir): + """Check that the uninstall/removal worked.""" + assert (not os.path.exists(str(viewdir.join('.spack'))) or + os.listdir(str(viewdir.join('.spack'))) == ['projections.yaml']) + + def test_add(): e = ev.create('test') e.add('mpileaks') @@ -606,22 +619,18 @@ def test_uninstall_removes_from_env(mock_stage, mock_fetch, install_mockery): def test_env_updates_view_install( - tmpdir, mock_stage, mock_fetch, install_mockery -): + tmpdir, mock_stage, mock_fetch, install_mockery): view_dir = tmpdir.mkdir('view') env('create', '--with-view=%s' % view_dir, 'test') with ev.read('test'): add('mpileaks') install('--fake') - assert os.path.exists(str(view_dir.join('.spack/mpileaks'))) - # Check that dependencies got in too - assert os.path.exists(str(view_dir.join('.spack/libdwarf'))) + check_mpileaks_install(view_dir) def test_env_without_view_install( - tmpdir, mock_stage, mock_fetch, install_mockery -): + tmpdir, mock_stage, mock_fetch, install_mockery): # Test enabling a view after installing specs env('create', '--without-view', 'test') @@ -639,13 +648,11 @@ def test_env_without_view_install( # After enabling the view, the specs should be linked into the environment # view dir - assert os.path.exists(str(view_dir.join('.spack/mpileaks'))) - assert os.path.exists(str(view_dir.join('.spack/libdwarf'))) + check_mpileaks_install(view_dir) def test_env_config_view_default( - tmpdir, mock_stage, mock_fetch, install_mockery -): + tmpdir, mock_stage, mock_fetch, install_mockery): # This config doesn't mention whether a view is enabled test_config = """\ env: @@ -665,21 +672,17 @@ env: def test_env_updates_view_install_package( - tmpdir, mock_stage, mock_fetch, install_mockery -): + tmpdir, mock_stage, mock_fetch, install_mockery): view_dir = tmpdir.mkdir('view') env('create', '--with-view=%s' % view_dir, 'test') with ev.read('test'): install('--fake', 'mpileaks') - assert os.path.exists(str(view_dir.join('.spack/mpileaks'))) - # Check that dependencies got in too - assert os.path.exists(str(view_dir.join('.spack/libdwarf'))) + check_mpileaks_install(view_dir) def test_env_updates_view_add_concretize( - tmpdir, mock_stage, mock_fetch, install_mockery -): + tmpdir, mock_stage, mock_fetch, install_mockery): view_dir = tmpdir.mkdir('view') env('create', '--with-view=%s' % view_dir, 'test') install('--fake', 'mpileaks') @@ -687,33 +690,26 @@ def test_env_updates_view_add_concretize( add('mpileaks') concretize() - assert os.path.exists(str(view_dir.join('.spack/mpileaks'))) - # Check that dependencies got in too - assert os.path.exists(str(view_dir.join('.spack/libdwarf'))) + check_mpileaks_install(view_dir) def test_env_updates_view_uninstall( - tmpdir, mock_stage, mock_fetch, install_mockery -): + tmpdir, mock_stage, mock_fetch, install_mockery): view_dir = tmpdir.mkdir('view') env('create', '--with-view=%s' % view_dir, 'test') with ev.read('test'): install('--fake', 'mpileaks') - assert os.path.exists(str(view_dir.join('.spack/mpileaks'))) - # Check that dependencies got in too - assert os.path.exists(str(view_dir.join('.spack/libdwarf'))) + check_mpileaks_install(view_dir) with ev.read('test'): uninstall('-ay') - assert (not os.path.exists(str(view_dir.join('.spack'))) or - os.listdir(str(view_dir.join('.spack'))) == ['projections.yaml']) + check_viewdir_removal(view_dir) def test_env_updates_view_uninstall_referenced_elsewhere( - tmpdir, mock_stage, mock_fetch, install_mockery -): + tmpdir, mock_stage, mock_fetch, install_mockery): view_dir = tmpdir.mkdir('view') env('create', '--with-view=%s' % view_dir, 'test') install('--fake', 'mpileaks') @@ -721,20 +717,16 @@ def test_env_updates_view_uninstall_referenced_elsewhere( add('mpileaks') concretize() - assert os.path.exists(str(view_dir.join('.spack/mpileaks'))) - # Check that dependencies got in too - assert os.path.exists(str(view_dir.join('.spack/libdwarf'))) + check_mpileaks_install(view_dir) with ev.read('test'): uninstall('-ay') - assert (not os.path.exists(str(view_dir.join('.spack'))) or - os.listdir(str(view_dir.join('.spack'))) == ['projections.yaml']) + check_viewdir_removal(view_dir) def test_env_updates_view_remove_concretize( - tmpdir, mock_stage, mock_fetch, install_mockery -): + tmpdir, mock_stage, mock_fetch, install_mockery): view_dir = tmpdir.mkdir('view') env('create', '--with-view=%s' % view_dir, 'test') install('--fake', 'mpileaks') @@ -742,40 +734,32 @@ def test_env_updates_view_remove_concretize( add('mpileaks') concretize() - assert os.path.exists(str(view_dir.join('.spack/mpileaks'))) - # Check that dependencies got in too - assert os.path.exists(str(view_dir.join('.spack/libdwarf'))) + check_mpileaks_install(view_dir) with ev.read('test'): remove('mpileaks') concretize() - assert (not os.path.exists(str(view_dir.join('.spack'))) or - os.listdir(str(view_dir.join('.spack'))) == ['projections.yaml']) + check_viewdir_removal(view_dir) def test_env_updates_view_force_remove( - tmpdir, mock_stage, mock_fetch, install_mockery -): + tmpdir, mock_stage, mock_fetch, install_mockery): view_dir = tmpdir.mkdir('view') env('create', '--with-view=%s' % view_dir, 'test') with ev.read('test'): install('--fake', 'mpileaks') - assert os.path.exists(str(view_dir.join('.spack/mpileaks'))) - # Check that dependencies got in too - assert os.path.exists(str(view_dir.join('.spack/libdwarf'))) + check_mpileaks_install(view_dir) with ev.read('test'): remove('-f', 'mpileaks') - assert (not os.path.exists(str(view_dir.join('.spack'))) or - os.listdir(str(view_dir.join('.spack'))) == ['projections.yaml']) + check_viewdir_removal(view_dir) def test_env_activate_view_fails( - tmpdir, mock_stage, mock_fetch, install_mockery -): + tmpdir, mock_stage, mock_fetch, install_mockery): """Sanity check on env activate to make sure it requires shell support""" out = env('activate', 'test') assert "To initialize spack's shell commands:" in out diff --git a/lib/spack/spack/test/cmd/install.py b/lib/spack/spack/test/cmd/install.py index 3cc6f4a0bf..05971da904 100644 --- a/lib/spack/spack/test/cmd/install.py +++ b/lib/spack/spack/test/cmd/install.py @@ -125,7 +125,7 @@ def test_package_output(tmpdir, capsys, install_mockery, mock_fetch): pkg = spec.package pkg.do_install(verbose=True) - log_file = os.path.join(spec.prefix, '.spack', 'build.txt') + log_file = pkg.build_log_path with open(log_file) as f: out = f.read() diff --git a/lib/spack/spack/test/directory_layout.py b/lib/spack/spack/test/directory_layout.py index 10e77143a7..619b617bab 100644 --- a/lib/spack/spack/test/directory_layout.py +++ b/lib/spack/spack/test/directory_layout.py @@ -29,9 +29,7 @@ def layout_and_dir(tmpdir): spack.store.layout = old_layout -def test_yaml_directory_layout_parameters( - tmpdir, config -): +def test_yaml_directory_layout_parameters(tmpdir, config): """This tests the various parameters that can be used to configure the install location """ spec = Spec('python') @@ -84,9 +82,7 @@ def test_yaml_directory_layout_parameters( path_scheme=scheme_package7) -def test_read_and_write_spec( - layout_and_dir, config, mock_packages -): +def test_read_and_write_spec(layout_and_dir, config, mock_packages): """This goes through each package in spack and creates a directory for it. It then ensures that the spec for the directory's installed package can be read back in consistently, and @@ -162,9 +158,7 @@ def test_read_and_write_spec( assert not os.path.exists(install_dir) -def test_handle_unknown_package( - layout_and_dir, config, mock_packages -): +def test_handle_unknown_package(layout_and_dir, config, mock_packages): """This test ensures that spack can at least do *some* operations with packages that are installed but that it does not know about. This is actually not such an uncommon @@ -234,3 +228,14 @@ def test_find(layout_and_dir, config, mock_packages): for name, spec in found_specs.items(): assert name in found_specs assert found_specs[name].eq_dag(spec) + + +def test_yaml_directory_layout_build_path(tmpdir, config): + """This tests build path method.""" + spec = Spec('python') + spec.concretize() + + layout = YamlDirectoryLayout(str(tmpdir)) + rel_path = os.path.join(layout.metadata_dir, layout.packages_dir) + assert layout.build_packages_path(spec) == os.path.join(spec.prefix, + rel_path) diff --git a/lib/spack/spack/test/install.py b/lib/spack/spack/test/install.py index 8e1d5c7940..ec70bdec42 100644 --- a/lib/spack/spack/test/install.py +++ b/lib/spack/spack/test/install.py @@ -5,11 +5,15 @@ import os import pytest +import shutil + +from llnl.util.filesystem import mkdirp, touch, working_dir import spack.patch import spack.repo import spack.store from spack.spec import Spec +from spack.package import _spack_build_envfile, _spack_build_logfile def test_install_and_uninstall(install_mockery, mock_fetch, monkeypatch): @@ -269,3 +273,97 @@ def test_failing_build(install_mockery, mock_fetch): class MockInstallError(spack.error.SpackError): pass + + +def test_pkg_build_paths(install_mockery): + # Get a basic concrete spec for the trivial install package. + spec = Spec('trivial-install-test-package').concretized() + + log_path = spec.package.log_path + assert log_path.endswith(_spack_build_logfile) + + env_path = spec.package.env_path + assert env_path.endswith(_spack_build_envfile) + + # Backward compatibility checks + log_dir = os.path.dirname(log_path) + mkdirp(log_dir) + with working_dir(log_dir): + # Start with the older of the previous log filenames + older_log = 'spack-build.out' + touch(older_log) + assert spec.package.log_path.endswith(older_log) + + # Now check the newer log filename + last_log = 'spack-build.txt' + os.rename(older_log, last_log) + assert spec.package.log_path.endswith(last_log) + + # Check the old environment file + last_env = 'spack-build.env' + os.rename(last_log, last_env) + assert spec.package.env_path.endswith(last_env) + + # Cleanup + shutil.rmtree(log_dir) + + +def test_pkg_install_paths(install_mockery): + # Get a basic concrete spec for the trivial install package. + spec = Spec('trivial-install-test-package').concretized() + + log_path = os.path.join(spec.prefix, '.spack', _spack_build_logfile) + assert spec.package.install_log_path == log_path + + env_path = os.path.join(spec.prefix, '.spack', _spack_build_envfile) + assert spec.package.install_env_path == env_path + + # Backward compatibility checks + log_dir = os.path.dirname(log_path) + mkdirp(log_dir) + with working_dir(log_dir): + # Start with the older of the previous install log filenames + older_log = 'build.out' + touch(older_log) + assert spec.package.install_log_path.endswith(older_log) + + # Now check the newer install log filename + last_log = 'build.txt' + os.rename(older_log, last_log) + assert spec.package.install_log_path.endswith(last_log) + + # Check the old install environment file + last_env = 'build.env' + os.rename(last_log, last_env) + assert spec.package.install_env_path.endswith(last_env) + + # Cleanup + shutil.rmtree(log_dir) + + +def test_pkg_install_log(install_mockery): + # Get a basic concrete spec for the trivial install package. + spec = Spec('trivial-install-test-package').concretized() + + # Attempt installing log without the build log file + with pytest.raises(IOError, match="No such file or directory"): + spec.package.log() + + # Set up mock build files and try again + log_path = spec.package.log_path + log_dir = os.path.dirname(log_path) + mkdirp(log_dir) + with working_dir(log_dir): + touch(log_path) + touch(spec.package.env_path) + + install_path = os.path.dirname(spec.package.install_log_path) + mkdirp(install_path) + + spec.package.log() + + assert os.path.exists(spec.package.install_log_path) + assert os.path.exists(spec.package.install_env_path) + + # Cleanup + shutil.rmtree(log_dir) |