summaryrefslogtreecommitdiff
path: root/lib
diff options
context:
space:
mode:
Diffstat (limited to 'lib')
-rw-r--r--lib/spack/docs/packaging_guide.rst38
-rw-r--r--lib/spack/llnl/util/filesystem.py9
-rw-r--r--lib/spack/spack/build_environment.py7
-rw-r--r--lib/spack/spack/directory_layout.py56
-rw-r--r--lib/spack/spack/fetch_strategy.py7
-rw-r--r--lib/spack/spack/mirror.py13
-rw-r--r--lib/spack/spack/package.py115
-rw-r--r--lib/spack/spack/stage.py48
8 files changed, 193 insertions, 100 deletions
diff --git a/lib/spack/docs/packaging_guide.rst b/lib/spack/docs/packaging_guide.rst
index 07a17ebd0a..b7e0b6a4f3 100644
--- a/lib/spack/docs/packaging_guide.rst
+++ b/lib/spack/docs/packaging_guide.rst
@@ -401,6 +401,35 @@ construct the new one for ``8.2.1``.
When you supply a custom URL for a version, Spack uses that URL
*verbatim* and does not perform extrapolation.
+Skipping the expand step
+~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Spack normally expands archives automatically after downloading
+them. If you want to skip this step (e.g., for self-extracting
+executables and other custom archive types), you can add
+``expand=False`` to a ``version`` directive.
+
+.. code-block:: python
+
+ version('8.2.1', '4136d7b4c04df68b686570afa26988ac',
+ url='http://example.com/foo-8.2.1-special-version.tar.gz', 'expand=False')
+
+When ``expand`` is set to ``False``, Spack sets the current working
+directory to the directory containing the downloaded archive before it
+calls your ``install`` method. Within ``install``, the path to the
+downloaded archive is available as ``self.stage.archive_file``.
+
+Here is an example snippet for packages distribuetd as self-extracting
+archives. The example sets permissions on the downloaded file to make
+it executable, then runs it with some arguments.
+
+.. code-block:: python
+
+ def install(self, spec, prefix):
+ set_executable(self.stage.archive_file)
+ installer = Executable(self.stage.archive_file)
+ installer('--prefix=%s' % prefix, 'arg1', 'arg2', 'etc.')
+
Checksums
~~~~~~~~~~~~~~~~~
@@ -2108,6 +2137,15 @@ Filtering functions
Examples:
+ #. Filtering a Makefile to force it to use Spack's compiler wrappers:
+
+ .. code-block:: python
+
+ filter_file(r'^CC\s*=.*', spack_cc, 'Makefile')
+ filter_file(r'^CXX\s*=.*', spack_cxx, 'Makefile')
+ filter_file(r'^F77\s*=.*', spack_f77, 'Makefile')
+ filter_file(r'^FC\s*=.*', spack_fc, 'Makefile')
+
#. Replacing ``#!/usr/bin/perl`` with ``#!/usr/bin/env perl`` in ``bib2xhtml``:
.. code-block:: python
diff --git a/lib/spack/llnl/util/filesystem.py b/lib/spack/llnl/util/filesystem.py
index 9ba662d0e3..c4665c284c 100644
--- a/lib/spack/llnl/util/filesystem.py
+++ b/lib/spack/llnl/util/filesystem.py
@@ -26,7 +26,8 @@ __all__ = ['set_install_permissions', 'install', 'install_tree', 'traverse_tree'
'expand_user', 'working_dir', 'touch', 'touchp', 'mkdirp',
'force_remove', 'join_path', 'ancestor', 'can_access', 'filter_file',
'FileFilter', 'change_sed_delimiter', 'is_exe', 'force_symlink',
- 'remove_dead_links', 'remove_linked_tree', 'copy_mode', 'unset_executable_mode']
+ 'set_executable', 'copy_mode', 'unset_executable_mode',
+ 'remove_dead_links', 'remove_linked_tree']
import os
import sys
@@ -354,6 +355,12 @@ def traverse_tree(source_root, dest_root, rel_path='', **kwargs):
if order == 'post':
yield (source_path, dest_path)
+
+def set_executable(path):
+ st = os.stat(path)
+ os.chmod(path, st.st_mode | stat.S_IEXEC)
+
+
def remove_dead_links(root):
"""
Removes any dead link that is present in root
diff --git a/lib/spack/spack/build_environment.py b/lib/spack/spack/build_environment.py
index 1b87778080..e22597a789 100644
--- a/lib/spack/spack/build_environment.py
+++ b/lib/spack/spack/build_environment.py
@@ -214,6 +214,13 @@ def set_module_variables_for_package(pkg, m):
m.std_cmake_args.append('-DCMAKE_INSTALL_RPATH_USE_LINK_PATH=FALSE')
m.std_cmake_args.append('-DCMAKE_INSTALL_RPATH=%s' % ":".join(get_rpaths(pkg)))
+ # Put spack compiler paths in module scope.
+ link_dir = spack.build_env_path
+ m.spack_cc = join_path(link_dir, pkg.compiler.link_paths['cc'])
+ m.spack_cxx = join_path(link_dir, pkg.compiler.link_paths['cxx'])
+ m.spack_f77 = join_path(link_dir, pkg.compiler.link_paths['f77'])
+ m.spack_f90 = join_path(link_dir, pkg.compiler.link_paths['fc'])
+
# Emulate some shell commands for convenience
m.pwd = os.getcwd
m.cd = os.chdir
diff --git a/lib/spack/spack/directory_layout.py b/lib/spack/spack/directory_layout.py
index 08c23627f4..242eb1afa0 100644
--- a/lib/spack/spack/directory_layout.py
+++ b/lib/spack/spack/directory_layout.py
@@ -85,6 +85,16 @@ class DirectoryLayout(object):
raise NotImplementedError()
+ def check_installed(self, spec):
+ """Checks whether a spec is installed.
+
+ Return the spec's prefix, if it is installed, None otherwise.
+
+ Raise an exception if the install is inconsistent or corrupt.
+ """
+ raise NotImplementedError()
+
+
def extension_map(self, spec):
"""Get a dict of currently installed extension packages for a spec.
@@ -246,26 +256,36 @@ class YamlDirectoryLayout(DirectoryLayout):
def create_install_directory(self, spec):
_check_concrete(spec)
+ prefix = self.check_installed(spec)
+ if prefix:
+ raise InstallDirectoryAlreadyExistsError(prefix)
+
+ mkdirp(self.metadata_path(spec))
+ self.write_spec(spec, self.spec_file_path(spec))
+
+
+ def check_installed(self, spec):
+ _check_concrete(spec)
path = self.path_for_spec(spec)
spec_file_path = self.spec_file_path(spec)
- if os.path.isdir(path):
- if not os.path.isfile(spec_file_path):
- raise InconsistentInstallDirectoryError(
- 'No spec file found at path %s' % spec_file_path)
+ if not os.path.isdir(path):
+ return None
- installed_spec = self.read_spec(spec_file_path)
- if installed_spec == self.spec:
- raise InstallDirectoryAlreadyExistsError(path)
+ if not os.path.isfile(spec_file_path):
+ raise InconsistentInstallDirectoryError(
+ 'Inconsistent state: install prefix exists but contains no spec.yaml:',
+ " " + path)
- if spec.dag_hash() == installed_spec.dag_hash():
- raise SpecHashCollisionError(installed_hash, spec_hash)
- else:
- raise InconsistentInstallDirectoryError(
- 'Spec file in %s does not match hash!' % spec_file_path)
+ installed_spec = self.read_spec(spec_file_path)
+ if installed_spec == spec:
+ return path
- mkdirp(self.metadata_path(spec))
- self.write_spec(spec, spec_file_path)
+ if spec.dag_hash() == installed_spec.dag_hash():
+ raise SpecHashCollisionError(installed_hash, spec_hash)
+ else:
+ raise InconsistentInstallDirectoryError(
+ 'Spec file in %s does not match hash!' % spec_file_path)
def all_specs(self):
@@ -399,8 +419,8 @@ class YamlDirectoryLayout(DirectoryLayout):
class DirectoryLayoutError(SpackError):
"""Superclass for directory layout errors."""
- def __init__(self, message):
- super(DirectoryLayoutError, self).__init__(message)
+ def __init__(self, message, long_msg=None):
+ super(DirectoryLayoutError, self).__init__(message, long_msg)
class SpecHashCollisionError(DirectoryLayoutError):
@@ -422,8 +442,8 @@ class RemoveFailedError(DirectoryLayoutError):
class InconsistentInstallDirectoryError(DirectoryLayoutError):
"""Raised when a package seems to be installed to the wrong place."""
- def __init__(self, message):
- super(InconsistentInstallDirectoryError, self).__init__(message)
+ def __init__(self, message, long_msg=None):
+ super(InconsistentInstallDirectoryError, self).__init__(message, long_msg)
class InstallDirectoryAlreadyExistsError(DirectoryLayoutError):
diff --git a/lib/spack/spack/fetch_strategy.py b/lib/spack/spack/fetch_strategy.py
index ec17cb97f1..0d0a7db8a9 100644
--- a/lib/spack/spack/fetch_strategy.py
+++ b/lib/spack/spack/fetch_strategy.py
@@ -82,7 +82,6 @@ class FetchStrategy(object):
class __metaclass__(type):
"""This metaclass registers all fetch strategies in a list."""
-
def __init__(cls, name, bases, dict):
type.__init__(cls, name, bases, dict)
if cls.enabled: all_strategies.append(cls)
@@ -145,6 +144,8 @@ class URLFetchStrategy(FetchStrategy):
self.digest = kwargs.get('md5', None)
if not self.digest: self.digest = digest
+ self.expand_archive = kwargs.get('expand', True)
+
if not self.url:
raise ValueError("URLFetchStrategy requires a url for fetching.")
@@ -218,6 +219,10 @@ class URLFetchStrategy(FetchStrategy):
@_needs_stage
def expand(self):
+ if not self.expand_archive:
+ tty.msg("Skipping expand step for %s" % self.archive_file)
+ return
+
tty.msg("Staging archive: %s" % self.archive_file)
self.stage.chdir()
diff --git a/lib/spack/spack/mirror.py b/lib/spack/spack/mirror.py
index fdc4e7967f..6981f69ac0 100644
--- a/lib/spack/spack/mirror.py
+++ b/lib/spack/spack/mirror.py
@@ -51,13 +51,20 @@ def mirror_archive_filename(spec, fetcher):
raise ValueError("mirror.path requires spec with concrete version.")
if isinstance(fetcher, fs.URLFetchStrategy):
- # If we fetch this version with a URLFetchStrategy, use URL's archive type
- ext = url.downloaded_file_extension(fetcher.url)
+ if fetcher.expand_archive:
+ # If we fetch this version with a URLFetchStrategy, use URL's archive type
+ ext = url.downloaded_file_extension(fetcher.url)
+ else:
+ # If the archive shouldn't be expanded, don't check for its extension.
+ ext = None
else:
# Otherwise we'll make a .tar.gz ourselves
ext = 'tar.gz'
- return "%s-%s.%s" % (spec.package.name, spec.version, ext)
+ filename = "%s-%s" % (spec.package.name, spec.version)
+ if ext:
+ filename += ".%s" % ext
+ return filename
def mirror_archive_path(spec, fetcher):
diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py
index d4acbf5024..972a0410b9 100644
--- a/lib/spack/spack/package.py
+++ b/lib/spack/spack/package.py
@@ -845,7 +845,8 @@ class Package(object):
if not self.spec.concrete:
raise ValueError("Can only install concrete packages.")
- if os.path.exists(self.prefix):
+ # Ensure package is not already installed
+ if spack.install_layout.check_installed(self.spec):
tty.msg("%s is already installed in %s" % (self.name, self.prefix))
return
@@ -857,18 +858,11 @@ class Package(object):
keep_prefix=keep_prefix, keep_stage=keep_stage, ignore_deps=ignore_deps,
fake=fake, skip_patch=skip_patch, verbose=verbose, make_jobs=make_jobs)
- def cleanup():
- """Handles removing install prefix on error."""
- if not keep_prefix:
- self.remove_prefix()
- else:
- tty.warn("Keeping install prefix in place despite error.",
- "Spack will think this package is installed. " +
- "Manually remove this directory to fix:",
- self.prefix, wrap=True)
+ # Set parallelism before starting build.
+ self.make_jobs = make_jobs
# Then install the package itself.
- def real_work():
+ def build_process():
"""Forked for each build. Has its own process and python
module space set up by build_environment.fork()."""
start_time = time.time()
@@ -878,30 +872,24 @@ class Package(object):
else:
self.do_stage()
- # create the install directory. The install layout
- # handles this in case so that it can use whatever
- # package naming scheme it likes.
- spack.install_layout.create_install_directory(self.spec)
-
- try:
- tty.msg("Building %s" % self.name)
+ tty.msg("Building %s" % self.name)
- self.stage.keep = keep_stage
- with self.stage:
- # Run the pre-install hook in the child process after
- # the directory is created.
- spack.hooks.pre_install(self)
+ self.stage.keep = keep_stage
+ with self.stage:
+ # Run the pre-install hook in the child process after
+ # the directory is created.
+ spack.hooks.pre_install(self)
- if fake:
- self.do_fake_install()
-
- else:
- # Do the real install in the source directory.
- self.stage.chdir_to_source()
+ if fake:
+ self.do_fake_install()
+ else:
+ # Do the real install in the source directory.
+ self.stage.chdir_to_source()
- # Save the build environment in a file before building.
- env_path = join_path(os.getcwd(), 'spack-build.env')
+ # Save the build environment in a file before building.
+ env_path = join_path(os.getcwd(), 'spack-build.env')
+ try:
# Redirect I/O to a build log (and optionally to the terminal)
log_path = join_path(os.getcwd(), 'spack-build.out')
log_file = open(log_path, 'w')
@@ -909,43 +897,46 @@ class Package(object):
dump_environment(env_path)
self.install(self.spec, self.prefix)
- # Ensure that something was actually installed.
- self._sanity_check_install()
+ except ProcessError as e:
+ # Annotate ProcessErrors with the location of the build log.
+ e.build_log = log_path
+ raise e
- # Copy provenance into the install directory on success
- log_install_path = spack.install_layout.build_log_path(self.spec)
- env_install_path = spack.install_layout.build_env_path(self.spec)
- packages_dir = spack.install_layout.build_packages_path(self.spec)
+ # Ensure that something was actually installed.
+ self._sanity_check_install()
- install(log_path, log_install_path)
- install(env_path, env_install_path)
- dump_packages(self.spec, packages_dir)
+ # Copy provenance into the install directory on success
+ log_install_path = spack.install_layout.build_log_path(self.spec)
+ env_install_path = spack.install_layout.build_env_path(self.spec)
+ packages_dir = spack.install_layout.build_packages_path(self.spec)
- # Stop timer.
- self._total_time = time.time() - start_time
- build_time = self._total_time - self._fetch_time
+ install(log_path, log_install_path)
+ install(env_path, env_install_path)
+ dump_packages(self.spec, packages_dir)
- tty.msg("Successfully installed %s" % self.name,
- "Fetch: %s. Build: %s. Total: %s."
- % (_hms(self._fetch_time), _hms(build_time), _hms(self._total_time)))
- print_pkg(self.prefix)
+ # Stop timer.
+ self._total_time = time.time() - start_time
+ build_time = self._total_time - self._fetch_time
- except ProcessError as e:
- # Annotate with location of build log.
- e.build_log = log_path
- cleanup()
- raise e
+ tty.msg("Successfully installed %s" % self.name,
+ "Fetch: %s. Build: %s. Total: %s."
+ % (_hms(self._fetch_time), _hms(build_time), _hms(self._total_time)))
+ print_pkg(self.prefix)
- except:
- # other exceptions just clean up and raise.
- cleanup()
- raise
-
- # Set parallelism before starting build.
- self.make_jobs = make_jobs
-
- # Do the build.
- spack.build_environment.fork(self, real_work)
+ try:
+ # Create the install prefix and fork the build process.
+ spack.install_layout.create_install_directory(self.spec)
+ spack.build_environment.fork(self, build_process)
+ except:
+ # remove the install prefix if anything went wrong during install.
+ if not keep_prefix:
+ self.remove_prefix()
+ else:
+ tty.warn("Keeping install prefix in place despite error.",
+ "Spack will think this package is installed. " +
+ "Manually remove this directory to fix:",
+ self.prefix, wrap=True)
+ raise
# note: PARENT of the build process adds the new package to
# the database, so that we don't need to re-read from file.
diff --git a/lib/spack/spack/stage.py b/lib/spack/spack/stage.py
index 9404e12e84..0d35511c34 100644
--- a/lib/spack/spack/stage.py
+++ b/lib/spack/spack/stage.py
@@ -229,13 +229,22 @@ class Stage(object):
@property
def source_path(self):
- """Returns the path to the expanded/checked out source code
- within this fetch strategy's path.
+ """Returns the path to the expanded/checked out source code.
- This assumes nothing else is going ot be put in the
- FetchStrategy's path. It searches for the first
- subdirectory of the path it can find, then returns that.
+ To find the source code, this method searches for the first
+ subdirectory of the stage that it can find, and returns it.
+ This assumes nothing besides the archive file will be in the
+ stage path, but it has the advantage that we don't need to
+ know the name of the archive or its contents.
+
+ If the fetch strategy is not supposed to expand the downloaded
+ file, it will just return the stage path. If the archive needs
+ to be expanded, it will return None when no archive is found.
"""
+ if isinstance(self.fetcher, fs.URLFetchStrategy):
+ if not self.fetcher.expand_archive:
+ return self.path
+
for p in [os.path.join(self.path, f) for f in os.listdir(self.path)]:
if os.path.isdir(p):
return p
@@ -416,21 +425,15 @@ class ResourceStage(Stage):
shutil.move(source_path, destination_path)
-@pattern.composite(method_list=['fetch', 'create', 'check', 'expand_archive', 'restage', 'destroy'])
+@pattern.composite(method_list=['fetch', 'create', 'check', 'expand_archive', 'restage', 'destroy'])
class StageComposite:
"""
Composite for Stage type objects. The first item in this composite is considered to be the root package, and
operations that return a value are forwarded to it.
"""
-
- @property
- def source_path(self):
- return self[0].source_path
-
- @property
- def path(self):
- return self[0].path
-
+ #
+ # __enter__ and __exit__ delegate to all stages in the composite.
+ #
def __enter__(self):
for item in self:
item.__enter__()
@@ -441,9 +444,24 @@ class StageComposite:
item.delete_on_exit = getattr(self, 'delete_on_exit', True)
item.__exit__(exc_type, exc_val, exc_tb)
+ #
+ # Below functions act only on the *first* stage in the composite.
+ #
+ @property
+ def source_path(self):
+ return self[0].source_path
+
+ @property
+ def path(self):
+ return self[0].path
+
def chdir_to_source(self):
return self[0].chdir_to_source()
+ @property
+ def archive_file(self):
+ return self[0].archive_file
+
class DIYStage(object):
"""Simple class that allows any directory to be a spack stage."""
generated by cgit v1.2.3-70-g09d2 (git 2.46.0) at 2024-08-26 12:58:49 +0000