From 8d8cf6201ba6ed3d6c852392af1861a61c8e81bf Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Sun, 26 Jul 2020 22:41:55 -0700 Subject: bugfix: don't redundantly print ChildErrors (#17709) A bug was introduced in #13100 where ChildErrors would be redundantly printed when raised during a build. We should eventually revisit error handling in builds and figure out what the right separation of responsibilities is for distributed builds, but for now just skip printing. - [x] SpackErrors were designed to be printed by the forked process, not by the parent, so check if they've already been printed. - [x] update tests --- lib/spack/spack/installer.py | 11 ++++++++--- lib/spack/spack/test/cmd/install.py | 6 ++++-- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/lib/spack/spack/installer.py b/lib/spack/spack/installer.py index d5d15f4f99..2d4b488ac3 100644 --- a/lib/spack/spack/installer.py +++ b/lib/spack/spack/installer.py @@ -1568,9 +1568,14 @@ class PackageInstaller(object): except (Exception, SystemExit) as exc: # Best effort installs suppress the exception and mark the # package as a failure UNLESS this is the explicit package. - err = 'Failed to install {0} due to {1}: {2}' - tty.error(err.format(pkg.name, exc.__class__.__name__, - str(exc))) + if (not isinstance(exc, spack.error.SpackError) or + not exc.printed): + # SpackErrors can be printed by the build process or at + # lower levels -- skip printing if already printed. + # TODO: sort out this and SpackEror.print_context() + err = 'Failed to install {0} due to {1}: {2}' + tty.error( + err.format(pkg.name, exc.__class__.__name__, str(exc))) self._update_failed(task, True, exc) diff --git a/lib/spack/spack/test/cmd/install.py b/lib/spack/spack/test/cmd/install.py index 9a46475c18..9eb2338649 100644 --- a/lib/spack/spack/test/cmd/install.py +++ b/lib/spack/spack/test/cmd/install.py @@ -180,10 +180,12 @@ def test_show_log_on_error(mock_packages, mock_archive, mock_fetch, assert install.error.pkg.name == 'build-error' assert 'Full build log:' in out - # Message shows up for ProcessError (1), ChildError (1), and output (1) + print(out) + + # Message shows up for ProcessError (1) and output (1) errors = [line for line in out.split('\n') if 'configure: error: cannot run C compiled programs' in line] - assert len(errors) == 3 + assert len(errors) == 2 def test_install_overwrite( -- cgit v1.2.3-70-g09d2 From 9cc01dc57467223570cb924bc095fc0ea2f52ffa Mon Sep 17 00:00:00 2001 From: Greg Becker Date: Mon, 27 Jul 2020 01:17:58 -0700 Subject: add tutorial setup script to share/spack (#17705) * add tutorial setup script to share/spack * Add check for Ubuntu 18, fix xvda check, fix apt-get errors - now works on t2.micro, t2.small, and m instances - apt-get needs retries around it to work Co-authored-by: Todd Gamblin --- share/spack/setup-tutorial-env.sh | 123 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 123 insertions(+) create mode 100755 share/spack/setup-tutorial-env.sh diff --git a/share/spack/setup-tutorial-env.sh b/share/spack/setup-tutorial-env.sh new file mode 100755 index 0000000000..bb1fd423df --- /dev/null +++ b/share/spack/setup-tutorial-env.sh @@ -0,0 +1,123 @@ +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +############################################################################### +# +# This file is part of Spack and sets up the environment for the Spack tutorial +# It is intended to be run on ubuntu-18.04 or an ubuntu-18.04 container or AWS +# cloud9 environment +# +# Components: +# 1. apt installs for packages used in the tutorial +# these include compilers and externals used by the tutorial and +# basic spack requirements like python and curl +# 2. spack configuration files +# these set the default configuration for Spack to use x86_64 and suppress +# certain gpg warnings. The gpg warnings are not relevant for the tutorial +# and the default x86_64 architecture allows us to run the same tutorial on +# any x86_64 architecture without needing new binary packages. +# 3. aws cloud9 configuration to expand available storage +# when we run on aws cloud9 we have to expand the storage from 10G to 30G +# because we install too much software for a default cloud9 instance +############################################################################### + +#### +# Ensure we're on Ubuntu 18.04 +#### + +if [ -f /etc/os-release ]; then + . /etc/os-release +fi +if [ x"$UBUNTU_CODENAME" != "xbionic" ]; then + echo "The tutorial setup script must be run on Ubuntu 18.04." + return 1 &>/dev/null || exit 1 # works if sourced or run +fi + +#### +# Install packages needed for tutorial +#### + +# compilers, basic system components, externals +# There are retries around these because apt fails frequently on new instances, +# due to unattended updates running in the background and taking the lock. +until sudo apt-get update -y; do + echo "==> apt-get update failed. retrying..." + sleep 5 +done + +until sudo apt-get install -y --no-install-recommends \ + autoconf make python3 python3-pip \ + build-essential ca-certificates curl git gnupg2 iproute2 emacs \ + file openssh-server tcl unzip vim wget \ + clang g++ g++-6 gcc gcc-6 gfortran gfortran-6 \ + zlib1g zlib1g-dev mpich; do + echo "==> apt-get install failed. retrying..." + sleep 5 +done + +#### +# Spack configuration settings for tutorial +#### + +# create spack system config +sudo mkdir -p /etc/spack + +# set default arch to x86_64 +sudo tee /etc/spack/packages.yaml << EOF > /dev/null +packages: + all: + target: [x86_64] +EOF + +# suppress gpg warnings +sudo tee /etc/spack/config.yaml << EOF > /dev/null +config: + suppress_gpg_warnings: true +EOF + +#### +# AWS set volume size to at least 30G +#### + +# Hardcode the specified size to 30G +SIZE=30 + +# Get the ID of the environment host Amazon EC2 instance. +INSTANCEID=$(curl http://169.254.169.254/latest/meta-data//instance-id) + +# Get the ID of the Amazon EBS volume associated with the instance. +VOLUMEID=$(aws ec2 describe-instances \ + --instance-id $INSTANCEID \ + --query "Reservations[0].Instances[0].BlockDeviceMappings[0].Ebs.VolumeId" \ + --output text) + +# Resize the EBS volume. +aws ec2 modify-volume --volume-id $VOLUMEID --size $SIZE + +# Wait for the resize to finish. +while [ \ + "$(aws ec2 describe-volumes-modifications \ + --volume-id $VOLUMEID \ + --filters Name=modification-state,Values="optimizing","completed" \ + --query "length(VolumesModifications)"\ + --output text)" != "1" ]; do + sleep 1 +done + +if [ -e /dev/xvda1 ] +then + # Rewrite the partition table so that the partition takes up all the space that it can. + sudo growpart /dev/xvda 1 + + # Expand the size of the file system. + sudo resize2fs /dev/xvda1 + +else + # Rewrite the partition table so that the partition takes up all the space that it can. + sudo growpart /dev/nvme0n1 1 + + # Expand the size of the file system. + sudo resize2fs /dev/nvme0n1p1 +fi -- cgit v1.2.3-70-g09d2 From ce772420dd32dfc9b1a170db39554eae9798ca14 Mon Sep 17 00:00:00 2001 From: Patrick Gartung Date: Tue, 7 Jul 2020 16:46:39 -0500 Subject: Relocate rpaths for all binaries, then do text bin replacement if the rpaths still exist after running patchelf/otool (#17418) --- lib/spack/spack/binary_distribution.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/lib/spack/spack/binary_distribution.py b/lib/spack/spack/binary_distribution.py index ccfe614720..8658f7aa51 100644 --- a/lib/spack/spack/binary_distribution.py +++ b/lib/spack/spack/binary_distribution.py @@ -604,13 +604,9 @@ def relocate_package(spec, allow_root): # If we are installing back to the same location don't replace anything if old_layout_root != new_layout_root: - paths_to_relocate = [old_spack_prefix, old_layout_root] - paths_to_relocate.extend(prefix_to_hash.keys()) - files_to_relocate = list(filter( - lambda pathname: not relocate.file_is_relocatable( - pathname, paths_to_relocate=paths_to_relocate), - map(lambda filename: os.path.join(workdir, filename), - buildinfo['relocate_binaries']))) + files_to_relocate = [os.path.join(workdir, filename) + for filename in buildinfo.get('relocate_binaries') + ] # If the buildcache was not created with relativized rpaths # do the relocation of path in binaries if (spec.architecture.platform == 'darwin' or @@ -646,6 +642,13 @@ def relocate_package(spec, allow_root): new_spack_prefix, prefix_to_prefix) + paths_to_relocate = [old_prefix, old_layout_root] + paths_to_relocate.extend(prefix_to_hash.keys()) + files_to_relocate = list(filter( + lambda pathname: not relocate.file_is_relocatable( + pathname, paths_to_relocate=paths_to_relocate), + map(lambda filename: os.path.join(workdir, filename), + buildinfo['relocate_binaries']))) # relocate the install prefixes in binary files including dependencies relocate.relocate_text_bin(files_to_relocate, old_prefix, new_prefix, -- cgit v1.2.3-70-g09d2 From 69775fcc0725ce29b3936fcc930f1818ee48f0f8 Mon Sep 17 00:00:00 2001 From: Patrick Gartung Date: Thu, 9 Jul 2020 22:28:51 -0500 Subject: Relocation of sbang needs to be done when the spack prefix changes even if the install tree has not changed. (#17455) --- lib/spack/spack/binary_distribution.py | 13 ++++++++++++- lib/spack/spack/relocate.py | 8 +++++--- 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/lib/spack/spack/binary_distribution.py b/lib/spack/spack/binary_distribution.py index 8658f7aa51..0beac413e1 100644 --- a/lib/spack/spack/binary_distribution.py +++ b/lib/spack/spack/binary_distribution.py @@ -602,7 +602,7 @@ def relocate_package(spec, allow_root): if not is_backup_file(text_name): text_names.append(text_name) -# If we are installing back to the same location don't replace anything +# If we are not installing back to the same install tree do the relocation if old_layout_root != new_layout_root: files_to_relocate = [os.path.join(workdir, filename) for filename in buildinfo.get('relocate_binaries') @@ -656,6 +656,17 @@ def relocate_package(spec, allow_root): new_spack_prefix, prefix_to_prefix) +# If we are installing back to the same location +# relocate the sbang location if the spack directory changed + else: + if old_spack_prefix != new_spack_prefix: + relocate.relocate_text(text_names, + old_layout_root, new_layout_root, + old_prefix, new_prefix, + old_spack_prefix, + new_spack_prefix, + prefix_to_prefix) + def extract_tarball(spec, filename, allow_root=False, unsigned=False, force=False): diff --git a/lib/spack/spack/relocate.py b/lib/spack/spack/relocate.py index 56e7c6632c..e299f1c5c1 100644 --- a/lib/spack/spack/relocate.py +++ b/lib/spack/spack/relocate.py @@ -804,15 +804,17 @@ def relocate_text( where they should be relocated """ # TODO: reduce the number of arguments (8 seems too much) - sbang_regex = r'#!/bin/bash {0}/bin/sbang'.format(orig_spack) - new_sbang = r'#!/bin/bash {0}/bin/sbang'.format(new_spack) + orig_sbang = '#!/bin/bash {0}/bin/sbang'.format(orig_spack) + new_sbang = '#!/bin/bash {0}/bin/sbang'.format(new_spack) for file in files: _replace_prefix_text(file, orig_install_prefix, new_install_prefix) for orig_dep_prefix, new_dep_prefix in new_prefixes.items(): _replace_prefix_text(file, orig_dep_prefix, new_dep_prefix) _replace_prefix_text(file, orig_layout_root, new_layout_root) - _replace_prefix_text(file, sbang_regex, new_sbang) + # relocate the sbang location only if the spack directory changed + if orig_spack != new_spack: + _replace_prefix_text(file, orig_sbang, new_sbang) def relocate_text_bin( -- cgit v1.2.3-70-g09d2 From 0efb8ef412c4287c05bc5561a299fdd360c2f061 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Mon, 27 Jul 2020 15:38:27 -0700 Subject: tutorial: Add boto3 installation to setup script --- share/spack/setup-tutorial-env.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/share/spack/setup-tutorial-env.sh b/share/spack/setup-tutorial-env.sh index bb1fd423df..1f46f15c2d 100755 --- a/share/spack/setup-tutorial-env.sh +++ b/share/spack/setup-tutorial-env.sh @@ -57,6 +57,12 @@ until sudo apt-get install -y --no-install-recommends \ sleep 5 done +#### +# Upgrade boto3 python package on AWS systems +#### +pip3 install --upgrade boto3 + + #### # Spack configuration settings for tutorial #### -- cgit v1.2.3-70-g09d2 From 24bd9e3039367d6eae556daa2f41d57067c6c21f Mon Sep 17 00:00:00 2001 From: Greg Becker Date: Mon, 27 Jul 2020 23:44:56 -0700 Subject: bugfix: allow relative view paths (#17721) Relative paths in views have been broken since #17608 or earlier. - [x] Fix by passing base path of the environment into the `ViewDescriptor`. Relative paths are calculated from this path. --- lib/spack/spack/environment.py | 42 ++++++++++++++++++++++++++++-------------- 1 file changed, 28 insertions(+), 14 deletions(-) diff --git a/lib/spack/spack/environment.py b/lib/spack/spack/environment.py index 151e73a0fd..99aa3963d5 100644 --- a/lib/spack/spack/environment.py +++ b/lib/spack/spack/environment.py @@ -463,8 +463,9 @@ def _eval_conditional(string): class ViewDescriptor(object): - def __init__(self, root, projections={}, select=[], exclude=[], + def __init__(self, base_path, root, projections={}, select=[], exclude=[], link=default_view_link): + self.base = base_path self.root = root self.projections = projections self.select = select @@ -494,15 +495,19 @@ class ViewDescriptor(object): return ret @staticmethod - def from_dict(d): - return ViewDescriptor(d['root'], + def from_dict(base_path, d): + return ViewDescriptor(base_path, + d['root'], d.get('projections', {}), d.get('select', []), d.get('exclude', []), d.get('link', default_view_link)) def view(self): - return YamlFilesystemView(self.root, spack.store.layout, + root = self.root + if not os.path.isabs(root): + root = os.path.normpath(os.path.join(self.base, self.root)) + return YamlFilesystemView(root, spack.store.layout, ignore_conflicts=True, projections=self.projections) @@ -548,8 +553,11 @@ class ViewDescriptor(object): # that cannot be resolved or have repos that have been removed # we always regenerate the view from scratch. We must first make # sure the root directory exists for the very first time though. - fs.mkdirp(self.root) - with fs.replace_directory_transaction(self.root): + root = self.root + if not os.path.isabs(root): + root = os.path.normpath(os.path.join(self.base, self.root)) + fs.mkdirp(root) + with fs.replace_directory_transaction(root): view = self.view() view.clean() @@ -609,9 +617,11 @@ class Environment(object): self.views = {} elif with_view is True: self.views = { - default_view_name: ViewDescriptor(self.view_path_default)} + default_view_name: ViewDescriptor(self.path, + self.view_path_default)} elif isinstance(with_view, six.string_types): - self.views = {default_view_name: ViewDescriptor(with_view)} + self.views = {default_view_name: ViewDescriptor(self.path, + with_view)} # If with_view is None, then defer to the view settings determined by # the manifest file @@ -682,11 +692,14 @@ class Environment(object): # enable_view can be boolean, string, or None if enable_view is True or enable_view is None: self.views = { - default_view_name: ViewDescriptor(self.view_path_default)} + default_view_name: ViewDescriptor(self.path, + self.view_path_default)} elif isinstance(enable_view, six.string_types): - self.views = {default_view_name: ViewDescriptor(enable_view)} + self.views = {default_view_name: ViewDescriptor(self.path, + enable_view)} elif enable_view: - self.views = dict((name, ViewDescriptor.from_dict(values)) + path = self.path + self.views = dict((name, ViewDescriptor.from_dict(path, values)) for name, values in enable_view.items()) else: self.views = {} @@ -1120,7 +1133,7 @@ class Environment(object): if name in self.views: self.default_view.root = viewpath else: - self.views[name] = ViewDescriptor(viewpath) + self.views[name] = ViewDescriptor(self.path, viewpath) else: self.views.pop(name, None) @@ -1531,9 +1544,10 @@ class Environment(object): default_name = default_view_name if self.views and len(self.views) == 1 and default_name in self.views: path = self.default_view.root - if self.default_view == ViewDescriptor(self.view_path_default): + if self.default_view == ViewDescriptor(self.path, + self.view_path_default): view = True - elif self.default_view == ViewDescriptor(path): + elif self.default_view == ViewDescriptor(self.path, path): view = path else: view = dict((name, view.to_dict()) -- cgit v1.2.3-70-g09d2 From ae4bbbd24126ef992ab7bbca62073cdc571c0e83 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Tue, 28 Jul 2020 02:05:26 -0700 Subject: bump version number for 0.15.3 --- lib/spack/spack/__init__.py | 2 +- lib/spack/spack/container/images.json | 12 ++++++++---- lib/spack/spack/schema/container.py | 1 + 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/lib/spack/spack/__init__.py b/lib/spack/spack/__init__.py index 2b87833f7e..862c8baa60 100644 --- a/lib/spack/spack/__init__.py +++ b/lib/spack/spack/__init__.py @@ -5,7 +5,7 @@ #: major, minor, patch version for Spack, in a tuple -spack_version_info = (0, 15, 2) +spack_version_info = (0, 15, 3) #: String containing Spack version joined with .'s spack_version = '.'.join(str(v) for v in spack_version_info) diff --git a/lib/spack/spack/container/images.json b/lib/spack/spack/container/images.json index 7a7e98be1f..24271e0721 100644 --- a/lib/spack/spack/container/images.json +++ b/lib/spack/spack/container/images.json @@ -14,7 +14,8 @@ "0.15": "0.15", "0.15.0": "0.15.0", "0.15.1": "0.15.1", - "0.15.2": "0.15.2" + "0.15.2": "0.15.2", + "0.15.3": "0.15.3" } }, "ubuntu:16.04": { @@ -32,7 +33,8 @@ "0.15": "0.15", "0.15.0": "0.15.0", "0.15.1": "0.15.1", - "0.15.2": "0.15.2" + "0.15.2": "0.15.2", + "0.15.3": "0.15.3" } }, "centos:7": { @@ -50,7 +52,8 @@ "0.15": "0.15", "0.15.0": "0.15.0", "0.15.1": "0.15.1", - "0.15.2": "0.15.2" + "0.15.2": "0.15.2", + "0.15.3": "0.15.3" } }, "centos:6": { @@ -68,7 +71,8 @@ "0.15": "0.15", "0.15.0": "0.15.0", "0.15.1": "0.15.1", - "0.15.2": "0.15.2" + "0.15.2": "0.15.2", + "0.15.3": "0.15.3" } } } diff --git a/lib/spack/spack/schema/container.py b/lib/spack/spack/schema/container.py index 8b7f8dac3b..c6e54732e6 100644 --- a/lib/spack/spack/schema/container.py +++ b/lib/spack/spack/schema/container.py @@ -33,6 +33,7 @@ container_schema = { 'develop', '0.14', '0.14.0', '0.14.1', '0.14.2', '0.15', '0.15.0', '0.15.1', '0.15.2', + '0.15.3', ] } }, -- cgit v1.2.3-70-g09d2 From 0f25462ea63119e3378da11f1e511bedee15b07d Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Tue, 28 Jul 2020 02:11:06 -0700 Subject: update CHANGELOG.md for 0.15.3 --- CHANGELOG.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index d098c0631e..7140e90f29 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,15 @@ +# v0.15.3 (2020-07-28) + +This release contains the following bugfixes: + +* Fix handling of relative view paths (#17721) +* Fixes for binary relocation (#17418, #17455) +* Fix redundant printing of error messages in build environment (#17709) + +It also adds a support script for Spack tutorials: + +* Add a tutorial setup script to share/spack (#17705, #17722) + # v0.15.2 (2020-07-23) This minor release includes two new features: -- cgit v1.2.3-70-g09d2