summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorGreg Becker <becker33@llnl.gov>2021-03-04 22:29:48 -0800
committerGitHub <noreply@github.com>2021-03-04 22:29:48 -0800
commit92b7805e40bbda74a925128936999cd46a8fc850 (patch)
tree4e4f3b6caafff11ad031119f2d029c6534efa429
parent8bdd6c6f6d6d56a835d40a7c4cd491cb2a810bbb (diff)
downloadspack-92b7805e40bbda74a925128936999cd46a8fc850.tar.gz
spack-92b7805e40bbda74a925128936999cd46a8fc850.tar.bz2
spack-92b7805e40bbda74a925128936999cd46a8fc850.tar.xz
spack-92b7805e40bbda74a925128936999cd46a8fc850.zip
Allow relative paths in config files (relative to file dirname) (#21996)
This allows users to use relative paths for mirrors and repos and other things that may be part of a Spack environment. There are two ways to do it. 1. Relative to the file ```yaml spack: repos: - local_dir/my_repository ``` Which will refer to a repository like this in the directory where `spack.yaml` lives: ``` env/ spack.yaml <-- the config file above local_dir/ my_repository/ <-- this repository repo.yaml packages/ ``` 2. Relative to the environment ```yaml spack: repos: - $env/local_dir/my_repository ``` Both of these would refer to the same directory, but they differ for included files. For example, if you had this layout: ``` env/ spack.yaml repository/ includes/ repos.yaml repository/ ``` And this `spack.yaml`: ```yaml spack: include: includes/repos.yaml ``` Then, these two `repos.yaml` files are functionally different: ```yaml repos: - $env/repository # refers to env/repository/ above repos: - repository # refers to env/includes/repository/ above ``` The $env variable will not be evaluated if there is no active environment. This generally means that it should not be used outside of an environment's spack.yaml file. However, if other aspects of your workflow guarantee that there is always an active environment, it may be used in other config scopes.
-rw-r--r--lib/spack/spack/test/config.py35
-rw-r--r--lib/spack/spack/util/path.py36
2 files changed, 63 insertions, 8 deletions
diff --git a/lib/spack/spack/test/config.py b/lib/spack/spack/test/config.py
index b5c3dd657b..06cfd018f1 100644
--- a/lib/spack/spack/test/config.py
+++ b/lib/spack/spack/test/config.py
@@ -16,6 +16,7 @@ import pytest
import spack.paths
import spack.config
import spack.main
+import spack.environment
import spack.schema.compilers
import spack.schema.config
import spack.schema.env
@@ -267,7 +268,12 @@ def test_write_list_in_memory(mock_low_high_config):
assert config == repos_high['repos'] + repos_low['repos']
-def test_substitute_config_variables(mock_low_high_config):
+class MockEnv(object):
+ def __init__(self, path):
+ self.path = path
+
+
+def test_substitute_config_variables(mock_low_high_config, monkeypatch):
prefix = spack.paths.prefix.lstrip('/')
assert os.path.join(
@@ -298,6 +304,33 @@ def test_substitute_config_variables(mock_low_high_config):
'/foo/bar/baz', prefix, 'foo/bar/baz'
) != spack_path.canonicalize_path('/foo/bar/baz/${spack/foo/bar/baz/')
+ # $env replacement is a no-op when no environment is active
+ assert spack_path.canonicalize_path(
+ '/foo/bar/baz/$env'
+ ) == '/foo/bar/baz/$env'
+
+ # Fake an active environment and $env is replaced properly
+ fake_env_path = '/quux/quuux'
+ monkeypatch.setattr(spack.environment, 'get_env',
+ lambda x, y: MockEnv(fake_env_path))
+ assert spack_path.canonicalize_path(
+ '$env/foo/bar/baz'
+ ) == os.path.join(fake_env_path, 'foo/bar/baz')
+
+ # relative paths without source information are relative to cwd
+ assert spack_path.canonicalize_path(
+ 'foo/bar/baz'
+ ) == os.path.abspath('foo/bar/baz')
+
+ # relative paths with source information are relative to the file
+ spack.config.set(
+ 'config:module_roots', {'lmod': 'foo/bar/baz'}, scope='low')
+ spack.config.config.clear_caches()
+ path = spack.config.get('config:module_roots:lmod')
+ assert spack_path.canonicalize_path(path) == os.path.normpath(
+ os.path.join(mock_low_high_config.scopes['low'].path,
+ 'foo/bar/baz'))
+
packages_merge_low = {
'packages': {
diff --git a/lib/spack/spack/util/path.py b/lib/spack/spack/util/path.py
index 5e717cbb93..0735133b35 100644
--- a/lib/spack/spack/util/path.py
+++ b/lib/spack/spack/util/path.py
@@ -17,7 +17,7 @@ import llnl.util.tty as tty
from llnl.util.lang import memoized
import spack.paths
-
+import spack.util.spack_yaml as syaml
__all__ = [
'substitute_config_variables',
@@ -72,12 +72,22 @@ def substitute_config_variables(path):
- $spack The Spack instance's prefix
- $user The current user's username
- $tempdir Default temporary directory returned by tempfile.gettempdir()
+ - $env The active Spack environment.
These are substituted case-insensitively into the path, and users can
- use either ``$var`` or ``${var}`` syntax for the variables.
-
+ use either ``$var`` or ``${var}`` syntax for the variables. $env is only
+ replaced if there is an active environment, and should only be used in
+ environment yaml files.
"""
- # Look up replacements for re.sub in the replacements dict.
+ import spack.environment as ev # break circular
+ env = ev.get_env({}, '')
+ if env:
+ replacements.update({'env': env.path})
+ else:
+ # If a previous invocation added env, remove it
+ replacements.pop('env', None)
+
+ # Look up replacements
def repl(match):
m = match.group(0).strip('${}')
return replacements.get(m.lower(), match.group(0))
@@ -132,7 +142,19 @@ def add_padding(path, length):
def canonicalize_path(path):
"""Same as substitute_path_variables, but also take absolute path."""
- path = substitute_path_variables(path)
- path = os.path.abspath(path)
+ # Get file in which path was written in case we need to make it absolute
+ # relative to that path.
+ filename = None
+ if isinstance(path, syaml.syaml_str):
+ filename = os.path.dirname(path._start_mark.name)
+ assert path._start_mark.name == path._end_mark.name
- return path
+ path = substitute_path_variables(path)
+ if not os.path.isabs(path):
+ if filename:
+ path = os.path.join(filename, path)
+ else:
+ path = os.path.abspath(path)
+ tty.debug("Using current working directory as base for abspath")
+
+ return os.path.normpath(path)