From a236fce31f5d5018c788f51cda50492f1f241cba Mon Sep 17 00:00:00 2001 From: Massimiliano Culpo Date: Thu, 28 Sep 2023 18:21:52 +0200 Subject: Partial removal of circular dependencies between `spack` and `llnl` (#40090) Modifications: - [x] Move `spack.util.string` to `llnl.string` - [x] Remove dependency of `llnl` on `spack.error` - [x] Move path of `spack.util.path` to `llnl.path` - [x] Move `spack.util.environment.get_host_*` to `spack.spec` --- lib/spack/llnl/path.py | 105 ++++++++++++++++++++++++++++ lib/spack/llnl/string.py | 67 ++++++++++++++++++ lib/spack/llnl/util/filesystem.py | 6 +- lib/spack/llnl/util/lock.py | 4 +- lib/spack/llnl/util/symlink.py | 5 +- lib/spack/spack/build_environment.py | 2 +- lib/spack/spack/cmd/__init__.py | 4 +- lib/spack/spack/cmd/buildcache.py | 2 +- lib/spack/spack/cmd/env.py | 2 +- lib/spack/spack/cmd/make_installer.py | 3 +- lib/spack/spack/cmd/tags.py | 3 +- lib/spack/spack/compiler.py | 4 +- lib/spack/spack/directory_layout.py | 4 +- lib/spack/spack/environment/environment.py | 2 +- lib/spack/spack/fetch_strategy.py | 2 +- lib/spack/spack/install_test.py | 2 +- lib/spack/spack/repo.py | 3 +- lib/spack/spack/spec.py | 51 ++++++++++++-- lib/spack/spack/stage.py | 4 +- lib/spack/spack/test/build_environment.py | 2 +- lib/spack/spack/test/cmd/bootstrap.py | 3 +- lib/spack/spack/test/directory_layout.py | 3 +- lib/spack/spack/test/llnl/llnl_string.py | 43 ++++++++++++ lib/spack/spack/test/util/util_string.py | 14 ---- lib/spack/spack/url.py | 2 +- lib/spack/spack/util/environment.py | 44 +----------- lib/spack/spack/util/path.py | 106 ----------------------------- lib/spack/spack/util/string.py | 57 ---------------- lib/spack/spack/util/url.py | 4 +- lib/spack/spack/variant.py | 2 +- 30 files changed, 297 insertions(+), 258 deletions(-) create mode 100644 lib/spack/llnl/path.py create mode 100644 lib/spack/llnl/string.py create mode 100644 lib/spack/spack/test/llnl/llnl_string.py delete mode 100644 lib/spack/spack/test/util/util_string.py delete mode 100644 lib/spack/spack/util/string.py diff --git a/lib/spack/llnl/path.py b/lib/spack/llnl/path.py new file mode 100644 index 0000000000..bef9e14ada --- /dev/null +++ b/lib/spack/llnl/path.py @@ -0,0 +1,105 @@ +# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) +"""Path primitives that just require Python standard library.""" +import functools +import sys +from typing import List, Optional +from urllib.parse import urlparse + + +class Path: + """Enum to identify the path-style.""" + + unix: int = 0 + windows: int = 1 + platform_path: int = windows if sys.platform == "win32" else unix + + +def format_os_path(path: str, mode: int = Path.unix) -> str: + """Formats the input path to use consistent, platform specific separators. + + Absolute paths are converted between drive letters and a prepended '/' as per platform + requirement. + + Parameters: + path: the path to be normalized, must be a string or expose the replace method. + mode: the path file separator style to normalize the passed path to. + Default is unix style, i.e. '/' + """ + if not path: + return path + if mode == Path.windows: + path = path.replace("/", "\\") + else: + path = path.replace("\\", "/") + return path + + +def convert_to_posix_path(path: str) -> str: + """Converts the input path to POSIX style.""" + return format_os_path(path, mode=Path.unix) + + +def convert_to_windows_path(path: str) -> str: + """Converts the input path to Windows style.""" + return format_os_path(path, mode=Path.windows) + + +def convert_to_platform_path(path: str) -> str: + """Converts the input path to the current platform's native style.""" + return format_os_path(path, mode=Path.platform_path) + + +def path_to_os_path(*parameters: str) -> List[str]: + """Takes an arbitrary number of positional parameters, converts each argument of type + string to use a normalized filepath separator, and returns a list of all values. + """ + + def _is_url(path_or_url: str) -> bool: + if "\\" in path_or_url: + return False + url_tuple = urlparse(path_or_url) + return bool(url_tuple.scheme) and len(url_tuple.scheme) > 1 + + result = [] + for item in parameters: + if isinstance(item, str) and not _is_url(item): + item = convert_to_platform_path(item) + result.append(item) + return result + + +def system_path_filter(_func=None, arg_slice: Optional[slice] = None): + """Filters function arguments to account for platform path separators. + Optional slicing range can be specified to select specific arguments + + This decorator takes all (or a slice) of a method's positional arguments + and normalizes usage of filepath separators on a per platform basis. + + Note: `**kwargs`, urls, and any type that is not a string are ignored + so in such cases where path normalization is required, that should be + handled by calling path_to_os_path directly as needed. + + Parameters: + arg_slice: a slice object specifying the slice of arguments + in the decorated method over which filepath separators are + normalized + """ + + def holder_func(func): + @functools.wraps(func) + def path_filter_caller(*args, **kwargs): + args = list(args) + if arg_slice: + args[arg_slice] = path_to_os_path(*args[arg_slice]) + else: + args = path_to_os_path(*args) + return func(*args, **kwargs) + + return path_filter_caller + + if _func: + return holder_func(_func) + return holder_func diff --git a/lib/spack/llnl/string.py b/lib/spack/llnl/string.py new file mode 100644 index 0000000000..a203be8d34 --- /dev/null +++ b/lib/spack/llnl/string.py @@ -0,0 +1,67 @@ +# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) +"""String manipulation functions that do not have other dependencies than Python +standard library +""" +from typing import List, Optional + + +def comma_list(sequence: List[str], article: str = "") -> str: + if type(sequence) is not list: + sequence = list(sequence) + + if not sequence: + return "" + if len(sequence) == 1: + return sequence[0] + + out = ", ".join(str(s) for s in sequence[:-1]) + if len(sequence) != 2: + out += "," # oxford comma + out += " " + if article: + out += article + " " + out += str(sequence[-1]) + return out + + +def comma_or(sequence: List[str]) -> str: + """Return a string with all the elements of the input joined by comma, but the last + one (which is joined by 'or'). + """ + return comma_list(sequence, "or") + + +def comma_and(sequence: List[str]) -> str: + """Return a string with all the elements of the input joined by comma, but the last + one (which is joined by 'and'). + """ + return comma_list(sequence, "and") + + +def quote(sequence: List[str], q: str = "'") -> List[str]: + """Quotes each item in the input list with the quote character passed as second argument.""" + return [f"{q}{e}{q}" for e in sequence] + + +def plural(n: int, singular: str, plural: Optional[str] = None, show_n: bool = True) -> str: + """Pluralize word by adding an s if n != 1. + + Arguments: + n: number of things there are + singular: singular form of word + plural: optional plural form, for when it's not just singular + 's' + show_n: whether to include n in the result string (default True) + + Returns: + "1 thing" if n == 1 or "n things" if n != 1 + """ + number = f"{n} " if show_n else "" + if n == 1: + return f"{number}{singular}" + elif plural is not None: + return f"{number}{plural}" + else: + return f"{number}{singular}s" diff --git a/lib/spack/llnl/util/filesystem.py b/lib/spack/llnl/util/filesystem.py index bd203ef200..8f4217049d 100644 --- a/lib/spack/llnl/util/filesystem.py +++ b/lib/spack/llnl/util/filesystem.py @@ -28,7 +28,8 @@ from llnl.util.lang import dedupe, memoized from llnl.util.symlink import islink, readlink, resolve_link_target_relative_to_the_link, symlink from spack.util.executable import Executable, which -from spack.util.path import path_to_os_path, system_path_filter + +from ..path import path_to_os_path, system_path_filter if sys.platform != "win32": import grp @@ -336,8 +337,7 @@ def filter_file( if string: regex = re.escape(regex) - filenames = path_to_os_path(*filenames) - for filename in filenames: + for filename in path_to_os_path(*filenames): msg = 'FILTER FILE: {0} [replacing "{1}"]' tty.debug(msg.format(filename, regex)) diff --git a/lib/spack/llnl/util/lock.py b/lib/spack/llnl/util/lock.py index 74a325acd2..156300b891 100644 --- a/lib/spack/llnl/util/lock.py +++ b/lib/spack/llnl/util/lock.py @@ -14,7 +14,7 @@ from typing import IO, Any, Callable, ContextManager, Dict, Generator, Optional, from llnl.util import lang, tty -import spack.util.string +from ..string import plural if sys.platform != "win32": import fcntl @@ -169,7 +169,7 @@ def _attempts_str(wait_time, nattempts): if nattempts <= 1: return "" - attempts = spack.util.string.plural(nattempts, "attempt") + attempts = plural(nattempts, "attempt") return " after {} and {}".format(lang.pretty_seconds(wait_time), attempts) diff --git a/lib/spack/llnl/util/symlink.py b/lib/spack/llnl/util/symlink.py index ab6654e847..4f4f965f13 100644 --- a/lib/spack/llnl/util/symlink.py +++ b/lib/spack/llnl/util/symlink.py @@ -11,8 +11,7 @@ import tempfile from llnl.util import lang, tty -from spack.error import SpackError -from spack.util.path import system_path_filter +from ..path import system_path_filter if sys.platform == "win32": from win32file import CreateHardLink @@ -338,7 +337,7 @@ def resolve_link_target_relative_to_the_link(link): return os.path.join(link_dir, target) -class SymlinkError(SpackError): +class SymlinkError(RuntimeError): """Exception class for errors raised while creating symlinks, junctions and hard links """ diff --git a/lib/spack/spack/build_environment.py b/lib/spack/spack/build_environment.py index 723f398ae6..881fcb5c9c 100644 --- a/lib/spack/spack/build_environment.py +++ b/lib/spack/spack/build_environment.py @@ -43,6 +43,7 @@ import types from typing import List, Tuple import llnl.util.tty as tty +from llnl.string import plural from llnl.util.filesystem import join_path from llnl.util.lang import dedupe from llnl.util.symlink import symlink @@ -82,7 +83,6 @@ from spack.util.environment import ( from spack.util.executable import Executable from spack.util.log_parse import make_log_context, parse_log_events from spack.util.module_cmd import load_module, module, path_from_modules -from spack.util.string import plural # # This can be set by the user to globally disable parallel builds. diff --git a/lib/spack/spack/cmd/__init__.py b/lib/spack/spack/cmd/__init__.py index fd5e81fe7a..c26ab181c1 100644 --- a/lib/spack/spack/cmd/__init__.py +++ b/lib/spack/spack/cmd/__init__.py @@ -11,6 +11,7 @@ import sys from textwrap import dedent from typing import List, Match, Tuple +import llnl.string import llnl.util.tty as tty from llnl.util.filesystem import join_path from llnl.util.lang import attr_setdefault, index_by @@ -29,7 +30,6 @@ import spack.traverse as traverse import spack.user_environment as uenv import spack.util.spack_json as sjson import spack.util.spack_yaml as syaml -import spack.util.string # cmd has a submodule called "list" so preserve the python list module python_list = list @@ -516,7 +516,7 @@ def print_how_many_pkgs(specs, pkg_type=""): category, e.g. if pkg_type is "installed" then the message would be "3 installed packages" """ - tty.msg("%s" % spack.util.string.plural(len(specs), pkg_type + " package")) + tty.msg("%s" % llnl.string.plural(len(specs), pkg_type + " package")) def spack_is_git_repo(): diff --git a/lib/spack/spack/cmd/buildcache.py b/lib/spack/spack/cmd/buildcache.py index f611956c36..5eb0ea1ed4 100644 --- a/lib/spack/spack/cmd/buildcache.py +++ b/lib/spack/spack/cmd/buildcache.py @@ -13,6 +13,7 @@ from typing import List import llnl.util.tty as tty import llnl.util.tty.color as clr +from llnl.string import plural from llnl.util.lang import elide_list import spack.binary_distribution as bindist @@ -32,7 +33,6 @@ import spack.util.web as web_util from spack.cmd import display_specs from spack.spec import Spec, save_dependency_specfiles from spack.stage import Stage -from spack.util.string import plural description = "create, download and install binary packages" section = "packaging" diff --git a/lib/spack/spack/cmd/env.py b/lib/spack/spack/cmd/env.py index 8a2f129bda..6c22e70a5d 100644 --- a/lib/spack/spack/cmd/env.py +++ b/lib/spack/spack/cmd/env.py @@ -9,6 +9,7 @@ import shutil import sys import tempfile +import llnl.string as string import llnl.util.filesystem as fs import llnl.util.tty as tty from llnl.util.tty.colify import colify @@ -28,7 +29,6 @@ import spack.environment.shell import spack.schema.env import spack.spec import spack.tengine -import spack.util.string as string from spack.util.environment import EnvironmentModifications description = "manage virtual environments" diff --git a/lib/spack/spack/cmd/make_installer.py b/lib/spack/spack/cmd/make_installer.py index 7f43659429..986631d39f 100644 --- a/lib/spack/spack/cmd/make_installer.py +++ b/lib/spack/spack/cmd/make_installer.py @@ -6,10 +6,11 @@ import os import posixpath import sys +from llnl.path import convert_to_posix_path + import spack.paths import spack.util.executable from spack.spec import Spec -from spack.util.path import convert_to_posix_path description = "generate Windows installer" section = "admin" diff --git a/lib/spack/spack/cmd/tags.py b/lib/spack/spack/cmd/tags.py index 0aa6acfd5f..0fbc91b6f4 100644 --- a/lib/spack/spack/cmd/tags.py +++ b/lib/spack/spack/cmd/tags.py @@ -5,6 +5,7 @@ import io import sys +import llnl.string import llnl.util.tty as tty import llnl.util.tty.colify as colify @@ -24,7 +25,7 @@ def report_tags(category, tags): if isatty: num = len(tags) fmt = "{0} package tag".format(category) - buffer.write("{0}:\n".format(spack.util.string.plural(num, fmt))) + buffer.write("{0}:\n".format(llnl.string.plural(num, fmt))) if tags: colify.colify(tags, output=buffer, tty=isatty, indent=4) diff --git a/lib/spack/spack/compiler.py b/lib/spack/spack/compiler.py index 3ec7d1421a..70da3ca2c4 100644 --- a/lib/spack/spack/compiler.py +++ b/lib/spack/spack/compiler.py @@ -13,6 +13,7 @@ import sys import tempfile from typing import List, Optional, Sequence +import llnl.path import llnl.util.lang import llnl.util.tty as tty from llnl.util.filesystem import path_contains_subdirectory, paths_containing_libs @@ -24,7 +25,6 @@ import spack.util.executable import spack.util.module_cmd import spack.version from spack.util.environment import filter_system_paths -from spack.util.path import system_path_filter __all__ = ["Compiler"] @@ -160,7 +160,7 @@ def _parse_link_paths(string): return implicit_link_dirs -@system_path_filter +@llnl.path.system_path_filter def _parse_non_system_link_dirs(string: str) -> List[str]: """Parses link paths out of compiler debug output. diff --git a/lib/spack/spack/directory_layout.py b/lib/spack/spack/directory_layout.py index dd4e6dd5e1..46bb6c8557 100644 --- a/lib/spack/spack/directory_layout.py +++ b/lib/spack/spack/directory_layout.py @@ -120,10 +120,8 @@ class DirectoryLayout: versioning. We use it in the case that an analysis later needs to easily access this information. """ - from spack.util.environment import get_host_environment_metadata - env_file = self.env_metadata_path(spec) - environ = get_host_environment_metadata() + environ = spack.spec.get_host_environment_metadata() with open(env_file, "w") as fd: sjson.dump(environ, fd) diff --git a/lib/spack/spack/environment/environment.py b/lib/spack/spack/environment/environment.py index d8f04c7d4b..496a8b332a 100644 --- a/lib/spack/spack/environment/environment.py +++ b/lib/spack/spack/environment/environment.py @@ -404,7 +404,7 @@ def _write_yaml(data, str_or_file): def _eval_conditional(string): """Evaluate conditional definitions using restricted variable scope.""" - valid_variables = spack.util.environment.get_host_environment() + valid_variables = spack.spec.get_host_environment() valid_variables.update({"re": re, "env": os.environ}) return eval(string, valid_variables) diff --git a/lib/spack/spack/fetch_strategy.py b/lib/spack/spack/fetch_strategy.py index 90ff8527fd..8578b110fc 100644 --- a/lib/spack/spack/fetch_strategy.py +++ b/lib/spack/spack/fetch_strategy.py @@ -35,6 +35,7 @@ import llnl.url import llnl.util import llnl.util.filesystem as fs import llnl.util.tty as tty +from llnl.string import comma_and, quote from llnl.util.filesystem import get_single_file, mkdirp, temp_cwd, temp_rename, working_dir from llnl.util.symlink import symlink @@ -49,7 +50,6 @@ import spack.version import spack.version.git_ref_lookup from spack.util.compression import decompressor_for from spack.util.executable import CommandNotFoundError, which -from spack.util.string import comma_and, quote #: List of all fetch strategies, created by FetchStrategy metaclass. all_strategies = [] diff --git a/lib/spack/spack/install_test.py b/lib/spack/spack/install_test.py index 90359796d8..0d8fa782b6 100644 --- a/lib/spack/spack/install_test.py +++ b/lib/spack/spack/install_test.py @@ -17,6 +17,7 @@ from typing import Callable, List, Optional, Tuple, Type, TypeVar, Union import llnl.util.filesystem as fs import llnl.util.tty as tty +from llnl.string import plural from llnl.util.lang import nullcontext from llnl.util.tty.color import colorize @@ -26,7 +27,6 @@ import spack.util.spack_json as sjson from spack.installer import InstallError from spack.spec import Spec from spack.util.prefix import Prefix -from spack.util.string import plural #: Stand-alone test failure info type TestFailureType = Tuple[BaseException, str] diff --git a/lib/spack/spack/repo.py b/lib/spack/spack/repo.py index a4362d9d31..4391d9a9a2 100644 --- a/lib/spack/spack/repo.py +++ b/lib/spack/spack/repo.py @@ -26,6 +26,7 @@ import types import uuid from typing import Any, Dict, List, Union +import llnl.path import llnl.util.filesystem as fs import llnl.util.lang import llnl.util.tty as tty @@ -563,7 +564,7 @@ class RepoIndex: self.checker = package_checker self.packages_path = self.checker.packages_path if sys.platform == "win32": - self.packages_path = spack.util.path.convert_to_posix_path(self.packages_path) + self.packages_path = llnl.path.convert_to_posix_path(self.packages_path) self.namespace = namespace self.indexers: Dict[str, Indexer] = {} diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index 13bf66f506..85a638b602 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -54,10 +54,14 @@ import enum import io import itertools import os +import platform import re +import socket import warnings -from typing import Callable, List, Optional, Tuple, Union +from typing import Any, Callable, Dict, List, Optional, Tuple, Union +import llnl.path +import llnl.string import llnl.util.filesystem as fs import llnl.util.lang as lang import llnl.util.tty as tty @@ -82,11 +86,9 @@ import spack.util.crypto import spack.util.executable import spack.util.hash import spack.util.module_cmd as md -import spack.util.path as pth import spack.util.prefix import spack.util.spack_json as sjson import spack.util.spack_yaml as syaml -import spack.util.string import spack.variant as vt import spack.version as vn import spack.version.git_ref_lookup @@ -1390,7 +1392,7 @@ class Spec: @property def external_path(self): - return pth.path_to_os_path(self._external_path)[0] + return llnl.path.path_to_os_path(self._external_path)[0] @external_path.setter def external_path(self, ext_path): @@ -1799,7 +1801,7 @@ class Spec: @prefix.setter def prefix(self, value): - self._prefix = spack.util.prefix.Prefix(pth.convert_to_platform_path(value)) + self._prefix = spack.util.prefix.Prefix(llnl.path.convert_to_platform_path(value)) def spec_hash(self, hash): """Utility method for computing different types of Spec hashes. @@ -5148,6 +5150,43 @@ def save_dependency_specfiles(root: Spec, output_directory: str, dependencies: L fd.write(spec.to_json(hash=ht.dag_hash)) +def get_host_environment_metadata() -> Dict[str, str]: + """Get the host environment, reduce to a subset that we can store in + the install directory, and add the spack version. + """ + import spack.main + + environ = get_host_environment() + return { + "host_os": environ["os"], + "platform": environ["platform"], + "host_target": environ["target"], + "hostname": environ["hostname"], + "spack_version": spack.main.get_version(), + "kernel_version": platform.version(), + } + + +def get_host_environment() -> Dict[str, Any]: + """Return a dictionary (lookup) with host information (not including the + os.environ). + """ + host_platform = spack.platforms.host() + host_target = host_platform.target("default_target") + host_os = host_platform.operating_system("default_os") + arch_fmt = "platform={0} os={1} target={2}" + arch_spec = Spec(arch_fmt.format(host_platform, host_os, host_target)) + return { + "target": str(host_target), + "os": str(host_os), + "platform": str(host_platform), + "arch": arch_spec, + "architecture": arch_spec, + "arch_str": str(arch_spec), + "hostname": socket.gethostname(), + } + + class SpecParseError(spack.error.SpecError): """Wrapper for ParseError for when we're parsing specs.""" @@ -5208,7 +5247,7 @@ class InvalidDependencyError(spack.error.SpecError): def __init__(self, pkg, deps): self.invalid_deps = deps super().__init__( - "Package {0} does not depend on {1}".format(pkg, spack.util.string.comma_or(deps)) + "Package {0} does not depend on {1}".format(pkg, llnl.string.comma_or(deps)) ) diff --git a/lib/spack/spack/stage.py b/lib/spack/spack/stage.py index 065f74eb8b..73b82c1378 100644 --- a/lib/spack/spack/stage.py +++ b/lib/spack/spack/stage.py @@ -14,6 +14,7 @@ import sys import tempfile from typing import Callable, Dict, Iterable, Optional +import llnl.string import llnl.util.lang import llnl.util.tty as tty from llnl.util.filesystem import ( @@ -37,7 +38,6 @@ import spack.spec import spack.util.lock import spack.util.path as sup import spack.util.pattern as pattern -import spack.util.string import spack.util.url as url_util from spack.util.crypto import bit_length, prefix_bits @@ -897,7 +897,7 @@ def get_checksums_for_versions( num_ver = len(sorted_versions) tty.msg( - f"Found {spack.util.string.plural(num_ver, 'version')} of {package_name}:", + f"Found {llnl.string.plural(num_ver, 'version')} of {package_name}:", "", *llnl.util.lang.elide_list( ["{0:{1}} {2}".format(str(v), max_len, url_by_version[v]) for v in sorted_versions] diff --git a/lib/spack/spack/test/build_environment.py b/lib/spack/spack/test/build_environment.py index 90f3fb378e..2eb80fded3 100644 --- a/lib/spack/spack/test/build_environment.py +++ b/lib/spack/spack/test/build_environment.py @@ -9,6 +9,7 @@ import posixpath import pytest +from llnl.path import Path, convert_to_platform_path from llnl.util.filesystem import HeaderList, LibraryList import spack.build_environment @@ -21,7 +22,6 @@ from spack.paths import build_env_path from spack.util.cpus import determine_number_of_jobs from spack.util.environment import EnvironmentModifications from spack.util.executable import Executable -from spack.util.path import Path, convert_to_platform_path def os_pathsep_join(path, *pths): diff --git a/lib/spack/spack/test/cmd/bootstrap.py b/lib/spack/spack/test/cmd/bootstrap.py index 35f6987bec..eff9bf042d 100644 --- a/lib/spack/spack/test/cmd/bootstrap.py +++ b/lib/spack/spack/test/cmd/bootstrap.py @@ -7,13 +7,14 @@ import sys import pytest +from llnl.path import convert_to_posix_path + import spack.bootstrap import spack.bootstrap.core import spack.config import spack.environment as ev import spack.main import spack.mirror -from spack.util.path import convert_to_posix_path _bootstrap = spack.main.SpackCommand("bootstrap") diff --git a/lib/spack/spack/test/directory_layout.py b/lib/spack/spack/test/directory_layout.py index 995707db23..64676bfebc 100644 --- a/lib/spack/spack/test/directory_layout.py +++ b/lib/spack/spack/test/directory_layout.py @@ -12,11 +12,12 @@ from pathlib import Path import pytest +from llnl.path import path_to_os_path + import spack.paths import spack.repo from spack.directory_layout import DirectoryLayout, InvalidDirectoryLayoutParametersError from spack.spec import Spec -from spack.util.path import path_to_os_path # number of packages to test (to reduce test time) max_packages = 10 diff --git a/lib/spack/spack/test/llnl/llnl_string.py b/lib/spack/spack/test/llnl/llnl_string.py new file mode 100644 index 0000000000..93d7e662c5 --- /dev/null +++ b/lib/spack/spack/test/llnl/llnl_string.py @@ -0,0 +1,43 @@ +# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) +import pytest + +import llnl.string + + +@pytest.mark.parametrize( + "arguments,expected", + [ + ((0, "thing"), "0 things"), + ((1, "thing"), "1 thing"), + ((2, "thing"), "2 things"), + ((1, "thing", "wombats"), "1 thing"), + ((2, "thing", "wombats"), "2 wombats"), + ((2, "thing", "wombats", False), "wombats"), + ], +) +def test_plural(arguments, expected): + assert llnl.string.plural(*arguments) == expected + + +@pytest.mark.parametrize( + "arguments,expected", + [((["one", "two"],), ["'one'", "'two'"]), ((["one", "two"], "^"), ["^one^", "^two^"])], +) +def test_quote(arguments, expected): + assert llnl.string.quote(*arguments) == expected + + +@pytest.mark.parametrize( + "input,expected_and,expected_or", + [ + (["foo"], "foo", "foo"), + (["foo", "bar"], "foo and bar", "foo or bar"), + (["foo", "bar", "baz"], "foo, bar, and baz", "foo, bar, or baz"), + ], +) +def test_comma_and_or(input, expected_and, expected_or): + assert llnl.string.comma_and(input) == expected_and + assert llnl.string.comma_or(input) == expected_or diff --git a/lib/spack/spack/test/util/util_string.py b/lib/spack/spack/test/util/util_string.py deleted file mode 100644 index f4de373859..0000000000 --- a/lib/spack/spack/test/util/util_string.py +++ /dev/null @@ -1,14 +0,0 @@ -# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other -# Spack Project Developers. See the top-level COPYRIGHT file for details. -# -# SPDX-License-Identifier: (Apache-2.0 OR MIT) - -from spack.util.string import plural - - -def test_plural(): - assert plural(0, "thing") == "0 things" - assert plural(1, "thing") == "1 thing" - assert plural(2, "thing") == "2 things" - assert plural(1, "thing", "wombats") == "1 thing" - assert plural(2, "thing", "wombats") == "2 wombats" diff --git a/lib/spack/spack/url.py b/lib/spack/spack/url.py index c5e47232c0..080c924596 100644 --- a/lib/spack/spack/url.py +++ b/lib/spack/spack/url.py @@ -31,12 +31,12 @@ import pathlib import re import llnl.url +from llnl.path import convert_to_posix_path from llnl.util.tty.color import cescape, colorize import spack.error import spack.util.web import spack.version -from spack.util.path import convert_to_posix_path # # Note: We call the input to most of these functions a "path" but the functions diff --git a/lib/spack/spack/util/environment.py b/lib/spack/spack/util/environment.py index c18be76cc6..246df65cb8 100644 --- a/lib/spack/spack/util/environment.py +++ b/lib/spack/spack/util/environment.py @@ -10,21 +10,16 @@ import json import os import os.path import pickle -import platform import re -import socket import sys from functools import wraps from typing import Any, Callable, Dict, List, MutableMapping, Optional, Tuple, Union +from llnl.path import path_to_os_path, system_path_filter from llnl.util import tty from llnl.util.lang import dedupe -import spack.platforms -import spack.spec - from .executable import Executable, which -from .path import path_to_os_path, system_path_filter if sys.platform == "win32": SYSTEM_PATHS = [ @@ -224,43 +219,6 @@ def pickle_environment(path: Path, environment: Optional[Dict[str, str]] = None) pickle.dump(dict(environment if environment else os.environ), pickle_file, protocol=2) -def get_host_environment_metadata() -> Dict[str, str]: - """Get the host environment, reduce to a subset that we can store in - the install directory, and add the spack version. - """ - import spack.main - - environ = get_host_environment() - return { - "host_os": environ["os"], - "platform": environ["platform"], - "host_target": environ["target"], - "hostname": environ["hostname"], - "spack_version": spack.main.get_version(), - "kernel_version": platform.version(), - } - - -def get_host_environment() -> Dict[str, Any]: - """Return a dictionary (lookup) with host information (not including the - os.environ). - """ - host_platform = spack.platforms.host() - host_target = host_platform.target("default_target") - host_os = host_platform.operating_system("default_os") - arch_fmt = "platform={0} os={1} target={2}" - arch_spec = spack.spec.Spec(arch_fmt.format(host_platform, host_os, host_target)) - return { - "target": str(host_target), - "os": str(host_os), - "platform": str(host_platform), - "arch": arch_spec, - "architecture": arch_spec, - "arch_str": str(arch_spec), - "hostname": socket.gethostname(), - } - - @contextlib.contextmanager def set_env(**kwargs): """Temporarily sets and restores environment variables. diff --git a/lib/spack/spack/util/path.py b/lib/spack/spack/util/path.py index 3dc0ea676c..a46443c083 100644 --- a/lib/spack/spack/util/path.py +++ b/lib/spack/spack/util/path.py @@ -15,7 +15,6 @@ import subprocess import sys import tempfile from datetime import date -from urllib.parse import urlparse import llnl.util.tty as tty from llnl.util.lang import memoized @@ -98,31 +97,10 @@ SPACK_MAX_INSTALL_PATH_LENGTH = 300 SPACK_PATH_PADDING_CHARS = "__spack_path_placeholder__" -def is_path_url(path): - if "\\" in path: - return False - url_tuple = urlparse(path) - return bool(url_tuple.scheme) and len(url_tuple.scheme) > 1 - - def win_exe_ext(): return ".exe" -def path_to_os_path(*pths): - """ - Takes an arbitrary number of positional parameters - converts each arguemnt of type string to use a normalized - filepath separator, and returns a list of all values - """ - ret_pths = [] - for pth in pths: - if isinstance(pth, str) and not is_path_url(pth): - pth = convert_to_platform_path(pth) - ret_pths.append(pth) - return ret_pths - - def sanitize_filename(filename: str) -> str: """ Replaces unsupported characters (for the host) in a filename with underscores. @@ -145,42 +123,6 @@ def sanitize_filename(filename: str) -> str: return re.sub(r'[\x00-\x1F\x7F"*/:<>?\\|]', "_", filename) -def system_path_filter(_func=None, arg_slice=None): - """ - Filters function arguments to account for platform path separators. - Optional slicing range can be specified to select specific arguments - - This decorator takes all (or a slice) of a method's positional arguments - and normalizes usage of filepath separators on a per platform basis. - - Note: **kwargs, urls, and any type that is not a string are ignored - so in such cases where path normalization is required, that should be - handled by calling path_to_os_path directly as needed. - - Parameters: - arg_slice (slice): a slice object specifying the slice of arguments - in the decorated method over which filepath separators are - normalized - """ - from functools import wraps - - def holder_func(func): - @wraps(func) - def path_filter_caller(*args, **kwargs): - args = list(args) - if arg_slice: - args[arg_slice] = path_to_os_path(*args[arg_slice]) - else: - args = path_to_os_path(*args) - return func(*args, **kwargs) - - return path_filter_caller - - if _func: - return holder_func(_func) - return holder_func - - @memoized def get_system_path_max(): # Choose a conservative default @@ -202,54 +144,6 @@ def get_system_path_max(): return sys_max_path_length -class Path: - """ - Describes the filepath separator types - in an enum style - with a helper attribute - exposing the path type of - the current platform. - """ - - unix = 0 - windows = 1 - platform_path = windows if sys.platform == "win32" else unix - - -def format_os_path(path, mode=Path.unix): - """ - Format path to use consistent, platform specific - separators. Absolute paths are converted between - drive letters and a prepended '/' as per platform - requirement. - - Parameters: - path (str): the path to be normalized, must be a string - or expose the replace method. - mode (Path): the path filesperator style to normalize the - passed path to. Default is unix style, i.e. '/' - """ - if not path: - return path - if mode == Path.windows: - path = path.replace("/", "\\") - else: - path = path.replace("\\", "/") - return path - - -def convert_to_posix_path(path): - return format_os_path(path, mode=Path.unix) - - -def convert_to_windows_path(path): - return format_os_path(path, mode=Path.windows) - - -def convert_to_platform_path(path): - return format_os_path(path, mode=Path.platform_path) - - def substitute_config_variables(path): """Substitute placeholders into paths. diff --git a/lib/spack/spack/util/string.py b/lib/spack/spack/util/string.py deleted file mode 100644 index 03dc2bb850..0000000000 --- a/lib/spack/spack/util/string.py +++ /dev/null @@ -1,57 +0,0 @@ -# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other -# Spack Project Developers. See the top-level COPYRIGHT file for details. -# -# SPDX-License-Identifier: (Apache-2.0 OR MIT) - - -def comma_list(sequence, article=""): - if type(sequence) is not list: - sequence = list(sequence) - - if not sequence: - return - elif len(sequence) == 1: - return sequence[0] - else: - out = ", ".join(str(s) for s in sequence[:-1]) - if len(sequence) != 2: - out += "," # oxford comma - out += " " - if article: - out += article + " " - out += str(sequence[-1]) - return out - - -def comma_or(sequence): - return comma_list(sequence, "or") - - -def comma_and(sequence): - return comma_list(sequence, "and") - - -def quote(sequence, q="'"): - return ["%s%s%s" % (q, e, q) for e in sequence] - - -def plural(n, singular, plural=None, show_n=True): - """Pluralize word by adding an s if n != 1. - - Arguments: - n (int): number of things there are - singular (str): singular form of word - plural (str or None): optional plural form, for when it's not just - singular + 's' - show_n (bool): whether to include n in the result string (default True) - - Returns: - (str): "1 thing" if n == 1 or "n things" if n != 1 - """ - number = "%s " % n if show_n else "" - if n == 1: - return "%s%s" % (number, singular) - elif plural is not None: - return "%s%s" % (number, plural) - else: - return "%s%ss" % (number, singular) diff --git a/lib/spack/spack/util/url.py b/lib/spack/spack/util/url.py index 9b3c55f332..0644a17ada 100644 --- a/lib/spack/spack/util/url.py +++ b/lib/spack/spack/util/url.py @@ -14,7 +14,9 @@ import sys import urllib.parse import urllib.request -from spack.util.path import convert_to_posix_path, sanitize_filename +from llnl.path import convert_to_posix_path + +from spack.util.path import sanitize_filename def validate_scheme(scheme): diff --git a/lib/spack/spack/variant.py b/lib/spack/spack/variant.py index e0b9a5540b..7b045d6262 100644 --- a/lib/spack/spack/variant.py +++ b/lib/spack/spack/variant.py @@ -15,10 +15,10 @@ import re import llnl.util.lang as lang import llnl.util.tty.color +from llnl.string import comma_or import spack.directives import spack.error as error -from spack.util.string import comma_or special_variant_values = [None, "none", "*"] -- cgit v1.2.3-60-g2f50