diff options
author | Tamara Dahlgren <35777542+tldahlgren@users.noreply.github.com> | 2024-01-31 17:07:16 -0800 |
---|---|---|
committer | GitHub <noreply@github.com> | 2024-02-01 10:07:16 +0900 |
commit | 2fc0d05a5544d0e3e7927ec350af79d7e9f82643 (patch) | |
tree | cb2fdb3ba829f5a2f2ed17c8376307dfa588f78f | |
parent | faf64f1a26fe7de97e8724e98df4ed82df32a354 (diff) | |
download | spack-2fc0d05a5544d0e3e7927ec350af79d7e9f82643.tar.gz spack-2fc0d05a5544d0e3e7927ec350af79d7e9f82643.tar.bz2 spack-2fc0d05a5544d0e3e7927ec350af79d7e9f82643.tar.xz spack-2fc0d05a5544d0e3e7927ec350af79d7e9f82643.zip |
Environments: Add support for including views (#42250)
* Environments: Add support for including views (take 2)
* schema type hint fixes
29 files changed, 590 insertions, 284 deletions
diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py index be6aeea7f0..cc15a7d36e 100644 --- a/lib/spack/spack/config.py +++ b/lib/spack/spack/config.py @@ -63,10 +63,11 @@ from spack.error import SpackError from spack.util.cpus import cpus_available #: Dict from section names -> schema for that section -SECTION_SCHEMAS = { +SECTION_SCHEMAS: Dict[str, Any] = { "compilers": spack.schema.compilers.schema, "concretizer": spack.schema.concretizer.schema, "definitions": spack.schema.definitions.schema, + "view": spack.schema.view.schema, "develop": spack.schema.develop.schema, "mirrors": spack.schema.mirrors.schema, "repos": spack.schema.repos.schema, @@ -81,7 +82,7 @@ SECTION_SCHEMAS = { # Same as above, but including keys for environments # this allows us to unify config reading between configs and environments -_ALL_SCHEMAS = copy.deepcopy(SECTION_SCHEMAS) +_ALL_SCHEMAS: Dict[str, Any] = copy.deepcopy(SECTION_SCHEMAS) _ALL_SCHEMAS.update({spack.schema.env.TOP_LEVEL_KEY: spack.schema.env.schema}) #: Path to the default configuration @@ -1096,7 +1097,7 @@ def read_config_file( data = syaml.load_config(f) if data: - if not schema: + if schema is None: key = next(iter(data)) schema = _ALL_SCHEMAS[key] validate(data, schema) diff --git a/lib/spack/spack/environment/environment.py b/lib/spack/spack/environment/environment.py index 605eaf222c..4320667274 100644 --- a/lib/spack/spack/environment/environment.py +++ b/lib/spack/spack/environment/environment.py @@ -871,9 +871,55 @@ class Environment: else: self.spec_lists[name] = user_specs + def _process_view(self, env_view: Optional[Union[bool, str, Dict]]): + """Process view option(s), which can be boolean, string, or None. + + A boolean environment view option takes precedence over any that may + be included. So ``view: True`` results in the default view only. And + ``view: False`` means the environment will have no view. + + Args: + env_view: view option provided in the manifest or configuration + """ + + def add_view(name, values): + """Add the view with the name and the string or dict values.""" + if isinstance(values, str): + self.views[name] = ViewDescriptor(self.path, values) + elif isinstance(values, dict): + self.views[name] = ViewDescriptor.from_dict(self.path, values) + else: + tty.error(f"Cannot add view named {name} for {type(values)} values {values}") + + # If the configuration specifies 'view: False' then we are done + # processing views. If this is called with the environment's view + # view (versus an included view), then there are to be NO views. + if env_view is False: + return + + # If the configuration specifies 'view: True' then only the default + # view will be created for the environment and we are done processing + # views. + if env_view is True: + add_view(default_view_name, self.view_path_default) + return + + # Otherwise, the configuration has a subdirectory or dictionary. + if isinstance(env_view, str): + add_view(default_view_name, env_view) + elif env_view: + for name, values in env_view.items(): + add_view(name, values) + + # If we reach this point without an explicit view option then we + # provide the default view. + if self.views == dict(): + self.views[default_view_name] = ViewDescriptor(self.path, self.view_path_default) + def _construct_state_from_manifest(self): """Set up user specs and views from the manifest file.""" self.spec_lists = collections.OrderedDict() + self.views = {} for item in spack.config.get("definitions", []): self._process_definition(item) @@ -885,20 +931,7 @@ class Environment: ) self.spec_lists[user_speclist_name] = user_specs - enable_view = env_configuration.get("view") - # enable_view can be boolean, string, or None - if enable_view is True or enable_view is None: - self.views = {default_view_name: ViewDescriptor(self.path, self.view_path_default)} - elif isinstance(enable_view, str): - self.views = {default_view_name: ViewDescriptor(self.path, enable_view)} - elif enable_view: - path = self.path - self.views = dict( - (name, ViewDescriptor.from_dict(path, values)) - for name, values in enable_view.items() - ) - else: - self.views = {} + self._process_view(spack.config.get("view", True)) @property def user_specs(self): diff --git a/lib/spack/spack/schema/bootstrap.py b/lib/spack/spack/schema/bootstrap.py index 0007852d1c..ecd2f0f834 100644 --- a/lib/spack/spack/schema/bootstrap.py +++ b/lib/spack/spack/schema/bootstrap.py @@ -3,16 +3,17 @@ # # SPDX-License-Identifier: (Apache-2.0 OR MIT) """Schema for bootstrap.yaml configuration file.""" +from typing import Any, Dict #: Schema of a single source -_source_schema = { +_source_schema: Dict[str, Any] = { "type": "object", "properties": {"name": {"type": "string"}, "metadata": {"type": "string"}}, "additionalProperties": False, "required": ["name", "metadata"], } -properties = { +properties: Dict[str, Any] = { "bootstrap": { "type": "object", "properties": { diff --git a/lib/spack/spack/schema/buildcache_spec.py b/lib/spack/spack/schema/buildcache_spec.py index e04a8e9519..70aa3a32f1 100644 --- a/lib/spack/spack/schema/buildcache_spec.py +++ b/lib/spack/spack/schema/buildcache_spec.py @@ -6,27 +6,31 @@ """Schema for a buildcache spec.yaml file .. literalinclude:: _spack_root/lib/spack/spack/schema/buildcache_spec.py - :lines: 13- + :lines: 15- """ +from typing import Any, Dict + import spack.schema.spec +properties: Dict[str, Any] = { + # `buildinfo` is no longer needed as of Spack 0.21 + "buildinfo": {"type": "object"}, + "spec": { + "type": "object", + "additionalProperties": True, + "items": spack.schema.spec.properties, + }, + "binary_cache_checksum": { + "type": "object", + "properties": {"hash_algorithm": {"type": "string"}, "hash": {"type": "string"}}, + }, + "buildcache_layout_version": {"type": "number"}, +} + schema = { "$schema": "http://json-schema.org/draft-07/schema#", "title": "Spack buildcache specfile schema", "type": "object", "additionalProperties": False, - "properties": { - # `buildinfo` is no longer needed as of Spack 0.21 - "buildinfo": {"type": "object"}, - "spec": { - "type": "object", - "additionalProperties": True, - "items": spack.schema.spec.properties, - }, - "binary_cache_checksum": { - "type": "object", - "properties": {"hash_algorithm": {"type": "string"}, "hash": {"type": "string"}}, - }, - "buildcache_layout_version": {"type": "number"}, - }, + "properties": properties, } diff --git a/lib/spack/spack/schema/cdash.py b/lib/spack/spack/schema/cdash.py index 42d40a5c3a..5360038f61 100644 --- a/lib/spack/spack/schema/cdash.py +++ b/lib/spack/spack/schema/cdash.py @@ -2,16 +2,15 @@ # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) - """Schema for cdash.yaml configuration file. .. literalinclude:: ../spack/schema/cdash.py :lines: 13- """ - +from typing import Any, Dict #: Properties for inclusion in other schemas -properties = { +properties: Dict[str, Any] = { "cdash": { "type": "object", "additionalProperties": False, diff --git a/lib/spack/spack/schema/ci.py b/lib/spack/spack/schema/ci.py index 9cc9baf385..3706415e30 100644 --- a/lib/spack/spack/schema/ci.py +++ b/lib/spack/spack/schema/ci.py @@ -2,12 +2,12 @@ # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) - """Schema for gitlab-ci.yaml configuration file. .. literalinclude:: ../spack/schema/ci.py - :lines: 13- + :lines: 16- """ +from typing import Any, Dict from llnl.util.lang import union_dicts @@ -164,7 +164,7 @@ ci_properties = { } #: Properties for inclusion in other schemas -properties = { +properties: Dict[str, Any] = { "ci": { "oneOf": [ # TODO: Replace with core-shared-properties in Spack 0.23 diff --git a/lib/spack/spack/schema/compilers.py b/lib/spack/spack/schema/compilers.py index 831ef9ad2b..1df696cda9 100644 --- a/lib/spack/spack/schema/compilers.py +++ b/lib/spack/spack/schema/compilers.py @@ -2,16 +2,17 @@ # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) - """Schema for compilers.yaml configuration file. .. literalinclude:: _spack_root/lib/spack/spack/schema/compilers.py - :lines: 13- + :lines: 15- """ +from typing import Any, Dict + import spack.schema.environment #: Properties for inclusion in other schemas -properties = { +properties: Dict[str, Any] = { "compilers": { "type": "array", "items": { diff --git a/lib/spack/spack/schema/concretizer.py b/lib/spack/spack/schema/concretizer.py index 57f0c06b54..bc9253cb25 100644 --- a/lib/spack/spack/schema/concretizer.py +++ b/lib/spack/spack/schema/concretizer.py @@ -2,14 +2,14 @@ # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) - """Schema for concretizer.yaml configuration file. .. literalinclude:: _spack_root/lib/spack/spack/schema/concretizer.py - :lines: 13- + :lines: 12- """ +from typing import Any, Dict -properties = { +properties: Dict[str, Any] = { "concretizer": { "type": "object", "additionalProperties": False, diff --git a/lib/spack/spack/schema/config.py b/lib/spack/spack/schema/config.py index add66118f7..2a6499058d 100644 --- a/lib/spack/spack/schema/config.py +++ b/lib/spack/spack/schema/config.py @@ -5,15 +5,16 @@ """Schema for config.yaml configuration file. .. literalinclude:: _spack_root/lib/spack/spack/schema/config.py - :lines: 13- + :lines: 17- """ +from typing import Any, Dict from llnl.util.lang import union_dicts import spack.schema.projections #: Properties for inclusion in other schemas -properties = { +properties: Dict[str, Any] = { "config": { "type": "object", "default": {}, diff --git a/lib/spack/spack/schema/container.py b/lib/spack/spack/schema/container.py index 287ed945eb..12b157c5cc 100644 --- a/lib/spack/spack/schema/container.py +++ b/lib/spack/spack/schema/container.py @@ -3,6 +3,7 @@ # # SPDX-License-Identifier: (Apache-2.0 OR MIT) """Schema for the 'container' subsection of Spack environments.""" +from typing import Any, Dict _stages_from_dockerhub = { "type": "object", @@ -85,4 +86,4 @@ container_schema = { }, } -properties = {"container": container_schema} +properties: Dict[str, Any] = {"container": container_schema} diff --git a/lib/spack/spack/schema/cray_manifest.py b/lib/spack/spack/schema/cray_manifest.py index c922bb2c31..ff76ecf915 100644 --- a/lib/spack/spack/schema/cray_manifest.py +++ b/lib/spack/spack/schema/cray_manifest.py @@ -11,112 +11,115 @@ This does not specify a configuration - it is an input format that is consumed and transformed into Spack DB records. """ +from typing import Any, Dict -schema = { - "$schema": "http://json-schema.org/schema#", - "title": "CPE manifest schema", - "type": "object", - "additionalProperties": False, - "properties": { - "_meta": { +properties: Dict[str, Any] = { + "_meta": { + "type": "object", + "additionalProperties": False, + "properties": { + "file-type": {"type": "string", "minLength": 1}, + "cpe-version": {"type": "string", "minLength": 1}, + "system-type": {"type": "string", "minLength": 1}, + "schema-version": {"type": "string", "minLength": 1}, + # Older schemas use did not have "cpe-version", just the + # schema version; in that case it was just called "version" + "version": {"type": "string", "minLength": 1}, + }, + }, + "compilers": { + "type": "array", + "items": { "type": "object", "additionalProperties": False, "properties": { - "file-type": {"type": "string", "minLength": 1}, - "cpe-version": {"type": "string", "minLength": 1}, - "system-type": {"type": "string", "minLength": 1}, - "schema-version": {"type": "string", "minLength": 1}, - # Older schemas use did not have "cpe-version", just the - # schema version; in that case it was just called "version" + "name": {"type": "string", "minLength": 1}, "version": {"type": "string", "minLength": 1}, - }, - }, - "compilers": { - "type": "array", - "items": { - "type": "object", - "additionalProperties": False, - "properties": { - "name": {"type": "string", "minLength": 1}, - "version": {"type": "string", "minLength": 1}, - "prefix": {"type": "string", "minLength": 1}, - "executables": { - "type": "object", - "additionalProperties": False, - "properties": { - "cc": {"type": "string", "minLength": 1}, - "cxx": {"type": "string", "minLength": 1}, - "fc": {"type": "string", "minLength": 1}, - }, + "prefix": {"type": "string", "minLength": 1}, + "executables": { + "type": "object", + "additionalProperties": False, + "properties": { + "cc": {"type": "string", "minLength": 1}, + "cxx": {"type": "string", "minLength": 1}, + "fc": {"type": "string", "minLength": 1}, }, - "arch": { - "type": "object", - "required": ["os", "target"], - "additionalProperties": False, - "properties": { - "os": {"type": "string", "minLength": 1}, - "target": {"type": "string", "minLength": 1}, - }, + }, + "arch": { + "type": "object", + "required": ["os", "target"], + "additionalProperties": False, + "properties": { + "os": {"type": "string", "minLength": 1}, + "target": {"type": "string", "minLength": 1}, }, }, }, }, - "specs": { - "type": "array", - "items": { - "type": "object", - "required": ["name", "version", "arch", "compiler", "prefix", "hash"], - "additionalProperties": False, - "properties": { - "name": {"type": "string", "minLength": 1}, - "version": {"type": "string", "minLength": 1}, - "arch": { - "type": "object", - "required": ["platform", "platform_os", "target"], - "additioanlProperties": False, - "properties": { - "platform": {"type": "string", "minLength": 1}, - "platform_os": {"type": "string", "minLength": 1}, - "target": { - "type": "object", - "additionalProperties": False, - "required": ["name"], - "properties": {"name": {"type": "string", "minLength": 1}}, - }, + }, + "specs": { + "type": "array", + "items": { + "type": "object", + "required": ["name", "version", "arch", "compiler", "prefix", "hash"], + "additionalProperties": False, + "properties": { + "name": {"type": "string", "minLength": 1}, + "version": {"type": "string", "minLength": 1}, + "arch": { + "type": "object", + "required": ["platform", "platform_os", "target"], + "additionalProperties": False, + "properties": { + "platform": {"type": "string", "minLength": 1}, + "platform_os": {"type": "string", "minLength": 1}, + "target": { + "type": "object", + "additionalProperties": False, + "required": ["name"], + "properties": {"name": {"type": "string", "minLength": 1}}, }, }, - "compiler": { - "type": "object", - "required": ["name", "version"], - "additionalProperties": False, - "properties": { - "name": {"type": "string", "minLength": 1}, - "version": {"type": "string", "minLength": 1}, - }, + }, + "compiler": { + "type": "object", + "required": ["name", "version"], + "additionalProperties": False, + "properties": { + "name": {"type": "string", "minLength": 1}, + "version": {"type": "string", "minLength": 1}, }, - "dependencies": { - "type": "object", - "patternProperties": { - "\\w[\\w-]*": { - "type": "object", - "required": ["hash"], - "additionalProperties": False, - "properties": { - "hash": {"type": "string", "minLength": 1}, - "type": { - "type": "array", - "items": {"type": "string", "minLength": 1}, - }, + }, + "dependencies": { + "type": "object", + "patternProperties": { + "\\w[\\w-]*": { + "type": "object", + "required": ["hash"], + "additionalProperties": False, + "properties": { + "hash": {"type": "string", "minLength": 1}, + "type": { + "type": "array", + "items": {"type": "string", "minLength": 1}, }, - } - }, + }, + } }, - "prefix": {"type": "string", "minLength": 1}, - "rpm": {"type": "string", "minLength": 1}, - "hash": {"type": "string", "minLength": 1}, - "parameters": {"type": "object"}, }, + "prefix": {"type": "string", "minLength": 1}, + "rpm": {"type": "string", "minLength": 1}, + "hash": {"type": "string", "minLength": 1}, + "parameters": {"type": "object"}, }, }, }, } + +schema = { + "$schema": "http://json-schema.org/schema#", + "title": "CPE manifest schema", + "type": "object", + "additionalProperties": False, + "properties": properties, +} diff --git a/lib/spack/spack/schema/database_index.py b/lib/spack/spack/schema/database_index.py index 4b25b415a3..eaa0d2ece9 100644 --- a/lib/spack/spack/schema/database_index.py +++ b/lib/spack/spack/schema/database_index.py @@ -6,12 +6,41 @@ """Schema for database index.json file .. literalinclude:: _spack_root/lib/spack/spack/schema/database_index.py - :lines: 36- + :lines: 17- """ +from typing import Any, Dict + import spack.schema.spec # spack.schema.spec.properties +properties: Dict[str, Any] = { + "database": { + "type": "object", + "required": ["installs", "version"], + "additionalProperties": False, + "properties": { + "installs": { + "type": "object", + "patternProperties": { + r"^[\w\d]{32}$": { + "type": "object", + "properties": { + "spec": spack.schema.spec.properties, + "path": {"oneOf": [{"type": "string"}, {"type": "null"}]}, + "installed": {"type": "boolean"}, + "ref_count": {"type": "integer", "minimum": 0}, + "explicit": {"type": "boolean"}, + "installation_time": {"type": "number"}, + }, + } + }, + }, + "version": {"type": "string"}, + }, + } +} + #: Full schema with metadata schema = { "$schema": "http://json-schema.org/draft-07/schema#", @@ -19,30 +48,5 @@ schema = { "type": "object", "required": ["database"], "additionalProperties": False, - "properties": { - "database": { - "type": "object", - "required": ["installs", "version"], - "additionalProperties": False, - "properties": { - "installs": { - "type": "object", - "patternProperties": { - r"^[\w\d]{32}$": { - "type": "object", - "properties": { - "spec": spack.schema.spec.properties, - "path": {"oneOf": [{"type": "string"}, {"type": "null"}]}, - "installed": {"type": "boolean"}, - "ref_count": {"type": "integer", "minimum": 0}, - "explicit": {"type": "boolean"}, - "installation_time": {"type": "number"}, - }, - } - }, - }, - "version": {"type": "string"}, - }, - } - }, + "properties": properties, } diff --git a/lib/spack/spack/schema/definitions.py b/lib/spack/spack/schema/definitions.py index 81579811b2..1f8b8b4833 100644 --- a/lib/spack/spack/schema/definitions.py +++ b/lib/spack/spack/schema/definitions.py @@ -6,13 +6,14 @@ """Schema for definitions .. literalinclude:: _spack_root/lib/spack/spack/schema/definitions.py - :lines: 13- + :lines: 16- """ +from typing import Any, Dict import spack.schema #: Properties for inclusion in other schemas -properties = { +properties: Dict[str, Any] = { "definitions": { "type": "array", "default": [], diff --git a/lib/spack/spack/schema/develop.py b/lib/spack/spack/schema/develop.py index 7fa2ec5b07..13391dcdb0 100644 --- a/lib/spack/spack/schema/develop.py +++ b/lib/spack/spack/schema/develop.py @@ -2,9 +2,9 @@ # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) +from typing import Any, Dict - -properties = { +properties: Dict[str, Any] = { "develop": { "type": "object", "default": {}, diff --git a/lib/spack/spack/schema/env.py b/lib/spack/spack/schema/env.py index b5f1294722..676fbeb277 100644 --- a/lib/spack/spack/schema/env.py +++ b/lib/spack/spack/schema/env.py @@ -6,8 +6,10 @@ """Schema for env.yaml configuration file. .. literalinclude:: _spack_root/lib/spack/spack/schema/env.py - :lines: 36- + :lines: 19- """ +from typing import Any, Dict + from llnl.util.lang import union_dicts import spack.schema.gitlab_ci # DEPRECATED @@ -19,61 +21,31 @@ TOP_LEVEL_KEY = "spack" projections_scheme = spack.schema.projections.properties["projections"] +properties: Dict[str, Any] = { + "spack": { + "type": "object", + "default": {}, + "additionalProperties": False, + "properties": union_dicts( + # Include deprecated "gitlab-ci" section + spack.schema.gitlab_ci.properties, + # merged configuration scope schemas + spack.schema.merged.properties, + # extra environment schema properties + { + "include": {"type": "array", "default": [], "items": {"type": "string"}}, + "specs": spack.schema.spec_list_schema, + }, + ), + } +} + schema = { "$schema": "http://json-schema.org/draft-07/schema#", "title": "Spack environment file schema", "type": "object", "additionalProperties": False, - "properties": { - "spack": { - "type": "object", - "default": {}, - "additionalProperties": False, - "properties": union_dicts( - # Include deprecated "gitlab-ci" section - spack.schema.gitlab_ci.properties, - # merged configuration scope schemas - spack.schema.merged.properties, - # extra environment schema properties - { - "include": {"type": "array", "default": [], "items": {"type": "string"}}, - "specs": spack.schema.spec_list_schema, - "view": { - "anyOf": [ - {"type": "boolean"}, - {"type": "string"}, - { - "type": "object", - "patternProperties": { - r"\w+": { - "required": ["root"], - "additionalProperties": False, - "properties": { - "root": {"type": "string"}, - "link": { - "type": "string", - "pattern": "(roots|all|run)", - }, - "link_type": {"type": "string"}, - "select": { - "type": "array", - "items": {"type": "string"}, - }, - "exclude": { - "type": "array", - "items": {"type": "string"}, - }, - "projections": projections_scheme, - }, - } - }, - }, - ] - }, - }, - ), - } - }, + "properties": properties, } diff --git a/lib/spack/spack/schema/environment.py b/lib/spack/spack/schema/environment.py index 1c5070092e..193a07b657 100644 --- a/lib/spack/spack/schema/environment.py +++ b/lib/spack/spack/schema/environment.py @@ -6,6 +6,7 @@ schemas. """ import collections.abc +from typing import Any, Dict array_of_strings_or_num = { "type": "array", @@ -18,7 +19,7 @@ dictionary_of_strings_or_num = { "patternProperties": {r"\w[\w-]*": {"anyOf": [{"type": "string"}, {"type": "number"}]}}, } -definition = { +definition: Dict[str, Any] = { "type": "object", "default": {}, "additionalProperties": False, diff --git a/lib/spack/spack/schema/gitlab_ci.py b/lib/spack/spack/schema/gitlab_ci.py index 85acd9cc51..a180777aca 100644 --- a/lib/spack/spack/schema/gitlab_ci.py +++ b/lib/spack/spack/schema/gitlab_ci.py @@ -6,8 +6,9 @@ """Schema for gitlab-ci.yaml configuration file. .. literalinclude:: ../spack/schema/gitlab_ci.py - :lines: 13- + :lines: 15- """ +from typing import Any, Dict from llnl.util.lang import union_dicts @@ -112,7 +113,7 @@ gitlab_ci_properties = { } #: Properties for inclusion in other schemas -properties = {"gitlab-ci": gitlab_ci_properties} +properties: Dict[str, Any] = {"gitlab-ci": gitlab_ci_properties} #: Full schema with metadata schema = { diff --git a/lib/spack/spack/schema/merged.py b/lib/spack/spack/schema/merged.py index f4d70f8241..a883f5af62 100644 --- a/lib/spack/spack/schema/merged.py +++ b/lib/spack/spack/schema/merged.py @@ -6,8 +6,10 @@ """Schema for configuration merged into one file. .. literalinclude:: _spack_root/lib/spack/spack/schema/merged.py - :lines: 39- + :lines: 32- """ +from typing import Any, Dict + from llnl.util.lang import union_dicts import spack.schema.bootstrap @@ -24,9 +26,10 @@ import spack.schema.modules import spack.schema.packages import spack.schema.repos import spack.schema.upstreams +import spack.schema.view #: Properties for inclusion in other schemas -properties = union_dicts( +properties: Dict[str, Any] = union_dicts( spack.schema.bootstrap.properties, spack.schema.cdash.properties, spack.schema.compilers.properties, @@ -41,6 +44,7 @@ properties = union_dicts( spack.schema.packages.properties, spack.schema.repos.properties, spack.schema.upstreams.properties, + spack.schema.view.properties, ) diff --git a/lib/spack/spack/schema/mirrors.py b/lib/spack/spack/schema/mirrors.py index 13ed1c746e..c3959374a7 100644 --- a/lib/spack/spack/schema/mirrors.py +++ b/lib/spack/spack/schema/mirrors.py @@ -6,8 +6,9 @@ """Schema for mirrors.yaml configuration file. .. literalinclude:: _spack_root/lib/spack/spack/schema/mirrors.py - :lines: 12-69 + :lines: 13- """ +from typing import Any, Dict #: Common properties for connection specification connection = { @@ -50,7 +51,7 @@ mirror_entry = { } #: Properties for inclusion in other schemas -properties = { +properties: Dict[str, Any] = { "mirrors": { "type": "object", "default": {}, diff --git a/lib/spack/spack/schema/modules.py b/lib/spack/spack/schema/modules.py index 48db98c485..02c8f757f2 100644 --- a/lib/spack/spack/schema/modules.py +++ b/lib/spack/spack/schema/modules.py @@ -6,8 +6,10 @@ """Schema for modules.yaml configuration file. .. literalinclude:: _spack_root/lib/spack/spack/schema/modules.py - :lines: 13- + :lines: 16- """ +from typing import Any, Dict + import spack.schema.environment import spack.schema.projections @@ -141,7 +143,7 @@ module_config_properties = { # Properties for inclusion into other schemas (requires definitions) -properties = { +properties: Dict[str, Any] = { "modules": { "type": "object", "additionalProperties": False, diff --git a/lib/spack/spack/schema/packages.py b/lib/spack/spack/schema/packages.py index 365536990e..492180f70e 100644 --- a/lib/spack/spack/schema/packages.py +++ b/lib/spack/spack/schema/packages.py @@ -5,8 +5,10 @@ """Schema for packages.yaml configuration files. .. literalinclude:: _spack_root/lib/spack/spack/schema/packages.py - :lines: 13- + :lines: 14- """ +from typing import Any, Dict + import spack.schema.environment permissions = { @@ -91,7 +93,7 @@ package_attributes = { REQUIREMENT_URL = "https://spack.readthedocs.io/en/latest/packages_yaml.html#package-requirements" #: Properties for inclusion in other schemas -properties = { +properties: Dict[str, Any] = { "packages": { "type": "object", "default": {}, diff --git a/lib/spack/spack/schema/projections.py b/lib/spack/spack/schema/projections.py index 00f28d8bf9..60f956bba6 100644 --- a/lib/spack/spack/schema/projections.py +++ b/lib/spack/spack/schema/projections.py @@ -6,12 +6,12 @@ """Schema for projections.yaml configuration file. .. literalinclude:: _spack_root/lib/spack/spack/schema/projections.py - :lines: 13- + :lines: 14- """ - +from typing import Any, Dict #: Properties for inclusion in other schemas -properties = { +properties: Dict[str, Any] = { "projections": {"type": "object", "patternProperties": {r"all|\w[\w-]*": {"type": "string"}}} } diff --git a/lib/spack/spack/schema/repos.py b/lib/spack/spack/schema/repos.py index 46f775fb4e..654efe0f00 100644 --- a/lib/spack/spack/schema/repos.py +++ b/lib/spack/spack/schema/repos.py @@ -6,12 +6,14 @@ """Schema for repos.yaml configuration file. .. literalinclude:: _spack_root/lib/spack/spack/schema/repos.py - :lines: 13- + :lines: 14- """ - +from typing import Any, Dict #: Properties for inclusion in other schemas -properties = {"repos": {"type": "array", "default": [], "items": {"type": "string"}}} +properties: Dict[str, Any] = { + "repos": {"type": "array", "default": [], "items": {"type": "string"}} +} #: Full schema with metadata diff --git a/lib/spack/spack/schema/spack.py b/lib/spack/spack/schema/spack.py new file mode 100644 index 0000000000..7d5a13c17e --- /dev/null +++ b/lib/spack/spack/schema/spack.py @@ -0,0 +1,46 @@ +# Copyright 2013-2024 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +"""Schema for spack environment + +.. literalinclude:: _spack_root/lib/spack/spack/schema/spack.py + :lines: 20- +""" +from typing import Any, Dict + +from llnl.util.lang import union_dicts + +import spack.schema +import spack.schema.gitlab_ci as ci_schema # DEPRECATED +import spack.schema.merged as merged_schema + +#: Properties for inclusion in other schemas +properties: Dict[str, Any] = { + "spack": { + "type": "object", + "default": {}, + "additionalProperties": False, + "properties": union_dicts( + # Include deprecated "gitlab-ci" section + ci_schema.properties, + # merged configuration scope schemas + merged_schema.properties, + # extra environment schema properties + { + "include": {"type": "array", "default": [], "items": {"type": "string"}}, + "specs": spack.schema.spec_list_schema, + }, + ), + } +} + +#: Full schema with metadata +schema = { + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Spack environment file schema", + "type": "object", + "additionalProperties": False, + "properties": properties, +} diff --git a/lib/spack/spack/schema/spec.py b/lib/spack/spack/schema/spec.py index 3cbb8a65d2..5ae59d9ffa 100644 --- a/lib/spack/spack/schema/spec.py +++ b/lib/spack/spack/schema/spec.py @@ -8,9 +8,9 @@ TODO: This needs to be updated? Especially the hashes under properties. .. literalinclude:: _spack_root/lib/spack/spack/schema/spec.py - :lines: 13- + :lines: 15- """ - +from typing import Any, Dict target = { "oneOf": [ @@ -57,7 +57,7 @@ build_spec = { } #: Properties for inclusion in other schemas -properties = { +properties: Dict[str, Any] = { "spec": { "type": "object", "additionalProperties": False, diff --git a/lib/spack/spack/schema/upstreams.py b/lib/spack/spack/schema/upstreams.py index eee9d050ba..6ce313ff89 100644 --- a/lib/spack/spack/schema/upstreams.py +++ b/lib/spack/spack/schema/upstreams.py @@ -2,10 +2,10 @@ # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) - +from typing import Any, Dict #: Properties for inclusion in other schemas -properties = { +properties: Dict[str, Any] = { "upstreams": { "type": "object", "default": {}, diff --git a/lib/spack/spack/schema/view.py b/lib/spack/spack/schema/view.py new file mode 100644 index 0000000000..6c24501ba9 --- /dev/null +++ b/lib/spack/spack/schema/view.py @@ -0,0 +1,49 @@ +# Copyright 2013-2024 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +"""Schema for view + +.. literalinclude:: _spack_root/lib/spack/spack/schema/view.py + :lines: 15- +""" +from typing import Any, Dict + +import spack.schema + +projections_scheme = spack.schema.projections.properties["projections"] + +#: Properties for inclusion in other schemas +properties: Dict[str, Any] = { + "view": { + "anyOf": [ + {"type": "boolean"}, + {"type": "string"}, + { + "type": "object", + "patternProperties": { + r"\w+": { + "required": ["root"], + "additionalProperties": False, + "properties": { + "root": {"type": "string"}, + "link": {"type": "string", "pattern": "(roots|all|run)"}, + "link_type": {"type": "string"}, + "select": {"type": "array", "items": {"type": "string"}}, + "exclude": {"type": "array", "items": {"type": "string"}}, + "projections": projections_scheme, + }, + } + }, + }, + ] + } +} + +#: Full schema with metadata +schema = { + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Spack view configuration file schema", + "properties": properties, +} diff --git a/lib/spack/spack/test/cmd/env.py b/lib/spack/spack/test/cmd/env.py index c9137e6aea..cfb167320e 100644 --- a/lib/spack/spack/test/cmd/env.py +++ b/lib/spack/spack/test/cmd/env.py @@ -2537,58 +2537,88 @@ spack: assert viewdir not in shell +@pytest.mark.parametrize("include_views", [True, False, "split"]) def test_stack_view_multiple_views( - tmpdir, mock_fetch, mock_packages, mock_archive, install_mockery + tmp_path, + mock_fetch, + mock_packages, + mock_archive, + install_mockery, + mutable_config, + include_views, ): - filename = str(tmpdir.join("spack.yaml")) - default_viewdir = str(tmpdir.join("default-view")) - combin_viewdir = str(tmpdir.join("combinatorial-view")) - with open(filename, "w") as f: - f.write( - """\ -spack: + """Test multiple views as both included views (True), as both environment + views (False), or as one included and the other in the environment.""" + # Write the view configuration and or manifest file + view_filename = tmp_path / "view.yaml" + base_content = """\ definitions: - packages: [mpileaks, cmake] - - compilers: ['%%gcc', '%%clang'] + - compilers: ['%gcc', '%clang'] specs: - matrix: - [$packages] - [$compilers] +""" - view: - default: - root: %s - select: ['%%gcc'] - combinatorial: - root: %s - exclude: [callpath %%gcc] - projections: - 'all': '{name}/{version}-{compiler.name}'""" - % (default_viewdir, combin_viewdir) - ) - with tmpdir.as_cwd(): - env("create", "test", "./spack.yaml") - with ev.read("test"): - install() + include_content = f" include:\n - {view_filename}\n" + view_line = " view:\n" - shell = env("activate", "--sh", "test") - assert "PATH" in shell - assert os.path.join(default_viewdir, "bin") in shell + comb_dir = tmp_path / "combinatorial-view" + comb_view = """\ +{0}combinatorial: +{0} root: {1} +{0} exclude: [callpath%gcc] +{0} projections: +""" - test = ev.read("test") - for spec in test._get_environment_specs(): + projection = " 'all': '{name}/{version}-{compiler.name}'" + + default_dir = tmp_path / "default-view" + default_view = """\ +{0}default: +{0} root: {1} +{0} select: ['%gcc'] +""" + + content = "spack:\n" + indent = " " + if include_views is True: + # Include both the gcc and combinatorial views + view = "view:\n" + default_view.format(indent, str(default_dir)) + view += comb_view.format(indent, str(comb_dir)) + indent + projection + view_filename.write_text(view) + content += include_content + base_content + elif include_views == "split": + # Include the gcc view and inline the combinatorial view + view = "view:\n" + default_view.format(indent, str(default_dir)) + view_filename.write_text(view) + content += include_content + base_content + view_line + indent += " " + content += comb_view.format(indent, str(comb_dir)) + indent + projection + else: + # Inline both the gcc and combinatorial views in the environment. + indent += " " + content += base_content + view_line + content += default_view.format(indent, str(default_dir)) + content += comb_view.format(indent, str(comb_dir)) + indent + projection + + filename = tmp_path / ev.manifest_name + filename.write_text(content) + + env("create", "test", str(filename)) + with ev.read("test"): + install() + + with ev.read("test") as e: + assert os.path.exists(str(default_dir / "bin")) + for spec in e._get_environment_specs(): + spec_subdir = f"{spec.version}-{spec.compiler.name}" + comb_spec_dir = str(comb_dir / spec.name / spec_subdir) if not spec.satisfies("callpath%gcc"): - assert os.path.exists( - os.path.join( - combin_viewdir, spec.name, "%s-%s" % (spec.version, spec.compiler.name) - ) - ) + assert os.path.exists(comb_spec_dir) else: - assert not os.path.exists( - os.path.join( - combin_viewdir, spec.name, "%s-%s" % (spec.version, spec.compiler.name) - ) - ) + assert not os.path.exists(comb_spec_dir) def test_env_activate_sh_prints_shell_output(tmpdir, mock_stage, mock_fetch, install_mockery): @@ -3714,3 +3744,150 @@ def test_environment_created_from_lockfile_has_view(mock_packages, temporary_sto # Make sure the view was created with ev.Environment(env_b) as e: assert os.path.isdir(e.view_path_default) + + +def test_env_view_disabled(tmp_path, mutable_mock_env_path): + """Ensure an inlined view being disabled means not even the default view + is created (since the case doesn't appear to be covered in this module).""" + spack_yaml = tmp_path / ev.manifest_name + spack_yaml.write_text( + """\ +spack: + specs: + - mpileaks + view: false +""" + ) + env("create", "disabled", str(spack_yaml)) + with ev.read("disabled") as e: + e.concretize() + + assert len(e.views) == 0 + assert not os.path.exists(e.view_path_default) + + +@pytest.mark.parametrize("first", ["false", "true", "custom"]) +def test_env_include_mixed_views(tmp_path, mutable_mock_env_path, mutable_config, first): + """Ensure including path and boolean views in different combinations result + in the creation of only the first view if it is not disabled.""" + false_yaml = tmp_path / "false-view.yaml" + false_yaml.write_text("view: false\n") + + true_yaml = tmp_path / "true-view.yaml" + true_yaml.write_text("view: true\n") + + custom_name = "my-test-view" + custom_view = tmp_path / custom_name + custom_yaml = tmp_path / "custom-view.yaml" + custom_yaml.write_text( + f""" +view: + {custom_name}: + root: {custom_view} +""" + ) + + if first == "false": + order = [false_yaml, true_yaml, custom_yaml] + elif first == "true": + order = [true_yaml, custom_yaml, false_yaml] + else: + order = [custom_yaml, false_yaml, true_yaml] + includes = [f" - {yaml}\n" for yaml in order] + + spack_yaml = tmp_path / ev.manifest_name + spack_yaml.write_text( + f"""\ +spack: + include: +{''.join(includes)} + specs: + - mpileaks + packages: + mpileaks: + compiler: [gcc] +""" + ) + + env("create", "test", str(spack_yaml)) + with ev.read("test") as e: + concretize() + + # Only the first included view should be created if view not disabled by it + assert len(e.views) == 0 if first == "false" else 1 + if first == "true": + assert os.path.exists(e.view_path_default) + else: + assert not os.path.exists(e.view_path_default) + + if first == "custom": + assert os.path.exists(custom_view) + else: + assert not os.path.exists(custom_view) + + +def test_stack_view_multiple_views_same_name( + tmp_path, mock_fetch, mock_packages, mock_archive, install_mockery, mutable_config +): + """Test multiple views with the same name combine settings with precedence + given to the options in spack.yaml.""" + # Write the view configuration and or manifest file + + view_filename = tmp_path / "view.yaml" + default_dir = tmp_path / "default-view" + default_view = f"""\ +view: + default: + root: {default_dir} + select: ['%gcc'] + projections: + all: '{{name}}/{{version}}-{{compiler.name}}' +""" + view_filename.write_text(default_view) + + view_dir = tmp_path / "view" + content = f"""\ +spack: + include: + - {view_filename} + definitions: + - packages: [mpileaks, cmake] + - compilers: ['%gcc', '%clang'] + specs: + - matrix: + - [$packages] + - [$compilers] + + view: + default: + root: {view_dir} + exclude: ['cmake'] + projections: + all: '{{name}}/{{compiler.name}}-{{version}}' +""" + + filename = tmp_path / ev.manifest_name + filename.write_text(content) + + env("create", "test", str(filename)) + with ev.read("test"): + install() + + with ev.read("test") as e: + # the view root in the included view should NOT exist + assert not os.path.exists(str(default_dir)) + + for spec in e._get_environment_specs(): + # no specs will exist in the included view projection + included_spec_subdir = f"{spec.version}-{spec.compiler.name}" + included_spec_dir = str(view_dir / spec.name / included_spec_subdir) + assert not os.path.exists(included_spec_dir) + + # only specs compiled with %gcc (selected in the included view) that + # are also not cmake (excluded in the environment view) should exist + env_spec_subdir = f"{spec.compiler.name}-{spec.version}" + env_spec_dir = str(view_dir / spec.name / env_spec_subdir) + if spec.satisfies("cmake") or spec.satisfies("%clang"): + assert not os.path.exists(env_spec_dir) + else: + assert os.path.exists(env_spec_dir) diff --git a/share/spack/spack-completion.fish b/share/spack/spack-completion.fish index 52def8de23..b2c0a89362 100755 --- a/share/spack/spack-completion.fish +++ b/share/spack/spack-completion.fish @@ -1179,19 +1179,19 @@ complete -c spack -n '__fish_spack_using_command config' -l scope -r -d 'configu # spack config get set -g __fish_spack_optspecs_spack_config_get h/help -complete -c spack -n '__fish_spack_using_command_pos 0 config get' -f -a 'bootstrap cdash ci compilers concretizer config definitions develop mirrors modules packages repos upstreams' +complete -c spack -n '__fish_spack_using_command_pos 0 config get' -f -a 'bootstrap cdash ci compilers concretizer config definitions develop mirrors modules packages repos upstreams view' complete -c spack -n '__fish_spack_using_command config get' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command config get' -s h -l help -d 'show this help message and exit' # spack config blame set -g __fish_spack_optspecs_spack_config_blame h/help -complete -c spack -n '__fish_spack_using_command_pos 0 config blame' -f -a 'bootstrap cdash ci compilers concretizer config definitions develop mirrors modules packages repos upstreams' +complete -c spack -n '__fish_spack_using_command_pos 0 config blame' -f -a 'bootstrap cdash ci compilers concretizer config definitions develop mirrors modules packages repos upstreams view' complete -c spack -n '__fish_spack_using_command config blame' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command config blame' -s h -l help -d 'show this help message and exit' # spack config edit set -g __fish_spack_optspecs_spack_config_edit h/help print-file -complete -c spack -n '__fish_spack_using_command_pos 0 config edit' -f -a 'bootstrap cdash ci compilers concretizer config definitions develop mirrors modules packages repos upstreams' +complete -c spack -n '__fish_spack_using_command_pos 0 config edit' -f -a 'bootstrap cdash ci compilers concretizer config definitions develop mirrors modules packages repos upstreams view' complete -c spack -n '__fish_spack_using_command config edit' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command config edit' -s h -l help -d 'show this help message and exit' complete -c spack -n '__fish_spack_using_command config edit' -l print-file -f -a print_file |