From 85dc20cb555b67410a7eaa324f3dc34e60a8e4db Mon Sep 17 00:00:00 2001 From: Chuck Atkins Date: Fri, 17 Jun 2022 10:29:08 -0400 Subject: Spec: Add a new virtual-customizable `home` attribute (#30917) * Spec: Add a new virtual-customizable home attribute * java: Use the new builtin home attribute * python: Use the new builtin home attribute --- lib/spack/docs/packaging_guide.rst | 268 +++++++++++++++++++++++++++++++++++++ lib/spack/spack/package_base.py | 4 + lib/spack/spack/spec.py | 24 ++-- lib/spack/spack/test/install.py | 44 ++++++ 4 files changed, 331 insertions(+), 9 deletions(-) (limited to 'lib') diff --git a/lib/spack/docs/packaging_guide.rst b/lib/spack/docs/packaging_guide.rst index dcc8f73b5d..3764a54e4b 100644 --- a/lib/spack/docs/packaging_guide.rst +++ b/lib/spack/docs/packaging_guide.rst @@ -2794,6 +2794,256 @@ Suppose a user invokes ``spack install`` like this: Spack will fail with a constraint violation, because the version of MPICH requested is too low for the ``mpi`` requirement in ``foo``. +.. _custom-attributes: + +------------------ +Custom attributes +------------------ + +Often a package will need to provide attributes for dependents to query +various details about what it provides. While any number of custom defined +attributes can be implemented by a package, the four specific attributes +described below are always available on every package with default +implementations and the ability to customize with alternate implementations +in the case of virtual packages provided: + +=========== =========================================== ===================== +Attribute Purpose Default +=========== =========================================== ===================== +``home`` The installation path for the package ``spec.prefix`` +``command`` An executable command for the package | ``spec.name`` found + in + | ``.home.bin`` +``headers`` A list of headers provided by the package | All headers + searched + | recursively in + ``.home.include`` +``libs`` A list of libraries provided by the package | ``lib{spec.name}`` + searched + | recursively in + ``.home`` starting + | with ``lib``, + ``lib64``, then the + | rest of ``.home`` +=========== =========================================== ===================== + +Each of these can be customized by implementing the relevant attribute +as a ``@property`` in the package's class: + +.. code-block:: python + :linenos: + + class Foo(Package): + ... + @property + def libs(self): + # The library provided by Foo is libMyFoo.so + return find_libraries('libMyFoo', root=self.home, recursive=True) + +A package may also provide a custom implementation of each attribute +for the virtual packages it provides by implementing the +``virtualpackagename_attributename`` property in the package's class. +The implementation used is the first one found from: + +#. Specialized virtual: ``Package.virtualpackagename_attributename`` +#. Generic package: ``Package.attributename`` +#. Default + +The use of customized attributes is demonstrated in the next example. + +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Example: Customized attributes for virtual packages +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Consider a package ``foo`` that can optionally provide two virtual +packages ``bar`` and ``baz``. When both are enabled the installation tree +appears as follows: + +.. code-block:: console + + include/foo.h + include/bar/bar.h + lib64/libFoo.so + lib64/libFooBar.so + baz/include/baz/baz.h + baz/lib/libFooBaz.so + +The install tree shows that ``foo`` is providing the header ``include/foo.h`` +and library ``lib64/libFoo.so`` in it's install prefix. The virtual +package ``bar`` is providing ``include/bar/bar.h`` and library +``lib64/libFooBar.so``, also in ``foo``'s install prefix. The ``baz`` +package, however, is provided in the ``baz`` subdirectory of ``foo``'s +prefix with the ``include/baz/baz.h`` header and ``lib/libFooBaz.so`` +library. Such a package could implement the optional attributes as +follows: + +.. code-block:: python + :linenos: + + class Foo(Package): + ... + variant('bar', default=False, description='Enable the Foo implementation of bar') + variant('baz', default=False, description='Enable the Foo implementation of baz') + ... + provides('bar', when='+bar') + provides('baz', when='+baz') + .... + + # Just the foo headers + @property + def headers(self): + return find_headers('foo', root=self.home.include, recursive=False) + + # Just the foo libraries + @property + def libs(self): + return find_libraries('libFoo', root=self.home, recursive=True) + + # The header provided by the bar virutal package + @property + def bar_headers(self): + return find_headers('bar/bar.h', root=self.home.include, recursive=False) + + # The libary provided by the bar virtual package + @property + def bar_libs(self): + return find_libraries('libFooBar', root=sef.home, recursive=True) + + # The baz virtual package home + @property + def baz_home(self): + return self.prefix.baz + + # The header provided by the baz virtual package + @property + def baz_headers(self): + return find_headers('baz/baz', root=self.baz_home.include, recursive=False) + + # The library provided by the baz virtual package + @property + def baz_libs(self): + return find_libraries('libFooBaz', root=self.baz_home, recursive=True) + +Now consider another package, ``foo-app``, depending on all three: + +.. code-block:: python + :linenos: + + class FooApp(CMakePackage): + ... + depends_on('foo') + depends_on('bar') + depends_on('baz') + +The resulting spec objects for it's dependencies shows the result of +the above attribute implementations: + +.. code-block:: python + + # The core headers and libraries of the foo package + + >>> spec['foo'] + foo@1.0%gcc@11.3.1+bar+baz arch=linux-fedora35-haswell + >>> spec['foo'].prefix + '/opt/spack/linux-fedora35-haswell/gcc-11.3.1/foo-1.0-ca3rczp5omy7dfzoqw4p7oc2yh3u7lt6' + + # home defaults to the package install prefix without an explicit implementation + >>> spec['foo'].home + '/opt/spack/linux-fedora35-haswell/gcc-11.3.1/foo-1.0-ca3rczp5omy7dfzoqw4p7oc2yh3u7lt6' + + # foo headers from the foo prefix + >>> spec['foo'].headers + HeaderList([ + '/opt/spack/linux-fedora35-haswell/gcc-11.3.1/foo-1.0-ca3rczp5omy7dfzoqw4p7oc2yh3u7lt6/include/foo.h', + ]) + + # foo include directories from the foo prefix + >>> spec['foo'].headers.directories + ['/opt/spack/linux-fedora35-haswell/gcc-11.3.1/foo-1.0-ca3rczp5omy7dfzoqw4p7oc2yh3u7lt6/include'] + + # foo libraries from the foo prefix + >>> spec['foo'].libs + LibraryList([ + '/opt/spack/linux-fedora35-haswell/gcc-11.3.1/foo-1.0-ca3rczp5omy7dfzoqw4p7oc2yh3u7lt6/lib64/libFoo.so', + ]) + + # foo library directories from the foo prefix + >>> spec['foo'].libs.directories + ['/opt/spack/linux-fedora35-haswell/gcc-11.3.1/foo-1.0-ca3rczp5omy7dfzoqw4p7oc2yh3u7lt6/lib64'] + +.. code-block:: python + + # The virtual bar package in the same prefix as foo + + # bar resolves to the foo package + >>> spec['bar'] + foo@1.0%gcc@11.3.1+bar+baz arch=linux-fedora35-haswell + >>> spec['bar'].prefix + '/opt/spack/linux-fedora35-haswell/gcc-11.3.1/foo-1.0-ca3rczp5omy7dfzoqw4p7oc2yh3u7lt6' + + # home defaults to the foo prefix without either a Foo.bar_home + # or Foo.home implementation + >>> spec['bar'].home + '/opt/spack/linux-fedora35-haswell/gcc-11.3.1/foo-1.0-ca3rczp5omy7dfzoqw4p7oc2yh3u7lt6' + + # bar header in the foo prefix + >>> spec['bar'].headers + HeaderList([ + '/opt/spack/linux-fedora35-haswell/gcc-11.3.1/foo-1.0-ca3rczp5omy7dfzoqw4p7oc2yh3u7lt6/include/bar/bar.h' + ]) + + # bar include dirs from the foo prefix + >>> spec['bar'].headers.directories + ['/opt/spack/linux-fedora35-haswell/gcc-11.3.1/foo-1.0-ca3rczp5omy7dfzoqw4p7oc2yh3u7lt6/include'] + + # bar library from the foo prefix + >>> spec['bar'].libs + LibraryList([ + '/opt/spack/linux-fedora35-haswell/gcc-11.3.1/foo-1.0-ca3rczp5omy7dfzoqw4p7oc2yh3u7lt6/lib64/libFooBar.so' + ]) + + # bar library directories from the foo prefix + >>> spec['bar'].libs.directories + ['/opt/spack/linux-fedora35-haswell/gcc-11.3.1/foo-1.0-ca3rczp5omy7dfzoqw4p7oc2yh3u7lt6/lib64'] + +.. code-block:: python + + # The virtual baz package in a subdirectory of foo's prefix + + # baz resolves to the foo package + >>> spec['baz'] + foo@1.0%gcc@11.3.1+bar+baz arch=linux-fedora35-haswell + >>> spec['baz'].prefix + '/opt/spack/linux-fedora35-haswell/gcc-11.3.1/foo-1.0-ca3rczp5omy7dfzoqw4p7oc2yh3u7lt6' + + # baz_home implementation provides the subdirectory inside the foo prefix + >>> spec['baz'].home + '/opt/spack/linux-fedora35-haswell/gcc-11.3.1/foo-1.0-ca3rczp5omy7dfzoqw4p7oc2yh3u7lt6/baz' + + # baz headers in the baz subdirectory of the foo prefix + >>> spec['baz'].headers + HeaderList([ + '/opt/spack/linux-fedora35-haswell/gcc-11.3.1/foo-1.0-ca3rczp5omy7dfzoqw4p7oc2yh3u7lt6/baz/include/baz/baz.h' + ]) + + # baz include directories in the baz subdirectory of the foo prefix + >>> spec['baz'].headers.directories + [ + '/opt/spack/linux-fedora35-haswell/gcc-11.3.1/foo-1.0-ca3rczp5omy7dfzoqw4p7oc2yh3u7lt6/baz/include' + ] + + # baz libraries in the baz subdirectory of the foo prefix + >>> spec['baz'].libs + LibraryList([ + '/opt/spack/linux-fedora35-haswell/gcc-11.3.1/foo-1.0-ca3rczp5omy7dfzoqw4p7oc2yh3u7lt6/baz/lib/libFooBaz.so' + ]) + + # baz library directories in the baz subdirectory of the foo porefix + >>> spec['baz'].libs.directories + [ + '/opt/spack/linux-fedora35-haswell/gcc-11.3.1/foo-1.0-ca3rczp5omy7dfzoqw4p7oc2yh3u7lt6/baz/lib' + ] + .. _abstract-and-concrete: ------------------------- @@ -5495,6 +5745,24 @@ Version Lists Spack packages should list supported versions with the newest first. +^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Using ``home`` vs ``prefix`` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +``home`` and ``prefix`` are both attributes that can be queried on a +package's dependencies, often when passing configure arguments pointing to the +location of a dependency. The difference is that while ``prefix`` is the +location on disk where a concrete package resides, ``home`` is the `logical` +location that a package resides, which may be different than ``prefix`` in +the case of virtual packages or other special circumstances. For most use +cases inside a package, it's dependency locations can be accessed via either +``self.spec['foo'].home`` or ``self.spec['foo'].prefix``. Specific packages +that should be consumed by dependents via ``.home`` instead of ``.prefix`` +should be noted in their respective documentation. + +See :ref:`custom-attributes` for more details and an example implementing +a custom ``home`` attribute. + --------------------------- Packaging workflow commands --------------------------- diff --git a/lib/spack/spack/package_base.py b/lib/spack/spack/package_base.py index 0760f832f7..9153310bd2 100644 --- a/lib/spack/spack/package_base.py +++ b/lib/spack/spack/package_base.py @@ -1447,6 +1447,10 @@ class PackageBase(six.with_metaclass(PackageMeta, PackageViewMixin, object)): """Get the prefix into which this package should be installed.""" return self.spec.prefix + @property + def home(self): + return self.prefix + @property # type: ignore[misc] @memoized def compiler(self): diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index 1952ed1a50..602dcb09e8 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -896,7 +896,7 @@ class _EdgeMap(Mapping): def _command_default_handler(descriptor, spec, cls): """Default handler when looking for the 'command' attribute. - Tries to search for ``spec.name`` in the ``spec.prefix.bin`` directory. + Tries to search for ``spec.name`` in the ``spec.home.bin`` directory. Parameters: descriptor (ForwardQueryToPackage): descriptor that triggered the call @@ -910,20 +910,21 @@ def _command_default_handler(descriptor, spec, cls): Raises: RuntimeError: If the command is not found """ - path = os.path.join(spec.prefix.bin, spec.name) + home = getattr(spec.package, 'home') + path = os.path.join(home.bin, spec.name) if fs.is_exe(path): return spack.util.executable.Executable(path) else: msg = 'Unable to locate {0} command in {1}' - raise RuntimeError(msg.format(spec.name, spec.prefix.bin)) + raise RuntimeError(msg.format(spec.name, home.bin)) def _headers_default_handler(descriptor, spec, cls): """Default handler when looking for the 'headers' attribute. Tries to search for ``*.h`` files recursively starting from - ``spec.prefix.include``. + ``spec.package.home.include``. Parameters: descriptor (ForwardQueryToPackage): descriptor that triggered the call @@ -937,21 +938,22 @@ def _headers_default_handler(descriptor, spec, cls): Raises: NoHeadersError: If no headers are found """ - headers = fs.find_headers('*', root=spec.prefix.include, recursive=True) + home = getattr(spec.package, 'home') + headers = fs.find_headers('*', root=home.include, recursive=True) if headers: return headers else: msg = 'Unable to locate {0} headers in {1}' raise spack.error.NoHeadersError( - msg.format(spec.name, spec.prefix.include)) + msg.format(spec.name, home)) def _libs_default_handler(descriptor, spec, cls): """Default handler when looking for the 'libs' attribute. Tries to search for ``lib{spec.name}`` recursively starting from - ``spec.prefix``. If ``spec.name`` starts with ``lib``, searches for + ``spec.package.home``. If ``spec.name`` starts with ``lib``, searches for ``{spec.name}`` instead. Parameters: @@ -978,6 +980,7 @@ def _libs_default_handler(descriptor, spec, cls): # get something like 'libabcXabc.so, but for now we consider this # unlikely). name = spec.name.replace('-', '?') + home = getattr(spec.package, 'home') # Avoid double 'lib' for packages whose names already start with lib if not name.startswith('lib'): @@ -990,12 +993,12 @@ def _libs_default_handler(descriptor, spec, cls): for shared in search_shared: libs = fs.find_libraries( - name, spec.prefix, shared=shared, recursive=True) + name, home, shared=shared, recursive=True) if libs: return libs msg = 'Unable to recursively locate {0} libraries in {1}' - raise spack.error.NoLibrariesError(msg.format(spec.name, spec.prefix)) + raise spack.error.NoLibrariesError(msg.format(spec.name, home)) class ForwardQueryToPackage(object): @@ -1116,6 +1119,9 @@ QueryState = collections.namedtuple( class SpecBuildInterface(lang.ObjectWrapper): + # home is available in the base Package so no default is needed + home = ForwardQueryToPackage('home', default_handler=None) + command = ForwardQueryToPackage( 'command', default_handler=_command_default_handler diff --git a/lib/spack/spack/test/install.py b/lib/spack/spack/test/install.py index d1f11aa4f4..c8c697fb7e 100644 --- a/lib/spack/spack/test/install.py +++ b/lib/spack/spack/test/install.py @@ -53,6 +53,50 @@ def test_install_and_uninstall(install_mockery, mock_fetch, monkeypatch): raise +def test_pkg_attributes(install_mockery, mock_fetch, monkeypatch): + # Get a basic concrete spec for the dummy package. + spec = Spec('attributes-foo-app ^attributes-foo') + spec.concretize() + assert spec.concrete + + pkg = spec.package + pkg.do_install() + foo = 'attributes-foo' + assert spec['bar'].prefix == spec[foo].prefix + assert spec['baz'].prefix == spec[foo].prefix + + assert spec[foo].home == spec[foo].prefix + assert spec['bar'].home == spec[foo].home + assert spec['baz'].home == spec[foo].prefix.baz + + foo_headers = spec[foo].headers + # assert foo_headers.basenames == ['foo.h'] + assert foo_headers.directories == [spec[foo].home.include] + bar_headers = spec['bar'].headers + # assert bar_headers.basenames == ['bar.h'] + assert bar_headers.directories == [spec['bar'].home.include] + baz_headers = spec['baz'].headers + # assert baz_headers.basenames == ['baz.h'] + assert baz_headers.directories == [spec['baz'].home.include] + + if 'platform=windows' in spec: + lib_suffix = '.lib' + elif 'platform=darwin' in spec: + lib_suffix = '.dylib' + else: + lib_suffix = '.so' + + foo_libs = spec[foo].libs + assert foo_libs.basenames == ['libFoo' + lib_suffix] + assert foo_libs.directories == [spec[foo].home.lib64] + bar_libs = spec['bar'].libs + assert bar_libs.basenames == ['libFooBar' + lib_suffix] + assert bar_libs.directories == [spec['bar'].home.lib64] + baz_libs = spec['baz'].libs + assert baz_libs.basenames == ['libFooBaz' + lib_suffix] + assert baz_libs.directories == [spec['baz'].home.lib] + + def mock_remove_prefix(*args): raise MockInstallError( "Intentional error", -- cgit v1.2.3-70-g09d2