From 7f1659786b32dd492b062347c6ca1538a8c71495 Mon Sep 17 00:00:00 2001 From: Tom Scogland Date: Mon, 9 May 2022 21:54:38 -0700 Subject: Add a Lua build-system (#28854) Reworking lua to allow easier substitution of the base lua implementation. Also adding in a maintained version of luajit and re-factoring the entire stack to use a custom build-system to centralize functionality like environment variable management and luarocks installation. The `lua-lang` virtual is now versioned so that a package that requires Lua 5.1 semantics can get any lua, but one that requires 5.2 will only get upstream lua. The luaposix package requires lua-bit32, but only when built with a lua conforming to version 5.1. This adds the package, and the dependencies, but exposed a problem with luarocks dependency detection. Since we're installing each package in its own "tree" and there's no environment variable to list extra trees, spack now generates a luarocks config file that lists all the trees of all the dependencies, and references it by setting `LUAROCKS_CONFIG` in the build environment of every LuaPackage. This allows luarocks to find the spack installed dependencies correctly rather than trying (and failing) to download them. Co-authored-by: Adam J. Stewart Co-authored-by: Tom Scogland Co-authored-by: Massimiliano Culpo --- lib/spack/docs/build_systems.rst | 1 + lib/spack/docs/build_systems/luapackage.rst | 105 ++++++++++++++++++++++++++++ lib/spack/spack/build_systems/lua.py | 102 +++++++++++++++++++++++++++ lib/spack/spack/cmd/create.py | 26 +++++++ lib/spack/spack/directives.py | 10 +-- lib/spack/spack/pkgkit.py | 1 + lib/spack/spack/solver/concretize.lp | 2 + lib/spack/spack/test/build_system_guess.py | 41 +++++------ lib/spack/spack/url.py | 8 +++ 9 files changed, 269 insertions(+), 27 deletions(-) create mode 100644 lib/spack/docs/build_systems/luapackage.rst create mode 100644 lib/spack/spack/build_systems/lua.py (limited to 'lib') diff --git a/lib/spack/docs/build_systems.rst b/lib/spack/docs/build_systems.rst index 77fb5c7838..11f0df4f49 100644 --- a/lib/spack/docs/build_systems.rst +++ b/lib/spack/docs/build_systems.rst @@ -47,6 +47,7 @@ on these ideas for each distinct build system that Spack supports: :maxdepth: 1 :caption: Language-specific + build_systems/luapackage build_systems/octavepackage build_systems/perlpackage build_systems/pythonpackage diff --git a/lib/spack/docs/build_systems/luapackage.rst b/lib/spack/docs/build_systems/luapackage.rst new file mode 100644 index 0000000000..6332edfc20 --- /dev/null +++ b/lib/spack/docs/build_systems/luapackage.rst @@ -0,0 +1,105 @@ +.. Copyright 2013-2022 Lawrence Livermore National Security, LLC and other + Spack Project Developers. See the top-level COPYRIGHT file for details. + + SPDX-License-Identifier: (Apache-2.0 OR MIT) + +.. _luapackage: + +------------ +LuaPackage +------------ + +LuaPackage is a helper for the common case of Lua packages that provide +a rockspec file. This is not meant to take a rock archive, but to build +a source archive or repository that provides a rockspec, which should cover +most lua packages. In the case a Lua package builds by Make rather than +luarocks, prefer MakefilePackage. + +^^^^^^ +Phases +^^^^^^ + +The ``LuaPackage`` base class comes with the following phases: + +#. ``unpack`` - if using a rock, unpacks the rock and moves into the source directory +#. ``preprocess`` - adjust sources or rockspec to fix build +#. ``install`` - install the project + +By default, these phases run: + +.. code-block:: console + + # If the archive is a source rock + $ luarocks unpack .src.rock + $ # preprocess is a noop by default + $ luarocks make .rockspec + + +Any of these phases can be overridden in your package as necessary. + +^^^^^^^^^^^^^^^ +Important files +^^^^^^^^^^^^^^^ + +Packages that use the Lua/LuaRocks build system can be identified by the +presence of a ``*.rockspec`` file in their sourcetree, or can be fetched as +a source rock archive (``.src.rock``). This file declares things like build +instructions and dependencies, the ``.src.rock`` also contains all code. + +It is common for the rockspec file to list the lua version required in +a dependency. The LuaPackage class adds appropriate dependencies on a Lua +implementation, but it is a good idea to specify the version required with +a ``depends_on`` statement. The block normally will be a table definition like +this: + +.. code-block:: lua + + dependencies = { + "lua >= 5.1", + } + +The LuaPackage class supports source repositories and archives containing +a rockspec and directly downloading source rock files. It *does not* support +downloading dependencies listed inside a rockspec, and thus does not support +directly downloading a rockspec as an archive. + +^^^^^^^^^^^^^^^^^^^^^^^^^ +Build system dependencies +^^^^^^^^^^^^^^^^^^^^^^^^^ + +All base dependencies are added by the build system, but LuaRocks is run to +avoid downloading extra Lua dependencies during build. If the package needs +Lua libraries outside the standard set, they should be added as dependencies. + +To specify a Lua version constraint but allow all lua implementations, prefer +to use ``depends_on("lua-lang@5.1:5.1.99")`` to express any 5.1 compatible +version. If the package requires LuaJit rather than Lua, +a ``depends_on("luajit")`` should be used to ensure a LuaJit distribution is +used instead of the Lua interpreter. Alternately, if only interpreted Lua will +work ``depends_on("lua")`` will express that. + +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Passing arguments to luarocks make +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +If you need to pass any arguments to the ``luarocks make`` call, you can +override the ``luarocks_args`` method like so: + +.. code-block:: python + + def luarocks_args(self): + return ['flag1', 'flag2'] + +One common use of this is to override warnings or flags for newer compilers, as in: + +.. code-block:: python + + def luarocks_args(self): + return ["CFLAGS='-Wno-error=implicit-function-declaration'"] + +^^^^^^^^^^^^^^^^^^^^^^ +External documentation +^^^^^^^^^^^^^^^^^^^^^^ + +For more information on the LuaRocks build system, see: +https://luarocks.org/ diff --git a/lib/spack/spack/build_systems/lua.py b/lib/spack/spack/build_systems/lua.py new file mode 100644 index 0000000000..3a836de012 --- /dev/null +++ b/lib/spack/spack/build_systems/lua.py @@ -0,0 +1,102 @@ +# Copyright 2013-2022 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + + +import os + +from llnl.util.filesystem import find + +from spack.directives import depends_on, extends +from spack.multimethod import when +from spack.package import PackageBase +from spack.util.executable import Executable + + +class LuaPackage(PackageBase): + """Specialized class for lua packages""" + + phases = ['unpack', 'generate_luarocks_config', 'preprocess', 'install'] + #: This attribute is used in UI queries that need to know the build + #: system base class + build_system_class = 'LuaPackage' + + list_depth = 1 # LuaRocks requires at least one level of spidering to find versions + depends_on('lua-lang') + extends('lua', when='^lua') + with when('^lua-luajit'): + extends('lua-luajit') + depends_on('luajit') + depends_on('lua-luajit+lualinks') + with when('^lua-luajit-openresty'): + extends('lua-luajit-openresty') + depends_on('luajit') + depends_on('lua-luajit-openresty+lualinks') + + def unpack(self, spec, prefix): + if os.path.splitext(self.stage.archive_file)[1] == '.rock': + directory = self.luarocks('unpack', self.stage.archive_file, output=str) + dirlines = directory.split('\n') + # TODO: figure out how to scope this better + os.chdir(dirlines[2]) + + def _generate_tree_line(self, name, prefix): + return """{{ name = "{name}", root = "{prefix}" }};""".format( + name=name, + prefix=prefix, + ) + + def _luarocks_config_path(self): + return os.path.join(self.stage.source_path, 'spack_luarocks.lua') + + def generate_luarocks_config(self, spec, prefix): + spec = self.spec + table_entries = [] + for d in spec.traverse( + deptypes=("build", "run"), deptype_query="run" + ): + if d.package.extends(self.extendee_spec): + table_entries.append(self._generate_tree_line(d.name, d.prefix)) + + path = self._luarocks_config_path() + with open(path, 'w') as config: + config.write( + """ + deps_mode="all" + rocks_trees={{ + {} + }} + """.format( + "\n".join(table_entries) + ) + ) + return path + + def setup_build_environment(self, env): + env.set('LUAROCKS_CONFIG', self._luarocks_config_path()) + + def preprocess(self, spec, prefix): + """Override this to preprocess source before building with luarocks""" + pass + + @property + def lua(self): + return Executable(self.spec['lua-lang'].prefix.bin.lua) + + @property + def luarocks(self): + lr = Executable(self.spec['lua-lang'].prefix.bin.luarocks) + return lr + + def luarocks_args(self): + return [] + + def install(self, spec, prefix): + rock = '.' + specs = find('.', '*.rockspec', recursive=False) + if specs: + rock = specs[0] + rocks_args = self.luarocks_args() + rocks_args.append(rock) + self.luarocks('--tree=' + prefix, 'make', *rocks_args) diff --git a/lib/spack/spack/cmd/create.py b/lib/spack/spack/cmd/create.py index 1e3e89760b..94ddc7d292 100644 --- a/lib/spack/spack/cmd/create.py +++ b/lib/spack/spack/cmd/create.py @@ -187,6 +187,27 @@ class CMakePackageTemplate(PackageTemplate): return args""" +class LuaPackageTemplate(PackageTemplate): + """Provides appropriate overrides for LuaRocks-based packages""" + + base_class_name = 'LuaPackage' + + body_def = """\ + def luarocks_args(self): + # FIXME: Add arguments to `luarocks make` other than rockspec path + # FIXME: If not needed delete this function + args = [] + return args""" + + def __init__(self, name, url, *args, **kwargs): + # If the user provided `--name lua-lpeg`, don't rename it lua-lua-lpeg + if not name.startswith('lua-'): + # Make it more obvious that we are renaming the package + tty.msg("Changing package name from {0} to lua-{0}".format(name)) + name = 'lua-{0}'.format(name) + super(LuaPackageTemplate, self).__init__(name, url, *args, **kwargs) + + class MesonPackageTemplate(PackageTemplate): """Provides appropriate overrides for meson-based packages""" @@ -580,6 +601,7 @@ templates = { 'makefile': MakefilePackageTemplate, 'intel': IntelPackageTemplate, 'meson': MesonPackageTemplate, + 'lua': LuaPackageTemplate, 'sip': SIPPackageTemplate, 'generic': PackageTemplate, } @@ -644,6 +666,9 @@ class BuildSystemGuesser: if url.endswith('.whl') or '.whl#' in url: self.build_system = 'python' return + if url.endswith('.rock'): + self.build_system = 'lua' + return # A list of clues that give us an idea of the build system a package # uses. If the regular expression matches a file contained in the @@ -668,6 +693,7 @@ class BuildSystemGuesser: (r'/Rakefile$', 'ruby'), (r'/setup\.rb$', 'ruby'), (r'/.*\.pro$', 'qmake'), + (r'/.*\.rockspec$', 'lua'), (r'/(GNU)?[Mm]akefile$', 'makefile'), (r'/DESCRIPTION$', 'octave'), (r'/meson\.build$', 'meson'), diff --git a/lib/spack/spack/directives.py b/lib/spack/spack/directives.py index 7d92a203e5..2de6552a82 100644 --- a/lib/spack/spack/directives.py +++ b/lib/spack/spack/directives.py @@ -48,13 +48,13 @@ from spack.fetch_strategy import from_kwargs from spack.resource import Resource from spack.version import Version, VersionChecksumError -__all__ = ['DirectiveError', 'DirectiveMeta'] +__all__ = ['DirectiveError', 'DirectiveMeta', 'version', 'conflicts', 'depends_on', + 'extends', 'provides', 'patch', 'variant', 'resource'] #: These are variant names used by Spack internally; packages can't use them reserved_names = ['patches', 'dev_path'] -#: Names of possible directives. This list is populated elsewhere in the file and then -#: added to `__all__` at the bottom. +#: Names of possible directives. This list is populated elsewhere in the file. directive_names = [] _patch_order_index = 0 @@ -731,7 +731,3 @@ class DependencyPatchError(DirectiveError): class UnsupportedPackageDirective(DirectiveError): """Raised when an invalid or unsupported package directive is specified.""" - - -#: add all directive names to __all__ -__all__.extend(directive_names) diff --git a/lib/spack/spack/pkgkit.py b/lib/spack/spack/pkgkit.py index 3fde349b9a..c7fbfe1af8 100644 --- a/lib/spack/spack/pkgkit.py +++ b/lib/spack/spack/pkgkit.py @@ -25,6 +25,7 @@ from spack.build_systems.cmake import CMakePackage from spack.build_systems.cuda import CudaPackage from spack.build_systems.gnu import GNUMirrorPackage from spack.build_systems.intel import IntelPackage +from spack.build_systems.lua import LuaPackage from spack.build_systems.makefile import MakefilePackage from spack.build_systems.maven import MavenPackage from spack.build_systems.meson import MesonPackage diff --git a/lib/spack/spack/solver/concretize.lp b/lib/spack/spack/solver/concretize.lp index 19bec37908..65dbd0b19d 100644 --- a/lib/spack/spack/solver/concretize.lp +++ b/lib/spack/spack/solver/concretize.lp @@ -288,6 +288,7 @@ possible_provider_weight(Dependency, Virtual, 100, "fallback") :- provider(Depen % These allow us to easily define conditional dependency and conflict rules % without enumerating all spec attributes every time. node(Package) :- attr("node", Package). +virtual_node(Virtual) :- attr("virtual_node", Virtual). hash(Package, Hash) :- attr("hash", Package, Hash). version(Package, Version) :- attr("version", Package, Version). version_satisfies(Package, Constraint) :- attr("version_satisfies", Package, Constraint). @@ -306,6 +307,7 @@ node_compiler_version_satisfies(Package, Compiler, Version) :- attr("node_compiler_version_satisfies", Package, Compiler, Version). attr("node", Package) :- node(Package). +attr("virtual_node", Virtual) :- virtual_node(Virtual). attr("hash", Package, Hash) :- hash(Package, Hash). attr("version", Package, Version) :- version(Package, Version). attr("version_satisfies", Package, Constraint) :- version_satisfies(Package, Constraint). diff --git a/lib/spack/spack/test/build_system_guess.py b/lib/spack/spack/test/build_system_guess.py index 9e97dfbfb2..7850d70fff 100644 --- a/lib/spack/spack/test/build_system_guess.py +++ b/lib/spack/spack/test/build_system_guess.py @@ -18,26 +18,27 @@ pytestmark = pytest.mark.skipif(sys.platform == "win32", @pytest.fixture( scope='function', params=[ - ('configure', 'autotools'), - ('CMakeLists.txt', 'cmake'), - ('project.pro', 'qmake'), - ('pom.xml', 'maven'), - ('SConstruct', 'scons'), - ('waf', 'waf'), - ('setup.py', 'python'), - ('NAMESPACE', 'r'), - ('WORKSPACE', 'bazel'), - ('Makefile.PL', 'perlmake'), - ('Build.PL', 'perlbuild'), - ('foo.gemspec', 'ruby'), - ('Rakefile', 'ruby'), - ('setup.rb', 'ruby'), - ('GNUmakefile', 'makefile'), - ('makefile', 'makefile'), - ('Makefile', 'makefile'), - ('meson.build', 'meson'), - ('configure.py', 'sip'), - ('foobar', 'generic') + ('configure', 'autotools'), + ('CMakeLists.txt', 'cmake'), + ('project.pro', 'qmake'), + ('pom.xml', 'maven'), + ('SConstruct', 'scons'), + ('waf', 'waf'), + ('argbah.rockspec', 'lua'), + ('setup.py', 'python'), + ('NAMESPACE', 'r'), + ('WORKSPACE', 'bazel'), + ('Makefile.PL', 'perlmake'), + ('Build.PL', 'perlbuild'), + ('foo.gemspec', 'ruby'), + ('Rakefile', 'ruby'), + ('setup.rb', 'ruby'), + ('GNUmakefile', 'makefile'), + ('makefile', 'makefile'), + ('Makefile', 'makefile'), + ('meson.build', 'meson'), + ('configure.py', 'sip'), + ('foobar', 'generic') ] ) def url_and_build_system(request, tmpdir): diff --git a/lib/spack/spack/url.py b/lib/spack/spack/url.py index 2eb2ab7ed7..8ad7b196af 100644 --- a/lib/spack/spack/url.py +++ b/lib/spack/spack/url.py @@ -58,6 +58,7 @@ def find_list_urls(url): BitBucket https://bitbucket.org///downloads/?tab=tags CRAN https://\*.r-project.org/src/contrib/Archive/ PyPI https://pypi.org/simple// + LuaRocks https://luarocks.org/modules// ========= ======================================================= Note: this function is called by `spack versions`, `spack checksum`, @@ -106,6 +107,13 @@ def find_list_urls(url): # e.g. https://pypi.io/packages/py2.py3/o/opencensus-context/opencensus_context-0.1.1-py2.py3-none-any.whl (r'(?:pypi|pythonhosted)[^/]+/packages/[^/]+/./([^/]+)', lambda m: 'https://pypi.org/simple/' + m.group(1) + '/'), + + # LuaRocks + # e.g. https://luarocks.org/manifests/gvvaughan/lpeg-1.0.2-1.src.rock + # e.g. https://luarocks.org/manifests/openresty/lua-cjson-2.1.0-1.src.rock + (r'luarocks[^/]+/(?:modules|manifests)/(?P[^/]+)/' + + r'(?P.+?)-[0-9.-]*\.src\.rock', + lambda m: 'https://luarocks.org/modules/' + m.group('org') + '/' + m.group('name') + '/'), ] list_urls = set([os.path.dirname(url)]) -- cgit v1.2.3-60-g2f50