From c3593e5b486f3f824d5070151ed07c10fdeccaa3 Mon Sep 17 00:00:00 2001 From: Greg Becker Date: Thu, 4 May 2023 14:36:21 -0700 Subject: Allow choosing the name of the packages subdirectory in repositories (#36643) Co-authored-by: becker33 --- lib/spack/docs/repositories.rst | 29 ++++++++++++++++++++++++++--- lib/spack/spack/cmd/repo.py | 13 ++++++++++++- lib/spack/spack/repo.py | 23 +++++++++++++---------- lib/spack/spack/test/repo.py | 18 ++++++++++-------- share/spack/spack-completion.bash | 2 +- 5 files changed, 62 insertions(+), 23 deletions(-) diff --git a/lib/spack/docs/repositories.rst b/lib/spack/docs/repositories.rst index e04104c586..9cb93af21f 100644 --- a/lib/spack/docs/repositories.rst +++ b/lib/spack/docs/repositories.rst @@ -32,11 +32,16 @@ A package repository a directory structured like this:: ... The top-level ``repo.yaml`` file contains configuration metadata for the -repository, and the ``packages`` directory contains subdirectories for -each package in the repository. Each package directory contains a -``package.py`` file and any patches or other files needed to build the +repository. The packages subdirectory, typically ``packages``, contains +subdirectories for each package in the repository. Each package directory +contains a ``package.py`` file and any patches or other files needed to build the package. +The ``repo.yaml`` file may also contain a ``subdirectory`` key, +which can modify the name of the subdirectory used for packages. As seen above, +the default value is ``packages``. An empty string (``subdirectory: ''``) requires +a flattened repo structure in which the package names are top-level subdirectories. + Package repositories allow you to: 1. Maintain your own packages separately from Spack; @@ -373,6 +378,24 @@ You can supply a custom namespace with a second argument, e.g.: repo: namespace: 'llnl.comp' +You can also create repositories with custom structure with the ``-d/--subdirectory`` +argument, e.g.: + +.. code-block:: console + + $ spack repo create -d applications myrepo apps + ==> Created repo with namespace 'apps'. + ==> To register it with Spack, run this command: + spack repo add ~/myrepo + + $ ls myrepo + applications/ repo.yaml + + $ cat myrepo/repo.yaml + repo: + namespace: apps + subdirectory: applications + ^^^^^^^^^^^^^^^^^^ ``spack repo add`` ^^^^^^^^^^^^^^^^^^ diff --git a/lib/spack/spack/cmd/repo.py b/lib/spack/spack/cmd/repo.py index 16285efa56..5dd5e1677f 100644 --- a/lib/spack/spack/cmd/repo.py +++ b/lib/spack/spack/cmd/repo.py @@ -32,6 +32,17 @@ def setup_parser(subparser): help="namespace to identify packages in the repository. " "defaults to the directory name", nargs="?", ) + create_parser.add_argument( + "-d", + "--subdirectory", + action="store", + dest="subdir", + default=spack.repo.packages_dir_name, + help=( + "subdirectory to store packages in the repository." + " Default 'packages'. Use an empty string for no subdirectory." + ), + ) # List list_parser = sp.add_parser("list", help=repo_list.__doc__) @@ -70,7 +81,7 @@ def setup_parser(subparser): def repo_create(args): """Create a new package repository.""" - full_path, namespace = spack.repo.create_repo(args.directory, args.namespace) + full_path, namespace = spack.repo.create_repo(args.directory, args.namespace, args.subdir) tty.msg("Created repo with namespace '%s'." % namespace) tty.msg("To register it with spack, run this command:", "spack repo add %s" % full_path) diff --git a/lib/spack/spack/repo.py b/lib/spack/spack/repo.py index f9bf3a8926..f68ce7ebdc 100644 --- a/lib/spack/spack/repo.py +++ b/lib/spack/spack/repo.py @@ -935,12 +935,6 @@ class Repo(object): self.config_file = os.path.join(self.root, repo_config_name) check(os.path.isfile(self.config_file), "No %s found in '%s'" % (repo_config_name, root)) - self.packages_path = os.path.join(self.root, packages_dir_name) - check( - os.path.isdir(self.packages_path), - "No directory '%s' found in '%s'" % (packages_dir_name, root), - ) - # Read configuration and validate namespace config = self._read_config() check( @@ -961,6 +955,13 @@ class Repo(object): # Keep name components around for checking prefixes. self._names = self.full_namespace.split(".") + packages_dir = config.get("subdirectory", packages_dir_name) + self.packages_path = os.path.join(self.root, packages_dir) + check( + os.path.isdir(self.packages_path), + "No directory '%s' found in '%s'" % (packages_dir, root), + ) + # These are internal cache variables. self._modules = {} self._classes = {} @@ -1150,7 +1151,7 @@ class Repo(object): def package_path(self, name): """Get path to package.py file for this repo.""" - return os.path.join(self.root, packages_dir_name, name, package_file_name) + return os.path.join(self.packages_path, name, package_file_name) def all_package_paths(self): for name in self.all_package_names(): @@ -1287,7 +1288,7 @@ class Repo(object): RepoType = Union[Repo, RepoPath] -def create_repo(root, namespace=None): +def create_repo(root, namespace=None, subdir=packages_dir_name): """Create a new repository in root with the specified namespace. If the namespace is not provided, use basename of root. @@ -1318,12 +1319,14 @@ def create_repo(root, namespace=None): try: config_path = os.path.join(root, repo_config_name) - packages_path = os.path.join(root, packages_dir_name) + packages_path = os.path.join(root, subdir) fs.mkdirp(packages_path) with open(config_path, "w") as config: config.write("repo:\n") - config.write(" namespace: '%s'\n" % namespace) + config.write(f" namespace: '{namespace}'\n") + if subdir != packages_dir_name: + config.write(f" subdirectory: '{subdir}'\n") except (IOError, OSError) as e: # try to clean up. diff --git a/lib/spack/spack/test/repo.py b/lib/spack/spack/test/repo.py index 8c54844644..a73056d810 100644 --- a/lib/spack/spack/test/repo.py +++ b/lib/spack/spack/test/repo.py @@ -11,11 +11,11 @@ import spack.paths import spack.repo -@pytest.fixture() -def extra_repo(tmpdir_factory): +@pytest.fixture(params=["packages", "", "foo"]) +def extra_repo(tmpdir_factory, request): repo_namespace = "extra_test_repo" repo_dir = tmpdir_factory.mktemp(repo_namespace) - repo_dir.ensure("packages", dir=True) + repo_dir.ensure(request.param, dir=True) with open(str(repo_dir.join("repo.yaml")), "w") as f: f.write( @@ -24,7 +24,9 @@ repo: namespace: extra_test_repo """ ) - return spack.repo.Repo(str(repo_dir)) + if request.param != "packages": + f.write(f" subdirectory: '{request.param}'") + return (spack.repo.Repo(str(repo_dir)), request.param) def test_repo_getpkg(mutable_mock_repo): @@ -33,13 +35,13 @@ def test_repo_getpkg(mutable_mock_repo): def test_repo_multi_getpkg(mutable_mock_repo, extra_repo): - mutable_mock_repo.put_first(extra_repo) + mutable_mock_repo.put_first(extra_repo[0]) mutable_mock_repo.get_pkg_class("a") mutable_mock_repo.get_pkg_class("builtin.mock.a") def test_repo_multi_getpkgclass(mutable_mock_repo, extra_repo): - mutable_mock_repo.put_first(extra_repo) + mutable_mock_repo.put_first(extra_repo[0]) mutable_mock_repo.get_pkg_class("a") mutable_mock_repo.get_pkg_class("builtin.mock.a") @@ -63,9 +65,9 @@ def test_repo_last_mtime(): def test_repo_invisibles(mutable_mock_repo, extra_repo): - with open(os.path.join(extra_repo.root, "packages", ".invisible"), "w"): + with open(os.path.join(extra_repo[0].root, extra_repo[1], ".invisible"), "w"): pass - extra_repo.all_package_names() + extra_repo[0].all_package_names() @pytest.mark.parametrize("attr_name,exists", [("cmake", True), ("__sphinx_mock__", False)]) diff --git a/share/spack/spack-completion.bash b/share/spack/spack-completion.bash index cae6d0cfce..f79b13287a 100755 --- a/share/spack/spack-completion.bash +++ b/share/spack/spack-completion.bash @@ -1606,7 +1606,7 @@ _spack_repo() { _spack_repo_create() { if $list_options then - SPACK_COMPREPLY="-h --help" + SPACK_COMPREPLY="-h --help -d --subdirectory" else _repos fi -- cgit v1.2.3-60-g2f50