summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--lib/spack/docs/packaging_guide.rst3
-rw-r--r--lib/spack/spack/fetch_strategy.py12
-rw-r--r--lib/spack/spack/test/conftest.py27
-rw-r--r--lib/spack/spack/test/git_fetch.py57
-rw-r--r--var/spack/repos/builtin/packages/py-torch/package.py3
5 files changed, 97 insertions, 5 deletions
diff --git a/lib/spack/docs/packaging_guide.rst b/lib/spack/docs/packaging_guide.rst
index e566cb4545..6e1271b12c 100644
--- a/lib/spack/docs/packaging_guide.rst
+++ b/lib/spack/docs/packaging_guide.rst
@@ -929,6 +929,9 @@ Git fetching supports the following parameters to ``version``:
* ``tag``: Name of a tag to fetch.
* ``commit``: SHA hash (or prefix) of a commit to fetch.
* ``submodules``: Also fetch submodules recursively when checking out this repository.
+* ``submodules_delete``: A list of submodules to forcibly delete from the repository
+ after fetching. Useful if a version in the repository has submodules that
+ have disappeared/are no longer accessible.
* ``get_full_repo``: Ensure the full git history is checked out with all remote
branch information. Normally (``get_full_repo=False``, the default), the git
option ``--depth 1`` will be used if the version of git and the specified
diff --git a/lib/spack/spack/fetch_strategy.py b/lib/spack/spack/fetch_strategy.py
index cf8ec0b095..5ae01286c4 100644
--- a/lib/spack/spack/fetch_strategy.py
+++ b/lib/spack/spack/fetch_strategy.py
@@ -714,7 +714,8 @@ class GitFetchStrategy(VCSFetchStrategy):
Repositories are cloned into the standard stage source path directory.
"""
url_attr = 'git'
- optional_attrs = ['tag', 'branch', 'commit', 'submodules', 'get_full_repo']
+ optional_attrs = ['tag', 'branch', 'commit', 'submodules',
+ 'get_full_repo', 'submodules_delete']
def __init__(self, **kwargs):
# Discards the keywords in kwargs that may conflict with the next call
@@ -725,6 +726,7 @@ class GitFetchStrategy(VCSFetchStrategy):
self._git = None
self.submodules = kwargs.get('submodules', False)
+ self.submodules_delete = kwargs.get('submodules_delete', False)
self.get_full_repo = kwargs.get('get_full_repo', False)
@property
@@ -858,6 +860,14 @@ class GitFetchStrategy(VCSFetchStrategy):
git(*pull_args, ignore_errors=1)
git(*co_args)
+ if self.submodules_delete:
+ with working_dir(self.stage.source_path):
+ for submodule_to_delete in self.submodules_delete:
+ args = ['rm', submodule_to_delete]
+ if not spack.config.get('config:debug'):
+ args.insert(1, '--quiet')
+ git(*args)
+
# Init submodules if the user asked for them.
if self.submodules:
with working_dir(self.stage.source_path):
diff --git a/lib/spack/spack/test/conftest.py b/lib/spack/spack/test/conftest.py
index d4c11e1693..97bbb69b52 100644
--- a/lib/spack/spack/test/conftest.py
+++ b/lib/spack/spack/test/conftest.py
@@ -744,11 +744,31 @@ def mock_archive(request, tmpdir_factory):
@pytest.fixture(scope='session')
def mock_git_repository(tmpdir_factory):
- """Creates a very simple git repository with two branches and
- two commits.
+ """Creates a simple git repository with two branches,
+ two commits and two submodules. Each submodule has one commit.
"""
git = spack.util.executable.which('git', required=True)
+ suburls = []
+ for submodule_count in range(2):
+ tmpdir = tmpdir_factory.mktemp('mock-git-repo-submodule-dir-{0}'
+ .format(submodule_count))
+ tmpdir.ensure(spack.stage._source_path_subdir, dir=True)
+ repodir = tmpdir.join(spack.stage._source_path_subdir)
+ suburls.append((submodule_count, 'file://' + str(repodir)))
+
+ # Initialize the repository
+ with repodir.as_cwd():
+ git('init')
+ git('config', 'user.name', 'Spack')
+ git('config', 'user.email', 'spack@spack.io')
+
+ # r0 is just the first commit
+ submodule_file = 'r0_file_{0}'.format(submodule_count)
+ repodir.ensure(submodule_file)
+ git('add', submodule_file)
+ git('commit', '-m', 'mock-git-repo r0 {0}'.format(submodule_count))
+
tmpdir = tmpdir_factory.mktemp('mock-git-repo-dir')
tmpdir.ensure(spack.stage._source_path_subdir, dir=True)
repodir = tmpdir.join(spack.stage._source_path_subdir)
@@ -759,6 +779,9 @@ def mock_git_repository(tmpdir_factory):
git('config', 'user.name', 'Spack')
git('config', 'user.email', 'spack@spack.io')
url = 'file://' + str(repodir)
+ for number, suburl in suburls:
+ git('submodule', 'add', suburl,
+ 'third_party/submodule{0}'.format(number))
# r0 is just the first commit
r0_file = 'r0_file'
diff --git a/lib/spack/spack/test/git_fetch.py b/lib/spack/spack/test/git_fetch.py
index 57474e56b7..8dc57da793 100644
--- a/lib/spack/spack/test/git_fetch.py
+++ b/lib/spack/spack/test/git_fetch.py
@@ -19,7 +19,6 @@ from spack.version import ver
from spack.fetch_strategy import GitFetchStrategy
from spack.util.executable import which
-
pytestmark = pytest.mark.skipif(
not which('git'), reason='requires git to be installed')
@@ -217,3 +216,59 @@ def test_get_full_repo(get_full_repo, git_version, mock_git_repository,
else:
assert(nbranches == 2)
assert(ncommits == 1)
+
+
+@pytest.mark.disable_clean_stage_check
+@pytest.mark.parametrize("submodules", [True, False])
+def test_gitsubmodule(submodules, mock_git_repository, config,
+ mutable_mock_repo):
+ """
+ Test GitFetchStrategy behavior with submodules
+ """
+ type_of_test = 'tag-branch'
+ t = mock_git_repository.checks[type_of_test]
+
+ # Construct the package under test
+ spec = Spec('git-test')
+ spec.concretize()
+ pkg = spack.repo.get(spec)
+ args = copy.copy(t.args)
+ args['submodules'] = submodules
+ pkg.versions[ver('git')] = args
+ pkg.do_stage()
+ with working_dir(pkg.stage.source_path):
+ for submodule_count in range(2):
+ file_path = os.path.join(pkg.stage.source_path,
+ 'third_party/submodule{0}/r0_file_{0}'
+ .format(submodule_count))
+ if submodules:
+ assert os.path.isfile(file_path)
+ else:
+ assert not os.path.isfile(file_path)
+
+
+@pytest.mark.disable_clean_stage_check
+def test_gitsubmodules_delete(mock_git_repository, config, mutable_mock_repo):
+ """
+ Test GitFetchStrategy behavior with submodules_delete
+ """
+ type_of_test = 'tag-branch'
+ t = mock_git_repository.checks[type_of_test]
+
+ # Construct the package under test
+ spec = Spec('git-test')
+ spec.concretize()
+ pkg = spack.repo.get(spec)
+ args = copy.copy(t.args)
+ args['submodules'] = True
+ args['submodules_delete'] = ['third_party/submodule0',
+ 'third_party/submodule1']
+ pkg.versions[ver('git')] = args
+ pkg.do_stage()
+ with working_dir(pkg.stage.source_path):
+ file_path = os.path.join(pkg.stage.source_path,
+ 'third_party/submodule0')
+ assert not os.path.isdir(file_path)
+ file_path = os.path.join(pkg.stage.source_path,
+ 'third_party/submodule1')
+ assert not os.path.isdir(file_path)
diff --git a/var/spack/repos/builtin/packages/py-torch/package.py b/var/spack/repos/builtin/packages/py-torch/package.py
index 2a20235bef..e40bc840fe 100644
--- a/var/spack/repos/builtin/packages/py-torch/package.py
+++ b/var/spack/repos/builtin/packages/py-torch/package.py
@@ -58,7 +58,8 @@ class PyTorch(PythonPackage, CudaPackage):
version('1.1.0', tag='v1.1.0', submodules=True)
version('1.0.1', tag='v1.0.1', submodules=True)
version('1.0.0', tag='v1.0.0', submodules=True)
- version('0.4.1', tag='v0.4.1', submodules=True)
+ version('0.4.1', tag='v0.4.1', submodules=True,
+ submodules_delete=['third_party/nervanagpu'])
version('0.4.0', tag='v0.4.0', submodules=True)
version('0.3.1', tag='v0.3.1', submodules=True)