diff options
| -rw-r--r-- | lib/spack/docs/packaging_guide.rst | 3 | ||||
| -rw-r--r-- | lib/spack/spack/fetch_strategy.py | 12 | ||||
| -rw-r--r-- | lib/spack/spack/test/conftest.py | 27 | ||||
| -rw-r--r-- | lib/spack/spack/test/git_fetch.py | 57 | ||||
| -rw-r--r-- | var/spack/repos/builtin/packages/py-torch/package.py | 3 |
5 files changed, 97 insertions, 5 deletions
diff --git a/lib/spack/docs/packaging_guide.rst b/lib/spack/docs/packaging_guide.rst index e566cb4545..6e1271b12c 100644 --- a/lib/spack/docs/packaging_guide.rst +++ b/lib/spack/docs/packaging_guide.rst @@ -929,6 +929,9 @@ Git fetching supports the following parameters to ``version``: * ``tag``: Name of a tag to fetch. * ``commit``: SHA hash (or prefix) of a commit to fetch. * ``submodules``: Also fetch submodules recursively when checking out this repository. +* ``submodules_delete``: A list of submodules to forcibly delete from the repository + after fetching. Useful if a version in the repository has submodules that + have disappeared/are no longer accessible. * ``get_full_repo``: Ensure the full git history is checked out with all remote branch information. Normally (``get_full_repo=False``, the default), the git option ``--depth 1`` will be used if the version of git and the specified diff --git a/lib/spack/spack/fetch_strategy.py b/lib/spack/spack/fetch_strategy.py index cf8ec0b095..5ae01286c4 100644 --- a/lib/spack/spack/fetch_strategy.py +++ b/lib/spack/spack/fetch_strategy.py @@ -714,7 +714,8 @@ class GitFetchStrategy(VCSFetchStrategy): Repositories are cloned into the standard stage source path directory. """ url_attr = 'git' - optional_attrs = ['tag', 'branch', 'commit', 'submodules', 'get_full_repo'] + optional_attrs = ['tag', 'branch', 'commit', 'submodules', + 'get_full_repo', 'submodules_delete'] def __init__(self, **kwargs): # Discards the keywords in kwargs that may conflict with the next call @@ -725,6 +726,7 @@ class GitFetchStrategy(VCSFetchStrategy): self._git = None self.submodules = kwargs.get('submodules', False) + self.submodules_delete = kwargs.get('submodules_delete', False) self.get_full_repo = kwargs.get('get_full_repo', False) @property @@ -858,6 +860,14 @@ class GitFetchStrategy(VCSFetchStrategy): git(*pull_args, ignore_errors=1) git(*co_args) + if self.submodules_delete: + with working_dir(self.stage.source_path): + for submodule_to_delete in self.submodules_delete: + args = ['rm', submodule_to_delete] + if not spack.config.get('config:debug'): + args.insert(1, '--quiet') + git(*args) + # Init submodules if the user asked for them. if self.submodules: with working_dir(self.stage.source_path): diff --git a/lib/spack/spack/test/conftest.py b/lib/spack/spack/test/conftest.py index d4c11e1693..97bbb69b52 100644 --- a/lib/spack/spack/test/conftest.py +++ b/lib/spack/spack/test/conftest.py @@ -744,11 +744,31 @@ def mock_archive(request, tmpdir_factory): @pytest.fixture(scope='session') def mock_git_repository(tmpdir_factory): - """Creates a very simple git repository with two branches and - two commits. + """Creates a simple git repository with two branches, + two commits and two submodules. Each submodule has one commit. """ git = spack.util.executable.which('git', required=True) + suburls = [] + for submodule_count in range(2): + tmpdir = tmpdir_factory.mktemp('mock-git-repo-submodule-dir-{0}' + .format(submodule_count)) + tmpdir.ensure(spack.stage._source_path_subdir, dir=True) + repodir = tmpdir.join(spack.stage._source_path_subdir) + suburls.append((submodule_count, 'file://' + str(repodir))) + + # Initialize the repository + with repodir.as_cwd(): + git('init') + git('config', 'user.name', 'Spack') + git('config', 'user.email', 'spack@spack.io') + + # r0 is just the first commit + submodule_file = 'r0_file_{0}'.format(submodule_count) + repodir.ensure(submodule_file) + git('add', submodule_file) + git('commit', '-m', 'mock-git-repo r0 {0}'.format(submodule_count)) + tmpdir = tmpdir_factory.mktemp('mock-git-repo-dir') tmpdir.ensure(spack.stage._source_path_subdir, dir=True) repodir = tmpdir.join(spack.stage._source_path_subdir) @@ -759,6 +779,9 @@ def mock_git_repository(tmpdir_factory): git('config', 'user.name', 'Spack') git('config', 'user.email', 'spack@spack.io') url = 'file://' + str(repodir) + for number, suburl in suburls: + git('submodule', 'add', suburl, + 'third_party/submodule{0}'.format(number)) # r0 is just the first commit r0_file = 'r0_file' diff --git a/lib/spack/spack/test/git_fetch.py b/lib/spack/spack/test/git_fetch.py index 57474e56b7..8dc57da793 100644 --- a/lib/spack/spack/test/git_fetch.py +++ b/lib/spack/spack/test/git_fetch.py @@ -19,7 +19,6 @@ from spack.version import ver from spack.fetch_strategy import GitFetchStrategy from spack.util.executable import which - pytestmark = pytest.mark.skipif( not which('git'), reason='requires git to be installed') @@ -217,3 +216,59 @@ def test_get_full_repo(get_full_repo, git_version, mock_git_repository, else: assert(nbranches == 2) assert(ncommits == 1) + + +@pytest.mark.disable_clean_stage_check +@pytest.mark.parametrize("submodules", [True, False]) +def test_gitsubmodule(submodules, mock_git_repository, config, + mutable_mock_repo): + """ + Test GitFetchStrategy behavior with submodules + """ + type_of_test = 'tag-branch' + t = mock_git_repository.checks[type_of_test] + + # Construct the package under test + spec = Spec('git-test') + spec.concretize() + pkg = spack.repo.get(spec) + args = copy.copy(t.args) + args['submodules'] = submodules + pkg.versions[ver('git')] = args + pkg.do_stage() + with working_dir(pkg.stage.source_path): + for submodule_count in range(2): + file_path = os.path.join(pkg.stage.source_path, + 'third_party/submodule{0}/r0_file_{0}' + .format(submodule_count)) + if submodules: + assert os.path.isfile(file_path) + else: + assert not os.path.isfile(file_path) + + +@pytest.mark.disable_clean_stage_check +def test_gitsubmodules_delete(mock_git_repository, config, mutable_mock_repo): + """ + Test GitFetchStrategy behavior with submodules_delete + """ + type_of_test = 'tag-branch' + t = mock_git_repository.checks[type_of_test] + + # Construct the package under test + spec = Spec('git-test') + spec.concretize() + pkg = spack.repo.get(spec) + args = copy.copy(t.args) + args['submodules'] = True + args['submodules_delete'] = ['third_party/submodule0', + 'third_party/submodule1'] + pkg.versions[ver('git')] = args + pkg.do_stage() + with working_dir(pkg.stage.source_path): + file_path = os.path.join(pkg.stage.source_path, + 'third_party/submodule0') + assert not os.path.isdir(file_path) + file_path = os.path.join(pkg.stage.source_path, + 'third_party/submodule1') + assert not os.path.isdir(file_path) diff --git a/var/spack/repos/builtin/packages/py-torch/package.py b/var/spack/repos/builtin/packages/py-torch/package.py index 2a20235bef..e40bc840fe 100644 --- a/var/spack/repos/builtin/packages/py-torch/package.py +++ b/var/spack/repos/builtin/packages/py-torch/package.py @@ -58,7 +58,8 @@ class PyTorch(PythonPackage, CudaPackage): version('1.1.0', tag='v1.1.0', submodules=True) version('1.0.1', tag='v1.0.1', submodules=True) version('1.0.0', tag='v1.0.0', submodules=True) - version('0.4.1', tag='v0.4.1', submodules=True) + version('0.4.1', tag='v0.4.1', submodules=True, + submodules_delete=['third_party/nervanagpu']) version('0.4.0', tag='v0.4.0', submodules=True) version('0.3.1', tag='v0.3.1', submodules=True) |