From b51be2bb1bc9dd3c422af2f159c94bc0277fcecc Mon Sep 17 00:00:00 2001 From: Glenn Johnson Date: Sat, 23 Jul 2016 18:11:09 -0500 Subject: Have fetch use list_url This PR allows archive file retrieval from urls derived from the `list_url` setting in a package file. This allows for continued retrieval of checksummed archive files even when they are moved to a new remote location when a package is updated upstream. --- lib/spack/spack/stage.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) (limited to 'lib') diff --git a/lib/spack/spack/stage.py b/lib/spack/spack/stage.py index b08cce43b8..0914afe3a7 100644 --- a/lib/spack/spack/stage.py +++ b/lib/spack/spack/stage.py @@ -37,6 +37,7 @@ import spack import spack.config import spack.fetch_strategy as fs import spack.error +from spack.version import Version STAGE_PREFIX = 'spack-stage-' @@ -306,6 +307,18 @@ class Stage(object): fetchers.insert(0, fs.URLFetchStrategy(url, digest)) fetchers.insert(0, spack.cache.fetcher(self.mirror_path, digest)) + # Look for the archive in list_url + archive_version = spack.url.parse_version(self.default_fetcher.url) + package_name = os.path.dirname(self.mirror_path) + pkg = spack.repo.get(package_name) + versions = pkg.fetch_remote_versions() + try: + url_from_list = versions[Version(archive_version)] + fetchers.append(fs.URLFetchStrategy(url_from_list, digest)) + except KeyError: + tty.msg("Can not find version %s in url_list" % + archive_version) + for fetcher in fetchers: try: fetcher.set_stage(self) -- cgit v1.2.3-60-g2f50 From 4181fd79cd6993ccb5cb95c4d3dd8d2cdf0b17c3 Mon Sep 17 00:00:00 2001 From: Glenn Johnson Date: Sat, 23 Jul 2016 19:00:57 -0500 Subject: Fix flake8 errors related to lines Fixed the flake 8 errors that involved too many blank lines or not enough blank lines. Basically, all of the flake8 errors except line length errors. --- lib/spack/spack/stage.py | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/stage.py b/lib/spack/spack/stage.py index 0914afe3a7..cf1c9d7b50 100644 --- a/lib/spack/spack/stage.py +++ b/lib/spack/spack/stage.py @@ -144,7 +144,6 @@ class Stage(object): # Flag to decide whether to delete the stage folder on exit or not self.keep = keep - def __enter__(self): """ Entering a stage context will create the stage directory @@ -155,7 +154,6 @@ class Stage(object): self.create() return self - def __exit__(self, exc_type, exc_val, exc_tb): """ Exiting from a stage context will delete the stage directory unless: @@ -174,7 +172,6 @@ class Stage(object): if exc_type is None and not self.keep: self.destroy() - def _need_to_create_path(self): """Makes sure nothing weird has happened since the last time we looked at path. Returns True if path already exists and is ok. @@ -334,7 +331,6 @@ class Stage(object): self.fetcher = self.default_fetcher raise fs.FetchError(errMessage, None) - def check(self): """Check the downloaded archive against a checksum digest. No-op if this stage checks code out of a repository.""" @@ -348,11 +344,9 @@ class Stage(object): else: self.fetcher.check() - def cache_local(self): spack.cache.store(self.fetcher, self.mirror_path) - def expand_archive(self): """Changes to the stage directory and attempt to expand the downloaded archive. Fail if the stage is not set up or if the archive is not yet @@ -509,8 +503,11 @@ class DIYStage(object): raise ChdirError("Setup failed: no such directory: " + self.path) # DIY stages do nothing as context managers. - def __enter__(self): pass - def __exit__(self, exc_type, exc_val, exc_tb): pass + def __enter__(self): + pass + + def __exit__(self, exc_type, exc_val, exc_tb): + pass def chdir_to_source(self): self.chdir() -- cgit v1.2.3-60-g2f50 From 4f09e8c9759473257ca8857bfa09eeeeca08cafd Mon Sep 17 00:00:00 2001 From: Glenn Johnson Date: Sat, 23 Jul 2016 22:56:08 -0500 Subject: Only use list if list_url set This commit will make urls from list_url only checked if `list_url` is set in the package file. This makes more sense as there is no need to check for those if the attribute is not present. If `url` is present and `list_url` is not then it would result in the same url. If `url_for_version` is used then that will not work anyway. --- lib/spack/spack/stage.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/stage.py b/lib/spack/spack/stage.py index cf1c9d7b50..6cf736b3f0 100644 --- a/lib/spack/spack/stage.py +++ b/lib/spack/spack/stage.py @@ -37,7 +37,7 @@ import spack import spack.config import spack.fetch_strategy as fs import spack.error -from spack.version import Version +from spack.version import * STAGE_PREFIX = 'spack-stage-' @@ -308,13 +308,14 @@ class Stage(object): archive_version = spack.url.parse_version(self.default_fetcher.url) package_name = os.path.dirname(self.mirror_path) pkg = spack.repo.get(package_name) - versions = pkg.fetch_remote_versions() - try: - url_from_list = versions[Version(archive_version)] - fetchers.append(fs.URLFetchStrategy(url_from_list, digest)) - except KeyError: - tty.msg("Can not find version %s in url_list" % - archive_version) + if pkg.list_url is not None: + versions = pkg.fetch_remote_versions() + try: + url_from_list = versions[Version(archive_version)] + fetchers.append(fs.URLFetchStrategy(url_from_list, digest)) + except KeyError: + tty.msg("Can not find version %s in url_list" % + archive_version) for fetcher in fetchers: try: -- cgit v1.2.3-60-g2f50 From b1e5ec05739b05ed3781473b0a27b04f7b053ce6 Mon Sep 17 00:00:00 2001 From: Glenn Johnson Date: Sun, 24 Jul 2016 09:21:11 -0500 Subject: Make sure package has the `url` attribute. In addition to `list_url` make sure the package has the `url` attribute set before attempting to add urls from a list. This is to cover the case where there may be a `list_url` specified in tandem with a `url_for_version`. --- lib/spack/spack/stage.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/spack/spack/stage.py b/lib/spack/spack/stage.py index 6cf736b3f0..8f3f0e163a 100644 --- a/lib/spack/spack/stage.py +++ b/lib/spack/spack/stage.py @@ -308,7 +308,7 @@ class Stage(object): archive_version = spack.url.parse_version(self.default_fetcher.url) package_name = os.path.dirname(self.mirror_path) pkg = spack.repo.get(package_name) - if pkg.list_url is not None: + if pkg.list_url is not None and pkg.url is not None: versions = pkg.fetch_remote_versions() try: url_from_list = versions[Version(archive_version)] -- cgit v1.2.3-60-g2f50 From 19578d954fd9b99db657cb4e9dc0da7290191093 Mon Sep 17 00:00:00 2001 From: Glenn Johnson Date: Mon, 1 Aug 2016 13:37:42 -0500 Subject: Fix flake8 line length errors --- lib/spack/spack/stage.py | 85 +++++++++++++++++++++++++++++------------------- 1 file changed, 52 insertions(+), 33 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/stage.py b/lib/spack/spack/stage.py index 8f3f0e163a..1a8b1a169a 100644 --- a/lib/spack/spack/stage.py +++ b/lib/spack/spack/stage.py @@ -52,10 +52,12 @@ class Stage(object): lifecycle looks like this: ``` - with Stage() as stage: # Context manager creates and destroys the stage directory + with Stage() as stage: # Context manager creates and destroys the + # stage directory stage.fetch() # Fetch a source archive into the stage. stage.expand_archive() # Expand the source archive. - # Build and install the archive. (handled by user of Stage) + # Build and install the archive. (handled by + # user of Stage) ``` When used as a context manager, the stage is automatically @@ -72,7 +74,8 @@ class Stage(object): stage.create() # Explicitly create the stage directory. stage.fetch() # Fetch a source archive into the stage. stage.expand_archive() # Expand the source archive. - # Build and install the archive. (handled by user of Stage) + # Build and install the archive. (handled by + # user of Stage) finally: stage.destroy() # Explicitly destroy the stage directory. ``` @@ -121,13 +124,17 @@ class Stage(object): elif isinstance(url_or_fetch_strategy, fs.FetchStrategy): self.fetcher = url_or_fetch_strategy else: - raise ValueError("Can't construct Stage without url or fetch strategy") + raise ValueError( + "Can't construct Stage without url or fetch strategy") self.fetcher.set_stage(self) - self.default_fetcher = self.fetcher # self.fetcher can change with mirrors. - self.skip_checksum_for_mirror = True # used for mirrored archives of repositories. + # self.fetcher can change with mirrors. + self.default_fetcher = self.fetcher + # used for mirrored archives of repositories. + self.skip_checksum_for_mirror = True - # TODO : this uses a protected member of tempfile, but seemed the only way to get a temporary name - # TODO : besides, the temporary link name won't be the same as the temporary stage area in tmp_root + # TODO : this uses a protected member of tempfile, but seemed the only + # TODO : way to get a temporary name besides, the temporary link name + # TODO : won't be the same as the temporary stage area in tmp_root self.name = name if name is None: self.name = STAGE_PREFIX + next(tempfile._get_candidate_names()) @@ -175,8 +182,7 @@ class Stage(object): def _need_to_create_path(self): """Makes sure nothing weird has happened since the last time we looked at path. Returns True if path already exists and is ok. - Returns False if path needs to be created. - """ + Returns False if path needs to be created.""" # Path doesn't exist yet. Will need to create it. if not os.path.exists(self.path): return True @@ -194,7 +200,8 @@ class Stage(object): if spack.use_tmp_stage: # If we're using a tmp dir, it's a link, and it points at the # right spot, then keep it. - if (real_path.startswith(real_tmp) and os.path.exists(real_path)): + if (real_path.startswith(real_tmp) and + os.path.exists(real_path)): return False else: # otherwise, just unlink it and start over. @@ -202,7 +209,8 @@ class Stage(object): return True else: - # If we're not tmp mode, then it's a link and we want a directory. + # If we're not tmp mode, then it's a link and we want a + # directory. os.unlink(self.path) return True @@ -213,10 +221,12 @@ class Stage(object): """Possible archive file paths.""" paths = [] if isinstance(self.fetcher, fs.URLFetchStrategy): - paths.append(os.path.join(self.path, os.path.basename(self.fetcher.url))) + paths.append(os.path.join( + self.path, os.path.basename(self.fetcher.url))) if self.mirror_path: - paths.append(os.path.join(self.path, os.path.basename(self.mirror_path))) + paths.append(os.path.join( + self.path, os.path.basename(self.mirror_path))) return paths @@ -225,10 +235,12 @@ class Stage(object): """Path to the source archive within this stage directory.""" paths = [] if isinstance(self.fetcher, fs.URLFetchStrategy): - paths.append(os.path.join(self.path, os.path.basename(self.fetcher.url))) + paths.append(os.path.join( + self.path, os.path.basename(self.fetcher.url))) if self.mirror_path: - paths.append(os.path.join(self.path, os.path.basename(self.mirror_path))) + paths.append(os.path.join( + self.path, os.path.basename(self.mirror_path))) for path in paths: if os.path.exists(path): @@ -260,7 +272,8 @@ class Stage(object): return None def chdir(self): - """Changes directory to the stage path. Or dies if it is not set up.""" + """Changes directory to the stage path. Or dies if it is not set + up.""" if os.path.isdir(self.path): os.chdir(self.path) else: @@ -335,7 +348,8 @@ class Stage(object): def check(self): """Check the downloaded archive against a checksum digest. No-op if this stage checks code out of a repository.""" - if self.fetcher is not self.default_fetcher and self.skip_checksum_for_mirror: + if self.fetcher is not self.default_fetcher and \ + self.skip_checksum_for_mirror: tty.warn("Fetching from mirror without a checksum!", "This package is normally checked out from a version " "control system, but it has been archived on a spack " @@ -350,9 +364,8 @@ class Stage(object): def expand_archive(self): """Changes to the stage directory and attempt to expand the downloaded - archive. Fail if the stage is not set up or if the archive is not yet - downloaded. - """ + archive. Fail if the stage is not set up or if the archive is not yet + downloaded.""" archive_dir = self.source_path if not archive_dir: self.fetcher.expand() @@ -394,8 +407,8 @@ class Stage(object): # Create the top-level stage directory mkdirp(spack.stage_path) remove_dead_links(spack.stage_path) - # If a tmp_root exists then create a directory there and then link it in the stage area, - # otherwise create the stage directory in self.path + # If a tmp_root exists then create a directory there and then link it + # in the stage area, otherwise create the stage directory in self.path if self._need_to_create_path(): if self.tmp_root: tmp_dir = tempfile.mkdtemp('', STAGE_PREFIX, self.tmp_root) @@ -417,6 +430,7 @@ class Stage(object): class ResourceStage(Stage): + def __init__(self, url_or_fetch_strategy, root, resource, **kwargs): super(ResourceStage, self).__init__(url_or_fetch_strategy, **kwargs) self.root_stage = root @@ -426,12 +440,15 @@ class ResourceStage(Stage): super(ResourceStage, self).expand_archive() root_stage = self.root_stage resource = self.resource - placement = os.path.basename(self.source_path) if resource.placement is None else resource.placement + placement = os.path.basename(self.source_path) \ + if resource.placement is None \ + else resource.placement if not isinstance(placement, dict): placement = {'': placement} # Make the paths in the dictionary absolute and link for key, value in placement.iteritems(): - target_path = join_path(root_stage.source_path, resource.destination) + target_path = join_path( + root_stage.source_path, resource.destination) destination_path = join_path(target_path, value) source_path = join_path(self.source_path, key) @@ -445,21 +462,23 @@ class ResourceStage(Stage): if not os.path.exists(destination_path): # Create a symlink - tty.info('Moving resource stage\n\tsource : {stage}\n\tdestination : {destination}'.format( - stage=source_path, destination=destination_path - )) + tty.info('Moving resource stage\n\tsource : ' + '{stage}\n\tdestination : {destination}'.format( + stage=source_path, destination=destination_path + )) shutil.move(source_path, destination_path) -@pattern.composite(method_list=['fetch', 'create', 'check', 'expand_archive', 'restage', 'destroy', 'cache_local']) +@pattern.composite(method_list=['fetch', 'create', 'check', 'expand_archive', + 'restage', 'destroy', 'cache_local']) class StageComposite: - """ - Composite for Stage type objects. The first item in this composite is considered to be the root package, and - operations that return a value are forwarded to it. - """ + """Composite for Stage type objects. The first item in this composite is + considered to be the root package, and operations that return a value are + forwarded to it.""" # # __enter__ and __exit__ delegate to all stages in the composite. # + def __enter__(self): for item in self: item.__enter__() -- cgit v1.2.3-60-g2f50