summaryrefslogtreecommitdiff
path: root/lib
diff options
context:
space:
mode:
authorBen Boeckel <ben.boeckel@kitware.com>2016-04-27 14:46:57 -0400
committerBen Boeckel <ben.boeckel@kitware.com>2016-04-27 14:49:27 -0400
commite53571d2f00d2640005a10d69edbad838cb38fef (patch)
treeabf5a0415da11b447c2a988660565e943112f932 /lib
parent92afa52eecb379d99ed14a4e805355e6fc7760db (diff)
downloadspack-e53571d2f00d2640005a10d69edbad838cb38fef.tar.gz
spack-e53571d2f00d2640005a10d69edbad838cb38fef.tar.bz2
spack-e53571d2f00d2640005a10d69edbad838cb38fef.tar.xz
spack-e53571d2f00d2640005a10d69edbad838cb38fef.zip
fetch_strategy: download to temporary files
This supports graceful recovery if spack is killed via a signal (e.g., SIGINT) while downloading a file. Fixes #287.
Diffstat (limited to 'lib')
-rw-r--r--lib/spack/spack/fetch_strategy.py24
-rw-r--r--lib/spack/spack/stage.py12
2 files changed, 34 insertions, 2 deletions
diff --git a/lib/spack/spack/fetch_strategy.py b/lib/spack/spack/fetch_strategy.py
index 4ea87bea7e..ce2c4e30c7 100644
--- a/lib/spack/spack/fetch_strategy.py
+++ b/lib/spack/spack/fetch_strategy.py
@@ -157,12 +157,26 @@ class URLFetchStrategy(FetchStrategy):
tty.msg("Already downloaded %s" % self.archive_file)
return
+ possible_files = self.stage.expected_archive_files
+ save_file = None
+ partial_file = None
+ if possible_files:
+ save_file = self.stage.expected_archive_files[0]
+ partial_file = self.stage.expected_archive_files[0] + '.part'
+
tty.msg("Trying to fetch from %s" % self.url)
- curl_args = ['-O', # save file to disk
+ if partial_file:
+ save_args = ['-C', '-', # continue partial downloads
+ '-o', partial_file] # use a .part file
+ else:
+ save_args = ['-O']
+
+ curl_args = save_args + [
'-f', # fail on >400 errors
'-D', '-', # print out HTML headers
- '-L', self.url, ]
+ '-L', # resolve 3xx redirects
+ self.url, ]
if sys.stdout.isatty():
curl_args.append('-#') # status bar when using a tty
@@ -178,6 +192,9 @@ class URLFetchStrategy(FetchStrategy):
if self.archive_file:
os.remove(self.archive_file)
+ if partial_file and os.path.exists(partial_file):
+ os.remove(partial_file)
+
if spack.curl.returncode == 22:
# This is a 404. Curl will print the error.
raise FailedDownloadError(
@@ -209,6 +226,9 @@ class URLFetchStrategy(FetchStrategy):
"'spack clean <package>' to remove the bad archive, then fix",
"your internet gateway issue and install again.")
+ if save_file:
+ os.rename(partial_file, save_file)
+
if not self.archive_file:
raise FailedDownloadError(self.url)
diff --git a/lib/spack/spack/stage.py b/lib/spack/spack/stage.py
index d711752c20..84c47ee660 100644
--- a/lib/spack/spack/stage.py
+++ b/lib/spack/spack/stage.py
@@ -211,6 +211,18 @@ class Stage(object):
return False
@property
+ def expected_archive_files(self):
+ """Possible archive file paths."""
+ paths = []
+ if isinstance(self.fetcher, fs.URLFetchStrategy):
+ paths.append(os.path.join(self.path, os.path.basename(self.fetcher.url)))
+
+ if self.mirror_path:
+ paths.append(os.path.join(self.path, os.path.basename(self.mirror_path)))
+
+ return paths
+
+ @property
def archive_file(self):
"""Path to the source archive within this stage directory."""
paths = []