summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorTamara Dahlgren <35777542+tldahlgren@users.noreply.github.com>2020-08-19 12:10:18 -0700
committerGitHub <noreply@github.com>2020-08-19 12:10:18 -0700
commit1c0a92662bee39bbd56ea5457aad3a31e45151b6 (patch)
treecc538dc3a76ce6992766ef3e7846a429e2080ecd
parent8e8071ffb6def6544f8c7d312d646312be8edaae (diff)
downloadspack-1c0a92662bee39bbd56ea5457aad3a31e45151b6.tar.gz
spack-1c0a92662bee39bbd56ea5457aad3a31e45151b6.tar.bz2
spack-1c0a92662bee39bbd56ea5457aad3a31e45151b6.tar.xz
spack-1c0a92662bee39bbd56ea5457aad3a31e45151b6.zip
Restore curl progress output (#18127)
Restores the fetching progress bar sans failure outputs; restores non-debug reporting of using fetch cache for installed packages; and adds a unit test. * Add status bar check to test and fetch output when already installed
-rw-r--r--lib/spack/spack/fetch_strategy.py20
-rw-r--r--lib/spack/spack/test/url_fetch.py21
2 files changed, 36 insertions, 5 deletions
diff --git a/lib/spack/spack/fetch_strategy.py b/lib/spack/spack/fetch_strategy.py
index fb87373897..97d95c2fe9 100644
--- a/lib/spack/spack/fetch_strategy.py
+++ b/lib/spack/spack/fetch_strategy.py
@@ -295,6 +295,9 @@ class URLFetchStrategy(FetchStrategy):
url = None
errors = []
for url in self.candidate_urls:
+ if not self._existing_url(url):
+ continue
+
try:
partial_file, save_file = self._fetch_from_url(url)
if save_file:
@@ -309,13 +312,22 @@ class URLFetchStrategy(FetchStrategy):
if not self.archive_file:
raise FailedDownloadError(url)
+ def _existing_url(self, url):
+ tty.debug('Checking existence of {0}'.format(url))
+ curl = self.curl
+ # Telling curl to fetch the first byte (-r 0-0) is supposed to be
+ # portable.
+ curl_args = ['--stderr', '-', '-s', '-f', '-r', '0-0', url]
+ _ = curl(*curl_args, fail_on_error=False, output=os.devnull)
+ return curl.returncode == 0
+
def _fetch_from_url(self, url):
save_file = None
partial_file = None
if self.stage.save_filename:
save_file = self.stage.save_filename
partial_file = self.stage.save_filename + '.part'
- tty.debug('Fetching {0}'.format(url))
+ tty.msg('Fetching {0}'.format(url))
if partial_file:
save_args = ['-C',
'-', # continue partial downloads
@@ -330,8 +342,6 @@ class URLFetchStrategy(FetchStrategy):
'-', # print out HTML headers
'-L', # resolve 3xx redirects
url,
- '--stderr', # redirect stderr output
- '-', # redirect to stdout
]
if not spack.config.get('config:verify_ssl'):
@@ -340,7 +350,7 @@ class URLFetchStrategy(FetchStrategy):
if sys.stdout.isatty() and tty.msg_enabled():
curl_args.append('-#') # status bar when using a tty
else:
- curl_args.append('-sS') # just errors when not.
+ curl_args.append('-sS') # show errors if fail
connect_timeout = spack.config.get('config:connect_timeout', 10)
@@ -569,7 +579,7 @@ class CacheURLFetchStrategy(URLFetchStrategy):
raise
# Notify the user how we fetched.
- tty.debug('Using cached archive: {0}'.format(path))
+ tty.msg('Using cached archive: {0}'.format(path))
class VCSFetchStrategy(FetchStrategy):
diff --git a/lib/spack/spack/test/url_fetch.py b/lib/spack/spack/test/url_fetch.py
index 20648b4766..8a9a175a37 100644
--- a/lib/spack/spack/test/url_fetch.py
+++ b/lib/spack/spack/test/url_fetch.py
@@ -6,8 +6,10 @@
import collections
import os
import pytest
+import sys
from llnl.util.filesystem import working_dir, is_exe
+import llnl.util.tty as tty
import spack.repo
import spack.config
@@ -173,6 +175,25 @@ def test_unknown_hash(checksum_type):
crypto.Checker('a')
+def test_url_with_status_bar(tmpdir, mock_archive, monkeypatch, capfd):
+ """Ensure fetch with status bar option succeeds."""
+ def is_true():
+ return True
+
+ testpath = str(tmpdir)
+
+ monkeypatch.setattr(sys.stdout, 'isatty', is_true)
+ monkeypatch.setattr(tty, 'msg_enabled', is_true)
+
+ fetcher = fs.URLFetchStrategy(mock_archive.url)
+ with Stage(fetcher, path=testpath) as stage:
+ assert fetcher.archive_file is None
+ stage.fetch()
+
+ status = capfd.readouterr()[1]
+ assert '##### 100.0%' in status
+
+
def test_url_extra_fetch(tmpdir, mock_archive):
"""Ensure a fetch after downloading is effectively a no-op."""
testpath = str(tmpdir)