summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorHarmen Stoppels <harmenstoppels@gmail.com>2021-07-19 13:39:17 +0200
committerGitHub <noreply@github.com>2021-07-19 13:39:17 +0200
commitfc50e04b594e4f76bcd037e0ec82d508d1be2c7d (patch)
treeadf463d4d39e317e18632be36a1aecf2849f5029
parent1bf9c10f0c4ec9732bf303b0a00fea674800ec91 (diff)
downloadspack-fc50e04b594e4f76bcd037e0ec82d508d1be2c7d.tar.gz
spack-fc50e04b594e4f76bcd037e0ec82d508d1be2c7d.tar.bz2
spack-fc50e04b594e4f76bcd037e0ec82d508d1be2c7d.tar.xz
spack-fc50e04b594e4f76bcd037e0ec82d508d1be2c7d.zip
Fix S3 urlparse scheme error and add test (#24952)
-rw-r--r--lib/spack/spack/test/web.py5
-rw-r--r--lib/spack/spack/util/s3.py12
2 files changed, 13 insertions, 4 deletions
diff --git a/lib/spack/spack/test/web.py b/lib/spack/spack/test/web.py
index c0ed098b2a..2ccbf51225 100644
--- a/lib/spack/spack/test/web.py
+++ b/lib/spack/spack/test/web.py
@@ -279,3 +279,8 @@ def test_s3_url_exists(monkeypatch, capfd):
fake_s3_url_does_not_exist = 's3://my-bucket/subdirectory/my-notfound-file'
assert(not spack.util.web.url_exists(fake_s3_url_does_not_exist))
+
+
+def test_s3_url_parsing():
+ assert(spack.util.s3._parse_s3_endpoint_url("example.com") == 'https://example.com')
+ assert(spack.util.s3._parse_s3_endpoint_url("http://example.com") == 'http://example.com')
diff --git a/lib/spack/spack/util/s3.py b/lib/spack/spack/util/s3.py
index 31a18093e1..b9b56e0498 100644
--- a/lib/spack/spack/util/s3.py
+++ b/lib/spack/spack/util/s3.py
@@ -11,6 +11,13 @@ import spack
import spack.util.url as url_util
+def _parse_s3_endpoint_url(endpoint_url):
+ if not urllib_parse.urlparse(endpoint_url, scheme='').scheme:
+ endpoint_url = '://'.join(('https', endpoint_url))
+
+ return endpoint_url
+
+
def create_s3_session(url):
url = url_util.parse(url)
if url.scheme != 's3':
@@ -30,10 +37,7 @@ def create_s3_session(url):
endpoint_url = os.environ.get('S3_ENDPOINT_URL')
if endpoint_url:
- if urllib_parse.urlparse(endpoint_url, scheme=None).scheme is None:
- endpoint_url = '://'.join(('https', endpoint_url))
-
- s3_client_args['endpoint_url'] = endpoint_url
+ s3_client_args['endpoint_url'] = _parse_s3_endpoint_url(endpoint_url)
# if no access credentials provided above, then access anonymously
if not session.get_credentials():