From fc50e04b594e4f76bcd037e0ec82d508d1be2c7d Mon Sep 17 00:00:00 2001 From: Harmen Stoppels Date: Mon, 19 Jul 2021 13:39:17 +0200 Subject: Fix S3 urlparse scheme error and add test (#24952) --- lib/spack/spack/test/web.py | 5 +++++ lib/spack/spack/util/s3.py | 12 ++++++++---- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/lib/spack/spack/test/web.py b/lib/spack/spack/test/web.py index c0ed098b2a..2ccbf51225 100644 --- a/lib/spack/spack/test/web.py +++ b/lib/spack/spack/test/web.py @@ -279,3 +279,8 @@ def test_s3_url_exists(monkeypatch, capfd): fake_s3_url_does_not_exist = 's3://my-bucket/subdirectory/my-notfound-file' assert(not spack.util.web.url_exists(fake_s3_url_does_not_exist)) + + +def test_s3_url_parsing(): + assert(spack.util.s3._parse_s3_endpoint_url("example.com") == 'https://example.com') + assert(spack.util.s3._parse_s3_endpoint_url("http://example.com") == 'http://example.com') diff --git a/lib/spack/spack/util/s3.py b/lib/spack/spack/util/s3.py index 31a18093e1..b9b56e0498 100644 --- a/lib/spack/spack/util/s3.py +++ b/lib/spack/spack/util/s3.py @@ -11,6 +11,13 @@ import spack import spack.util.url as url_util +def _parse_s3_endpoint_url(endpoint_url): + if not urllib_parse.urlparse(endpoint_url, scheme='').scheme: + endpoint_url = '://'.join(('https', endpoint_url)) + + return endpoint_url + + def create_s3_session(url): url = url_util.parse(url) if url.scheme != 's3': @@ -30,10 +37,7 @@ def create_s3_session(url): endpoint_url = os.environ.get('S3_ENDPOINT_URL') if endpoint_url: - if urllib_parse.urlparse(endpoint_url, scheme=None).scheme is None: - endpoint_url = '://'.join(('https', endpoint_url)) - - s3_client_args['endpoint_url'] = endpoint_url + s3_client_args['endpoint_url'] = _parse_s3_endpoint_url(endpoint_url) # if no access credentials provided above, then access anonymously if not session.get_credentials(): -- cgit v1.2.3-70-g09d2