diff options
| -rw-r--r-- | lib/spack/spack/mirror.py | 26 | ||||
| -rw-r--r-- | lib/spack/spack/test/web.py | 23 | ||||
| -rw-r--r-- | lib/spack/spack/util/s3.py | 42 |
3 files changed, 62 insertions, 29 deletions
diff --git a/lib/spack/spack/mirror.py b/lib/spack/spack/mirror.py index f6e5d38091..9b6fd5ba3c 100644 --- a/lib/spack/spack/mirror.py +++ b/lib/spack/spack/mirror.py @@ -90,7 +90,9 @@ class Mirror(object): def to_dict(self): if self._push_url is None: - return self._fetch_url + return syaml_dict([ + ('fetch', self._fetch_url), + ('push', self._fetch_url)]) else: return syaml_dict([ ('fetch', self._fetch_url), @@ -105,12 +107,12 @@ class Mirror(object): def display(self, max_len=0): if self._push_url is None: - _display_mirror_entry(max_len, self._name, self._fetch_url) + _display_mirror_entry(max_len, self._name, self.fetch_url) else: _display_mirror_entry( - max_len, self._name, self._fetch_url, "fetch") + max_len, self._name, self.fetch_url, "fetch") _display_mirror_entry( - max_len, self._name, self._push_url, "push") + max_len, self._name, self.push_url, "push") def __str__(self): name = self._name @@ -145,8 +147,8 @@ class Mirror(object): def get_profile(self, url_type): if isinstance(self._fetch_url, dict): if url_type == "push": - return self._push_url['profile'] - return self._fetch_url['profile'] + return self._push_url.get('profile', None) + return self._fetch_url.get('profile', None) else: return None @@ -159,8 +161,8 @@ class Mirror(object): def get_access_pair(self, url_type): if isinstance(self._fetch_url, dict): if url_type == "push": - return self._push_url['access_pair'] - return self._fetch_url['access_pair'] + return self._push_url.get('access_pair', None) + return self._fetch_url.get('access_pair', None) else: return None @@ -173,8 +175,8 @@ class Mirror(object): def get_endpoint_url(self, url_type): if isinstance(self._fetch_url, dict): if url_type == "push": - return self._push_url['endpoint_url'] - return self._fetch_url['endpoint_url'] + return self._push_url.get('endpoint_url', None) + return self._fetch_url.get('endpoint_url', None) else: return None @@ -187,8 +189,8 @@ class Mirror(object): def get_access_token(self, url_type): if isinstance(self._fetch_url, dict): if url_type == "push": - return self._push_url['access_token'] - return self._fetch_url['access_token'] + return self._push_url.get('access_token', None) + return self._fetch_url.get('access_token', None) else: return None diff --git a/lib/spack/spack/test/web.py b/lib/spack/spack/test/web.py index 4ac0da162b..41aa1e6121 100644 --- a/lib/spack/spack/test/web.py +++ b/lib/spack/spack/test/web.py @@ -246,6 +246,29 @@ class MockS3Client(object): raise self.ClientError +def test_gather_s3_information(monkeypatch, capfd): + mock_connection_data = {"access_token": "AAAAAAA", + "profile": "SPacKDeV", + "access_pair": ("SPA", "CK"), + "endpoint_url": "https://127.0.0.1:8888"} + + session_args, client_args = spack.util.s3.get_mirror_s3_connection_info(mock_connection_data) # noqa: E501 + + # Session args are used to create the S3 Session object + assert "aws_session_token" in session_args + assert session_args.get("aws_session_token") == "AAAAAAA" + assert "aws_access_key_id" in session_args + assert session_args.get("aws_access_key_id") == "SPA" + assert "aws_secret_access_key" in session_args + assert session_args.get("aws_secret_access_key") == "CK" + assert "profile_name" in session_args + assert session_args.get("profile_name") == "SPacKDeV" + + # In addition to the session object, use the client_args to create the s3 + # Client object + assert "endpoint_url" in client_args + + def test_remove_s3_url(monkeypatch, capfd): fake_s3_url = 's3://my-bucket/subdirectory/mirror' diff --git a/lib/spack/spack/util/s3.py b/lib/spack/spack/util/s3.py index c120b4d5c5..82d51eb72e 100644 --- a/lib/spack/spack/util/s3.py +++ b/lib/spack/spack/util/s3.py @@ -27,6 +27,30 @@ def _parse_s3_endpoint_url(endpoint_url): return endpoint_url +def get_mirror_s3_connection_info(connection): + s3_connection = {} + + s3_connection_is_dict = connection and isinstance(connection, dict) + if s3_connection_is_dict: + if connection.get("access_token"): + s3_connection["aws_session_token"] = connection["access_token"] + if connection.get("access_pair"): + s3_connection["aws_access_key_id"] = connection["access_pair"][0] + s3_connection["aws_secret_access_key"] = connection["access_pair"][1] + if connection.get("profile"): + s3_connection["profile_name"] = connection["profile"] + + s3_client_args = {"use_ssl": spack.config.get('config:verify_ssl')} + + endpoint_url = os.environ.get('S3_ENDPOINT_URL') + if endpoint_url: + s3_client_args['endpoint_url'] = _parse_s3_endpoint_url(endpoint_url) + elif s3_connection_is_dict and connection.get("endpoint_url"): + s3_client_args["endpoint_url"] = _parse_s3_endpoint_url(connection["endpoint_url"]) # noqa: E501 + + return (s3_connection, s3_client_args) + + def create_s3_session(url, connection={}): url = url_util.parse(url) if url.scheme != 's3': @@ -40,25 +64,9 @@ def create_s3_session(url, connection={}): from boto3 import Session from botocore.exceptions import ClientError - s3_connection = {} - - if connection: - if connection['access_token']: - s3_connection["aws_session_token"] = connection["access_token"] - if connection["access_pair"][0]: - s3_connection["aws_access_key_id"] = connection["access_pair"][0] - s3_connection["aws_secret_access_key"] = connection["access_pair"][1] - if connection["profile"]: - s3_connection["profile_name"] = connection["profile"] + s3_connection, s3_client_args = get_mirror_s3_connection_info(connection) session = Session(**s3_connection) - s3_client_args = {"use_ssl": spack.config.get('config:verify_ssl')} - - endpoint_url = os.environ.get('S3_ENDPOINT_URL') - if endpoint_url: - s3_client_args['endpoint_url'] = _parse_s3_endpoint_url(endpoint_url) - elif connection and 'endpoint_url' in connection: - s3_client_args["endpoint_url"] = _parse_s3_endpoint_url(connection["endpoint_url"]) # noqa: E501 # if no access credentials provided above, then access anonymously if not session.get_credentials(): from botocore import UNSIGNED |