From 7659912dc40bc42df3eb108471974260b3e3b6e8 Mon Sep 17 00:00:00 2001 From: roottreej <119509268+roottreej@users.noreply.github.com> Date: Tue, 17 Jan 2023 14:19:13 +0100 Subject: Reduce verbosity in mirrors.yaml (#34210) Ensure `spack mirror add ` without further arguments translates to `: ` key value pairs in mirrors.yaml. If --s3-* flags are provided, only store the provided ones. Co-authored-by: Harmen Stoppels --- lib/spack/spack/ci.py | 10 ++++++++-- lib/spack/spack/cmd/ci.py | 10 ++++++---- lib/spack/spack/cmd/mirror.py | 21 ++++++++++++++++++++- lib/spack/spack/mirror.py | 25 ++++++++----------------- 4 files changed, 42 insertions(+), 24 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/ci.py b/lib/spack/spack/ci.py index 11b5f608d6..5ea451a94e 100644 --- a/lib/spack/spack/ci.py +++ b/lib/spack/spack/ci.py @@ -727,13 +727,19 @@ def generate_gitlab_ci_yaml( # --check-index-only, then the override mirror needs to be added to # the configured mirrors when bindist.update() is run, or else we # won't fetch its index and include in our local cache. - spack.mirror.add("ci_pr_mirror", remote_mirror_override, cfg.default_modify_scope()) + spack.mirror.add( + spack.mirror.Mirror(remote_mirror_override, name="ci_pr_mirror"), + cfg.default_modify_scope(), + ) shared_pr_mirror = None if spack_pipeline_type == "spack_pull_request": stack_name = os.environ.get("SPACK_CI_STACK_NAME", "") shared_pr_mirror = url_util.join(SHARED_PR_MIRROR_URL, stack_name) - spack.mirror.add("ci_shared_pr_mirror", shared_pr_mirror, cfg.default_modify_scope()) + spack.mirror.add( + spack.mirror.Mirror(shared_pr_mirror, name="ci_shared_pr_mirror"), + cfg.default_modify_scope(), + ) pipeline_artifacts_dir = artifacts_root if not pipeline_artifacts_dir: diff --git a/lib/spack/spack/cmd/ci.py b/lib/spack/spack/cmd/ci.py index 54770f6ae3..c674911de1 100644 --- a/lib/spack/spack/cmd/ci.py +++ b/lib/spack/spack/cmd/ci.py @@ -453,9 +453,8 @@ def ci_rebuild(args): # mirror now so it's used when we check for a hash match already # built for this spec. if pipeline_mirror_url: - spack.mirror.add( - spack_ci.TEMP_STORAGE_MIRROR_NAME, pipeline_mirror_url, cfg.default_modify_scope() - ) + mirror = spack.mirror.Mirror(pipeline_mirror_url, name=spack_ci.TEMP_STORAGE_MIRROR_NAME) + spack.mirror.add(mirror, cfg.default_modify_scope()) pipeline_mirrors.append(pipeline_mirror_url) # Check configured mirrors for a built spec with a matching hash @@ -470,7 +469,10 @@ def ci_rebuild(args): # could be installed from either the override mirror or any other configured # mirror (e.g. remote_mirror_url which is defined in the environment or # pipeline_mirror_url), which is also what we want. - spack.mirror.add("mirror_override", remote_mirror_override, cfg.default_modify_scope()) + spack.mirror.add( + spack.mirror.Mirror(remote_mirror_override, name="mirror_override"), + cfg.default_modify_scope(), + ) pipeline_mirrors.append(remote_mirror_override) if spack_pipeline_type == "spack_pull_request": diff --git a/lib/spack/spack/cmd/mirror.py b/lib/spack/spack/cmd/mirror.py index 5c5ae7abc9..7d5f96c5c2 100644 --- a/lib/spack/spack/cmd/mirror.py +++ b/lib/spack/spack/cmd/mirror.py @@ -145,7 +145,26 @@ def setup_parser(subparser): def mirror_add(args): """Add a mirror to Spack.""" - spack.mirror.add(args.name, args.url, args.scope, args) + if ( + args.s3_access_key_id + or args.s3_access_key_secret + or args.s3_access_token + or args.s3_profile + or args.s3_endpoint_url + ): + connection = {"url": args.url} + if args.s3_access_key_id and args.s3_access_key_secret: + connection["access_pair"] = (args.s3_access_key_id, args.s3_access_key_secret) + if args.s3_access_token: + connection["access_token"] = args.s3_access_token + if args.s3_profile: + connection["profile"] = args.s3_profile + if args.s3_endpoint_url: + connection["endpoint_url"] = args.s3_endpoint_url + mirror = spack.mirror.Mirror(fetch_url=connection, push_url=connection, name=args.name) + else: + mirror = spack.mirror.Mirror(args.url, name=args.name) + spack.mirror.add(mirror, args.scope) def mirror_remove(args): diff --git a/lib/spack/spack/mirror.py b/lib/spack/spack/mirror.py index 2b741b06ad..59f2c1ec7c 100644 --- a/lib/spack/spack/mirror.py +++ b/lib/spack/spack/mirror.py @@ -119,6 +119,10 @@ class Mirror(object): return Mirror(fetch_url=url) def to_dict(self): + # Keep it a key-value pair : when possible. + if isinstance(self._fetch_url, str) and self._push_url is None: + return self._fetch_url + if self._push_url is None: return syaml_dict([("fetch", self._fetch_url), ("push", self._fetch_url)]) else: @@ -556,30 +560,17 @@ def mirror_cache_and_stats(path, skip_unstable_versions=False): return mirror_cache, mirror_stats -def add(name, url, scope, args={}): +def add(mirror: Mirror, scope=None): """Add a named mirror in the given scope""" mirrors = spack.config.get("mirrors", scope=scope) if not mirrors: mirrors = syaml_dict() - if name in mirrors: - tty.die("Mirror with name %s already exists." % name) + if mirror.name in mirrors: + tty.die("Mirror with name {} already exists.".format(mirror.name)) items = [(n, u) for n, u in mirrors.items()] - mirror_data = url - key_values = ["s3_access_key_id", "s3_access_token", "s3_profile"] - # On creation, assume connection data is set for both - if any(value for value in key_values if value in args): - url_dict = { - "url": url, - "access_pair": (args.s3_access_key_id, args.s3_access_key_secret), - "access_token": args.s3_access_token, - "profile": args.s3_profile, - "endpoint_url": args.s3_endpoint_url, - } - mirror_data = {"fetch": url_dict, "push": url_dict} - - items.insert(0, (name, mirror_data)) + items.insert(0, (mirror.name, mirror.to_dict())) mirrors = syaml_dict(items) spack.config.set("mirrors", mirrors, scope=scope) -- cgit v1.2.3-60-g2f50