diff options
| -rw-r--r-- | lib/spack/spack/ci.py | 356 |
1 files changed, 192 insertions, 164 deletions
diff --git a/lib/spack/spack/ci.py b/lib/spack/spack/ci.py index 7c2f3f2eb6..96dd58a4f4 100644 --- a/lib/spack/spack/ci.py +++ b/lib/spack/spack/ci.py @@ -70,7 +70,7 @@ SHARED_PR_MIRROR_URL = "s3://spack-binaries-prs/shared_pr_mirror" JOB_NAME_FORMAT = ( "{name}{@version} {/hash:7} {%compiler.name}{@compiler.version}{arch=architecture}" ) - +IS_WINDOWS = sys.platform == "win32" spack_gpg = spack.main.SpackCommand("gpg") spack_compiler = spack.main.SpackCommand("compiler") @@ -103,7 +103,7 @@ def get_job_name(spec: spack.spec.Spec, build_group: str = ""): job_name = spec.format(JOB_NAME_FORMAT) if build_group: - job_name = "{0} {1}".format(job_name, build_group) + job_name = f"{job_name} {build_group}" return job_name[:255] @@ -114,7 +114,7 @@ def _remove_reserved_tags(tags): def _spec_deps_key(s): - return "{0}/{1}".format(s.name, s.dag_hash(7)) + return f"{s.name}/{s.dag_hash(7)}" def _add_dependency(spec_label, dep_label, deps): @@ -213,7 +213,7 @@ def _print_staging_summary(spec_labels, stages, mirrors_to_check, rebuild_decisi mirrors = spack.mirror.MirrorCollection(mirrors=mirrors_to_check, binary=True) tty.msg("Checked the following mirrors for binaries:") for m in mirrors.values(): - tty.msg(" {0}".format(m.fetch_url)) + tty.msg(f" {m.fetch_url}") tty.msg("Staging summary ([x] means a job needs rebuilding):") for stage_index, stage in enumerate(stages): @@ -296,7 +296,7 @@ def _compute_spec_deps(spec_list): for spec in spec_list: for s in spec.traverse(deptype="all"): if s.external: - tty.msg("Will not stage external pkg: {0}".format(s)) + tty.msg(f"Will not stage external pkg: {s}") continue skey = _spec_deps_key(s) @@ -305,7 +305,7 @@ def _compute_spec_deps(spec_list): for d in s.dependencies(deptype="all"): dkey = _spec_deps_key(d) if d.external: - tty.msg("Will not stage external dep: {0}".format(d)) + tty.msg(f"Will not stage external dep: {d}") continue append_dep(skey, dkey) @@ -374,8 +374,8 @@ def get_stack_changed(env_path, rev1="HEAD^", rev2="HEAD"): for path in lines: if ".gitlab-ci.yml" in path or path in env_path: - tty.debug("env represented by {0} changed".format(env_path)) - tty.debug("touched file: {0}".format(path)) + tty.debug(f"env represented by {env_path} changed") + tty.debug(f"touched file: {path}") return True return False @@ -419,7 +419,7 @@ def get_spec_filter_list(env, affected_pkgs, dependent_traverse_depth=None): all_concrete_specs = env.all_specs() tty.debug("All concrete environment specs:") for s in all_concrete_specs: - tty.debug(" {0}/{1}".format(s.name, s.dag_hash()[:7])) + tty.debug(f" {s.name}/{s.dag_hash()[:7]}") affected_pkgs = frozenset(affected_pkgs) env_matches = [s for s in all_concrete_specs if s.name in affected_pkgs] visited = set() @@ -510,7 +510,7 @@ class SpackCI: and if so return the name otherwise return none. """ for _name in self.named_jobs: - keys = ["{0}-job".format(_name), "{0}-job-remove".format(_name)] + keys = [f"{_name}-job", f"{_name}-job-remove"] if any([key for key in keys if key in section]): return _name @@ -525,9 +525,9 @@ class SpackCI: jname = name if suffix: - jname = "{0}-job{1}".format(name, suffix) + jname = f"{name}-job{suffix}" else: - jname = "{0}-job".format(name) + jname = f"{name}-job" return jname @@ -739,7 +739,7 @@ def generate_gitlab_ci_yaml( # Requested to prune untouched packages, but assume we won't do that # unless we're actually in a git repo. rev1, rev2 = get_change_revisions() - tty.debug("Got following revisions: rev1={0}, rev2={1}".format(rev1, rev2)) + tty.debug(f"Got following revisions: rev1={rev1}, rev2={rev2}") if rev1 and rev2: # If the stack file itself did not change, proceed with pruning if not get_stack_changed(env.manifest_path, rev1, rev2): @@ -747,13 +747,13 @@ def generate_gitlab_ci_yaml( affected_pkgs = compute_affected_packages(rev1, rev2) tty.debug("affected pkgs:") for p in affected_pkgs: - tty.debug(" {0}".format(p)) + tty.debug(f" {p}") affected_specs = get_spec_filter_list( env, affected_pkgs, dependent_traverse_depth=dependent_depth ) tty.debug("all affected specs:") for s in affected_specs: - tty.debug(" {0}/{1}".format(s.name, s.dag_hash()[:7])) + tty.debug(f" {s.name}/{s.dag_hash()[:7]}") # Allow overriding --prune-dag cli opt with environment variable prune_dag_override = os.environ.get("SPACK_PRUNE_UP_TO_DATE", None) @@ -978,7 +978,7 @@ def generate_gitlab_ci_yaml( rebuild_decisions = {} for stage_jobs in stages: - stage_name = "stage-{0}".format(stage_id) + stage_name = f"stage-{stage_id}" stage_names.append(stage_name) stage_id += 1 @@ -1009,7 +1009,7 @@ def generate_gitlab_ci_yaml( job_object = spack_ci_ir["jobs"][release_spec_dag_hash]["attributes"] if not job_object: - tty.warn("No match found for {0}, skipping it".format(release_spec)) + tty.warn(f"No match found for {release_spec}, skipping it") continue if spack_pipeline_type is not None: @@ -1119,7 +1119,7 @@ def generate_gitlab_ci_yaml( if artifacts_root: job_object["needs"].append( - {"job": generate_job_name, "pipeline": "{0}".format(parent_pipeline_id)} + {"job": generate_job_name, "pipeline": f"{parent_pipeline_id}"} ) # Let downstream jobs know whether the spec needed rebuilding, regardless @@ -1185,19 +1185,17 @@ def generate_gitlab_ci_yaml( if spack_pipeline_type == "spack_pull_request": spack.mirror.remove("ci_shared_pr_mirror", cfg.default_modify_scope()) - tty.debug("{0} build jobs generated in {1} stages".format(job_id, stage_id)) + tty.debug(f"{job_id} build jobs generated in {stage_id} stages") if job_id > 0: - tty.debug( - "The max_needs_job is {0}, with {1} needs".format(max_needs_job, max_length_needs) - ) + tty.debug(f"The max_needs_job is {max_needs_job}, with {max_length_needs} needs") # Use "all_job_names" to populate the build group for this set if cdash_handler and cdash_handler.auth_token: try: cdash_handler.populate_buildgroup(all_job_names) except (SpackError, HTTPError, URLError) as err: - tty.warn("Problem populating buildgroup: {0}".format(err)) + tty.warn(f"Problem populating buildgroup: {err}") else: tty.warn("Unable to populate buildgroup without CDash credentials") @@ -1211,9 +1209,7 @@ def generate_gitlab_ci_yaml( sync_job = copy.deepcopy(spack_ci_ir["jobs"]["copy"]["attributes"]) sync_job["stage"] = "copy" if artifacts_root: - sync_job["needs"] = [ - {"job": generate_job_name, "pipeline": "{0}".format(parent_pipeline_id)} - ] + sync_job["needs"] = [{"job": generate_job_name, "pipeline": f"{parent_pipeline_id}"}] if "variables" not in sync_job: sync_job["variables"] = {} @@ -1230,6 +1226,7 @@ def generate_gitlab_ci_yaml( # TODO: Remove this condition in Spack 0.23 buildcache_source = os.environ.get("SPACK_SOURCE_MIRROR", None) sync_job["variables"]["SPACK_BUILDCACHE_SOURCE"] = buildcache_source + sync_job["dependencies"] = [] output_object["copy"] = sync_job job_id += 1 @@ -1348,7 +1345,7 @@ def generate_gitlab_ci_yaml( copy_specs_file = os.path.join( copy_specs_dir, - "copy_{}_specs.json".format(spack_stack_name if spack_stack_name else "rebuilt"), + f"copy_{spack_stack_name if spack_stack_name else 'rebuilt'}_specs.json", ) with open(copy_specs_file, "w") as fd: @@ -1440,7 +1437,7 @@ def import_signing_key(base64_signing_key): fd.write(decoded_key) key_import_output = spack_gpg("trust", sign_key_path, output=str) - tty.debug("spack gpg trust {0}".format(sign_key_path)) + tty.debug(f"spack gpg trust {sign_key_path}") tty.debug(key_import_output) # Now print the keys we have for verifying and signing @@ -1469,7 +1466,7 @@ def can_verify_binaries(): def _push_mirror_contents(input_spec, sign_binaries, mirror_url): """Unchecked version of the public API, for easier mocking""" unsigned = not sign_binaries - tty.debug("Creating buildcache ({0})".format("unsigned" if unsigned else "signed")) + tty.debug(f"Creating buildcache ({'unsigned' if unsigned else 'signed'})") push_url = spack.mirror.Mirror.from_url(mirror_url).push_url return bindist.push(input_spec, push_url, bindist.PushOptions(force=True, unsigned=unsigned)) @@ -1498,9 +1495,9 @@ def push_mirror_contents(input_spec: spack.spec.Spec, mirror_url, sign_binaries) # Exception # BaseException # object - err_msg = "Error msg: {0}".format(inst) + err_msg = f"Error msg: {inst}" if any(x in err_msg for x in ["Access Denied", "InvalidAccessKeyId"]): - tty.msg("Permission problem writing to {0}".format(mirror_url)) + tty.msg(f"Permission problem writing to {mirror_url}") tty.msg(err_msg) return False else: @@ -1531,8 +1528,9 @@ def copy_files_to_artifacts(src, artifacts_dir): try: fs.copy(src, artifacts_dir) except Exception as err: - msg = ("Unable to copy files ({0}) to artifacts {1} due to " "exception: {2}").format( - src, artifacts_dir, str(err) + msg = ( + f"Unable to copy files ({src}) to artifacts {artifacts_dir} due to " + f"exception: {str(err)}" ) tty.warn(msg) @@ -1548,23 +1546,23 @@ def copy_stage_logs_to_artifacts(job_spec: spack.spec.Spec, job_log_dir: str) -> job_spec: spec associated with spack install log job_log_dir: path into which build log should be copied """ - tty.debug("job spec: {0}".format(job_spec)) + tty.debug(f"job spec: {job_spec}") if not job_spec: - msg = "Cannot copy stage logs: job spec ({0}) is required" - tty.error(msg.format(job_spec)) + msg = f"Cannot copy stage logs: job spec ({job_spec}) is required" + tty.error(msg) return try: pkg_cls = spack.repo.PATH.get_pkg_class(job_spec.name) job_pkg = pkg_cls(job_spec) - tty.debug("job package: {0}".format(job_pkg)) + tty.debug(f"job package: {job_pkg}") except AssertionError: - msg = "Cannot copy stage logs: job spec ({0}) must be concrete" - tty.error(msg.format(job_spec)) + msg = f"Cannot copy stage logs: job spec ({job_spec}) must be concrete" + tty.error(msg) return stage_dir = job_pkg.stage.path - tty.debug("stage dir: {0}".format(stage_dir)) + tty.debug(f"stage dir: {stage_dir}") for file in [job_pkg.log_path, job_pkg.env_mods_path, *job_pkg.builder.archive_files]: copy_files_to_artifacts(file, job_log_dir) @@ -1577,10 +1575,10 @@ def copy_test_logs_to_artifacts(test_stage, job_test_dir): test_stage (str): test stage path job_test_dir (str): the destination artifacts test directory """ - tty.debug("test stage: {0}".format(test_stage)) + tty.debug(f"test stage: {test_stage}") if not os.path.exists(test_stage): - msg = "Cannot copy test logs: job test stage ({0}) does not exist" - tty.error(msg.format(test_stage)) + msg = f"Cannot copy test logs: job test stage ({test_stage}) does not exist" + tty.error(msg) return copy_files_to_artifacts(os.path.join(test_stage, "*", "*.txt"), job_test_dir) @@ -1595,7 +1593,7 @@ def download_and_extract_artifacts(url, work_dir): url (str): Complete url to artifacts.zip file work_dir (str): Path to destination where artifacts should be extracted """ - tty.msg("Fetching artifacts from: {0}\n".format(url)) + tty.msg(f"Fetching artifacts from: {url}\n") headers = {"Content-Type": "application/zip"} @@ -1612,7 +1610,7 @@ def download_and_extract_artifacts(url, work_dir): response_code = response.getcode() if response_code != 200: - msg = "Error response code ({0}) in reproduce_ci_job".format(response_code) + msg = f"Error response code ({response_code}) in reproduce_ci_job" raise SpackError(msg) artifacts_zip_path = os.path.join(work_dir, "artifacts.zip") @@ -1642,7 +1640,7 @@ def get_spack_info(): return git_log - return "no git repo, use spack {0}".format(spack.spack_version) + return f"no git repo, use spack {spack.spack_version}" def setup_spack_repro_version(repro_dir, checkout_commit, merge_commit=None): @@ -1665,8 +1663,8 @@ def setup_spack_repro_version(repro_dir, checkout_commit, merge_commit=None): """ # figure out the path to the spack git version being used for the # reproduction - print("checkout_commit: {0}".format(checkout_commit)) - print("merge_commit: {0}".format(merge_commit)) + print(f"checkout_commit: {checkout_commit}") + print(f"merge_commit: {merge_commit}") dot_git_path = os.path.join(spack.paths.prefix, ".git") if not os.path.exists(dot_git_path): @@ -1685,14 +1683,14 @@ def setup_spack_repro_version(repro_dir, checkout_commit, merge_commit=None): git("log", "-1", checkout_commit, output=str, error=os.devnull, fail_on_error=False) if git.returncode != 0: - tty.error("Missing commit: {0}".format(checkout_commit)) + tty.error(f"Missing commit: {checkout_commit}") return False if merge_commit: git("log", "-1", merge_commit, output=str, error=os.devnull, fail_on_error=False) if git.returncode != 0: - tty.error("Missing commit: {0}".format(merge_commit)) + tty.error(f"Missing commit: {merge_commit}") return False # Next attempt to clone your local spack repo into the repro dir @@ -1715,7 +1713,7 @@ def setup_spack_repro_version(repro_dir, checkout_commit, merge_commit=None): ) if git.returncode != 0: - tty.error("Unable to checkout {0}".format(checkout_commit)) + tty.error(f"Unable to checkout {checkout_commit}") tty.msg(co_out) return False @@ -1734,7 +1732,7 @@ def setup_spack_repro_version(repro_dir, checkout_commit, merge_commit=None): ) if git.returncode != 0: - tty.error("Unable to merge {0}".format(merge_commit)) + tty.error(f"Unable to merge {merge_commit}") tty.msg(merge_out) return False @@ -1755,6 +1753,7 @@ def reproduce_ci_job(url, work_dir, autostart, gpg_url, runtime): commands to run to reproduce the build once inside the container. """ work_dir = os.path.realpath(work_dir) + platform_script_ext = "ps1" if IS_WINDOWS else "sh" download_and_extract_artifacts(url, work_dir) gpg_path = None @@ -1765,13 +1764,13 @@ def reproduce_ci_job(url, work_dir, autostart, gpg_url, runtime): lock_file = fs.find(work_dir, "spack.lock")[0] repro_lock_dir = os.path.dirname(lock_file) - tty.debug("Found lock file in: {0}".format(repro_lock_dir)) + tty.debug(f"Found lock file in: {repro_lock_dir}") yaml_files = fs.find(work_dir, ["*.yaml", "*.yml"]) tty.debug("yaml files:") for yaml_file in yaml_files: - tty.debug(" {0}".format(yaml_file)) + tty.debug(f" {yaml_file}") pipeline_yaml = None @@ -1786,10 +1785,10 @@ def reproduce_ci_job(url, work_dir, autostart, gpg_url, runtime): pipeline_yaml = yaml_obj if pipeline_yaml: - tty.debug("\n{0} is likely your pipeline file".format(yf)) + tty.debug(f"\n{yf} is likely your pipeline file") relative_concrete_env_dir = pipeline_yaml["variables"]["SPACK_CONCRETE_ENV_DIR"] - tty.debug("Relative environment path used by cloud job: {0}".format(relative_concrete_env_dir)) + tty.debug(f"Relative environment path used by cloud job: {relative_concrete_env_dir}") # Using the relative concrete environment path found in the generated # pipeline variable above, copy the spack environment files so they'll @@ -1803,10 +1802,11 @@ def reproduce_ci_job(url, work_dir, autostart, gpg_url, runtime): shutil.copyfile(orig_yaml_path, copy_yaml_path) # Find the install script in the unzipped artifacts and make it executable - install_script = fs.find(work_dir, "install.sh")[0] - st = os.stat(install_script) - os.chmod(install_script, st.st_mode | stat.S_IEXEC) - + install_script = fs.find(work_dir, f"install.{platform_script_ext}")[0] + if not IS_WINDOWS: + # pointless on Windows + st = os.stat(install_script) + os.chmod(install_script, st.st_mode | stat.S_IEXEC) # Find the repro details file. This just includes some values we wrote # during `spack ci rebuild` to make reproduction easier. E.g. the job # name is written here so we can easily find the configuration of the @@ -1844,7 +1844,7 @@ def reproduce_ci_job(url, work_dir, autostart, gpg_url, runtime): job_image = job_image_elt["name"] else: job_image = job_image_elt - tty.msg("Job ran with the following image: {0}".format(job_image)) + tty.msg(f"Job ran with the following image: {job_image}") # Because we found this job was run with a docker image, so we will try # to print a "docker run" command that bind-mounts the directory where @@ -1919,65 +1919,75 @@ def reproduce_ci_job(url, work_dir, autostart, gpg_url, runtime): job_tags = None if "tags" in job_yaml: job_tags = job_yaml["tags"] - tty.msg("Job ran with the following tags: {0}".format(job_tags)) + tty.msg(f"Job ran with the following tags: {job_tags}") entrypoint_script = [ ["git", "config", "--global", "--add", "safe.directory", mount_as_dir], - [".", os.path.join(mount_as_dir if job_image else work_dir, "share/spack/setup-env.sh")], + [ + ".", + os.path.join( + mount_as_dir if job_image else work_dir, + f"share/spack/setup-env.{platform_script_ext}", + ), + ], ["spack", "gpg", "trust", mounted_gpg_path if job_image else gpg_path] if gpg_path else [], ["spack", "env", "activate", mounted_env_dir if job_image else repro_dir], - [os.path.join(mounted_repro_dir, "install.sh") if job_image else install_script], + [ + ( + os.path.join(mounted_repro_dir, f"install.{platform_script_ext}") + if job_image + else install_script + ) + ], ] - + entry_script = os.path.join(mounted_workdir, f"entrypoint.{platform_script_ext}") inst_list = [] # Finally, print out some instructions to reproduce the build if job_image: # Allow interactive - entrypoint_script.extend( - [ - [ - "echo", - "Re-run install script using:\n\t{0}".format( - os.path.join(mounted_repro_dir, "install.sh") - if job_image - else install_script - ), - ], - # Allow interactive - ["exec", "$@"], - ] + install_mechanism = ( + os.path.join(mounted_repro_dir, f"install.{platform_script_ext}") + if job_image + else install_script ) + entrypoint_script.append(["echo", f"Re-run install script using:\n\t{install_mechanism}"]) + # Allow interactive + if IS_WINDOWS: + entrypoint_script.extend(["&", "($args -Join ' ')", "-NoExit"]) + else: + entrypoint_script.extend(["exec", "$@"]) + process_command( "entrypoint", entrypoint_script, work_dir, run=False, exit_on_failure=False ) docker_command = [ - [ - runtime, - "run", - "-i", - "-t", - "--rm", - "--name", - "spack_reproducer", - "-v", - ":".join([work_dir, mounted_workdir, "Z"]), - "-v", - ":".join( - [ - os.path.join(work_dir, "jobs_scratch_dir"), - os.path.join(mount_as_dir, "jobs_scratch_dir"), - "Z", - ] - ), - "-v", - ":".join([os.path.join(work_dir, "spack"), mount_as_dir, "Z"]), - "--entrypoint", - os.path.join(mounted_workdir, "entrypoint.sh"), - job_image, - "bash", - ] + runtime, + "run", + "-i", + "-t", + "--rm", + "--name", + "spack_reproducer", + "-v", + ":".join([work_dir, mounted_workdir, "Z"]), + "-v", + ":".join( + [ + os.path.join(work_dir, "jobs_scratch_dir"), + os.path.join(mount_as_dir, "jobs_scratch_dir"), + "Z", + ] + ), + "-v", + ":".join([os.path.join(work_dir, "spack"), mount_as_dir, "Z"]), + "--entrypoint", ] + if IS_WINDOWS: + docker_command.extend(["powershell.exe", job_image, entry_script, "powershell.exe"]) + else: + docker_command.extend([entry_script, job_image, "bash"]) + docker_command = [docker_command] autostart = autostart and setup_result process_command("start", docker_command, work_dir, run=autostart) @@ -1986,22 +1996,20 @@ def reproduce_ci_job(url, work_dir, autostart, gpg_url, runtime): inst_list.extend( [ " - Start the docker container install", - " $ {0}/start.sh".format(work_dir), + f" $ {work_dir}/start.{platform_script_ext}", ] ) else: process_command("reproducer", entrypoint_script, work_dir, run=False) inst_list.append("\nOnce on the tagged runner:\n\n") - inst_list.extent( - [" - Run the reproducer script", " $ {0}/reproducer.sh".format(work_dir)] - ) + inst_list.extent([" - Run the reproducer script", f" $ {work_dir}/reproducer.py"]) if not setup_result: inst_list.append("\n - Clone spack and acquire tested commit") - inst_list.append("\n {0}\n".format(spack_info)) + inst_list.append(f"\n {spack_info}\n") inst_list.append("\n") - inst_list.append("\n Path to clone spack: {0}/spack\n\n".format(work_dir)) + inst_list.append(f"\n Path to clone spack: {work_dir}/spack\n\n") tty.msg("".join(inst_list)) @@ -2020,50 +2028,78 @@ def process_command(name, commands, repro_dir, run=True, exit_on_failure=True): Returns: the exit code from processing the command """ - tty.debug("spack {0} arguments: {1}".format(name, commands)) + tty.debug(f"spack {name} arguments: {commands}") if len(commands) == 0 or isinstance(commands[0], str): commands = [commands] - # Create a string [command 1] && [command 2] && ... && [command n] with commands - # quoted using double quotes. - args_to_string = lambda args: " ".join('"{}"'.format(arg) for arg in args) - full_command = " \n ".join(map(args_to_string, commands)) - - # Write the command to a shell script - script = "{0}.sh".format(name) - with open(script, "w") as fd: - fd.write("#!/bin/sh\n\n") - fd.write("\n# spack {0} command\n".format(name)) + def compose_command_err_handling(args): + if not IS_WINDOWS: + args = [f'"{arg}"' for arg in args] + arg_str = " ".join(args) + result = arg_str + "\n" + # ErrorActionPreference will handle PWSH commandlets (Spack calls), + # but we need to handle EXEs (git, etc) ourselves + catch_exe_failure = ( + """ +if ($LASTEXITCODE -ne 0){ + throw "Command {} has failed" +} +""" + if IS_WINDOWS + else "" + ) + if exit_on_failure and catch_exe_failure: + result += catch_exe_failure.format(arg_str) + return result + + # Create a string [command 1] \n [command 2] \n ... \n [command n] with + # commands composed into a platform dependent shell script, pwsh on Windows, + full_command = "\n".join(map(compose_command_err_handling, commands)) + # Write the command to a python script + if IS_WINDOWS: + script = f"{name}.ps1" + script_content = [f"\n# spack {name} command\n"] + if exit_on_failure: + script_content.append('$ErrorActionPreference = "Stop"\n') + if os.environ.get("SPACK_VERBOSE_SCRIPT"): + script_content.append("Set-PSDebug -Trace 2\n") + else: + script = f"{name}.sh" + script_content = ["#!/bin/sh\n\n", f"\n# spack {name} command\n"] if exit_on_failure: - fd.write("set -e\n") + script_content.append("set -e\n") if os.environ.get("SPACK_VERBOSE_SCRIPT"): - fd.write("set -x\n") - fd.write(full_command) - fd.write("\n") + script_content.append("set -x\n") + script_content.append(full_command) + script_content.append("\n") - st = os.stat(script) - os.chmod(script, st.st_mode | stat.S_IEXEC) + with open(script, "w") as fd: + for line in script_content: + fd.write(line) copy_path = os.path.join(repro_dir, script) shutil.copyfile(script, copy_path) - st = os.stat(copy_path) - os.chmod(copy_path, st.st_mode | stat.S_IEXEC) + if not IS_WINDOWS: + st = os.stat(copy_path) + os.chmod(copy_path, st.st_mode | stat.S_IEXEC) - # Run the generated install.sh shell script as if it were being run in + # Run the generated shell script as if it were being run in # a login shell. exit_code = None if run: try: - cmd_process = subprocess.Popen(["/bin/sh", "./{0}".format(script)]) + # We use sh as executor on Linux like platforms, pwsh on Windows + interpreter = "powershell.exe" if IS_WINDOWS else "/bin/sh" + cmd_process = subprocess.Popen([interpreter, f"./{script}"]) cmd_process.wait() exit_code = cmd_process.returncode except (ValueError, subprocess.CalledProcessError, OSError) as err: - tty.error("Encountered error running {0} script".format(name)) + tty.error(f"Encountered error running {name} script") tty.error(err) exit_code = 1 - tty.debug("spack {0} exited {1}".format(name, exit_code)) + tty.debug(f"spack {name} exited {exit_code}") else: # Delete the script, it is copied to the destination dir os.remove(script) @@ -2122,7 +2158,7 @@ def write_broken_spec(url, pkg_name, stack_name, job_url, pipeline_url, spec_dic # If there is an S3 error (e.g., access denied or connection # error), the first non boto-specific class in the exception # hierarchy is Exception. Just print a warning and return - msg = "Error writing to broken specs list {0}: {1}".format(url, err) + msg = f"Error writing to broken specs list {url}: {err}" tty.warn(msg) finally: shutil.rmtree(tmpdir) @@ -2135,7 +2171,7 @@ def read_broken_spec(broken_spec_url): try: _, _, fs = web_util.read_from_url(broken_spec_url) except (URLError, web_util.SpackWebError, HTTPError): - tty.warn("Unable to read broken spec from {0}".format(broken_spec_url)) + tty.warn(f"Unable to read broken spec from {broken_spec_url}") return None broken_spec_contents = codecs.getreader("utf-8")(fs).read() @@ -2150,14 +2186,14 @@ def display_broken_spec_messages(base_url, hashes): for spec_hash, broken_spec in [tup for tup in broken_specs if tup[1]]: details = broken_spec["broken-spec"] if "job-name" in details: - item_name = "{0}/{1}".format(details["job-name"], spec_hash[:7]) + item_name = f"{details['job-name']}/{spec_hash[:7]}" else: item_name = spec_hash if "job-stack" in details: - item_name = "{0} (in stack {1})".format(item_name, details["job-stack"]) + item_name = f"{item_name} (in stack {details['job-stack']})" - msg = " {0} was reported broken here: {1}".format(item_name, details["job-url"]) + msg = f" {item_name} was reported broken here: {details['job-url']}" tty.msg(msg) @@ -2180,7 +2216,7 @@ def run_standalone_tests(**kwargs): log_file = kwargs.get("log_file") if cdash and log_file: - tty.msg("The test log file {0} option is ignored with CDash reporting".format(log_file)) + tty.msg(f"The test log file {log_file} option is ignored with CDash reporting") log_file = None # Error out but do NOT terminate if there are missing required arguments. @@ -2206,10 +2242,10 @@ def run_standalone_tests(**kwargs): test_args.extend(["--log-file", log_file]) test_args.append(job_spec.name) - tty.debug("Running {0} stand-alone tests".format(job_spec.name)) + tty.debug(f"Running {job_spec.name} stand-alone tests") exit_code = process_command("test", test_args, repro_dir) - tty.debug("spack test exited {0}".format(exit_code)) + tty.debug(f"spack test exited {exit_code}") class CDashHandler: @@ -2232,7 +2268,7 @@ class CDashHandler: # append runner description to the site if available runner = os.environ.get("CI_RUNNER_DESCRIPTION") if runner: - self.site += " ({0})".format(runner) + self.site += f" ({runner})" # track current spec, if any self.current_spec = None @@ -2260,21 +2296,13 @@ class CDashHandler: Returns: (str) current spec's CDash build name.""" spec = self.current_spec if spec: - build_name = "{0}@{1}%{2} hash={3} arch={4} ({5})".format( - spec.name, - spec.version, - spec.compiler, - spec.dag_hash(), - spec.architecture, - self.build_group, - ) - tty.debug( - "Generated CDash build name ({0}) from the {1}".format(build_name, spec.name) - ) + build_name = f"{spec.name}@{spec.version}%{spec.compiler} \ +hash={spec.dag_hash()} arch={spec.architecture} ({self.build_group})" + tty.debug(f"Generated CDash build name ({build_name}) from the {spec.name}") return build_name build_name = os.environ.get("SPACK_CDASH_BUILD_NAME") - tty.debug("Using CDash build name ({0}) from the environment".format(build_name)) + tty.debug(f"Using CDash build name ({build_name}) from the environment") return build_name @property # type: ignore @@ -2288,25 +2316,25 @@ class CDashHandler: Returns: (str) current CDash build stamp""" build_stamp = os.environ.get("SPACK_CDASH_BUILD_STAMP") if build_stamp: - tty.debug("Using build stamp ({0}) from the environment".format(build_stamp)) + tty.debug(f"Using build stamp ({build_stamp}) from the environment") return build_stamp build_stamp = cdash_build_stamp(self.build_group, time.time()) - tty.debug("Generated new build stamp ({0})".format(build_stamp)) + tty.debug(f"Generated new build stamp ({build_stamp})") return build_stamp @property # type: ignore @memoized def project_enc(self): - tty.debug("Encoding project ({0}): {1})".format(type(self.project), self.project)) + tty.debug(f"Encoding project ({type(self.project)}): {self.project})") encode = urlencode({"project": self.project}) index = encode.find("=") + 1 return encode[index:] @property def upload_url(self): - url_format = "{0}/submit.php?project={1}" - return url_format.format(self.url, self.project_enc) + url_format = f"{self.url}/submit.php?project={self.project_enc}" + return url_format def copy_test_results(self, source, dest): """Copy test results to artifacts directory.""" @@ -2324,7 +2352,7 @@ class CDashHandler: response_code = response.getcode() if response_code not in [200, 201]: - msg = "Creating buildgroup failed (response code = {0})".format(response_code) + msg = f"Creating buildgroup failed (response code = {response_code})" tty.warn(msg) return None @@ -2335,10 +2363,10 @@ class CDashHandler: return build_group_id def populate_buildgroup(self, job_names): - url = "{0}/api/v1/buildgroup.php".format(self.url) + url = f"{self.url}/api/v1/buildgroup.php" headers = { - "Authorization": "Bearer {0}".format(self.auth_token), + "Authorization": f"Bearer {self.auth_token}", "Content-Type": "application/json", } @@ -2346,11 +2374,11 @@ class CDashHandler: parent_group_id = self.create_buildgroup(opener, headers, url, self.build_group, "Daily") group_id = self.create_buildgroup( - opener, headers, url, "Latest {0}".format(self.build_group), "Latest" + opener, headers, url, f"Latest {self.build_group}", "Latest" ) if not parent_group_id or not group_id: - msg = "Failed to create or retrieve buildgroups for {0}".format(self.build_group) + msg = f"Failed to create or retrieve buildgroups for {self.build_group}" tty.warn(msg) return @@ -2370,7 +2398,7 @@ class CDashHandler: response_code = response.getcode() if response_code != 200: - msg = "Error response code ({0}) in populate_buildgroup".format(response_code) + msg = f"Error response code ({response_code}) in populate_buildgroup" tty.warn(msg) def report_skipped(self, spec: spack.spec.Spec, report_dir: str, reason: Optional[str]): |