diff options
| author | Scott Wittenburg <scott.wittenburg@kitware.com> | 2021-10-07 08:59:51 -0600 |
|---|---|---|
| committer | GitHub <noreply@github.com> | 2021-10-07 08:59:51 -0600 |
| commit | 0561af197511d5b949417195ffb3eaa0cff6f4f5 (patch) | |
| tree | c67514d28dc1b0599ee257569f4195657b8387cf /lib | |
| parent | b33a0923e191aefe1d730a43baa7e3d326ea2991 (diff) | |
| download | spack-0561af197511d5b949417195ffb3eaa0cff6f4f5.tar.gz spack-0561af197511d5b949417195ffb3eaa0cff6f4f5.tar.bz2 spack-0561af197511d5b949417195ffb3eaa0cff6f4f5.tar.xz spack-0561af197511d5b949417195ffb3eaa0cff6f4f5.zip | |
Pipelines: retry service job on system errors (#26508)
Retry rebuild-index, cleanup, and no-op jobs automatically if they fail
due to infrastructure-related problems.
Diffstat (limited to 'lib')
| -rw-r--r-- | lib/spack/spack/ci.py | 12 |
1 files changed, 12 insertions, 0 deletions
diff --git a/lib/spack/spack/ci.py b/lib/spack/spack/ci.py index d269e87a59..32b5da451a 100644 --- a/lib/spack/spack/ci.py +++ b/lib/spack/spack/ci.py @@ -1004,6 +1004,14 @@ def generate_gitlab_ci_yaml(env, print_summary, output_file, 'after_script', ] + service_job_retries = { + 'max': 2, + 'when': [ + 'runner_system_failure', + 'stuck_or_timeout_failure' + ] + } + if job_id > 0: if temp_storage_url_prefix: # There were some rebuild jobs scheduled, so we will need to @@ -1023,6 +1031,7 @@ def generate_gitlab_ci_yaml(env, print_summary, output_file, temp_storage_url_prefix) ] cleanup_job['when'] = 'always' + cleanup_job['retry'] = service_job_retries output_object['cleanup'] = cleanup_job @@ -1046,6 +1055,7 @@ def generate_gitlab_ci_yaml(env, print_summary, output_file, index_target_mirror) ] final_job['when'] = 'always' + final_job['retry'] = service_job_retries output_object['rebuild-index'] = final_job @@ -1109,6 +1119,8 @@ def generate_gitlab_ci_yaml(env, print_summary, output_file, 'echo "All specs already up to date, nothing to rebuild."', ] + noop_job['retry'] = service_job_retries + sorted_output = {'no-specs-to-rebuild': noop_job} if known_broken_specs_encountered: |