From 8e61f542601083f363c5910a5ed74c8dae44e69d Mon Sep 17 00:00:00 2001 From: Vanessasaurus <814322+vsoch@users.noreply.github.com> Date: Thu, 2 Sep 2021 00:46:27 -0600 Subject: start of work to add spack audit packages-https checker (#25670) This PR will add a new audit, specifically for spack package homepage urls (and eventually other kinds I suspect) to see if there is an http address that can be changed to https. Usage is as follows: ```bash $ spack audit packages-https ``` And in list view: ```bash $ spack audit list generic: Generic checks relying on global variables configs: Sanity checks on compilers.yaml Sanity checks on packages.yaml packages: Sanity checks on specs used in directives packages-https: Sanity checks on https checks of package urls, etc. ``` I think it would be unwise to include with packages, because when run for all, since we do requests it takes a long time. I also like the idea of more well scoped checks - likely there will be other addresses for http/https within a package that we eventually check. For now, there are two error cases - one is when an https url is tried but there is some SSL error (or other error that means we cannot update to https): ```bash $ spack audit packages-https zoltan PKG-HTTPS-DIRECTIVES: 1 issue found 1. Error with attempting https for "zoltan": ``` This is either not fixable, or could be fixed with a change to the url or (better) contacting the site owners to ask about some certificate or similar. The second case is when there is an http that needs to be https, which is a huge issue now, but hopefully not after this spack PR. ```bash $ spack audit packages-https xman Package "xman" uses http but has a valid https endpoint. ``` And then when a package is fixed: ```bash $ spack audit packages-https zlib PKG-HTTPS-DIRECTIVES: 0 issues found. ``` And that's mostly it. :) Signed-off-by: vsoch Co-authored-by: vsoch --- lib/spack/spack/audit.py | 43 +++++++++++++++++++++++++++++++++++++++ lib/spack/spack/cmd/audit.py | 33 ++++++++++++++++++++++++++---- lib/spack/spack/test/cmd/audit.py | 20 ++++++++++++++++++ 3 files changed, 92 insertions(+), 4 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/audit.py b/lib/spack/spack/audit.py index 76616f31e0..47894383a5 100644 --- a/lib/spack/spack/audit.py +++ b/lib/spack/spack/audit.py @@ -37,12 +37,16 @@ as input. """ import collections import itertools +import re + +from six.moves.urllib.request import urlopen try: from collections.abc import Sequence # novm except ImportError: from collections import Sequence + #: Map an audit tag to a list of callables implementing checks CALLBACKS = {} @@ -261,6 +265,45 @@ package_directives = AuditClass( kwargs=('pkgs',) ) +#: Sanity checks on linting +# This can take some time, so it's run separately from packages +package_https_directives = AuditClass( + group='packages-https', + tag='PKG-HTTPS-DIRECTIVES', + description='Sanity checks on https checks of package urls, etc.', + kwargs=('pkgs',) +) + + +@package_https_directives +def _linting_package_file(pkgs, error_cls): + """Check for correctness of links + """ + import llnl.util.lang + + import spack.repo + import spack.spec + + errors = [] + for pkg_name in pkgs: + pkg = spack.repo.get(pkg_name) + + # Does the homepage have http, and if so, does https work? + if pkg.homepage.startswith('http://'): + https = re.sub("http", "https", pkg.homepage, 1) + try: + response = urlopen(https) + except Exception as e: + msg = 'Error with attempting https for "{0}": ' + errors.append(error_cls(msg.format(pkg.name), [str(e)])) + continue + + if response.getcode() == 200: + msg = 'Package "{0}" uses http but has a valid https endpoint.' + errors.append(msg.format(pkg.name)) + + return llnl.util.lang.dedupe(errors) + @package_directives def _unknown_variants_in_directives(pkgs, error_cls): diff --git a/lib/spack/spack/cmd/audit.py b/lib/spack/spack/cmd/audit.py index 67d761b7d8..0e87925021 100644 --- a/lib/spack/spack/cmd/audit.py +++ b/lib/spack/spack/cmd/audit.py @@ -2,6 +2,7 @@ # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) +import llnl.util.tty as tty import llnl.util.tty.color as cl import spack.audit @@ -19,12 +20,24 @@ def setup_parser(subparser): # Audit configuration files sp.add_parser('configs', help='audit configuration files') + # Https and other linting + https_parser = sp.add_parser('packages-https', help='check https in packages') + https_parser.add_argument( + '--all', + action='store_true', + default=False, + dest='check_all', + help="audit all packages" + ) + # Audit package recipes pkg_parser = sp.add_parser('packages', help='audit package recipes') - pkg_parser.add_argument( - 'name', metavar='PKG', nargs='*', - help='package to be analyzed (if none all packages will be processed)', - ) + + for group in [pkg_parser, https_parser]: + group.add_argument( + 'name', metavar='PKG', nargs='*', + help='package to be analyzed (if none all packages will be processed)', + ) # List all checks sp.add_parser('list', help='list available checks and exits') @@ -41,6 +54,17 @@ def packages(parser, args): _process_reports(reports) +def packages_https(parser, args): + + # Since packages takes a long time, --all is required without name + if not args.check_all and not args.name: + tty.die("Please specify one or more packages to audit, or --all.") + + pkgs = args.name or spack.repo.path.all_package_names() + reports = spack.audit.run_group(args.subcommand, pkgs=pkgs) + _process_reports(reports) + + def list(parser, args): for subcommand, check_tags in spack.audit.GROUPS.items(): print(cl.colorize('@*b{' + subcommand + '}:')) @@ -58,6 +82,7 @@ def audit(parser, args): subcommands = { 'configs': configs, 'packages': packages, + 'packages-https': packages_https, 'list': list } subcommands[args.subcommand](parser, args) diff --git a/lib/spack/spack/test/cmd/audit.py b/lib/spack/spack/test/cmd/audit.py index 84bbdbde5a..2c9dc6b124 100644 --- a/lib/spack/spack/test/cmd/audit.py +++ b/lib/spack/spack/test/cmd/audit.py @@ -31,3 +31,23 @@ def test_audit_configs(mutable_config, mock_packages): audit('configs', fail_on_error=False) # The mock configuration has duplicate definitions of some compilers assert audit.returncode == 1 + + +def test_audit_packages_https(mutable_config, mock_packages): + + # Without providing --all should fail + audit('packages-https', fail_on_error=False) + # The mock configuration has duplicate definitions of some compilers + assert audit.returncode == 1 + + # This uses http and should fail + audit('packages-https', "preferred-test", fail_on_error=False) + assert audit.returncode == 1 + + # providing one or more package names with https should work + audit('packages-https', "cmake", fail_on_error=True) + assert audit.returncode == 0 + + # providing one or more package names with https should work + audit('packages-https', "cmake", "conflict", fail_on_error=True) + assert audit.returncode == 0 -- cgit v1.2.3-70-g09d2