diff options
| -rw-r--r-- | lib/spack/spack/cmd/url.py | 152 | ||||
| -rw-r--r-- | lib/spack/spack/test/cmd/url.py | 6 |
2 files changed, 93 insertions, 65 deletions
diff --git a/lib/spack/spack/cmd/url.py b/lib/spack/spack/cmd/url.py index 3fe4e0ac6a..10257b2608 100644 --- a/lib/spack/spack/cmd/url.py +++ b/lib/spack/spack/cmd/url.py @@ -239,80 +239,104 @@ def url_summary(args): def url_stats(args): - stats = {} # stats about fetchers in packages. - nvers = 0 # total number of versions - npkgs = 0 # total number of packages - - def inc(fstype, category, attr=None): - """Increment statistics in the stats dict.""" - categories = stats.setdefault(fstype, {}) - if attr: - cat_stats = categories.setdefault(category, {}) - val = cat_stats.setdefault(attr, 0) - stats[fstype][category][attr] = val + 1 - else: - val = categories.setdefault(category, 0) - stats[fstype][category] = val + 1 + class UrlStats(object): + def __init__(self): + self.total = 0 + self.schemes = defaultdict(lambda: 0) + self.checksums = defaultdict(lambda: 0) + self.url_type = defaultdict(lambda: 0) + self.git_type = defaultdict(lambda: 0) + + def add(self, fetcher): + self.total += 1 + + url_type = fetcher.url_attr + self.url_type[url_type or 'no code'] += 1 + + if url_type == 'url': + digest = getattr(fetcher, 'digest', None) + if digest: + algo = crypto.hash_algo_for_digest(digest) + else: + algo = 'no checksum' + self.checksums[algo] += 1 + + # parse out the URL scheme (https/http/ftp/etc.) + urlinfo = urlparse(fetcher.url) + self.schemes[urlinfo.scheme] += 1 + + elif url_type == 'git': + if getattr(fetcher, 'commit', None): + self.git_type['commit'] += 1 + elif getattr(fetcher, 'branch', None): + self.git_type['branch'] += 1 + elif getattr(fetcher, 'tag', None): + self.git_type['tag'] += 1 + else: + self.git_type['no ref'] += 1 + + npkgs = 0 + version_stats = UrlStats() + resource_stats = UrlStats() - # over all packages for pkg in spack.repo.path.all_packages(): npkgs += 1 - if not pkg.has_code: - for _ in pkg.versions: - inc('No code', 'total') - nvers += 1 - continue - - # look at each version for v, args in pkg.versions.items(): - # figure out what type of fetcher it is fetcher = fs.for_package_version(pkg, v) - nvers += 1 + version_stats.add(fetcher) - fstype = fetcher.url_attr - inc(fstype, 'total') + for _, resources in pkg.resources.items(): + for resource in resources: + resource_stats.add(resource.fetcher) - # put some special stats in for particular types of fetchers. - if fstype == 'git': - if 'commit' in args: - inc('git', 'security', 'commit') - else: - inc('git', 'security', 'no commit') - elif fstype == 'url': - for h in crypto.hashes: - if h in args: - inc('url', 'checksums', h) - break - else: - if 'checksum' in args: - h = crypto.hash_algo_for_digest(args['checksum']) - inc('url', 'checksums', h) - else: - inc('url', 'checksums', 'no checksum') + # print a nice summary table + tty.msg("URL stats for %d packages:" % npkgs) - # parse out the URL scheme (https/http/ftp/etc.) - urlinfo = urlparse(fetcher.url) - inc('url', 'schemes', urlinfo.scheme) + def print_line(): + print("-" * 62) - # print a nice summary table - tty.msg("%d total versions for %d packages:" % (nvers, npkgs)) - line_width = 36 - print("-" * line_width) - for fetcher, fetcher_stats in sorted(stats.items(), reverse=True): - fs_total = fetcher_stats['total'] - fs_pct = float(fs_total) / nvers * 100 - print("%-22s%5d%8.1f%%" % (fetcher, fs_total, fs_pct)) - - for category, cat_stats in sorted(fetcher_stats.items(), reverse=True): - if category == 'total': - continue - print(" %s" % category) - - for name, number in sorted(cat_stats.items(), reverse=True): - pct = float(number) / fs_total * 100 - print(" %-18s%5d%8.1f%%" % (name, number, pct)) - print("-" * line_width) + def print_stat(indent, name, stat_name=None): + width = 20 - indent + fmt = " " * indent + fmt += "%%-%ds" % width + if stat_name is None: + print(fmt % name) + else: + fmt += "%12d%8.1f%%%12d%8.1f%%" + v = getattr(version_stats, stat_name).get(name, 0) + r = getattr(resource_stats, stat_name).get(name, 0) + print(fmt % (name, + v, v / version_stats.total * 100, + r, r / resource_stats.total * 100)) + + print_line() + print("%-20s%12s%9s%12s%9s" % ("stat", "versions", "%", "resources", "%")) + print_line() + print_stat(0, "url", "url_type") + + print_stat(4, "schemes") + schemes = set(version_stats.schemes) | set(resource_stats.schemes) + for scheme in schemes: + print_stat(8, scheme, "schemes") + + print_stat(4, "checksums") + checksums = set(version_stats.checksums) | set(resource_stats.checksums) + for checksum in checksums: + print_stat(8, checksum, "checksums") + print_line() + + types = set(version_stats.url_type) | set(resource_stats.url_type) + types -= set(["url", "git"]) + for url_type in sorted(types): + print_stat(0, url_type, "url_type") + print_line() + + print_stat(0, "git", "url_type") + git_types = set(version_stats.git_type) | set(resource_stats.git_type) + for git_type in sorted(git_types): + print_stat(4, git_type, "git_type") + print_line() def print_name_and_version(url): diff --git a/lib/spack/spack/test/cmd/url.py b/lib/spack/spack/test/cmd/url.py index e147f84291..6593c94d98 100644 --- a/lib/spack/spack/test/cmd/url.py +++ b/lib/spack/spack/test/cmd/url.py @@ -131,4 +131,8 @@ def test_url_stats(capfd): output = url('stats') npkgs = '%d packages' % len(spack.repo.all_package_names()) assert npkgs in output - assert 'total versions' in output + assert 'url' in output + assert 'git' in output + assert 'schemes' in output + assert 'versions' in output + assert 'resources' in output |