diff options
| -rw-r--r-- | arches | 6 | ||||
| -rw-r--r-- | depver_test.py | 142 | ||||
| -rw-r--r-- | output.py | 136 | ||||
| -rw-r--r-- | pkgver_test.py | 137 | ||||
| -rw-r--r-- | system-specific (renamed from sys-specific) | 0 | ||||
| -rw-r--r-- | tester.py | 88 | ||||
| -rw-r--r-- | version.py | 75 | ||||
| -rw-r--r-- | vertest.py | 138 |
8 files changed, 490 insertions, 232 deletions
@@ -1,6 +0,0 @@ -aarch64 -armv7 -pmmx -ppc -ppc64 -x86_64 diff --git a/depver_test.py b/depver_test.py new file mode 100644 index 0000000..3dcf596 --- /dev/null +++ b/depver_test.py @@ -0,0 +1,142 @@ +#!/usr/bin/env python3 +# Adélie Linux architecture package tester +# Ensure all packages depend on latest versions of each other +# +# Copyright © 2019-2020 Adélie Linux team. All rights reserved. +# NCSA license. +# +import argparse +import collections +import functools +import os +import sys + +from apkkit.base.index import Index + +from version import is_older, is_same, verkey, ver_is, APK_OPS +from output import FORMATTERS, No, Partial + +def atomize(spec): + if not any(i in spec for i in APK_OPS): + return spec, None + + for op in APK_OPS: + try: + name, ver = spec.split(op, maxsplit=1) + except ValueError: + continue + + return name, functools.partial(ver_is, op=op, b=ver) + + # Not reached + assert False + +def analyze(url, repos, arch): + pkgs = collections.defaultdict(dict) + newest = {} + providers = collections.defaultdict(list) + + print("Loading " + arch + "...", file=sys.stderr) + + index = [] + for repo in repos: + index.extend(Index(url=url + f"/{repo}/{arch}/APKINDEX.tar.gz").packages) + + for pkg in index: + new = pkg.version + pkgs[pkg.name][new] = pkg + + curr = newest.get(pkg.name, None) + if curr is None: + newest[pkg.name] = new + elif is_older(curr, new): + newest[pkg.name] = new + + providers[pkg.name].append((pkg.name, new)) + for i in pkg.provides: + i = i.split("=", maxsplit=1) + if len(i) == 2: + i, pver = i + else: + i, pver = i[0], new + + curr = newest.get(i, None) + if curr is None: + newest[i] = pver + elif is_older(curr, pver): + newest[i] = pver + + providers[i].append((pkg.name, pver)) + + yield ["arch", "package", "version", "issue"] + + for name in sorted(pkgs.keys()): + # DON'T use newest[] here. It is possible that a package + # provides= another package's name with a newer version + ver = sorted(pkgs[name].keys(), key=verkey)[-1] + pkg = pkgs[name][ver] + + for dep in pkg.depends: + if dep.startswith("!"): + continue + + spec = dep + dep, constraint = atomize(dep) + + if dep in pkg.provides: + continue + if dep not in providers: + yield [arch, name, ver, No(f"Missing {spec}")] + continue + + if constraint: + for provider, pver in providers[dep]: + if constraint(pver): + break + else: + yield [arch, name, ver, No(f"Missing {spec}")] + + for _, pver in providers[dep]: + if is_same(pver, newest[dep]): + break + else: + yield [arch, name, ver, Partial(f"Old {spec}")] + +if __name__ == "__main__": + opts = argparse.ArgumentParser( + description="""Scan the REPO/ARCH repositories under URL and + look for outdated or missing dependencies as required by the + most recent version of each package. This script examines both + main packages and subpackages. + + Be sure to include any repositories on which the repository of + interest depends, otherwise a lot of dependencies will be + incorrectly marked as missing.""", + ) + opts.add_argument( + "-f", "--format", choices=FORMATTERS.keys(), + default="pretty" if os.isatty(sys.stdout.fileno()) else "tab", + help="display format", + ) + opts.add_argument( + "url", metavar="URL", + help="base URL (no repository or arch)", + ) + opts.add_argument( + "repos", metavar="REPOS", + help="repositories (comma separated)", + ) + opts.add_argument( + "arches", metavar="ARCHES", + help="architectures (comma separated)", + ) + opts = opts.parse_args() + opts.repos = opts.repos.split(",") + opts.arches = opts.arches.split(",") + + for arch in opts.arches: + # FIXME: with html, wrapping will repeat for > 1 arch + FORMATTERS[opts.format]( + opts, + analyze(opts.url, opts.repos, arch), + ) diff --git a/output.py b/output.py new file mode 100644 index 0000000..870d618 --- /dev/null +++ b/output.py @@ -0,0 +1,136 @@ +#!/usr/bin/env python3 +# +# Copyright © 2019-2020 Adélie Linux team. All rights reserved. +# NCSA license. +# +import subprocess + +class _Cell: + def __init__(self, content): + self.content = content + +_ANSI_CLEAR = "\033[0;00m" + +class Yes(_Cell): + symbol = "✓" + cls = "cell-yes" + color = "\033[1;32m" + +class No(_Cell): + symbol = "✗" + cls = "cell-no" + color = "\033[1;31m" + +class Partial(_Cell): + symbol = "?" + cls = "cell-partial" + color = "\033[1;33m" + +PARTIAL_MISSING = Partial("missing") +YES_MISSING = Yes("missing") + +_HEADER = r"""<!DOCTYPE html> +<html> + <head> + <title>APKINDEX analysis - {date}</title> + <meta http-equiv="content-type" content="text/html; charset=utf-8"> + <style type="text/css"> + table { + background-color: #F8F9FA; + color: #222; + margin: 1em 0; + border: 1px solid #A2A9B1; + border-collapse: collapse; + } + td, th { + border: 1px solid #A2A9B1; + padding: 0.2em 0.4em; + } + th { + background-color: #EAECF0; + text-align: center; + position: sticky; + top: 0; + } + .cell-yes { background-color: #DFF0D8; color: black; } + .cell-no { background-color: #F2DEDE; color: black; } + .cell-partial { background: #FFF176; color: black; } + </style> + </head> + <body> + <h1>APKINDEX analysis - {date}</h1> + <p> + Base URL: <a href="{url}">{url}</a><br> + Repositories: {repos}<br> + Architectures: {arches}<br> + </p> +""" +_FOOTER = r""" </body> +</html> +""" + +def format_tab(_, rows): + for row in rows: + for i, j in enumerate(row): + if isinstance(j, _Cell): + row[i] = f"{j.symbol} {j.content}" + + print(*row, sep="\t") + +def format_pretty(_, rows): + proc = subprocess.Popen( + ("column", "-ts", "\t"), + stdin=subprocess.PIPE, + encoding="utf-8", + ) + + for row in rows: + for i, j in enumerate(row): + if isinstance(j, _Cell): + row[i] = f"{j.color}{j.symbol} {j.content}{_ANSI_CLEAR}" + else: + # Work around column(1) not liking ANSI escapes by + # always printing some + row[i] = f"{_ANSI_CLEAR}{j}{_ANSI_CLEAR}" + + print(*row, file=proc.stdin, sep="\t") + + proc.communicate() + +def format_html(opts, rows): + print(_HEADER) + + first = True + for row in rows: + if first: + print("<table>") + print( + "<thead><tr><th>", + "</th><th>".join(row), + "</th></tr></thead>", sep="" + ) + print("<tbody>") + first = False + continue + + for i, j in enumerate(row[1:]): + i += 1 + + if isinstance(j, _Cell): + row[i] = f" class='{j.cls}'>{j.symbol} {j.content}" + else: + row[i] = ">" + j + + print("<tr><td>", "</td><td".join(row), "</td></tr>", sep="") + + if not first: + print("</tbody>") + print("</table>") + + print(_FOOTER) + +FORMATTERS = { + "tab": format_tab, + "pretty": format_pretty, + "html": format_html, +} diff --git a/pkgver_test.py b/pkgver_test.py new file mode 100644 index 0000000..9566908 --- /dev/null +++ b/pkgver_test.py @@ -0,0 +1,137 @@ +#!/usr/bin/env python3 +# Adélie Linux architecture package tester +# Ensure all packages are all on arches +# +# Copyright © 2018-2020 Adélie Linux team. All rights reserved. +# NCSA license. +# +import argparse +import os +import sys +from pathlib import Path + +from apkkit.base.index import Index + +from output import FORMATTERS, Yes, No, YES_MISSING, PARTIAL_MISSING +from version import is_older + +def analyze(url, repo, arches): + newest = dict() + arch_newest = {arch: dict() for arch in arches} + + for arch in arches: + print(f"Loading {arch}...", file=sys.stderr) + + for pkg in Index(url=f"{url}/{repo}/{arch}/APKINDEX.tar.gz").packages: + if pkg.name != pkg.origin: + continue + new = pkg.version + + for mydict in (newest, arch_newest[arch]): + curr = mydict.get(pkg.name, None) + if curr is None: + mydict[pkg.name] = new + elif is_older(curr, new): + mydict[pkg.name] = new + + return newest, arch_newest + +def order_arch(arches, newest, arch_newest, ign): + yield ["arch", "package", "version for arch", "newest version"] + + for arch in arches: + old = list() + + for pkg in newest.keys(): + ver = newest[pkg] + archver = arch_newest[arch].get(pkg, None) + + if archver is None and pkg not in ign: + yield [arch, pkg, PARTIAL_MISSING, ver] + + elif archver is not None and is_older(archver, ver): + yield [arch, pkg, archver, ver] + +def order_pkg(arches, newest, arch_newest, ign): + yield ["package", *arches] + + for pkg in sorted(newest.keys()): + for i in arches: + archver = arch_newest[i].get(pkg, None) + if archver is None: + break + if archver is not None and is_older(archver, newest[pkg]): + break + else: + continue + + row = [pkg] + for i in arches: + archver = arch_newest[i].get(pkg, None) + if archver is None and pkg in ign: + row.append(YES_MISSING) + elif archver is None: + row.append(PARTIAL_MISSING) + elif is_older(archver, newest[pkg]): + row.append(No(archver)) + else: + row.append(Yes(archver)) + + yield row + +ORDERS = { + "pkg": order_pkg, + "arch": order_arch, +} + +if __name__ == "__main__": + opts = argparse.ArgumentParser( + description="""Scan the REPO/ARCH repositories under URL and + look for outdated or missing packages when comparing across + architectures. This script only examines main packages, not + subpackages. + + At least two architectures must be given in order to make a + comparison.""" + ) + opts.add_argument( + "-f", "--format", choices=FORMATTERS.keys(), + default="pretty" if os.isatty(sys.stdout.fileno()) else "tab", + help="display format", + ) + opts.add_argument( + "-o", "--order", choices=ORDERS.keys(), + default="pkg", + help="display order", + ) + opts.add_argument( + "url", metavar="URL", + help="base URL (no repository or arch)", + ) + opts.add_argument( + "repo", metavar="REPO", + help="repository", + ) + opts.add_argument( + "arches", metavar="ARCHES", + help="architectures (comma separated, at least 2)", + ) + opts = opts.parse_args() + opts.arches = opts.arches.split(",") + + if len(opts.arches) < 2: + print("At least two arches are required", file=sys.stderr) + sys.exit(1) + + if Path(opts.repo + "-specific").is_file(): + with open(opts.repo + '-specific', 'r') as ignore_file: + ign = set(pkg[:-1] for pkg in ignore_file.readlines()) + else: + ign = set() + + newest, arch_newest = analyze(opts.url, opts.repo, opts.arches) + # FIXME: support multiple repos + FORMATTERS[opts.format]( + opts, + ORDERS[opts.order](opts.arches, newest, arch_newest, ign), + ) diff --git a/sys-specific b/system-specific index e69de29..e69de29 100644 --- a/sys-specific +++ b/system-specific diff --git a/tester.py b/tester.py deleted file mode 100644 index 797216f..0000000 --- a/tester.py +++ /dev/null @@ -1,88 +0,0 @@ -#!/usr/bin/env python3 -# Adélie Linux architecture package tester -# Ensure all packages are all on arches -# -# Copyright © 2018 Adélie Linux team. All rights reserved. -# NCSA license. -# - -from apkkit.base.index import Index - -BASE_URL = "https://mirrormaster.adelielinux.org/adelie/{version}/{repo}/{arch}/APKINDEX.tar.gz" -"""The base URL to use for downloading package indexes.""" - -VERSION = "1.0-alpha6" -"""The version to check. Bump this after each release.""" - -def main(): - """The main function.""" - - arches = set() - sys_ign = set() - user_ign = set() - with open('arches', 'r') as arch_file: - arches = [arch[:-1] for arch in arch_file.readlines()] - with open('sys-specific', 'r') as ignore_file: - sys_ign = set(pkg[:-1] for pkg in ignore_file.readlines()) - with open('user-specific', 'r') as ignore_file: - user_ign = set(pkg[:-1] for pkg in ignore_file.readlines()) - - all_sys_pkgs = set() - all_user_pkgs = set() - pkgs = {} - - broken_sys = list() - broken_user = list() - not_broken = list() - - for arch in arches: - print("Loading " + arch + "...") - arch_sys = Index(url=BASE_URL.format(version=VERSION, repo='system', arch=arch)) - arch_user = Index(url=BASE_URL.format(version=VERSION, repo='user', arch=arch)) - - all_sys_pkgs = all_sys_pkgs.union(arch_sys.origins) - all_user_pkgs = all_user_pkgs.union(arch_user.origins) - - pkgs[arch] = {'system': arch_sys.origins, 'user': arch_user.origins} - - for pkg in sys_ign: - all_sys_pkgs.discard(pkg) - for pkg in user_ign: - all_user_pkgs.discard(pkg) - - for arch in arches: - missing_sys = all_sys_pkgs - pkgs[arch]['system'] - missing_user = all_user_pkgs - pkgs[arch]['user'] - - if len(missing_sys) > 0: - print("Missing in {arch}/system:".format(arch=arch)) - print("=========================") - missing = list(missing_sys) - missing.sort() - for pkg in missing: print(pkg) - print("\n\n") - broken_sys.append(arch) - - if len(missing_user) > 0: - print("Missing in {arch}/user:".format(arch=arch)) - print("=========================") - missing = list(missing_user) - missing.sort() - for pkg in missing: print(pkg) - print("\n\n") - broken_user.append(arch) - - if len(missing_sys) == 0 and len(missing_user) == 0: - not_broken.append(arch) - - print("\033[1;32mNo issues\033[1;0m: {a}".format(a=", ".join(not_broken))) - print("\033[1;33mMissing user\033[1;0m: {a}".format(a=", ".join( - ["{x} ({y})".format(x=arch, y=len(all_user_pkgs - pkgs[arch]['user'])) - for arch in broken_user]))) - print("\033[1;31mMissing system\033[1;0m: {a}".format(a=", ".join( - ["{x} ({y})".format(x=arch, y=len(all_sys_pkgs - pkgs[arch]['system'])) - for arch in broken_sys]))) - -if __name__ == "__main__": - main() - diff --git a/version.py b/version.py new file mode 100644 index 0000000..1de0442 --- /dev/null +++ b/version.py @@ -0,0 +1,75 @@ +#!/usr/bin/env python3 +# +# Copyright © 2018-2020 Adélie Linux team. All rights reserved. +# NCSA license. +# +import ctypes +import enum +import functools + +class APK_VER(enum.IntFlag): + UNKNOWN = 0 + EQUAL = 1 + LESS = 2 + GREATER = 4 + FUZZY = 8 + +APK_OPS = { + ">=": APK_VER.GREATER | APK_VER.EQUAL, + "<=": APK_VER.LESS | APK_VER.EQUAL, + ">~": APK_VER.GREATER | APK_VER.EQUAL, + "<~": APK_VER.LESS | APK_VER.EQUAL, + ">": APK_VER.GREATER, + "<": APK_VER.LESS, + "=": APK_VER.EQUAL, + "~": APK_VER.EQUAL, +} + +class _apk_blob_t(ctypes.Structure): + _fields_ = [ + ("len", ctypes.c_long), + ("ptr", ctypes.c_char_p), + ] + + def __init__(self, s): + s = s.encode("utf-8") + self.len = len(s) + self.ptr = ctypes.c_char_p(s) + +def vercmp(a, b, fuzzy=False): + a = _apk_blob_t(a) + b = _apk_blob_t(b) + fuzzy = 1 if fuzzy else 0 + return APK_VER(_LIBAPK.apk_version_compare_blob_fuzzy(a, b, fuzzy)) + +@functools.cmp_to_key +def verkey(a, b): + r = vercmp(a, b) + + if r == APK_VER.LESS: + return -1 + if r == APK_VER.EQUAL: + return 0 + if r == APK_VER.GREATER: + return 1 + + return None + +def ver_is(a, op, b): + if op not in APK_OPS: + raise ValueError("Invalid op " + repr(op)) + + fuzzy = "~" in op + return vercmp(a, b, fuzzy) & APK_OPS[op] + +def is_older(old, new): + return vercmp(old, new) == APK_VER.LESS + +def is_same(old, new): + return vercmp(old, new) == APK_VER.EQUAL + +_LIBAPK = ctypes.CDLL("libapk.so") +_LIBAPK.apk_version_compare_blob_fuzzy.argtypes = [ + _apk_blob_t, _apk_blob_t, ctypes.c_int, +] +_LIBAPK.apk_version_compare_blob_fuzzy.restype = ctypes.c_int diff --git a/vertest.py b/vertest.py deleted file mode 100644 index be5eaa3..0000000 --- a/vertest.py +++ /dev/null @@ -1,138 +0,0 @@ -#!/usr/bin/env python3 -# Adélie Linux architecture package tester -# Ensure all packages are all on arches -# -# Copyright © 2018 Adélie Linux team. All rights reserved. -# NCSA license. -# - -from apkkit.base.index import Index - -BASE_URL = "https://mirrormaster.adelielinux.org/adelie/{version}/{repo}/{arch}/APKINDEX.tar.gz" -"""The base URL to use for downloading package indexes.""" - -VERSION = "1.0-alpha6" -"""The version to check. Bump this after each release.""" - -def compare_versions(old, new): - """Compare two versions. Return True if equal.""" - return (old == new) - -def compare_older(old, new): - """Compare two versions. Return True if old is older.""" - for part in zip(old.split('.'), new.split('.')): - # part[0] = next component of old ver, part[1] = new - try: - oldpart = int(part[0]) - newpart = int(part[1]) - except ValueError: - oldpart = part[0] - newpart = part[1] - if oldpart < newpart: - return True - elif oldpart == newpart: - continue - else: - return False - return False - -def process_one_pkg(mydict, next_pkg): - """Process a package.""" - new = next_pkg.version - if next_pkg.origin in mydict: - curr = mydict[next_pkg.origin] - - if not compare_versions(curr, new) and compare_older(curr, new): - mydict[next_pkg.origin] = new - else: - pass # Do nothing if this is older. - else: # Doesn't exist in this dict, put it in. - mydict[next_pkg.origin] = new - -def main(): - """The main function.""" - - arches = set() - sys_ign = set() - user_ign = set() - with open('arches', 'r') as arch_file: - arches = [arch[:-1] for arch in arch_file.readlines()] - with open('sys-specific', 'r') as ignore_file: - sys_ign = set(pkg[:-1] for pkg in ignore_file.readlines()) - with open('user-specific', 'r') as ignore_file: - user_ign = set(pkg[:-1] for pkg in ignore_file.readlines()) - - all_sys_pkgs = dict() - all_user_pkgs = dict() - pkgs = {} - - arch_sys_pkgs = {arch: dict() for arch in arches} - arch_user_pkgs = {arch: dict() for arch in arches} - broken_sys = list() - broken_user = list() - not_broken = list() - - for arch in arches: - print("Loading " + arch + "...") - arch_sys = Index(url=BASE_URL.format(version=VERSION, repo='system', arch=arch)) - arch_user = Index(url=BASE_URL.format(version=VERSION, repo='user', arch=arch)) - - for next_pkg in arch_sys.packages: - for mydict in (all_sys_pkgs, arch_sys_pkgs[arch]): - process_one_pkg(mydict, next_pkg) - - for next_pkg in arch_user.packages: - for mydict in (all_user_pkgs, arch_user_pkgs[arch]): - process_one_pkg(mydict, next_pkg) - - all_sys_pkgs.pop('firefox-esr') - all_sys_pkgs.pop('gcc') - - for arch in arches: - old_sys = list() - old_user = list() - - for pkg in all_sys_pkgs.keys(): - sysver = all_sys_pkgs[pkg] - if pkg in arch_sys_pkgs[arch]: - archver = arch_sys_pkgs[arch][pkg] - if compare_older(archver, sysver): - old_sys.append((pkg, sysver, archver)) - - for pkg in all_user_pkgs.keys(): - userver = all_user_pkgs[pkg] - if pkg in arch_user_pkgs[arch]: - archver = arch_user_pkgs[arch][pkg] - if compare_older(archver, userver): - old_user.append((pkg, userver, archver)) - - if len(old_sys) > 0: - print("Outdated in {arch}/system:".format(arch=arch)) - print("=========================") - old_sys.sort() - for pkg, sys, archver in old_sys: - print("{0}: {1} is older than {2}".format(pkg, archver, sys)) - print("\n\n") - broken_sys.append(arch) - - if len(old_user) > 0: - print("Outdated in {arch}/user:".format(arch=arch)) - print("=========================") - old_user.sort() - for pkg, user, archver in old_user: - print("{0}: {1} is older than {2}".format(pkg, archver, user)) - print("\n\n") - broken_user.append(arch) - - if len(old_sys) == 0 and len(old_user) == 0: - not_broken.append(arch) - - print("\033[1;32mNo issues\033[1;0m: {a}".format(a=", ".join(not_broken))) - print("\033[1;33mOutdated user\033[1;0m: {a}".format(a=", ".join( - ["{x}".format(x=arch) for arch in broken_user]))) - print("\033[1;31mOutdated system\033[1;0m: {a}".format(a=", ".join( - ["{x}".format(x=arch) for arch in broken_sys]))) - -if __name__ == "__main__": - main() - |