Overhaul with multiple formats/orders + libapk for version comparison

8 files changed, 490 insertions, 232 deletions

diff --git a/arches b/arches
deleted file mode 100644
index 90e6a18..0000000
--- a/arches
+++ /dev/null
@@ -1,6 +0,0 @@
-aarch64
-armv7
-pmmx
-ppc
-ppc64
-x86_64
diff --git a/depver_test.py b/depver_test.py
new file mode 100644
index 0000000..3dcf596
--- /dev/null
+++ b/depver_test.py
@@ -0,0 +1,142 @@
+#!/usr/bin/env python3
+# Adélie Linux architecture package tester
+# Ensure all packages depend on latest versions of each other
+#
+# Copyright © 2019-2020 Adélie Linux team.  All rights reserved.
+# NCSA license.
+#
+import argparse
+import collections
+import functools
+import os
+import sys
+
+from apkkit.base.index import Index
+
+from version import is_older, is_same, verkey, ver_is, APK_OPS
+from output import FORMATTERS, No, Partial
+
+def atomize(spec):
+    if not any(i in spec for i in APK_OPS):
+        return spec, None
+
+    for op in APK_OPS:
+        try:
+            name, ver = spec.split(op, maxsplit=1)
+        except ValueError:
+            continue
+
+        return name, functools.partial(ver_is, op=op, b=ver)
+
+    # Not reached
+    assert False
+
+def analyze(url, repos, arch):
+    pkgs = collections.defaultdict(dict)
+    newest = {}
+    providers = collections.defaultdict(list)
+
+    print("Loading " + arch + "...", file=sys.stderr)
+
+    index = []
+    for repo in repos:
+        index.extend(Index(url=url + f"/{repo}/{arch}/APKINDEX.tar.gz").packages)
+
+    for pkg in index:
+        new = pkg.version
+        pkgs[pkg.name][new] = pkg
+
+        curr = newest.get(pkg.name, None)
+        if curr is None:
+            newest[pkg.name] = new
+        elif is_older(curr, new):
+            newest[pkg.name] = new
+
+        providers[pkg.name].append((pkg.name, new))
+        for i in pkg.provides:
+            i = i.split("=", maxsplit=1)
+            if len(i) == 2:
+                i, pver = i
+            else:
+                i, pver = i[0], new
+
+            curr = newest.get(i, None)
+            if curr is None:
+                newest[i] = pver
+            elif is_older(curr, pver):
+                newest[i] = pver
+
+            providers[i].append((pkg.name, pver))
+
+    yield ["arch", "package", "version", "issue"]
+
+    for name in sorted(pkgs.keys()):
+        # DON'T use newest[] here. It is possible that a package
+        # provides= another package's name with a newer version
+        ver = sorted(pkgs[name].keys(), key=verkey)[-1]
+        pkg = pkgs[name][ver]
+
+        for dep in pkg.depends:
+            if dep.startswith("!"):
+                continue
+
+            spec = dep
+            dep, constraint = atomize(dep)
+
+            if dep in pkg.provides:
+                continue
+            if dep not in providers:
+                yield [arch, name, ver, No(f"Missing {spec}")]
+                continue
+
+            if constraint:
+                for provider, pver in providers[dep]:
+                    if constraint(pver):
+                        break
+                else:
+                    yield [arch, name, ver, No(f"Missing {spec}")]
+
+            for _, pver in providers[dep]:
+                if is_same(pver, newest[dep]):
+                    break
+            else:
+                yield [arch, name, ver, Partial(f"Old {spec}")]
+
+if __name__ == "__main__":
+    opts = argparse.ArgumentParser(
+        description="""Scan the REPO/ARCH repositories under URL and
+        look for outdated or missing dependencies as required by the
+        most recent version of each package. This script examines both
+        main packages and subpackages.
+
+        Be sure to include any repositories on which the repository of
+        interest depends, otherwise a lot of dependencies will be
+        incorrectly marked as missing.""",
+    )
+    opts.add_argument(
+        "-f", "--format", choices=FORMATTERS.keys(),
+        default="pretty" if os.isatty(sys.stdout.fileno()) else "tab",
+        help="display format",
+    )
+    opts.add_argument(
+        "url", metavar="URL",
+        help="base URL (no repository or arch)",
+    )
+    opts.add_argument(
+        "repos", metavar="REPOS",
+        help="repositories (comma separated)",
+    )
+    opts.add_argument(
+        "arches", metavar="ARCHES",
+        help="architectures (comma separated)",
+    )
+    opts = opts.parse_args()
+    opts.repos = opts.repos.split(",")
+    opts.arches = opts.arches.split(",")
+
+    for arch in opts.arches:
+        # FIXME: with html, wrapping will repeat for > 1 arch
+        FORMATTERS[opts.format](
+            opts,
+            analyze(opts.url, opts.repos, arch),
+        )
diff --git a/output.py b/output.py
new file mode 100644
index 0000000..870d618
--- /dev/null
+++ b/output.py
@@ -0,0 +1,136 @@
+#!/usr/bin/env python3
+#
+# Copyright © 2019-2020 Adélie Linux team.  All rights reserved.
+# NCSA license.
+#
+import subprocess
+
+class _Cell:
+    def __init__(self, content):
+        self.content = content
+
+_ANSI_CLEAR = "\033[0;00m"
+
+class Yes(_Cell):
+    symbol = "✓"
+    cls = "cell-yes"
+    color = "\033[1;32m"
+
+class No(_Cell):
+    symbol = "✗"
+    cls = "cell-no"
+    color = "\033[1;31m"
+
+class Partial(_Cell):
+    symbol = "?"
+    cls = "cell-partial"
+    color = "\033[1;33m"
+
+PARTIAL_MISSING = Partial("missing")
+YES_MISSING = Yes("missing")
+
+_HEADER = r"""<!DOCTYPE html>
+<html>
+ <head>
+  <title>APKINDEX analysis - {date}</title>
+  <meta http-equiv="content-type" content="text/html; charset=utf-8">
+  <style type="text/css">
+   table {
+    background-color: #F8F9FA;
+    color: #222;
+    margin: 1em 0;
+    border: 1px solid #A2A9B1;
+    border-collapse: collapse;
+   }
+   td, th {
+    border: 1px solid #A2A9B1;
+    padding: 0.2em 0.4em;
+   }
+   th {
+    background-color: #EAECF0;
+    text-align: center;
+    position: sticky;
+    top: 0;
+   }
+   .cell-yes { background-color: #DFF0D8; color: black; }
+   .cell-no { background-color: #F2DEDE; color: black; }
+   .cell-partial { background: #FFF176; color: black; }
+  </style>
+ </head>
+ <body>
+  <h1>APKINDEX analysis - {date}</h1>
+  <p>
+   Base URL: <a href="{url}">{url}</a><br>
+   Repositories: {repos}<br>
+   Architectures: {arches}<br>
+  </p>
+"""
+_FOOTER = r""" </body>
+</html>
+"""
+
+def format_tab(_, rows):
+    for row in rows:
+        for i, j in enumerate(row):
+            if isinstance(j, _Cell):
+                row[i] = f"{j.symbol} {j.content}"
+
+        print(*row, sep="\t")
+
+def format_pretty(_, rows):
+    proc = subprocess.Popen(
+        ("column", "-ts", "\t"),
+        stdin=subprocess.PIPE,
+        encoding="utf-8",
+    )
+
+    for row in rows:
+        for i, j in enumerate(row):
+            if isinstance(j, _Cell):
+                row[i] = f"{j.color}{j.symbol} {j.content}{_ANSI_CLEAR}"
+            else:
+                # Work around column(1) not liking ANSI escapes by
+                # always printing some
+                row[i] = f"{_ANSI_CLEAR}{j}{_ANSI_CLEAR}"
+
+        print(*row, file=proc.stdin, sep="\t")
+
+    proc.communicate()
+
+def format_html(opts, rows):
+    print(_HEADER)
+
+    first = True
+    for row in rows:
+        if first:
+            print("<table>")
+            print(
+                "<thead><tr><th>",
+                "</th><th>".join(row),
+                "</th></tr></thead>", sep=""
+            )
+            print("<tbody>")
+            first = False
+            continue
+
+        for i, j in enumerate(row[1:]):
+            i += 1
+
+            if isinstance(j, _Cell):
+                row[i] = f" class='{j.cls}'>{j.symbol} {j.content}"
+            else:
+                row[i] = ">" + j
+
+        print("<tr><td>", "</td><td".join(row), "</td></tr>", sep="")
+
+    if not first:
+        print("</tbody>")
+        print("</table>")
+
+    print(_FOOTER)
+
+FORMATTERS = {
+    "tab": format_tab,
+    "pretty": format_pretty,
+    "html": format_html,
+}
diff --git a/pkgver_test.py b/pkgver_test.py
new file mode 100644
index 0000000..9566908
--- /dev/null
+++ b/pkgver_test.py
@@ -0,0 +1,137 @@
+#!/usr/bin/env python3
+# Adélie Linux architecture package tester
+# Ensure all packages are all on arches
+#
+# Copyright © 2018-2020 Adélie Linux team.  All rights reserved.
+# NCSA license.
+#
+import argparse
+import os
+import sys
+from pathlib import Path
+
+from apkkit.base.index import Index
+
+from output import FORMATTERS, Yes, No, YES_MISSING, PARTIAL_MISSING
+from version import is_older
+
+def analyze(url, repo, arches):
+    newest = dict()
+    arch_newest = {arch: dict() for arch in arches}
+
+    for arch in arches:
+        print(f"Loading {arch}...", file=sys.stderr)
+
+        for pkg in Index(url=f"{url}/{repo}/{arch}/APKINDEX.tar.gz").packages:
+            if pkg.name != pkg.origin:
+                continue
+            new = pkg.version
+
+            for mydict in (newest, arch_newest[arch]):
+                curr = mydict.get(pkg.name, None)
+                if curr is None:
+                    mydict[pkg.name] = new
+                elif is_older(curr, new):
+                    mydict[pkg.name] = new
+
+    return newest, arch_newest
+
+def order_arch(arches, newest, arch_newest, ign):
+    yield ["arch", "package", "version for arch", "newest version"]
+
+    for arch in arches:
+        old = list()
+
+        for pkg in newest.keys():
+            ver = newest[pkg]
+            archver = arch_newest[arch].get(pkg, None)
+
+            if archver is None and pkg not in ign:
+                yield [arch, pkg, PARTIAL_MISSING, ver]
+
+            elif archver is not None and is_older(archver, ver):
+                yield [arch, pkg, archver, ver]
+
+def order_pkg(arches, newest, arch_newest, ign):
+    yield ["package", *arches]
+
+    for pkg in sorted(newest.keys()):
+        for i in arches:
+            archver = arch_newest[i].get(pkg, None)
+            if archver is None:
+                break
+            if archver is not None and is_older(archver, newest[pkg]):
+                break
+        else:
+            continue
+
+        row = [pkg]
+        for i in arches:
+            archver = arch_newest[i].get(pkg, None)
+            if archver is None and pkg in ign:
+                row.append(YES_MISSING)
+            elif archver is None:
+                row.append(PARTIAL_MISSING)
+            elif is_older(archver, newest[pkg]):
+                row.append(No(archver))
+            else:
+                row.append(Yes(archver))
+
+        yield row
+
+ORDERS = {
+    "pkg": order_pkg,
+    "arch": order_arch,
+}
+
+if __name__ == "__main__":
+    opts = argparse.ArgumentParser(
+        description="""Scan the REPO/ARCH repositories under URL and
+        look for outdated or missing packages when comparing across
+        architectures. This script only examines main packages, not
+        subpackages.
+
+        At least two architectures must be given in order to make a
+        comparison."""
+    )
+    opts.add_argument(
+        "-f", "--format", choices=FORMATTERS.keys(),
+        default="pretty" if os.isatty(sys.stdout.fileno()) else "tab",
+        help="display format",
+    )
+    opts.add_argument(
+        "-o", "--order", choices=ORDERS.keys(),
+        default="pkg",
+        help="display order",
+    )
+    opts.add_argument(
+        "url", metavar="URL",
+        help="base URL (no repository or arch)",
+    )
+    opts.add_argument(
+        "repo", metavar="REPO",
+        help="repository",
+    )
+    opts.add_argument(
+        "arches", metavar="ARCHES",
+        help="architectures (comma separated, at least 2)",
+    )
+    opts = opts.parse_args()
+    opts.arches = opts.arches.split(",")
+
+    if len(opts.arches) < 2:
+        print("At least two arches are required", file=sys.stderr)
+        sys.exit(1)
+
+    if Path(opts.repo + "-specific").is_file():
+        with open(opts.repo + '-specific', 'r') as ignore_file:
+            ign = set(pkg[:-1] for pkg in ignore_file.readlines())
+    else:
+        ign = set()
+
+    newest, arch_newest = analyze(opts.url, opts.repo, opts.arches)
+    # FIXME: support multiple repos
+    FORMATTERS[opts.format](
+        opts,
+        ORDERS[opts.order](opts.arches, newest, arch_newest, ign),
+    )
diff --git a/sys-specific b/system-specific
index e69de29..e69de29 100644
--- a/sys-specific
+++ b/system-specific
diff --git a/tester.py b/tester.py
deleted file mode 100644
index 797216f..0000000
--- a/tester.py
+++ /dev/null
@@ -1,88 +0,0 @@
-#!/usr/bin/env python3
-# Adélie Linux architecture package tester
-# Ensure all packages are all on arches
-#
-# Copyright © 2018 Adélie Linux team.  All rights reserved.
-# NCSA license.
-#
-
-from apkkit.base.index import Index
-
-BASE_URL = "https://mirrormaster.adelielinux.org/adelie/{version}/{repo}/{arch}/APKINDEX.tar.gz"
-"""The base URL to use for downloading package indexes."""
-
-VERSION = "1.0-alpha6"
-"""The version to check.  Bump this after each release."""
-
-def main():
-    """The main function."""
-
-    arches = set()
-    sys_ign = set()
-    user_ign = set()
-    with open('arches', 'r') as arch_file:
-        arches = [arch[:-1] for arch in arch_file.readlines()]
-    with open('sys-specific', 'r') as ignore_file:
-        sys_ign = set(pkg[:-1] for pkg in ignore_file.readlines())
-    with open('user-specific', 'r') as ignore_file:
-        user_ign = set(pkg[:-1] for pkg in ignore_file.readlines())
-
-    all_sys_pkgs = set()
-    all_user_pkgs = set()
-    pkgs = {}
-
-    broken_sys = list()
-    broken_user = list()
-    not_broken = list()
-
-    for arch in arches:
-        print("Loading " + arch + "...")
-        arch_sys = Index(url=BASE_URL.format(version=VERSION, repo='system', arch=arch))
-        arch_user = Index(url=BASE_URL.format(version=VERSION, repo='user', arch=arch))
-
-        all_sys_pkgs = all_sys_pkgs.union(arch_sys.origins)
-        all_user_pkgs = all_user_pkgs.union(arch_user.origins)
-
-        pkgs[arch] = {'system': arch_sys.origins, 'user': arch_user.origins}
-
-    for pkg in sys_ign:
-        all_sys_pkgs.discard(pkg)
-    for pkg in user_ign:
-        all_user_pkgs.discard(pkg)
-
-    for arch in arches:
-        missing_sys = all_sys_pkgs - pkgs[arch]['system']
-        missing_user = all_user_pkgs - pkgs[arch]['user']
-
-        if len(missing_sys) > 0:
-            print("Missing in {arch}/system:".format(arch=arch))
-            print("=========================")
-            missing = list(missing_sys)
-            missing.sort()
-            for pkg in missing: print(pkg)
-            print("\n\n")
-            broken_sys.append(arch)
-
-        if len(missing_user) > 0:
-            print("Missing in {arch}/user:".format(arch=arch))
-            print("=========================")
-            missing = list(missing_user)
-            missing.sort()
-            for pkg in missing: print(pkg)
-            print("\n\n")
-            broken_user.append(arch)
-
-        if len(missing_sys) == 0 and len(missing_user) == 0:
-            not_broken.append(arch)
-
-    print("\033[1;32mNo issues\033[1;0m: {a}".format(a=", ".join(not_broken)))
-    print("\033[1;33mMissing user\033[1;0m: {a}".format(a=", ".join(
-        ["{x} ({y})".format(x=arch, y=len(all_user_pkgs - pkgs[arch]['user']))
-            for arch in broken_user])))
-    print("\033[1;31mMissing system\033[1;0m: {a}".format(a=", ".join(
-        ["{x} ({y})".format(x=arch, y=len(all_sys_pkgs - pkgs[arch]['system']))
-            for arch in broken_sys])))
-
-if __name__ == "__main__":
-    main()
-
diff --git a/version.py b/version.py
new file mode 100644
index 0000000..1de0442
--- /dev/null
+++ b/version.py
@@ -0,0 +1,75 @@
+#!/usr/bin/env python3
+#
+# Copyright © 2018-2020 Adélie Linux team.  All rights reserved.
+# NCSA license.
+#
+import ctypes
+import enum
+import functools
+
+class APK_VER(enum.IntFlag):
+    UNKNOWN = 0
+    EQUAL = 1
+    LESS = 2
+    GREATER = 4
+    FUZZY = 8
+
+APK_OPS = {
+   ">=": APK_VER.GREATER | APK_VER.EQUAL,
+   "<=": APK_VER.LESS | APK_VER.EQUAL,
+   ">~": APK_VER.GREATER | APK_VER.EQUAL,
+   "<~": APK_VER.LESS | APK_VER.EQUAL,
+   ">": APK_VER.GREATER,
+   "<": APK_VER.LESS,
+   "=": APK_VER.EQUAL,
+   "~": APK_VER.EQUAL,
+}
+
+class _apk_blob_t(ctypes.Structure):
+    _fields_ = [
+        ("len", ctypes.c_long),
+        ("ptr", ctypes.c_char_p),
+    ]
+
+    def __init__(self, s):
+        s = s.encode("utf-8")
+        self.len = len(s)
+        self.ptr = ctypes.c_char_p(s)
+
+def vercmp(a, b, fuzzy=False):
+    a = _apk_blob_t(a)
+    b = _apk_blob_t(b)
+    fuzzy = 1 if fuzzy else 0
+    return APK_VER(_LIBAPK.apk_version_compare_blob_fuzzy(a, b, fuzzy))
+
+@functools.cmp_to_key
+def verkey(a, b):
+    r = vercmp(a, b)
+
+    if r == APK_VER.LESS:
+        return -1
+    if r == APK_VER.EQUAL:
+        return 0
+    if r == APK_VER.GREATER:
+        return 1
+
+    return None
+
+def ver_is(a, op, b):
+    if op not in APK_OPS:
+        raise ValueError("Invalid op " + repr(op))
+
+    fuzzy = "~" in op
+    return vercmp(a, b, fuzzy) & APK_OPS[op]
+
+def is_older(old, new):
+    return vercmp(old, new) == APK_VER.LESS
+
+def is_same(old, new):
+    return vercmp(old, new) == APK_VER.EQUAL
+
+_LIBAPK = ctypes.CDLL("libapk.so")
+_LIBAPK.apk_version_compare_blob_fuzzy.argtypes = [
+    _apk_blob_t, _apk_blob_t, ctypes.c_int,
+]
+_LIBAPK.apk_version_compare_blob_fuzzy.restype = ctypes.c_int
diff --git a/vertest.py b/vertest.py
deleted file mode 100644
index be5eaa3..0000000
--- a/vertest.py
+++ /dev/null
@@ -1,138 +0,0 @@
-#!/usr/bin/env python3
-# Adélie Linux architecture package tester
-# Ensure all packages are all on arches
-#
-# Copyright © 2018 Adélie Linux team.  All rights reserved.
-# NCSA license.
-#
-
-from apkkit.base.index import Index
-
-BASE_URL = "https://mirrormaster.adelielinux.org/adelie/{version}/{repo}/{arch}/APKINDEX.tar.gz"
-"""The base URL to use for downloading package indexes."""
-
-VERSION = "1.0-alpha6"
-"""The version to check.  Bump this after each release."""
-
-def compare_versions(old, new):
-    """Compare two versions.  Return True if equal."""
-    return (old == new)
-
-def compare_older(old, new):
-    """Compare two versions.  Return True if old is older."""
-    for part in zip(old.split('.'), new.split('.')):
-        # part[0] = next component of old ver, part[1] = new
-        try:
-            oldpart = int(part[0])
-            newpart = int(part[1])
-        except ValueError:
-            oldpart = part[0]
-            newpart = part[1]
-        if oldpart < newpart:
-            return True
-        elif oldpart == newpart:
-            continue
-        else:
-            return False
-    return False
-
-def process_one_pkg(mydict, next_pkg):
-    """Process a package."""
-    new = next_pkg.version
-    if next_pkg.origin in mydict:
-        curr = mydict[next_pkg.origin]
-
-        if not compare_versions(curr, new) and compare_older(curr, new):
-            mydict[next_pkg.origin] = new
-        else:
-            pass  # Do nothing if this is older.
-    else:  # Doesn't exist in this dict, put it in.
-        mydict[next_pkg.origin] = new
-
-def main():
-    """The main function."""
-
-    arches = set()
-    sys_ign = set()
-    user_ign = set()
-    with open('arches', 'r') as arch_file:
-        arches = [arch[:-1] for arch in arch_file.readlines()]
-    with open('sys-specific', 'r') as ignore_file:
-        sys_ign = set(pkg[:-1] for pkg in ignore_file.readlines())
-    with open('user-specific', 'r') as ignore_file:
-        user_ign = set(pkg[:-1] for pkg in ignore_file.readlines())
-
-    all_sys_pkgs = dict()
-    all_user_pkgs = dict()
-    pkgs = {}
-
-    arch_sys_pkgs = {arch: dict() for arch in arches}
-    arch_user_pkgs = {arch: dict() for arch in arches}
-    broken_sys = list()
-    broken_user = list()
-    not_broken = list()
-
-    for arch in arches:
-        print("Loading " + arch + "...")
-        arch_sys = Index(url=BASE_URL.format(version=VERSION, repo='system', arch=arch))
-        arch_user = Index(url=BASE_URL.format(version=VERSION, repo='user', arch=arch))
-
-        for next_pkg in arch_sys.packages:
-            for mydict in (all_sys_pkgs, arch_sys_pkgs[arch]):
-                process_one_pkg(mydict, next_pkg)
-
-        for next_pkg in arch_user.packages:
-            for mydict in (all_user_pkgs, arch_user_pkgs[arch]):
-                process_one_pkg(mydict, next_pkg)
-
-    all_sys_pkgs.pop('firefox-esr')
-    all_sys_pkgs.pop('gcc')
-
-    for arch in arches:
-        old_sys = list()
-        old_user = list()
-
-        for pkg in all_sys_pkgs.keys():
-            sysver = all_sys_pkgs[pkg]
-            if pkg in arch_sys_pkgs[arch]:
-                archver = arch_sys_pkgs[arch][pkg]
-                if compare_older(archver, sysver):
-                    old_sys.append((pkg, sysver, archver))
-
-        for pkg in all_user_pkgs.keys():
-            userver = all_user_pkgs[pkg]
-            if pkg in arch_user_pkgs[arch]:
-                archver = arch_user_pkgs[arch][pkg]
-                if compare_older(archver, userver):
-                    old_user.append((pkg, userver, archver))
-
-        if len(old_sys) > 0:
-            print("Outdated in {arch}/system:".format(arch=arch))
-            print("=========================")
-            old_sys.sort()
-            for pkg, sys, archver in old_sys:
-                print("{0}: {1} is older than {2}".format(pkg, archver, sys))
-            print("\n\n")
-            broken_sys.append(arch)
-
-        if len(old_user) > 0:
-            print("Outdated in {arch}/user:".format(arch=arch))
-            print("=========================")
-            old_user.sort()
-            for pkg, user, archver in old_user:
-                print("{0}: {1} is older than {2}".format(pkg, archver, user))
-            print("\n\n")
-            broken_user.append(arch)
-
-        if len(old_sys) == 0 and len(old_user) == 0:
-            not_broken.append(arch)
-
-    print("\033[1;32mNo issues\033[1;0m: {a}".format(a=", ".join(not_broken)))
-    print("\033[1;33mOutdated user\033[1;0m: {a}".format(a=", ".join(
-        ["{x}".format(x=arch) for arch in broken_user])))
-    print("\033[1;31mOutdated system\033[1;0m: {a}".format(a=", ".join(
-        ["{x}".format(x=arch) for arch in broken_sys])))
-
-if __name__ == "__main__":
-    main()
-