tools: rewrite support.py

tools/support.py

@@ -1,4 +1,5 @@
 #!/usr/bin/env python3
+import re
 import os
 import sys
 import click
@@ -6,125 +7,280 @@ import coin_info
 import json
 
 SUPPORT_INFO = coin_info.get_support_data()
+MISSING_MEANS_NO = ("connect", "webwallet")
+VERSIONED_SUPPORT_INFO = ("trezor1", "trezor2")
 
-MANDATORY_ENTRIES = ("trezor1", "trezor2", "connect", "webwallet")
+VERSION_RE = re.compile(r"\d+.\d+.\d+")
-
-
-def update_support(key, entry, value):
-    # template entry
-    support = {k: None for k in MANDATORY_ENTRIES}
-    support["other"] = {}
-    # fill out actual support info, if it exists
-    support.update(SUPPORT_INFO.get(key, {}))
-
-    if entry in MANDATORY_ENTRIES:
-        if entry.startswith("trezor") and not value:
-            value = None
-        support[entry] = value
-    else:
-        support["other"][entry] = value
-
-    for k in support["other"]:
-        if not support["other"][k]:
-            del support["other"][k]
-
-    if not support["other"]:
-        del support["other"]
-
-    SUPPORT_INFO[key] = support
-    return support
 
 
 def write_support_info():
     with open(os.path.join(coin_info.DEFS_DIR, "support.json"), "w") as f:
-        json.dump(SUPPORT_INFO, f, indent=4)
+        json.dump(SUPPORT_INFO, f, indent=2, sort_keys=True)
         f.write("\n")
 
 
+def print_support(coin):
+    def support_value(where, key, missing_means_no=False):
+        if "supported" in where and key in where["supported"]:
+            val = where["supported"][key]
+            if val is True:
+                return "YES"
+            elif val == "soon":
+                return "SOON"
+            elif VERSION_RE.match(val):
+                return f"YES since {val}"
+            else:
+                return f"BAD VALUE {val}"
+        elif "unsupported" in where and key in where["unsupported"]:
+            val = where["unsupported"][key]
+            return f"NO (reason: {val})"
+        elif missing_means_no:
+            return "NO"
+        else:
+            return "support info missing"
+
+    key, name, shortcut = coin["key"], coin["name"], coin["shortcut"]
+    print(f"{key} - {name} ({shortcut})")
+    if coin.get("duplicate"):
+        print(" * DUPLICATE SYMBOL (no support)")
+    else:
+        for dev, where in SUPPORT_INFO.items():
+            missing_means_no = dev in MISSING_MEANS_NO
+            print(" *", dev, ":", support_value(where, key, missing_means_no))
+
+
+# ====== validation functions ====== #
+
+
+def check_support_values():
+    def _check_value_version_soon(val):
+        if not isinstance(value, str):
+            raise ValueError(f"non-str value: {value}")
+
+        is_version = VERSION_RE.match(value)
+        is_soon = value == "soon"
+        if not (is_version or is_soon):
+            raise ValueError(f"expected version or 'soon', found '{value}'")
+
+    errors = []
+    for device, values in SUPPORT_INFO.items():
+        supported = values.get("supported")
+        if not isinstance(supported, dict):
+            errors.append(f"Missing 'supported' dict for {device}")
+        else:
+            for key, value in supported.items():
+                try:
+                    if device in VERSIONED_SUPPORT_INFO:
+                        _check_value_version_soon(value)
+                    else:
+                        if value is not True:
+                            raise ValueError(f"only allowed is True, but found {value}")
+                except Exception as e:
+                    errors.append(f"{device}.supported.{key}: {e}")
+
+        unsupported = values.get("unsupported")
+        if not isinstance(unsupported, dict):
+            errors.append(f"Missing 'supported' dict for {device}")
+        else:
+            for key, value in unsupported.items():
+                if not isinstance(value, str) or not value:
+                    errors.append(f"{device}.unsupported.{key}: missing reason")
+
+    return errors
+
+
+def find_unsupported_coins(coins_dict):
+    result = {}
+    for device in VERSIONED_SUPPORT_INFO:
+        values = SUPPORT_INFO[device]
+        support_set = set()
+        support_set.update(values["supported"].keys())
+        support_set.update(values["unsupported"].keys())
+
+        result[device] = unsupported = []
+        for key, coin in coins_dict.items():
+            if coin.get("duplicate"):
+                continue
+            if key not in support_set:
+                unsupported.append(coin)
+
+    return result
+
+
+def find_orphaned_support_keys(coins_dict):
+    result = {}
+    for device, values in SUPPORT_INFO.items():
+        device_res = {}
+        for supkey, supvalues in values.items():
+            orphans = set()
+            for coin_key in supvalues.keys():
+                if coin_key not in coins_dict:
+                    orphans.add(coin_key)
+            device_res[supkey] = orphans
+        result[device] = device_res
+
+    return result
+
+
 @click.group()
 def cli():
     pass
 
 
 @cli.command()
-def rewrite():
+# fmt: off
-    """Regenerate support.json to match predefined structure and field order."""
+@click.option("-p", "--prune-orphans", is_flag=True, help="Remove orphaned keys for which there is no corresponding coin info")
-    for key, coin in SUPPORT_INFO.items():
+@click.option("-t", "--ignore-tokens", is_flag=True, help="Ignore unsupported ERC20 tokens")
-        d = {"trezor1": None, "trezor2": None, "connect": None, "webwallet": None}
+# fmt: on
-        d.update(coin)
+def check(prune_orphans, ignore_tokens):
-        if "electrum" in d:
-            del d["electrum"]
-        if "other" in d and not d["other"]:
-            del d["other"]
-        SUPPORT_INFO[key] = d
-
-    write_support_info()
-
-
-@cli.command()
-def check():
     """Check validity of support information.
 
-    The relevant code is actually part of 'coin_gen.py'. It can be invoked from
+    Ensures that `support.json` data is well formed, there are no keys without
-    here for convenience and because it makes sense. But it's preferable to run it
+    corresponding coins, and there are no coins without corresponding keys.
-    as part of 'coin_gen.py check'.
-    """
-    defs = coin_info.get_all()
-    support_data = coin_info.get_support_data()
-    import coin_gen
 
-    if not coin_gen.check_support(defs, support_data, fail_missing=True):
+    If `--prune-orphans` is specified, orphaned keys (no corresponding coin)
+    will be deleted from `support.json`.
+
+    If `--ignore-tokens` is specified, the check will ignore ERC20 tokens
+    without support info. This is useful because there is usually a lot of ERC20
+    tokens.
+    """
+    coins_dict = coin_info.get_all(deduplicate=False).as_dict()
+    checks_ok = True
+
+    errors = check_support_values()
+    if errors:
+        for error in errors:
+            print(error)
+        checks_ok = False
+
+    orphaned = find_orphaned_support_keys(coins_dict)
+    for device, values in orphaned.items():
+        for supkey, supvalues in values.items():
+            for key in supvalues:
+                print(f"orphaned key {device} -> {supkey} -> {key}")
+                if prune_orphans:
+                    del SUPPORT_INFO[device][supkey][key]
+                else:
+                    checks_ok = False
+
+    if prune_orphans:
+        write_support_info()
+
+    missing = find_unsupported_coins(coins_dict)
+    for device, values in missing.items():
+        if ignore_tokens:
+            values = [coin for coin in values if not coin["key"].startswith("erc20:")]
+        if values:
+            checks_ok = False
+            print(f"Device {device} has missing support infos:")
+            for coin in values:
+                print(f"{coin['key']} - {coin['name']}")
+
+    if not checks_ok:
+        print("Some checks have failed")
         sys.exit(1)
 
 
 @cli.command()
-@click.argument("keyword", nargs=-1)
+# fmt: off
-def show(keyword):
+@click.argument("version")
-    """Show support status of specified coins.
+@click.option("--git-tag/--no-git-tag", "-g", default=False, help="create a corresponding Git tag")
+@click.option("--soon/--no-soon", default=True, help="Release coins marked 'soon'")
+@click.option("--missing/--no-missing", default=True, help="Release coins with missing support info")
+@click.option("-n", "--dry-run", is_flag=True, help="Do not write changes")
+# fmt: on
+def release(version, git_tag, soon, missing, dry_run):
+    """Release a new Trezor firmware.
 
-    Keywords match against key, name or shortcut (ticker symbol) of coin. If no
+    Update support infos so that all coins have a clear support status.
-    keywords are provided, show all supported coins.
+    By default, marks duplicate tokens as unsupported, and all coins that either
+    don't have support info, or they are supported "soon", are set to the
+    released firmware version.
 
-    Only coins listed in support.json are considered "supported". That means that
+    Optionally tags the repository with the given version.
-    Ethereum networks, ERC20 tokens and NEM mosaics will probably show up wrong.
     """
-    defs = coin_info.get_all().as_list()
+    version_tuple = list(map(int, version.split(".")))
+    device = f"trezor{version_tuple[0]}"
 
-    if keyword:
+    print(f"Releasing {device} firmware version {version}")
-        for coin in defs:
+
+    defs = coin_info.get_all(deduplicate=False)
+    coin_info.mark_duplicate_shortcuts(defs.as_list())
+    coins_dict = defs.as_dict()
+
+    if missing:
+        missing_list = find_unsupported_coins(coins_dict)[device]
+        for coin in missing_list:
             key = coin["key"]
-            name, shortcut = coin["name"], coin["shortcut"]
+            if coin.get("duplicate"):
-            for kw in keyword:
+                print(f"UNsupporting duplicate coin {key} ({coin['name']})")
-                kwl = kw.lower()
+                SUPPORT_INFO[device]["unsupported"][key] = "duplicate key"
-                if kwl == key.lower() or kwl in name.lower() or kwl == shortcut.lower():
+            else:
-                    print("{} - {} ({})".format(key, name, shortcut), end=" - ")
+                print(f"Adding missing {key} ({coin['name']})")
-                    if key in SUPPORT_INFO:
+                SUPPORT_INFO[device]["supported"][key] = version
-                        print(json.dumps(SUPPORT_INFO[key], indent=4))
-                    else:
-                        print("no support info")
-                    break
 
+    if soon:
+        soon_list = [
+            coins_dict[key]
+            for key, val in SUPPORT_INFO[device]["supported"].items()
+            if val == "soon" and key in coins_dict
+        ]
+        for coin in soon_list:
+            key = coin["key"]
+            print(f"Adding soon-released {key} ({coin['name']})")
+            SUPPORT_INFO[device]["supported"][key] = version
+
+    if git_tag:
+        print("git tag not supported yet")
+
+    if not dry_run:
+        write_support_info()
     else:
-        print(json.dumps(SUPPORT_INFO, indent=4))
+        print("No changes written")
 
 
 @cli.command()
-@click.argument("support_key", required=True)
+@click.argument("keyword", nargs=-1, required=True)
-@click.argument(
+def show(keyword):
-    "entries", nargs=-1, required=True, metavar="entry=value [entry=value]..."
+    """Show support status of specified coins.
-)
+
-@click.option(
+    Keywords match against key, name or shortcut (ticker symbol) of coin.
-    "-n",
+    """
-    "--dry-run",
+    defs = coin_info.get_all(deduplicate=False).as_list()
-    is_flag=True,
+    coin_info.mark_duplicate_shortcuts(defs)
-    help="Only print updated support info, do not write back",
+
-)
+    for coin in defs:
-def set(support_key, entries, dry_run):
+        key = coin["key"].lower()
+        name = coin["name"].lower()
+        shortcut = coin["shortcut"].lower()
+        symsplit = shortcut.split(" ", maxsplit=1)
+        symbol = symsplit[0]
+        suffix = symsplit[1] if len(symsplit) > 1 else ""
+        for kw in keyword:
+            kwl = kw.lower()
+            if (
+                kwl == key
+                or kwl in name
+                or kwl == shortcut
+                or kwl == symbol
+                or kwl in suffix
+            ):
+                print_support(coin)
+
+
+@cli.command(name="set")
+# fmt: off
+@click.argument("key", required=True)
+@click.argument("entries", nargs=-1, required=True, metavar="entry=value [entry=value]...")
+@click.option("-r", "--reason", help="Reason for not supporting")
+# fmt: on
+def set_support_value(key, entries, reason):
     """Set a support info variable.
 
     Examples:
-    support.py coin:BTC trezor1=soon trezor2=2.0.7 webwallet=yes connect=no
+    support.py set coin:BTC trezor1=soon trezor2=2.0.7 webwallet=yes connect=no
-    support.py coin:LTC trezor1=yes "Electrum-LTC=https://electrum-ltc.org" Electrum=
+    support.py set coin:LTC trezor1=yes connect=
 
     Setting a variable to "yes", "true" or "1" sets support to true.
     Setting a variable to "no", "false" or "0" sets support to false.
@@ -138,33 +294,51 @@ def set(support_key, entries, dry_run):
     Entries with other names will be inserted into "others". This is a good place
     to store links to 3rd party software, such as Electrum forks or claim tools.
     """
-    coins = coin_info.get_all().as_dict()
+    coins = coin_info.get_all(deduplicate=False).as_dict()
-    if support_key not in coins:
+    coin_info.mark_duplicate_shortcuts(coins.values())
-        click.echo("Failed to find key {}".format(support_key))
+    if key not in coins:
+        click.echo(f"Failed to find key {key}")
         click.echo("Use 'support.py show' to search for the right one.")
         sys.exit(1)
 
-    print("{} - {}".format(support_key, coins[support_key]["name"]))
+    if coins[key].get("duplicate"):
+        shortcut = coins[key]["shortcut"]
+        click.echo(f"Note: shortcut {shortcut} is a duplicate.")
+        click.echo(f"Coin will NOT be listed regardless of support.json status.")
 
     for entry in entries:
         try:
-            key, value = entry.split("=", maxsplit=1)
+            device, value = entry.split("=", maxsplit=1)
         except ValueError:
-            click.echo("Invalid entry: {}".format(entry))
+            click.echo(f"Invalid entry: {entry}")
             sys.exit(2)
 
+        if device not in SUPPORT_INFO:
+            raise click.ClickException(f"unknown device: {device}")
+
+        where = SUPPORT_INFO[device]
+        # clear existing info
+        where["supported"].pop(key, None)
+        where["unsupported"].pop(key, None)
+
         if value in ("yes", "true", "1"):
-            value = True
+            where["supported"][key] = True
-        elif value in ("no", "false", "2"):
+        elif value in ("no", "false", "0"):
-            value = False
+            if device in MISSING_MEANS_NO:
+                click.echo("Setting explicitly unsupported for {device}.")
+                click.echo("Perhaps you meant removing support, i.e., '{device}=' ?")
+            if not reason:
+                reason = click.prompt(f"Enter reason for not supporting on {device}:")
+            where["unsupported"][key] = reason
         elif value == "":
-            value = None
+            # do nothing, existing info is cleared
+            pass
+        else:
+            # arbitrary string?
+            where["supported"][key] = value
 
-        support = update_support(support_key, key, value)
+    print_support(coins[key])
-
+    write_support_info()
-    print(json.dumps(support, indent=4))
-    if not dry_run:
-        write_support_info()
 
 
 if __name__ == "__main__":