tools: add support tool and coin_gen tool

support.py - query and modify info in support.json
coin_gen.py - generate coins_json, render mako templates, run checks

tools/coin_gen.py

@@ -0,0 +1,264 @@
+#!/usr/bin/env python3
+import io
+import json
+import re
+import sys
+import os
+import glob
+
+import click
+
+import coin_defs
+
+try:
+    import mako
+    import mako.template
+    from munch import Munch
+
+    CAN_RENDER = True
+except ImportError:
+    CAN_RENDER = False
+
+try:
+    import requests
+except ImportError:
+    requests = None
+
+try:
+    from hashlib import sha256
+    import ed25519
+    from PIL import Image
+    from trezorlib.protobuf import dump_message
+    from coindef import CoinDef
+
+    CAN_BUILD_DEFS = True
+except ImportError:
+    CAN_BUILD_DEFS = False
+
+
+# ======= Jinja2 management ======
+
+
+def c_str_filter(b):
+    if b is None:
+        return "NULL"
+
+    def hexescape(c):
+        return r"\x{:02x}".format(c)
+
+    if isinstance(b, bytes):
+        return '"' + "".join(map(hexescape, b)) + '"'
+    else:
+        return json.dumps(b)
+
+
+def ascii_filter(s):
+    return re.sub("[^ -\x7e]", "_", s)
+
+
+MAKO_FILTERS = {"c_str": c_str_filter, "ascii": ascii_filter}
+
+
+def render_file(filename, coins, support_info):
+    """Opens `filename.j2`, renders the template and stores the result in `filename`."""
+    template = mako.template.Template(filename=filename + ".mako")
+    result = template.render(support_info=support_info, **coins, **MAKO_FILTERS)
+    with open(filename, "w") as f:
+        f.write(result)
+
+
+# ====== validation functions ======
+
+
+def check_support(defs, support_data):
+    check_passed = True
+
+    for key, support in support_data.items():
+        errors = coin_defs.validate_support(support)
+        if errors:
+            check_passed = False
+            print("ERR:", "invalid definition for", key)
+            print("\n".join(errors))
+
+    expected_coins = set(coin["key"] for coin in defs["coins"] + defs["misc"])
+
+    # detect missing support info for expected
+    for coin in expected_coins:
+        if coin not in support_data:
+            check_passed = False
+            print("ERR: Missing support info for", coin)
+
+    # detect non-matching support info
+    coin_list = sum(defs.values(), [])
+    coin_set = set(coin["key"] for coin in coin_list)
+    for key in support_data:
+        # detect non-matching support info
+        if key not in coin_set:
+            check_passed = False
+            print("ERR: Support info found for unknown coin", key)
+
+        # detect override - info only, doesn't fail check
+        if key not in expected_coins:
+            print("INFO: Override present for coin", key)
+
+    return check_passed
+
+
+def check_btc(coins):
+    check_passed = True
+
+    for coin in coins:
+        errors = coin_defs.validate_btc(coin)
+        if errors:
+            check_passed = False
+            print("ERR:", "invalid definition for", coin["name"])
+            print("\n".join(errors))
+
+    collisions = coin_defs.find_address_collisions(coins)
+    # warning only
+    for key, dups in collisions.items():
+        if dups:
+            print("WARN: collisions found in", key)
+            for k, v in dups.items():
+                print("-", k, ":", ", ".join(map(str, v)))
+
+    return check_passed
+
+
+def check_backends(coins):
+    check_passed = True
+    for coin in coins:
+        genesis_block = coin.get("hash_genesis_block")
+        if not genesis_block:
+            continue
+        backends = coin.get("blockbook", []) + coin.get("bitcore", [])
+        for backend in backends:
+            print("checking", backend, "... ", end="", flush=True)
+            try:
+                j = requests.get(backend + "/block-index/0").json()
+                if j["blockHash"] != genesis_block:
+                    raise RuntimeError("genesis block mismatch")
+            except Exception as e:
+                print(e)
+                check_passed = False
+            else:
+                print("OK")
+    return check_passed
+
+
+# ====== click command handlers ======
+
+
+@click.group()
+def cli():
+    pass
+
+
+@cli.command()
+@click.option(
+    "--backend-check/--no-backend-check",
+    "-b",
+    help="Also check blockbook/bitcore responses",
+)
+def check(backend_check):
+    """Validate coin definitions.
+
+    Checks that every btc-like coin is properly filled out, reports address collisions
+    and missing support information.
+    """
+    if backend_check and requests is None:
+        raise click.ClickException("You must install requests for backend check")
+
+    defs = coin_defs.get_all()
+    all_checks_passed = True
+
+    print("Checking BTC-like coins...")
+    if not check_btc(defs["coins"]):
+        all_checks_passed = False
+
+    print("Checking support data...")
+    if not check_support(defs, coin_defs.get_support_data()):
+        all_checks_passed = False
+
+    if backend_check:
+        print("Checking backend responses...")
+        if not check_backends(defs["coins"]):
+            all_checks_passed = False
+
+    if not all_checks_passed:
+        print("Some checks failed.")
+        sys.exit(1)
+    else:
+        print("Everything is OK.")
+
+
+@cli.command()
+@click.option("-o", "--outfile", type=click.File(mode="w"), default="./coins.json")
+def coins_json(outfile):
+    """Generate coins.json for consumption in python-trezor and Connect/Wallet"""
+    defs = coin_defs.get_all()
+    coins = defs["coins"]
+    support_info = coin_defs.support_info(coins)
+    by_name = {}
+    for coin in coins:
+        coin["support"] = support_info[coin["key"]]
+        by_name[coin["name"]] = coin
+
+    with outfile:
+        json.dump(by_name, outfile, indent=4, sort_keys=True)
+
+
+@cli.command()
+@click.argument("paths", metavar="[path]...", nargs=-1)
+def render(paths):
+    """Generate source code from Jinja2 templates.
+
+    For every "foo.bar.j2" filename passed, runs the template and
+    saves the result as "foo.bar".
+
+    For every directory name passed, processes all ".j2" files found
+    in that directory.
+
+    If no arguments are given, processes the current directory.
+    """
+    if not CAN_RENDER:
+        raise click.ClickException("Please install 'mako' and 'munch'")
+
+    if not paths:
+        paths = ["."]
+
+    files = []
+    for path in paths:
+        if not os.path.exists(path):
+            click.echo("Path {} does not exist".format(path))
+        elif os.path.isdir(path):
+            files += glob.glob(os.path.join(path, "*.mako"))
+        else:
+            files.append(path)
+
+    defs = coin_defs.get_all()
+    all_coins = sum(defs.values(), [])
+    versions = coin_defs.latest_releases()
+    support_info = coin_defs.support_info(all_coins, erc20_versions=versions)
+
+    # munch dicts - make them attribute-accessable
+    for key, value in defs.items():
+        defs[key] = [Munch(coin) for coin in value]
+    for key, value in support_info.items():
+        support_info[key] = Munch(value)
+
+    for file in files:
+        if not file.endswith(".mako"):
+            click.echo("File {} does not end with .mako".format(file))
+        else:
+            target = file[: -len(".mako")]
+            click.echo("Rendering {} => {}".format(file, target))
+            try:
+                render_file(target, defs, support_info)
+            except Exception as e:
+                click.echo("Error occured: {}".format(e))
+                raise
+
+
+if __name__ == "__main__":
+    cli()

tools/support.py

@@ -0,0 +1,171 @@
+#!/usr/bin/env python3
+import os
+import sys
+import click
+import coin_defs
+import json
+
+SUPPORT_INFO = coin_defs.get_support_data()
+
+MANDATORY_ENTRIES = ("trezor1", "trezor2", "connect", "webwallet")
+
+
+def update_support(key, entry, value):
+    # template entry
+    support = {k: None for k in MANDATORY_ENTRIES}
+    support["other"] = {}
+    # fill out actual support info, if it exists
+    support.update(SUPPORT_INFO.get(key, {}))
+
+    if entry in MANDATORY_ENTRIES:
+        if entry.startswith("trezor") and not value:
+            value = None
+        support[entry] = value
+    else:
+        support["other"][entry] = value
+
+    for k in support["other"]:
+        if not support["other"][k]:
+            del support["other"][k]
+
+    if not support["other"]:
+        del support["other"]
+
+    SUPPORT_INFO[key] = support
+    return support
+
+
+def write_support_info():
+    with open(os.path.join(coin_defs.DEFS_DIR, "support.json"), "w") as f:
+        json.dump(SUPPORT_INFO, f, indent=4)
+        f.write("\n")
+
+
+@click.group()
+def cli():
+    pass
+
+
+@cli.command()
+def rewrite():
+    """Regenerate support.json to match predefined structure and field order."""
+    for key, coin in SUPPORT_INFO.items():
+        d = {"trezor1": None, "trezor2": None, "connect": None, "webwallet": None}
+        d.update(coin)
+        if "electrum" in d:
+            del d["electrum"]
+        if "other" in d and not d["other"]:
+            del d["other"]
+        SUPPORT_INFO[key] = d
+
+    write_support_info()
+
+
+@cli.command()
+def check():
+    """Check validity of support information.
+
+    The relevant code is actually part of 'coin_gen.py'. It can be invoked from
+    here for convenience and because it makes sense. But it's preferable to run it
+    as part of 'coin_gen.py check'.
+    """
+    defs = coin_defs.get_all()
+    support_data = coin_defs.get_support_data()
+    import coin_gen
+
+    if not coin_gen.check_support(defs, support_data):
+        sys.exit(1)
+
+
+@cli.command()
+@click.argument("keyword", nargs=-1)
+def show(keyword):
+    """Show support status of specified coins.
+
+    Keywords match against key, name or shortcut (ticker symbol) of coin. If no
+    keywords are provided, show all supported coins.
+
+    Only coins listed in support.json are considered "supported". That means that
+    Ethereum networks, ERC20 tokens and NEM mosaics will probably show up wrong.
+    """
+    defs = coin_defs.get_list()
+
+    if keyword:
+        for coin in defs:
+            key = coin["key"]
+            name, shortcut = coin["name"], coin["shortcut"]
+            for kw in keyword:
+                kwl = kw.lower()
+                if kwl == key.lower() or kwl in name.lower() or kwl == shortcut.lower():
+                    print("{} - {} ({})".format(key, name, shortcut), end=" - ")
+                    if key in SUPPORT_INFO:
+                        print(json.dumps(SUPPORT_INFO[key], indent=4))
+                    else:
+                        print("no support info")
+                    break
+
+    else:
+        print(json.dumps(SUPPORT_INFO, indent=4))
+
+
+@cli.command()
+@click.argument("support_key", required=True)
+@click.argument(
+    "entries", nargs=-1, required=True, metavar="entry=value [entry=value]..."
+)
+@click.option(
+    "-n",
+    "--dry-run",
+    is_flag=True,
+    help="Only print updated support info, do not write back",
+)
+def set(support_key, entries, dry_run):
+    """Set a support info variable.
+
+    Examples:
+    support.py coin:BTC trezor1=soon trezor2=2.0.7 webwallet=yes connect=no
+    support.py coin:LTC trezor1=yes "Electrum-LTC=https://electrum-ltc.org" Electrum=
+
+    Setting a variable to "yes", "true" or "1" sets support to true.
+    Setting a variable to "no", "false" or "0" sets support to false.
+    (or null, in case of trezor1/2)
+    Setting variable to empty ("trezor1=") will set to null, or clear the entry.
+    Setting to "soon", "planned", "2.1.1" etc. will set the literal string.
+
+    Entries that are always present:
+    trezor1 trezor2 webwallet connect
+
+    Entries with other names will be inserted into "others". This is a good place
+    to store links to 3rd party software, such as Electrum forks or claim tools.
+    """
+    coins = coin_defs.get_dict()
+    if support_key not in coins:
+        click.echo("Failed to find key {}".format(support_key))
+        click.echo("Use 'support.py show' to search for the right one.")
+        sys.exit(1)
+
+    print("{} - {}".format(support_key, coins[support_key]["name"]))
+
+    for entry in entries:
+        try:
+            key, value = entry.split("=", maxsplit=1)
+        except ValueError:
+            click.echo("Invalid entry: {}".format(entry))
+            sys.exit(2)
+
+        if value in ("yes", "true", "1"):
+            value = True
+        elif value in ("no", "false", "2"):
+            value = False
+        elif value == "":
+            value = None
+
+        support = update_support(support_key, key, value)
+
+    print(json.dumps(support, indent=4))
+    if not dry_run:
+        write_support_info()
+
+
+if __name__ == "__main__":
+    cli()