tools: coin_defs renamed to coin_info and interface

improved a little

tools/coin_gen.py

@@ -8,7 +8,7 @@ import glob
 
 import click
 
-import coin_defs
+import coin_info
 
 try:
     import mako
@@ -25,10 +25,13 @@ except ImportError:
     requests = None
 
 try:
+    import binascii
+    import struct
+    import zlib
     from hashlib import sha256
     import ed25519
     from PIL import Image
-    from trezorlib.protobuf import dump_message
+    from trezorlib import protobuf
     from coindef import CoinDef
 
     CAN_BUILD_DEFS = True
@@ -36,7 +39,7 @@ except ImportError:
     CAN_BUILD_DEFS = False
 
 
-# ======= Jinja2 management ======
+# ======= Mako management ======
 
 
 def c_str_filter(b):
@@ -70,26 +73,36 @@ def render_file(filename, coins, support_info):
 # ====== validation functions ======
 
 
-def check_support(defs, support_data):
+def check_support(defs, support_data, fail_missing=False):
     check_passed = True
+    coin_list = defs.as_list()
+    coin_names = {coin["key"]: coin["name"] for coin in coin_list}
+
+    def coin_name(key):
+        if key in coin_names:
+            return "{} ({})".format(key, coin_names[key])
+        else:
+            return "{} <unknown key>".format(key)
 
     for key, support in support_data.items():
-        errors = coin_defs.validate_support(support)
+        errors = coin_info.validate_support(support)
         if errors:
             check_passed = False
-            print("ERR:", "invalid definition for", key)
+            print("ERR:", "invalid definition for", coin_name(key))
             print("\n".join(errors))
 
-    expected_coins = set(coin["key"] for coin in defs["coins"] + defs["misc"])
+    expected_coins = set(coin["key"] for coin in defs.coins + defs.misc)
 
     # detect missing support info for expected
     for coin in expected_coins:
         if coin not in support_data:
-            check_passed = False
+            if fail_missing:
-            print("ERR: Missing support info for", coin)
+                check_passed = False
+                print("ERR: Missing support info for", coin_name(coin))
+            else:
+                print("WARN: Missing support info for", coin_name(coin))
 
     # detect non-matching support info
-    coin_list = sum(defs.values(), [])
     coin_set = set(coin["key"] for coin in coin_list)
     for key in support_data:
         # detect non-matching support info
@@ -97,9 +110,9 @@ def check_support(defs, support_data):
             check_passed = False
             print("ERR: Support info found for unknown coin", key)
 
-        # detect override - info only, doesn't fail check
+        # detect override - doesn't fail check
         if key not in expected_coins:
-            print("INFO: Override present for coin", key)
+            print("INFO: Override present for coin", coin_name(key))
 
     return check_passed
 
@@ -108,13 +121,13 @@ def check_btc(coins):
     check_passed = True
 
     for coin in coins:
-        errors = coin_defs.validate_btc(coin)
+        errors = coin_info.validate_btc(coin)
         if errors:
             check_passed = False
             print("ERR:", "invalid definition for", coin["name"])
             print("\n".join(errors))
 
-    collisions = coin_defs.find_address_collisions(coins)
+    collisions = coin_info.find_address_collisions(coins)
     # warning only
     for key, dups in collisions.items():
         if dups:
@@ -135,7 +148,7 @@ def check_backends(coins):
         for backend in backends:
             print("checking", backend, "... ", end="", flush=True)
             try:
-                j = requests.get(backend + "/block-index/0").json()
+                j = requests.get(backend + "/api/block-index/0").json()
                 if j["blockHash"] != genesis_block:
                     raise RuntimeError("genesis block mismatch")
             except Exception as e:
@@ -146,6 +159,59 @@ def check_backends(coins):
     return check_passed
 
 
+# ====== coindefs generators ======
+
+
+def convert_icon(icon):
+    """Convert PIL icon to TOIF format"""
+    # TODO: move this to python-trezor at some point
+    DIM = 32
+    icon = icon.resize((DIM, DIM), Image.LANCZOS)
+    # remove alpha channel, replace with black
+    bg = Image.new("RGBA", icon.size, (0, 0, 0, 255))
+    icon = Image.alpha_composite(bg, icon)
+    # process pixels
+    pix = icon.load()
+    data = bytes()
+    for y in range(DIM):
+        for x in range(DIM):
+            r, g, b, _ = pix[x, y]
+            c = ((r & 0xF8) << 8) | ((g & 0xFC) << 3) | ((b & 0xF8) >> 3)
+            data += struct.pack(">H", c)
+    z = zlib.compressobj(level=9, wbits=10)
+    zdata = z.compress(data) + z.flush()
+    zdata = zdata[2:-4]  # strip header and checksum
+    return zdata
+
+
+def coindef_from_dict(coin):
+    proto = CoinDef()
+    for fname, _, fflags in CoinDef.FIELDS.values():
+        val = coin.get(fname)
+        if val is None and fflags & protobuf.FLAG_REPEATED:
+            val = []
+        elif fname == "signed_message_header":
+            val = val.encode("utf-8")
+        elif fname == "hash_genesis_block":
+            val = binascii.unhexlify(val)
+        setattr(proto, fname, val)
+
+    return proto
+
+
+def serialize_coindef(proto, icon):
+    proto.icon = icon
+    buf = io.BytesIO()
+    protobuf.dump_message(buf, proto)
+    return buf.getvalue()
+
+
+def sign(data):
+    h = sha256(data).digest()
+    sign_key = ed25519.SigningKey(b"A" * 32)
+    return sign_key.sign(h)
+
+
 # ====== click command handlers ======
 
 
@@ -155,12 +221,17 @@ def cli():
 
 
 @cli.command()
+@click.option(
+    "--check-missing-support/--no-check-missing-support",
+    "-s",
+    help="Fail if support info for a coin is missing",
+)
 @click.option(
     "--backend-check/--no-backend-check",
     "-b",
     help="Also check blockbook/bitcore responses",
 )
-def check(backend_check):
+def check(check_missing_support, backend_check):
     """Validate coin definitions.
 
     Checks that every btc-like coin is properly filled out, reports address collisions
@@ -169,20 +240,21 @@ def check(backend_check):
     if backend_check and requests is None:
         raise click.ClickException("You must install requests for backend check")
 
-    defs = coin_defs.get_all()
+    defs = coin_info.get_all()
     all_checks_passed = True
 
     print("Checking BTC-like coins...")
-    if not check_btc(defs["coins"]):
+    if not check_btc(defs.coins):
         all_checks_passed = False
 
     print("Checking support data...")
-    if not check_support(defs, coin_defs.get_support_data()):
+    support_data = coin_info.get_support_data()
+    if not check_support(defs, support_data, fail_missing=check_missing_support):
         all_checks_passed = False
 
     if backend_check:
         print("Checking backend responses...")
-        if not check_backends(defs["coins"]):
+        if not check_backends(defs.coins):
             all_checks_passed = False
 
     if not all_checks_passed:
@@ -196,9 +268,8 @@ def check(backend_check):
 @click.option("-o", "--outfile", type=click.File(mode="w"), default="./coins.json")
 def coins_json(outfile):
     """Generate coins.json for consumption in python-trezor and Connect/Wallet"""
-    defs = coin_defs.get_all()
+    coins = coin_info.get_all().coins
-    coins = defs["coins"]
+    support_info = coin_info.support_info(coins)
-    support_info = coin_defs.support_info(coins)
     by_name = {}
     for coin in coins:
         coin["support"] = support_info[coin["key"]]
@@ -206,6 +277,30 @@ def coins_json(outfile):
 
     with outfile:
         json.dump(by_name, outfile, indent=4, sort_keys=True)
+        outfile.write("\n")
+
+
+@cli.command()
+@click.option("-o", "--outfile", type=click.File(mode="w"), default="./coindefs.json")
+def coindefs(outfile):
+    """Generate signed coin definitions for python-trezor and others
+
+    This is currently unused but should enable us to add new coins without having to
+    update firmware.
+    """
+    coins = coin_info.get_all().coins
+    coindefs = {}
+    for coin in coins:
+        key = coin["key"]
+        icon = Image.open(coin["icon"])
+        ser = serialize_coindef(coindef_from_dict(coin), convert_icon(icon))
+        sig = sign(ser)
+        definition = binascii.hexlify(sig + ser).decode("ascii")
+        coindefs[key] = definition
+
+    with outfile:
+        json.dump(coindefs, outfile, indent=4, sort_keys=True)
+        outfile.write("\n")
 
 
 @cli.command()
@@ -236,10 +331,9 @@ def render(paths):
         else:
             files.append(path)
 
-    defs = coin_defs.get_all()
+    defs = coin_info.get_all()
-    all_coins = sum(defs.values(), [])
+    versions = coin_info.latest_releases()
-    versions = coin_defs.latest_releases()
+    support_info = coin_info.support_info(defs, erc20_versions=versions)
-    support_info = coin_defs.support_info(all_coins, erc20_versions=versions)
 
     # munch dicts - make them attribute-accessable
     for key, value in defs.items():

tools/coin_info.py

@@ -30,6 +30,35 @@ def load_json(*path):
         return json.load(f, object_pairs_hook=OrderedDict)
 
 
+# ====== CoinsInfo ======
+
+
+class CoinsInfo(dict):
+    """Collection of information about all known kinds of coins.
+
+    It contains the following lists:
+    `coins` for btc-like coins,
+    `eth` for ethereum networks,
+    `erc20` for ERC20 tokens,
+    `nem` for NEM mosaics,
+    `misc` for other networks.
+
+    Accessible as a dict or by attribute: `info["coins"] == info.coins`
+    """
+
+    def as_list(self):
+        return sum(self.values(), [])
+
+    def as_dict(self):
+        return {coin["key"]: coin for coin in self.as_list()}
+
+    def __getattr__(self, attr):
+        if attr in self:
+            return self[attr]
+        else:
+            raise AttributeError(attr)
+
+
 # ====== coin validation ======
 
 
@@ -186,6 +215,7 @@ def _load_btc_coins():
             name=coin["coin_name"],
             shortcut=coin["coin_shortcut"],
             key="coin:{}".format(coin["coin_shortcut"]),
+            icon=filename.replace(".json", ".png"),
         )
         coins.append(coin)
 
@@ -324,11 +354,14 @@ def support_info_erc20(coins, versions):
 def support_info(coins, erc20_versions=None, skip_missing=False):
     """Generate Trezor support information.
 
-    Takes a dict of coins and generates a support-info entry for each.
+    Takes a collection of coins and generates a support-info entry for each.
     The support-info is a dict with a number of known keys:
     `trezor1`, `trezor2`, `webwallet`, `connect`. An optional `other` entry
     is a dict of name-url pairs for third-party software.
 
+    The `coins` argument can be a `CoinsInfo` object, a list or a dict of
+    coin items.
+
     For btc-like coins and misc networks, this is taken from `support.json`.
     For NEM mosaics and ethereum networks, the support is presumed to be "yes"
     for both Trezors. Webwallet and Connect info is not filled out.
@@ -346,6 +379,11 @@ def support_info(coins, erc20_versions=None, skip_missing=False):
     and a warning emitted. "No support information" means that the coin is not
     listed in `support.json` and we have no heuristic to determine the support.
     """
+    if isinstance(coins, CoinsInfo):
+        coins = coins.as_list()
+    elif isinstance(coins, dict):
+        coins = coins.values()
+
     support_data = get_support_data()
     support = {}
     for coin in coins:
@@ -428,6 +466,10 @@ def _filter_duplicate_shortcuts(coins):
     retained_coins = OrderedDict()
 
     for coin in coins:
+        if "Testnet" in coin["name"] and coin["shortcut"] == "tETH":
+            # special case for Ethereum testnets
+            continue
+
         key = coin["shortcut"]
         if key in dup_keys:
             pass
@@ -458,7 +500,7 @@ def get_all():
     `nem` for NEM mosaics,
     `misc` for other networks.
     """
-    all_coins = dict(
+    all_coins = CoinsInfo(
         coins=_load_btc_coins(),
         eth=_load_ethereum_networks(),
         erc20=_load_erc20_tokens(),
@@ -476,22 +518,13 @@ def get_all():
             coins.sort(key=lambda c: c["key"].upper())
 
         _ensure_mandatory_values(coins)
-        if k != "eth":
+        dup_keys = _filter_duplicate_shortcuts(coins)
-            dup_keys = _filter_duplicate_shortcuts(coins)
+        if dup_keys:
-            if dup_keys:
+            if k == "erc20":
-                log.warning(
+                severity = logging.INFO
-                    "{}: removing duplicate symbols: {}".format(k, ", ".join(dup_keys))
+            else:
-                )
+                severity = logging.WARNING
+            dup_str = ", ".join(dup_keys)
+            log.log(severity, "{}: removing duplicate symbols: {}".format(k, dup_str))
 
     return all_coins
-
-
-def get_list():
-    """Return all definitions as a single list of coins."""
-    all_coins = get_all()
-    return sum(all_coins.values(), [])
-
-
-def get_dict():
-    """Return all definitions as a dict indexed by coin keys."""
-    return {coin["key"]: coin for coin in get_list()}

tools/coins_details.py

@@ -5,15 +5,15 @@ import time
 import json
 import logging
 import requests
-import coin_defs
+import coin_info
 
 LOG = logging.getLogger(__name__)
 
 OPTIONAL_KEYS = ("links", "notes", "wallet")
 ALLOWED_SUPPORT_STATUS = ("yes", "no", "planned", "soon")
 
-OVERRIDES = coin_defs.load_json("coins_details.override.json")
+OVERRIDES = coin_info.load_json("coins_details.override.json")
-VERSIONS = coin_defs.latest_releases()
+VERSIONS = coin_info.latest_releases()
 
 COINMAKETCAP_CACHE = os.path.join(os.path.dirname(__file__), "coinmarketcap.json")
 
@@ -292,16 +292,15 @@ def apply_overrides(coins):
 
 
 if __name__ == "__main__":
-    defs = coin_defs.get_all()
+    defs = coin_info.get_all()
-    all_coins = sum(defs.values(), [])
+    support_info = coin_info.support_info(defs, erc20_versions=VERSIONS)
-    support_info = coin_defs.support_info(all_coins, erc20_versions=VERSIONS)
 
     coins = {}
-    coins.update(update_coins(defs["coins"], support_info))
+    coins.update(update_coins(defs.coins, support_info))
-    coins.update(update_erc20(defs["erc20"], support_info))
+    coins.update(update_erc20(defs.erc20, support_info))
-    coins.update(update_ethereum_networks(defs["eth"], support_info))
+    coins.update(update_ethereum_networks(defs.eth, support_info))
-    coins.update(update_simple(defs["nem"], support_info, "mosaic"))
+    coins.update(update_simple(defs.nem, support_info, "mosaic"))
-    coins.update(update_simple(defs["misc"], support_info, "coin"))
+    coins.update(update_simple(defs.misc, support_info, "coin"))
 
     apply_overrides(coins)
     update_marketcaps(coins)
@@ -311,5 +310,5 @@ if __name__ == "__main__":
     details = dict(coins=coins, info=info)
 
     print(json.dumps(info, sort_keys=True, indent=4))
-    with open(os.path.join(coin_defs.DEFS_DIR, "coins_details.json"), "w") as f:
+    with open(os.path.join(coin_info.DEFS_DIR, "coins_details.json"), "w") as f:
         json.dump(details, f, sort_keys=True, indent=4)

tools/support.py

@@ -2,10 +2,10 @@
 import os
 import sys
 import click
-import coin_defs
+import coin_info
 import json
 
-SUPPORT_INFO = coin_defs.get_support_data()
+SUPPORT_INFO = coin_info.get_support_data()
 
 MANDATORY_ENTRIES = ("trezor1", "trezor2", "connect", "webwallet")
 
@@ -36,7 +36,7 @@ def update_support(key, entry, value):
 
 
 def write_support_info():
-    with open(os.path.join(coin_defs.DEFS_DIR, "support.json"), "w") as f:
+    with open(os.path.join(coin_info.DEFS_DIR, "support.json"), "w") as f:
         json.dump(SUPPORT_INFO, f, indent=4)
         f.write("\n")
 
@@ -69,11 +69,11 @@ def check():
     here for convenience and because it makes sense. But it's preferable to run it
     as part of 'coin_gen.py check'.
     """
-    defs = coin_defs.get_all()
+    defs = coin_info.get_all()
-    support_data = coin_defs.get_support_data()
+    support_data = coin_info.get_support_data()
     import coin_gen
 
-    if not coin_gen.check_support(defs, support_data):
+    if not coin_gen.check_support(defs, support_data, fail_missing=True):
         sys.exit(1)
 
 
@@ -88,7 +88,7 @@ def show(keyword):
     Only coins listed in support.json are considered "supported". That means that
     Ethereum networks, ERC20 tokens and NEM mosaics will probably show up wrong.
     """
-    defs = coin_defs.get_list()
+    defs = coin_info.get_all().as_list()
 
     if keyword:
         for coin in defs:
@@ -138,7 +138,7 @@ def set(support_key, entries, dry_run):
     Entries with other names will be inserted into "others". This is a good place
     to store links to 3rd party software, such as Electrum forks or claim tools.
     """
-    coins = coin_defs.get_dict()
+    coins = coin_info.get_all().as_dict()
     if support_key not in coins:
         click.echo("Failed to find key {}".format(support_key))
         click.echo("Use 'support.py show' to search for the right one.")