From b5443af4c5bea643181ecabc2a2d5d6290b67287 Mon Sep 17 00:00:00 2001 From: matejcik Date: Mon, 30 Jul 2018 14:25:53 +0200 Subject: [PATCH] tools: coin_defs renamed to coin_info and interface improved a little --- tools/coin_gen.py | 148 ++++++++++++++++++++++----- tools/{coin_defs.py => coin_info.py} | 71 +++++++++---- tools/coins_details.py | 23 ++--- tools/support.py | 16 +-- 4 files changed, 192 insertions(+), 66 deletions(-) rename tools/{coin_defs.py => coin_info.py} (90%) diff --git a/tools/coin_gen.py b/tools/coin_gen.py index 04f775399a..771b0c33f7 100755 --- a/tools/coin_gen.py +++ b/tools/coin_gen.py @@ -8,7 +8,7 @@ import glob import click -import coin_defs +import coin_info try: import mako @@ -25,10 +25,13 @@ except ImportError: requests = None try: + import binascii + import struct + import zlib from hashlib import sha256 import ed25519 from PIL import Image - from trezorlib.protobuf import dump_message + from trezorlib import protobuf from coindef import CoinDef CAN_BUILD_DEFS = True @@ -36,7 +39,7 @@ except ImportError: CAN_BUILD_DEFS = False -# ======= Jinja2 management ====== +# ======= Mako management ====== def c_str_filter(b): @@ -70,26 +73,36 @@ def render_file(filename, coins, support_info): # ====== validation functions ====== -def check_support(defs, support_data): +def check_support(defs, support_data, fail_missing=False): check_passed = True + coin_list = defs.as_list() + coin_names = {coin["key"]: coin["name"] for coin in coin_list} + + def coin_name(key): + if key in coin_names: + return "{} ({})".format(key, coin_names[key]) + else: + return "{} ".format(key) for key, support in support_data.items(): - errors = coin_defs.validate_support(support) + errors = coin_info.validate_support(support) if errors: check_passed = False - print("ERR:", "invalid definition for", key) + print("ERR:", "invalid definition for", coin_name(key)) print("\n".join(errors)) - expected_coins = set(coin["key"] for coin in defs["coins"] + defs["misc"]) + expected_coins = set(coin["key"] for coin in defs.coins + defs.misc) # detect missing support info for expected for coin in expected_coins: if coin not in support_data: - check_passed = False - print("ERR: Missing support info for", coin) + if fail_missing: + check_passed = False + print("ERR: Missing support info for", coin_name(coin)) + else: + print("WARN: Missing support info for", coin_name(coin)) # detect non-matching support info - coin_list = sum(defs.values(), []) coin_set = set(coin["key"] for coin in coin_list) for key in support_data: # detect non-matching support info @@ -97,9 +110,9 @@ def check_support(defs, support_data): check_passed = False print("ERR: Support info found for unknown coin", key) - # detect override - info only, doesn't fail check + # detect override - doesn't fail check if key not in expected_coins: - print("INFO: Override present for coin", key) + print("INFO: Override present for coin", coin_name(key)) return check_passed @@ -108,13 +121,13 @@ def check_btc(coins): check_passed = True for coin in coins: - errors = coin_defs.validate_btc(coin) + errors = coin_info.validate_btc(coin) if errors: check_passed = False print("ERR:", "invalid definition for", coin["name"]) print("\n".join(errors)) - collisions = coin_defs.find_address_collisions(coins) + collisions = coin_info.find_address_collisions(coins) # warning only for key, dups in collisions.items(): if dups: @@ -135,7 +148,7 @@ def check_backends(coins): for backend in backends: print("checking", backend, "... ", end="", flush=True) try: - j = requests.get(backend + "/block-index/0").json() + j = requests.get(backend + "/api/block-index/0").json() if j["blockHash"] != genesis_block: raise RuntimeError("genesis block mismatch") except Exception as e: @@ -146,6 +159,59 @@ def check_backends(coins): return check_passed +# ====== coindefs generators ====== + + +def convert_icon(icon): + """Convert PIL icon to TOIF format""" + # TODO: move this to python-trezor at some point + DIM = 32 + icon = icon.resize((DIM, DIM), Image.LANCZOS) + # remove alpha channel, replace with black + bg = Image.new("RGBA", icon.size, (0, 0, 0, 255)) + icon = Image.alpha_composite(bg, icon) + # process pixels + pix = icon.load() + data = bytes() + for y in range(DIM): + for x in range(DIM): + r, g, b, _ = pix[x, y] + c = ((r & 0xF8) << 8) | ((g & 0xFC) << 3) | ((b & 0xF8) >> 3) + data += struct.pack(">H", c) + z = zlib.compressobj(level=9, wbits=10) + zdata = z.compress(data) + z.flush() + zdata = zdata[2:-4] # strip header and checksum + return zdata + + +def coindef_from_dict(coin): + proto = CoinDef() + for fname, _, fflags in CoinDef.FIELDS.values(): + val = coin.get(fname) + if val is None and fflags & protobuf.FLAG_REPEATED: + val = [] + elif fname == "signed_message_header": + val = val.encode("utf-8") + elif fname == "hash_genesis_block": + val = binascii.unhexlify(val) + setattr(proto, fname, val) + + return proto + + +def serialize_coindef(proto, icon): + proto.icon = icon + buf = io.BytesIO() + protobuf.dump_message(buf, proto) + return buf.getvalue() + + +def sign(data): + h = sha256(data).digest() + sign_key = ed25519.SigningKey(b"A" * 32) + return sign_key.sign(h) + + # ====== click command handlers ====== @@ -155,12 +221,17 @@ def cli(): @cli.command() +@click.option( + "--check-missing-support/--no-check-missing-support", + "-s", + help="Fail if support info for a coin is missing", +) @click.option( "--backend-check/--no-backend-check", "-b", help="Also check blockbook/bitcore responses", ) -def check(backend_check): +def check(check_missing_support, backend_check): """Validate coin definitions. Checks that every btc-like coin is properly filled out, reports address collisions @@ -169,20 +240,21 @@ def check(backend_check): if backend_check and requests is None: raise click.ClickException("You must install requests for backend check") - defs = coin_defs.get_all() + defs = coin_info.get_all() all_checks_passed = True print("Checking BTC-like coins...") - if not check_btc(defs["coins"]): + if not check_btc(defs.coins): all_checks_passed = False print("Checking support data...") - if not check_support(defs, coin_defs.get_support_data()): + support_data = coin_info.get_support_data() + if not check_support(defs, support_data, fail_missing=check_missing_support): all_checks_passed = False if backend_check: print("Checking backend responses...") - if not check_backends(defs["coins"]): + if not check_backends(defs.coins): all_checks_passed = False if not all_checks_passed: @@ -196,9 +268,8 @@ def check(backend_check): @click.option("-o", "--outfile", type=click.File(mode="w"), default="./coins.json") def coins_json(outfile): """Generate coins.json for consumption in python-trezor and Connect/Wallet""" - defs = coin_defs.get_all() - coins = defs["coins"] - support_info = coin_defs.support_info(coins) + coins = coin_info.get_all().coins + support_info = coin_info.support_info(coins) by_name = {} for coin in coins: coin["support"] = support_info[coin["key"]] @@ -206,6 +277,30 @@ def coins_json(outfile): with outfile: json.dump(by_name, outfile, indent=4, sort_keys=True) + outfile.write("\n") + + +@cli.command() +@click.option("-o", "--outfile", type=click.File(mode="w"), default="./coindefs.json") +def coindefs(outfile): + """Generate signed coin definitions for python-trezor and others + + This is currently unused but should enable us to add new coins without having to + update firmware. + """ + coins = coin_info.get_all().coins + coindefs = {} + for coin in coins: + key = coin["key"] + icon = Image.open(coin["icon"]) + ser = serialize_coindef(coindef_from_dict(coin), convert_icon(icon)) + sig = sign(ser) + definition = binascii.hexlify(sig + ser).decode("ascii") + coindefs[key] = definition + + with outfile: + json.dump(coindefs, outfile, indent=4, sort_keys=True) + outfile.write("\n") @cli.command() @@ -236,10 +331,9 @@ def render(paths): else: files.append(path) - defs = coin_defs.get_all() - all_coins = sum(defs.values(), []) - versions = coin_defs.latest_releases() - support_info = coin_defs.support_info(all_coins, erc20_versions=versions) + defs = coin_info.get_all() + versions = coin_info.latest_releases() + support_info = coin_info.support_info(defs, erc20_versions=versions) # munch dicts - make them attribute-accessable for key, value in defs.items(): diff --git a/tools/coin_defs.py b/tools/coin_info.py similarity index 90% rename from tools/coin_defs.py rename to tools/coin_info.py index 52e7e3637b..31237f666b 100755 --- a/tools/coin_defs.py +++ b/tools/coin_info.py @@ -30,6 +30,35 @@ def load_json(*path): return json.load(f, object_pairs_hook=OrderedDict) +# ====== CoinsInfo ====== + + +class CoinsInfo(dict): + """Collection of information about all known kinds of coins. + + It contains the following lists: + `coins` for btc-like coins, + `eth` for ethereum networks, + `erc20` for ERC20 tokens, + `nem` for NEM mosaics, + `misc` for other networks. + + Accessible as a dict or by attribute: `info["coins"] == info.coins` + """ + + def as_list(self): + return sum(self.values(), []) + + def as_dict(self): + return {coin["key"]: coin for coin in self.as_list()} + + def __getattr__(self, attr): + if attr in self: + return self[attr] + else: + raise AttributeError(attr) + + # ====== coin validation ====== @@ -186,6 +215,7 @@ def _load_btc_coins(): name=coin["coin_name"], shortcut=coin["coin_shortcut"], key="coin:{}".format(coin["coin_shortcut"]), + icon=filename.replace(".json", ".png"), ) coins.append(coin) @@ -324,11 +354,14 @@ def support_info_erc20(coins, versions): def support_info(coins, erc20_versions=None, skip_missing=False): """Generate Trezor support information. - Takes a dict of coins and generates a support-info entry for each. + Takes a collection of coins and generates a support-info entry for each. The support-info is a dict with a number of known keys: `trezor1`, `trezor2`, `webwallet`, `connect`. An optional `other` entry is a dict of name-url pairs for third-party software. + The `coins` argument can be a `CoinsInfo` object, a list or a dict of + coin items. + For btc-like coins and misc networks, this is taken from `support.json`. For NEM mosaics and ethereum networks, the support is presumed to be "yes" for both Trezors. Webwallet and Connect info is not filled out. @@ -346,6 +379,11 @@ def support_info(coins, erc20_versions=None, skip_missing=False): and a warning emitted. "No support information" means that the coin is not listed in `support.json` and we have no heuristic to determine the support. """ + if isinstance(coins, CoinsInfo): + coins = coins.as_list() + elif isinstance(coins, dict): + coins = coins.values() + support_data = get_support_data() support = {} for coin in coins: @@ -428,6 +466,10 @@ def _filter_duplicate_shortcuts(coins): retained_coins = OrderedDict() for coin in coins: + if "Testnet" in coin["name"] and coin["shortcut"] == "tETH": + # special case for Ethereum testnets + continue + key = coin["shortcut"] if key in dup_keys: pass @@ -458,7 +500,7 @@ def get_all(): `nem` for NEM mosaics, `misc` for other networks. """ - all_coins = dict( + all_coins = CoinsInfo( coins=_load_btc_coins(), eth=_load_ethereum_networks(), erc20=_load_erc20_tokens(), @@ -476,22 +518,13 @@ def get_all(): coins.sort(key=lambda c: c["key"].upper()) _ensure_mandatory_values(coins) - if k != "eth": - dup_keys = _filter_duplicate_shortcuts(coins) - if dup_keys: - log.warning( - "{}: removing duplicate symbols: {}".format(k, ", ".join(dup_keys)) - ) + dup_keys = _filter_duplicate_shortcuts(coins) + if dup_keys: + if k == "erc20": + severity = logging.INFO + else: + severity = logging.WARNING + dup_str = ", ".join(dup_keys) + log.log(severity, "{}: removing duplicate symbols: {}".format(k, dup_str)) return all_coins - - -def get_list(): - """Return all definitions as a single list of coins.""" - all_coins = get_all() - return sum(all_coins.values(), []) - - -def get_dict(): - """Return all definitions as a dict indexed by coin keys.""" - return {coin["key"]: coin for coin in get_list()} diff --git a/tools/coins_details.py b/tools/coins_details.py index e12d18e556..5c4b407fb9 100755 --- a/tools/coins_details.py +++ b/tools/coins_details.py @@ -5,15 +5,15 @@ import time import json import logging import requests -import coin_defs +import coin_info LOG = logging.getLogger(__name__) OPTIONAL_KEYS = ("links", "notes", "wallet") ALLOWED_SUPPORT_STATUS = ("yes", "no", "planned", "soon") -OVERRIDES = coin_defs.load_json("coins_details.override.json") -VERSIONS = coin_defs.latest_releases() +OVERRIDES = coin_info.load_json("coins_details.override.json") +VERSIONS = coin_info.latest_releases() COINMAKETCAP_CACHE = os.path.join(os.path.dirname(__file__), "coinmarketcap.json") @@ -292,16 +292,15 @@ def apply_overrides(coins): if __name__ == "__main__": - defs = coin_defs.get_all() - all_coins = sum(defs.values(), []) - support_info = coin_defs.support_info(all_coins, erc20_versions=VERSIONS) + defs = coin_info.get_all() + support_info = coin_info.support_info(defs, erc20_versions=VERSIONS) coins = {} - coins.update(update_coins(defs["coins"], support_info)) - coins.update(update_erc20(defs["erc20"], support_info)) - coins.update(update_ethereum_networks(defs["eth"], support_info)) - coins.update(update_simple(defs["nem"], support_info, "mosaic")) - coins.update(update_simple(defs["misc"], support_info, "coin")) + coins.update(update_coins(defs.coins, support_info)) + coins.update(update_erc20(defs.erc20, support_info)) + coins.update(update_ethereum_networks(defs.eth, support_info)) + coins.update(update_simple(defs.nem, support_info, "mosaic")) + coins.update(update_simple(defs.misc, support_info, "coin")) apply_overrides(coins) update_marketcaps(coins) @@ -311,5 +310,5 @@ if __name__ == "__main__": details = dict(coins=coins, info=info) print(json.dumps(info, sort_keys=True, indent=4)) - with open(os.path.join(coin_defs.DEFS_DIR, "coins_details.json"), "w") as f: + with open(os.path.join(coin_info.DEFS_DIR, "coins_details.json"), "w") as f: json.dump(details, f, sort_keys=True, indent=4) diff --git a/tools/support.py b/tools/support.py index c9659df395..fe610baca0 100755 --- a/tools/support.py +++ b/tools/support.py @@ -2,10 +2,10 @@ import os import sys import click -import coin_defs +import coin_info import json -SUPPORT_INFO = coin_defs.get_support_data() +SUPPORT_INFO = coin_info.get_support_data() MANDATORY_ENTRIES = ("trezor1", "trezor2", "connect", "webwallet") @@ -36,7 +36,7 @@ def update_support(key, entry, value): def write_support_info(): - with open(os.path.join(coin_defs.DEFS_DIR, "support.json"), "w") as f: + with open(os.path.join(coin_info.DEFS_DIR, "support.json"), "w") as f: json.dump(SUPPORT_INFO, f, indent=4) f.write("\n") @@ -69,11 +69,11 @@ def check(): here for convenience and because it makes sense. But it's preferable to run it as part of 'coin_gen.py check'. """ - defs = coin_defs.get_all() - support_data = coin_defs.get_support_data() + defs = coin_info.get_all() + support_data = coin_info.get_support_data() import coin_gen - if not coin_gen.check_support(defs, support_data): + if not coin_gen.check_support(defs, support_data, fail_missing=True): sys.exit(1) @@ -88,7 +88,7 @@ def show(keyword): Only coins listed in support.json are considered "supported". That means that Ethereum networks, ERC20 tokens and NEM mosaics will probably show up wrong. """ - defs = coin_defs.get_list() + defs = coin_info.get_all().as_list() if keyword: for coin in defs: @@ -138,7 +138,7 @@ def set(support_key, entries, dry_run): Entries with other names will be inserted into "others". This is a good place to store links to 3rd party software, such as Electrum forks or claim tools. """ - coins = coin_defs.get_dict() + coins = coin_info.get_all().as_dict() if support_key not in coins: click.echo("Failed to find key {}".format(support_key)) click.echo("Use 'support.py show' to search for the right one.")