1
0
mirror of https://github.com/trezor/trezor-firmware.git synced 2024-12-22 22:38:08 +00:00

tools: coin_defs renamed to coin_info and interface

improved a little
This commit is contained in:
matejcik 2018-07-30 14:25:53 +02:00
parent 1e032d4da5
commit b5443af4c5
4 changed files with 192 additions and 66 deletions

View File

@ -8,7 +8,7 @@ import glob
import click
import coin_defs
import coin_info
try:
import mako
@ -25,10 +25,13 @@ except ImportError:
requests = None
try:
import binascii
import struct
import zlib
from hashlib import sha256
import ed25519
from PIL import Image
from trezorlib.protobuf import dump_message
from trezorlib import protobuf
from coindef import CoinDef
CAN_BUILD_DEFS = True
@ -36,7 +39,7 @@ except ImportError:
CAN_BUILD_DEFS = False
# ======= Jinja2 management ======
# ======= Mako management ======
def c_str_filter(b):
@ -70,26 +73,36 @@ def render_file(filename, coins, support_info):
# ====== validation functions ======
def check_support(defs, support_data):
def check_support(defs, support_data, fail_missing=False):
check_passed = True
coin_list = defs.as_list()
coin_names = {coin["key"]: coin["name"] for coin in coin_list}
def coin_name(key):
if key in coin_names:
return "{} ({})".format(key, coin_names[key])
else:
return "{} <unknown key>".format(key)
for key, support in support_data.items():
errors = coin_defs.validate_support(support)
errors = coin_info.validate_support(support)
if errors:
check_passed = False
print("ERR:", "invalid definition for", key)
print("ERR:", "invalid definition for", coin_name(key))
print("\n".join(errors))
expected_coins = set(coin["key"] for coin in defs["coins"] + defs["misc"])
expected_coins = set(coin["key"] for coin in defs.coins + defs.misc)
# detect missing support info for expected
for coin in expected_coins:
if coin not in support_data:
check_passed = False
print("ERR: Missing support info for", coin)
if fail_missing:
check_passed = False
print("ERR: Missing support info for", coin_name(coin))
else:
print("WARN: Missing support info for", coin_name(coin))
# detect non-matching support info
coin_list = sum(defs.values(), [])
coin_set = set(coin["key"] for coin in coin_list)
for key in support_data:
# detect non-matching support info
@ -97,9 +110,9 @@ def check_support(defs, support_data):
check_passed = False
print("ERR: Support info found for unknown coin", key)
# detect override - info only, doesn't fail check
# detect override - doesn't fail check
if key not in expected_coins:
print("INFO: Override present for coin", key)
print("INFO: Override present for coin", coin_name(key))
return check_passed
@ -108,13 +121,13 @@ def check_btc(coins):
check_passed = True
for coin in coins:
errors = coin_defs.validate_btc(coin)
errors = coin_info.validate_btc(coin)
if errors:
check_passed = False
print("ERR:", "invalid definition for", coin["name"])
print("\n".join(errors))
collisions = coin_defs.find_address_collisions(coins)
collisions = coin_info.find_address_collisions(coins)
# warning only
for key, dups in collisions.items():
if dups:
@ -135,7 +148,7 @@ def check_backends(coins):
for backend in backends:
print("checking", backend, "... ", end="", flush=True)
try:
j = requests.get(backend + "/block-index/0").json()
j = requests.get(backend + "/api/block-index/0").json()
if j["blockHash"] != genesis_block:
raise RuntimeError("genesis block mismatch")
except Exception as e:
@ -146,6 +159,59 @@ def check_backends(coins):
return check_passed
# ====== coindefs generators ======
def convert_icon(icon):
"""Convert PIL icon to TOIF format"""
# TODO: move this to python-trezor at some point
DIM = 32
icon = icon.resize((DIM, DIM), Image.LANCZOS)
# remove alpha channel, replace with black
bg = Image.new("RGBA", icon.size, (0, 0, 0, 255))
icon = Image.alpha_composite(bg, icon)
# process pixels
pix = icon.load()
data = bytes()
for y in range(DIM):
for x in range(DIM):
r, g, b, _ = pix[x, y]
c = ((r & 0xF8) << 8) | ((g & 0xFC) << 3) | ((b & 0xF8) >> 3)
data += struct.pack(">H", c)
z = zlib.compressobj(level=9, wbits=10)
zdata = z.compress(data) + z.flush()
zdata = zdata[2:-4] # strip header and checksum
return zdata
def coindef_from_dict(coin):
proto = CoinDef()
for fname, _, fflags in CoinDef.FIELDS.values():
val = coin.get(fname)
if val is None and fflags & protobuf.FLAG_REPEATED:
val = []
elif fname == "signed_message_header":
val = val.encode("utf-8")
elif fname == "hash_genesis_block":
val = binascii.unhexlify(val)
setattr(proto, fname, val)
return proto
def serialize_coindef(proto, icon):
proto.icon = icon
buf = io.BytesIO()
protobuf.dump_message(buf, proto)
return buf.getvalue()
def sign(data):
h = sha256(data).digest()
sign_key = ed25519.SigningKey(b"A" * 32)
return sign_key.sign(h)
# ====== click command handlers ======
@ -155,12 +221,17 @@ def cli():
@cli.command()
@click.option(
"--check-missing-support/--no-check-missing-support",
"-s",
help="Fail if support info for a coin is missing",
)
@click.option(
"--backend-check/--no-backend-check",
"-b",
help="Also check blockbook/bitcore responses",
)
def check(backend_check):
def check(check_missing_support, backend_check):
"""Validate coin definitions.
Checks that every btc-like coin is properly filled out, reports address collisions
@ -169,20 +240,21 @@ def check(backend_check):
if backend_check and requests is None:
raise click.ClickException("You must install requests for backend check")
defs = coin_defs.get_all()
defs = coin_info.get_all()
all_checks_passed = True
print("Checking BTC-like coins...")
if not check_btc(defs["coins"]):
if not check_btc(defs.coins):
all_checks_passed = False
print("Checking support data...")
if not check_support(defs, coin_defs.get_support_data()):
support_data = coin_info.get_support_data()
if not check_support(defs, support_data, fail_missing=check_missing_support):
all_checks_passed = False
if backend_check:
print("Checking backend responses...")
if not check_backends(defs["coins"]):
if not check_backends(defs.coins):
all_checks_passed = False
if not all_checks_passed:
@ -196,9 +268,8 @@ def check(backend_check):
@click.option("-o", "--outfile", type=click.File(mode="w"), default="./coins.json")
def coins_json(outfile):
"""Generate coins.json for consumption in python-trezor and Connect/Wallet"""
defs = coin_defs.get_all()
coins = defs["coins"]
support_info = coin_defs.support_info(coins)
coins = coin_info.get_all().coins
support_info = coin_info.support_info(coins)
by_name = {}
for coin in coins:
coin["support"] = support_info[coin["key"]]
@ -206,6 +277,30 @@ def coins_json(outfile):
with outfile:
json.dump(by_name, outfile, indent=4, sort_keys=True)
outfile.write("\n")
@cli.command()
@click.option("-o", "--outfile", type=click.File(mode="w"), default="./coindefs.json")
def coindefs(outfile):
"""Generate signed coin definitions for python-trezor and others
This is currently unused but should enable us to add new coins without having to
update firmware.
"""
coins = coin_info.get_all().coins
coindefs = {}
for coin in coins:
key = coin["key"]
icon = Image.open(coin["icon"])
ser = serialize_coindef(coindef_from_dict(coin), convert_icon(icon))
sig = sign(ser)
definition = binascii.hexlify(sig + ser).decode("ascii")
coindefs[key] = definition
with outfile:
json.dump(coindefs, outfile, indent=4, sort_keys=True)
outfile.write("\n")
@cli.command()
@ -236,10 +331,9 @@ def render(paths):
else:
files.append(path)
defs = coin_defs.get_all()
all_coins = sum(defs.values(), [])
versions = coin_defs.latest_releases()
support_info = coin_defs.support_info(all_coins, erc20_versions=versions)
defs = coin_info.get_all()
versions = coin_info.latest_releases()
support_info = coin_info.support_info(defs, erc20_versions=versions)
# munch dicts - make them attribute-accessable
for key, value in defs.items():

View File

@ -30,6 +30,35 @@ def load_json(*path):
return json.load(f, object_pairs_hook=OrderedDict)
# ====== CoinsInfo ======
class CoinsInfo(dict):
"""Collection of information about all known kinds of coins.
It contains the following lists:
`coins` for btc-like coins,
`eth` for ethereum networks,
`erc20` for ERC20 tokens,
`nem` for NEM mosaics,
`misc` for other networks.
Accessible as a dict or by attribute: `info["coins"] == info.coins`
"""
def as_list(self):
return sum(self.values(), [])
def as_dict(self):
return {coin["key"]: coin for coin in self.as_list()}
def __getattr__(self, attr):
if attr in self:
return self[attr]
else:
raise AttributeError(attr)
# ====== coin validation ======
@ -186,6 +215,7 @@ def _load_btc_coins():
name=coin["coin_name"],
shortcut=coin["coin_shortcut"],
key="coin:{}".format(coin["coin_shortcut"]),
icon=filename.replace(".json", ".png"),
)
coins.append(coin)
@ -324,11 +354,14 @@ def support_info_erc20(coins, versions):
def support_info(coins, erc20_versions=None, skip_missing=False):
"""Generate Trezor support information.
Takes a dict of coins and generates a support-info entry for each.
Takes a collection of coins and generates a support-info entry for each.
The support-info is a dict with a number of known keys:
`trezor1`, `trezor2`, `webwallet`, `connect`. An optional `other` entry
is a dict of name-url pairs for third-party software.
The `coins` argument can be a `CoinsInfo` object, a list or a dict of
coin items.
For btc-like coins and misc networks, this is taken from `support.json`.
For NEM mosaics and ethereum networks, the support is presumed to be "yes"
for both Trezors. Webwallet and Connect info is not filled out.
@ -346,6 +379,11 @@ def support_info(coins, erc20_versions=None, skip_missing=False):
and a warning emitted. "No support information" means that the coin is not
listed in `support.json` and we have no heuristic to determine the support.
"""
if isinstance(coins, CoinsInfo):
coins = coins.as_list()
elif isinstance(coins, dict):
coins = coins.values()
support_data = get_support_data()
support = {}
for coin in coins:
@ -428,6 +466,10 @@ def _filter_duplicate_shortcuts(coins):
retained_coins = OrderedDict()
for coin in coins:
if "Testnet" in coin["name"] and coin["shortcut"] == "tETH":
# special case for Ethereum testnets
continue
key = coin["shortcut"]
if key in dup_keys:
pass
@ -458,7 +500,7 @@ def get_all():
`nem` for NEM mosaics,
`misc` for other networks.
"""
all_coins = dict(
all_coins = CoinsInfo(
coins=_load_btc_coins(),
eth=_load_ethereum_networks(),
erc20=_load_erc20_tokens(),
@ -476,22 +518,13 @@ def get_all():
coins.sort(key=lambda c: c["key"].upper())
_ensure_mandatory_values(coins)
if k != "eth":
dup_keys = _filter_duplicate_shortcuts(coins)
if dup_keys:
log.warning(
"{}: removing duplicate symbols: {}".format(k, ", ".join(dup_keys))
)
dup_keys = _filter_duplicate_shortcuts(coins)
if dup_keys:
if k == "erc20":
severity = logging.INFO
else:
severity = logging.WARNING
dup_str = ", ".join(dup_keys)
log.log(severity, "{}: removing duplicate symbols: {}".format(k, dup_str))
return all_coins
def get_list():
"""Return all definitions as a single list of coins."""
all_coins = get_all()
return sum(all_coins.values(), [])
def get_dict():
"""Return all definitions as a dict indexed by coin keys."""
return {coin["key"]: coin for coin in get_list()}

View File

@ -5,15 +5,15 @@ import time
import json
import logging
import requests
import coin_defs
import coin_info
LOG = logging.getLogger(__name__)
OPTIONAL_KEYS = ("links", "notes", "wallet")
ALLOWED_SUPPORT_STATUS = ("yes", "no", "planned", "soon")
OVERRIDES = coin_defs.load_json("coins_details.override.json")
VERSIONS = coin_defs.latest_releases()
OVERRIDES = coin_info.load_json("coins_details.override.json")
VERSIONS = coin_info.latest_releases()
COINMAKETCAP_CACHE = os.path.join(os.path.dirname(__file__), "coinmarketcap.json")
@ -292,16 +292,15 @@ def apply_overrides(coins):
if __name__ == "__main__":
defs = coin_defs.get_all()
all_coins = sum(defs.values(), [])
support_info = coin_defs.support_info(all_coins, erc20_versions=VERSIONS)
defs = coin_info.get_all()
support_info = coin_info.support_info(defs, erc20_versions=VERSIONS)
coins = {}
coins.update(update_coins(defs["coins"], support_info))
coins.update(update_erc20(defs["erc20"], support_info))
coins.update(update_ethereum_networks(defs["eth"], support_info))
coins.update(update_simple(defs["nem"], support_info, "mosaic"))
coins.update(update_simple(defs["misc"], support_info, "coin"))
coins.update(update_coins(defs.coins, support_info))
coins.update(update_erc20(defs.erc20, support_info))
coins.update(update_ethereum_networks(defs.eth, support_info))
coins.update(update_simple(defs.nem, support_info, "mosaic"))
coins.update(update_simple(defs.misc, support_info, "coin"))
apply_overrides(coins)
update_marketcaps(coins)
@ -311,5 +310,5 @@ if __name__ == "__main__":
details = dict(coins=coins, info=info)
print(json.dumps(info, sort_keys=True, indent=4))
with open(os.path.join(coin_defs.DEFS_DIR, "coins_details.json"), "w") as f:
with open(os.path.join(coin_info.DEFS_DIR, "coins_details.json"), "w") as f:
json.dump(details, f, sort_keys=True, indent=4)

View File

@ -2,10 +2,10 @@
import os
import sys
import click
import coin_defs
import coin_info
import json
SUPPORT_INFO = coin_defs.get_support_data()
SUPPORT_INFO = coin_info.get_support_data()
MANDATORY_ENTRIES = ("trezor1", "trezor2", "connect", "webwallet")
@ -36,7 +36,7 @@ def update_support(key, entry, value):
def write_support_info():
with open(os.path.join(coin_defs.DEFS_DIR, "support.json"), "w") as f:
with open(os.path.join(coin_info.DEFS_DIR, "support.json"), "w") as f:
json.dump(SUPPORT_INFO, f, indent=4)
f.write("\n")
@ -69,11 +69,11 @@ def check():
here for convenience and because it makes sense. But it's preferable to run it
as part of 'coin_gen.py check'.
"""
defs = coin_defs.get_all()
support_data = coin_defs.get_support_data()
defs = coin_info.get_all()
support_data = coin_info.get_support_data()
import coin_gen
if not coin_gen.check_support(defs, support_data):
if not coin_gen.check_support(defs, support_data, fail_missing=True):
sys.exit(1)
@ -88,7 +88,7 @@ def show(keyword):
Only coins listed in support.json are considered "supported". That means that
Ethereum networks, ERC20 tokens and NEM mosaics will probably show up wrong.
"""
defs = coin_defs.get_list()
defs = coin_info.get_all().as_list()
if keyword:
for coin in defs:
@ -138,7 +138,7 @@ def set(support_key, entries, dry_run):
Entries with other names will be inserted into "others". This is a good place
to store links to 3rd party software, such as Electrum forks or claim tools.
"""
coins = coin_defs.get_dict()
coins = coin_info.get_all().as_dict()
if support_key not in coins:
click.echo("Failed to find key {}".format(support_key))
click.echo("Use 'support.py show' to search for the right one.")