Scripts moved to tools/

tools/build_coins.py

@@ -0,0 +1,248 @@
+#!/usr/bin/env python3
+
+# This script generates coins.json files from the definitions in defs/
+#
+# - `./build_coins.py` generates a big file with everything
+# - `./build_coins.py XXX` generates a file with coins supported by XXX
+#      for example: `./build_coins.py webwallet` or `./build_coins.py trezor1`
+# - `./build_coins.py XXX --defs` also adds protobuf definitions with TOIF icon
+#
+# generated file is coins.json in current directory,
+# and coindefs.json if --def is enabled
+
+import json
+import glob
+import re
+import os
+import sys
+
+if '--defs' in sys.argv:
+    from binascii import unhexlify
+    from hashlib import sha256
+    import ed25519
+    from PIL import Image
+    from trezorlib.protobuf import dump_message
+    from coindef import CoinDef
+    BUILD_DEFS = True
+else:
+    BUILD_DEFS = False
+
+
+def check_type(val, types, nullable=False, empty=False, regex=None, choice=None):  # noqa:E501
+    # check nullable
+    if nullable and val is None:
+        return True
+    # check empty
+    try:
+        if not empty and len(val) == 0:
+            return False
+    except TypeError:
+        pass
+    # check regex
+    if regex is not None:
+        if types is not str:
+            return False
+        m = re.match(regex, val)
+        if not m:
+            return False
+    # check choice
+    if choice is not None:
+        if val not in choice:
+            return False
+    # check type
+    if isinstance(types, list):
+        return True in [isinstance(val, t) for t in types]
+    else:
+        return isinstance(val, types)
+
+
+def validate_coin(coin):
+    assert check_type(coin['coin_name'], str, regex=r'^[A-Z]')
+    assert check_type(coin['coin_shortcut'], str, regex=r'^[A-Zt][A-Z][A-Z]+$')
+    assert check_type(coin['coin_label'], str, regex=r'^[A-Z]')
+    assert check_type(coin['website'], str, regex=r'^http.*[^/]$')
+    assert check_type(coin['github'], str, regex=r'^https://github.com/.*[^/]$')  # noqa:E501
+    assert check_type(coin['maintainer'], str)
+    assert check_type(coin['curve_name'], str, choice=['secp256k1', 'secp256k1_decred', 'secp256k1_groestl'])  # noqa:E501
+    assert check_type(coin['address_type'], int)
+    assert check_type(coin['address_type_p2sh'], int)
+    assert coin['address_type'] != coin['address_type_p2sh']
+    assert check_type(coin['maxfee_kb'], int)
+    assert check_type(coin['minfee_kb'], int)
+    assert coin['maxfee_kb'] >= coin['minfee_kb']
+    assert check_type(coin['hash_genesis_block'], str, regex=r'^[0-9a-f]{64}$')
+    assert check_type(coin['xprv_magic'], int)
+    assert check_type(coin['xpub_magic'], int)
+    assert check_type(coin['xpub_magic_segwit_p2sh'], int, nullable=True)
+    assert check_type(coin['xpub_magic_segwit_native'], int, nullable=True)
+    assert coin['xprv_magic'] != coin['xpub_magic']
+    assert coin['xprv_magic'] != coin['xpub_magic_segwit_p2sh']
+    assert coin['xprv_magic'] != coin['xpub_magic_segwit_native']
+    assert coin['xpub_magic'] != coin['xpub_magic_segwit_p2sh']
+    assert coin['xpub_magic'] != coin['xpub_magic_segwit_native']
+    assert coin['xpub_magic_segwit_p2sh'] is None or coin['xpub_magic_segwit_native'] is None or coin['xpub_magic_segwit_p2sh'] != coin['xpub_magic_segwit_native']  # noqa:E501
+    assert check_type(coin['slip44'], int)
+    assert check_type(coin['segwit'], bool)
+    assert check_type(coin['decred'], bool)
+    assert check_type(coin['fork_id'], int, nullable=True)
+    assert check_type(coin['force_bip143'], bool)
+    assert check_type(coin['version_group_id'], int, nullable=True)
+    assert check_type(coin['default_fee_b'], dict)
+    assert check_type(coin['dust_limit'], int)
+    assert check_type(coin['blocktime_seconds'], int)
+    assert check_type(coin['signed_message_header'], str)
+    assert check_type(coin['address_prefix'], str, regex=r'^.*:$')
+    assert check_type(coin['min_address_length'], int)
+    assert check_type(coin['max_address_length'], int)
+    assert coin['max_address_length'] >= coin['min_address_length']
+    assert check_type(coin['bech32_prefix'], str, nullable=True)
+    assert check_type(coin['cashaddr_prefix'], str, nullable=True)
+    assert check_type(coin['bitcore'], list, empty=True)
+    for bc in coin['bitcore']:
+        assert not bc.endswith('/')
+
+
+def validate_icon(icon):
+    assert icon.size == (96, 96)
+    assert icon.mode == 'RGBA'
+
+
+class Writer:
+
+    def __init__(self):
+        self.buf = bytearray()
+
+    def write(self, buf):
+        self.buf.extend(buf)
+
+
+def serialize(coin, icon):
+    c = dict(coin)
+    c['signed_message_header'] = c['signed_message_header'].encode()
+    c['hash_genesis_block'] = unhexlify(c['hash_genesis_block'])
+    c['icon'] = icon
+    msg = CoinDef(**c)
+    w = Writer()
+    dump_message(w, msg)
+    return bytes(w.buf)
+
+
+def sign(data):
+    h = sha256(data).digest()
+    sign_key = ed25519.SigningKey(b'A' * 32)
+    return sign_key.sign(h)
+
+
+# conversion copied from trezor-core/tools/png2toi
+# TODO: extract into common module in python-trezor
+def convert_icon(icon):
+    import struct
+    import zlib
+    w, h = 32, 32
+    icon = icon.resize((w, h), Image.LANCZOS)
+    # remove alpha channel, replace with black
+    bg = Image.new('RGBA', icon.size, (0, 0, 0, 255))
+    icon = Image.alpha_composite(bg, icon)
+    # process pixels
+    pix = icon.load()
+    data = bytes()
+    for j in range(h):
+        for i in range(w):
+            r, g, b, _ = pix[i, j]
+            c = ((r & 0xF8) << 8) | ((g & 0xFC) << 3) | ((b & 0xF8) >> 3)
+            data += struct.pack('>H', c)
+    z = zlib.compressobj(level=9, wbits=10)
+    zdata = z.compress(data) + z.flush()
+    zdata = zdata[2:-4]  # strip header and checksum
+    return zdata
+
+
+def process_json(fn):
+    print(os.path.basename(fn), end=' ... ')
+    j = json.load(open(fn))
+    if BUILD_DEFS:
+        i = Image.open(fn.replace('.json', '.png'))
+        validate_coin(j)
+        validate_icon(i)
+        ser = serialize(j, convert_icon(i))
+        sig = sign(ser)
+        definition = (sig + ser).hex()
+        print('OK')
+        return j, definition
+    else:
+        print('OK')
+        return j, None
+
+def process(for_device=None):
+    scriptdir = os.path.dirname(os.path.realpath(__file__))
+
+    support_json = json.load(open(scriptdir + '/../defs/support.json'))
+    if for_device is not None:
+        support_list = support_json[for_device].keys()
+    else:
+        support_list = None
+
+    coins = {}
+    defs = {}
+    for fn in glob.glob(scriptdir + '/../defs/coins/*.json'):
+        c, d = process_json(fn)
+        n = c['coin_name']
+        c['support'] = {}
+        for s in support_json.keys():
+            c['support'][s] = support_json[s][n] if n in support_json[s] else None
+        if support_list is None or n in support_list:
+            coins[n] = c
+            defs[n] = d
+
+    return (coins, defs)
+
+if __name__ == '__main__':
+    if len(sys.argv) > 1 and not sys.argv[1].startswith('-'):
+        for_device = sys.argv[1]
+    else:
+        for_device = None
+    
+    (coins, defs) = process(for_device)
+
+    json.dump(coins, open('coins.json', 'w'), indent=4, sort_keys=True)
+    if BUILD_DEFS:
+        json.dump(defs, open('coindefs.json', 'w'), indent=4, sort_keys=True)
+
+    # check for colliding address versions
+    at_p2pkh = {}
+    at_p2sh = {}
+    slip44 = {}
+
+    for n, c in coins.items():
+        s = c['slip44']
+        if s not in slip44:
+            slip44[s] = []
+        if not(n.endswith('Testnet') and s == 1):
+            slip44[s].append(n)
+        if c['cashaddr_prefix']:  # skip cashaddr currencies
+            continue
+        a1, a2 = c['address_type'], c['address_type_p2sh']
+        if a1 not in at_p2pkh:
+            at_p2pkh[a1] = []
+        if a2 not in at_p2sh:
+            at_p2sh[a2] = []
+        at_p2pkh[a1].append(n)
+        at_p2sh[a2].append(n)
+
+    print()
+    print('Colliding address_types for P2PKH:')
+    for k, v in at_p2pkh.items():
+        if len(v) >= 2:
+            print('-', k, ':', ','.join(v))
+
+    print()
+    print('Colliding address_types for P2SH:')
+    for k, v in at_p2sh.items():
+        if len(v) >= 2:
+            print('-', k, ':', ','.join(v))
+
+    print()
+    print('Colliding SLIP44 constants:')
+    for k, v in slip44.items():
+        if len(v) >= 2:
+            print('-', k, ':', ','.join(v))

tools/coindef.py

@@ -0,0 +1,110 @@
+from trezorlib import protobuf as p
+
+
+class CoinDef(p.MessageType):
+    FIELDS = {
+        1: ('coin_name', p.UnicodeType, 0),
+        2: ('coin_shortcut', p.UnicodeType, 0),
+        3: ('coin_label', p.UnicodeType, 0),
+        4: ('curve_name', p.UnicodeType, 0),
+        5: ('address_type', p.UVarintType, 0),
+        6: ('address_type_p2sh', p.UVarintType, 0),
+        7: ('maxfee_kb', p.UVarintType, 0),
+        8: ('minfee_kb', p.UVarintType, 0),
+        9: ('signed_message_header', p.BytesType, 0),
+        10: ('hash_genesis_block', p.BytesType, 0),
+        11: ('xprv_magic', p.UVarintType, 0),
+        12: ('xpub_magic', p.UVarintType, 0),
+        13: ('xpub_magic_segwit_p2sh', p.UVarintType, 0),
+        14: ('xpub_magic_segwit_native', p.UVarintType, 0),
+        15: ('bech32_prefix', p.UnicodeType, 0),
+        16: ('cashaddr_prefix', p.UnicodeType, 0),
+        17: ('slip44', p.UVarintType, 0),
+        18: ('segwit', p.BoolType, 0),
+        19: ('decred', p.BoolType, 0),
+        20: ('fork_id', p.UVarintType, 0),
+        21: ('force_bip143', p.BoolType, 0),
+        22: ('dust_limit', p.UVarintType, 0),
+        23: ('address_prefix', p.UnicodeType, 0),
+        24: ('min_address_length', p.UVarintType, 0),
+        25: ('max_address_length', p.UVarintType, 0),
+        26: ('icon', p.BytesType, 0),
+        27: ('version_group_id', p.UVarintType, 0),
+        28: ('website', p.UnicodeType, 0),
+        29: ('github', p.UnicodeType, 0),
+        30: ('maintainer', p.UnicodeType, 0),
+        31: ('blocktime_seconds', p.UVarintType, 0),
+    }
+
+    def __init__(
+        self,
+        coin_name: str = None,
+        coin_shortcut: str = None,
+        coin_label: str = None,
+        curve_name: str = None,
+        address_type: int = None,
+        address_type_p2sh: int = None,
+        maxfee_kb: int = None,
+        minfee_kb: int = None,
+        signed_message_header: bytes = None,
+        hash_genesis_block: bytes = None,
+        xprv_magic: int = None,
+        xpub_magic: int = None,
+        xpub_magic_segwit_p2sh: int = None,
+        xpub_magic_segwit_native: int = None,
+        bech32_prefix: str = None,
+        cashaddr_prefix: str = None,
+        slip44: int = None,
+        segwit: bool = None,
+        decred: bool = None,
+        fork_id: int = None,
+        force_bip143: bool = None,
+        dust_limit: int = None,
+        address_prefix: str = None,
+        min_address_length: int = None,
+        max_address_length: int = None,
+        icon: bytes = None,
+        version_group_id: int = None,
+        website: str = None,
+        github: str = None,
+        maintainer: str = None,
+        blocktime_seconds: int = None,
+        default_fee_b: dict = None,
+        bitcore: dict = None,
+        blockbook: dict = None
+    ):
+        self.coin_name = coin_name
+        self.coin_shortcut = coin_shortcut
+        self.coin_label = coin_label
+        self.curve_name = curve_name
+        self.address_type = address_type
+        self.address_type_p2sh = address_type_p2sh
+        self.maxfee_kb = maxfee_kb
+        self.minfee_kb = minfee_kb
+        self.signed_message_header = signed_message_header
+        self.hash_genesis_block = hash_genesis_block
+        self.xprv_magic = xprv_magic
+        self.xpub_magic = xpub_magic
+        self.xpub_magic_segwit_p2sh = xpub_magic_segwit_p2sh
+        self.xpub_magic_segwit_native = xpub_magic_segwit_native
+        self.bech32_prefix = bech32_prefix
+        self.cashaddr_prefix = cashaddr_prefix
+        self.slip44 = slip44
+        self.segwit = segwit
+        self.decred = decred
+        self.fork_id = fork_id
+        self.force_bip143 = force_bip143
+        self.dust_limit = dust_limit
+        self.address_prefix = address_prefix
+        self.min_address_length = min_address_length
+        self.max_address_length = max_address_length
+        self.icon = icon
+        self.version_group_id = version_group_id
+        self.website = website
+        self.github = github
+        self.maintainer = maintainer
+        self.blocktime_seconds = blocktime_seconds
+        self.default_fee_b = default_fee_b
+        self.bitcore = bitcore
+        self.blockbook = blockbook
+        p.MessageType.__init__(self)

tools/coins_details.py

@@ -5,6 +5,7 @@ import json
 import requests
 import pprint
 import ethereum_tokens_gen
+import build_coins
 
 COINS = {}
 
@@ -68,8 +69,8 @@ def update_info(details):
     details['info']['updated_at'] = int(time.time())
     details['info']['updated_at_readable'] = time.asctime()
 
-    details['info']['t1_coins'] = len([True for _, c in details['coins'].items() if c['t1_enabled'] == 'yes' and not c.get('hidden', False)])
-    details['info']['t2_coins'] = len([True for _, c in details['coins'].items() if c['t2_enabled'] == 'yes' and not c.get('hidden', False)])
+    details['info']['t1_coins'] = len([True for _, c in details['coins'].items() if c.get('t1_enabled') == 'yes' and not c.get('hidden', False)])
+    details['info']['t2_coins'] = len([True for _, c in details['coins'].items() if c.get('t2_enabled') == 'yes' and not c.get('hidden', False)])
 
     try:
         details['info']['total_marketcap_usd'] = int(coinmarketcap_global()['data']['quotes']['USD']['total_market_cap'])
@@ -90,12 +91,13 @@ def check_unsupported(details, prefix, supported):
             print("%s not supported by Trezor? (Possible manual entry)" % k)
 
 def update_coins(details):
-    coins = json.load(open('coins.json', 'r'))
+    (coins, _) = build_coins.process(None)
+    firmware = json.load(open('../defs/support.json', 'r'))
 
     supported = []
-    for coin in coins:
-        if coin['firmware'] != 'stable':
-            continue
+    for key, coin in coins.items():
+        t1_enabled = key in firmware['trezor1']
+        t2_enabled = key in firmware['trezor2']
 
         # print("Updating", coin['coin_label'], coin['coin_shortcut'])
         key = "coin:%s" % coin['coin_shortcut']
@@ -105,8 +107,11 @@ def update_coins(details):
         set_default(out, 'shortcut', coin['coin_shortcut'])
         set_default(out, 'name', coin['coin_label'])
         set_default(out, 'links', {})
-        set_default(out, 't1_enabled', 'yes')
-        set_default(out, 't2_enabled', 'yes')
+        set_default(out['links'], 'Homepage', coin['website'])
+        set_default(out['links'], 'Github', coin['github'])
+        set_default(out, 't1_enabled', 'yes' if t1_enabled else 'no')
+        set_default(out, 't2_enabled', 'yes' if t2_enabled else 'no')
+
         update_marketcap(out, coin.get('coinmarketcap_alias', coin['coin_label']))
 
     check_unsupported(details, 'coin:', supported)
@@ -235,7 +240,7 @@ def update_ethereum(details):
     set_default(out, 't2_enabled', 'yes')
     update_marketcap(out, 'etsc')
 
-    ut = details['coins'].setdefault('coin2:EOSC', {})
+    out = details['coins'].setdefault('coin2:EOSC', {})
     out['type'] = 'coin'
     set_default(out, 'shortcut', 'EOSC')
     set_default(out, 'name', 'EOS Classic')
@@ -244,7 +249,7 @@ def update_ethereum(details):
     update_marketcap(out, 'eosc')
 
 def update_mosaics(details):
-    d = json.load(open('defs/nem/nem_mosaics.json'))
+    d = json.load(open('../defs/nem/nem_mosaics.json'))
     supported = []
     for mosaic in d:
         # print('Updating', mosaic['name'], mosaic['ticker'])
@@ -270,7 +275,7 @@ def check_missing_details(details):
         if 'links' not in coin:
             print("%s: Missing links" % k)
             hide = True
-        if 'Homepage' not in coin['links']:
+        if 'Homepage' not in coin.get('links', {}):
             print("%s: Missing homepage" % k)
             hide = True
         if coin['t1_enabled'] not in ('yes', 'no', 'planned', 'soon'):
@@ -279,11 +284,11 @@ def check_missing_details(details):
         if coin['t2_enabled'] not in ('yes', 'no', 'planned', 'soon'):
             print("%s: Unknown t2_enabled" % k)
             hide = True
-        if 'TREZOR Wallet' in coin['links'] and coin['links']['TREZOR Wallet'] != 'https://wallet.trezor.io':
-            print("%s: Strange URL for TREZOR Wallet" % k)
+        if 'Trezor Wallet' in coin.get('links', {}) and coin['links']['Trezor Wallet'] != 'https://wallet.trezor.io':
+            print("%s: Strange URL for Trezor Wallet" % k)
             hide = True
 
-        for w in [ x.lower() for x in coin['links'].keys() ]:
+        for w in [ x.lower() for x in coin.get('links', {}).keys() ]:
             if 'wallet' in w or 'electrum' in w:
                 break
         else:
@@ -294,6 +299,9 @@ def check_missing_details(details):
                 print("%s: Missing wallet, but not hiding" % k)
 
         if hide:
+            if coin.get('hidden') != 1:
+                print("%s: HIDING COIN!" % k)
+
             # If any of important detail is missing, hide coin from list
             coin['hidden'] = 1
 
@@ -306,7 +314,7 @@ def check_missing_details(details):
 
 if __name__ == '__main__':
     try:
-        details = json.load(open('coins_details.json', 'r'))
+        details = json.load(open('../defs/coins_details.json', 'r'))
     except FileNotFoundError:
         details = {'coins': {}, 'info': {}}
 
@@ -319,4 +327,4 @@ if __name__ == '__main__':
     check_missing_details(details)
 
     print(json.dumps(details['info'], sort_keys=True, indent=4))
-    json.dump(details, open('coins_details.json', 'w'), sort_keys=True, indent=4)
+    json.dump(details, open('../defs/coins_details.json', 'w'), sort_keys=True, indent=4)

tools/ethereum_tokens_gen.py

@@ -7,7 +7,7 @@ def get_tokens():
     tokens = []
     for s, i in networks:
         try:
-            files = os.scandir('defs/ethereum/tokens/tokens/%s' % s)
+            files = os.scandir('../defs/ethereum/tokens/tokens/%s' % s)
         except FileNotFoundError:
             continue