tools: unified retrieval of coin data, regenerated coins_details

tools/coin_defs.py

@@ -0,0 +1,404 @@
+#!/usr/bin/env python3
+from binascii import unhexlify
+from collections import defaultdict, OrderedDict
+import re
+import os
+import json
+import glob
+import logging
+
+try:
+    import requests
+except ImportError:
+    requests = None
+
+log = logging.getLogger(__name__)
+
+DEFS_DIR = os.path.abspath(
+    os.environ.get("DEFS_DIR") or os.path.join(os.path.dirname(__file__), "..", "defs")
+)
+
+
+def load_json(*path):
+    if len(path) == 1 and path[0].startswith("/"):
+        filename = path[0]
+    else:
+        filename = os.path.join(DEFS_DIR, *path)
+
+    with open(filename) as f:
+        return json.load(f, object_pairs_hook=OrderedDict)
+
+
+# ====== coin validation ======
+
+
+def check_type(val, types, nullable=False, empty=False, regex=None, choice=None):
+    # check nullable
+    if val is None:
+        if nullable:
+            return
+        else:
+            raise ValueError("Missing required value")
+
+    # check type
+    if not isinstance(val, types):
+        raise TypeError("Wrong type (expected: {})".format(types))
+
+    # check empty
+    if isinstance(val, (list, dict)) and not empty and not val:
+        raise ValueError("Empty collection")
+
+    # check regex
+    if regex is not None:
+        if types is not str:
+            raise TypeError("Wrong type for regex check")
+        if not re.search(regex, val):
+            raise ValueError("Value does not match regex {}".format(regex))
+
+    # check choice
+    if choice is not None and val not in choice:
+        raise ValueError("Value not allowed, use one of: {}".format(", ".join(choice)))
+
+
+def check_key(key, types, **kwargs):
+    def do_check(coin):
+        if not key in coin:
+            raise KeyError("{}: Missing key".format(key))
+        try:
+            check_type(coin[key], types, **kwargs)
+        except Exception as e:
+            raise ValueError("{}: {}".format(key, e)) from e
+
+    return do_check
+
+
+COIN_CHECKS = [
+    check_key("coin_name", str, regex=r"^[A-Z]"),
+    check_key("coin_shortcut", str, regex=r"^t?[A-Z]{3,}$"),
+    check_key("coin_label", str, regex=r"^[A-Z]"),
+    check_key("website", str, regex=r"^http.*[^/]$"),
+    check_key("github", str, regex=r"^https://github.com/.*[^/]$"),
+    check_key("maintainer", str),
+    check_key(
+        "curve_name", str, choice=["secp256k1", "secp256k1_decred", "secp256k1_groestl"]
+    ),
+    check_key("address_type", int),
+    check_key("address_type_p2sh", int),
+    check_key("maxfee_kb", int),
+    check_key("minfee_kb", int),
+    check_key("hash_genesis_block", str, regex=r"^[0-9a-f]{64}$"),
+    check_key("xprv_magic", int),
+    check_key("xpub_magic", int),
+    check_key("xpub_magic_segwit_p2sh", int, nullable=True),
+    check_key("xpub_magic_segwit_native", int, nullable=True),
+    check_key("slip44", int),
+    check_key("segwit", bool),
+    check_key("decred", bool),
+    check_key("fork_id", int, nullable=True),
+    check_key("force_bip143", bool),
+    check_key("bip115", bool),
+    check_key("version_group_id", int, nullable=True),
+    check_key("default_fee_b", dict),
+    check_key("dust_limit", int),
+    check_key("blocktime_seconds", int),
+    check_key("signed_message_header", str),
+    check_key("address_prefix", str, regex=r":$"),
+    check_key("min_address_length", int),
+    check_key("max_address_length", int),
+    check_key("bech32_prefix", str, nullable=True),
+    check_key("cashaddr_prefix", str, nullable=True),
+    check_key("bitcore", list, empty=True),
+    check_key("blockbook", list, empty=True),
+]
+
+
+def validate_coin(coin):
+    errors = []
+    for check in COIN_CHECKS:
+        try:
+            check(coin)
+        except Exception as e:
+            errors.append(str(e))
+
+    magics = [
+        coin[k]
+        for k in (
+            "xprv_magic",
+            "xpub_magic",
+            "xpub_magic_segwit_p2sh",
+            "xpub_magic_segwit_native",
+        )
+        if coin[k] is not None
+    ]
+    # each of those must be unique
+    # therefore length of list == length of set of unique values
+    if len(magics) != len(set(magics)):
+        errors.append("XPUB/XPRV magic numbers must be unique")
+
+    if coin["address_type"] == coin["address_type_p2sh"]:
+        errors.append("address_type must be distinct from address_type_p2sh")
+
+    if not coin["maxfee_kb"] >= coin["minfee_kb"]:
+        errors.append("max fee must not be smaller than min fee")
+
+    if not coin["max_address_length"] >= coin["min_address_length"]:
+        errors.append("max address length must not be smaller than min address length")
+
+    for bc in coin["bitcore"] + coin["blockbook"]:
+        if bc.endswith("/"):
+            errors.append("make sure URLs don't end with '/'")
+
+    return errors
+
+
+# ======= Coin json loaders =======
+
+
+def get_coins():
+    coins = []
+    for filename in glob.glob(os.path.join(DEFS_DIR, "coins", "*.json")):
+        coin = load_json(filename)
+        coin.update(
+            name=coin["coin_name"],
+            shortcut=coin["coin_shortcut"],
+            key="btc:{}".format(coin["coin_shortcut"]),
+        )
+        coins.append(coin)
+
+    return coins
+
+
+def get_ethereum_networks():
+    networks = load_json("ethereum", "networks.json")
+    for network in networks:
+        network.update(key="eth:{}".format(network["shortcut"]))
+    return networks
+
+
+def get_erc20_tokens():
+    networks = get_ethereum_networks()
+    tokens = []
+    for network in networks:
+        if network["name"].startswith("Ethereum Testnet "):
+            idx = len("Ethereum Testnet ")
+            chain = network["name"][idx : idx + 3]
+        else:
+            chain = network["shortcut"]
+        chain = chain.lower()
+        if not chain:
+            continue
+
+        chain_path = os.path.join(DEFS_DIR, "ethereum", "tokens", "tokens", chain)
+        for filename in glob.glob(os.path.join(chain_path, "*.json")):
+            token = load_json(filename)
+            token.update(
+                chain=chain,
+                chain_id=network["chain_id"],
+                address_bytes=unhexlify(token["address"][2:]),
+                shortcut=token["symbol"],
+                key="erc20:{}:{}".format(chain, token["symbol"]),
+            )
+            tokens.append(token)
+
+    return tokens
+
+
+def get_nem_mosaics():
+    mosaics = load_json("nem", "nem_mosaics.json")
+    for mosaic in mosaics:
+        shortcut = mosaic["ticker"].strip()
+        mosaic.update(shortcut=shortcut, key="nem:{}".format(shortcut))
+    return mosaics
+
+
+def get_others():
+    others = load_json("others.json")
+    for other in others:
+        other.update(key="network:{}".format(other["shortcut"]))
+    return others
+
+
+# ====== support info ======
+
+RELEASES_URL = "https://wallet.trezor.io/data/firmware/{}/releases.json"
+ETHEREUM_TOKENS = {
+    "1": "https://raw.githubusercontent.com/trezor/trezor-mcu/v{}/firmware/ethereum_tokens.c",
+    "2": "https://raw.githubusercontent.com/trezor/trezor-core/v{}/src/apps/ethereum/tokens.py",
+}
+
+TOKEN_MATCH = {
+    "1": re.compile(r'\{.*" ([^"]+)".*\},'),
+    "2": re.compile(r'\(.*["\']([^"\']+)["\'].*\),'),
+}
+
+
+def latest_releases():
+    if not requests:
+        raise RuntimeError("requests library is required for getting release info")
+
+    latest = {}
+    for v in ("1", "2"):
+        releases = requests.get(RELEASES_URL.format(v)).json()
+        latest[v] = max(tuple(r["version"]) for r in releases)
+    return latest
+
+
+def support_info_erc20(coins, versions):
+    supported_tokens = {}
+    for trezor, version in versions.items():
+        tokens = set()
+        version_str = ".".join(map(str, version))
+
+        token_file = requests.get(ETHEREUM_TOKENS[trezor].format(version_str)).text
+        token_match = TOKEN_MATCH[trezor]
+
+        for line in token_file.split("\n"):
+            m = token_match.search(line)
+            if m:
+                tokens.add(m.group(1))
+
+        supported_tokens[trezor] = tokens
+
+    support = {}
+    for coin in coins:
+        key = coin["key"]
+        if not key.startswith("erc20:"):
+            continue
+
+        support[key] = dict(
+            trezor1="yes" if coin["shortcut"] in supported_tokens["1"] else "soon",
+            trezor2="yes" if coin["shortcut"] in supported_tokens["2"] else "soon",
+        )
+
+    return support
+
+
+def support_info(coins, erc20_versions=None):
+    support_data = load_json("support.json")
+    support = {}
+    for coin in coins:
+        key = coin["key"]
+        typ = key.split(":", maxsplit=1)[0]
+        if key in support_data:
+            support[key] = support_data[key]
+
+        elif typ in ("nem", "eth"):
+            support[key] = dict(trezor1="yes", trezor2="yes")
+
+        elif typ == "erc20":
+            # you must call a separate function to get that
+            continue
+
+        else:
+            log.warning("support info missing for {}".format(key))
+            support[key] = {}
+
+    if erc20_versions:
+        erc20 = support_info_erc20(coins, erc20_versions)
+        erc20.update(support)
+        return erc20
+    else:
+        return support
+
+
+# ====== data cleanup functions ======
+
+
+def find_address_collisions(coins):
+    slip44 = defaultdict(list)
+    at_p2pkh = defaultdict(list)
+    at_p2sh = defaultdict(list)
+
+    for name, coin in coins.items():
+        s = coin["slip44"]
+        # ignore m/1 testnets
+        if not (name.endswith("Testnet") and s == 1):
+            slip44[s].append(s)
+
+        # skip address types on cashaddr currencies
+        if coin["cashaddr_prefix"]:
+            continue
+
+        at_p2pkh[coin["address_type"]].append(name)
+        at_p2sh[coin["address_type_p2sh"]].append(name)
+
+    def prune(d):
+        ret = d.copy()
+        for key in d.keys():
+            if len(d[key]) < 2:
+                del ret[key]
+        return ret
+
+    return dict(
+        slip44=prune(slip44),
+        address_type=prune(at_p2pkh),
+        address_type_p2sh=prune(at_p2sh),
+    )
+
+
+def ensure_mandatory_values(coins):
+    for coin in coins:
+        if not all(coin.get(k) for k in ("name", "shortcut", "key")):
+            raise ValueError(coin)
+
+
+def filter_duplicate_shortcuts(coins):
+    dup_keys = set()
+    retained_coins = OrderedDict()
+
+    for coin in coins:
+        key = coin["shortcut"]
+        if key in dup_keys:
+            pass
+        elif key in retained_coins:
+            dup_keys.add(key)
+            del retained_coins[key]
+        else:
+            retained_coins[key] = coin
+
+    # modify original list
+    coins[:] = retained_coins.values()
+    # return duplicates
+    return dup_keys
+
+
+def _btc_sort_key(coin):
+    if coin["name"] in ("Bitcoin", "Testnet"):
+        return "000000" + coin["name"]
+    else:
+        return coin["name"]
+
+
+def get_all():
+    all_coins = dict(
+        btc=get_coins(),
+        eth=get_ethereum_networks(),
+        erc20=get_erc20_tokens(),
+        nem=get_nem_mosaics(),
+        other=get_others(),
+    )
+
+    for k, coins in all_coins.items():
+        if k == "btc":
+            coins.sort(key=_btc_sort_key)
+        else:
+            coins.sort(key=lambda c: c["key"].upper())
+
+        ensure_mandatory_values(coins)
+        if k != "eth":
+            dup_keys = filter_duplicate_shortcuts(coins)
+            if dup_keys:
+                log.warning(
+                    "{}: removing duplicate symbols: {}".format(k, ", ".join(dup_keys))
+                )
+
+    return all_coins
+
+
+def get_list():
+    all_coins = get_all()
+    return sum(all_coins.values(), [])
+
+
+def get_dict():
+    return {coin["key"]: coin for coin in get_list()}

tools/coins_details.py

@@ -2,360 +2,343 @@
 """Fetch information about coins and tokens supported by Trezor and update it in coins_details.json."""
 import time
 import json
+import logging
 import requests
-import ethereum_tokens_gen
+import coin_defs
-import build_coins
-from distutils.version import LooseVersion
 
-T1_LATEST='1.6.2'
+OPTIONAL_KEYS = ("links", "notes", "wallet")
-T2_LATEST='2.0.7'
+OVERRIDES = coin_defs.load_json("coins_details.override.json")
+
+VERSIONS = coin_defs.latest_releases()
 
 COINS = {}
 
+log = logging.getLogger(__name__)
+
+
 def coinmarketcap_init():
     global COINS
 
     try:
-        COINS = json.load(open('coinmarketcap.json', 'r'))
+        marketcap_json = json.load(open("coinmarketcap.json", "r"))
     except FileNotFoundError:
         pass
     else:
-        if COINS["1"]["last_updated"] > time.time() - 3600:
+        pass
-            print("Using local cache of coinmarketcap")
+        # if COINS["1"]["last_updated"] > time.time() - 3600:
-            return
+        #     print("Using local cache of coinmarketcap")
+        #     return
+
+    for coin in marketcap_json.values():
+        slug = coin["website_slug"]
+        market_cap = coin["quotes"]["USD"]["market_cap"]
+        if market_cap is not None:
+            COINS[slug] = int(float(market_cap))
+
+    return
 
     print("Updating coins from coinmarketcap")
     total = None
     COINS = {}
 
     while total is None or len(COINS) < total:
-        url = 'https://api.coinmarketcap.com/v2/ticker/?start=%d&convert=USD&limit=100' % (len(COINS))
+        url = (
+            "https://api.coinmarketcap.com/v2/ticker/?start=%d&convert=USD&limit=100"
+            % (len(COINS) + 1)
+        )
         data = requests.get(url).json()
-        COINS.update(data['data'])
+        COINS.update(data["data"])
         if total is None:
-            total = data['metadata']['num_cryptocurrencies']
+            total = data["metadata"]["num_cryptocurrencies"]
 
         print("Fetched %d of %d coins" % (len(COINS), total))
         time.sleep(1)
 
-    json.dump(COINS, open('coinmarketcap.json', 'w'), sort_keys=True, indent=4)
+    json.dump(COINS, open("coinmarketcap.json", "w"), sort_keys=True, indent=4)
 
 
-def coinmarketcap_info(shortcut):
+def marketcap(coin):
-    global COINS
+    cap = None
-    shortcut = shortcut.replace(' ', '-').lower()
 
-    for _id in COINS:
+    if "coinmarketcap_alias" in coin:
-        coin = COINS[_id]
+        cap = COINS.get(coin["coinmarketcap_alias"])
-        # print(shortcut, coin['website_slug'])
+    
-        if shortcut == coin['website_slug']:
+    if not cap:
-            # print(coin)
+        slug = coin["name"].replace(" ", "-").lower()
-            return coin
+        cap = COINS.get(slug)
+    
+    if not cap:
+        cap = COINS.get(coin["shortcut"].lower())
+    
+    return cap
 
 
-def update_marketcap(obj, *shortcuts):
+def update_marketcap(coins):
-    for sym in shortcuts:
+    for coin in coins.values():
-        try:
+        cap = marketcap(coin)
-            obj['marketcap_usd'] = int(float(coinmarketcap_info(sym)['quotes']['USD']['market_cap']))
+        if cap:
-            return
+            coin["marketcap_usd"] = cap
-        except:
-            pass
-        # print("Marketcap info not found for", shortcut)
 
 
 def coinmarketcap_global():
-    url = 'https://api.coinmarketcap.com/v2/global'
+    url = "https://api.coinmarketcap.com/v2/global"
     ret = requests.get(url)
     data = ret.json()
     return data
 
 
-def set_default(obj, key, default_value):
-    obj[key] = obj.setdefault(key, default_value)
-
-
 def update_info(details):
-    details['info']['updated_at'] = int(time.time())
+    details["info"]["updated_at"] = int(time.time())
-    details['info']['updated_at_readable'] = time.asctime()
+    details["info"]["updated_at_readable"] = time.asctime()
 
-    details['info']['t1_coins'] = len([True for _, c in details['coins'].items() if c.get('t1_enabled') == 'yes' and not c.get('hidden', False)])
+    details["info"]["t1_coins"] = len(
-    details['info']['t2_coins'] = len([True for _, c in details['coins'].items() if c.get('t2_enabled') == 'yes' and not c.get('hidden', False)])
+        [
+            True
+            for _, c in details["coins"].items()
+            if c.get("t1_enabled") == "yes" and not c.get("hidden", False)
+        ]
+    )
+    details["info"]["t2_coins"] = len(
+        [
+            True
+            for _, c in details["coins"].items()
+            if c.get("t2_enabled") == "yes" and not c.get("hidden", False)
+        ]
+    )
 
     try:
-        details['info']['total_marketcap_usd'] = int(coinmarketcap_global()['data']['quotes']['USD']['total_market_cap'])
+        details["info"]["total_marketcap_usd"] = int(
+            coinmarketcap_global()["data"]["quotes"]["USD"]["total_market_cap"]
+        )
     except:
         pass
 
     marketcap = 0
-    for k, c in details['coins'].items():
+    for k, c in details["coins"].items():
-        if c['t1_enabled'] == 'yes' or c['t2_enabled'] == 'yes':
+        if c["t1_enabled"] == "yes" or c["t2_enabled"] == "yes":
-            marketcap += details['coins'][k].setdefault('marketcap_usd', 0)
+            marketcap += details["coins"][k].setdefault("marketcap_usd", 0)
-    details['info']['marketcap_usd'] = marketcap
+    details["info"]["marketcap_usd"] = marketcap
 
 
 def check_unsupported(details, prefix, supported):
-    for k in details['coins'].keys():
+    for k in details["coins"].keys():
         if not k.startswith(prefix):
             continue
         if k not in supported:
             print("%s not supported by Trezor? (Possible manual entry)" % k)
 
 
-def update_coins(details):
+def _is_supported(support, trezor_version):
-    (coins, _) = build_coins.process(None)
+    version = VERSIONS[str(trezor_version)]
-    firmware = json.load(open('../defs/support.json', 'r'))
+    nominal = support.get("trezor" + str(trezor_version))
+    if nominal is None:
+        return "no"
+    elif isinstance(nominal, bool):
+        return "yes" if nominal else "no"
 
-    supported = []
+    try:
-    for key, coin in coins.items():
+        nominal_version = tuple(map(int, nominal.split(".")))
-
+        return "yes" if nominal_version <= version else "soon"
-        t1_enabled = 'no'
+    except ValueError:
-        if key in firmware['trezor1']:
+        return nominal
-            if LooseVersion(firmware['trezor1'][key]) <= LooseVersion(T1_LATEST):
-                t1_enabled = 'yes'
-            else:
-                t1_enabled = 'soon'
-
-        t2_enabled = 'no'
-        if key in firmware['trezor2']:
-            if LooseVersion(firmware['trezor2'][key]) <= LooseVersion(T2_LATEST):
-                t2_enabled = 'yes'
-            else:
-                t2_enabled = 'soon'
-
-        # print("Updating", coin['coin_label'], coin['coin_shortcut'])
-        key = "coin:%s" % coin['coin_shortcut']
-        supported.append(key)
-        out = details['coins'].setdefault(key, {})
-        out['type'] = 'coin'
-        out['t1_enabled'] = t1_enabled
-        out['t2_enabled'] = t2_enabled
-
-        set_default(out, 'shortcut', coin['coin_shortcut'])
-        set_default(out, 'name', coin['coin_label'])
-        set_default(out, 'links', {})
-        set_default(out['links'], 'Homepage', coin['website'])
-        set_default(out['links'], 'Github', coin['github'])
-        set_default(out, 'wallet', {})
-        update_marketcap(out, coin.get('coinmarketcap_alias', coin['coin_label']))
-
-    check_unsupported(details, 'coin:', supported)
 
 
-def update_erc20(details):
+def _webwallet_support(coin, support):
-    networks = [x[0] for x in ethereum_tokens_gen.networks]
+    """Check the "webwallet" support property.
+    If set, check that at least one of the backends run on trezor.io.
+    If yes, assume we support the coin in our wallet.
+    Otherwise it's probably working with a custom backend, which means don't
+    link to our wallet.
+    """
+    if not support.get("webwallet"):
+        return False
+    return any(".trezor.io" in url for url in coin["blockbook"] + coin["bitcore"])
 
-    LATEST_T1 = 'https://raw.githubusercontent.com/trezor/trezor-mcu/v%s/firmware/ethereum_tokens.c' % T1_LATEST
-    LATEST_T2 = 'https://raw.githubusercontent.com/trezor/trezor-core/v%s/src/apps/ethereum/tokens.py' % T2_LATEST
 
-    tokens = ethereum_tokens_gen.get_tokens()
+def update_coins(coins, support_info):
-    tokens_t1 = requests.get(LATEST_T1).text
+    res = {}
-    tokens_t2 = requests.get(LATEST_T2).text
+    for coin in coins:
+        key = coin["key"]
+        support = support_info[key]
+        details = dict(
+            type="coin",
+            shortcut=coin["shortcut"],
+            name=coin["coin_label"],
+            links=dict(Homepage=coin["website"], Github=coin["github"]),
+            t1_enabled=_is_supported(support, 1),
+            t2_enabled=_is_supported(support, 2),
+            wallet={},
+        )
+        if _webwallet_support(coin, support):
+            details["wallet"]["Trezor"] = "https://wallet.trezor.io"
+        if support.get("other"):
+            details["wallet"].update(support["other"])
 
-    supported = []
+        # XXX get rid of this in favor of res[key]
-    for t in tokens:
+        res["coin:{}".format(coin["shortcut"])] = details
-        # print('Updating', t['symbol'])
 
-        if t['chain'] not in networks:
+    return res
-            print('Skipping, %s is disabled' % t['chain'])
-            continue
 
-        key = "erc20:%s:%s" % (t['chain'], t['symbol'])
-        supported.append(key)
-        out = details['coins'].setdefault(key, {})
-        out['type'] = 'erc20'
-        out['network'] = t['chain']
-        out['address'] = t['address']
 
-        set_default(out, 'shortcut', t['symbol'])
+def update_erc20(coins, support_info):
-        set_default(out, 'name', t['name'])
+    # TODO skip disabled networks?
-        set_default(out, 'links', {})
+    res = {}
-        set_default(out, 'wallet', {})
+    for coin in coins:
+        key = coin["key"]
+        support = support_info[key]
+        details = dict(
+            type="erc20",
+            network=coin["chain"],
+            address=coin["address"],
+            shortcut=coin["shortcut"],
+            name=coin["name"],
+            links={},
+            wallet=dict(
+                MyCrypto="https://mycrypto.com",
+                MyEtherWallet="https://www.myetherwallet.com",
+            ),
+            t1_enabled=support["trezor1"],
+            t2_enabled=support["trezor2"],
+        )
+        if coin.get("website"):
+            details["links"]["Homepage"] = coin["website"]
+        if coin.get("social", {}).get("github"):
+            details["links"]["Github"] = coin["social"]["github"]
 
-        if "\" %s\"" % t['symbol'] in tokens_t1:
+        res[key] = details
-            out['t1_enabled'] = 'yes'
+
+    return res
+
+
+def update_simple(coins, support_info, type):
+    res = {}
+    for coin in coins:
+        key = coin["key"]
+        support = support_info[key]
+
+        # XXX drop newkey
+        if type == "mosaic":
+            newkey = "mosaic:{}".format(coin["shortcut"])
         else:
-            out['t1_enabled'] = 'soon'
+            newkey = "coin2:{}".format(coin["shortcut"])
 
-        if "'%s'" % t['symbol'] in tokens_t2:
+        details = dict(
-            out['t2_enabled'] = 'yes'
+            name=coin["name"],
-        else:
+            shortcut=coin["shortcut"],
-            out['t2_enabled'] = 'soon'
+            type=type,
+            t1_enabled=_is_supported(support, 1),
+            t2_enabled=_is_supported(support, 2),
+        )
+        for key in OPTIONAL_KEYS:
+            if key in coin:
+                details[key] = coin[key]
 
-        out['wallet']['MyCrypto'] = 'https://mycrypto.com'
+        res[newkey] = details
-        out['wallet']['MyEtherWallet'] = 'https://www.myetherwallet.com'
 
-        if t.get('website'):
+    return res
-            out['links']['Homepage'] = t['website']
-        if t.get('social', {}).get('github', None):
-            out['links']['Github'] = t['social']['github']
-
-        update_marketcap(out, out.get('coinmarketcap_alias'), t['name'], t['symbol'])
-
-    check_unsupported(details, 'erc20:', supported)
 
 
-def update_ethereum(details):
+def update_ethereum_networks(coins, support_info):
-    out = details['coins'].setdefault('coin2:ETH', {})
+    res = update_simple(coins, support_info, "coin")
-    out['type'] = 'coin'
+    for coin in coins:
-    set_default(out, 'shortcut', 'ETH')
+        newkey = "coin2:{}".format(coin["shortcut"])
-    set_default(out, 'name', 'Ethereum')
+        res[newkey]["wallet"] = dict(
-    set_default(out, 't1_enabled', 'yes')
+            MyCrypto="https://mycrypto.com",
-    set_default(out, 't2_enabled', 'yes')
+            MyEtherWallet="https://www.myetherwallet.com",
-    update_marketcap(out, 'ethereum')
+        )
-
+        res[newkey]["links"] = dict(Homepage=coin.get("url"))
-    out = details['coins'].setdefault('coin2:ETC', {})
+    return res
-    out['type'] = 'coin'
-    set_default(out, 'shortcut', 'ETC')
-    set_default(out, 'name', 'Ethereum Classic')
-    set_default(out, 't1_enabled', 'yes')
-    set_default(out, 't2_enabled', 'yes')
-    update_marketcap(out, 'ethereum-classic')
-
-    out = details['coins'].setdefault('coin2:RSK', {})
-    out['type'] = 'coin'
-    set_default(out, 'shortcut', 'RSK')
-    set_default(out, 'name', 'Rootstock')
-    set_default(out, 't1_enabled', 'yes')
-    set_default(out, 't2_enabled', 'yes')
-    update_marketcap(out, 'rootstock')
-
-    out = details['coins'].setdefault('coin2:EXP', {})
-    out['type'] = 'coin'
-    set_default(out, 'shortcut', 'EXP')
-    set_default(out, 'name', 'Expanse')
-    set_default(out, 't1_enabled', 'yes')
-    set_default(out, 't2_enabled', 'yes')
-    update_marketcap(out, 'expanse')
-
-    out = details['coins'].setdefault('coin2:UBQ', {})
-    out['type'] = 'coin'
-    set_default(out, 'shortcut', 'UBQ')
-    set_default(out, 'name', 'Ubiq')
-    set_default(out, 't1_enabled', 'yes')
-    set_default(out, 't2_enabled', 'yes')
-    update_marketcap(out, 'ubiq')
-
-    out = details['coins'].setdefault('coin2:ELLA', {})
-    out['type'] = 'coin'
-    set_default(out, 'shortcut', 'ELLA')
-    set_default(out, 'name', 'Ellaism')
-    set_default(out, 't1_enabled', 'yes')
-    set_default(out, 't2_enabled', 'yes')
-    update_marketcap(out, 'ellaism')
-
-    out = details['coins'].setdefault('coin2:EGEM', {})
-    out['type'] = 'coin'
-    set_default(out, 'shortcut', 'EGEM')
-    set_default(out, 'name', 'EtherGem')
-    set_default(out, 't1_enabled', 'yes')
-    set_default(out, 't2_enabled', 'yes')
-    update_marketcap(out, 'egem')
-
-    out = details['coins'].setdefault('coin2:ETSC', {})
-    out['type'] = 'coin'
-    set_default(out, 'shortcut', 'ETSC')
-    set_default(out, 'name', 'Ethereum Social')
-    set_default(out, 't1_enabled', 'yes')
-    set_default(out, 't2_enabled', 'yes')
-    update_marketcap(out, 'etsc')
-
-    out = details['coins'].setdefault('coin2:EOSC', {})
-    out['type'] = 'coin'
-    set_default(out, 'shortcut', 'EOSC')
-    set_default(out, 'name', 'EOS Classic')
-    set_default(out, 't1_enabled', 'yes')
-    set_default(out, 't2_enabled', 'yes')
-    update_marketcap(out, 'eosc')
-
-    out = details['coins'].setdefault('coin2:ADA', {})
-    out['type'] = 'coin'
-    set_default(out, 'shortcut', 'ADA')
-    set_default(out, 'name', 'Cardano')
-    set_default(out, 't1_enabled', 'no')
-    set_default(out, 't2_enabled', 'soon')
-    update_marketcap(out, 'cardano')
-
-    out = details['coins'].setdefault('coin2:XTZ', {})
-    out['type'] = 'coin'
-    set_default(out, 'shortcut', 'XTZ')
-    set_default(out, 'name', 'Tezos')
-    set_default(out, 't1_enabled', 'no')
-    set_default(out, 't2_enabled', 'soon')
-    update_marketcap(out, 'tezos')
-
-
-def update_mosaics(details):
-    d = json.load(open('../defs/nem/nem_mosaics.json'))
-    supported = []
-    for mosaic in d:
-        # print('Updating', mosaic['name'], mosaic['ticker'])
-
-        key = "mosaic:%s" % mosaic['ticker'].strip()
-        supported.append(key)
-        out = details['coins'].setdefault(key, {})
-        out['type'] = 'mosaic'
-        set_default(out, 'shortcut', mosaic['ticker'].strip())
-        set_default(out, 'name', mosaic['name'])
-        set_default(out, 't1_enabled', 'yes')
-        set_default(out, 't2_enabled', 'yes')
-
-        update_marketcap(out, out.get('coinmarketcap_alias', out['name']))
-
-    check_unsupported(details, 'mosaic:', supported)
 
 
 def check_missing_details(details):
-    for k in details['coins'].keys():
+    for k in details["coins"].keys():
-        coin = details['coins'][k]
+        coin = details["coins"][k]
         hide = False
 
-        if 'links' not in coin:
+        if "links" not in coin:
             print("%s: Missing links" % k)
             hide = True
-        if 'Homepage' not in coin.get('links', {}):
+        if "Homepage" not in coin.get("links", {}):
             print("%s: Missing homepage" % k)
             hide = True
-        if coin['t1_enabled'] not in ('yes', 'no', 'planned', 'soon'):
+        if coin["t1_enabled"] not in ("yes", "no", "planned", "soon"):
             print("%s: Unknown t1_enabled" % k)
             hide = True
-        if coin['t2_enabled'] not in ('yes', 'no', 'planned', 'soon'):
+        if coin["t2_enabled"] not in ("yes", "no", "planned", "soon"):
             print("%s: Unknown t2_enabled" % k)
             hide = True
-        if 'Trezor' in coin.get('wallet', {}) and coin['wallet']['Trezor'] != 'https://wallet.trezor.io':
+        if (
+            "Trezor" in coin.get("wallet", {})
+            and coin["wallet"]["Trezor"] != "https://wallet.trezor.io"
+        ):
             print("%s: Strange URL for Trezor Wallet" % k)
             hide = True
 
-        if len(coin.get('wallet', {})) == 0:
+        if len(coin.get("wallet", {})) == 0:
             print("%s: Missing wallet" % k)
 
+        if "Testnet" in coin["name"]:
+            print("%s: Hiding testnet" % k)
+            hide = True
+
         if hide:
-            if coin.get('hidden') != 1:
+            if coin.get("hidden") != 1:
                 print("%s: HIDING COIN!" % k)
 
             # If any of important detail is missing, hide coin from list
-            coin['hidden'] = 1
+            coin["hidden"] = 1
 
-        if not hide and coin.get('hidden'):
+        if not hide and coin.get("hidden"):
             print("%s: Details are OK, but coin is still hidden" % k)
 
-    for k in details['coins'].keys():
+    for k in details["coins"].keys():
-        if details['coins'][k].get('hidden') == 1:
+        if details["coins"][k].get("hidden") == 1:
             print("%s: Coin is hidden" % k)
 
 
-if __name__ == '__main__':
+def apply_overrides(coins):
-    try:
+    for key, override in OVERRIDES.items():
-        details = json.load(open('../defs/coins_details.json', 'r'))
+        if key not in coins:
-    except FileNotFoundError:
+            log.warning("override without coin: {}".format(key))
-        details = {'coins': {}, 'info': {}}
+            continue
+
+        def recursive_update(orig, new):
+            if isinstance(new, dict) and isinstance(orig, dict):
+                for k, v in new.items():
+                    orig[k] = recursive_update(orig.get(k), v)
+            else:
+                return new
+
+        coin = coins[key]
+        recursive_update(coin, override)
+
+
+if __name__ == "__main__":
+    # try:
+    #     details = json.load(open('../defs/coins_details.json', 'r'))
+    # except FileNotFoundError:
+    #     details = {'coins': {}, 'info': {}}
 
     coinmarketcap_init()
-    update_coins(details)
+
-    update_erc20(details)
+    defs = coin_defs.get_all()
-    update_ethereum(details)
+    all_coins = sum(defs.values(), [])
-    update_mosaics(details)
+    support_info = coin_defs.support_info(all_coins, erc20_versions=VERSIONS)
+
+    coins = {}
+    coins.update(update_coins(defs["btc"], support_info))
+    coins.update(update_erc20(defs["erc20"], support_info))
+    coins.update(update_ethereum_networks(defs["eth"], support_info))
+    coins.update(update_simple(defs["nem"], support_info, "mosaic"))
+    coins.update(update_simple(defs["other"], support_info, "coin"))
+
+    apply_overrides(coins)
+    update_marketcap(coins)
+
+    details = dict(coins=coins, info={})
     update_info(details)
     check_missing_details(details)
 
-    print(json.dumps(details['info'], sort_keys=True, indent=4))
+    print(json.dumps(details["info"], sort_keys=True, indent=4))
-    json.dump(details, open('../defs/coins_details.json', 'w'), sort_keys=True, indent=4)
+    json.dump(details, open("../defs/coins_details.json", "w"), sort_keys=True, indent=4)
+