feat(common): better checks for delisted tokens

common/defs/coins_details.override.json

@@ -21,5 +21,15 @@
         "wallet": {
             "MyEtherWallet": null
         }
+    },
+    "eth:EOS": {
+        "hidden": true,
+        "ignore_cmc_rank": true,
+        "reason": "this exists as misc:EOS and the eth: entry is probably a mistake"
+    },
+    "misc:LSK": {
+        "hidden": true,
+        "ignore_cmc_rank": true,
+        "reason": "delisted incompatible hardfork"
     }
 }

common/defs/duplicity_overrides.json

@@ -3,6 +3,8 @@
     "erc20:eth:BTL (Bitlle)": true,
     "erc20:eth:LINK Platform": true,
     "erc20:eth:NXX": false,
+    "erc20:eth:SNX:c011": false,
+    "erc20:eth:TUSD": false,
     "erc20:eth:Hdp": true,
     "erc20:eth:Hdp.ф": true,
     "misc:BNB": false,

common/tools/coins_details.py

@@ -259,13 +259,15 @@ def check_missing_data(coins):
             LOG.info(f"{k}: Details are OK, but coin is still hidden")
 
         if hide:
+            data = marketcap.get_coin(coin)
+            if data and data["cmc_rank"] < 150 and not coin.get("ignore_cmc_rank"):
+                LOG.warning(f"{k}: Hiding coin ranked {data['cmc_rank']} on CMC")
             coin["hidden"] = 1
 
     # summary of hidden coins
     hidden_coins = [k for k, coin in coins.items() if coin.get("hidden")]
     for key in hidden_coins:
         del coins[key]
-        LOG.debug(f"{key}: Coin is hidden")
 
 
 def apply_overrides(coins):
@@ -310,7 +312,7 @@ def main(refresh, api_key, verbose):
 
     marketcap.init(api_key, refresh=refresh)
 
-    defs = coin_info.coin_info()
+    defs, _ = coin_info.coin_info_with_duplicates()
     support_info = coin_info.support_info(defs)
 
     coins = {}

common/tools/marketcap.py

@@ -9,8 +9,7 @@ import requests
 COINMAKETCAP_CACHE = os.path.join(os.path.dirname(__file__), "coinmarketcap.json")
 COINMARKETCAP_API_BASE = "https://pro-api.coinmarketcap.com/v1/"
 
-MARKET_CAPS = {}
+COINS_SEARCHABLE = {}
-PRICES = {}
 
 
 def call(endpoint, api_key, params=None):
@@ -21,7 +20,7 @@ def call(endpoint, api_key, params=None):
 
 
 def init(api_key, refresh=None):
-    global MARKET_CAPS, PRICES
+    global COINS_SEARCHABLE
 
     force_refresh = refresh is True
     disable_refresh = refresh is False
@@ -60,41 +59,53 @@ def init(api_key, refresh=None):
     except Exception as e:
         raise RuntimeError("market cap data unavailable") from e
 
-    cap_data = {}
+    data_searchable = {}
-    price_data = {}
     for coin in coinmarketcap_data["data"]:
         slug = coin["slug"]
         symbol = coin["symbol"]
         platform = coin["meta"]["platform"]
-        market_cap = coin["quote"]["USD"]["market_cap"]
+        data_searchable[slug] = data_searchable[symbol] = coin
-        price = coin["quote"]["USD"]["price"]
+        if platform is not None and platform["name"] == "Ethereum":
-        if market_cap is not None:
+            address = platform["token_address"].lower()
-            cap_data[slug] = cap_data[symbol] = int(market_cap)
+            data_searchable[address] = coin
-            price_data[symbol] = price
+        for explorer in coin["meta"]["urls"]["explorer"]:
-            if platform is not None and platform["name"] == "Ethereum":
+            # some tokens exist in multiple places, such as BNB and ETH
-                address = platform["token_address"].lower()
+            # then the "platform" field might list the wrong thing
-                cap_data[address] = int(market_cap)
+            # to be sure, walk the list of explorers and look for etherscan.io
-                price_data[symbol] = price
+            if explorer.startswith("https://etherscan.io/token/"):
+                address = explorer.rsplit("/", 1)[-1].lower()
+                data_searchable[address] = coin
 
-    MARKET_CAPS = cap_data
+    COINS_SEARCHABLE = data_searchable
-    PRICES = price_data
+
+
+def get_coin(coin):
+    if coin["type"] == "erc20":
+        address = coin["address"].lower()
+        return COINS_SEARCHABLE.get(address)
+
+    data = None
+    if "coinmarketcap_alias" in coin:
+        data = COINS_SEARCHABLE.get(coin["coinmarketcap_alias"])
+    if data is None:
+        slug = coin["name"].replace(" ", "-").lower()
+        data = COINS_SEARCHABLE.get(slug)
+    if data is None:
+        data = COINS_SEARCHABLE.get(coin["shortcut"].lower())
+    return data
 
 
 def marketcap(coin):
-    cap = None
+    data = get_coin(coin)
-    if coin["type"] == "erc20":
+    if data is None:
-        address = coin["address"].lower()
+        return None
-        return MARKET_CAPS.get(address)
 
-    if "coinmarketcap_alias" in coin:
+    return int(data["quote"]["USD"]["market_cap"])
-        cap = MARKET_CAPS.get(coin["coinmarketcap_alias"])
-    if cap is None:
-        slug = coin["name"].replace(" ", "-").lower()
-        cap = MARKET_CAPS.get(slug)
-    if cap is None:
-        cap = MARKET_CAPS.get(coin["shortcut"].lower())
-    return cap
 
 
 def fiat_price(coin_symbol):
-    return PRICES.get(coin_symbol)
+    data = COINS_SEARCHABLE.get(coin_symbol)
+    if data is None:
+        return None
+
+    return data["quote"]["USD"]["price"]