coins_details: update to new CMC API, improve script, refresh data

tools/coins_details.py

@@ -8,6 +8,8 @@ import requests
 import sys
 import coin_info
 
+import click
+
 LOG = logging.getLogger(__name__)
 
 OPTIONAL_KEYS = ("links", "notes", "wallet")
@@ -17,54 +19,55 @@ OVERRIDES = coin_info.load_json("coins_details.override.json")
 VERSIONS = coin_info.latest_releases()
 
 COINMAKETCAP_CACHE = os.path.join(os.path.dirname(__file__), "coinmarketcap.json")
+COINMARKETCAP_API_BASE = "https://pro-api.coinmarketcap.com/v1/"
 
-COINMARKETCAP_TICKERS_URL = (
+MARKET_CAPS = {}
-    "https://api.coinmarketcap.com/v2/ticker/?start={}&convert=USD&limit=100"
-)
-COINMARKETCAP_GLOBAL_URL = "https://api.coinmarketcap.com/v2/global/"
 
 
-def coinmarketcap_init():
+def coinmarketcap_call(endpoint, api_key, params=None):
+    url = COINMARKETCAP_API_BASE + endpoint
+    r = requests.get(url, params=params, headers={"X-CMC_PRO_API_KEY": api_key})
+    r.raise_for_status()
+    return r.json()
+
+
+def coinmarketcap_init(api_key, refresh=None):
+    global MARKET_CAPS
+
+    force_refresh = refresh is True
+    disable_refresh = refresh is False
     try:
         mtime = os.path.getmtime(COINMAKETCAP_CACHE)
-        if mtime > time.time() - 3600:
+        cache_is_fresh = mtime > time.time() - 3600
+        if disable_refresh or (cache_is_fresh and not force_refresh):
             print("Using cached market cap data")
             with open(COINMAKETCAP_CACHE) as f:
-                return json.load(f)
+                coinmarketcap_data = json.load(f)
-    except Exception:
+        else:
-        pass
+            print("Fetching market cap data")
+            coinmarketcap_data = coinmarketcap_call(
+                "cryptocurrency/listings/latest",
+                api_key,
+                params={"limit": 5000, "convert": "USD"},
+            )
+    except Exception as e:
+        raise RuntimeError("market cap data unavailable") from e
 
-    print("Updating coins from coinmarketcap")
-    total = None
-    ctr = 0
     coin_data = {}
+    for coin in coinmarketcap_data["data"]:
+        slug = coin["slug"]
+        market_cap = coin["quote"]["USD"]["market_cap"]
+        if market_cap is not None:
+            coin_data[slug] = int(market_cap)
 
-    while total is None or ctr < total:
+    MARKET_CAPS = coin_data
-        url = COINMARKETCAP_TICKERS_URL.format(ctr + 1)
-        data = requests.get(url).json()
-
-        if total is None:
-            total = data["metadata"]["num_cryptocurrencies"]
-        ctr += len(data["data"])
-
-        for coin in data["data"].values():
-            slug = coin["website_slug"]
-            market_cap = coin["quotes"]["USD"]["market_cap"]
-            if market_cap is not None:
-                coin_data[slug] = int(market_cap)
-
-        print("Fetched {} of {} coins".format(ctr, total))
-        time.sleep(1)
 
     with open(COINMAKETCAP_CACHE, "w") as f:
-        json.dump(coin_data, f)
+        json.dump(coinmarketcap_data, f)
 
     return coin_data
 
 
-MARKET_CAPS = coinmarketcap_init()
-
-
 def marketcap(coin):
     cap = None
     if "coinmarketcap_alias" in coin:
@@ -82,7 +85,7 @@ def update_marketcaps(coins):
         coin["marketcap_usd"] = marketcap(coin) or 0
 
 
-def summary(coins):
+def summary(coins, api_key):
     t1_coins = 0
     t2_coins = 0
     supported_marketcap = 0
@@ -101,8 +104,8 @@ def summary(coins):
 
     total_marketcap = None
     try:
-        ret = requests.get(COINMARKETCAP_GLOBAL_URL).json()
+        ret = coinmarketcap_call("global-metrics/quotes/latest", api_key)
-        total_marketcap = int(ret["data"]["quotes"]["USD"]["total_market_cap"])
+        total_marketcap = int(ret["data"]["quote"]["USD"]["total_market_cap"])
     except:
         pass
 
@@ -117,8 +120,8 @@ def summary(coins):
 
 
 def _is_supported(support, trezor_version):
-    version = VERSIONS[str(trezor_version)]
+    version = VERSIONS[trezor_version]
-    nominal = support.get("trezor" + str(trezor_version))
+    nominal = support.get(trezor_version)
     if nominal is None:
         return "no"
     elif isinstance(nominal, bool):
@@ -228,7 +231,6 @@ def check_missing_data(coins):
     for k, coin in coins.items():
         hide = False
 
-        print(coin)
         if "Homepage" not in coin.get("links", {}):
             print("%s: Missing homepage" % k)
             hide = True
@@ -271,7 +273,7 @@ def check_missing_data(coins):
 def apply_overrides(coins):
     for key, override in OVERRIDES.items():
         if key not in coins:
-            LOG.warning("override without coin: {}".format(key))
+            LOG.warning(f"override without coin: {key}")
             continue
 
         def recursive_update(orig, new):
@@ -286,7 +288,13 @@ def apply_overrides(coins):
         recursive_update(coin, override)
 
 
-if __name__ == "__main__":
+@click.command()
+# fmt: off
+@click.option("-r", "--refresh", "refresh", flag_value=True, help="Force refresh market cap info")
+@click.option("-R", "--no-refresh", "refresh", flag_value=False, help="Force use cached market cap info")
+@click.option("-A", "--api-key", required=True, envvar="COINMARKETCAP_API_KEY", help="Coinmarketcap API key")
+# fmt: on
+def main(refresh, api_key):
     # setup logging
     root = logging.getLogger()
     root.setLevel(logging.DEBUG)
@@ -294,6 +302,8 @@ if __name__ == "__main__":
     handler.setLevel(logging.DEBUG)
     root.addHandler(handler)
 
+    coinmarketcap_init(api_key, refresh=refresh)
+
     defs = coin_info.coin_info()
     support_info = coin_info.support_info(defs)
 
@@ -308,9 +318,13 @@ if __name__ == "__main__":
     update_marketcaps(coins)
     check_missing_data(coins)
 
-    info = summary(coins)
+    info = summary(coins, api_key)
     details = dict(coins=coins, info=info)
 
     print(json.dumps(info, sort_keys=True, indent=4))
     with open(os.path.join(coin_info.DEFS_DIR, "coins_details.json"), "w") as f:
         json.dump(details, f, sort_keys=True, indent=4)
+
+
+if __name__ == "__main__":
+    main()