From fbacd4cc07bf356ee593ed5c76929a819de0dc6a Mon Sep 17 00:00:00 2001 From: Martin Novak Date: Mon, 5 Sep 2022 13:38:11 +0200 Subject: [PATCH] feat(common): added built-in Ethereum definitions --- common/defs/README.md | 41 ++--- common/defs/coins_details.override.json | 35 ---- common/defs/duplicity_overrides.json | 37 +---- .../defs/ethereum/eth_builtin_networks.json | 2 + common/defs/ethereum/eth_builtin_tokens.json | 2 + common/defs/wallets.json | 11 -- common/tools/README.md | 40 ++--- common/tools/coin_info.py | 96 +++++------ common/tools/coins_details.py | 42 ++--- common/tools/cointool.py | 29 +--- common/tools/ethereum_definitions.py | 150 +++++++++++------- common/tools/release.sh | 2 + 12 files changed, 197 insertions(+), 290 deletions(-) create mode 100644 common/defs/ethereum/eth_builtin_networks.json create mode 100644 common/defs/ethereum/eth_builtin_tokens.json diff --git a/common/defs/README.md b/common/defs/README.md index be106f076b..69f63027a4 100644 --- a/common/defs/README.md +++ b/common/defs/README.md @@ -33,19 +33,21 @@ Testnet is considered a separate coin, so it must have its own JSON and icon. We will not support coins that have `address_type` 0, i.e., same as Bitcoin. -#### `eth` +#### `eth` and `erc20` -The file [`ethereum/networks.json`](ethereum/networks.json) has a list of descriptions -of Ethereum networks. Each network must also have a PNG icon in `ethereum/.png` -file. +Definitions for Ethereum chains(networks) and tokens(erc20) are splitted in two parts: +1. built-in definitions - some of the chain and token definitions are built into the firmware + image. List of built-in chains is stored in [`ethereum/eth_builtin_networks.json`](ethereum/eth_builtin_networks.json) + and tokens in [`ethereum/eth_builtin_tokens.json`](ethereum/eth_builtin_tokens.json). +2. external definitions - external definitions are dynamically generated from multiple + sources - [`coingecko.com`](https://www.coingecko.com/), [defillama](https://defillama.com/), + [Ethereum Lists - chains](https://github.com/ethereum-lists/chains) + and [Ethereum Lists - tokens](https://github.com/ethereum-lists/tokens). + They are re-generated and signed on every release of the firmware. Signed + definitions are available at #TODO: add link!! -#### `erc20` - -`ethereum/tokens` is a submodule linking to [Ethereum Lists](https://github.com/ethereum-lists/tokens) -project with descriptions of ERC20 tokens. If you want to add or update a token -definition in Trezor, you need to get your change to the `tokens` repository first. - -Trezor will only support tokens that have a unique symbol. +If you want to add or update a token definition in Trezor, you need to get your change +to the [Ethereum Lists - tokens](https://github.com/ethereum-lists/tokens) repository first. #### `nem` @@ -71,12 +73,12 @@ generated from the coin's type and shortcut: If a token shortcut has a suffix, such as `CAT (BlockCat)`, the whole thing is part of the key (so the key is `erc20:eth:CAT (BlockCat)`). -Sometimes coins end up with duplicate symbols, which in case of ERC20 tokens leads to -key collisions. We do not allow duplicate symbols in the data, so this doesn't affect -everyday use (see below). However, for validation purposes, it is sometimes useful -to work with unfiltered data that includes the duplicates. In such cases, keys are -deduplicated by adding a counter at end, e.g.: `erc20:eth:SMT:0`, `erc20:eth:SMT:1`. -Note that the suffix _is not stable_, so these coins can't be reliably uniquely identified. +Sometimes coins end up with duplicate symbols (keys), which in case of ERC20 tokens leads to +key collisions. We do not allow duplicate symbols in the built-in data, so this doesn't affect +everyday use (see below). + +External definitions are generated based on data from external APIs on which we rely to have +the collisions solved. ## Duplicate Detection @@ -84,6 +86,9 @@ Note that the suffix _is not stable_, so these coins can't be reliably uniquely are removed from the data set before processing. The duplicate status is mentioned in `support.json` (see below), but it is impossible to override from there. +We try to minimize the occurence of duplicates in built-in tokens, but sometimes its +unavoidable. + Duplicate detection works as follows: 1. a _symbol_ is split off from the shortcut string. E.g., for `CAT (BlockCat)`, symbol @@ -132,7 +137,7 @@ External contributors should not touch this file unless asked to. # Support Information -We keep track of support status of each coin over our devices. That is +We keep track of support status of each built-in coin over our devices. That is `trezor1` for Trezor One, `trezor2` for Trezor T, `connect` for [Connect](https://github.com/trezor/connect) and `suite` for [Trezor Suite](https://suite.trezor.io/). In further description, the word "device" applies to Connect and Suite as well. diff --git a/common/defs/coins_details.override.json b/common/defs/coins_details.override.json index 26675039fe..8710ff9bfd 100644 --- a/common/defs/coins_details.override.json +++ b/common/defs/coins_details.override.json @@ -1,42 +1,7 @@ { - "erc20:eth:BAT": { - "name": "Basic Attention Token" - }, - "erc20:eth:LINK (Chainlink)": { - "name": "Chainlink" - }, - "erc20:eth:SOL": { - "shortcut": "SOLA" - }, - "eth:CLO": { - "coinmarketcap_alias": "callisto-network" - }, - "eth:ESN": { - "coinmarketcap_alias": "ethersocial" - }, "nem:DIMTOK": { "coinmarketcap_alias": "dimcoin" }, - "eth:AUX": { - "links": { - "Github": "https://github.com/auxiliumglobal" - }, - "wallet": { - "MyEtherWallet": null - } - }, - "eth:EOS": { - "hidden": true, - "ignore_cmc_rank": true, - "reason": "this exists as misc:EOS and the eth: entry is probably a mistake" - }, - "eth:XDC": { - "wallet": { - "MyCrypto": null, - "MyEtherWallet": null, - "XDC Wallet": "https://wallet.xinfin.network" - } - }, "misc:LSK": { "hidden": true, "ignore_cmc_rank": true, diff --git a/common/defs/duplicity_overrides.json b/common/defs/duplicity_overrides.json index 51c0cedb44..d05c7b1886 100644 --- a/common/defs/duplicity_overrides.json +++ b/common/defs/duplicity_overrides.json @@ -1,38 +1,3 @@ { - "erc20:eth:BTL (Battle)": true, - "erc20:eth:BTL (Bitlle)": true, - "erc20:eth:LINK Platform": true, - "erc20:eth:NXX": false, - "erc20:eth:SNX:c011": false, - "erc20:eth:TUSD": false, - "erc20:eth:Hdp": true, - "erc20:eth:Hdp.ф": true, - "erc20:eth:HEX:2b59": false, - "erc20:eth:JOB:dfbc": false, - "misc:BNB": false, - "eth:BNB": false, - "eth:ONE:1666600000": false, - "eth:ONE:1666600001": false, - "eth:ONE:1666600002": false, - "eth:ONE:1666600003": false, - "eth:tGOR:5": false, - "eth:tGOR:420": false, - "eth:tCELO:44787": false, - "eth:tCELO:62320": false, - "eth:QKC:100000": false, - "eth:QKC:100001": false, - "eth:QKC:100002": false, - "eth:QKC:100003": false, - "eth:QKC:100004": false, - "eth:QKC:100005": false, - "eth:QKC:100006": false, - "eth:QKC:100007": false, - "eth:QKC:100008": false, - "eth:xDAI:100": false, - "eth:xDAI:200": false, - "eth:CPAY:3000": false, - "eth:CPAY:3001": false, - "eth:CPAY:21337": false, - "erc20:eth:USDT": false, - "erc20:avax:USDT": false + "misc:BNB": false } diff --git a/common/defs/ethereum/eth_builtin_networks.json b/common/defs/ethereum/eth_builtin_networks.json new file mode 100644 index 0000000000..0d4f101c7a --- /dev/null +++ b/common/defs/ethereum/eth_builtin_networks.json @@ -0,0 +1,2 @@ +[ +] diff --git a/common/defs/ethereum/eth_builtin_tokens.json b/common/defs/ethereum/eth_builtin_tokens.json new file mode 100644 index 0000000000..2c63c08510 --- /dev/null +++ b/common/defs/ethereum/eth_builtin_tokens.json @@ -0,0 +1,2 @@ +{ +} diff --git a/common/defs/wallets.json b/common/defs/wallets.json index 2b918515d3..b506151900 100644 --- a/common/defs/wallets.json +++ b/common/defs/wallets.json @@ -120,16 +120,5 @@ }, "bitcoin:ZCR": { "Electrum-ZCR": "https://github.com/zcore-coin/electrum-wallet/" - }, - "eth:WAN": { - "Wanchain Wallet": "https://www.wanchain.org/getstarted/" - }, - "eth:AUX": { - "MyEtherWallet": null - }, - "eth:XDC": { - "MyCrypto": null, - "MyEtherWallet": null, - "XDC Wallet": "https://wallet.xinfin.network" } } diff --git a/common/tools/README.md b/common/tools/README.md index 3cfe818772..500a16204b 100644 --- a/common/tools/README.md +++ b/common/tools/README.md @@ -21,7 +21,6 @@ the following commands: * **`check`**: check validity of json definitions and associated data. Used in CI. * **`dump`**: dump coin information, including support status, in JSON format. Various filtering options are available, check help for details. -* **`coindefs`**: generate signed protobuf definitions for Ethereum networks (chains) and tokens. Use `cointool.py command --help` to get more information on each command. @@ -52,6 +51,19 @@ Compares generated `coins_details.json` to the released version currently served on https://trezor.io/coins, in a format that is nicely readable to humans and hard(er) to mess up by diff. +### `ethereum_definitions.py` + +Script used to work with Ethereum definitions. + +Definitions for Ethereum chains(networks) and tokens(erc20) are dynamically generated +from multiple sources - [`coingecko.com`](https://www.coingecko.com/), [defillama](https://defillama.com/), +[Ethereum Lists - chains](https://github.com/ethereum-lists/chains) +and [Ethereum Lists - tokens](https://github.com/ethereum-lists/tokens). + +The following commands are available: +* **`prepare-definitions`**: collect and process definitions for Ethereum networks (chains) and tokens. +* **`sign-definitions`**: generate signed protobuf definitions for Ethereum networks (chains) and tokens. + ### `coin_info.py` In case where code generation with `cointool.py render` is impractical or not sufficient, @@ -129,26 +141,12 @@ Or mark them as unsupported explicitly. ## Releasing a new firmware -#### **Step 1:** update the tokens repo +#### **Step 1:** run the release script ```sh -pushd defs/ethereum/tokens -git checkout master -git pull -popd -git add defs/ethereum/tokens +./tools/release.sh ``` -#### **Step 2:** run the release flow - -```sh -./tools/support.py release 2 -``` - -The number `2` indicates that you are releasing Trezor 2. The version will be -automatically determined, based on currently released firmwares. Or you can explicitly -specify the version with `-r 2.1.0`. - All currently known unreleased ERC20 tokens are automatically set to the given version. All coins marked _soon_ are set to the current version. This is automatic - coins that @@ -169,13 +167,7 @@ Use `-g` or `--git-tag` to automatically tag the current `HEAD` with a version, XXX this should also commit the changes though, otherwise the tag will apply to the wrong commit. -#### **Step 3:** review and commit your changes +#### **Step 2:** review and commit your changes Use `git diff` to review changes made, commit and push. If you tagged the commit in the previous step, don't forget to `git push --tags` too. - -#### **Step 4:** update submodule in your target repository - -Go to `trezor-core` or `trezor-mcu` checkout and update the submodule. Checkout the -appropriate tag if you created it. If you're in `trezor-core`, run `make templates` -to update source files. diff --git a/common/tools/coin_info.py b/common/tools/coin_info.py index e673b0dae5..5cc1b11f30 100755 --- a/common/tools/coin_info.py +++ b/common/tools/coin_info.py @@ -118,7 +118,6 @@ class Coin(TypedDict): address: str address_bytes: bytes dup_key_nontoken: bool - deprecation: dict[str, str] # Special NEM fields ticker: str @@ -366,6 +365,41 @@ def _load_btc_coins() -> Coins: return coins +def _load_builtin_ethereum_networks() -> Coins: + """Load ethereum networks from `ethereum/eth_builtin_networks.json`""" + chains_data = load_json("ethereum", "eth_builtin_networks.json") + networks: Coins = [] + for chain_data in chains_data: + chain_data.update( + chain_id=str(chain_data["chain_id"]), + key=f"eth:{chain_data['shortcut']}", + ) + networks.append(cast(Coin, chain_data)) + + return networks + + +def _load_builtin_erc20_tokens() -> Coins: + """Load ERC20 tokens from `ethereum/eth_builtin_tokens.json`.""" + tokens_data = load_json("ethereum", "eth_builtin_tokens.json") + all_tokens: Coins = [] + + for chain_id_and_chain, tokens in tokens_data.items(): + chain_id, chain = chain_id_and_chain.split(";", maxsplit=1) + for token in tokens: + token.update( + chain=chain, + chain_id=chain_id, + address=token["address"].lower(), + address_bytes=bytes.fromhex(token["address"][2:]), + symbol=token["shortcut"], + key=f"erc20:{chain}:{token['shortcut']}", + ) + all_tokens.append(cast(Coin, token)) + + return all_tokens + + def _load_nem_mosaics() -> Coins: """Loads NEM mosaics from `nem/nem_mosaics.json`""" mosaics: Coins = load_json("nem/nem_mosaics.json") @@ -491,10 +525,6 @@ def support_info(coins: Iterable[Coin] | CoinsInfo | dict[str, Coin]) -> Support WALLET_SUITE = {"Trezor Suite": "https://suite.trezor.io"} WALLET_NEM = {"Nano Wallet": "https://nemplatform.com/wallets/#desktop"} -WALLETS_ETH_3RDPARTY = { - "MyEtherWallet": "https://www.myetherwallet.com", - "MyCrypto": "https://mycrypto.com", -} def get_wallet_data() -> WalletInfo: @@ -516,7 +546,6 @@ def _suite_support(coin: Coin, support: SupportInfoItem) -> bool: def wallet_info_single( support_data: SupportInfo, - eth_networks_support_data: SupportInfo, wallet_data: WalletInfo, coin: Coin, ) -> WalletItems: @@ -533,26 +562,16 @@ def wallet_info_single( if key.startswith("bitcoin:"): if _suite_support(coin, support_data[key]): wallets.update(WALLET_SUITE) - elif key.startswith("eth:"): - if support_data[key]["suite"]: - wallets.update(WALLET_SUITE) - else: - wallets.update(WALLETS_ETH_3RDPARTY) - elif key.startswith("erc20:"): - if eth_networks_support_data[coin["chain"]]["suite"]: - wallets.update(WALLET_SUITE) - else: - wallets.update(WALLETS_ETH_3RDPARTY) elif key.startswith("nem:"): wallets.update(WALLET_NEM) - elif key.startswith("misc:"): + elif key.startswith(("eth:", "erc20:", "misc:")): pass # no special logic here else: raise ValueError(f"Unknown coin category: {key}") # Add wallets from `wallets.json` # This must come last as it offers the ability to override existing wallets - # (for example with `"MyEtherWallet": null` we delete the MyEtherWallet from the coin) + # (for example with `"Trezor Suite": null` we delete the "Trezor Suite" from the coin) wallets.update(wallet_data.get(key, {})) # Removing potentially disabled wallets from the last step @@ -581,17 +600,9 @@ def wallet_info(coins: Iterable[Coin] | CoinsInfo | dict[str, Coin]) -> WalletIn support_data = support_info(coins) wallet_data = get_wallet_data() - # Needed to find out suitable wallets for all the erc20 coins (Suite vs 3rd party) - eth_networks = [coin for coin in coins if coin["key"].startswith("eth:")] - eth_networks_support_data = { - n["chain"]: support_data[n["key"]] for n in eth_networks - } - wallet: WalletInfo = {} for coin in coins: - wallet[coin["key"]] = wallet_info_single( - support_data, eth_networks_support_data, wallet_data, coin - ) + wallet[coin["key"]] = wallet_info_single(support_data, wallet_data, coin) return wallet @@ -671,8 +682,7 @@ def deduplicate_erc20(buckets: CoinBuckets, networks: Coins) -> None: are still considered duplicate.) 2. If there is only one "main" token in the bucket, the bucket is cleared. - That means that all other tokens must either be on testnets, or they must be marked - as deprecated, with a deprecation pointing to the "main" token. + That means that all other tokens must be on testnets. """ testnet_networks = {n["chain"] for n in networks if n["slip44"] == 1} @@ -697,25 +707,8 @@ def deduplicate_erc20(buckets: CoinBuckets, networks: Coins) -> None: # protected categories: testnets = [coin for coin in bucket if coin["chain"] in testnet_networks] - deprecated_by_same = [ - coin - for coin in bucket - if "deprecation" in coin - and any( - other["address"] == coin["deprecation"]["new_address"] - for other in bucket - ) - ] - remaining = [ - coin - for coin in bucket - if coin not in testnets and coin not in deprecated_by_same - ] + remaining = [coin for coin in bucket if coin not in testnets] if len(remaining) <= 1: - for coin in deprecated_by_same: - deprecated_symbol = "[deprecated] " + coin["symbol"] - coin["shortcut"] = coin["symbol"] = deprecated_symbol - coin["key"] += ":deprecated" clear_bucket(bucket) @@ -766,8 +759,8 @@ def collect_coin_info() -> CoinsInfo: """ all_coins = CoinsInfo( bitcoin=_load_btc_coins(), - eth=_load_ethereum_networks(), - erc20=_load_erc20_tokens(), + eth=_load_builtin_ethereum_networks(), + erc20=_load_builtin_erc20_tokens(), nem=_load_nem_mosaics(), misc=_load_misc(), ) @@ -803,7 +796,7 @@ def coin_info_with_duplicates() -> tuple[CoinsInfo, CoinBuckets]: coin_list = all_coins.as_list() # generate duplicity buckets based on shortcuts buckets = mark_duplicate_shortcuts(all_coins.as_list()) - # apply further processing to ERC20 tokens, generate deprecations etc. + # apply further processing to ERC20 tokens deduplicate_erc20(buckets, all_coins.eth) # ensure the whole list has unique keys (taking into account changes from deduplicate_erc20) deduplicate_keys(coin_list) @@ -820,9 +813,6 @@ def coin_info() -> CoinsInfo: Does not auto-delete duplicates. This should now be based on support info. """ all_coins, _ = coin_info_with_duplicates() - # all_coins["erc20"] = [ - # coin for coin in all_coins["erc20"] if not coin.get("duplicate") - # ] return all_coins diff --git a/common/tools/coins_details.py b/common/tools/coins_details.py index 806e99b091..e9ec83b9e2 100755 --- a/common/tools/coins_details.py +++ b/common/tools/coins_details.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -"""Fetch information about coins and tokens supported by Trezor and update it in coins_details.json.""" +"""Fetch information about coins supported by Trezor and update it in coins_details.json.""" import json import logging import os @@ -20,14 +20,6 @@ WALLETS = coin_info.load_json("wallets.json") OVERRIDES = coin_info.load_json("coins_details.override.json") VERSIONS = coin_info.latest_releases() -# automatic wallet entries -WALLET_SUITE = {"Trezor Suite": "https://suite.trezor.io"} -WALLET_NEM = {"Nano Wallet": "https://nemplatform.com/wallets/#desktop"} -WALLETS_ETH_3RDPARTY = { - "MyEtherWallet": "https://www.myetherwallet.com", - "MyCrypto": "https://mycrypto.com", -} - TREZOR_KNOWN_URLS = ( "https://suite.trezor.io", @@ -136,7 +128,7 @@ def update_bitcoin(coins, support_info): details = dict( name=coin["coin_label"], links=dict(Homepage=coin["website"], Github=coin["github"]), - wallet=WALLET_SUITE if _suite_support(coin, support) else {}, + wallet=coin_info.WALLET_SUITE if _suite_support(coin, support) else {}, ) dict_merge(res[key], details) @@ -152,30 +144,18 @@ def update_erc20(coins, networks, support_info): key = coin["key"] chain = coin["chain"] - hidden = False - if chain in network_testnets: - hidden = True - if "deprecation" in coin: - hidden = True - - if network_support.get(chain, {}).get("suite"): - wallets = WALLET_SUITE - else: - wallets = WALLETS_ETH_3RDPARTY - details = dict( network=chain, address=coin["address"], shortcut=coin["shortcut"], links={}, - wallet=wallets, ) - if hidden: + + if chain in network_testnets: details["hidden"] = True - if coin.get("website"): - details["links"]["Homepage"] = coin["website"] - if coin.get("social", {}).get("github"): - details["links"]["Github"] = coin["social"]["github"] + + if network_support.get(chain, {}).get("suite"): + details["wallets"] = coin_info.WALLET_SUITE dict_merge(res[key], details) @@ -186,11 +166,9 @@ def update_ethereum_networks(coins, support_info): res = update_simple(coins, support_info, "coin") for coin in coins: key = coin["key"] + details = dict(links=dict(Homepage=coin.get("url"))) if support_info[key].get("suite"): - wallets = WALLET_SUITE - else: - wallets = WALLETS_ETH_3RDPARTY - details = dict(links=dict(Homepage=coin.get("url")), wallet=wallets) + details["wallets"] = coin_info.WALLET_SUITE dict_merge(res[key], details) return res @@ -200,7 +178,7 @@ def update_nem_mosaics(coins, support_info): res = update_simple(coins, support_info, "mosaic") for coin in coins: key = coin["key"] - details = dict(wallet=WALLET_NEM) + details = dict(wallet=coin_info.WALLET_NEM) dict_merge(res[key], details) return res diff --git a/common/tools/cointool.py b/common/tools/cointool.py index 68960affc0..839297e5d3 100755 --- a/common/tools/cointool.py +++ b/common/tools/cointool.py @@ -283,11 +283,9 @@ def check_btc(coins: Coins) -> bool: return check_passed -def check_dups(buckets: CoinBuckets, print_at_level: int = logging.WARNING) -> bool: +def check_dups(buckets: CoinBuckets) -> bool: """Analyze and pretty-print results of `coin_info.mark_duplicate_shortcuts`. - `print_at_level` can be one of logging levels. - The results are buckets of colliding symbols. If the collision is only between ERC20 tokens, it's DEBUG. If the collision includes one non-token, it's INFO. @@ -350,10 +348,6 @@ def check_dups(buckets: CoinBuckets, print_at_level: int = logging.WARNING) -> b # At most 1 supported coin, at most 1 non-token. This is informational only. level = logging.DEBUG - # deciding whether to print - if level < print_at_level: - continue - if symbol == "_override": print_log(level, "force-set duplicates:", dup_str) else: @@ -599,9 +593,8 @@ def cli(colors: bool) -> None: # fmt: off @click.option("--backend/--no-backend", "-b", default=False, help="Check blockbook/bitcore responses") @click.option("--icons/--no-icons", default=True, help="Check icon files") -@click.option("-d", "--show-duplicates", type=click.Choice(("all", "nontoken", "errors")), default="errors", help="How much information about duplicate shortcuts should be shown.") # fmt: on -def check(backend: bool, icons: bool, show_duplicates: str) -> None: +def check(backend: bool, icons: bool) -> None: """Validate coin definitions. Checks that every btc-like coin is properly filled out, reports duplicate symbols, @@ -611,14 +604,7 @@ def check(backend: bool, icons: bool, show_duplicates: str) -> None: Uniformity check ignores NEM mosaics and ERC20 tokens, where non-uniformity is expected. - The `--show-duplicates` option can be set to: - - - all: all shortcut collisions are shown, including colliding ERC20 tokens - - - nontoken: only collisions that affect non-ERC20 coins are shown - - - errors: only collisions between non-ERC20 tokens are shown. This is the default, - as a collision between two or more non-ERC20 tokens is an error. + All shortcut collisions are shown, including colliding ERC20 tokens. In the output, duplicate ERC tokens will be shown in cyan; duplicate non-tokens in red. An asterisk (*) next to symbol name means that even though it was detected @@ -653,14 +639,7 @@ def check(backend: bool, icons: bool, show_duplicates: str) -> None: if not check_eth(defs.eth): all_checks_passed = False - if show_duplicates == "all": - dup_level = logging.DEBUG - elif show_duplicates == "nontoken": - dup_level = logging.INFO - else: - dup_level = logging.WARNING - print("Checking unexpected duplicates...") - if not check_dups(buckets, dup_level): + if not check_dups(buckets): all_checks_passed = False nontoken_dups = [coin for coin in defs.as_list() if "dup_key_nontoken" in coin] diff --git a/common/tools/ethereum_definitions.py b/common/tools/ethereum_definitions.py index 47a44a4a70..4a167a273b 100755 --- a/common/tools/ethereum_definitions.py +++ b/common/tools/ethereum_definitions.py @@ -18,7 +18,15 @@ import requests from requests.adapters import HTTPAdapter from urllib3.util.retry import Retry -from coin_info import Coin, CoinBuckets, Coins, coin_info, load_json +from coin_info import ( + Coin, + CoinBuckets, + Coins, + _load_builtin_erc20_tokens, + _load_builtin_ethereum_networks, + coin_info, + load_json, +) from trezorlib import protobuf from trezorlib.messages import ( EthereumDefinitionType, @@ -282,6 +290,29 @@ def _load_erc20_tokens_from_repo( return tokens +def remove_builtin_definitions(networks: List[Dict], tokens: List[Dict]) -> None: + builtin_networks = _load_builtin_ethereum_networks() + builtin_tokens = _load_builtin_erc20_tokens() + + networks_by_chain_id = defaultdict(list) + for n in networks: + networks_by_chain_id[n["chain_id"]].append(n) + + tokens_by_chain_id_and_address = defaultdict(list) + for t in tokens: + tokens_by_chain_id_and_address[(t["chain_id"], t["address"])].append(t) + + for bn in builtin_networks: + for n in networks_by_chain_id.get(int(bn["chain_id"]), []): + networks.remove(n) + + for bt in builtin_tokens: + for t in tokens_by_chain_id_and_address.get( + (int(bt["chain_id"]), bt["address"]), [] + ): + tokens.remove(t) + + def _set_definition_metadata( definition: Dict, old_definition: Dict | None = None, @@ -437,6 +468,7 @@ def check_definitions_list( def_name: str, interactive: bool, force: bool, + top100_coingecko_ids: List[str] | None = None, ) -> None: # store already processed definitions deleted_definitions: List[Dict] = [] @@ -460,7 +492,7 @@ def check_definitions_list( hash_dict_on_keys(d, main_keys): d for d in old_defs } - # mark all deleted, moved or changed definitions + # mark all resurrected, moved, modified or deleted definitions for old_def in old_defs: old_def_hash_only_main_keys = hash_dict_on_keys(old_def, main_keys) old_def_hash_no_metadata = hash_dict_on_keys(old_def, exclude_keys=["metadata"]) @@ -533,18 +565,17 @@ def check_definitions_list( if (orig_def, new_def) in modified_definitions: modified_definitions.remove((orig_def, new_def)) - no_of_changes = ( - len(moved_definitions) - + len(modified_definitions) - + len(deleted_definitions) - + len(resurrected_definitions) - ) - if no_of_changes > 0: - print(f"Number of changes: {no_of_changes}") + def any_in_top_100(*definitions) -> bool: + if definitions is not None: + for d in definitions: + if d is not None and d.get("coingecko_id") in top100_coingecko_ids: + return True + return False # go through changes and ask for confirmation for old_def, new_def, orig_def in moved_definitions: accept_change = True + print_change = any_in_top_100(old_def, new_def, orig_def) # if the change contains symbol change "--force" parameter must be used to be able to accept this change if ( orig_def is not None @@ -555,14 +586,19 @@ def check_definitions_list( "\nERROR: Symbol change in this definition! To be able to approve this change re-run with `--force` argument." ) accept_change = False + print_change = True - answer = print_definition_change( - def_name.upper(), - "MOVED", - old_def, - new_def, - orig_def, - prompt=interactive and accept_change, + answer = ( + print_definition_change( + def_name.upper(), + "MOVED", + old_def, + new_def, + orig_def, + prompt=interactive and accept_change, + ) + if print_change + else None ) if answer is False or answer is None and not accept_change: # revert change - replace "new_def" with "old_def" and "orig_def" @@ -587,19 +623,28 @@ def check_definitions_list( for old_def, new_def in modified_definitions: accept_change = True + print_change = any_in_top_100(old_def, new_def) # if the change contains symbol change "--force" parameter must be used to be able to accept this change - if old_def.get("shortcut") != new_def.get("shortcut") and not force: + if ( + old_def.get("shortcut") != new_def.get("shortcut") + and not force + ): print( "\nERROR: Symbol change in this definition! To be able to approve this change re-run with `--force` argument." ) accept_change = False + print_change = True - answer = print_definition_change( - def_name.upper(), - "MODIFIED", - old_def, - new_def, - prompt=interactive and accept_change, + answer = ( + print_definition_change( + def_name.upper(), + "MODIFIED", + old_def, + new_def, + prompt=interactive and accept_change, + ) + if print_change + else None ) if answer is False or answer is None and not accept_change: # revert change - replace "new_def" with "old_def" @@ -608,7 +653,8 @@ def check_definitions_list( for definition in deleted_definitions: if ( - print_definition_change( + any_in_top_100(definition) + and print_definition_change( def_name.upper(), "DELETED", definition, prompt=interactive ) is False @@ -621,7 +667,8 @@ def check_definitions_list( for definition in resurrected_definitions: if ( - print_definition_change( + any_in_top_100(definition) + and print_definition_change( def_name.upper(), "RESURRECTED", definition, prompt=interactive ) is not False @@ -630,19 +677,6 @@ def check_definitions_list( _set_definition_metadata(definition) -def filter_top100(top100: List[Dict], definitions: List[Dict]) -> List[Dict]: - top100_ids = [d["id"] for d in top100] - - return { - "networks": [ - n for n in definitions["networks"] if n.get("coingecko_id") in top100_ids - ], - "tokens": [ - t for t in definitions["tokens"] if t.get("coingecko_id") in top100_ids - ], - } - - def _load_prepared_definitions(): # set keys for the networks and tokens # networks.append(cast(Coin, network)) @@ -818,51 +852,57 @@ def prepare_definitions( # merge tokens tokens: List[Dict] = [] - tokens_by_chain_id_and_address = defaultdict(list) + cg_tokens_chain_id_and_address = [] for t in cg_tokens: if t not in tokens: # add only unique tokens tokens.append(t) - tokens_by_chain_id_and_address[(t["chain_id"], t["address"])].append(t) + cg_tokens_chain_id_and_address.append((t["chain_id"], t["address"])) for t in repo_tokens: if ( t not in tokens - and (t["chain_id"], t["address"]) not in tokens_by_chain_id_and_address + and (t["chain_id"], t["address"]) not in cg_tokens_chain_id_and_address ): - # add only unique tokens and CoinGecko tokens are preffered + # add only unique tokens and prefer CoinGecko in case of collision of chain id and token address tokens.append(t) - tokens_by_chain_id_and_address[(t["chain_id"], t["address"])].append(t) old_defs = None if deffile.exists(): # load old definitions old_defs = load_json(deffile) + remove_builtin_definitions(networks, tokens) + check_tokens_collisions( tokens, old_defs["tokens"] if old_defs is not None else None ) # map coingecko ids to tokens + tokens_by_chain_id_and_address = {(t["chain_id"], t["address"]): t for t in tokens} cg_coin_list = downloader.get_coingecko_coins_list() for coin in cg_coin_list: for platform_name, address in coin.get("platforms", dict()).items(): key = (coingecko_id_to_chain_id.get(platform_name), address) if key in tokens_by_chain_id_and_address: - for token in tokens_by_chain_id_and_address[key]: - token["coingecko_id"] = coin["id"] + tokens_by_chain_id_and_address[key]["coingecko_id"] = coin["id"] - # load top 100 definitions from CoinGecko - cg_top100 = downloader.get_coingecko_top100() + # load top 100 (by market cap) definitions from CoinGecko + cg_top100_ids = [d["id"] for d in downloader.get_coingecko_top100()] + + # save cache + downloader.save_cache() # check changes in definitions if old_defs is not None: - # filter to top 100 based on Coingecko market cap order - if not show_all: - old_defs = filter_top100(cg_top100, old_defs) - # check networks and tokens check_definitions_list( - old_defs["networks"], networks, ["chain_id"], "network", interactive, force + old_defs["networks"], + networks, + ["chain_id"], + "network", + interactive, + force, + cg_top100_ids if not show_all else None, ) check_definitions_list( old_defs["tokens"], @@ -871,6 +911,7 @@ def prepare_definitions( "token", interactive, force, + cg_top100_ids if not show_all else None, ) # sort networks and tokens @@ -888,9 +929,6 @@ def prepare_definitions( ) f.write("\n") - # save cache - downloader.save_cache() - # TODO: separate function to generate built-in defs??? diff --git a/common/tools/release.sh b/common/tools/release.sh index d7e4e4779a..d1c86dd2d4 100755 --- a/common/tools/release.sh +++ b/common/tools/release.sh @@ -32,3 +32,5 @@ make -C $HERE/../.. gen diff $CHECK_OUTPUT/pre.txt $CHECK_OUTPUT/post.txt $HERE/coins_details.py + +$HERE/ethereum_definitions.py prepare-definitions