2018-07-27 10:05:09 +00:00
#!/usr/bin/env python3
2018-09-14 13:57:11 +00:00
import fnmatch
2019-04-18 14:27:27 +00:00
import glob
2018-07-27 10:05:09 +00:00
import json
2018-08-23 11:05:41 +00:00
import logging
2018-07-27 10:05:09 +00:00
import os
2019-04-18 14:27:27 +00:00
import re
import sys
2018-08-23 11:05:41 +00:00
from collections import defaultdict
2018-07-30 12:45:01 +00:00
from hashlib import sha256
2018-07-27 10:05:09 +00:00
import click
2018-07-30 12:25:53 +00:00
import coin_info
2018-07-30 12:45:01 +00:00
2018-08-23 11:05:41 +00:00
try :
import termcolor
except ImportError :
termcolor = None
2018-07-27 10:05:09 +00:00
try :
import mako
import mako . template
from munch import Munch
CAN_RENDER = True
except ImportError :
CAN_RENDER = False
try :
import requests
except ImportError :
requests = None
try :
from PIL import Image
2018-07-30 15:28:20 +00:00
2021-05-31 13:11:05 +00:00
CAN_CHECK_ICONS = True
2018-07-27 10:05:09 +00:00
except ImportError :
2021-05-31 13:11:05 +00:00
CAN_CHECK_ICONS = False
2018-07-27 10:05:09 +00:00
2018-08-23 11:05:41 +00:00
# ======= Crayon colors ======
USE_COLORS = False
def crayon ( color , string , bold = False , dim = False ) :
if not termcolor or not USE_COLORS :
return string
else :
if bold :
attrs = [ " bold " ]
elif dim :
attrs = [ " dark " ]
else :
attrs = [ ]
return termcolor . colored ( string , color , attrs = attrs )
def print_log ( level , * args , * * kwargs ) :
prefix = logging . getLevelName ( level )
if level == logging . DEBUG :
prefix = crayon ( " blue " , prefix , bold = False )
elif level == logging . INFO :
prefix = crayon ( " blue " , prefix , bold = True )
elif level == logging . WARNING :
prefix = crayon ( " red " , prefix , bold = False )
elif level == logging . ERROR :
prefix = crayon ( " red " , prefix , bold = True )
print ( prefix , * args , * * kwargs )
2018-07-30 12:25:53 +00:00
# ======= Mako management ======
2018-07-27 10:05:09 +00:00
def c_str_filter ( b ) :
if b is None :
return " NULL "
def hexescape ( c ) :
2021-10-13 13:45:53 +00:00
return rf " \ x { c : 02x } "
2018-07-27 10:05:09 +00:00
if isinstance ( b , bytes ) :
return ' " ' + " " . join ( map ( hexescape , b ) ) + ' " '
else :
return json . dumps ( b )
2018-08-15 15:33:33 +00:00
def black_repr_filter ( val ) :
if isinstance ( val , str ) :
if ' " ' in val :
return repr ( val )
else :
return c_str_filter ( val )
elif isinstance ( val , bytes ) :
return " b " + c_str_filter ( val )
else :
return repr ( val )
2018-07-27 10:05:09 +00:00
def ascii_filter ( s ) :
return re . sub ( " [^ - \x7e ] " , " _ " , s )
2018-07-30 15:28:20 +00:00
def make_support_filter ( support_info ) :
def supported_on ( device , coins ) :
2021-09-20 10:37:56 +00:00
return ( c for c in coins if support_info [ c . key ] . get ( device ) )
2018-08-07 11:00:30 +00:00
2018-07-30 15:28:20 +00:00
return supported_on
2018-08-15 15:33:33 +00:00
MAKO_FILTERS = {
" c_str " : c_str_filter ,
" ascii " : ascii_filter ,
" black_repr " : black_repr_filter ,
}
2018-07-27 10:05:09 +00:00
2018-08-07 11:00:30 +00:00
def render_file ( src , dst , coins , support_info ) :
""" Renders `src` template into `dst`.
` src ` is a filename , ` dst ` is an open file object .
"""
template = mako . template . Template ( filename = src )
2018-07-30 15:28:20 +00:00
result = template . render (
support_info = support_info ,
supported_on = make_support_filter ( support_info ) ,
* * coins ,
2018-08-15 15:33:33 +00:00
* * MAKO_FILTERS ,
2018-07-30 15:28:20 +00:00
)
2018-08-07 11:00:30 +00:00
dst . write ( result )
2018-07-27 10:05:09 +00:00
# ====== validation functions ======
2019-04-29 16:04:19 +00:00
def mark_unsupported ( support_info , coins ) :
for coin in coins :
key = coin [ " key " ]
# checking for explicit False because None means unknown
coin [ " unsupported " ] = all ( v is False for v in support_info [ key ] . values ( ) )
2018-08-23 11:05:41 +00:00
def highlight_key ( coin , color ) :
2018-08-23 17:32:28 +00:00
""" Return a colorful string where the SYMBOL part is bold. """
2018-08-23 11:05:41 +00:00
keylist = coin [ " key " ] . split ( " : " )
if keylist [ - 1 ] . isdigit ( ) :
keylist [ - 2 ] = crayon ( color , keylist [ - 2 ] , bold = True )
else :
keylist [ - 1 ] = crayon ( color , keylist [ - 1 ] , bold = True )
key = crayon ( color , " : " . join ( keylist ) )
2021-09-27 10:13:51 +00:00
name = crayon ( None , f " ( { coin [ ' name ' ] } ) " , dim = True )
return f " { key } { name } "
2018-08-23 11:05:41 +00:00
2018-09-07 11:17:47 +00:00
def find_collisions ( coins , field ) :
2018-08-23 11:05:41 +00:00
""" Detects collisions in a given field. Returns buckets of colliding coins. """
collisions = defaultdict ( list )
for coin in coins :
2019-11-26 17:11:09 +00:00
values = coin [ field ]
if not isinstance ( values , list ) :
values = [ values ]
for value in values :
collisions [ value ] . append ( coin )
2018-08-23 11:05:41 +00:00
return { k : v for k , v in collisions . items ( ) if len ( v ) > 1 }
2018-09-07 11:17:47 +00:00
def check_eth ( coins ) :
check_passed = True
chains = find_collisions ( coins , " chain " )
for key , bucket in chains . items ( ) :
2021-09-27 10:13:51 +00:00
bucket_str = " , " . join ( f " { coin [ ' key ' ] } ( { coin [ ' name ' ] } ) " for coin in bucket )
2018-09-07 11:17:47 +00:00
chain_name_str = " colliding chain name " + crayon ( None , key , bold = True ) + " : "
print_log ( logging . ERROR , chain_name_str , bucket_str )
check_passed = False
return check_passed
2018-07-27 10:05:09 +00:00
def check_btc ( coins ) :
check_passed = True
2018-08-23 17:32:28 +00:00
# validate individual coin data
2018-07-27 10:05:09 +00:00
for coin in coins :
2018-07-30 12:25:53 +00:00
errors = coin_info . validate_btc ( coin )
2018-07-27 10:05:09 +00:00
if errors :
check_passed = False
2018-08-23 11:05:41 +00:00
print_log ( logging . ERROR , " invalid definition for " , coin [ " name " ] )
2018-07-27 10:05:09 +00:00
print ( " \n " . join ( errors ) )
2018-08-23 11:05:41 +00:00
def collision_str ( bucket ) :
2018-08-23 17:32:28 +00:00
""" Generate a colorful string out of a bucket of colliding coins. """
2018-08-23 11:05:41 +00:00
coin_strings = [ ]
for coin in bucket :
name = coin [ " name " ]
prefix = " "
2019-04-27 14:15:35 +00:00
if name . endswith ( " Testnet " ) or name . endswith ( " Regtest " ) :
2018-08-23 11:05:41 +00:00
color = " green "
elif name == " Bitcoin " :
color = " red "
2019-04-29 16:04:19 +00:00
elif coin [ " unsupported " ] :
2018-08-23 11:05:41 +00:00
color = " grey "
prefix = crayon ( " blue " , " (X) " , bold = True )
else :
color = " blue "
hl = highlight_key ( coin , color )
coin_strings . append ( prefix + hl )
return " , " . join ( coin_strings )
2018-12-17 22:11:30 +00:00
def print_collision_buckets ( buckets , prefix , maxlevel = logging . ERROR , strict = False ) :
2018-08-23 17:32:28 +00:00
""" Intelligently print collision buckets.
For each bucket , if there are any collision with a mainnet , print it .
If the collision is with unsupported networks or testnets , it ' s just INFO.
If the collision is with supported mainnets , it ' s WARNING.
If the collision with any supported network includes Bitcoin , it ' s an ERROR.
"""
2018-08-23 11:05:41 +00:00
failed = False
for key , bucket in buckets . items ( ) :
2019-04-27 14:15:35 +00:00
mainnets = [
c
for c in bucket
if not c [ " name " ] . endswith ( " Testnet " )
and not c [ " name " ] . endswith ( " Regtest " )
]
2018-08-23 11:05:41 +00:00
2019-04-29 16:04:19 +00:00
have_bitcoin = any ( coin [ " name " ] == " Bitcoin " for coin in mainnets )
supported_mainnets = [ c for c in mainnets if not c [ " unsupported " ] ]
supported_networks = [ c for c in bucket if not c [ " unsupported " ] ]
2018-08-23 11:05:41 +00:00
if len ( mainnets ) > 1 :
2018-12-17 22:11:30 +00:00
if ( have_bitcoin or strict ) and len ( supported_networks ) > 1 :
2018-08-23 17:09:35 +00:00
# ANY collision with Bitcoin is bad
2018-08-28 11:56:53 +00:00
level = maxlevel
2018-08-23 17:09:35 +00:00
failed = True
elif len ( supported_mainnets ) > 1 :
# collision between supported networks is still pretty bad
level = logging . WARNING
2018-08-23 11:05:41 +00:00
else :
2018-08-23 17:09:35 +00:00
# collision between some unsupported networks is OK
2018-08-23 11:05:41 +00:00
level = logging . INFO
2021-09-27 10:13:51 +00:00
print_log ( level , f " { prefix } { key } : " , collision_str ( bucket ) )
2018-08-23 11:05:41 +00:00
return failed
# slip44 collisions
2018-12-17 22:11:30 +00:00
print ( " Checking SLIP44 values collisions... " )
2018-09-07 11:17:47 +00:00
slip44 = find_collisions ( coins , " slip44 " )
2018-12-17 22:11:30 +00:00
if print_collision_buckets ( slip44 , " value " , strict = True ) :
2018-08-23 11:05:41 +00:00
check_passed = False
2018-08-23 17:32:28 +00:00
# only check address_type on coins that don't use cashaddr
2018-08-23 11:05:41 +00:00
nocashaddr = [ coin for coin in coins if not coin . get ( " cashaddr_prefix " ) ]
2018-08-28 11:56:53 +00:00
2018-08-23 11:05:41 +00:00
print ( " Checking address_type collisions... " )
2018-09-07 11:17:47 +00:00
address_type = find_collisions ( nocashaddr , " address_type " )
2018-08-23 11:05:41 +00:00
if print_collision_buckets ( address_type , " address type " ) :
check_passed = False
print ( " Checking address_type_p2sh collisions... " )
2018-09-07 11:17:47 +00:00
address_type_p2sh = find_collisions ( nocashaddr , " address_type_p2sh " )
2018-08-23 11:05:41 +00:00
# we ignore failed checks on P2SH, because reasons
2018-08-28 11:56:53 +00:00
print_collision_buckets ( address_type_p2sh , " address type " , logging . WARNING )
2018-07-27 10:05:09 +00:00
2019-01-16 12:52:18 +00:00
print ( " Checking genesis block collisions... " )
genesis = find_collisions ( coins , " hash_genesis_block " )
print_collision_buckets ( genesis , " genesis block " , logging . WARNING )
2018-07-27 10:05:09 +00:00
return check_passed
2019-04-29 16:04:19 +00:00
def check_dups ( buckets , print_at_level = logging . WARNING ) :
2018-08-23 17:32:28 +00:00
""" Analyze and pretty-print results of `coin_info.mark_duplicate_shortcuts`.
` print_at_level ` can be one of logging levels .
The results are buckets of colliding symbols .
If the collision is only between ERC20 tokens , it ' s DEBUG.
If the collision includes one non - token , it ' s INFO.
If the collision includes more than one non - token , it ' s ERROR and printed always.
"""
2018-08-28 11:56:53 +00:00
2018-08-23 11:05:41 +00:00
def coin_str ( coin ) :
2018-08-23 17:32:28 +00:00
""" Colorize coins. Tokens are cyan, nontokens are red. Coins that are NOT
marked duplicate get a green asterisk .
"""
2019-04-29 16:04:19 +00:00
prefix = " "
if coin [ " unsupported " ] :
color = " grey "
prefix = crayon ( " blue " , " (X) " , bold = True )
elif coin_info . is_token ( coin ) :
2018-08-23 11:05:41 +00:00
color = " cyan "
else :
color = " red "
2019-04-29 16:04:19 +00:00
2018-08-23 11:05:41 +00:00
if not coin . get ( " duplicate " ) :
2019-04-29 16:04:19 +00:00
prefix = crayon ( " green " , " * " , bold = True ) + prefix
highlighted = highlight_key ( coin , color )
2021-09-27 10:13:51 +00:00
return f " { prefix } { highlighted } "
2018-08-23 11:05:41 +00:00
2018-08-15 17:20:15 +00:00
check_passed = True
2018-08-23 11:05:41 +00:00
for symbol in sorted ( buckets . keys ( ) ) :
bucket = buckets [ symbol ]
if not bucket :
2018-08-15 17:20:15 +00:00
continue
2018-08-23 11:05:41 +00:00
2019-04-29 16:04:19 +00:00
supported = [ coin for coin in bucket if not coin [ " unsupported " ] ]
2020-11-10 15:05:40 +00:00
nontokens = [
coin
for coin in bucket
2020-11-11 10:41:33 +00:00
if not coin [ " unsupported " ]
and coin . get ( " duplicate " )
and not coin_info . is_token ( coin )
2020-11-10 15:05:40 +00:00
] # we do not count override-marked coins as duplicates here
2019-04-29 16:04:19 +00:00
cleared = not any ( coin . get ( " duplicate " ) for coin in bucket )
2018-08-23 11:05:41 +00:00
# string generation
dup_str = " , " . join ( coin_str ( coin ) for coin in bucket )
2019-04-29 16:04:19 +00:00
if len ( nontokens ) > 1 :
# Two or more colliding nontokens. This is always fatal.
# XXX consider allowing two nontokens as long as only one is supported?
2018-08-23 11:05:41 +00:00
level = logging . ERROR
2018-08-15 17:20:15 +00:00
check_passed = False
2019-04-29 16:04:19 +00:00
elif len ( supported ) > 1 :
# more than one supported coin in bucket
if cleared :
# some previous step has explicitly marked them as non-duplicate
level = logging . INFO
else :
2020-11-10 15:05:40 +00:00
# at most 1 non-token - we tentatively allow token collisions
2019-04-29 16:04:19 +00:00
# when explicitly marked as supported
level = logging . WARNING
else :
# At most 1 supported coin, at most 1 non-token. This is informational only.
level = logging . DEBUG
2018-08-15 17:20:15 +00:00
2018-08-23 11:05:41 +00:00
# deciding whether to print
2018-08-23 17:32:28 +00:00
if level < print_at_level :
2018-08-23 11:05:41 +00:00
continue
if symbol == " _override " :
print_log ( level , " force-set duplicates: " , dup_str )
else :
2021-09-27 10:13:51 +00:00
print_log ( level , f " duplicate symbol { symbol . upper ( ) } : " , dup_str )
2018-08-23 11:05:41 +00:00
2018-08-15 17:20:15 +00:00
return check_passed
2018-07-27 10:05:09 +00:00
def check_backends ( coins ) :
check_passed = True
for coin in coins :
genesis_block = coin . get ( " hash_genesis_block " )
if not genesis_block :
continue
backends = coin . get ( " blockbook " , [ ] ) + coin . get ( " bitcore " , [ ] )
for backend in backends :
print ( " checking " , backend , " ... " , end = " " , flush = True )
try :
2018-07-30 12:25:53 +00:00
j = requests . get ( backend + " /api/block-index/0 " ) . json ( )
2018-07-27 10:05:09 +00:00
if j [ " blockHash " ] != genesis_block :
raise RuntimeError ( " genesis block mismatch " )
except Exception as e :
print ( e )
check_passed = False
else :
print ( " OK " )
return check_passed
2018-07-30 12:45:01 +00:00
def check_icons ( coins ) :
check_passed = True
for coin in coins :
key = coin [ " key " ]
icon_file = coin . get ( " icon " )
if not icon_file :
print ( key , " : missing icon " )
check_passed = False
continue
try :
icon = Image . open ( icon_file )
except Exception :
print ( key , " : failed to open icon file " , icon_file )
check_passed = False
continue
if icon . size != ( 96 , 96 ) or icon . mode != " RGBA " :
print ( key , " : bad icon format (must be RGBA 96x96) " )
check_passed = False
return check_passed
2018-11-20 16:51:04 +00:00
IGNORE_NONUNIFORM_KEYS = frozenset ( ( " unsupported " , " duplicate " ) )
2018-08-23 17:09:54 +00:00
def check_key_uniformity ( coins ) :
keysets = defaultdict ( list )
for coin in coins :
keyset = frozenset ( coin . keys ( ) ) | IGNORE_NONUNIFORM_KEYS
keysets [ keyset ] . append ( coin )
if len ( keysets ) < = 1 :
return True
buckets = list ( keysets . values ( ) )
buckets . sort ( key = lambda x : len ( x ) )
majority = buckets [ - 1 ]
rest = sum ( buckets [ : - 1 ] , [ ] )
2018-11-20 16:51:04 +00:00
reference_keyset = set ( majority [ 0 ] . keys ( ) ) | IGNORE_NONUNIFORM_KEYS
print ( reference_keyset )
2018-08-23 17:09:54 +00:00
for coin in rest :
key = coin [ " key " ]
2018-11-20 16:51:04 +00:00
keyset = set ( coin . keys ( ) ) | IGNORE_NONUNIFORM_KEYS
2018-08-23 17:09:54 +00:00
missing = " , " . join ( reference_keyset - keyset )
if missing :
2021-09-27 10:13:51 +00:00
print_log ( logging . ERROR , f " coin { key } has missing keys: { missing } " )
2018-08-23 17:09:54 +00:00
additional = " , " . join ( keyset - reference_keyset )
if additional :
2018-11-29 14:42:49 +00:00
print_log (
logging . ERROR ,
2021-09-27 10:13:51 +00:00
f " coin { key } has superfluous keys: { additional } " ,
2018-11-29 14:42:49 +00:00
)
2018-08-23 17:09:54 +00:00
return False
2019-02-12 09:56:34 +00:00
def check_segwit ( coins ) :
for coin in coins :
segwit = coin [ " segwit " ]
2019-03-26 11:22:57 +00:00
segwit_fields = [
" bech32_prefix " ,
" xpub_magic_segwit_native " ,
" xpub_magic_segwit_p2sh " ,
2021-01-13 21:58:07 +00:00
" xpub_magic_multisig_segwit_native " ,
" xpub_magic_multisig_segwit_p2sh " ,
2019-03-26 11:22:57 +00:00
]
2019-02-12 09:56:34 +00:00
if segwit :
2019-03-26 11:22:57 +00:00
for field in segwit_fields :
if coin [ field ] is None :
print_log (
logging . ERROR ,
coin [ " name " ] ,
2021-09-27 10:13:51 +00:00
f " segwit is True => { field } should be set " ,
2019-03-26 11:22:57 +00:00
)
return False
2019-02-12 09:56:34 +00:00
else :
2019-03-26 11:22:57 +00:00
for field in segwit_fields :
if coin [ field ] is not None :
print_log (
logging . ERROR ,
coin [ " name " ] ,
2021-09-27 10:13:51 +00:00
f " segwit is False => { field } should NOT be set " ,
2019-03-26 11:22:57 +00:00
)
return False
2019-02-12 09:56:34 +00:00
return True
2019-11-26 17:11:09 +00:00
FIDO_KNOWN_KEYS = frozenset (
2019-12-11 10:47:02 +00:00
(
" key " ,
" u2f " ,
" webauthn " ,
2020-07-30 10:11:26 +00:00
" name " ,
2019-12-11 10:47:02 +00:00
" use_sign_count " ,
" use_self_attestation " ,
" no_icon " ,
" icon " ,
)
2019-11-26 17:11:09 +00:00
)
def check_fido ( apps ) :
check_passed = True
2020-07-30 10:11:26 +00:00
u2fs = find_collisions ( ( u for a in apps if " u2f " in a for u in a [ " u2f " ] ) , " app_id " )
for key , bucket in u2fs . items ( ) :
bucket_str = " , " . join ( u2f [ " label " ] for u2f in bucket )
app_id_str = " colliding U2F app ID " + crayon ( None , key , bold = True ) + " : "
print_log ( logging . ERROR , app_id_str , bucket_str )
2019-11-26 17:11:09 +00:00
check_passed = False
webauthn_domains = find_collisions ( ( a for a in apps if " webauthn " in a ) , " webauthn " )
for key , bucket in webauthn_domains . items ( ) :
bucket_str = " , " . join ( app [ " key " ] for app in bucket )
webauthn_str = " colliding WebAuthn domain " + crayon ( None , key , bold = True ) + " : "
print_log ( logging . ERROR , webauthn_str , bucket_str )
check_passed = False
2020-07-30 10:16:49 +00:00
domain_hashes = { }
for app in apps :
if " webauthn " in app :
for domain in app [ " webauthn " ] :
domain_hashes [ sha256 ( domain . encode ( ) ) . digest ( ) ] = domain
for app in apps :
if " u2f " in app :
for u2f in app [ " u2f " ] :
domain = domain_hashes . get ( bytes . fromhex ( u2f [ " app_id " ] ) )
if domain :
print_log (
logging . ERROR ,
" colliding WebAuthn domain "
+ crayon ( None , domain , bold = True )
+ " and U2F app_id "
+ crayon ( None , u2f [ " app_id " ] , bold = True )
+ " for "
+ u2f [ " label " ] ,
)
check_passed = False
2019-11-26 17:11:09 +00:00
for app in apps :
2020-07-30 10:11:26 +00:00
if " name " not in app :
print_log ( logging . ERROR , app [ " key " ] , " : missing name " )
2019-11-26 17:11:09 +00:00
check_passed = False
2020-07-30 10:11:26 +00:00
if " u2f " in app :
for u2f in app [ " u2f " ] :
if " app_id " not in u2f :
print_log ( logging . ERROR , app [ " key " ] , " : missing app_id " )
check_passed = False
if " label " not in u2f :
print_log ( logging . ERROR , app [ " key " ] , " : missing label " )
check_passed = False
2019-11-26 17:11:09 +00:00
if not app . get ( " u2f " ) and not app . get ( " webauthn " ) :
print_log ( logging . ERROR , app [ " key " ] , " : no U2F nor WebAuthn addresses " )
check_passed = False
unknown_keys = set ( app . keys ( ) ) - FIDO_KNOWN_KEYS
if unknown_keys :
print_log ( logging . ERROR , app [ " key " ] , " : unrecognized keys: " , unknown_keys )
2019-11-28 13:00:13 +00:00
check_passed = False
2019-11-26 17:11:09 +00:00
# check icons
2019-11-28 13:00:13 +00:00
if app [ " icon " ] is None :
2019-12-09 14:23:06 +00:00
if app . get ( " no_icon " ) :
continue
print_log ( logging . ERROR , app [ " key " ] , " : missing icon " )
check_passed = False
2019-11-28 13:00:13 +00:00
continue
2019-12-09 14:23:06 +00:00
elif app . get ( " no_icon " ) :
print_log ( logging . ERROR , app [ " key " ] , " : icon present for ' no_icon ' app " )
check_passed = False
2019-11-28 13:00:13 +00:00
try :
icon = Image . open ( app [ " icon " ] )
except Exception :
2019-12-09 14:23:06 +00:00
print_log (
logging . ERROR , app [ " key " ] , " : failed to open icon file " , app [ " icon " ]
)
2019-11-28 13:00:13 +00:00
check_passed = False
2019-11-26 17:11:09 +00:00
continue
if icon . size != ( 128 , 128 ) or icon . mode != " RGBA " :
print_log (
logging . ERROR , app [ " key " ] , " : bad icon format (must be RGBA 128x128) "
)
check_passed = False
2019-12-09 14:23:06 +00:00
2019-11-26 17:11:09 +00:00
return check_passed
2018-07-27 10:05:09 +00:00
# ====== click command handlers ======
@click.group ( )
2018-08-23 11:05:41 +00:00
@click.option (
" --colors/--no-colors " ,
" -c/-C " ,
default = sys . stdout . isatty ( ) ,
help = " Force colored output on/off " ,
)
def cli ( colors ) :
global USE_COLORS
USE_COLORS = colors
2018-07-27 10:05:09 +00:00
@cli.command ( )
2018-08-15 16:46:00 +00:00
# fmt: off
@click.option ( " --backend/--no-backend " , " -b " , default = False , help = " Check blockbook/bitcore responses " )
2018-07-30 15:28:20 +00:00
@click.option ( " --icons/--no-icons " , default = True , help = " Check icon files " )
2019-04-18 14:27:27 +00:00
@click.option ( " -d " , " --show-duplicates " , type = click . Choice ( ( " all " , " nontoken " , " errors " ) ) , default = " errors " , help = " How much information about duplicate shortcuts should be shown. " )
2018-08-15 16:46:00 +00:00
# fmt: on
2018-08-23 17:32:28 +00:00
def check ( backend , icons , show_duplicates ) :
2018-07-27 10:05:09 +00:00
""" Validate coin definitions.
2018-08-23 17:09:54 +00:00
Checks that every btc - like coin is properly filled out , reports duplicate symbols ,
missing or invalid icons , backend responses , and uniform key information - -
i . e . , that all coins of the same type have the same fields in their JSON data .
Uniformity check ignores NEM mosaics and ERC20 tokens , where non - uniformity is
expected .
2018-08-23 11:05:41 +00:00
The ` - - show - duplicates ` option can be set to :
2018-08-23 17:32:28 +00:00
- all : all shortcut collisions are shown , including colliding ERC20 tokens
- nontoken : only collisions that affect non - ERC20 coins are shown
- errors : only collisions between non - ERC20 tokens are shown . This is the default ,
as a collision between two or more non - ERC20 tokens is an error .
2018-08-23 11:05:41 +00:00
In the output , duplicate ERC tokens will be shown in cyan ; duplicate non - tokens
in red . An asterisk ( * ) next to symbol name means that even though it was detected
as duplicate , it is still included in results .
2018-08-23 17:09:35 +00:00
2018-08-23 17:32:28 +00:00
The collision detection checks that SLIP44 numbers don ' t collide between different
mainnets ( testnet collisions are allowed ) , that ` address_prefix ` doesn ' t collide
with Bitcoin ( other collisions are reported as warnings ) . ` address_prefix_p2sh `
2018-08-23 17:09:35 +00:00
is also checked but we have a bunch of collisions there and can ' t do much
about them , so it ' s not an error.
In the collision checks , Bitcoin is shown in red , other mainnets in blue ,
testnets in green and unsupported networks in gray , marked with ` ( X ) ` for
non - colored output .
2018-07-27 10:05:09 +00:00
"""
2018-07-30 12:45:01 +00:00
if backend and requests is None :
2018-07-27 10:05:09 +00:00
raise click . ClickException ( " You must install requests for backend check " )
2021-05-31 13:11:05 +00:00
if icons and not CAN_CHECK_ICONS :
2018-07-30 12:45:01 +00:00
raise click . ClickException ( " Missing requirements for icon check " )
2018-08-24 13:20:25 +00:00
defs , buckets = coin_info . coin_info_with_duplicates ( )
2019-04-29 16:04:19 +00:00
support_info = coin_info . support_info ( defs )
mark_unsupported ( support_info , defs . as_list ( ) )
2018-07-27 10:05:09 +00:00
all_checks_passed = True
print ( " Checking BTC-like coins... " )
2018-08-24 13:42:06 +00:00
if not check_btc ( defs . bitcoin ) :
2018-07-27 10:05:09 +00:00
all_checks_passed = False
2018-09-07 11:17:47 +00:00
print ( " Checking Ethereum networks... " )
if not check_eth ( defs . eth ) :
all_checks_passed = False
2018-08-23 11:05:41 +00:00
if show_duplicates == " all " :
2018-08-23 17:32:28 +00:00
dup_level = logging . DEBUG
2018-08-23 11:05:41 +00:00
elif show_duplicates == " nontoken " :
2018-08-23 17:32:28 +00:00
dup_level = logging . INFO
2018-08-23 11:05:41 +00:00
else :
2019-04-29 16:04:19 +00:00
dup_level = logging . WARNING
2018-08-15 17:20:15 +00:00
print ( " Checking unexpected duplicates... " )
2018-08-23 17:32:28 +00:00
if not check_dups ( buckets , dup_level ) :
2018-08-15 17:20:15 +00:00
all_checks_passed = False
2018-11-29 14:42:49 +00:00
nontoken_dups = [ coin for coin in defs . as_list ( ) if " dup_key_nontoken " in coin ]
if nontoken_dups :
nontoken_dup_str = " , " . join (
highlight_key ( coin , " red " ) for coin in nontoken_dups
)
print_log ( logging . ERROR , " Non-token duplicate keys: " + nontoken_dup_str )
all_checks_passed = False
2018-07-30 12:45:01 +00:00
if icons :
print ( " Checking icon files... " )
2018-08-24 13:42:06 +00:00
if not check_icons ( defs . bitcoin ) :
2018-07-30 12:45:01 +00:00
all_checks_passed = False
if backend :
2018-07-27 10:05:09 +00:00
print ( " Checking backend responses... " )
2018-08-24 13:42:06 +00:00
if not check_backends ( defs . bitcoin ) :
2018-07-27 10:05:09 +00:00
all_checks_passed = False
2019-02-12 09:56:34 +00:00
print ( " Checking segwit fields... " )
if not check_segwit ( defs . bitcoin ) :
all_checks_passed = False
2018-08-23 17:09:54 +00:00
print ( " Checking key uniformity... " )
for cointype , coinlist in defs . items ( ) :
if cointype in ( " erc20 " , " nem " ) :
continue
if not check_key_uniformity ( coinlist ) :
all_checks_passed = False
2019-11-26 17:11:09 +00:00
print ( " Checking FIDO app definitions... " )
if not check_fido ( coin_info . fido_info ( ) ) :
all_checks_passed = False
2018-07-27 10:05:09 +00:00
if not all_checks_passed :
print ( " Some checks failed. " )
sys . exit ( 1 )
else :
print ( " Everything is OK. " )
@cli.command ( )
2018-09-12 16:09:01 +00:00
# fmt: off
2018-09-14 13:57:11 +00:00
@click.option ( " -o " , " --outfile " , type = click . File ( mode = " w " ) , default = " - " )
2018-09-12 16:09:01 +00:00
@click.option ( " -s/-S " , " --support/--no-support " , default = True , help = " Include support data for each coin " )
@click.option ( " -p " , " --pretty " , is_flag = True , help = " Generate nicely formatted JSON " )
2018-10-08 11:57:42 +00:00
@click.option ( " -l " , " --list " , " flat_list " , is_flag = True , help = " Output a flat list of coins " )
2018-09-14 13:57:11 +00:00
@click.option ( " -i " , " --include " , metavar = " FIELD " , multiple = True , help = " Include only these fields " )
@click.option ( " -e " , " --exclude " , metavar = " FIELD " , multiple = True , help = " Exclude these fields " )
@click.option ( " -I " , " --include-type " , metavar = " TYPE " , multiple = True , help = " Include only these categories " )
@click.option ( " -E " , " --exclude-type " , metavar = " TYPE " , multiple = True , help = " Exclude these categories " )
@click.option ( " -f " , " --filter " , metavar = " FIELD=FILTER " , multiple = True , help = " Include only coins that match a filter " )
@click.option ( " -F " , " --filter-exclude " , metavar = " FIELD=FILTER " , multiple = True , help = " Exclude coins that match a filter " )
@click.option ( " -t " , " --exclude-tokens " , is_flag = True , help = " Exclude ERC20 tokens. Equivalent to ' -E erc20 ' " )
@click.option ( " -d " , " --device " , metavar = " NAME " , help = " Only include coins supported on a given device " )
2018-09-12 16:09:01 +00:00
# fmt: on
2018-09-14 13:57:11 +00:00
def dump (
outfile ,
support ,
pretty ,
flat_list ,
include ,
exclude ,
include_type ,
exclude_type ,
filter ,
filter_exclude ,
exclude_tokens ,
device ,
) :
2019-11-26 17:11:09 +00:00
""" Dump coin data in JSON format.
2018-09-04 13:28:06 +00:00
This file is structured the same as the internal data . That is , top - level object
is a dict with keys : ' bitcoin ' , ' eth ' , ' erc20 ' , ' nem ' and ' misc ' . Value for each
key is a list of dicts , each describing a known coin .
2018-09-14 13:57:11 +00:00
If ' --list ' is specified , the top - level object is instead a flat list of coins .
2018-09-04 13:28:06 +00:00
\b
Fields are category - specific , except for four common ones :
- ' name ' - human - readable name
- ' shortcut ' - currency symbol
- ' key ' - unique identifier , e . g . , ' bitcoin:BTC '
- ' support ' - a dict with entries per known device
2018-09-12 16:09:01 +00:00
To control the size and properties of the resulting file , you can specify whether
2018-09-14 13:57:11 +00:00
or not you want pretty - printing and whether or not to include support data with
each coin .
2018-09-12 16:09:01 +00:00
2018-09-14 13:57:11 +00:00
You can specify which categories and which fields will be included or excluded .
You cannot specify both include and exclude at the same time . Include is " stronger "
than exclude , in that _only_ the specified fields are included .
You can also specify filters , in the form ' -f field=value ' ( or ' -F ' for inverse
filter ) . Filter values are case - insensitive and support shell - style wildcards ,
so ' -f name=bit* ' finds all coins whose names start with " bit " or " Bit " .
2018-09-04 13:28:06 +00:00
"""
2018-09-14 13:57:11 +00:00
if exclude_tokens :
exclude_type = ( " erc20 " , )
if include and exclude :
raise click . ClickException (
" You cannot specify --include and --exclude at the same time. "
)
if include_type and exclude_type :
raise click . ClickException (
" You cannot specify --include-type and --exclude-type at the same time. "
)
2018-09-04 13:28:06 +00:00
coins = coin_info . coin_info ( )
2018-09-14 13:57:11 +00:00
support_info = coin_info . support_info ( coins . as_list ( ) )
2018-09-04 13:28:06 +00:00
2018-09-12 16:09:01 +00:00
if support :
for category in coins . values ( ) :
for coin in category :
coin [ " support " ] = support_info [ coin [ " key " ] ]
2018-09-04 13:28:06 +00:00
2018-09-14 13:57:11 +00:00
# filter types
if include_type :
coins_dict = { k : v for k , v in coins . items ( ) if k in include_type }
else :
coins_dict = { k : v for k , v in coins . items ( ) if k not in exclude_type }
# filter individual coins
include_filters = [ f . split ( " = " , maxsplit = 1 ) for f in filter ]
exclude_filters = [ f . split ( " = " , maxsplit = 1 ) for f in filter_exclude ]
# always exclude 'address_bytes', not encodable in JSON
exclude + = ( " address_bytes " , )
def should_include_coin ( coin ) :
for field , filter in include_filters :
filter = filter . lower ( )
if field not in coin :
return False
2019-02-01 11:58:20 +00:00
if not fnmatch . fnmatch ( str ( coin [ field ] ) . lower ( ) , filter ) :
2018-09-14 13:57:11 +00:00
return False
for field , filter in exclude_filters :
filter = filter . lower ( )
if field not in coin :
continue
2019-02-01 11:58:20 +00:00
if fnmatch . fnmatch ( str ( coin [ field ] ) . lower ( ) , filter ) :
2018-09-14 13:57:11 +00:00
return False
if device :
is_supported = support_info [ coin [ " key " ] ] . get ( device , None )
if not is_supported :
return False
return True
2018-09-12 16:09:01 +00:00
2018-09-14 13:57:11 +00:00
def modify_coin ( coin ) :
if include :
return { k : v for k , v in coin . items ( ) if k in include }
else :
return { k : v for k , v in coin . items ( ) if k not in exclude }
for key , coinlist in coins_dict . items ( ) :
coins_dict [ key ] = [ modify_coin ( c ) for c in coinlist if should_include_coin ( c ) ]
if flat_list :
output = sum ( coins_dict . values ( ) , [ ] )
else :
output = coins_dict
2018-07-27 10:05:09 +00:00
with outfile :
2018-09-14 13:57:11 +00:00
indent = 4 if pretty else None
json . dump ( output , outfile , indent = indent , sort_keys = True )
outfile . write ( " \n " )
2018-07-30 12:25:53 +00:00
2018-07-27 10:05:09 +00:00
@cli.command ( )
2018-08-15 16:46:00 +00:00
# fmt: off
2018-07-27 10:05:09 +00:00
@click.argument ( " paths " , metavar = " [path]... " , nargs = - 1 )
2018-08-07 11:00:30 +00:00
@click.option ( " -o " , " --outfile " , type = click . File ( " w " ) , help = " Alternate output file " )
@click.option ( " -v " , " --verbose " , is_flag = True , help = " Print rendered file names " )
2019-04-27 14:15:35 +00:00
@click.option ( " -b " , " --bitcoin-only " , is_flag = True , help = " Accept only Bitcoin coins " )
2018-08-15 16:46:00 +00:00
# fmt: on
2019-04-27 14:15:35 +00:00
def render ( paths , outfile , verbose , bitcoin_only ) :
2018-08-07 11:00:30 +00:00
""" Generate source code from Mako templates.
2018-07-27 10:05:09 +00:00
2018-08-07 11:00:30 +00:00
For every " foo.bar.mako " filename passed , runs the template and
saves the result as " foo.bar " . For every directory name passed ,
processes all " .mako " files found in that directory .
2018-07-27 10:05:09 +00:00
2018-08-07 11:00:30 +00:00
If ` - o ` is specified , renders a single file into the specified outfile .
2018-07-27 10:05:09 +00:00
If no arguments are given , processes the current directory .
"""
if not CAN_RENDER :
raise click . ClickException ( " Please install ' mako ' and ' munch ' " )
2018-08-07 11:00:30 +00:00
if outfile and ( len ( paths ) != 1 or not os . path . isfile ( paths [ 0 ] ) ) :
raise click . ClickException ( " Option -o can only be used with single input file " )
2018-07-27 10:05:09 +00:00
2018-08-07 11:00:30 +00:00
# prepare defs
2018-08-24 13:20:25 +00:00
defs = coin_info . coin_info ( )
2019-11-28 13:00:13 +00:00
defs [ " fido " ] = coin_info . fido_info ( )
2018-08-15 15:34:20 +00:00
support_info = coin_info . support_info ( defs )
2018-07-27 10:05:09 +00:00
2019-04-27 14:15:35 +00:00
if bitcoin_only :
defs [ " bitcoin " ] = [
x
for x in defs [ " bitcoin " ]
if x [ " coin_name " ] in ( " Bitcoin " , " Testnet " , " Regtest " )
]
2018-07-30 12:45:01 +00:00
# munch dicts - make them attribute-accessible
2018-07-27 10:05:09 +00:00
for key , value in defs . items ( ) :
defs [ key ] = [ Munch ( coin ) for coin in value ]
for key , value in support_info . items ( ) :
support_info [ key ] = Munch ( value )
2018-08-07 11:00:30 +00:00
def do_render ( src , dst ) :
if verbose :
2021-09-27 10:13:51 +00:00
click . echo ( f " Rendering { src } => { dst } " )
2018-08-07 11:00:30 +00:00
render_file ( src , dst , defs , support_info )
# single in-out case
if outfile :
do_render ( paths [ 0 ] , outfile )
return
# find files in directories
if not paths :
paths = [ " . " ]
files = [ ]
for path in paths :
if not os . path . exists ( path ) :
2021-09-27 10:13:51 +00:00
click . echo ( f " Path { path } does not exist " )
2018-08-07 11:00:30 +00:00
elif os . path . isdir ( path ) :
files + = glob . glob ( os . path . join ( path , " *.mako " ) )
else :
files . append ( path )
# render each file
2018-07-27 10:05:09 +00:00
for file in files :
if not file . endswith ( " .mako " ) :
2021-09-27 10:13:51 +00:00
click . echo ( f " File { file } does not end with .mako " )
2018-07-27 10:05:09 +00:00
else :
target = file [ : - len ( " .mako " ) ]
2018-08-07 11:00:30 +00:00
with open ( target , " w " ) as dst :
do_render ( file , dst )
2018-07-27 10:05:09 +00:00
if __name__ == " __main__ " :
cli ( )