From 72557614c4a2077f75d662ad5c6d9f9e8f4661ec Mon Sep 17 00:00:00 2001 From: Martin Milata Date: Mon, 7 Jun 2021 10:57:33 +0200 Subject: [PATCH] build(core): update build process for new protobuf - properly exclude in the trezor/enums folder - generate Rust protobuf blobs in scons Split from "tools: Generate special Protobuf blobs for the Rust codec" --- core/SConscript.firmware | 83 ++++++++++++++++++++++++++-------------- core/SConscript.unix | 82 +++++++++++++++++++++++++-------------- 2 files changed, 108 insertions(+), 57 deletions(-) diff --git a/core/SConscript.firmware b/core/SConscript.firmware index 0453aeeb94..f05765dcf2 100644 --- a/core/SConscript.firmware +++ b/core/SConscript.firmware @@ -448,13 +448,26 @@ env.Replace( MAKEVERSIONHDR='$PYTHON vendor/micropython/py/makeversionhdr.py', MAKEMODULEDEFS='$PYTHON vendor/micropython/py/makemoduledefs.py', MPY_TOOL='$PYTHON vendor/micropython/tools/mpy-tool.py', - MPY_CROSS='vendor/micropython/mpy-cross/mpy-cross -O' + PYOPT + MPY_CROSS='vendor/micropython/mpy-cross/mpy-cross -O' + PYOPT, + PB2PY='$PYTHON ../common/protob/pb2py', ) # # Qstrings # +PROTO_SOURCES_DIR = '../../../common/protob/' +PROTO_SOURCES = Glob(PROTO_SOURCES_DIR + '*.proto', + exclude=[PROTO_SOURCES_DIR + 'messages-bootloader.proto'] +) +qstr_protobuf = env.Command( + target=[ + 'genhdr/qstrdefs.protobuf.h', + ], + source=PROTO_SOURCES, + action='$PB2PY $SOURCES --qstr-out ${TARGET} --bitcoin-only=%s' % BITCOIN_ONLY, +) + qstr_micropython = 'vendor/micropython/py/qstrdefs.h' qstr_collected = env.CollectQstr( @@ -462,7 +475,7 @@ qstr_collected = env.CollectQstr( qstr_preprocessed = env.PreprocessQstr( target='genhdr/qstrdefs.preprocessed.h', - source=[qstr_micropython, qstr_collected]) + source=[qstr_micropython, qstr_protobuf, qstr_collected]) qstr_generated = env.GenerateQstrDefs( target='genhdr/qstrdefs.generated.h', source=qstr_preprocessed) @@ -520,19 +533,20 @@ if FROZEN: SOURCE_PY.extend(Glob(SOURCE_PY_DIR + 'storage/*.py')) - SOURCE_PY.extend(Glob(SOURCE_PY_DIR + 'trezor/messages/*.py', + SOURCE_PY.extend(Glob(SOURCE_PY_DIR + 'trezor/messages/__init__.py')) + SOURCE_PY.extend(Glob(SOURCE_PY_DIR + 'trezor/enums/*.py', exclude=[ - SOURCE_PY_DIR + 'trezor/messages/Binance*.py', - SOURCE_PY_DIR + 'trezor/messages/Cardano*.py', - SOURCE_PY_DIR + 'trezor/messages/DebugMonero*.py', - SOURCE_PY_DIR + 'trezor/messages/Eos*.py', - SOURCE_PY_DIR + 'trezor/messages/Ethereum*.py', - SOURCE_PY_DIR + 'trezor/messages/Lisk*.py', - SOURCE_PY_DIR + 'trezor/messages/Monero*.py', - SOURCE_PY_DIR + 'trezor/messages/NEM*.py', - SOURCE_PY_DIR + 'trezor/messages/Ripple*.py', - SOURCE_PY_DIR + 'trezor/messages/Stellar*.py', - SOURCE_PY_DIR + 'trezor/messages/Tezos*.py', + SOURCE_PY_DIR + 'trezor/enums/Binance*.py', + SOURCE_PY_DIR + 'trezor/enums/Cardano*.py', + SOURCE_PY_DIR + 'trezor/enums/DebugMonero*.py', + SOURCE_PY_DIR + 'trezor/enums/Eos*.py', + SOURCE_PY_DIR + 'trezor/enums/Ethereum*.py', + SOURCE_PY_DIR + 'trezor/enums/Lisk*.py', + SOURCE_PY_DIR + 'trezor/enums/Monero*.py', + SOURCE_PY_DIR + 'trezor/enums/NEM*.py', + SOURCE_PY_DIR + 'trezor/enums/Ripple*.py', + SOURCE_PY_DIR + 'trezor/enums/Stellar*.py', + SOURCE_PY_DIR + 'trezor/enums/Tezos*.py', ]) ) @@ -555,41 +569,41 @@ if FROZEN: if EVERYTHING: SOURCE_PY.extend(Glob(SOURCE_PY_DIR + 'apps/binance/*.py')) - SOURCE_PY.extend(Glob(SOURCE_PY_DIR + 'trezor/messages/Binance*.py')) + SOURCE_PY.extend(Glob(SOURCE_PY_DIR + 'trezor/enums/Binance*.py')) SOURCE_PY.extend(Glob(SOURCE_PY_DIR + 'apps/cardano/*.py')) SOURCE_PY.extend(Glob(SOURCE_PY_DIR + 'apps/cardano/*/*.py')) - SOURCE_PY.extend(Glob(SOURCE_PY_DIR + 'trezor/messages/Cardano*.py')) + SOURCE_PY.extend(Glob(SOURCE_PY_DIR + 'trezor/enums/Cardano*.py')) SOURCE_PY.extend(Glob(SOURCE_PY_DIR + 'apps/eos/*.py')) SOURCE_PY.extend(Glob(SOURCE_PY_DIR + 'apps/eos/*/*.py')) - SOURCE_PY.extend(Glob(SOURCE_PY_DIR + 'trezor/messages/Eos*.py')) + SOURCE_PY.extend(Glob(SOURCE_PY_DIR + 'trezor/enums/Eos*.py')) SOURCE_PY.extend(Glob(SOURCE_PY_DIR + 'apps/ethereum/*.py')) - SOURCE_PY.extend(Glob(SOURCE_PY_DIR + 'trezor/messages/Ethereum*.py')) + SOURCE_PY.extend(Glob(SOURCE_PY_DIR + 'trezor/enums/Ethereum*.py')) SOURCE_PY.extend(Glob(SOURCE_PY_DIR + 'apps/lisk/*.py')) - SOURCE_PY.extend(Glob(SOURCE_PY_DIR + 'trezor/messages/Lisk*.py')) + SOURCE_PY.extend(Glob(SOURCE_PY_DIR + 'trezor/enums/Lisk*.py')) SOURCE_PY.extend(Glob(SOURCE_PY_DIR + 'apps/monero/*.py')) SOURCE_PY.extend(Glob(SOURCE_PY_DIR + 'apps/monero/*/*.py')) SOURCE_PY.extend(Glob(SOURCE_PY_DIR + 'apps/monero/*/*/*.py')) - SOURCE_PY.extend(Glob(SOURCE_PY_DIR + 'trezor/messages/DebugMonero*.py')) - SOURCE_PY.extend(Glob(SOURCE_PY_DIR + 'trezor/messages/Monero*.py')) + SOURCE_PY.extend(Glob(SOURCE_PY_DIR + 'trezor/enums/DebugMonero*.py')) + SOURCE_PY.extend(Glob(SOURCE_PY_DIR + 'trezor/enums/Monero*.py')) SOURCE_PY.extend(Glob(SOURCE_PY_DIR + 'apps/nem/*.py')) SOURCE_PY.extend(Glob(SOURCE_PY_DIR + 'apps/nem/*/*.py')) - SOURCE_PY.extend(Glob(SOURCE_PY_DIR + 'trezor/messages/NEM*.py')) + SOURCE_PY.extend(Glob(SOURCE_PY_DIR + 'trezor/enums/NEM*.py')) SOURCE_PY.extend(Glob(SOURCE_PY_DIR + 'apps/ripple/*.py')) - SOURCE_PY.extend(Glob(SOURCE_PY_DIR + 'trezor/messages/Ripple*.py')) + SOURCE_PY.extend(Glob(SOURCE_PY_DIR + 'trezor/enums/Ripple*.py')) SOURCE_PY.extend(Glob(SOURCE_PY_DIR + 'apps/stellar/*.py')) SOURCE_PY.extend(Glob(SOURCE_PY_DIR + 'apps/stellar/*/*.py')) - SOURCE_PY.extend(Glob(SOURCE_PY_DIR + 'trezor/messages/Stellar*.py')) + SOURCE_PY.extend(Glob(SOURCE_PY_DIR + 'trezor/enums/Stellar*.py')) SOURCE_PY.extend(Glob(SOURCE_PY_DIR + 'apps/tezos/*.py')) - SOURCE_PY.extend(Glob(SOURCE_PY_DIR + 'trezor/messages/Tezos*.py')) + SOURCE_PY.extend(Glob(SOURCE_PY_DIR + 'trezor/enums/Tezos*.py')) SOURCE_PY.extend(Glob(SOURCE_PY_DIR + 'apps/webauthn/*.py')) @@ -628,9 +642,21 @@ if FEATURE_FLAGS["SECP256K1_ZKP"]: # Rust library # +protobuf_blobs = env.Command( + target=[ + f'rust/{RUST_TARGET}/proto_enums.data', + f'rust/{RUST_TARGET}/proto_msgs.data', + f'rust/{RUST_TARGET}/proto_names.data', + f'rust/{RUST_TARGET}/proto_wire.data', + ], + source=PROTO_SOURCES, + action='$PB2PY --bitcoin-only=%s --blob-outdir ${TARGET.dir} $SOURCES --qstr-defs build/firmware/genhdr/qstrdefs.generated.h' % BITCOIN_ONLY, +) +env.Depends(protobuf_blobs, qstr_generated) + RUST_PROFILE = 'release' RUST_LIB = 'trezor_lib' -RUST_LIBDIR = f'build/rust/{RUST_TARGET}/{RUST_PROFILE}' +RUST_LIBDIR = f'build/firmware/rust/{RUST_TARGET}/{RUST_PROFILE}' RUST_LIBPATH = f'{RUST_LIBDIR}/lib{RUST_LIB}.a' def cargo_build(): @@ -639,14 +665,13 @@ def cargo_build(): profile = '--release' else: profile = '' - return f'cd embed/rust; cargo build {profile} --target={RUST_TARGET}' + return f'cd embed/rust; cargo build {profile} --target={RUST_TARGET} --target-dir=../../build/firmware/rust' rust = env.Command( target=RUST_LIBPATH, source='', action=cargo_build(), ) - -env.Depends(rust, qstr_generated) +env.Depends(rust, protobuf_blobs) env.Append(LINKFLAGS=f' -L{RUST_LIBDIR}') env.Append(LINKFLAGS=f' -l{RUST_LIB}') diff --git a/core/SConscript.unix b/core/SConscript.unix index fb5f957fc1..1f3c351f40 100644 --- a/core/SConscript.unix +++ b/core/SConscript.unix @@ -400,13 +400,26 @@ env.Replace( MAKEVERSIONHDR='$PYTHON vendor/micropython/py/makeversionhdr.py', MAKEMODULEDEFS='$PYTHON vendor/micropython/py/makemoduledefs.py', MPY_TOOL='$PYTHON vendor/micropython/tools/mpy-tool.py', - MPY_CROSS='vendor/micropython/mpy-cross/mpy-cross -O' + PYOPT + MPY_CROSS='vendor/micropython/mpy-cross/mpy-cross -O' + PYOPT, + PB2PY='$PYTHON ../common/protob/pb2py', ) # # Qstrings # +PROTO_SOURCES_DIR = '../../../common/protob/' +PROTO_SOURCES = Glob(PROTO_SOURCES_DIR + '*.proto', + exclude=[PROTO_SOURCES_DIR + 'messages-bootloader.proto'] +) +qstr_protobuf = env.Command( + target=[ + 'genhdr/qstrdefs.protobuf.h', + ], + source=PROTO_SOURCES, + action='$PB2PY $SOURCES --qstr-out ${TARGET} --bitcoin-only=%s' % BITCOIN_ONLY, +) + qstr_micropython = 'vendor/micropython/py/qstrdefs.h' qstr_collected = env.CollectQstr( @@ -414,7 +427,7 @@ qstr_collected = env.CollectQstr( qstr_preprocessed = env.PreprocessQstr( target='genhdr/qstrdefs.preprocessed.h', - source=[qstr_micropython, qstr_collected]) + source=[qstr_micropython, qstr_protobuf, qstr_collected]) qstr_generated = env.GenerateQstrDefs( target='genhdr/qstrdefs.generated.h', source=qstr_preprocessed) @@ -472,19 +485,20 @@ if FROZEN: SOURCE_PY.extend(Glob(SOURCE_PY_DIR + 'storage/*.py')) - SOURCE_PY.extend(Glob(SOURCE_PY_DIR + 'trezor/messages/*.py', + SOURCE_PY.extend(Glob(SOURCE_PY_DIR + 'trezor/messages/__init__.py')) + SOURCE_PY.extend(Glob(SOURCE_PY_DIR + 'trezor/enums/*.py', exclude=[ - SOURCE_PY_DIR + 'trezor/messages/Binance*.py', - SOURCE_PY_DIR + 'trezor/messages/Cardano*.py', - SOURCE_PY_DIR + 'trezor/messages/DebugMonero*.py', - SOURCE_PY_DIR + 'trezor/messages/Eos*.py', - SOURCE_PY_DIR + 'trezor/messages/Ethereum*.py', - SOURCE_PY_DIR + 'trezor/messages/Lisk*.py', - SOURCE_PY_DIR + 'trezor/messages/Monero*.py', - SOURCE_PY_DIR + 'trezor/messages/NEM*.py', - SOURCE_PY_DIR + 'trezor/messages/Ripple*.py', - SOURCE_PY_DIR + 'trezor/messages/Stellar*.py', - SOURCE_PY_DIR + 'trezor/messages/Tezos*.py', + SOURCE_PY_DIR + 'trezor/enums/Binance*.py', + SOURCE_PY_DIR + 'trezor/enums/Cardano*.py', + SOURCE_PY_DIR + 'trezor/enums/DebugMonero*.py', + SOURCE_PY_DIR + 'trezor/enums/Eos*.py', + SOURCE_PY_DIR + 'trezor/enums/Ethereum*.py', + SOURCE_PY_DIR + 'trezor/enums/Lisk*.py', + SOURCE_PY_DIR + 'trezor/enums/Monero*.py', + SOURCE_PY_DIR + 'trezor/enums/NEM*.py', + SOURCE_PY_DIR + 'trezor/enums/Ripple*.py', + SOURCE_PY_DIR + 'trezor/enums/Stellar*.py', + SOURCE_PY_DIR + 'trezor/enums/Tezos*.py', ]) ) @@ -507,41 +521,41 @@ if FROZEN: if EVERYTHING: SOURCE_PY.extend(Glob(SOURCE_PY_DIR + 'apps/binance/*.py')) - SOURCE_PY.extend(Glob(SOURCE_PY_DIR + 'trezor/messages/Binance*.py')) + SOURCE_PY.extend(Glob(SOURCE_PY_DIR + 'trezor/enums/Binance*.py')) SOURCE_PY.extend(Glob(SOURCE_PY_DIR + 'apps/cardano/*.py')) SOURCE_PY.extend(Glob(SOURCE_PY_DIR + 'apps/cardano/*/*.py')) - SOURCE_PY.extend(Glob(SOURCE_PY_DIR + 'trezor/messages/Cardano*.py')) + SOURCE_PY.extend(Glob(SOURCE_PY_DIR + 'trezor/enums/Cardano*.py')) SOURCE_PY.extend(Glob(SOURCE_PY_DIR + 'apps/eos/*.py')) SOURCE_PY.extend(Glob(SOURCE_PY_DIR + 'apps/eos/*/*.py')) - SOURCE_PY.extend(Glob(SOURCE_PY_DIR + 'trezor/messages/Eos*.py')) + SOURCE_PY.extend(Glob(SOURCE_PY_DIR + 'trezor/enums/Eos*.py')) SOURCE_PY.extend(Glob(SOURCE_PY_DIR + 'apps/ethereum/*.py')) - SOURCE_PY.extend(Glob(SOURCE_PY_DIR + 'trezor/messages/Ethereum*.py')) + SOURCE_PY.extend(Glob(SOURCE_PY_DIR + 'trezor/enums/Ethereum*.py')) SOURCE_PY.extend(Glob(SOURCE_PY_DIR + 'apps/lisk/*.py')) - SOURCE_PY.extend(Glob(SOURCE_PY_DIR + 'trezor/messages/Lisk*.py')) + SOURCE_PY.extend(Glob(SOURCE_PY_DIR + 'trezor/enums/Lisk*.py')) SOURCE_PY.extend(Glob(SOURCE_PY_DIR + 'apps/monero/*.py')) SOURCE_PY.extend(Glob(SOURCE_PY_DIR + 'apps/monero/*/*.py')) SOURCE_PY.extend(Glob(SOURCE_PY_DIR + 'apps/monero/*/*/*.py')) - SOURCE_PY.extend(Glob(SOURCE_PY_DIR + 'trezor/messages/DebugMonero*.py')) - SOURCE_PY.extend(Glob(SOURCE_PY_DIR + 'trezor/messages/Monero*.py')) + SOURCE_PY.extend(Glob(SOURCE_PY_DIR + 'trezor/enums/DebugMonero*.py')) + SOURCE_PY.extend(Glob(SOURCE_PY_DIR + 'trezor/enums/Monero*.py')) SOURCE_PY.extend(Glob(SOURCE_PY_DIR + 'apps/nem/*.py')) SOURCE_PY.extend(Glob(SOURCE_PY_DIR + 'apps/nem/*/*.py')) - SOURCE_PY.extend(Glob(SOURCE_PY_DIR + 'trezor/messages/NEM*.py')) + SOURCE_PY.extend(Glob(SOURCE_PY_DIR + 'trezor/enums/NEM*.py')) SOURCE_PY.extend(Glob(SOURCE_PY_DIR + 'apps/ripple/*.py')) - SOURCE_PY.extend(Glob(SOURCE_PY_DIR + 'trezor/messages/Ripple*.py')) + SOURCE_PY.extend(Glob(SOURCE_PY_DIR + 'trezor/enums/Ripple*.py')) SOURCE_PY.extend(Glob(SOURCE_PY_DIR + 'apps/stellar/*.py')) SOURCE_PY.extend(Glob(SOURCE_PY_DIR + 'apps/stellar/*/*.py')) - SOURCE_PY.extend(Glob(SOURCE_PY_DIR + 'trezor/messages/Stellar*.py')) + SOURCE_PY.extend(Glob(SOURCE_PY_DIR + 'trezor/enums/Stellar*.py')) SOURCE_PY.extend(Glob(SOURCE_PY_DIR + 'apps/tezos/*.py')) - SOURCE_PY.extend(Glob(SOURCE_PY_DIR + 'trezor/messages/Tezos*.py')) + SOURCE_PY.extend(Glob(SOURCE_PY_DIR + 'trezor/enums/Tezos*.py')) SOURCE_PY.extend(Glob(SOURCE_PY_DIR + 'apps/webauthn/*.py')) @@ -580,9 +594,21 @@ if FEATURE_FLAGS["SECP256K1_ZKP"]: # Rust library # +protobuf_blobs = env.Command( + target=[ + 'rust/proto_enums.data', + 'rust/proto_msgs.data', + 'rust/proto_names.data', + 'rust/proto_wire.data', + ], + source=PROTO_SOURCES, + action='$PB2PY --bitcoin-only=%s --blob-outdir ${TARGET.dir} $SOURCES --qstr-defs build/unix/genhdr/qstrdefs.generated.h' % BITCOIN_ONLY, +) +env.Depends(protobuf_blobs, qstr_generated) + RUST_PROFILE = 'release' RUST_LIB = 'trezor_lib' -RUST_LIBDIR = f'build/rust/{RUST_PROFILE}' +RUST_LIBDIR = f'build/unix/rust/{RUST_PROFILE}' RUST_LIBPATH = f'{RUST_LIBDIR}/lib{RUST_LIB}.a' def cargo_build(): @@ -591,14 +617,14 @@ def cargo_build(): profile = '--release' else: profile = '' - return f'cd embed/rust; cargo build {profile}' + return f'cd embed/rust; cargo build {profile} --target-dir=../../build/unix/rust' rust = env.Command( target=RUST_LIBPATH, source='', action=cargo_build(), ) +env.Depends(rust, protobuf_blobs) -env.Depends(rust, qstr_generated) env.Append(LINKFLAGS=f'-L{RUST_LIBDIR}') env.Append(LINKFLAGS=f'-l{RUST_LIB}')