diff --git a/common/protob/messages-bitcoin.proto b/common/protob/messages-bitcoin.proto index 7b574742b..61487e8ad 100644 --- a/common/protob/messages-bitcoin.proto +++ b/common/protob/messages-bitcoin.proto @@ -5,6 +5,8 @@ package hw.trezor.messages.bitcoin; option java_package = "com.satoshilabs.trezor.lib.protobuf"; option java_outer_classname = "TrezorMessageBitcoin"; +option (include_in_bitcoin_only) = true; + import "messages.proto"; import "messages-common.proto"; @@ -42,7 +44,7 @@ enum DecredStakingSpendType { /** * Unit to be used when showing amounts on the display */ - enum AmountUnit { +enum AmountUnit { BITCOIN = 0; // BTC MILLIBITCOIN = 1; // mBTC MICROBITCOIN = 2; // uBTC @@ -418,7 +420,6 @@ message PrevOutput { */ message TxAckInput { option (wire_type) = 22; - option (include_in_bitcoin_only) = true; required TxAckInputWrapper tx = 1; @@ -438,7 +439,6 @@ message TxAckInput { */ message TxAckOutput { option (wire_type) = 22; - option (include_in_bitcoin_only) = true; required TxAckOutputWrapper tx = 1; @@ -458,7 +458,6 @@ message TxAckOutput { */ message TxAckPrevMeta { option (wire_type) = 22; - option (include_in_bitcoin_only) = true; required PrevTx tx = 1; } @@ -474,7 +473,6 @@ message TxAckPrevMeta { */ message TxAckPrevInput { option (wire_type) = 22; - option (include_in_bitcoin_only) = true; required TxAckPrevInputWrapper tx = 1; @@ -495,7 +493,6 @@ message TxAckPrevInput { */ message TxAckPrevOutput { option (wire_type) = 22; - option (include_in_bitcoin_only) = true; required TxAckPrevOutputWrapper tx = 1; @@ -514,7 +511,6 @@ message TxAckPrevOutput { */ message TxAckPrevExtraData { option (wire_type) = 22; - option (include_in_bitcoin_only) = true; required TxAckPrevExtraDataWrapper tx = 1; diff --git a/common/protob/messages-bootloader.proto b/common/protob/messages-bootloader.proto index 7151a8e3b..05dc27a09 100644 --- a/common/protob/messages-bootloader.proto +++ b/common/protob/messages-bootloader.proto @@ -5,6 +5,8 @@ package hw.trezor.messages.bootloader; option java_package = "com.satoshilabs.trezor.lib.protobuf"; option java_outer_classname = "TrezorMessageBootloader"; +import "messages.proto"; + /** * Request: Ask device to erase its firmware (so it can be replaced via FirmwareUpload) * @start diff --git a/common/protob/messages-common.proto b/common/protob/messages-common.proto index dbdbc0171..e7a0ac715 100644 --- a/common/protob/messages-common.proto +++ b/common/protob/messages-common.proto @@ -5,6 +5,10 @@ package hw.trezor.messages.common; option java_package = "com.satoshilabs.trezor.lib.protobuf"; option java_outer_classname = "TrezorMessageCommon"; +option (include_in_bitcoin_only) = true; + +import "messages.proto"; + /** * Response: Success of the previous request * @end diff --git a/common/protob/messages-crypto.proto b/common/protob/messages-crypto.proto index bb8569807..38429790d 100644 --- a/common/protob/messages-crypto.proto +++ b/common/protob/messages-crypto.proto @@ -5,6 +5,10 @@ package hw.trezor.messages.crypto; option java_package = "com.satoshilabs.trezor.lib.protobuf"; option java_outer_classname = "TrezorMessageCrypto"; +option (include_in_bitcoin_only) = true; + +import "messages.proto"; + /** * Request: Ask device to encrypt or decrypt value of given key * @start diff --git a/common/protob/messages-debug.proto b/common/protob/messages-debug.proto index 3ed839ad3..1b88a4488 100644 --- a/common/protob/messages-debug.proto +++ b/common/protob/messages-debug.proto @@ -5,6 +5,9 @@ package hw.trezor.messages.debug; option java_package = "com.satoshilabs.trezor.lib.protobuf"; option java_outer_classname = "TrezorMessageDebug"; +option (include_in_bitcoin_only) = true; + +import "messages.proto"; import "messages-common.proto"; import "messages-management.proto"; diff --git a/common/protob/messages-management.proto b/common/protob/messages-management.proto index c74eea495..cd4a02c0c 100644 --- a/common/protob/messages-management.proto +++ b/common/protob/messages-management.proto @@ -5,6 +5,8 @@ package hw.trezor.messages.management; option java_package = "com.satoshilabs.trezor.lib.protobuf"; option java_outer_classname = "TrezorMessageManagement"; +option (include_in_bitcoin_only) = true; + import "messages.proto"; /** diff --git a/common/protob/messages-webauthn.proto b/common/protob/messages-webauthn.proto index 10bf4604d..76871abc6 100644 --- a/common/protob/messages-webauthn.proto +++ b/common/protob/messages-webauthn.proto @@ -5,6 +5,8 @@ package hw.trezor.messages.webauthn; option java_package = "com.satoshilabs.trezor.lib.protobuf"; option java_outer_classname = "TrezorMessageWebAuthn"; +import "messages.proto"; + /** * Request: List resident credentials * @start diff --git a/common/protob/messages.proto b/common/protob/messages.proto index 995e4c17d..ef540934b 100644 --- a/common/protob/messages.proto +++ b/common/protob/messages.proto @@ -9,6 +9,8 @@ package hw.trezor.messages; option java_package = "com.satoshilabs.trezor.lib.protobuf"; option java_outer_classname = "TrezorMessage"; +option (include_in_bitcoin_only) = true; + import "google/protobuf/descriptor.proto"; /************************* WARNING *********************** @@ -18,7 +20,7 @@ EnumValueOptions and FieldOptions, MUST NOT have the same ID. Using the same ID indicates the same purpose (protobuf does not allow multiple extensions with the same name), such as EnumValueOptions.bitcoin_only and -MessageOptions.include_in_bitcoin_only. pb2py can then find the extension under +FileOptions.include_in_bitcoin_only. pb2py can then find the extension under either name. The convention to achieve this is as follows: @@ -54,9 +56,6 @@ extend google.protobuf.EnumOptions { extend google.protobuf.MessageOptions { optional bool unstable = 52001; // indicate that a message definition might change at any time optional uint32 wire_type = 52002; // override wire type specified in the MessageType enum - - optional bool include_in_bitcoin_only = 60000; // message is available on BITCOIN_ONLY build - // intentionally identical to `bitcoin_only` from enum } /** Options for tagging field types */ @@ -64,6 +63,12 @@ extend google.protobuf.FieldOptions { optional bool experimental = 53001; // indicate that a field is intended for development and beta testing only } +/** Options for tagging files with protobuf definitions */ +extend google.protobuf.FileOptions { + optional bool include_in_bitcoin_only = 60000; // definitions are available on BITCOIN_ONLY build + // intentionally identical to `bitcoin_only` from enum +} + /** * Mapping between Trezor wire identifier (uint) and a protobuf message diff --git a/common/protob/pb2py b/common/protob/pb2py index 4b84eb1d5..aab8cc86a 100755 --- a/common/protob/pb2py +++ b/common/protob/pb2py @@ -65,10 +65,15 @@ INT_TYPES = ( MESSAGE_TYPE_ENUM = "MessageType" +LengthDelimited = c.Struct( + "len" / c.VarInt, + "bytes" / c.Bytes(c.this.len), +) + ListOfSimpleValues = c.GreedyRange( c.Struct( "key" / c.VarInt, - "value" / c.VarInt, + "value" / c.Switch(c.this.key & 0b111, {0: c.VarInt, 2: LengthDelimited}), ) ) @@ -76,7 +81,8 @@ ListOfSimpleValues = c.GreedyRange( def parse_protobuf_simple(data): """Micro-parse protobuf-encoded data. - Assume every field is of type 0 (varint), and parse to a dict of fieldnum: value. + Assume every value is of type 0 (varint) or 2 (length-delimited), + and parse to a dict of fieldnum: value. """ return {v.key >> 3: v.value for v in ListOfSimpleValues.parse(data)} @@ -339,33 +345,28 @@ class Descriptor: ext.name: ext.number for file in self.files for ext in file.extension } + if self.bitcoin_only: + self.files = [ + f + for f in self.files + if self.get_extensions(f).get("include_in_bitcoin_only") + ] + logging.debug(f"found {len(self.files)} bitcoin-only files") + # find message_type enum top_level_enums = itertools.chain.from_iterable(f.enum_type for f in self.files) self.message_type_enum = find_by_name(top_level_enums, MESSAGE_TYPE_ENUM, ()) self.convert_enum_value_names(self.message_type_enum) - # top-level message inclusion filter that takes bitcoin_only into account - def should_include_message(message: ProtoMessage): - return ( - # include all messages when not in bitcoin_only mode - not self.bitcoin_only - # include all non-wire messages - or message.wire_type is None - # include messages that are marked bitcoin_only - or message.extensions.get("bitcoin_only") - ) - # find messages and enums self.messages = [] self.enums = [] for file in self.files: - messages = ( + messages = [ ProtoMessage.from_message(self, m) for m in self._filter_items(file.message_type) - ) - # use exclusion list on top-level messages - messages = [m for m in messages if should_include_message(m)] + ] self.messages += messages self.enums += self._filter_items(file.enum_type) @@ -379,7 +380,6 @@ class Descriptor: for enum in self.enums: self.convert_enum_value_names(enum) - def _filter_items(self, iter): return [ item @@ -402,7 +402,7 @@ class Descriptor: # the API provides access to unknown fields, it hides the extensions. # What we do is re-encode the options descriptor... options_bytes = something.options.SerializeToString() - # ...and re-parse it as a dict of uvarints... + # ...and re-parse it as a dict of uvarints/strings... simple_values = parse_protobuf_simple(options_bytes) # ...and extract the value corresponding to the extension we care about. return simple_values.get(extension_num, default) diff --git a/common/protob/protocol.md b/common/protob/protocol.md index 44a44106a..c75296f64 100644 --- a/common/protob/protocol.md +++ b/common/protob/protocol.md @@ -9,7 +9,7 @@ First packet has the following structure: | offset | length | type | contents | |--------|--------|-------------|---------------------------------------------------------------------------------------| | 0 | 3 | char[3] | '?##' magic constant | -| 3 | 2 | BE uint16_t | numerical [message type](messages.proto#L14) | +| 3 | 2 | BE uint16_t | numerical [message type](messages.proto#L76) | | 5 | 4 | BE uint32_t | message size | | 9 | 55 | uint8_t[55] | first 55 bytes of message encoded in Protocol Buffers (padded with zeroes if shorter) |