refactor(common/protob): make bitcoin_only options file-level

common/protob/messages-bitcoin.proto

@@ -5,6 +5,8 @@ package hw.trezor.messages.bitcoin;
 option java_package = "com.satoshilabs.trezor.lib.protobuf";
 option java_outer_classname = "TrezorMessageBitcoin";
 
+option (include_in_bitcoin_only) = true;
+
 import "messages.proto";
 import "messages-common.proto";
 
@@ -418,7 +420,6 @@ message PrevOutput {
  */
 message TxAckInput {
     option (wire_type) = 22;
-    option (include_in_bitcoin_only) = true;
 
     required TxAckInputWrapper tx = 1;
 
@@ -438,7 +439,6 @@ message TxAckInput {
  */
 message TxAckOutput {
     option (wire_type) = 22;
-    option (include_in_bitcoin_only) = true;
 
     required TxAckOutputWrapper tx = 1;
 
@@ -458,7 +458,6 @@ message TxAckOutput {
  */
 message TxAckPrevMeta {
     option (wire_type) = 22;
-    option (include_in_bitcoin_only) = true;
 
     required PrevTx tx = 1;
 }
@@ -474,7 +473,6 @@ message TxAckPrevMeta {
  */
 message TxAckPrevInput {
     option (wire_type) = 22;
-    option (include_in_bitcoin_only) = true;
 
     required TxAckPrevInputWrapper tx = 1;
 
@@ -495,7 +493,6 @@ message TxAckPrevInput {
  */
 message TxAckPrevOutput {
     option (wire_type) = 22;
-    option (include_in_bitcoin_only) = true;
 
     required TxAckPrevOutputWrapper tx = 1;
 
@@ -514,7 +511,6 @@ message TxAckPrevOutput {
  */
 message TxAckPrevExtraData {
     option (wire_type) = 22;
-    option (include_in_bitcoin_only) = true;
 
     required TxAckPrevExtraDataWrapper tx = 1;
 

common/protob/messages-bootloader.proto

@@ -5,6 +5,8 @@ package hw.trezor.messages.bootloader;
 option java_package = "com.satoshilabs.trezor.lib.protobuf";
 option java_outer_classname = "TrezorMessageBootloader";
 
+import "messages.proto";
+
 /**
  * Request: Ask device to erase its firmware (so it can be replaced via FirmwareUpload)
  * @start

common/protob/messages-common.proto

@@ -5,6 +5,10 @@ package hw.trezor.messages.common;
 option java_package = "com.satoshilabs.trezor.lib.protobuf";
 option java_outer_classname = "TrezorMessageCommon";
 
+option (include_in_bitcoin_only) = true;
+
+import "messages.proto";
+
 /**
  * Response: Success of the previous request
  * @end

common/protob/messages-crypto.proto

@@ -5,6 +5,10 @@ package hw.trezor.messages.crypto;
 option java_package = "com.satoshilabs.trezor.lib.protobuf";
 option java_outer_classname = "TrezorMessageCrypto";
 
+option (include_in_bitcoin_only) = true;
+
+import "messages.proto";
+
 /**
  * Request: Ask device to encrypt or decrypt value of given key
  * @start

common/protob/messages-debug.proto

@@ -5,6 +5,9 @@ package hw.trezor.messages.debug;
 option java_package = "com.satoshilabs.trezor.lib.protobuf";
 option java_outer_classname = "TrezorMessageDebug";
 
+option (include_in_bitcoin_only) = true;
+
+import "messages.proto";
 import "messages-common.proto";
 import "messages-management.proto";
 

common/protob/messages-management.proto

@@ -5,6 +5,8 @@ package hw.trezor.messages.management;
 option java_package = "com.satoshilabs.trezor.lib.protobuf";
 option java_outer_classname = "TrezorMessageManagement";
 
+option (include_in_bitcoin_only) = true;
+
 import "messages.proto";
 
 /**

common/protob/messages-webauthn.proto

@@ -5,6 +5,8 @@ package hw.trezor.messages.webauthn;
 option java_package = "com.satoshilabs.trezor.lib.protobuf";
 option java_outer_classname = "TrezorMessageWebAuthn";
 
+import "messages.proto";
+
 /**
  * Request: List resident credentials
  * @start

common/protob/messages.proto

@@ -9,6 +9,8 @@ package hw.trezor.messages;
 option java_package = "com.satoshilabs.trezor.lib.protobuf";
 option java_outer_classname = "TrezorMessage";
 
+option (include_in_bitcoin_only) = true;
+
 import "google/protobuf/descriptor.proto";
 
 /************************* WARNING ***********************
@@ -18,7 +20,7 @@ EnumValueOptions and FieldOptions, MUST NOT have the same ID.
 
 Using the same ID indicates the same purpose (protobuf does not allow multiple
 extensions with the same name), such as EnumValueOptions.bitcoin_only and
-MessageOptions.include_in_bitcoin_only. pb2py can then find the extension under
+FileOptions.include_in_bitcoin_only. pb2py can then find the extension under
 either name.
 
 The convention to achieve this is as follows:
@@ -54,9 +56,6 @@ extend google.protobuf.EnumOptions {
 extend google.protobuf.MessageOptions {
     optional bool unstable = 52001;   // indicate that a message definition might change at any time
     optional uint32 wire_type = 52002;   // override wire type specified in the MessageType enum
-
-    optional bool include_in_bitcoin_only = 60000;  // message is available on BITCOIN_ONLY build
-                                                    // intentionally identical to `bitcoin_only` from enum
 }
 
 /** Options for tagging field types */
@@ -64,6 +63,12 @@ extend google.protobuf.FieldOptions {
     optional bool experimental = 53001;   // indicate that a field is intended for development and beta testing only
 }
 
+/** Options for tagging files with protobuf definitions */
+extend google.protobuf.FileOptions {
+    optional bool include_in_bitcoin_only = 60000;  // definitions are available on BITCOIN_ONLY build
+                                                    // intentionally identical to `bitcoin_only` from enum
+}
+
 
 /**
  * Mapping between Trezor wire identifier (uint) and a protobuf message

common/protob/pb2py

@@ -65,10 +65,15 @@ INT_TYPES = (
 
 MESSAGE_TYPE_ENUM = "MessageType"
 
+LengthDelimited = c.Struct(
+    "len" / c.VarInt,
+    "bytes" / c.Bytes(c.this.len),
+)
+
 ListOfSimpleValues = c.GreedyRange(
     c.Struct(
         "key" / c.VarInt,
-        "value" / c.VarInt,
+        "value" / c.Switch(c.this.key & 0b111, {0: c.VarInt, 2: LengthDelimited}),
     )
 )
 
@@ -76,7 +81,8 @@ ListOfSimpleValues = c.GreedyRange(
 def parse_protobuf_simple(data):
     """Micro-parse protobuf-encoded data.
 
-    Assume every field is of type 0 (varint), and parse to a dict of fieldnum: value.
+    Assume every value is of type 0 (varint) or 2 (length-delimited),
+    and parse to a dict of fieldnum: value.
     """
     return {v.key >> 3: v.value for v in ListOfSimpleValues.parse(data)}
 
@@ -339,33 +345,28 @@ class Descriptor:
             ext.name: ext.number for file in self.files for ext in file.extension
         }
 
+        if self.bitcoin_only:
+            self.files = [
+                f
+                for f in self.files
+                if self.get_extensions(f).get("include_in_bitcoin_only")
+            ]
+            logging.debug(f"found {len(self.files)} bitcoin-only files")
+
         # find message_type enum
         top_level_enums = itertools.chain.from_iterable(f.enum_type for f in self.files)
         self.message_type_enum = find_by_name(top_level_enums, MESSAGE_TYPE_ENUM, ())
         self.convert_enum_value_names(self.message_type_enum)
 
-        # top-level message inclusion filter that takes bitcoin_only into account
-        def should_include_message(message: ProtoMessage):
-            return (
-                # include all messages when not in bitcoin_only mode
-                not self.bitcoin_only
-                # include all non-wire messages
-                or message.wire_type is None
-                # include messages that are marked bitcoin_only
-                or message.extensions.get("bitcoin_only")
-            )
-
         # find messages and enums
         self.messages = []
         self.enums = []
 
         for file in self.files:
-            messages = (
+            messages = [
                 ProtoMessage.from_message(self, m)
                 for m in self._filter_items(file.message_type)
-            )
+            ]
-            # use exclusion list on top-level messages
-            messages = [m for m in messages if should_include_message(m)]
             self.messages += messages
             self.enums += self._filter_items(file.enum_type)
 
@@ -379,7 +380,6 @@ class Descriptor:
         for enum in self.enums:
             self.convert_enum_value_names(enum)
 
-
     def _filter_items(self, iter):
         return [
             item
@@ -402,7 +402,7 @@ class Descriptor:
         # the API provides access to unknown fields, it hides the extensions.
         # What we do is re-encode the options descriptor...
         options_bytes = something.options.SerializeToString()
-        # ...and re-parse it as a dict of uvarints...
+        # ...and re-parse it as a dict of uvarints/strings...
         simple_values = parse_protobuf_simple(options_bytes)
         # ...and extract the value corresponding to the extension we care about.
         return simple_values.get(extension_num, default)

common/protob/protocol.md

@@ -9,7 +9,7 @@ First packet has the following structure:
 | offset | length | type        | contents                                                                              |
 |--------|--------|-------------|---------------------------------------------------------------------------------------|
 |      0 |      3 | char[3]     | '?##' magic constant                                                                  |
-|      3 |      2 | BE uint16_t | numerical [message type](messages.proto#L14)                                          |
+|      3 |      2 | BE uint16_t | numerical [message type](messages.proto#L76)                                          |
 |      5 |      4 | BE uint32_t | message size                                                                          |
 |      9 |     55 | uint8_t[55] | first 55 bytes of message encoded in Protocol Buffers (padded with zeroes if shorter) |