From 6e81c914435d775ae1b787a052fb72f8ee314513 Mon Sep 17 00:00:00 2001 From: Saleem Rashid Date: Sun, 17 Dec 2017 13:39:18 +0000 Subject: [PATCH 1/4] pb2py: Clean up argparse usage --- tools/pb2py | 32 +++++++++++--------------------- 1 file changed, 11 insertions(+), 21 deletions(-) diff --git a/tools/pb2py b/tools/pb2py index 0551cb291..c897af471 100755 --- a/tools/pb2py +++ b/tools/pb2py @@ -3,12 +3,17 @@ # to plain-python objects as used in TREZOR Core and python-trezor import sys +import importlib import os import argparse from google.protobuf.internal.enum_type_wrapper import EnumTypeWrapper +def import_pb2(name): + return importlib.import_module("pb2.%s_pb2" % name) + + def process_type(t, cls, msg_id, indexfile, is_upy): print(" * type %s" % t) @@ -149,8 +154,7 @@ def process_module(mod, genpath, indexfile, modlist, is_upy): types = dict([(name, cls) for name, cls in mod.__dict__.items() if isinstance(cls, type)]) - msg_types = __import__('pb2', globals(), locals(), [ - 'messages_pb2', ]).messages_pb2.MessageType.items() + msg_types = import_pb2('messages').MessageType.items() for t, cls in sorted(types.items()): # Find message type for given class @@ -185,29 +189,15 @@ def write_to_file(genpath, t, out): if __name__ == '__main__': parser = argparse.ArgumentParser() - parser.add_argument('modulename', type=str, help="Name of module to generate") + parser.add_argument('modulename', type=import_pb2, help="Name of module to generate") parser.add_argument('genpath', type=str, help="Directory for generated source code") - parser.add_argument('-i', '--indexfile', type=str, help="[optional] Generate index file of wire types") - parser.add_argument('-l', '--modlist', type=str, help="[optional] Generate list of modules") - parser.add_argument('-p', '--protopath', type=str, help="[optional] Path to search for pregenerated Google's python sources") + parser.add_argument('-i', '--indexfile', type=argparse.FileType('a'), help="Generate index file of wire types") + parser.add_argument('-l', '--modlist', type=argparse.FileType('a'), help="Generate list of modules") + parser.add_argument('-p', '--protopath', type=str, help="Path to search for pregenerated Google's python sources") parser.add_argument('-m', '--micropython', action='store_true', help="Use micropython-favoured source code") args = parser.parse_args() - if args.indexfile: - indexfile = open(args.indexfile, 'a') - else: - indexfile = None - - if args.modlist: - modlist = open(args.modlist, 'a') - else: - modlist = None - if args.protopath: sys.path.append(args.protopath) - # Dynamically load module from argv[1] - tmp = __import__('pb2', globals(), locals(), ['%s_pb2' % args.modulename]) - mod = getattr(tmp, "%s_pb2" % args.modulename) - - process_module(mod, args.genpath, indexfile, modlist, args.micropython) + process_module(args.modulename, args.genpath, args.indexfile, args.modlist, args.micropython) From fd5f232fddcd75289e37414e4135e619404671cc Mon Sep 17 00:00:00 2001 From: Saleem Rashid Date: Sun, 17 Dec 2017 15:12:39 +0000 Subject: [PATCH 2/4] pb2py: Refactor --- tools/build_protobuf | 3 +- tools/pb2py | 189 +++++++++++++++++++------------------------ 2 files changed, 83 insertions(+), 109 deletions(-) diff --git a/tools/build_protobuf b/tools/build_protobuf index 778d7e100..2bc74c0e3 100755 --- a/tools/build_protobuf +++ b/tools/build_protobuf @@ -8,7 +8,6 @@ mkdir -p ../trezorlib/messages INDEX=../trezorlib/messages/__init__.py rm -f $INDEX echo '# Automatically generated by pb2py' >> $INDEX -echo 'from __future__ import absolute_import' >> $INDEX echo '' >> $INDEX for i in types messages storage ; do @@ -24,7 +23,7 @@ sed -i 's/^import types_pb2/from . import types_pb2/g' $CURDIR/pb2/storage_pb2.p for i in types messages storage ; do # Convert google protobuf library to trezor's internal format cd $CURDIR - ./pb2py -p $CURDIR -l $INDEX $i ../trezorlib/messages/ + ./pb2py -P "trezorlib.protobuf" -p $CURDIR -l $INDEX $i ../trezorlib/messages/ done rm -rf $CURDIR/pb2/ diff --git a/tools/pb2py b/tools/pb2py index c897af471..8df4abcad 100755 --- a/tools/pb2py +++ b/tools/pb2py @@ -7,69 +7,73 @@ import importlib import os import argparse -from google.protobuf.internal.enum_type_wrapper import EnumTypeWrapper - def import_pb2(name): return importlib.import_module("pb2.%s_pb2" % name) -def process_type(t, cls, msg_id, indexfile, is_upy): - print(" * type %s" % t) - - imports = [] - out = ["", "", "class %s(p.MessageType):" % t, ] - - if cls.DESCRIPTOR.fields_by_name: - out.append(" FIELDS = {") - elif msg_id is None: - out.append(" pass") +def create_message_import(name): + return "from .%s import %s" % (name, name) - for v in sorted(cls.DESCRIPTOR.fields_by_name.values(), key=lambda x: x.number): - number = v.number - fieldname = v.name - type = None - repeated = v.label == 3 - required = v.label == 2 - # print v.has_default_value, v.default_value +def create_const(name, value, is_upy): + if is_upy: + return "%s = const(%s)" % (name, value) + else: + return "%s = %s" % (name, value) - if v.type in (4, 13, 14): - # TYPE_UINT64 = 4 - # TYPE_UINT32 = 13 - # TYPE_ENUM = 14 - type = 'p.UVarintType' - elif v.type in (17,): - # TYPE_SINT32 = 17 - type = 'p.Sint32Type' +def remove_from_start(s, prefix): + if s.startswith(prefix): + return s[len(prefix):] + else: + return s - elif v.type == 9: - # TYPE_STRING = 9 - type = 'p.UnicodeType' - elif v.type == 8: - # TYPE_BOOL = 8 - type = 'p.BoolType' +def process_message(descriptor, protobuf_module, msg_id, indexfile, is_upy): + print(" * type %s" % descriptor.name) - elif v.type == 12: - # TYPE_BYTES = 12 - type = 'p.BytesType' + imports = [] + out = ["", "", "class %s(p.MessageType):" % descriptor.name, ] - elif v.type == 11: - # TYPE_MESSAGE = 1 - type = v.message_type.name - imports.append("from .%s import %s" % - (v.message_type.name, v.message_type.name)) + if descriptor.fields_by_number: + out.append(" FIELDS = {") + elif msg_id is None: + out.append(" pass") + for number, field in descriptor.fields_by_number.items(): + field_name = field.name + field_type = None + repeated = (field.label == field.LABEL_REPEATED) + required = (field.label == field.LABEL_REQUIRED) + + types = { + field.TYPE_UINT64: 'p.UVarintType', + field.TYPE_UINT32: 'p.UVarintType', + field.TYPE_ENUM: 'p.UVarintType', + field.TYPE_SINT32: 'p.Sint32Type', + field.TYPE_STRING: 'p.UnicodeType', + field.TYPE_BOOL: 'p.BoolType', + field.TYPE_BYTES: 'p.BytesType' + } + + if field.type == field.TYPE_MESSAGE: + field_type = field.message_type.name + imports.append(create_message_import(field_type)) else: - raise Exception("Unknown field type %s for field %s" % - (v.type, fieldname)) + try: + field_type = types[field.type] + except KeyError: + raise ValueError("Unknown field type %d for field %s" % (field.type, field_name)) + comments = [] if required: - comment = ' # required' - elif v.has_default_value: - comment = ' # default=%s' % repr(v.default_value) + comments.append('required') + if field.has_default_value: + comments.append("default=%s" % repr(field.default_value)) + + if comments: + comment = " # %s" % ' '.join(comments) else: comment = '' @@ -79,101 +83,71 @@ def process_type(t, cls, msg_id, indexfile, is_upy): flags = '0' out.append(" %d: ('%s', %s, %s),%s" % - (number, fieldname, type, flags, comment)) - - # print fieldname, number, type, repeated, comment - # print v.__dict__ - # print v.CPPTYPE_STRING - # print v.LABEL_REPEATED - # print v.enum_type - # v.has_default_value, v.default_value - # v.label == 3 # repeated - # print v.number - - if cls.DESCRIPTOR.fields_by_name: + (number, field_name, field_type, flags, comment)) + + if descriptor.fields_by_name: out.append(" }") if msg_id is not None: out.append(" MESSAGE_WIRE_TYPE = %d" % msg_id) if indexfile is not None: - if is_upy: - indexfile.write("%s = const(%d)\n" % (t, msg_id)) - else: - indexfile.write("%s = %d\n" % (t, msg_id)) + indexfile.write(create_const(t, msg_id, is_upy)) # Remove duplicate imports - imports = list(set(imports)) - - if is_upy: - imports = ['import protobuf as p'] + imports - else: - imports = ['from __future__ import absolute_import', - 'from .. import protobuf as p'] + imports - + imports = ["import %s as p" % protobuf_module, *set(imports)] return imports + out -def process_enum(t, cls, is_upy): +def process_enum(descriptor, is_upy): out = [] if is_upy: out += ("from micropython import const", "") - print(" * enum %s" % t) + print(" * enum %s" % descriptor.name) - for k, v in cls.items(): + for name, value in descriptor.values_by_name.items(): # Remove type name from the beginning of the constant # For example "PinMatrixRequestType_Current" -> "Current" - if k.startswith("%s_" % t): - k = k.replace("%s_" % t, '') + enum_prefix = descriptor.name + name = remove_from_start(name, "%s_" % enum_prefix) # If type ends with *Type, but constant use type name without *Type, remove it too :) # For example "ButtonRequestType & ButtonRequest_Other" => "Other" - if t.endswith("Type") and k.startswith("%s_" % t.replace("Type", '')): - k = k.replace("%s_" % t.replace("Type", ''), '') + if enum_prefix.endswith("Type"): + enum_prefix, _ = enum_prefix.rsplit("Type", 1) + name = remove_from_start(name, "%s_" % enum_prefix) - if is_upy: - out.append("%s = const(%s)" % (k, v)) - else: - out.append("%s = %s" % (k, v)) + out.append(create_const(name, value.number, is_upy)) return out -def find_msg_type(msg_types, t): - for k, v in msg_types: - msg_name = k.replace('MessageType_', '') - if msg_name == t: - return v +def process_file(descriptor, protobuf_module, genpath, indexfile, modlist, is_upy): + print("Processing module %s" % descriptor.name) -def process_module(mod, genpath, indexfile, modlist, is_upy): + msg_types = import_pb2('messages').MessageType - print("Processing module %s" % mod.__name__) - types = dict([(name, cls) - for name, cls in mod.__dict__.items() if isinstance(cls, type)]) - - msg_types = import_pb2('messages').MessageType.items() - - for t, cls in sorted(types.items()): + for name, message_descriptor in descriptor.message_types_by_name.items(): # Find message type for given class - msg_id = find_msg_type(msg_types, t) + try: + msg_id = msg_types.Value("MessageType_%s" % name) + except ValueError: + msg_id = None - out = process_type(t, cls, msg_id, indexfile, is_upy) + out = process_message(message_descriptor, protobuf_module, msg_id, indexfile, is_upy) - write_to_file(genpath, t, out) + write_to_file(genpath, name, out) if modlist: - modlist.write("from .%s import *\n" % t) - - enums = dict([(name, cls) for name, cls in mod.__dict__.items() - if isinstance(cls, EnumTypeWrapper)]) + modlist.write(create_message_import(name) + "\n") - for t, cls in enums.items(): - out = process_enum(t, cls, is_upy) - write_to_file(genpath, t, out) + for name, enum_descriptor in descriptor.enum_types_by_name.items(): + out = process_enum(enum_descriptor, is_upy) + write_to_file(genpath, name, out) if modlist: - modlist.write("from . import %s\n" % t) + modlist.write("from . import %s\n" % name) def write_to_file(genpath, t, out): @@ -189,8 +163,9 @@ def write_to_file(genpath, t, out): if __name__ == '__main__': parser = argparse.ArgumentParser() - parser.add_argument('modulename', type=import_pb2, help="Name of module to generate") + parser.add_argument('module', type=import_pb2, help="Name of module to generate") parser.add_argument('genpath', type=str, help="Directory for generated source code") + parser.add_argument('-P', '--protobuf-module', default="protobuf", help="Name of protobuf module") parser.add_argument('-i', '--indexfile', type=argparse.FileType('a'), help="Generate index file of wire types") parser.add_argument('-l', '--modlist', type=argparse.FileType('a'), help="Generate list of modules") parser.add_argument('-p', '--protopath', type=str, help="Path to search for pregenerated Google's python sources") @@ -200,4 +175,4 @@ if __name__ == '__main__': if args.protopath: sys.path.append(args.protopath) - process_module(args.modulename, args.genpath, args.indexfile, args.modlist, args.micropython) + process_file(args.module.DESCRIPTOR, args.protobuf_module, args.genpath, args.indexfile, args.modlist, args.micropython) From ff5999162a028414de2c9cd75f3d6128e0364b40 Mon Sep 17 00:00:00 2001 From: Saleem Rashid Date: Sun, 24 Dec 2017 12:04:36 +0000 Subject: [PATCH 3/4] pb2py: Add pb2/ to sys.path This removes the need for sed (which was problematic on OS X as it ships with BSD Sed instead of GNU Sed) --- tools/build_protobuf | 7 +------ tools/pb2py | 11 +++++++---- 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/tools/build_protobuf b/tools/build_protobuf index 2bc74c0e3..e46db4c55 100755 --- a/tools/build_protobuf +++ b/tools/build_protobuf @@ -1,7 +1,6 @@ #!/bin/bash CURDIR=$(pwd) mkdir -p $CURDIR/pb2/ -touch $CURDIR/pb2/__init__.py mkdir -p ../trezorlib/messages @@ -16,14 +15,10 @@ for i in types messages storage ; do protoc --python_out=$CURDIR/pb2/ -I/usr/include -I. $i.proto done -# hack to make output python 3 compatible -sed -i 's/^import types_pb2/from . import types_pb2/g' $CURDIR/pb2/messages_pb2.py -sed -i 's/^import types_pb2/from . import types_pb2/g' $CURDIR/pb2/storage_pb2.py - for i in types messages storage ; do # Convert google protobuf library to trezor's internal format cd $CURDIR - ./pb2py -P "trezorlib.protobuf" -p $CURDIR -l $INDEX $i ../trezorlib/messages/ + ./pb2py -P "trezorlib.protobuf" -p $CURDIR/pb2 -l $INDEX $i ../trezorlib/messages/ done rm -rf $CURDIR/pb2/ diff --git a/tools/pb2py b/tools/pb2py index 8df4abcad..4d9fb5bb6 100755 --- a/tools/pb2py +++ b/tools/pb2py @@ -9,7 +9,7 @@ import argparse def import_pb2(name): - return importlib.import_module("pb2.%s_pb2" % name) + return importlib.import_module("%s_pb2" % name) def create_message_import(name): @@ -163,8 +163,8 @@ def write_to_file(genpath, t, out): if __name__ == '__main__': parser = argparse.ArgumentParser() - parser.add_argument('module', type=import_pb2, help="Name of module to generate") - parser.add_argument('genpath', type=str, help="Directory for generated source code") + parser.add_argument('module', help="Name of module to generate") + parser.add_argument('genpath', help="Directory for generated source code") parser.add_argument('-P', '--protobuf-module', default="protobuf", help="Name of protobuf module") parser.add_argument('-i', '--indexfile', type=argparse.FileType('a'), help="Generate index file of wire types") parser.add_argument('-l', '--modlist', type=argparse.FileType('a'), help="Generate list of modules") @@ -175,4 +175,7 @@ if __name__ == '__main__': if args.protopath: sys.path.append(args.protopath) - process_file(args.module.DESCRIPTOR, args.protobuf_module, args.genpath, args.indexfile, args.modlist, args.micropython) + # This must be done after sys.path.append + module = import_pb2(args.module) + + process_file(module.DESCRIPTOR, args.protobuf_module, args.genpath, args.indexfile, args.modlist, args.micropython) From bbed79f658a6843e2c26064b49b6c1877b743402 Mon Sep 17 00:00:00 2001 From: Saleem Rashid Date: Sun, 24 Dec 2017 12:14:29 +0000 Subject: [PATCH 4/4] build_protobuf: Refactor * Use `set -e` to fail on error * Change into script directory automatically --- tools/build_protobuf | 35 +++++++++++++++++++++-------------- 1 file changed, 21 insertions(+), 14 deletions(-) diff --git a/tools/build_protobuf b/tools/build_protobuf index e46db4c55..8a30407ed 100755 --- a/tools/build_protobuf +++ b/tools/build_protobuf @@ -1,24 +1,31 @@ #!/bin/bash -CURDIR=$(pwd) -mkdir -p $CURDIR/pb2/ +set -e -mkdir -p ../trezorlib/messages +cd "$(dirname "$0")" -INDEX=../trezorlib/messages/__init__.py -rm -f $INDEX -echo '# Automatically generated by pb2py' >> $INDEX -echo '' >> $INDEX +GENPATH="../trezorlib/messages" +INDEX="$GENPATH/__init__.py" +PROTO_PATH="../../trezor-common/protob" +PROTO_FILES=(types messages storage) +PB2_OUT="pb2" -for i in types messages storage ; do +mkdir -p "$GENPATH" + +cat > "$INDEX" << EOF +# Automatically generated by pb2py + +EOF + +mkdir -p "$PB2_OUT" + +for file in "${PROTO_FILES[@]}"; do # Compile .proto files to python2 modules using google protobuf library - cd $CURDIR/../../trezor-common/protob - protoc --python_out=$CURDIR/pb2/ -I/usr/include -I. $i.proto + protoc --python_out="$PB2_OUT" -I"$PROTO_PATH" "$file.proto" done -for i in types messages storage ; do +for file in "${PROTO_FILES[@]}"; do # Convert google protobuf library to trezor's internal format - cd $CURDIR - ./pb2py -P "trezorlib.protobuf" -p $CURDIR/pb2 -l $INDEX $i ../trezorlib/messages/ + ./pb2py -P "trezorlib.protobuf" -p "$PB2_OUT" -l "$INDEX" "$file" "$GENPATH" done -rm -rf $CURDIR/pb2/ +rm -rf "$PB2_OUT"