From 2091ae97bf395e63903af1177216c3c080b85195 Mon Sep 17 00:00:00 2001 From: matejcik Date: Mon, 2 Jul 2018 18:39:33 +0200 Subject: [PATCH] tools: use pb2py from trezor-common --- tools/build_protobuf | 73 +-------------- tools/pb2py | 217 ------------------------------------------- 2 files changed, 5 insertions(+), 285 deletions(-) delete mode 100755 tools/pb2py diff --git a/tools/build_protobuf b/tools/build_protobuf index 04cc8d107..88c2565c0 100755 --- a/tools/build_protobuf +++ b/tools/build_protobuf @@ -1,72 +1,9 @@ #!/bin/bash set -e -IS_CORE="" +../vendor/trezor-common/protob/pb2py \ + --no-init-py \ + -o ../src/trezor/messages \ + ../vendor/trezor-common/protob/{types,messages}.proto -if [ "$1" == "--core" ]; then - shift - IS_CORE=yes -elif [ "$1" == "--no-core" ]; then - shift -elif echo $PWD | grep -q "trezor-core"; then - IS_CORE=yes -fi - -if [ -n "$1" ]; then - OUTDIR=`readlink -f "$1"` -fi - -cd "$(dirname "$0")" - -# set up paths -INDEX="__init__.py" -GENPATH="${OUTDIR:-../src/trezor/messages}" -PROTO_PATH="../vendor/trezor-common/protob" -PROTO_FILES="types messages" - -# set up temporary directory & cleanup -TMPDIR=$(mktemp -d) -function cleanup { - rm -r $TMPDIR -} -trap cleanup EXIT - -# set up pb2 outdir -PB2_OUT="$TMPDIR/pb2" -mkdir -p "$PB2_OUT" - -# compile .proto files to python2 modules using google protobuf library -for file in $PROTO_FILES; do - protoc --python_out="$PB2_OUT" -I/usr/include -I"$PROTO_PATH" "$PROTO_PATH/$file.proto" -done - - -if [ -n "$IS_CORE" ]; then - # generate for micropython - PB2PY_OPTS="-m" -else - # create index (__init__.py) - echo "# Automatically generated by pb2py" > $TMPDIR/$INDEX - echo >> $TMPDIR/$INDEX - PB2PY_OPTS="-l $TMPDIR/$INDEX" -fi - -# convert google protobuf library to trezor's internal format -for file in $PROTO_FILES; do - ./pb2py $PB2PY_OPTS -P "trezorlib.protobuf" -p "$PB2_OUT" "$file" "$TMPDIR" -done - -if [ -n "$IS_CORE" ]; then - cp "$TMPDIR/MessageType.py" "$TMPDIR/wire_types.py" -fi - -# ensure $GENPATH exists and is empty of messages -mkdir -p "$GENPATH" -# only remove messages - there could possibly be other files not starting with capital letter -rm -f "$GENPATH"/[A-Z]*.py - -# move generated files to the destination -# (this assumes $INDEX is *.py, otherwise we'd have to add $INDEX separately) -mv "$TMPDIR"/*.py "$GENPATH" - -# the exit trap handles removing the tmp directory +cp ../src/trezor/messages/MessageType.py ../src/trezor/messages/wire_types.py diff --git a/tools/pb2py b/tools/pb2py deleted file mode 100755 index 6ae458366..000000000 --- a/tools/pb2py +++ /dev/null @@ -1,217 +0,0 @@ -#!/usr/bin/env python3 -# Converts Google's protobuf python definitions of TREZOR wire messages -# to plain-python objects as used in TREZOR Core and python-trezor - -import argparse -import importlib -import logging -import os -import sys -from collections import namedtuple - -ProtoField = namedtuple('ProtoField', 'name, number, proto_type, py_type, repeated, required, orig') - - -def parse_field(number, field): - FIELD_TYPES = { - field.TYPE_UINT64: ('p.UVarintType', 'int'), - field.TYPE_UINT32: ('p.UVarintType', 'int'), - field.TYPE_ENUM: ('p.UVarintType', 'int'), - field.TYPE_SINT32: ('p.SVarintType', 'int'), - field.TYPE_SINT64: ('p.SVarintType', 'int'), - field.TYPE_STRING: ('p.UnicodeType', 'str'), - field.TYPE_BOOL: ('p.BoolType', 'bool'), - field.TYPE_BYTES: ('p.BytesType', 'bytes'), - } - repeated = (field.label == field.LABEL_REPEATED) - required = (field.label == field.LABEL_REQUIRED) - if field.type == field.TYPE_MESSAGE: - proto_type = py_type = field.message_type.name - else: - try: - proto_type, py_type = FIELD_TYPES[field.type] - except KeyError: - raise ValueError("Unknown field type %d for field %s" % (field.type, field.name)) from None - - if repeated: - py_type = "List[%s]" % py_type - - return ProtoField( - name=field.name, - number=number, - proto_type=proto_type, - py_type=py_type, - repeated=repeated, - required=required, - orig=field, - ) - - -def import_pb2(name): - return importlib.import_module("%s_pb2" % name) - - -def create_message_import(name): - return "from .%s import %s" % (name, name) - - -def remove_from_start(s, prefix): - if s.startswith(prefix): - return s[len(prefix):] - else: - return s - - -def process_message_imports(descriptor): - imports = set() - - for field in descriptor.fields: - if field.type == field.TYPE_MESSAGE: - imports.add(field.message_type.name) - - for name in sorted(imports): - yield create_message_import(name) - - -def create_init_method(fields): - yield " def __init__(" # noqa: E271 - yield " self," # noqa: E271 - for field in fields[:-1]: - yield " %s: %s = None," % (field.name, field.py_type) # noqa: E271 - # last field must not have a traling comma - yield " %s: %s = None" % (fields[-1].name, fields[-1].py_type) # noqa: E271 - yield " ) -> None:" # noqa: E271 - - for field in fields: - if field.repeated: - yield " self.{0} = {0} if {0} is not None else []".format(field.name) - else: - yield " self.{0} = {0}".format(field.name) - - -def process_message(descriptor, protobuf_module, msg_id, is_upy): - logging.debug("Processing message %s", descriptor.name) - - if is_upy: - yield "import protobuf as p" - else: - yield "from .. import protobuf as p" - - fields = list(parse_field(number, field) - for number, field - in descriptor.fields_by_number.items()) - - if any(field.repeated for field in fields): - yield "if __debug__:" - yield " try:" - yield " from typing import List" - yield " except ImportError:" - yield " List = None" - - yield from process_message_imports(descriptor) - - yield "" - yield "" - yield "class %s(p.MessageType):" % descriptor.name - - if msg_id is not None: - yield " MESSAGE_WIRE_TYPE = %d" % msg_id - - if fields: - yield " FIELDS = {" - for field in fields: - comments = [] - if field.required: - comments.append('required') - if field.orig.has_default_value: - comments.append("default=%s" % repr(field.orig.default_value)) - - if comments: - comment = " # %s" % ' '.join(comments) - else: - comment = '' - - if field.repeated: - flags = 'p.FLAG_REPEATED' - else: - flags = '0' - - yield " %d: ('%s', %s, %s),%s" % (field.number, field.name, field.proto_type, flags, comment) - - yield " }" - yield "" - yield from create_init_method(fields) - - if not fields and not msg_id: - yield " pass" - - -def process_enum(descriptor, is_upy): - logging.debug("Processing enum %s", descriptor.name) - - for name, value in descriptor.values_by_name.items(): - # Remove type name from the beginning of the constant - # For example "PinMatrixRequestType_Current" -> "Current" - enum_prefix = descriptor.name - name = remove_from_start(name, "%s_" % enum_prefix) - - # If type ends with *Type, but constant use type name without *Type, remove it too :) - # For example "ButtonRequestType & ButtonRequest_Other" => "Other" - if enum_prefix.endswith("Type"): - enum_prefix, _ = enum_prefix.rsplit("Type", 1) - name = remove_from_start(name, "%s_" % enum_prefix) - - yield "%s = %s" % (name, value.number) - - -def process_file(descriptor, protobuf_module, genpath, modlist, is_upy): - logging.info("Processing module %s", descriptor.name) - - msg_types = import_pb2('messages').MessageType - - for name, message_descriptor in sorted(descriptor.message_types_by_name.items()): - # Find message type for given class - try: - msg_id = msg_types.Value("MessageType_%s" % name) - except ValueError: - msg_id = None - - out = process_message(message_descriptor, protobuf_module, msg_id, is_upy) - write_to_file(genpath, name, out) - if modlist: - modlist.write(create_message_import(name) + "\n") - - for name, enum_descriptor in descriptor.enum_types_by_name.items(): - out = process_enum(enum_descriptor, is_upy) - write_to_file(genpath, name, out) - if modlist: - modlist.write("from . import %s\n" % name) - - -def write_to_file(genpath, t, out): - # Write generated sourcecode to given file - with open(os.path.join(genpath, "%s.py" % t), 'w') as f: - f.write("# Automatically generated by pb2py\n") - for line in out: - f.write(line + "\n") - - -if __name__ == '__main__': - logging.basicConfig(level=logging.DEBUG) - - parser = argparse.ArgumentParser() - parser.add_argument('module', help="Name of module to generate") - parser.add_argument('genpath', help="Directory for generated source code") - parser.add_argument('-P', '--protobuf-module', default="protobuf", help="Name of protobuf module") - parser.add_argument('-l', '--modlist', type=argparse.FileType('a'), help="Generate list of modules") - parser.add_argument('-p', '--protopath', type=str, help="Path to search for pregenerated Google's python sources") - parser.add_argument('-m', '--micropython', action='store_true', help="Use micropython-favoured source code") - args = parser.parse_args() - - if args.protopath: - sys.path.append(args.protopath) - - # This must be done after sys.path.append - module = import_pb2(args.module) - - process_file(module.DESCRIPTOR, args.protobuf_module, args.genpath, args.modlist, args.micropython)