From 90fd0bb67aaaf77691a76ae80522a48d5101caab Mon Sep 17 00:00:00 2001 From: Dusan Klinec Date: Thu, 1 Nov 2018 17:18:40 +0100 Subject: [PATCH 1/7] xmr: mlsag memory optimizations, in-place computation - pub key matrix is not ge25519 as it consumes high amount of memory - in-place computation used to reduce fragmentation overhead --- src/apps/monero/xmr/crypto/__init__.py | 1 + src/apps/monero/xmr/mlsag.py | 119 ++++++++++++++++--------- 2 files changed, 80 insertions(+), 40 deletions(-) diff --git a/src/apps/monero/xmr/crypto/__init__.py b/src/apps/monero/xmr/crypto/__init__.py index 9f746eec9..70fde1c46 100644 --- a/src/apps/monero/xmr/crypto/__init__.py +++ b/src/apps/monero/xmr/crypto/__init__.py @@ -107,6 +107,7 @@ def sc_init_into(r, x): return tcry.init256_modm(r, x) +sc_copy = tcry.init256_modm sc_get64 = tcry.get256_modm sc_check = tcry.check256_modm check_sc = tcry.check256_modm diff --git a/src/apps/monero/xmr/mlsag.py b/src/apps/monero/xmr/mlsag.py index 9aef687cb..095c3bcc9 100644 --- a/src/apps/monero/xmr/mlsag.py +++ b/src/apps/monero/xmr/mlsag.py @@ -42,6 +42,9 @@ and `sk` is equal to: Mostly ported from official Monero client, but also inspired by Mininero. Author: Dusan Klinec, ph4r05, 2018 """ + +import gc + from apps.monero.xmr import crypto @@ -68,33 +71,45 @@ def generate_mlsag_full( for i in range(rows + 1): sk[i] = crypto.sc_0() + tmp_mi_rows = crypto.new_point(None) + tmp_pt = crypto.new_point(None) + for i in range(cols): - M[i][rows] = crypto.identity() + crypto.identity_into(tmp_mi_rows) # M[i][rows] + for j in range(rows): - M[i][j] = crypto.decodepoint(pubs[i][j].dest) - M[i][rows] = crypto.point_add( - M[i][rows], crypto.decodepoint(pubs[i][j].commitment) + M[i][j] = pubs[i][j].dest + crypto.point_add_into( + tmp_mi_rows, + tmp_mi_rows, + crypto.decodepoint_into(tmp_pt, pubs[i][j].commitment), ) + pubs[i] = None - sk[rows] = crypto.sc_0() - for j in range(rows): - sk[j] = in_sk[j].dest - sk[rows] = crypto.sc_add(sk[rows], in_sk[j].mask) # add masks in last row - - for i in range(cols): for j in range(len(out_pk_commitments)): - M[i][rows] = crypto.point_sub( - M[i][rows], crypto.decodepoint(out_pk_commitments[j]) + crypto.point_sub_into( + tmp_mi_rows, + tmp_mi_rows, + crypto.decodepoint_into(tmp_pt, out_pk_commitments[j]), ) # subtract output Ci's in last row # Subtract txn fee output in last row - M[i][rows] = crypto.point_sub(M[i][rows], txn_fee_key) + crypto.point_sub_into(tmp_mi_rows, tmp_mi_rows, txn_fee_key) + M[i][rows] = crypto.encodepoint(tmp_mi_rows) + + sk[rows] = crypto.sc_0() + for j in range(rows): + sk[j] = in_sk[j].dest + crypto.sc_add_into(sk[rows], sk[rows], in_sk[j].mask) # add masks in last row for j in range(len(out_pk_commitments)): - sk[rows] = crypto.sc_sub( - sk[rows], out_sk_mask[j] + crypto.sc_sub_into( + sk[rows], sk[rows], out_sk_mask[j] ) # subtract output masks in last row + del (pubs, tmp_mi_rows, tmp_pt) + gc.collect() + return generate_mlsag(message, M, sk, kLRki, index, rows) @@ -123,10 +138,19 @@ def generate_mlsag_simple(message, pubs, in_sk, a, cout, kLRki, index): sk[0] = in_sk.dest sk[1] = crypto.sc_sub(in_sk.mask, a) + tmp_pt = crypto.new_point() for i in range(cols): - M[i][0] = crypto.decodepoint(pubs[i].dest) - M[i][1] = crypto.point_sub(crypto.decodepoint(pubs[i].commitment), cout) + crypto.point_sub_into( + tmp_pt, crypto.decodepoint_into(tmp_pt, pubs[i].commitment), cout + ) + + M[i][0] = pubs[i].dest + M[i][1] = crypto.encodepoint(tmp_pt) + pubs[i] = None + + del (pubs) + gc.collect() return generate_mlsag(message, M, sk, kLRki, index, dsRows) @@ -174,18 +198,19 @@ def generate_first_c_and_key_images( :param rows: total number of rows :param cols: size of ring """ - Ip = _key_vector(dsRows) rv.II = _key_vector(dsRows) alpha = _key_vector(rows) - rv.ss = _key_matrix(rows, cols) tmp_buff = bytearray(32) + Hi = crypto.new_point() + aGi = crypto.new_point() + aHPi = crypto.new_point() hasher = _hasher_message(message) for i in range(dsRows): # this is somewhat extra as compared to the Ring Confidential Tx paper # see footnote in From Zero to Monero section 3.3 - hasher.update(crypto.encodepoint(pk[index][i])) + hasher.update(pk[index][i]) if kLRki: raise NotImplementedError("Multisig not implemented") # alpha[i] = kLRki.k @@ -194,33 +219,31 @@ def generate_first_c_and_key_images( # hash_point(hasher, kLRki.R, tmp_buff) else: - Hi = crypto.hash_to_point(crypto.encodepoint(pk[index][i])) + crypto.hash_to_point_into(Hi, pk[index][i]) alpha[i] = crypto.random_scalar() # L = alpha_i * G - aGi = crypto.scalarmult_base(alpha[i]) + crypto.scalarmult_base_into(aGi, alpha[i]) # Ri = alpha_i * H(P_i) - aHPi = crypto.scalarmult(Hi, alpha[i]) + crypto.scalarmult_into(aHPi, Hi, alpha[i]) # key image rv.II[i] = crypto.scalarmult(Hi, xx[i]) _hash_point(hasher, aGi, tmp_buff) _hash_point(hasher, aHPi, tmp_buff) - Ip[i] = rv.II[i] - for i in range(dsRows, rows): alpha[i] = crypto.random_scalar() # L = alpha_i * G - aGi = crypto.scalarmult_base(alpha[i]) + crypto.scalarmult_base_into(aGi, alpha[i]) # for some reasons we omit calculating R here, which seems # contrary to the paper, but it is in the Monero official client # see https://github.com/monero-project/monero/blob/636153b2050aa0642ba86842c69ac55a5d81618d/src/ringct/rctSigs.cpp#L191 - _hash_point(hasher, pk[index][i], tmp_buff) + hasher.update(pk[index][i]) _hash_point(hasher, aGi, tmp_buff) # the first c c_old = hasher.digest() c_old = crypto.decodeint(c_old) - return c_old, Ip, alpha + return c_old, alpha def generate_mlsag(message, pk, xx, kLRki, index, dsRows): @@ -240,17 +263,22 @@ def generate_mlsag(message, pk, xx, kLRki, index, dsRows): rows, cols = gen_mlsag_assert(pk, xx, kLRki, index, dsRows) rv = MgSig() - c, L, R, Hi = 0, None, None, None + rv.cc = crypto.new_scalar() + c = crypto.new_scalar() + L = crypto.new_point() + R = crypto.new_point() + Hi = crypto.new_point() # calculates the "first" c, key images and random scalars alpha - c_old, Ip, alpha = generate_first_c_and_key_images( + c_old, alpha = generate_first_c_and_key_images( message, rv, pk, xx, kLRki, index, dsRows, rows, cols ) i = (index + 1) % cols if i == 0: - rv.cc = c_old + crypto.sc_copy(rv.cc, c_old) + rv.ss = [None] * cols tmp_buff = bytearray(32) while i != index: rv.ss[i] = _generate_random_vector(rows) @@ -258,27 +286,38 @@ def generate_mlsag(message, pk, xx, kLRki, index, dsRows): for j in range(dsRows): # L = rv.ss[i][j] * G + c_old * pk[i][j] - L = crypto.add_keys2(rv.ss[i][j], c_old, pk[i][j]) - Hi = crypto.hash_to_point(crypto.encodepoint(pk[i][j])) + crypto.add_keys2_into( + L, rv.ss[i][j], c_old, crypto.decodepoint_into(Hi, pk[i][j]) + ) + crypto.hash_to_point_into(Hi, pk[i][j]) + # R = rv.ss[i][j] * H(pk[i][j]) + c_old * Ip[j] - R = crypto.add_keys3(rv.ss[i][j], Hi, c_old, rv.II[j]) - _hash_point(hasher, pk[i][j], tmp_buff) + crypto.add_keys3_into(R, rv.ss[i][j], Hi, c_old, rv.II[j]) + + hasher.update(pk[i][j]) _hash_point(hasher, L, tmp_buff) _hash_point(hasher, R, tmp_buff) for j in range(dsRows, rows): # again, omitting R here as discussed above - L = crypto.add_keys2(rv.ss[i][j], c_old, pk[i][j]) - _hash_point(hasher, pk[i][j], tmp_buff) + crypto.add_keys2_into( + L, rv.ss[i][j], c_old, crypto.decodepoint_into(Hi, pk[i][j]) + ) + hasher.update(pk[i][j]) _hash_point(hasher, L, tmp_buff) - c = crypto.decodeint(hasher.digest()) - c_old = c + crypto.decodeint_into(c, hasher.digest()) + crypto.sc_copy(c_old, c) + pk[i] = None i = (i + 1) % cols if i == 0: - rv.cc = c_old + crypto.sc_copy(rv.cc, c_old) + gc.collect() + + del rv.II + rv.ss[index] = [None] * rows for j in range(rows): rv.ss[index][j] = crypto.sc_mulsub(c, xx[j], alpha[j]) From a462ea35ceea77488f6f59ebf8d67b228460b2c0 Mon Sep 17 00:00:00 2001 From: Dusan Klinec Date: Thu, 1 Nov 2018 18:20:31 +0100 Subject: [PATCH 2/7] xmr: step_09_sign - manual MgSig serialization --- src/apps/monero/signing/step_09_sign_input.py | 98 +++++++++++++------ 1 file changed, 69 insertions(+), 29 deletions(-) diff --git a/src/apps/monero/signing/step_09_sign_input.py b/src/apps/monero/signing/step_09_sign_input.py index c9ea302e1..a176bdf16 100644 --- a/src/apps/monero/signing/step_09_sign_input.py +++ b/src/apps/monero/signing/step_09_sign_input.py @@ -10,7 +10,8 @@ from .state import State from apps.monero.layout import confirms from apps.monero.signing import RctType -from apps.monero.xmr import crypto, serialize +from apps.monero.xmr import crypto +from apps.monero.xmr.serialize import int_serialize if False: from trezor.messages.MoneroTransactionSourceEntry import ( @@ -66,7 +67,7 @@ async def sign_input( raise ValueError("HMAC is not correct") gc.collect() - state.mem_trace(1) + state.mem_trace(1, True) if state.rct_type == RctType.Simple: # both pseudo_out and its mask were offloaded so we need to @@ -78,8 +79,7 @@ async def sign_input( if not crypto.ct_equals(pseudo_out_hmac_comp, pseudo_out_hmac): raise ValueError("HMAC is not correct") - gc.collect() - state.mem_trace(2) + state.mem_trace(2, True) from apps.monero.xmr.crypto import chacha_poly @@ -102,8 +102,7 @@ async def sign_input( ) ) - gc.collect() - state.mem_trace(3) + state.mem_trace(3, True) # Basic setup, sanity check index = src_entr.real_output @@ -126,13 +125,15 @@ async def sign_input( "Real source entry's mask does not equal spend key's", ) - gc.collect() - state.mem_trace(4) + state.mem_trace(4, True) + mg_buff = bytearray(_mg_size(len(src_entr.outputs))) from apps.monero.xmr import mlsag if state.rct_type == RctType.Simple: ring_pubkeys = [x.key for x in src_entr.outputs] + src_entr = None + mg = mlsag.generate_mlsag_simple( state.full_message, ring_pubkeys, @@ -143,10 +144,14 @@ async def sign_input( index, ) + del (ring_pubkeys, input_secret_key, pseudo_out_alpha, pseudo_out_c) + else: # Full RingCt, only one input txn_fee_key = crypto.scalarmult_h(state.fee) ring_pubkeys = [[x.key] for x in src_entr.outputs] + src_entr = None + mg = mlsag.generate_mlsag_full( state.full_message, ring_pubkeys, @@ -158,35 +163,70 @@ async def sign_input( txn_fee_key, ) - gc.collect() - state.mem_trace(5) + del (ring_pubkeys, input_secret_key, txn_fee_key) - # Encode - mgs = _recode_msg([mg]) + del (mlsag, src_entr) + state.mem_trace(5, True) - gc.collect() - state.mem_trace(6) + # Encode + mg_buffer = _mg_serialize(mg, mg_buff) + state.mem_trace(6, True) from trezor.messages.MoneroTransactionSignInputAck import ( MoneroTransactionSignInputAck, ) - return MoneroTransactionSignInputAck( - signature=serialize.dump_msg_gc(mgs[0], preallocate=488) - ) + return MoneroTransactionSignInputAck(signature=mg_buffer) + + +def _mg_size(num_outs): + """ + Computes size of the MgSig + :param num_outs: + :return: + """ + size = 32 # cc + mg_cols = num_outs + mg_rows = 2 + cols_b_size = int_serialize.uvarint_size(mg_cols) + rows_b_size = 1 + size += cols_b_size + mg_cols * (rows_b_size + mg_rows * 32) + return size -def _recode_msg(mgs): +def _mg_serialize(mg, buff): """ - Recodes MGs signatures from raw forms to bytearrays so it works with serialization + Serializes MgSig structure: (("ss", KeyM), ("cc", ECKey)) + :param mg: + :return: """ - for idx in range(len(mgs)): - mgs[idx].cc = crypto.encodeint(mgs[idx].cc) - if hasattr(mgs[idx], "II") and mgs[idx].II: - for i in range(len(mgs[idx].II)): - mgs[idx].II[i] = crypto.encodepoint(mgs[idx].II[i]) - - for i in range(len(mgs[idx].ss)): - for j in range(len(mgs[idx].ss[i])): - mgs[idx].ss[i][j] = crypto.encodeint(mgs[idx].ss[i][j]) - return mgs + size = len(buff) + mg_cols = len(mg.ss) + mg_rows = len(mg.ss[0]) + cols_b_size = int_serialize.uvarint_size(mg_cols) + rows_b_size = int_serialize.uvarint_size(mg_rows) + offset = 0 + + int_serialize.dump_uvarint_b_into(mg_cols, buff, offset) + offset += cols_b_size + + for i in range(mg_cols): + utils.ensure(len(mg.ss[i]) == mg_rows, "Irregular matrix shape") + + int_serialize.dump_uvarint_b_into(mg_rows, buff, offset) + offset += rows_b_size + + for j in range(mg_rows): + crypto.encodeint_into(buff, mg.ss[i][j], offset) + offset += 32 + + mg.ss[i] = None + gc.collect() + + mg.ss = None + + crypto.encodeint_into(buff, mg.cc, offset) + offset += 32 + + utils.ensure(offset == size, "Invalid mg size computation") + return buff From ddcb836ff7eed2bf2971ac788ecb187fa36fb012 Mon Sep 17 00:00:00 2001 From: Dusan Klinec Date: Fri, 2 Nov 2018 07:03:44 +0100 Subject: [PATCH 3/7] xmr: mlsag - generating serialized signature directly --- src/apps/monero/signing/step_09_sign_input.py | 50 +--------- src/apps/monero/xmr/mlsag.py | 96 +++++++++++++------ .../monero/xmr/serialize_messages/tx_full.py | 11 --- 3 files changed, 70 insertions(+), 87 deletions(-) delete mode 100644 src/apps/monero/xmr/serialize_messages/tx_full.py diff --git a/src/apps/monero/signing/step_09_sign_input.py b/src/apps/monero/signing/step_09_sign_input.py index a176bdf16..101722fda 100644 --- a/src/apps/monero/signing/step_09_sign_input.py +++ b/src/apps/monero/signing/step_09_sign_input.py @@ -126,7 +126,7 @@ async def sign_input( ) state.mem_trace(4, True) - mg_buff = bytearray(_mg_size(len(src_entr.outputs))) + mg_buffer = bytearray(_mg_size(len(src_entr.outputs))) from apps.monero.xmr import mlsag @@ -134,7 +134,7 @@ async def sign_input( ring_pubkeys = [x.key for x in src_entr.outputs] src_entr = None - mg = mlsag.generate_mlsag_simple( + mlsag.generate_mlsag_simple( state.full_message, ring_pubkeys, input_secret_key, @@ -142,6 +142,7 @@ async def sign_input( pseudo_out_c, kLRki, index, + mg_buffer, ) del (ring_pubkeys, input_secret_key, pseudo_out_alpha, pseudo_out_c) @@ -152,7 +153,7 @@ async def sign_input( ring_pubkeys = [[x.key] for x in src_entr.outputs] src_entr = None - mg = mlsag.generate_mlsag_full( + mlsag.generate_mlsag_full( state.full_message, ring_pubkeys, [input_secret_key], @@ -161,6 +162,7 @@ async def sign_input( kLRki, index, txn_fee_key, + mg_buffer, ) del (ring_pubkeys, input_secret_key, txn_fee_key) @@ -168,10 +170,6 @@ async def sign_input( del (mlsag, src_entr) state.mem_trace(5, True) - # Encode - mg_buffer = _mg_serialize(mg, mg_buff) - state.mem_trace(6, True) - from trezor.messages.MoneroTransactionSignInputAck import ( MoneroTransactionSignInputAck, ) @@ -192,41 +190,3 @@ def _mg_size(num_outs): rows_b_size = 1 size += cols_b_size + mg_cols * (rows_b_size + mg_rows * 32) return size - - -def _mg_serialize(mg, buff): - """ - Serializes MgSig structure: (("ss", KeyM), ("cc", ECKey)) - :param mg: - :return: - """ - size = len(buff) - mg_cols = len(mg.ss) - mg_rows = len(mg.ss[0]) - cols_b_size = int_serialize.uvarint_size(mg_cols) - rows_b_size = int_serialize.uvarint_size(mg_rows) - offset = 0 - - int_serialize.dump_uvarint_b_into(mg_cols, buff, offset) - offset += cols_b_size - - for i in range(mg_cols): - utils.ensure(len(mg.ss[i]) == mg_rows, "Irregular matrix shape") - - int_serialize.dump_uvarint_b_into(mg_rows, buff, offset) - offset += rows_b_size - - for j in range(mg_rows): - crypto.encodeint_into(buff, mg.ss[i][j], offset) - offset += 32 - - mg.ss[i] = None - gc.collect() - - mg.ss = None - - crypto.encodeint_into(buff, mg.cc, offset) - offset += 32 - - utils.ensure(offset == size, "Invalid mg size computation") - return buff diff --git a/src/apps/monero/xmr/mlsag.py b/src/apps/monero/xmr/mlsag.py index 095c3bcc9..ac12f0255 100644 --- a/src/apps/monero/xmr/mlsag.py +++ b/src/apps/monero/xmr/mlsag.py @@ -45,11 +45,22 @@ Author: Dusan Klinec, ph4r05, 2018 import gc +from trezor import utils + from apps.monero.xmr import crypto +from apps.monero.xmr.serialize import int_serialize def generate_mlsag_full( - message, pubs, in_sk, out_sk_mask, out_pk_commitments, kLRki, index, txn_fee_key + message, + pubs, + in_sk, + out_sk_mask, + out_pk_commitments, + kLRki, + index, + txn_fee_key, + mg_buff, ): cols = len(pubs) if cols == 0: @@ -110,10 +121,10 @@ def generate_mlsag_full( del (pubs, tmp_mi_rows, tmp_pt) gc.collect() - return generate_mlsag(message, M, sk, kLRki, index, rows) + return generate_mlsag(message, M, sk, kLRki, index, rows, mg_buff) -def generate_mlsag_simple(message, pubs, in_sk, a, cout, kLRki, index): +def generate_mlsag_simple(message, pubs, in_sk, a, cout, kLRki, index, mg_buff): """ MLSAG for RctType.Simple :param message: the full message to be signed (actually its hash) @@ -123,7 +134,7 @@ def generate_mlsag_simple(message, pubs, in_sk, a, cout, kLRki, index): :param cout: pseudo output commitment; point, decoded; better name: pseudo_out_c :param kLRki: used only in multisig, currently not implemented :param index: specifies corresponding public key to the `in_sk` in the pubs array - :return: MgSig + :param mg_buff: buffer to store the signature to """ # Monero signs inputs separately, so `rows` always equals 2 (pubkey, commitment) # and `dsRows` is always 1 (denotes where the pubkeys "end") @@ -152,7 +163,7 @@ def generate_mlsag_simple(message, pubs, in_sk, a, cout, kLRki, index): del (pubs) gc.collect() - return generate_mlsag(message, M, sk, kLRki, index, dsRows) + return generate_mlsag(message, M, sk, kLRki, index, dsRows, mg_buff) def gen_mlsag_assert(pk, xx, kLRki, index, dsRows): @@ -183,13 +194,10 @@ def gen_mlsag_assert(pk, xx, kLRki, index, dsRows): return rows, cols -def generate_first_c_and_key_images( - message, rv, pk, xx, kLRki, index, dsRows, rows, cols -): +def generate_first_c_and_key_images(message, pk, xx, kLRki, index, dsRows, rows, cols): """ MLSAG computation - the part with secret keys :param message: the full message to be signed (actually its hash) - :param rv: MgSig :param pk: matrix of public keys and commitments :param xx: input secret array composed of a private key and commitment mask :param kLRki: used only in multisig, currently not implemented @@ -198,7 +206,7 @@ def generate_first_c_and_key_images( :param rows: total number of rows :param cols: size of ring """ - rv.II = _key_vector(dsRows) + II = _key_vector(dsRows) alpha = _key_vector(rows) tmp_buff = bytearray(32) @@ -226,7 +234,7 @@ def generate_first_c_and_key_images( # Ri = alpha_i * H(P_i) crypto.scalarmult_into(aHPi, Hi, alpha[i]) # key image - rv.II[i] = crypto.scalarmult(Hi, xx[i]) + II[i] = crypto.scalarmult(Hi, xx[i]) _hash_point(hasher, aGi, tmp_buff) _hash_point(hasher, aHPi, tmp_buff) @@ -243,10 +251,10 @@ def generate_first_c_and_key_images( # the first c c_old = hasher.digest() c_old = crypto.decodeint(c_old) - return c_old, alpha + return c_old, II, alpha -def generate_mlsag(message, pk, xx, kLRki, index, dsRows): +def generate_mlsag(message, pk, xx, kLRki, index, dsRows, mg_buff): """ Multilayered Spontaneous Anonymous Group Signatures (MLSAG signatures) @@ -256,43 +264,57 @@ def generate_mlsag(message, pk, xx, kLRki, index, dsRows): :param kLRki: used only in multisig, currently not implemented :param index: specifies corresponding public key to the `xx`'s private key in the `pk` array :param dsRows: separates pubkeys from commitment - :return MgSig + :param mg_buff: mg signature buffer """ - from apps.monero.xmr.serialize_messages.tx_full import MgSig - rows, cols = gen_mlsag_assert(pk, xx, kLRki, index, dsRows) + rows_b_size = int_serialize.uvarint_size(rows) + cols_b_size = int_serialize.uvarint_size(cols) + int_serialize.dump_uvarint_b_into(cols, mg_buff) - rv = MgSig() - rv.cc = crypto.new_scalar() + # Computes offset to the mg_buffer + # mg_buffer format: (("ss", KeyM), ("cc", ECKey)) + # ss[i][j], i over cols, j over rows + def buff_offset(col): + return cols_b_size + col * (rows_b_size + rows * 32) + + cc = crypto.new_scalar() # rv.cc c = crypto.new_scalar() L = crypto.new_point() R = crypto.new_point() Hi = crypto.new_point() # calculates the "first" c, key images and random scalars alpha - c_old, alpha = generate_first_c_and_key_images( - message, rv, pk, xx, kLRki, index, dsRows, rows, cols + c_old, II, alpha = generate_first_c_and_key_images( + message, pk, xx, kLRki, index, dsRows, rows, cols ) i = (index + 1) % cols if i == 0: - crypto.sc_copy(rv.cc, c_old) + crypto.sc_copy(cc, c_old) - rv.ss = [None] * cols + ss = [crypto.new_scalar() for _ in range(rows)] tmp_buff = bytearray(32) + while i != index: - rv.ss[i] = _generate_random_vector(rows) hasher = _hasher_message(message) + # Serialize size of the row + offset = buff_offset(i) + int_serialize.dump_uvarint_b_into(rows, mg_buff, offset) + offset += rows_b_size + + for x in ss: + crypto.random_scalar(x) + for j in range(dsRows): # L = rv.ss[i][j] * G + c_old * pk[i][j] crypto.add_keys2_into( - L, rv.ss[i][j], c_old, crypto.decodepoint_into(Hi, pk[i][j]) + L, ss[j], c_old, crypto.decodepoint_into(Hi, pk[i][j]) ) crypto.hash_to_point_into(Hi, pk[i][j]) # R = rv.ss[i][j] * H(pk[i][j]) + c_old * Ip[j] - crypto.add_keys3_into(R, rv.ss[i][j], Hi, c_old, rv.II[j]) + crypto.add_keys3_into(R, ss[j], Hi, c_old, II[j]) hasher.update(pk[i][j]) _hash_point(hasher, L, tmp_buff) @@ -301,27 +323,39 @@ def generate_mlsag(message, pk, xx, kLRki, index, dsRows): for j in range(dsRows, rows): # again, omitting R here as discussed above crypto.add_keys2_into( - L, rv.ss[i][j], c_old, crypto.decodepoint_into(Hi, pk[i][j]) + L, ss[j], c_old, crypto.decodepoint_into(Hi, pk[i][j]) ) hasher.update(pk[i][j]) _hash_point(hasher, L, tmp_buff) + for si in range(rows): + crypto.encodeint_into(tmp_buff, ss[si]) + utils.memcpy(mg_buff, offset + 32 * si, tmp_buff, 0, 32) + crypto.decodeint_into(c, hasher.digest()) crypto.sc_copy(c_old, c) pk[i] = None i = (i + 1) % cols if i == 0: - crypto.sc_copy(rv.cc, c_old) + crypto.sc_copy(cc, c_old) gc.collect() - del rv.II + del II + + # Finalizing rv.ss by processing rv.ss[index] + offset = buff_offset(index) + int_serialize.dump_uvarint_b_into(rows, mg_buff, offset) + offset += rows_b_size - rv.ss[index] = [None] * rows for j in range(rows): - rv.ss[index][j] = crypto.sc_mulsub(c, xx[j], alpha[j]) + crypto.sc_mulsub_into(ss[j], c, xx[j], alpha[j]) + crypto.encodeint_into(tmp_buff, ss[j]) + utils.memcpy(mg_buff, offset + 32 * j, tmp_buff, 0, 32) - return rv + # rv.cc + utils.memcpy(mg_buff, len(mg_buff) - 32, crypto.encodeint_into(tmp_buff, cc), 0, 32) + utils.ensure(buff_offset(cols) + 32 == len(mg_buff), "Invalid mg_buff size") def _key_vector(rows): diff --git a/src/apps/monero/xmr/serialize_messages/tx_full.py b/src/apps/monero/xmr/serialize_messages/tx_full.py deleted file mode 100644 index 1c98619b6..000000000 --- a/src/apps/monero/xmr/serialize_messages/tx_full.py +++ /dev/null @@ -1,11 +0,0 @@ -from apps.monero.xmr.serialize.message_types import MessageType -from apps.monero.xmr.serialize_messages.base import ECKey -from apps.monero.xmr.serialize_messages.ct_keys import KeyM - - -class MgSig(MessageType): - __slots__ = ("ss", "cc", "II") - - @classmethod - def f_specs(cls): - return (("ss", KeyM), ("cc", ECKey)) From cdf9d51ee2804dccf6701d20a4f0ad45f041b2a6 Mon Sep 17 00:00:00 2001 From: Dusan Klinec Date: Fri, 2 Nov 2018 07:25:12 +0100 Subject: [PATCH 4/7] xmr: step_09_sign - unimport after heavy ops --- src/apps/monero/signing/step_09_sign_input.py | 23 +++++++++++++------ 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/src/apps/monero/signing/step_09_sign_input.py b/src/apps/monero/signing/step_09_sign_input.py index 101722fda..3a87445fe 100644 --- a/src/apps/monero/signing/step_09_sign_input.py +++ b/src/apps/monero/signing/step_09_sign_input.py @@ -41,8 +41,6 @@ async def sign_input( :param spend_enc: one time address spending private key. Encrypted. :return: Generated signature MGs[i] """ - from apps.monero.signing import offloading_keys - await confirms.transaction_step( state.ctx, state.STEP_SIGN, state.current_input_index + 1, state.input_count ) @@ -58,8 +56,11 @@ async def sign_input( raise ValueError("Two and more inputs must imply SimpleRCT") input_position = state.source_permutation[state.current_input_index] + mods = utils.unimport_begin() # Check input's HMAC + from apps.monero.signing import offloading_keys + vini_hmac_comp = await offloading_keys.gen_hmac_vini( state.key_hmac, src_entr, vini_bin, input_position ) @@ -69,6 +70,8 @@ async def sign_input( gc.collect() state.mem_trace(1, True) + from apps.monero.xmr.crypto import chacha_poly + if state.rct_type == RctType.Simple: # both pseudo_out and its mask were offloaded so we need to # validate pseudo_out's HMAC and decrypt the alpha @@ -81,8 +84,6 @@ async def sign_input( state.mem_trace(2, True) - from apps.monero.xmr.crypto import chacha_poly - pseudo_out_alpha = crypto.decodeint( chacha_poly.decrypt_pack( offloading_keys.enc_key_txin_alpha(state.key_enc, input_position), @@ -92,9 +93,6 @@ async def sign_input( pseudo_out_c = crypto.decodepoint(pseudo_out) # Spending secret - from apps.monero.xmr.crypto import chacha_poly - from apps.monero.xmr.serialize_messages.ct_keys import CtKey - spend_key = crypto.decodeint( chacha_poly.decrypt_pack( offloading_keys.enc_key_spend(state.key_enc, input_position), @@ -102,8 +100,19 @@ async def sign_input( ) ) + del ( + offloading_keys, + chacha_poly, + pseudo_out, + pseudo_out_hmac, + pseudo_out_alpha_enc, + spend_enc, + ) + utils.unimport_end(mods) state.mem_trace(3, True) + from apps.monero.xmr.serialize_messages.ct_keys import CtKey + # Basic setup, sanity check index = src_entr.real_output input_secret_key = CtKey(dest=spend_key, mask=crypto.decodeint(src_entr.mask)) From 0414a8e74b098e4e3f51aedac3b8169b0d920cdd Mon Sep 17 00:00:00 2001 From: Dusan Klinec Date: Fri, 2 Nov 2018 08:10:35 +0100 Subject: [PATCH 5/7] protobuf: enable dumping large byte arrays in chunks --- src/protobuf.py | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/src/protobuf.py b/src/protobuf.py index 654c71e98..414f5c0a9 100644 --- a/src/protobuf.py +++ b/src/protobuf.py @@ -275,6 +275,11 @@ async def dump_message(writer, msg, fields=None): elif ftype is BoolType: await dump_uvarint(writer, int(svalue)) + elif ftype is BytesType and is_chunked(svalue): + await dump_uvarint(writer, len_list_bytes(svalue)) + for sub_svalue in svalue: + await writer.awrite(sub_svalue) + elif ftype is BytesType: await dump_uvarint(writer, len(svalue)) await writer.awrite(svalue) @@ -329,7 +334,9 @@ def count_message(msg, fields=None): elif ftype is BytesType: for svalue in fvalue: - svalue = len(svalue) + svalue = ( + len(svalue) if not is_chunked(svalue) else len_list_bytes(svalue) + ) nbytes += count_uvarint(svalue) nbytes += svalue @@ -351,3 +358,18 @@ def count_message(msg, fields=None): raise TypeError return nbytes + + +def is_chunked(svalue): + return ( + isinstance(svalue, list) + and len(svalue) > 0 + and not isinstance(svalue[0], (int, bool)) + ) + + +def len_list_bytes(svalue): + res = 0 + for x in svalue: + res += len(x) + return res From 3d66ba153507a23fd877b85be52514fc54c7e694 Mon Sep 17 00:00:00 2001 From: Dusan Klinec Date: Fri, 2 Nov 2018 08:12:00 +0100 Subject: [PATCH 6/7] xmr: mgsig generated in chunks --- src/apps/monero/signing/step_09_sign_input.py | 18 +--------- src/apps/monero/xmr/mlsag.py | 33 +++++++------------ 2 files changed, 12 insertions(+), 39 deletions(-) diff --git a/src/apps/monero/signing/step_09_sign_input.py b/src/apps/monero/signing/step_09_sign_input.py index 3a87445fe..8d4249545 100644 --- a/src/apps/monero/signing/step_09_sign_input.py +++ b/src/apps/monero/signing/step_09_sign_input.py @@ -11,7 +11,6 @@ from .state import State from apps.monero.layout import confirms from apps.monero.signing import RctType from apps.monero.xmr import crypto -from apps.monero.xmr.serialize import int_serialize if False: from trezor.messages.MoneroTransactionSourceEntry import ( @@ -135,7 +134,7 @@ async def sign_input( ) state.mem_trace(4, True) - mg_buffer = bytearray(_mg_size(len(src_entr.outputs))) + mg_buffer = [] from apps.monero.xmr import mlsag @@ -184,18 +183,3 @@ async def sign_input( ) return MoneroTransactionSignInputAck(signature=mg_buffer) - - -def _mg_size(num_outs): - """ - Computes size of the MgSig - :param num_outs: - :return: - """ - size = 32 # cc - mg_cols = num_outs - mg_rows = 2 - cols_b_size = int_serialize.uvarint_size(mg_cols) - rows_b_size = 1 - size += cols_b_size + mg_cols * (rows_b_size + mg_rows * 32) - return size diff --git a/src/apps/monero/xmr/mlsag.py b/src/apps/monero/xmr/mlsag.py index ac12f0255..944c1b8af 100644 --- a/src/apps/monero/xmr/mlsag.py +++ b/src/apps/monero/xmr/mlsag.py @@ -45,8 +45,6 @@ Author: Dusan Klinec, ph4r05, 2018 import gc -from trezor import utils - from apps.monero.xmr import crypto from apps.monero.xmr.serialize import int_serialize @@ -268,15 +266,12 @@ def generate_mlsag(message, pk, xx, kLRki, index, dsRows, mg_buff): """ rows, cols = gen_mlsag_assert(pk, xx, kLRki, index, dsRows) rows_b_size = int_serialize.uvarint_size(rows) - cols_b_size = int_serialize.uvarint_size(cols) - int_serialize.dump_uvarint_b_into(cols, mg_buff) - # Computes offset to the mg_buffer - # mg_buffer format: (("ss", KeyM), ("cc", ECKey)) - # ss[i][j], i over cols, j over rows - def buff_offset(col): - return cols_b_size + col * (rows_b_size + rows * 32) + # Preallocation of the chunked buffer, len + cols + cc + for _ in range(1 + cols + 1): + mg_buff.append(None) + mg_buff[0] = int_serialize.dump_uvarint_b(cols) cc = crypto.new_scalar() # rv.cc c = crypto.new_scalar() L = crypto.new_point() @@ -299,9 +294,8 @@ def generate_mlsag(message, pk, xx, kLRki, index, dsRows, mg_buff): hasher = _hasher_message(message) # Serialize size of the row - offset = buff_offset(i) - int_serialize.dump_uvarint_b_into(rows, mg_buff, offset) - offset += rows_b_size + mg_buff[i + 1] = bytearray(rows_b_size + 32 * rows) + int_serialize.dump_uvarint_b_into(rows, mg_buff[i + 1]) for x in ss: crypto.random_scalar(x) @@ -329,8 +323,7 @@ def generate_mlsag(message, pk, xx, kLRki, index, dsRows, mg_buff): _hash_point(hasher, L, tmp_buff) for si in range(rows): - crypto.encodeint_into(tmp_buff, ss[si]) - utils.memcpy(mg_buff, offset + 32 * si, tmp_buff, 0, 32) + crypto.encodeint_into(mg_buff[i + 1], ss[si], rows_b_size + 32 * si) crypto.decodeint_into(c, hasher.digest()) crypto.sc_copy(c_old, c) @@ -344,18 +337,14 @@ def generate_mlsag(message, pk, xx, kLRki, index, dsRows, mg_buff): del II # Finalizing rv.ss by processing rv.ss[index] - offset = buff_offset(index) - int_serialize.dump_uvarint_b_into(rows, mg_buff, offset) - offset += rows_b_size - + mg_buff[index + 1] = bytearray(rows_b_size + 32 * rows) + int_serialize.dump_uvarint_b_into(rows, mg_buff[index + 1]) for j in range(rows): crypto.sc_mulsub_into(ss[j], c, xx[j], alpha[j]) - crypto.encodeint_into(tmp_buff, ss[j]) - utils.memcpy(mg_buff, offset + 32 * j, tmp_buff, 0, 32) + crypto.encodeint_into(mg_buff[index + 1], ss[j], rows_b_size + 32 * j) # rv.cc - utils.memcpy(mg_buff, len(mg_buff) - 32, crypto.encodeint_into(tmp_buff, cc), 0, 32) - utils.ensure(buff_offset(cols) + 32 == len(mg_buff), "Invalid mg_buff size") + mg_buff[-1] = crypto.encodeint(cc) def _key_vector(rows): From 22add1d5151e199101750fe0bf4b8cc7fcd437be Mon Sep 17 00:00:00 2001 From: Dusan Klinec Date: Fri, 2 Nov 2018 14:17:53 +0100 Subject: [PATCH 7/7] xmr: full mlsag simplified --- src/apps/monero/signing/step_09_sign_input.py | 19 +++++------ src/apps/monero/xmr/mlsag.py | 33 +++++++------------ 2 files changed, 19 insertions(+), 33 deletions(-) diff --git a/src/apps/monero/signing/step_09_sign_input.py b/src/apps/monero/signing/step_09_sign_input.py index 8d4249545..73c7566eb 100644 --- a/src/apps/monero/signing/step_09_sign_input.py +++ b/src/apps/monero/signing/step_09_sign_input.py @@ -134,14 +134,14 @@ async def sign_input( ) state.mem_trace(4, True) - mg_buffer = [] from apps.monero.xmr import mlsag - if state.rct_type == RctType.Simple: - ring_pubkeys = [x.key for x in src_entr.outputs] - src_entr = None + mg_buffer = [] + ring_pubkeys = [x.key for x in src_entr.outputs] + del src_entr + if state.rct_type == RctType.Simple: mlsag.generate_mlsag_simple( state.full_message, ring_pubkeys, @@ -153,18 +153,15 @@ async def sign_input( mg_buffer, ) - del (ring_pubkeys, input_secret_key, pseudo_out_alpha, pseudo_out_c) + del (input_secret_key, pseudo_out_alpha, pseudo_out_c) else: # Full RingCt, only one input txn_fee_key = crypto.scalarmult_h(state.fee) - ring_pubkeys = [[x.key] for x in src_entr.outputs] - src_entr = None - mlsag.generate_mlsag_full( state.full_message, ring_pubkeys, - [input_secret_key], + input_secret_key, state.output_sk_masks, state.output_pk_commitments, kLRki, @@ -173,9 +170,9 @@ async def sign_input( mg_buffer, ) - del (ring_pubkeys, input_secret_key, txn_fee_key) + del (input_secret_key, txn_fee_key) - del (mlsag, src_entr) + del (mlsag, ring_pubkeys) state.mem_trace(5, True) from trezor.messages.MoneroTransactionSignInputAck import ( diff --git a/src/apps/monero/xmr/mlsag.py b/src/apps/monero/xmr/mlsag.py index 944c1b8af..06b47fd1b 100644 --- a/src/apps/monero/xmr/mlsag.py +++ b/src/apps/monero/xmr/mlsag.py @@ -63,22 +63,12 @@ def generate_mlsag_full( cols = len(pubs) if cols == 0: raise ValueError("Empty pubs") - rows = len(pubs[0]) - if rows == 0: - raise ValueError("Empty pub row") - for i in range(cols): - if len(pubs[i]) != rows: - raise ValueError("pub is not rectangular") - - if len(in_sk) != rows: - raise ValueError("Bad inSk size") + rows = 1 # Monero uses only one row if len(out_sk_mask) != len(out_pk_commitments): raise ValueError("Bad outsk/putpk size") sk = _key_vector(rows + 1) M = _key_matrix(rows + 1, cols) - for i in range(rows + 1): - sk[i] = crypto.sc_0() tmp_mi_rows = crypto.new_point(None) tmp_pt = crypto.new_point(None) @@ -86,13 +76,13 @@ def generate_mlsag_full( for i in range(cols): crypto.identity_into(tmp_mi_rows) # M[i][rows] - for j in range(rows): - M[i][j] = pubs[i][j].dest - crypto.point_add_into( - tmp_mi_rows, - tmp_mi_rows, - crypto.decodepoint_into(tmp_pt, pubs[i][j].commitment), - ) + # Should iterate over rows, simplified as rows == 1 + M[i][0] = pubs[i].dest + crypto.point_add_into( + tmp_mi_rows, + tmp_mi_rows, + crypto.decodepoint_into(tmp_pt, pubs[i].commitment), + ) pubs[i] = None for j in range(len(out_pk_commitments)): @@ -106,10 +96,9 @@ def generate_mlsag_full( crypto.point_sub_into(tmp_mi_rows, tmp_mi_rows, txn_fee_key) M[i][rows] = crypto.encodepoint(tmp_mi_rows) - sk[rows] = crypto.sc_0() - for j in range(rows): - sk[j] = in_sk[j].dest - crypto.sc_add_into(sk[rows], sk[rows], in_sk[j].mask) # add masks in last row + # Simplified as rows == 1 + sk[0] = in_sk.dest + sk[rows] = in_sk.mask # originally: sum of all in_sk[0..rows] in sk[rows] for j in range(len(out_pk_commitments)): crypto.sc_sub_into(