2016-09-27 12:06:57 +00:00
|
|
|
#!/usr/bin/env python3
|
2019-05-17 12:22:36 +00:00
|
|
|
import glob
|
2016-09-27 12:06:57 +00:00
|
|
|
import os
|
2019-05-17 12:22:36 +00:00
|
|
|
import re
|
|
|
|
import shutil
|
|
|
|
import subprocess
|
|
|
|
import sys
|
|
|
|
import tempfile
|
2021-11-18 11:30:27 +00:00
|
|
|
from pathlib import Path
|
2019-05-17 12:22:36 +00:00
|
|
|
|
2021-11-18 11:30:27 +00:00
|
|
|
CORE_DIR = Path(__file__).parent.parent.resolve()
|
|
|
|
EXTMOD_PATH = CORE_DIR / "embed" / "extmod"
|
|
|
|
MOCKS_PATH = CORE_DIR / "mocks" / "generated"
|
2016-09-27 12:06:57 +00:00
|
|
|
|
2018-07-31 09:35:09 +00:00
|
|
|
COMMENT_PREFIX = "/// "
|
2016-09-27 12:06:57 +00:00
|
|
|
|
2017-06-14 15:39:32 +00:00
|
|
|
current_indent = 0
|
|
|
|
current_class = None
|
2016-09-27 12:06:57 +00:00
|
|
|
current_method = None
|
|
|
|
current_package = None
|
|
|
|
|
2017-06-13 14:50:03 +00:00
|
|
|
|
2017-04-08 16:43:26 +00:00
|
|
|
def split_to_parts(line, mod_desc=None):
|
2017-06-14 15:39:32 +00:00
|
|
|
global current_indent
|
|
|
|
global current_class
|
2016-09-27 12:06:57 +00:00
|
|
|
global current_method
|
xmr: master merging commit
templates: build
style: correct math in comment
[260130c1] xmr: show address fix
[abd27e6c] xmr: reorganize module structure
[cd9e5a5d] xmr: simplify layout code
[a5b56f17] xmr: monero.crypto test fix
[60bc30ec] xmr: minor fixes
[f82bd9c0] xmr: use trezor.utils.ensure
[adf119ac] xmr: get rid of xmr.common module
[3531a42f] pipenv: temporary fix attempt for travis
- until pipenv bug is resolved
[d172f86f] xmr: protob messages refactored
[e83085ea] trezor-common version bump & messages regenerated
xmr: mlsag_hasher simplified
[feb5f1c6] xmr: simplify key_image_sync workflow
[d4cb0084] xmr: tiny note in README and typo
[62411cde] xmr: readme updates
[ff15b46e] xmr: rename and order mlsag functions
[3fb57dab] xmr: mlsag notes
[c27ae90b] xmr: output index check added in step6
[973c457b] xmr: grouping is mandatory
[684c7e19] xmr: range sig grouping check added
[012ca766] xmr: small refactor in borromean range proof
[1ba72b65] xmr: move range signatures to seperate file; rename mlsag2 to mlsag
[a89f3ab0] xmr: fix wrong annotation in modtrezorcrypto
[8303b42f] xmr: state's use_simple_rct and use_bulletproof modified to enums
[276712a9] xmr: re-export most of functions in apps.monero.xmr.crypto
[74165457] xmr: out_pk_masks changed to out_pk_commitments
[1e186727] xmr: state comments
[3c69a2e1] xmr: TrezorTxPrefixHashNotMatchingError note removed
only concerns multisig
[5af0feae] xmr: master merging commit
xmr: step 10 review
[d8e99375] xmr: step 09 review
[a510150d] travis: workaround form 6a0ea22b6eb3bdc9bce62d7c88fb8841ef8fdb0c
[03d27115] mocks: regenerate
[bd24bb35] mocks: add support for entering the global scope
[f75c1902] mocks: regenerate
[bce85968] modtrezorcrypto: define mock package
[7c077521] mocks: support package definition
[b3f1017b] xmr: step 05 and 06 masks and range proofs review
Masks are now always generated in step 5 and stored in state.
Range proofs were reviewed only in a high-level manner and will be
reviewed later.
[67f391cd] xmr: step 08 review
[673bf01f] xmr: steps 04, 05, 06 (almost) and 07 review
_range_proof in step 06 is still to be reviewed
[24c52517] xmr: simplify serialization, remove Archive
[896cdebf] xmr: redundant exception removed
[cb3813aa] xmr: serializer simplified
[471213b4] xmr: serializer flake8 fix
[9d4df17f] xmr: aescbc not needed
[52dd8b3b] xmr: serializer - erefs kicked out
[9e3be78f] xmr: sign cleanup, comments
- state cleanup, comments added, unused code removed
[164a7d6a] xmr: sign step 09 - fix in_memory artifact, dead branch
[8fa4066d] xmr: sign step 03 - permutation length check added
[55a593aa] xmr: multisig removed from protocol and functions
[6470678d] xmr: black styling
[02664401] xmr: step 03 review
[daf7b7d6] xmr: step 02 review
xmr: adding agent tests to travis
[c752866f] xmr: unused imports removed
[85115fd9] xmr: serializer test fixed
- removed unsupported messages after serialization simplification
[60874751] xmr: sign step 06 - comment on bulletproof hashing
- hash_bp(bp) != hash(serialize(bp)) because hash does not contain array lengths
[4abb5473] xmr: sign protocol - multisig logic removed
[813cb3a0] xmr: lite protocol removed
- backup left in xmr-total-full-with-lite branch
[e5f5b5bd] xmr: serialization slimming
- base types reduced, not needed for now
- some int serialization methods not used now
[fb515aaf] xmr: serialization - archive simplified
- simple parameter passing, no kwargs
- unused methods removed
- reader/writer passing removed for archive methods
[ec4c4adb] xmr: KeccakXmrArchive simplified, no archive used
- getting rid of container_size. We dont use containers with fixed size so this special case can be abandoned.
- KeccakXmrArchive is lighweight without need to touch main serialization
[90065bd4] xmr: serialization - serialize_archive removed
- custom serialization routine is not required at this moment
[b98c2f89] xmr: extra serialization refactored, manual serialization
- extra is serialized manually to reduce serialization overhead
- extra contains simple structures now:
- payment ID = already serialized manually
- tx pub key = easy to serialize manually
- tx additional pub keys = serialized manually with little effort, more efficient memory usage
[8ce28a54] xmr: state 6 - provided tx keys removed
- needed only in the multisig scenario which is pruned now
[8d827f46] xmr: PreMlsagHasher pseudo out hashing fix
[49e552da] xmr: redundant import removed
[4199943d] xmr: KeccakXmrArchive simplified
[69bbf5fa] xmr: PreMlsagHasher - KeyV import removed, comment added
[9a194fa2] xmr: step7 - manual hashing of Extra
[d8a09283] xmr: PreMlsagHasher state load/save removed
- not needed, state not serialized anymore
[d5f43fa4] xmr: serialize reimport removed
- complex types are not surviving protocol boundary anymore, no need to fix hierarchy problems due to unimporting
[3b045612] xmr: serialize - TupleType removed
[b9a56981] xmr: serialization schemes simplified
[a59dbb8d] xmr: HashWrapper removed
[d0d1f05b] xmr: step 01 cleanup and comments
[8f7a7782] xmr: isort, black, flake8 fixes
[14265eb2] xmr: getting rid of CtKey from the state
- only lightweight objects are kept in the state. CtKey is import heavy object. Each set_out call locally imports a new own version of the Ctkey which causes a memory leak.
[c0cfc203] xmr: set_out minor function call fix
[c11c4686] xmr: sign_tx unimport optimization to reduce fragmentation
[df0a1dfc] xmr: range_sig allocation reordering, large chunks first
[63cddd5f] xmr: remove misc.StdObj
[8c8e3f33] xmr: proper memory usage in workflow
[9be1e0ac] xmr: typos and renames
[87f718b5] xmr: back to flat workflow
[64751339] xmr: refactor builder to seperate steps
- lot of work to be done, but the general idea will probably stay
- the messages workflow works, but the signed tx was not accepted by
daemon, so there is a bug somewhere
- additional cleanup/refactoring is defintely needed
[14b0a851] xmr: iface modified to layout
[27d568e1] xmr: extmod refactoring
- *_into removed, replaced by generic methods
- point_add, point_sub added
- code cleanup (+1 squashed commit)
Squashed commits:
[fbe39496] monero support added
Squashed commits:
xmr: hmac/enc keys removed from builder
[41028dff] xmr: unused function removed
[358573e3] xmr: PR comments fixes
[4abf9dc9] xmr: test fixes after refactoring
[192785ad] template rebuilt
[57a1f25e] xmr: wrapper protocol messages removed
[6f40ce17] xmr: trezor-common version bump & sync
[03e71de2] xmr: check input permutation
[6fc8b0ec] xmr: code cleanup, refactoring
[85ecc151] xmr: crypto code cleanup
[20b4113c] xmr: chunked bulletproof vectors
- workaround for the heap fragmentation problems
[66786f9d] tools: enable to reset class level indentation for mocks gen
[dc6f84a9] xmr: extmod-monero comments added, for mocks
[b1d4ab11] xmr: code cleanup
[447a8625] xmr: tsx confirmation raises exception on cancellation
[00dd8f6c] xmr: protocol optimizations removed, flow unified
- in_memory optimization stored tx.vin parts in the memory which enabled to skip roundtrips with permutations and hash_vini. Optimizations was removed so the protocol flow is unified among inputs, independent of the tx specs
- many_inputs: optimization stored spending keys for UTXO in memory, now it is offloaded in the encrypted form.
[ea69c7ad] vendor: trezor-common version bump & pb sync
[5d81c2a8] xmr: manual BP serialization
- more memory effective as the memory is critical in the range proof section
[d64bda7c] xmr: range_proof C-impl deprecated
- using now partitioned implementation in Python, which is also quite fast and easier to maintain due to allocations and buffers.
[18604e0f] xmr: borromean range sig generated by partitions
- overcomes heap fragmentation problem
[65a5116b] xmr: comments removed
[ca2bd0cc] xmr: auto-generated intelliJ param comments removed
[a75ef32e] xmr: code cleanup, heap fragmentations fixes
Squashed commits:
[d2ac2eb6] xmr: addr cleanup
[7e4c1a9c] xmr: code cleanup, heap fragmentations fixes
[93af8afb] xmr: refactoring, typing, comments
[28df8662] xmr: comment fix
[8b4f4d9c] xmr: serializer test fix (+34 squashed commits)
Squashed commits:
[823ee192] xmr: crypto comment cleanup
[6debfb6b] xmr: ring_ct cleanup
[759f52bf] xmr: tsx signer code style, hintins
[0b175bc9] xmr: tsx builder external state removed
[fee4a5ad] xmr: builder state fix
[92736fa7] xmr: sign_tx unimport
[a570ecb9] xmr: misc code cleanup
[4a496bb7] xmr: hash wrapper not needed in writer
[fefdb834] xmr: signer serialization improved
[8fa6eec9] xmr: signer mem clean
[66c53fe5] xmr: isort
[6996bd90] xmr: black
[59915a85] xmr: tsx input serialization refactored
[326af131] xmr: msg dump with prefix
[6e39801d] xmr: manual serialization of tx prefix
[9e5e047a] xmr: manual serialization improvements
[d07cee60] xmr: manual serialization of txout elements
[8d56c80e] xmr: TxOut custom serialization optimized II
[c19ba122] xmr: TxOut custom serialization optimized
[ce0d9b0b] xmr: TxOut manual serialization
[44e38342] xmr: sing_tx unimport
[61ac61be] xmr: lite log trace rename
[176b4275] xmr: de-async overhaul
[89ae3bad] xmr: diag style
[5ccb2fbb] xmr: wrappers cleanup
[aa86fb19] xmr: py3 only inheritance
[8031b1b0] xmr: builder, log_trace -> mem_trace for clarity
[25bf70d7] xmr: debugging logging only in debug mode
[c7c8d3c8] xmr: iface cleanup
[b0373396] xmr: lite debug only
[b1f6ce04] xmr: diag only in debug
[de7d718b] xmr: tsx counter removed
[76729beb] xmr: tsx_sign removed
[c6e6ffa8] Merge commit 'ba500bf4ec1ef9cd953bdf5a47888c5226db8d0b' into xmr
[ee97ef9f] xmr: minor code cleanup
xmr: black
xmr: minor code cleanup
[bae3ecac] xmr: bp comments
[5e812e6f] xmr: sign - mem_trace, pydoc
[7216a8c6] xmr: pydoc removed
[e87365f4] xmr: layout cleanup
[8d21d82e] xmr: redundant constructors removed
[9aa82bed] xmr: redundant comments removed
[9b926d6c] xmr: preludes removed
[bc9e77f1] xmr: readme update
[cf62047a] xmr: aggregated bulletproofs + rsig offloading
xmr: change idx fix
xmr: iface refactoring, integrated address (+5 squashed commits)
xmr: layout pagination refactoring
xmr: addr - integrated address
pb: sync
vendor: trezor-common version bump
xmr: style fixes
xmr: handle sweep tsx correctly
- handle dummy change address correctly
xmr: integrated address generation
build: fix after trezor-crypto version bump
xmr: new protocol dispatch handlers
xmr: slip0010
[43cf4c3c] xmr: comment fix
xmr: extmod pointer aritm fix
xmr: _into api unified, result is the first parameter
xmr: bp cleanup
xmr: scalar nullity test fix
xmr: msg registration improved
- lite protocol optional
- diag protocol optional
xmr: unused imports (+33 squashed commits)
[b4d045ae] xmr: bp - noqa flake8 false positive
[2c79d4be] xmr: isort
[8b9d2835] xmr: code cleanup
[eb7496e9] xmr: iface - shorter timeouts for faster tests
[59520b63] xmr: ringct comment
[6b16088e] xmr: signer - comment fixes
[a08958e2] xmr: simple and bulletproof condition fix
[4e0289a9] vendor: trezor-common version bump
[de472e5a] xmr: black
[234d2249] xmr: lightening, fixes, KeccakXmrArchive
- builder keys
- unload mods before memory intensive operation
[abdec665] xmr: sign_tx logging
[989d8687] xmr: serialize lightening
[7d61f056] xmr: tsx sign refactoring, lightening
- wake_up state restore
- minimize import weight
[3a0daa8b] xmr: serialize thinning
[65ad1d2e] xmr: serialize thinning
[501221d5] xmr: bp - thinning
[3d980377] xmr: bp - generalization with proof_v8
[10d11d60] xmr: extended rsig - offloading protocol
[a8f5caa2] xmr: crypto - rsig params fix
[f5e130b8] xmr: crypto - inv8
[dbc3f9d8] xmr: rsig pb sync
[5748a13e] xmr: bp - data for bp4 fix (+18 squashed commits)
Squashed commits:
[5bcd54e3] xmr: bp - black
[e93e97dd] xmr: bp refactoring, large memory optimizations
- memoryview in __getitem__ requires new memory allocation so the refactored version uses to(), read() methods that can operate directly on buffers without need to create memory views.
[c30745ae] xmr: bp - black
[f5c40699] xmr: bp - tests extended
[8dae75de] xmr: bp - get_exponent optim
[3e59ff80] xmr: bp - precomputations for 4 statements
[d1d2e29c] xmr: bp - gc.collect
[1bb6b5ba] xmr: bp - optimizations, streamlining
[2a2b0cb7] xmr: bp - verification in log(MN) memory for 1 proof
- not allocating MN vectors
- sequential multiexec added for memory efficient verification
- bulletproofs: maintain -z4, -z5, and -y0 to avoid subtractions [8276d25]
- bulletproofs: merge multiexps as per sarang's new python code [acd64d2b]
[75aa7dee] xmr: bp - memory optimization
[a10d05a9] xmr: bp - deterministic mask generation init
[5060d6a7] xmr: bp optimizations
[dd69eb1b] xmr: bp - black
[19f0f64b] xmr: bp - optimizations, power key vector
[2ba63f8c] xmr: bp - minor cleanup, optimizations, scalarmultH
[31c9ca2d] xmr: bp - mem clean
[3fc2c793] xmr: bp - memory save
[5b16c9cc] bp: black
[f1040c97] xmr: crypto - memory leak fix
[ff863510] xmr: iface - flake
[6ebf69c2] xmr: lite - flake8, black
[eee55d62] xmr: bp - memory diag
[2767009b] xmr: bulletproofs upgrade, mainnet version, cleaning
[be6ebbd5] xmr: lite protocol
[d603e96d] xmr: pb sync
[5da15da9] vendor: trezor-common fix
[0373b97e] xmr: iface - output confirmation split, subaddr fix
[2cf32176] xmr: monero - subaddress fixed for index (0, 0)
[3bb8f08b] xmr: enc.aescbc added - for lite protocol (+1 squashed commit)
Squashed commits:
[011dbaab] TMP: trezor-common on master, crypto on ph4
- trezor-crypto on ph4r04 fork as it has all required stuff
- Lite protocol not merged in master, thus does not work in the PR
[795b34e1] xmr: get_address fix
[2d39c90c] xmr: bp - import fix (squashed commit)
Squashed commits:
[2d5c6cee] extmod: monero - reduce32 and ge25519_norm removed (squashed commit)
- not needed in trezor-core (+4 squashed commits)
Squashed commits:
[90e6b5c5] xmr: bp optimization
[4fda0d22] xmr: redundant ge_ functions removed
[68903767] xmr: crypto - sc_reduce32 not needed
[c8a6c807] xmr: test for inversion added (+12 squashed commits)
Squashed commits:
[378928db] xmr: adapting to new trezor-crypto
[8f4ff8c1] protob sync
[82dff70a] vendor: trezor-common version bump
[fabc67b3] extmod: monero - inversion mod curve order optimized a bit
[4f29fe4c] xmr: import fix
[f6f8e300] xmr: bp - code cleanup
[d54b4f3b] xmr: bp - memory cleaning
[1065abc3] xmr: tsx_signer - bulletproofs fixes
[9f8a700f] xmr: bp key vector iterator fix
[49c25977] xmr.serialize: bulletproof fix
[1ee77378] xmr: monero - format
[cf0a7104] xmr: bp last mask fix (+20 squashed commits)
Squashed commits:
[fa1c3623] xmr: black
[3f3e31f3] xmr: bulletproofs added to signer
[d23d9284] xmr: protocol.tsx_sign_builder - logger collects
[a28eb55f] xmr: bp - memory optimizations
[d2fcb23a] xmr: tests for bulletproofs added
[82eef146] xmr: bp - gc (+14 squashed commits)
Squashed commits:
[4cf70d97] xmr: bp - gc
[42877b05] xmr: bp - minor memory optimization
[2c612e45] xmr: bp - use sc_inv_into
[d7e9dab4] xmr: bp - KeyVEval fix
[1523f400] xmr: bp - blacked
[b264a65b] xmr: bp - KeyVEval - caching current element, avoid allocations
[83ba7a65] xmr: bp - memory view optimized
[b517906c] xmr: bp - gc() during inversion
[92d37c88] xmr: bp - gc.collect() after expensive inversion
[e7fad558] xmr: bp - hashing memory optimization
[4c278152] xmr: bp - deterministic masks optimization, prove_s1 optim
[cbf74a70] xmr: bp - detect which modular inversion is usable
[8ea1ec43] xmr: better memory tracing for bulletproofs
[2f4dd552] xmr: bulletproofs added
[1928e2d3] xmr: crypto - sc_inv_into added (+2 squashed commits)
Squashed commits:
[f895fa6e] xmr: crypto - hash to existing buffer
[b76c6b09] xmr: crypto - in-place crypto functions added
- required for Bulletproof to minimize the heap fragmentation
[cab4366e] extmod: monero - modular inversion mod curve order added (+2 squashed commits)
Squashed commits:
[52a6e487] extmod: monero - hash into buffer added
[695a3827] extmod: monero module - muladd256_modm added
- required for Bulletproof
[3f4498d7] xmr: crypto tests added
- basic unit tests for crypto, tests monero module and underlying trezor-crypto + basic address manipulation
[820d012d] pb sync
[49eeddd1] vendor: trezor-common version bump
[30382440] xmr: crypto - point norm not needed
[89701c41] tests: xmr - serializer tests added
[bfee46db] tests: support async unit tests, assertListEqual added
[55c14487] xmr: serialize - serialization logic cleaned, refactored
[4b771638] xmr: simplification, do not ask to confirm change tx output
- change address checked to match main address in the builder
[f334d8ad] xmr: protocol: simplification - require change address to equal the main address
[1a3416eb] xmr: unpack256_modm_noreduce added
- 32B array to integer mod curve order, without modular reduction after conversion
- required for bulletproofs
[1c94b5d4] xmr: readme added
[3cc9f9fa] extmod/monero: mul256_modm added, required for BP
[5cf77a4c] xmr: monero support added [for review]
depends on PRs:
https://github.com/trezor/trezor-crypto/pull/162
https://github.com/trezor/trezor-core/pull/286
2018-08-06 12:07:14 +00:00
|
|
|
global current_package
|
|
|
|
|
|
|
|
if line.startswith("package: "):
|
|
|
|
current_package = line[9:].strip()
|
|
|
|
return
|
|
|
|
|
|
|
|
if line.startswith("mock:global"):
|
|
|
|
current_indent = 0
|
|
|
|
current_class = None
|
|
|
|
return
|
2016-10-03 13:47:54 +00:00
|
|
|
|
2018-07-31 09:35:09 +00:00
|
|
|
if line.startswith("class "):
|
|
|
|
current_class = line[6:].split("(")[0].strip(":")
|
2017-06-14 15:39:32 +00:00
|
|
|
current_indent = 0
|
2016-09-27 12:06:57 +00:00
|
|
|
|
2019-05-17 12:22:36 +00:00
|
|
|
yield (current_package, "\n\n")
|
2018-07-31 09:35:09 +00:00
|
|
|
yield (current_package, "# " + mod_desc + "\n")
|
2017-06-14 15:39:32 +00:00
|
|
|
|
2018-07-31 09:35:09 +00:00
|
|
|
elif line.startswith("def "):
|
|
|
|
current_method = line[4:].split("(")[0]
|
2017-06-14 15:39:32 +00:00
|
|
|
|
|
|
|
if current_class is None:
|
2019-05-17 12:22:36 +00:00
|
|
|
yield (current_package, "\n\n")
|
2018-07-31 09:35:09 +00:00
|
|
|
yield (current_package, "# " + mod_desc + "\n")
|
2017-06-14 15:39:32 +00:00
|
|
|
else:
|
2019-05-17 12:22:36 +00:00
|
|
|
yield (current_package, "\n")
|
2017-06-14 15:39:32 +00:00
|
|
|
current_indent = 4
|
|
|
|
|
2018-07-31 09:35:09 +00:00
|
|
|
line = current_indent * " " + line
|
2016-10-03 13:47:54 +00:00
|
|
|
|
2016-09-27 12:06:57 +00:00
|
|
|
yield (current_package, line)
|
|
|
|
|
2017-06-13 14:50:03 +00:00
|
|
|
|
2016-09-27 12:06:57 +00:00
|
|
|
def store_to_file(dest, parts):
|
2017-06-14 15:39:32 +00:00
|
|
|
for package, line in parts:
|
xmr: master merging commit
templates: build
style: correct math in comment
[260130c1] xmr: show address fix
[abd27e6c] xmr: reorganize module structure
[cd9e5a5d] xmr: simplify layout code
[a5b56f17] xmr: monero.crypto test fix
[60bc30ec] xmr: minor fixes
[f82bd9c0] xmr: use trezor.utils.ensure
[adf119ac] xmr: get rid of xmr.common module
[3531a42f] pipenv: temporary fix attempt for travis
- until pipenv bug is resolved
[d172f86f] xmr: protob messages refactored
[e83085ea] trezor-common version bump & messages regenerated
xmr: mlsag_hasher simplified
[feb5f1c6] xmr: simplify key_image_sync workflow
[d4cb0084] xmr: tiny note in README and typo
[62411cde] xmr: readme updates
[ff15b46e] xmr: rename and order mlsag functions
[3fb57dab] xmr: mlsag notes
[c27ae90b] xmr: output index check added in step6
[973c457b] xmr: grouping is mandatory
[684c7e19] xmr: range sig grouping check added
[012ca766] xmr: small refactor in borromean range proof
[1ba72b65] xmr: move range signatures to seperate file; rename mlsag2 to mlsag
[a89f3ab0] xmr: fix wrong annotation in modtrezorcrypto
[8303b42f] xmr: state's use_simple_rct and use_bulletproof modified to enums
[276712a9] xmr: re-export most of functions in apps.monero.xmr.crypto
[74165457] xmr: out_pk_masks changed to out_pk_commitments
[1e186727] xmr: state comments
[3c69a2e1] xmr: TrezorTxPrefixHashNotMatchingError note removed
only concerns multisig
[5af0feae] xmr: master merging commit
xmr: step 10 review
[d8e99375] xmr: step 09 review
[a510150d] travis: workaround form 6a0ea22b6eb3bdc9bce62d7c88fb8841ef8fdb0c
[03d27115] mocks: regenerate
[bd24bb35] mocks: add support for entering the global scope
[f75c1902] mocks: regenerate
[bce85968] modtrezorcrypto: define mock package
[7c077521] mocks: support package definition
[b3f1017b] xmr: step 05 and 06 masks and range proofs review
Masks are now always generated in step 5 and stored in state.
Range proofs were reviewed only in a high-level manner and will be
reviewed later.
[67f391cd] xmr: step 08 review
[673bf01f] xmr: steps 04, 05, 06 (almost) and 07 review
_range_proof in step 06 is still to be reviewed
[24c52517] xmr: simplify serialization, remove Archive
[896cdebf] xmr: redundant exception removed
[cb3813aa] xmr: serializer simplified
[471213b4] xmr: serializer flake8 fix
[9d4df17f] xmr: aescbc not needed
[52dd8b3b] xmr: serializer - erefs kicked out
[9e3be78f] xmr: sign cleanup, comments
- state cleanup, comments added, unused code removed
[164a7d6a] xmr: sign step 09 - fix in_memory artifact, dead branch
[8fa4066d] xmr: sign step 03 - permutation length check added
[55a593aa] xmr: multisig removed from protocol and functions
[6470678d] xmr: black styling
[02664401] xmr: step 03 review
[daf7b7d6] xmr: step 02 review
xmr: adding agent tests to travis
[c752866f] xmr: unused imports removed
[85115fd9] xmr: serializer test fixed
- removed unsupported messages after serialization simplification
[60874751] xmr: sign step 06 - comment on bulletproof hashing
- hash_bp(bp) != hash(serialize(bp)) because hash does not contain array lengths
[4abb5473] xmr: sign protocol - multisig logic removed
[813cb3a0] xmr: lite protocol removed
- backup left in xmr-total-full-with-lite branch
[e5f5b5bd] xmr: serialization slimming
- base types reduced, not needed for now
- some int serialization methods not used now
[fb515aaf] xmr: serialization - archive simplified
- simple parameter passing, no kwargs
- unused methods removed
- reader/writer passing removed for archive methods
[ec4c4adb] xmr: KeccakXmrArchive simplified, no archive used
- getting rid of container_size. We dont use containers with fixed size so this special case can be abandoned.
- KeccakXmrArchive is lighweight without need to touch main serialization
[90065bd4] xmr: serialization - serialize_archive removed
- custom serialization routine is not required at this moment
[b98c2f89] xmr: extra serialization refactored, manual serialization
- extra is serialized manually to reduce serialization overhead
- extra contains simple structures now:
- payment ID = already serialized manually
- tx pub key = easy to serialize manually
- tx additional pub keys = serialized manually with little effort, more efficient memory usage
[8ce28a54] xmr: state 6 - provided tx keys removed
- needed only in the multisig scenario which is pruned now
[8d827f46] xmr: PreMlsagHasher pseudo out hashing fix
[49e552da] xmr: redundant import removed
[4199943d] xmr: KeccakXmrArchive simplified
[69bbf5fa] xmr: PreMlsagHasher - KeyV import removed, comment added
[9a194fa2] xmr: step7 - manual hashing of Extra
[d8a09283] xmr: PreMlsagHasher state load/save removed
- not needed, state not serialized anymore
[d5f43fa4] xmr: serialize reimport removed
- complex types are not surviving protocol boundary anymore, no need to fix hierarchy problems due to unimporting
[3b045612] xmr: serialize - TupleType removed
[b9a56981] xmr: serialization schemes simplified
[a59dbb8d] xmr: HashWrapper removed
[d0d1f05b] xmr: step 01 cleanup and comments
[8f7a7782] xmr: isort, black, flake8 fixes
[14265eb2] xmr: getting rid of CtKey from the state
- only lightweight objects are kept in the state. CtKey is import heavy object. Each set_out call locally imports a new own version of the Ctkey which causes a memory leak.
[c0cfc203] xmr: set_out minor function call fix
[c11c4686] xmr: sign_tx unimport optimization to reduce fragmentation
[df0a1dfc] xmr: range_sig allocation reordering, large chunks first
[63cddd5f] xmr: remove misc.StdObj
[8c8e3f33] xmr: proper memory usage in workflow
[9be1e0ac] xmr: typos and renames
[87f718b5] xmr: back to flat workflow
[64751339] xmr: refactor builder to seperate steps
- lot of work to be done, but the general idea will probably stay
- the messages workflow works, but the signed tx was not accepted by
daemon, so there is a bug somewhere
- additional cleanup/refactoring is defintely needed
[14b0a851] xmr: iface modified to layout
[27d568e1] xmr: extmod refactoring
- *_into removed, replaced by generic methods
- point_add, point_sub added
- code cleanup (+1 squashed commit)
Squashed commits:
[fbe39496] monero support added
Squashed commits:
xmr: hmac/enc keys removed from builder
[41028dff] xmr: unused function removed
[358573e3] xmr: PR comments fixes
[4abf9dc9] xmr: test fixes after refactoring
[192785ad] template rebuilt
[57a1f25e] xmr: wrapper protocol messages removed
[6f40ce17] xmr: trezor-common version bump & sync
[03e71de2] xmr: check input permutation
[6fc8b0ec] xmr: code cleanup, refactoring
[85ecc151] xmr: crypto code cleanup
[20b4113c] xmr: chunked bulletproof vectors
- workaround for the heap fragmentation problems
[66786f9d] tools: enable to reset class level indentation for mocks gen
[dc6f84a9] xmr: extmod-monero comments added, for mocks
[b1d4ab11] xmr: code cleanup
[447a8625] xmr: tsx confirmation raises exception on cancellation
[00dd8f6c] xmr: protocol optimizations removed, flow unified
- in_memory optimization stored tx.vin parts in the memory which enabled to skip roundtrips with permutations and hash_vini. Optimizations was removed so the protocol flow is unified among inputs, independent of the tx specs
- many_inputs: optimization stored spending keys for UTXO in memory, now it is offloaded in the encrypted form.
[ea69c7ad] vendor: trezor-common version bump & pb sync
[5d81c2a8] xmr: manual BP serialization
- more memory effective as the memory is critical in the range proof section
[d64bda7c] xmr: range_proof C-impl deprecated
- using now partitioned implementation in Python, which is also quite fast and easier to maintain due to allocations and buffers.
[18604e0f] xmr: borromean range sig generated by partitions
- overcomes heap fragmentation problem
[65a5116b] xmr: comments removed
[ca2bd0cc] xmr: auto-generated intelliJ param comments removed
[a75ef32e] xmr: code cleanup, heap fragmentations fixes
Squashed commits:
[d2ac2eb6] xmr: addr cleanup
[7e4c1a9c] xmr: code cleanup, heap fragmentations fixes
[93af8afb] xmr: refactoring, typing, comments
[28df8662] xmr: comment fix
[8b4f4d9c] xmr: serializer test fix (+34 squashed commits)
Squashed commits:
[823ee192] xmr: crypto comment cleanup
[6debfb6b] xmr: ring_ct cleanup
[759f52bf] xmr: tsx signer code style, hintins
[0b175bc9] xmr: tsx builder external state removed
[fee4a5ad] xmr: builder state fix
[92736fa7] xmr: sign_tx unimport
[a570ecb9] xmr: misc code cleanup
[4a496bb7] xmr: hash wrapper not needed in writer
[fefdb834] xmr: signer serialization improved
[8fa6eec9] xmr: signer mem clean
[66c53fe5] xmr: isort
[6996bd90] xmr: black
[59915a85] xmr: tsx input serialization refactored
[326af131] xmr: msg dump with prefix
[6e39801d] xmr: manual serialization of tx prefix
[9e5e047a] xmr: manual serialization improvements
[d07cee60] xmr: manual serialization of txout elements
[8d56c80e] xmr: TxOut custom serialization optimized II
[c19ba122] xmr: TxOut custom serialization optimized
[ce0d9b0b] xmr: TxOut manual serialization
[44e38342] xmr: sing_tx unimport
[61ac61be] xmr: lite log trace rename
[176b4275] xmr: de-async overhaul
[89ae3bad] xmr: diag style
[5ccb2fbb] xmr: wrappers cleanup
[aa86fb19] xmr: py3 only inheritance
[8031b1b0] xmr: builder, log_trace -> mem_trace for clarity
[25bf70d7] xmr: debugging logging only in debug mode
[c7c8d3c8] xmr: iface cleanup
[b0373396] xmr: lite debug only
[b1f6ce04] xmr: diag only in debug
[de7d718b] xmr: tsx counter removed
[76729beb] xmr: tsx_sign removed
[c6e6ffa8] Merge commit 'ba500bf4ec1ef9cd953bdf5a47888c5226db8d0b' into xmr
[ee97ef9f] xmr: minor code cleanup
xmr: black
xmr: minor code cleanup
[bae3ecac] xmr: bp comments
[5e812e6f] xmr: sign - mem_trace, pydoc
[7216a8c6] xmr: pydoc removed
[e87365f4] xmr: layout cleanup
[8d21d82e] xmr: redundant constructors removed
[9aa82bed] xmr: redundant comments removed
[9b926d6c] xmr: preludes removed
[bc9e77f1] xmr: readme update
[cf62047a] xmr: aggregated bulletproofs + rsig offloading
xmr: change idx fix
xmr: iface refactoring, integrated address (+5 squashed commits)
xmr: layout pagination refactoring
xmr: addr - integrated address
pb: sync
vendor: trezor-common version bump
xmr: style fixes
xmr: handle sweep tsx correctly
- handle dummy change address correctly
xmr: integrated address generation
build: fix after trezor-crypto version bump
xmr: new protocol dispatch handlers
xmr: slip0010
[43cf4c3c] xmr: comment fix
xmr: extmod pointer aritm fix
xmr: _into api unified, result is the first parameter
xmr: bp cleanup
xmr: scalar nullity test fix
xmr: msg registration improved
- lite protocol optional
- diag protocol optional
xmr: unused imports (+33 squashed commits)
[b4d045ae] xmr: bp - noqa flake8 false positive
[2c79d4be] xmr: isort
[8b9d2835] xmr: code cleanup
[eb7496e9] xmr: iface - shorter timeouts for faster tests
[59520b63] xmr: ringct comment
[6b16088e] xmr: signer - comment fixes
[a08958e2] xmr: simple and bulletproof condition fix
[4e0289a9] vendor: trezor-common version bump
[de472e5a] xmr: black
[234d2249] xmr: lightening, fixes, KeccakXmrArchive
- builder keys
- unload mods before memory intensive operation
[abdec665] xmr: sign_tx logging
[989d8687] xmr: serialize lightening
[7d61f056] xmr: tsx sign refactoring, lightening
- wake_up state restore
- minimize import weight
[3a0daa8b] xmr: serialize thinning
[65ad1d2e] xmr: serialize thinning
[501221d5] xmr: bp - thinning
[3d980377] xmr: bp - generalization with proof_v8
[10d11d60] xmr: extended rsig - offloading protocol
[a8f5caa2] xmr: crypto - rsig params fix
[f5e130b8] xmr: crypto - inv8
[dbc3f9d8] xmr: rsig pb sync
[5748a13e] xmr: bp - data for bp4 fix (+18 squashed commits)
Squashed commits:
[5bcd54e3] xmr: bp - black
[e93e97dd] xmr: bp refactoring, large memory optimizations
- memoryview in __getitem__ requires new memory allocation so the refactored version uses to(), read() methods that can operate directly on buffers without need to create memory views.
[c30745ae] xmr: bp - black
[f5c40699] xmr: bp - tests extended
[8dae75de] xmr: bp - get_exponent optim
[3e59ff80] xmr: bp - precomputations for 4 statements
[d1d2e29c] xmr: bp - gc.collect
[1bb6b5ba] xmr: bp - optimizations, streamlining
[2a2b0cb7] xmr: bp - verification in log(MN) memory for 1 proof
- not allocating MN vectors
- sequential multiexec added for memory efficient verification
- bulletproofs: maintain -z4, -z5, and -y0 to avoid subtractions [8276d25]
- bulletproofs: merge multiexps as per sarang's new python code [acd64d2b]
[75aa7dee] xmr: bp - memory optimization
[a10d05a9] xmr: bp - deterministic mask generation init
[5060d6a7] xmr: bp optimizations
[dd69eb1b] xmr: bp - black
[19f0f64b] xmr: bp - optimizations, power key vector
[2ba63f8c] xmr: bp - minor cleanup, optimizations, scalarmultH
[31c9ca2d] xmr: bp - mem clean
[3fc2c793] xmr: bp - memory save
[5b16c9cc] bp: black
[f1040c97] xmr: crypto - memory leak fix
[ff863510] xmr: iface - flake
[6ebf69c2] xmr: lite - flake8, black
[eee55d62] xmr: bp - memory diag
[2767009b] xmr: bulletproofs upgrade, mainnet version, cleaning
[be6ebbd5] xmr: lite protocol
[d603e96d] xmr: pb sync
[5da15da9] vendor: trezor-common fix
[0373b97e] xmr: iface - output confirmation split, subaddr fix
[2cf32176] xmr: monero - subaddress fixed for index (0, 0)
[3bb8f08b] xmr: enc.aescbc added - for lite protocol (+1 squashed commit)
Squashed commits:
[011dbaab] TMP: trezor-common on master, crypto on ph4
- trezor-crypto on ph4r04 fork as it has all required stuff
- Lite protocol not merged in master, thus does not work in the PR
[795b34e1] xmr: get_address fix
[2d39c90c] xmr: bp - import fix (squashed commit)
Squashed commits:
[2d5c6cee] extmod: monero - reduce32 and ge25519_norm removed (squashed commit)
- not needed in trezor-core (+4 squashed commits)
Squashed commits:
[90e6b5c5] xmr: bp optimization
[4fda0d22] xmr: redundant ge_ functions removed
[68903767] xmr: crypto - sc_reduce32 not needed
[c8a6c807] xmr: test for inversion added (+12 squashed commits)
Squashed commits:
[378928db] xmr: adapting to new trezor-crypto
[8f4ff8c1] protob sync
[82dff70a] vendor: trezor-common version bump
[fabc67b3] extmod: monero - inversion mod curve order optimized a bit
[4f29fe4c] xmr: import fix
[f6f8e300] xmr: bp - code cleanup
[d54b4f3b] xmr: bp - memory cleaning
[1065abc3] xmr: tsx_signer - bulletproofs fixes
[9f8a700f] xmr: bp key vector iterator fix
[49c25977] xmr.serialize: bulletproof fix
[1ee77378] xmr: monero - format
[cf0a7104] xmr: bp last mask fix (+20 squashed commits)
Squashed commits:
[fa1c3623] xmr: black
[3f3e31f3] xmr: bulletproofs added to signer
[d23d9284] xmr: protocol.tsx_sign_builder - logger collects
[a28eb55f] xmr: bp - memory optimizations
[d2fcb23a] xmr: tests for bulletproofs added
[82eef146] xmr: bp - gc (+14 squashed commits)
Squashed commits:
[4cf70d97] xmr: bp - gc
[42877b05] xmr: bp - minor memory optimization
[2c612e45] xmr: bp - use sc_inv_into
[d7e9dab4] xmr: bp - KeyVEval fix
[1523f400] xmr: bp - blacked
[b264a65b] xmr: bp - KeyVEval - caching current element, avoid allocations
[83ba7a65] xmr: bp - memory view optimized
[b517906c] xmr: bp - gc() during inversion
[92d37c88] xmr: bp - gc.collect() after expensive inversion
[e7fad558] xmr: bp - hashing memory optimization
[4c278152] xmr: bp - deterministic masks optimization, prove_s1 optim
[cbf74a70] xmr: bp - detect which modular inversion is usable
[8ea1ec43] xmr: better memory tracing for bulletproofs
[2f4dd552] xmr: bulletproofs added
[1928e2d3] xmr: crypto - sc_inv_into added (+2 squashed commits)
Squashed commits:
[f895fa6e] xmr: crypto - hash to existing buffer
[b76c6b09] xmr: crypto - in-place crypto functions added
- required for Bulletproof to minimize the heap fragmentation
[cab4366e] extmod: monero - modular inversion mod curve order added (+2 squashed commits)
Squashed commits:
[52a6e487] extmod: monero - hash into buffer added
[695a3827] extmod: monero module - muladd256_modm added
- required for Bulletproof
[3f4498d7] xmr: crypto tests added
- basic unit tests for crypto, tests monero module and underlying trezor-crypto + basic address manipulation
[820d012d] pb sync
[49eeddd1] vendor: trezor-common version bump
[30382440] xmr: crypto - point norm not needed
[89701c41] tests: xmr - serializer tests added
[bfee46db] tests: support async unit tests, assertListEqual added
[55c14487] xmr: serialize - serialization logic cleaned, refactored
[4b771638] xmr: simplification, do not ask to confirm change tx output
- change address checked to match main address in the builder
[f334d8ad] xmr: protocol: simplification - require change address to equal the main address
[1a3416eb] xmr: unpack256_modm_noreduce added
- 32B array to integer mod curve order, without modular reduction after conversion
- required for bulletproofs
[1c94b5d4] xmr: readme added
[3cc9f9fa] extmod/monero: mul256_modm added, required for BP
[5cf77a4c] xmr: monero support added [for review]
depends on PRs:
https://github.com/trezor/trezor-crypto/pull/162
https://github.com/trezor/trezor-core/pull/286
2018-08-06 12:07:14 +00:00
|
|
|
package = package.replace(".", "/")
|
2021-11-18 11:30:27 +00:00
|
|
|
dirpath = dest / os.path.dirname(package)
|
2019-06-09 09:24:06 +00:00
|
|
|
filename = os.path.basename(package) + ".pyi"
|
2021-11-18 11:30:27 +00:00
|
|
|
filepath = dirpath / filename
|
2016-10-03 13:47:54 +00:00
|
|
|
|
2021-11-18 11:30:27 +00:00
|
|
|
dirpath.mkdir(parents=True, exist_ok=True)
|
2019-05-17 12:22:36 +00:00
|
|
|
|
2021-11-18 11:30:27 +00:00
|
|
|
if (dest / package).is_dir():
|
2019-05-17 12:22:36 +00:00
|
|
|
if not line.strip():
|
|
|
|
continue
|
2021-09-27 10:13:51 +00:00
|
|
|
print(f"Package exists: {package}")
|
|
|
|
print(f"You should set 'package:' in {line.strip()}")
|
2019-05-17 12:22:36 +00:00
|
|
|
sys.exit(1)
|
2016-10-03 13:47:54 +00:00
|
|
|
|
2021-11-18 11:30:27 +00:00
|
|
|
if not filepath.exists():
|
|
|
|
filepath.write_text("from typing import *\n")
|
2017-06-14 17:27:02 +00:00
|
|
|
|
2018-07-31 09:35:09 +00:00
|
|
|
with open(filepath, "a") as f:
|
2017-06-14 15:39:32 +00:00
|
|
|
f.write(line)
|
2016-09-27 12:06:57 +00:00
|
|
|
|
2017-06-13 14:50:03 +00:00
|
|
|
|
2016-09-27 12:06:57 +00:00
|
|
|
def build_module(mod_file, dest):
|
2017-06-14 15:39:32 +00:00
|
|
|
global current_indent
|
|
|
|
global current_class
|
|
|
|
global current_package
|
|
|
|
|
2021-11-18 11:30:27 +00:00
|
|
|
assert mod_file.name.startswith("mod")
|
|
|
|
assert mod_file.suffix in (".c", ".h")
|
2019-05-17 12:22:36 +00:00
|
|
|
# modfoobar-xyz.h -> foobar-xyz
|
2021-11-18 11:30:27 +00:00
|
|
|
name = mod_file.name[3:-2]
|
2016-09-27 12:06:57 +00:00
|
|
|
|
2017-06-14 15:39:32 +00:00
|
|
|
current_indent = 0
|
|
|
|
current_class = None
|
2019-05-17 12:22:36 +00:00
|
|
|
current_package = name.split("-")[0]
|
2021-11-18 11:30:27 +00:00
|
|
|
mod_desc = str(mod_file.relative_to(CORE_DIR / "embed"))
|
2017-06-14 15:39:32 +00:00
|
|
|
|
2016-09-27 12:06:57 +00:00
|
|
|
for l in open(mod_file):
|
|
|
|
if not l.startswith(COMMENT_PREFIX):
|
|
|
|
continue
|
2016-10-03 13:47:54 +00:00
|
|
|
|
2018-07-31 09:35:09 +00:00
|
|
|
l = l[len(COMMENT_PREFIX) :] # .strip()
|
2017-04-08 16:43:26 +00:00
|
|
|
store_to_file(dest, split_to_parts(l, mod_desc))
|
2016-10-03 13:47:54 +00:00
|
|
|
|
2017-06-13 14:50:03 +00:00
|
|
|
|
2021-11-18 11:30:27 +00:00
|
|
|
def place_symlinks(dest):
|
|
|
|
# make symlinks for the non-generated files
|
|
|
|
for pyi in MOCKS_PATH.glob("../*.pyi"):
|
|
|
|
dest_file = dest / pyi.name
|
|
|
|
dest_file.symlink_to(os.path.relpath(pyi.resolve(), dest))
|
|
|
|
|
|
|
|
|
|
|
|
def build_directory(dest):
|
|
|
|
for modfile in sorted(EXTMOD_PATH.glob("**/mod*.[ch]")):
|
2019-05-17 12:22:36 +00:00
|
|
|
build_module(modfile, dest)
|
2021-11-18 11:30:27 +00:00
|
|
|
place_symlinks(dest)
|
2016-09-27 12:06:57 +00:00
|
|
|
|
2017-06-13 14:50:03 +00:00
|
|
|
|
2019-05-17 12:22:36 +00:00
|
|
|
def do_check():
|
|
|
|
with tempfile.TemporaryDirectory() as tmpdir:
|
2021-11-18 11:30:27 +00:00
|
|
|
build_directory(Path(tmpdir))
|
2019-05-17 12:22:36 +00:00
|
|
|
diff_out = subprocess.run(
|
2021-11-18 11:30:27 +00:00
|
|
|
["diff", "-ur", str(MOCKS_PATH), tmpdir],
|
2019-05-17 12:22:36 +00:00
|
|
|
stdout=subprocess.PIPE,
|
|
|
|
universal_newlines=True,
|
|
|
|
).stdout
|
|
|
|
if diff_out.strip():
|
|
|
|
print(diff_out, end="")
|
|
|
|
sys.exit(1)
|
|
|
|
|
2016-09-27 12:41:54 +00:00
|
|
|
|
2019-05-17 12:22:36 +00:00
|
|
|
def do_generate():
|
|
|
|
shutil.rmtree(MOCKS_PATH)
|
2021-11-18 11:30:27 +00:00
|
|
|
build_directory(MOCKS_PATH)
|
2017-09-05 21:15:47 +00:00
|
|
|
|
2018-07-31 09:35:09 +00:00
|
|
|
if __name__ == "__main__":
|
2019-05-17 12:22:36 +00:00
|
|
|
if len(sys.argv) > 1 and sys.argv[1] == "--check":
|
|
|
|
do_check()
|
|
|
|
else:
|
|
|
|
do_generate()
|