parent
0f07d74063
commit
e6a7b9ccaa
@ -0,0 +1 @@
|
||||
Signed Ethereum network and token definitions from host
|
@ -0,0 +1,145 @@
|
||||
import logging
|
||||
import tarfile
|
||||
import typing as t
|
||||
from pathlib import Path
|
||||
|
||||
import construct as c
|
||||
import requests
|
||||
from construct_classes import Struct, subcon
|
||||
|
||||
from . import cosi, merkle_tree
|
||||
from .messages import EthereumDefinitionType
|
||||
from .tools import EnumAdapter
|
||||
|
||||
LOG = logging.getLogger(__name__)
|
||||
|
||||
FORMAT_MAGIC = b"trzd1"
|
||||
DEFS_BASE_URL = "https://data.trezor.io/firmware/eth-definitions/"
|
||||
|
||||
DEFINITIONS_DEV_SIGS_REQUIRED = 1
|
||||
DEFINITIONS_DEV_PUBLIC_KEYS = [
|
||||
bytes.fromhex(key)
|
||||
for key in ("db995fe25169d141cab9bbba92baa01f9f2e1ece7df4cb2ac05190f37fcc1f9d",)
|
||||
]
|
||||
|
||||
DEFINITIONS_SIGS_REQUIRED = 2
|
||||
DEFINITIONS_PUBLIC_KEYS = [
|
||||
bytes.fromhex(key)
|
||||
for key in (
|
||||
"4334996343623e462f0fc93311fef1484ca23d2ff1eec6df1fa8eb7e3573b3db",
|
||||
"a9a22cc265a0cb1d6cb329bc0e60bc45df76b9ab28fb87b61136feaf8d8fdc96",
|
||||
"b8d2b21de27124f0511f903ae7e60e07961810a0b8f28ea755fa50367a8a2b8b",
|
||||
)
|
||||
]
|
||||
|
||||
|
||||
ProofFormat = c.PrefixedArray(c.Int8ul, c.Bytes(32))
|
||||
|
||||
|
||||
class DefinitionPayload(Struct):
|
||||
magic: bytes
|
||||
data_type: EthereumDefinitionType
|
||||
timestamp: int
|
||||
data: bytes
|
||||
|
||||
SUBCON = c.Struct(
|
||||
"magic" / c.Const(FORMAT_MAGIC),
|
||||
"data_type" / EnumAdapter(c.Int8ul, EthereumDefinitionType),
|
||||
"timestamp" / c.Int32ul,
|
||||
"data" / c.Prefixed(c.Int16ul, c.GreedyBytes),
|
||||
)
|
||||
|
||||
|
||||
class Definition(Struct):
|
||||
payload: DefinitionPayload = subcon(DefinitionPayload)
|
||||
proof: t.List[bytes]
|
||||
sigmask: int
|
||||
signature: bytes
|
||||
|
||||
SUBCON = c.Struct(
|
||||
"payload" / DefinitionPayload.SUBCON,
|
||||
"proof" / ProofFormat,
|
||||
"sigmask" / c.Int8ul,
|
||||
"signature" / c.Bytes(64),
|
||||
)
|
||||
|
||||
def verify(self, dev: bool = False) -> None:
|
||||
payload = self.payload.build()
|
||||
root = merkle_tree.evaluate_proof(payload, self.proof)
|
||||
cosi.verify(
|
||||
self.signature,
|
||||
root,
|
||||
DEFINITIONS_DEV_SIGS_REQUIRED,
|
||||
DEFINITIONS_DEV_PUBLIC_KEYS,
|
||||
self.sigmask,
|
||||
)
|
||||
|
||||
|
||||
class Source:
|
||||
def fetch_path(self, *components: str) -> t.Optional[bytes]:
|
||||
raise NotImplementedError
|
||||
|
||||
def get_network_by_slip44(self, slip44: int) -> t.Optional[bytes]:
|
||||
return self.fetch_path("slip44", str(slip44), "network.dat")
|
||||
|
||||
def get_network(self, chain_id: int) -> t.Optional[bytes]:
|
||||
return self.fetch_path("chain-id", str(chain_id), "network.dat")
|
||||
|
||||
def get_token(self, chain_id: int, address: t.AnyStr) -> t.Optional[bytes]:
|
||||
if isinstance(address, bytes):
|
||||
address_str = address.hex()
|
||||
elif address.startswith("0x"):
|
||||
address_str = address[2:]
|
||||
else:
|
||||
address_str = address
|
||||
|
||||
address_str = address_str.lower()
|
||||
|
||||
return self.fetch_path("chain-id", f"{chain_id}", f"token-{address_str}.dat")
|
||||
|
||||
|
||||
class NullSource(Source):
|
||||
def fetch_path(self, *components: str) -> t.Optional[bytes]:
|
||||
return None
|
||||
|
||||
|
||||
class FilesystemSource(Source):
|
||||
def __init__(self, root: Path) -> None:
|
||||
self.root = root
|
||||
|
||||
def fetch_path(self, *components: str) -> t.Optional[bytes]:
|
||||
path = self.root.joinpath(*components)
|
||||
if not path.exists():
|
||||
LOG.info("Requested definition at %s was not found", path)
|
||||
return None
|
||||
LOG.info("Reading definition from %s", path)
|
||||
return path.read_bytes()
|
||||
|
||||
|
||||
class UrlSource(Source):
|
||||
def __init__(self, base_url: str = DEFS_BASE_URL) -> None:
|
||||
self.base_url = base_url
|
||||
|
||||
def fetch_path(self, *components: str) -> t.Optional[bytes]:
|
||||
url = self.base_url + "/".join(components)
|
||||
LOG.info("Downloading definition from %s", url)
|
||||
r = requests.get(url)
|
||||
if r.status_code == 404:
|
||||
LOG.info("Requested definition at %s was not found", url)
|
||||
return None
|
||||
r.raise_for_status()
|
||||
return r.content
|
||||
|
||||
|
||||
class TarSource(Source):
|
||||
def __init__(self, path: Path) -> None:
|
||||
self.archive = tarfile.open(path)
|
||||
|
||||
def fetch_path(self, *components: str) -> t.Optional[bytes]:
|
||||
inner_name = "/".join(components)
|
||||
LOG.info("Extracting definition from %s:%s", self.archive.name, inner_name)
|
||||
try:
|
||||
return self.archive.extractfile(inner_name).read() # type: ignore [not a known member]
|
||||
except Exception:
|
||||
LOG.info("Requested definition at %s was not found", inner_name)
|
||||
return None
|
@ -0,0 +1,178 @@
|
||||
# This file is part of the Trezor project.
|
||||
#
|
||||
# Copyright (C) 2012-2022 SatoshiLabs and contributors
|
||||
#
|
||||
# This library is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Lesser General Public License version 3
|
||||
# as published by the Free Software Foundation.
|
||||
#
|
||||
# This library is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Lesser General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the License along with this library.
|
||||
# If not, see <https://www.gnu.org/licenses/lgpl-3.0.html>.
|
||||
|
||||
import typing as t
|
||||
from hashlib import sha256
|
||||
|
||||
from typing_extensions import Protocol
|
||||
|
||||
|
||||
def leaf_hash(value: bytes) -> bytes:
|
||||
"""Calculate a hash of a leaf node based on its value.
|
||||
|
||||
See documentation for `MerkleTree` for details.
|
||||
"""
|
||||
return sha256(b"\x00" + value).digest()
|
||||
|
||||
|
||||
def internal_hash(left: bytes, right: bytes) -> bytes:
|
||||
"""Calculate a hash of an internal node based on its child nodes.
|
||||
|
||||
See documentation for `MerkleTree` for details.
|
||||
"""
|
||||
hash_a = min(left, right)
|
||||
hash_b = max(left, right)
|
||||
return sha256(b"\x01" + hash_a + hash_b).digest()
|
||||
|
||||
|
||||
class NodeType(Protocol):
|
||||
"""Merkle tree node."""
|
||||
|
||||
tree_hash: bytes
|
||||
"""Merkle root hash of the subtree rooted at this node."""
|
||||
|
||||
def add_to_proof_list(self, proof_entry: bytes) -> None:
|
||||
"""Add a proof entry to the proof list of this node."""
|
||||
...
|
||||
|
||||
|
||||
class Leaf:
|
||||
"""Leaf of a Merkle tree."""
|
||||
|
||||
def __init__(self, value: bytes) -> None:
|
||||
self.tree_hash = leaf_hash(value)
|
||||
self.proof: t.List[bytes] = []
|
||||
|
||||
def add_to_proof_list(self, proof_entry: bytes) -> None:
|
||||
self.proof.append(proof_entry)
|
||||
|
||||
|
||||
class Node:
|
||||
"""Internal node of a Merkle tree.
|
||||
|
||||
Does not have its own proof, but helps to build the proof of its children by passing
|
||||
the respective proof entries to them.
|
||||
"""
|
||||
|
||||
def __init__(self, left: NodeType, right: NodeType) -> None:
|
||||
self.left = left
|
||||
self.right = right
|
||||
self.left.add_to_proof_list(self.right.tree_hash)
|
||||
self.right.add_to_proof_list(self.left.tree_hash)
|
||||
self.tree_hash = internal_hash(self.left.tree_hash, self.right.tree_hash)
|
||||
|
||||
def add_to_proof_list(self, proof_entry: bytes) -> None:
|
||||
self.left.add_to_proof_list(proof_entry)
|
||||
self.right.add_to_proof_list(proof_entry)
|
||||
|
||||
|
||||
class MerkleTree:
|
||||
"""Merkle tree for a list of byte values.
|
||||
|
||||
The tree is built up as follows:
|
||||
|
||||
1. Order the leaves by their hash.
|
||||
2. Build up the next level up by pairing the leaves in the current level from left
|
||||
to right.
|
||||
3. Any left-over odd node at the current level gets pushed to the next level.
|
||||
4. Repeat until there is only one node left.
|
||||
|
||||
Values are not saved in the tree, only their hashes. This allows us to construct a
|
||||
tree with very large values without having to keep them in memory.
|
||||
|
||||
Semantically, the tree operates as a set, but this implementation does not check for
|
||||
duplicates. If the same value is added multiple times, the resulting tree will be
|
||||
different from a tree with only one instance of the value. In addition, only one of
|
||||
the several possible proofs for the repeated value is retrievable.
|
||||
|
||||
Proof hashes are constructed as follows:
|
||||
|
||||
- Leaf node entries are hashes of b"\x00" + value.
|
||||
- Internal node entries are hashes of b"\x01" + min(left, right) + max(left, right).
|
||||
|
||||
The prefixes function to distinguish leaf nodes from internal nodes. This prevents
|
||||
two attacks:
|
||||
|
||||
(a) An attacker cannot misuse a proof for an internal node to claim that
|
||||
<internal-node-entry> is a member of the tree.
|
||||
(b) An attacker cannot insert a leaf node in the format of <internal-node-entry>
|
||||
that is itself an internal node of a different tree. This would allow the
|
||||
attacker to expand the tree with their own subtree.
|
||||
|
||||
Ordering the internal node entry as min(left, right) + max(left, right) simplifies
|
||||
the proof format and verifier code: when constructing the internal entry, the
|
||||
verifier does not need to distinguish between left and right subtree.
|
||||
"""
|
||||
|
||||
entries: t.Dict[bytes, Leaf]
|
||||
"""Map of leaf hash -> leaf node.
|
||||
|
||||
Use `leaf_hash` to calculate the hash of a value, or use `get_proof(value)`
|
||||
to access the proof directly.
|
||||
"""
|
||||
root: NodeType
|
||||
"""Root node of the tree."""
|
||||
|
||||
def __init__(self, values: t.Iterable[bytes]) -> None:
|
||||
leaves = [Leaf(value) for value in values]
|
||||
leaves.sort(key=lambda leaf: leaf.tree_hash)
|
||||
|
||||
if not leaves:
|
||||
raise ValueError("Merkle tree must have at least one value")
|
||||
|
||||
self.entries = {leaf.tree_hash: leaf for leaf in leaves}
|
||||
|
||||
# build the tree
|
||||
current_level = leaves
|
||||
while len(current_level) > 1:
|
||||
# build one level of the tree
|
||||
next_level = []
|
||||
while len(current_level) >= 2:
|
||||
left, right, *current_level = current_level
|
||||
next_level.append(Node(left, right))
|
||||
|
||||
# add the remaining one or zero nodes to the next level
|
||||
next_level.extend(current_level)
|
||||
|
||||
# switch levels and continue
|
||||
current_level = next_level
|
||||
|
||||
assert len(current_level) == 1, "Tree must have exactly one root node"
|
||||
# save the root
|
||||
self.root = current_level[0]
|
||||
|
||||
def get_root_hash(self) -> bytes:
|
||||
return self.root.tree_hash
|
||||
|
||||
def get_proof(self, value: bytes) -> t.List[bytes]:
|
||||
"""Get the proof for a given value."""
|
||||
try:
|
||||
return self.entries[leaf_hash(value)].proof
|
||||
except KeyError:
|
||||
raise KeyError("Value not found in Merkle tree") from None
|
||||
|
||||
|
||||
def evaluate_proof(value: bytes, proof: t.List[bytes]) -> bytes:
|
||||
"""Evaluate the provided proof of membership.
|
||||
|
||||
Reconstructs the Merkle root hash for a tree that contains `value` as a leaf node,
|
||||
proving membership in a Merkle tree with the given root hash. The result can be
|
||||
compared to a statically known root hash, or a signature of it can be verified.
|
||||
"""
|
||||
hash = leaf_hash(value)
|
||||
for proof_entry in proof:
|
||||
hash = internal_hash(hash, proof_entry)
|
||||
return hash
|
@ -0,0 +1,108 @@
|
||||
# This file is part of the Trezor project.
|
||||
#
|
||||
# Copyright (C) 2012-2022 SatoshiLabs and contributors
|
||||
#
|
||||
# This library is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Lesser General Public License version 3
|
||||
# as published by the Free Software Foundation.
|
||||
#
|
||||
# This library is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Lesser General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the License along with this library.
|
||||
# If not, see <https://www.gnu.org/licenses/lgpl-3.0.html>.
|
||||
|
||||
import pytest
|
||||
|
||||
import typing as t
|
||||
|
||||
from trezorlib.merkle_tree import (
|
||||
MerkleTree,
|
||||
Leaf,
|
||||
Node,
|
||||
leaf_hash,
|
||||
internal_hash,
|
||||
evaluate_proof,
|
||||
)
|
||||
|
||||
|
||||
NODE_VECTORS = ( # node, expected_hash
|
||||
( # leaf node
|
||||
Leaf(b"hello"),
|
||||
"8a2a5c9b768827de5a9552c38a044c66959c68f6d2f21b5260af54d2f87db827",
|
||||
),
|
||||
( # node with leaf nodes
|
||||
Node(left=Leaf(b"hello"), right=Leaf(b"world")),
|
||||
"24233339aadcedf287d262413f03c028eb8db397edd32a2878091151b99bf20f",
|
||||
),
|
||||
( # asymmetric node with leaf hanging on second level
|
||||
Node(left=Node(left=Leaf(b"hello"), right=Leaf(b"world")), right=Leaf(b"!")),
|
||||
"c3727420dc97c0dbd89678ee195957e44cfa69f5759b395a07bc171b21468633",
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
MERKLE_TREE_VECTORS = (
|
||||
( # one value
|
||||
# values
|
||||
[b"Merkle"],
|
||||
# expected root hash
|
||||
leaf_hash(b"Merkle"),
|
||||
# expected dict of proof lists
|
||||
{
|
||||
b"Merkle": [],
|
||||
},
|
||||
),
|
||||
( # two values
|
||||
# values
|
||||
[b"Haber", b"Stornetta"],
|
||||
# expected root hash
|
||||
internal_hash(
|
||||
leaf_hash(b"Haber"),
|
||||
leaf_hash(b"Stornetta"),
|
||||
),
|
||||
# expected dict of proof lists
|
||||
{
|
||||
b"Haber": [leaf_hash(b"Stornetta")],
|
||||
b"Stornetta": [leaf_hash(b"Haber")],
|
||||
},
|
||||
),
|
||||
( # three values
|
||||
# values
|
||||
[b"Andersen", b"Wuille", b"Maxwell"],
|
||||
# expected root hash
|
||||
internal_hash(
|
||||
internal_hash(
|
||||
leaf_hash(b"Maxwell"),
|
||||
leaf_hash(b"Wuille"),
|
||||
),
|
||||
leaf_hash(b"Andersen"),
|
||||
),
|
||||
# expected dict of proof lists
|
||||
{
|
||||
b"Andersen": [internal_hash(leaf_hash(b"Maxwell"), leaf_hash(b"Wuille"))],
|
||||
b"Maxwell": [leaf_hash(b"Wuille"), leaf_hash(b"Andersen")],
|
||||
b"Wuille": [leaf_hash(b"Maxwell"), leaf_hash(b"Andersen")],
|
||||
},
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("node, expected_hash", NODE_VECTORS)
|
||||
def test_node(node: t.Union[Node, Leaf], expected_hash: str) -> None:
|
||||
assert node.tree_hash.hex() == expected_hash
|
||||
|
||||
|
||||
@pytest.mark.parametrize("values, root_hash, proofs", MERKLE_TREE_VECTORS)
|
||||
def test_tree(
|
||||
values: t.List[bytes],
|
||||
root_hash: bytes,
|
||||
proofs: t.Dict[bytes, t.List[bytes]],
|
||||
) -> None:
|
||||
mt = MerkleTree(values)
|
||||
assert mt.get_root_hash() == root_hash
|
||||
for value, proof in proofs.items():
|
||||
assert mt.get_proof(value) == proof
|
||||
assert evaluate_proof(value, proof) == root_hash
|
@ -0,0 +1,107 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
# This file is part of the Trezor project.
|
||||
#
|
||||
# Copyright (C) 2012-2022 SatoshiLabs and contributors
|
||||
#
|
||||
# This library is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Lesser General Public License version 3
|
||||
# as published by the Free Software Foundation.
|
||||
#
|
||||
# This library is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Lesser General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the License along with this library.
|
||||
# If not, see <https://www.gnu.org/licenses/lgpl-3.0.html>.
|
||||
from __future__ import annotations
|
||||
|
||||
import click
|
||||
import requests
|
||||
import zipfile
|
||||
from pathlib import Path
|
||||
|
||||
from trezorlib import definitions, merkle_tree
|
||||
|
||||
ZIP_FILENAME = "definitions-sparse.zip"
|
||||
|
||||
TOPDIRS = ("chain-id", "slip44")
|
||||
|
||||
|
||||
class SparseZipSource(definitions.Source):
|
||||
def __init__(self, zip: Path | zipfile.ZipFile) -> None:
|
||||
if isinstance(zip, Path):
|
||||
self.zip = zipfile.ZipFile(zip)
|
||||
else:
|
||||
self.zip = zip
|
||||
|
||||
# extract signature
|
||||
self.signature = self.read_bytes("signature.dat")
|
||||
self.root_hash = self.read_bytes("root.dat")
|
||||
|
||||
# construct a Merkle tree
|
||||
entries = []
|
||||
for name in self.zip.namelist():
|
||||
if name.startswith("chain-id/"):
|
||||
entries.append(self.read_bytes(name))
|
||||
entries.sort()
|
||||
self.merkle_tree = merkle_tree.MerkleTree(entries)
|
||||
|
||||
if self.root_hash != self.merkle_tree.get_root_hash():
|
||||
raise ValueError("Failed to reconstruct the correct Merkle tree")
|
||||
|
||||
def read_bytes(self, path: str | Path) -> bytes:
|
||||
with self.zip.open(str(path)) as f:
|
||||
return f.read()
|
||||
|
||||
def fetch_path(self, *components: str) -> bytes | None:
|
||||
path = "/".join(components)
|
||||
data = self.read_bytes(path)
|
||||
proof = self.merkle_tree.get_proof(data)
|
||||
proof_bytes = definitions.ProofFormat.build(proof)
|
||||
return data + proof_bytes + self.signature
|
||||
|
||||
|
||||
@click.command()
|
||||
@click.option(
|
||||
"-z",
|
||||
"--definitions-zip",
|
||||
type=click.Path(exists=True, dir_okay=False, resolve_path=True, path_type=Path),
|
||||
help="Local zip file with stored definitions.",
|
||||
)
|
||||
@click.argument(
|
||||
"outdir",
|
||||
type=click.Path(resolve_path=True, file_okay=False, writable=True, path_type=Path),
|
||||
)
|
||||
def unpack_definitions(definitions_zip: Path, outdir: Path) -> None:
|
||||
"""Script that unpacks and completes (insert missing Merkle Tree proofs
|
||||
into the definitions) the Ethereum definitions (networks and tokens).
|
||||
|
||||
If no local zip is provided, the latest one will be downloaded from trezor.io.
|
||||
"""
|
||||
if definitions_zip is None:
|
||||
result = requests.get(definitions.DEFS_BASE_URL + ZIP_FILENAME)
|
||||
result.raise_for_status()
|
||||
zip = zipfile.ZipFile(result.raw)
|
||||
else:
|
||||
zip = zipfile.ZipFile(definitions_zip)
|
||||
|
||||
source = SparseZipSource(zip)
|
||||
|
||||
if not outdir.exists():
|
||||
outdir.mkdir()
|
||||
|
||||
for name in zip.namelist():
|
||||
if name == "signature.dat" or not name.endswith(".dat"):
|
||||
continue
|
||||
|
||||
local_path = outdir / name
|
||||
local_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
data = source.fetch_path(name)
|
||||
assert data is not None, f"Could not read data for: {name}"
|
||||
local_path.write_bytes(data)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unpack_definitions()
|
Loading…
Reference in new issue