feat(tools): add scripts for firmware.elf file analysis using binsize

[no changelog]

core/tools/size/.gitignore

@@ -0,0 +1 @@
+*.txt

core/tools/size/README.md

@@ -0,0 +1,44 @@
+# Size (binsize)
+
+Shell and python scripts acting as wrappers around `binsize` tool/command.
+
+Adding `--help` to any `<command>.sh` will show the help message for that specific command.
+
+Settings of this specific (`trezor-firmware`) project are forwarded in each command. Specifically, `core/build/firmware/firmware.elf` and `core/build/firmware/firmware.map` (optional) files are used and we are interested in `.flash` and `.flash2` sections.
+
+`bloaty` and `nm` tools are needed.
+
+For more info about `binsize` tool, visit [its repository](github.com/trezor/binsize).
+
+## Available scripts/commands
+
+### app.py
+Shows the statistics about each micropython app.
+
+### build.sh <file_suffix>
+Builds the firmware with optional renaming of the generated .elf file.
+
+### checker.py
+Checks the size of the current firmware against the size limits of its flash sections.
+
+### commit.sh <commit_id>
+Gets the size difference introduced by a specified commit
+
+### compare_master.py
+Compares the size of the current firmware with the size of the latest master.
+
+### compare.sh <old_elf_file> <new_elf_file>
+Compares the size of two firmware binaries.
+
+### get.sh
+Gets the size information about the current firmware.
+
+### groups.py
+Shows the groupings of all symbols into specific categories.
+
+### history.sh <commit_amount:int> <step_size:int>
+Shows the size history of latest `<commit_amount>` with `<step_size>` commits  between them.
+BEWARE: might not always work properly, as it needs to build firmware for each commit. It may happens that some commits are not buildable.
+
+### tree.sh
+Shows the tree-size view of all files in the current firmware with their sizes.

core/tools/size/apps.py

@@ -0,0 +1,45 @@
+#!/usr/bin/env python3
+"""
+Showing sizes of individual micropython apps.
+"""
+
+from __future__ import annotations
+
+import re
+import sys
+from pathlib import Path
+
+
+from binsize import BinarySize, StatisticsPlugin, DataRow
+
+HERE = Path(__file__).parent
+CORE_DIR = HERE.parent.parent
+
+if len(sys.argv) > 1:
+    BIN_TO_ANALYZE = sys.argv[1]
+else:
+    BIN_TO_ANALYZE = CORE_DIR / "build/firmware/firmware.elf"  # type: ignore
+
+
+def apps_categories(row: DataRow) -> str | None:
+    pattern = r"^src/apps/(\w+)/"  # dir name after apps/
+    match = re.search(pattern, row.module_name)
+    if not match:
+        return None
+    else:
+        return match.group(1)
+
+
+if __name__ == "__main__":
+    BS = (
+        BinarySize()
+        .load_file(BIN_TO_ANALYZE, sections=(".flash", ".flash2"))
+        .use_map_file(
+            CORE_DIR / "build/firmware/firmware.map", sections=(".flash", ".flash2")
+        )
+        .add_basic_info()
+        .aggregate()
+        .sort()
+        .add_definitions()
+    )
+    StatisticsPlugin(BS, apps_categories).show()

core/tools/size/build.sh

@@ -0,0 +1,7 @@
+#!/bin/sh
+
+CURR_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+CORE_DIR=$(realpath "${CURR_DIR}/../..")
+
+export BINSIZE_ROOT_DIR="${CORE_DIR}"
+binsize build $@

core/tools/size/checker.py

@@ -0,0 +1,56 @@
+#!/usr/bin/env python3
+"""
+Checking the size of the flash sections in firmware binary.
+
+Prints the info and fails if there is not enough free space.
+"""
+
+from __future__ import annotations
+
+import sys
+from pathlib import Path
+
+from binsize import get_sections_sizes, set_root_dir
+
+HERE = Path(__file__).parent
+CORE_DIR = HERE.parent.parent
+
+if len(sys.argv) > 1:
+    BIN_TO_ANALYZE = sys.argv[1]
+else:
+    BIN_TO_ANALYZE = CORE_DIR / "build/firmware/firmware.elf"  # type: ignore
+
+# Comes from `core/embed/firmware/memory_T.ld`
+FLASH_SIZE_KB = 768
+FLASH_2_SIZE_KB = 896
+
+MIN_KB_FREE_TO_SUCCEED = 15
+
+EXIT_CODE = 0
+
+
+def report_section(name: str, size: int, max_size: int) -> None:
+    percentage = 100 * size / max_size
+    free = max_size - size
+
+    print(f"{name}: {size}K / {max_size}K ({percentage:.2f}%) - {free}K free")
+
+    global EXIT_CODE
+    if free < MIN_KB_FREE_TO_SUCCEED:
+        print(
+            f"Less free space in {name} ({free}K) than expected ({MIN_KB_FREE_TO_SUCCEED}K). Failing"
+        )
+        EXIT_CODE = 1  # type: ignore
+
+
+if __name__ == "__main__":
+    print(f"Analyzing {BIN_TO_ANALYZE}")
+
+    set_root_dir(str(CORE_DIR))
+
+    sizes = get_sections_sizes(BIN_TO_ANALYZE, sections=(".flash", ".flash2"))
+
+    report_section(".flash", sizes[".flash"] // 1024, FLASH_SIZE_KB)
+    report_section(".flash2", sizes[".flash2"] // 1024, FLASH_2_SIZE_KB)
+
+    sys.exit(EXIT_CODE)

core/tools/size/commit.sh

@@ -0,0 +1,7 @@
+#!/bin/sh
+
+CURR_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+CORE_DIR=$(realpath "${CURR_DIR}/../..")
+
+export BINSIZE_ROOT_DIR="${CORE_DIR}"
+binsize commit -s ".flash" -s ".flash2" $@

core/tools/size/compare.sh

@@ -0,0 +1,7 @@
+#!/bin/sh
+
+CURR_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+CORE_DIR=$(realpath "${CURR_DIR}/../..")
+
+export BINSIZE_ROOT_DIR="${CORE_DIR}"
+binsize compare -s ".flash" -s ".flash2" $@

core/tools/size/compare_master.py

@@ -0,0 +1,97 @@
+#!/usr/bin/env python3
+"""
+Compares the current firmware build with a master one
+and prints the differences.
+
+Fails if the current changes are increasing the size by a lot.
+
+Also generates a thorough report of the current state of the binary
+with all the functions and their definitions.
+"""
+
+from __future__ import annotations
+
+import atexit
+import shutil
+import sys
+from io import BytesIO
+from pathlib import Path
+from zipfile import ZipFile
+
+import requests
+import click
+
+from binsize import BinarySize, get_sections_sizes, show_binaries_diff, set_root_dir
+
+HERE = Path(__file__).parent
+CORE_DIR = HERE.parent.parent
+
+FIRMWARE_ELF = CORE_DIR / "build/firmware/firmware.elf"
+
+MAX_KB_ADDITION_TO_SUCCEED = 5
+
+
+def download_and_get_latest_master_firmware_elf() -> Path:
+    url = "https://gitlab.com/satoshilabs/trezor/trezor-firmware/-/jobs/artifacts/master/download?job=core%20fw%20regular%20build"
+    req = requests.get(url)
+    tmp_dir = HERE / "tmp_for_master_elf"
+    zip_file = ZipFile(BytesIO(req.content))
+    zip_file.extractall(tmp_dir)
+
+    atexit.register(lambda: shutil.rmtree(tmp_dir))
+
+    return tmp_dir / "firmware.elf"
+
+
+def generate_report_file(fw_location: str, report_file: str | Path) -> None:
+    BinarySize().load_file(
+        fw_location, sections=(".flash", ".flash2")
+    ).add_basic_info().aggregate().sort(lambda row: row.size, reverse=True).show(
+        report_file
+    )
+
+
+@click.command()
+@click.argument("fw_location", required=False, default=FIRMWARE_ELF)
+@click.option("-r", "--report-file", help="Report file")
+def compare_master(fw_location: str, report_file: str | None) -> None:
+    print(f"Analyzing {fw_location}")
+    set_root_dir(str(CORE_DIR))
+
+    if report_file:
+        print(f"Generating report file under {report_file}")
+        generate_report_file(fw_location, report_file)
+
+    sections = (".flash", ".flash2")
+
+    master_bin = download_and_get_latest_master_firmware_elf()
+    if not master_bin.exists():
+        print("Master firmware not found")
+        sys.exit(1)
+
+    show_binaries_diff(old=master_bin, new=fw_location, sections=sections)
+
+    curr = get_sections_sizes(fw_location, sections)
+    curr_flash = curr[".flash"] // 1024
+    curr_flash_2 = curr[".flash2"] // 1024
+
+    master = get_sections_sizes(master_bin, sections)
+    master_flash = master[".flash"] // 1024
+    master_flash_2 = master[".flash2"] // 1024
+
+    print()
+    print(f"Current: flash={curr_flash}K flash2={curr_flash_2}K")
+    print(f"Master:  flash={master_flash}K flash2={master_flash_2}K")
+
+    size_diff = (curr_flash + curr_flash_2) - (master_flash + master_flash_2)
+    print(f"Size_diff: {size_diff} K")
+    if size_diff > MAX_KB_ADDITION_TO_SUCCEED:
+        print(f"Size of flash sections increased by {size_diff} K.")
+        print(f"More than allowed {MAX_KB_ADDITION_TO_SUCCEED} K. Failing.")
+        sys.exit(1)
+    else:
+        sys.exit(0)
+
+
+if __name__ == "__main__":
+    compare_master()

core/tools/size/get.sh

@@ -0,0 +1,10 @@
+#!/bin/sh
+
+CURR_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+CORE_DIR=$(realpath "${CURR_DIR}/../..")
+
+FIRMWARE_ELF="${CORE_DIR}/build/firmware/firmware.elf"
+MAP_FILE="${CORE_DIR}/build/firmware/firmware.map"
+
+export BINSIZE_ROOT_DIR="${CORE_DIR}"
+binsize get "${FIRMWARE_ELF}" -m "${MAP_FILE}" -s ".flash" -s ".flash2"  $@

core/tools/size/groups.py

@@ -0,0 +1,171 @@
+#!/usr/bin/env python3
+"""
+Grouping symbols in binary into coherent categories.
+"""
+
+from __future__ import annotations
+
+import sys
+from pathlib import Path
+from typing import Callable
+
+from binsize import BinarySize, DataRow, StatisticsPlugin, set_root_dir
+
+HERE = Path(__file__).resolve().parent
+CORE_DIR = HERE.parent.parent
+
+if len(sys.argv) > 1:
+    BIN_TO_ANALYZE = sys.argv[1]
+else:
+    BIN_TO_ANALYZE = CORE_DIR / "build/firmware/firmware.elf"  # type: ignore
+FILE_TO_SAVE = HERE / "size_binary_firmware_elf_results.txt"
+
+
+def _categories_func(row: DataRow) -> str | None:
+    # Defined inside the function so it can be seen in the function definition
+    # (which is optionally printed)
+    CATEGORIES: dict[str, Callable[[DataRow], bool]] = {
+        "UI": lambda row: (
+            row.source_definition.startswith(
+                ("src/trezor/ui/", "embed/extmod/modtrezorui/")
+            )
+        ),
+        "Crypto": lambda row: (
+            row.source_definition.startswith(
+                (
+                    "vendor/trezor-crypto/",
+                    "src/trezor/crypto/",
+                    "embed/extmod/modtrezorcrypto/",
+                )
+            )
+        ),
+        "Secp256": lambda row: (
+            row.source_definition.startswith("vendor/secp256k1-zkp/")
+        ),
+        "Storage": lambda row: (
+            row.source_definition.startswith(("src/storage/", "vendor/trezor-storage/"))
+        ),
+        "Micropython": lambda row: row.source_definition.startswith(
+            "vendor/micropython/"
+        ),
+        "Bitcoin app": lambda row: row.source_definition.startswith(
+            "src/apps/bitcoin/"
+        ),
+        "Ethereum app": lambda row: row.source_definition.startswith(
+            "src/apps/ethereum/"
+        ),
+        "Monero app": lambda row: row.source_definition.startswith("src/apps/monero/"),
+        "Cardano app": lambda row: row.source_definition.startswith(
+            "src/apps/cardano/"
+        ),
+        "Management app": lambda row: row.source_definition.startswith(
+            "src/apps/management/"
+        ),
+        "Common apps": lambda row: row.source_definition.startswith("src/apps/common/"),
+        "Webauthn app": lambda row: row.source_definition.startswith(
+            "src/apps/webauthn/"
+        ),
+        "Altcoin apps": lambda row: (
+            row.source_definition.startswith(
+                (
+                    "src/apps/nem/",
+                    "src/apps/stellar/",
+                    "src/apps/eos/",
+                    "src/apps/tezos/",
+                    "src/apps/ripple/",
+                    "src/apps/zcash/",
+                    "src/apps/binance/",
+                )
+            )
+        ),
+        "Other apps": lambda row: row.source_definition.startswith("src/apps/"),
+        "Rest of src/": lambda row: row.source_definition.startswith("src/"),
+        "Fonts": lambda row: row.source_definition.startswith("embed/lib/fonts/"),
+        "Embed firmware": lambda row: row.source_definition.startswith(
+            "embed/firmware/"
+        ),
+        "Trezorhal": lambda row: row.source_definition.startswith("embed/trezorhal/"),
+        "Trezorio": lambda row: row.source_definition.startswith(
+            "embed/extmod/modtrezorio/"
+        ),
+        "Trezorconfig": lambda row: row.source_definition.startswith(
+            "embed/extmod/modtrezorconfig/"
+        ),
+        "Trezorutils": lambda row: row.source_definition.startswith(
+            "embed/extmod/modtrezorutils/"
+        ),
+        "Embed extmod": lambda row: row.source_definition.startswith("embed/extmod/"),
+        "Embed lib": lambda row: row.source_definition.startswith("embed/lib/"),
+        "Rust": lambda row: (
+            row.language == "Rust"
+            or row.source_definition.startswith(("embed/rust/", "/cargo/registry"))
+            or row.symbol_name.startswith(("trezor_tjpgdec", "qrcodegen"))
+            or ".cargo/registry" in row.symbol_name
+        ),
+        "Frozen modules": lambda row: row.symbol_name.startswith("frozen_module"),
+        ".bootloader": lambda row: row.symbol_name == ".bootloader",
+        ".rodata + qstr + misc": lambda row: (
+            row.symbol_name.startswith(
+                (".rodata", "mp_qstr", "str1", "*fill*", ".text", "OUTLINED_FUNCTION")
+            )
+            or _has_32_hex(row.symbol_name)
+        ),
+    }
+
+    for category, func in CATEGORIES.items():
+        if func(row):
+            return category
+    return None
+
+
+def _has_32_hex(text: str) -> bool:
+    if "." in text:
+        text = text.split(".")[0]
+    return len(text) == 32 and all(c in "0123456789abcdef" for c in text)
+
+
+def show_categories_statistics(
+    STATS: StatisticsPlugin, include_categories_func: bool = False
+) -> None:
+    STATS.show(include_none=True, include_categories_func=include_categories_func)
+
+
+def show_data_with_categories(
+    STATS: StatisticsPlugin, file_to_save: str | Path | None = None
+) -> None:
+    STATS.show_data_with_categories(file_to_save, include_none=True)
+
+
+def show_only_one_category(
+    BS: BinarySize, category: str | None, file_to_save: str | Path | None = None
+) -> None:
+    BS.filter(lambda row: _categories_func(row) == category).show(
+        file_to_save, debug=True
+    )
+
+
+def show_raw_bloaty_data() -> None:
+    BinarySize().load_file(BIN_TO_ANALYZE, sections=(".flash", ".flash2")).show(
+        HERE / "size_binary_firmware_elf_results_no_aggregation.txt"
+    )
+
+
+if __name__ == "__main__":
+    set_root_dir(str(CORE_DIR))
+
+    BS = (
+        BinarySize()
+        .load_file(BIN_TO_ANALYZE, sections=(".flash", ".flash2"))
+        .use_map_file(
+            CORE_DIR / "build/firmware/firmware.map", sections=(".flash", ".flash2")
+        )
+        .add_basic_info()
+        .aggregate()
+        .sort()
+        .add_definitions()
+    )
+    STATS = StatisticsPlugin(BS, _categories_func)
+
+    show_categories_statistics(STATS, include_categories_func=True)
+    show_data_with_categories(STATS, FILE_TO_SAVE)
+    show_only_one_category(BS, None, HERE / "size_binary_firmware_elf_results_None.txt")

core/tools/size/history.sh

@@ -0,0 +1,7 @@
+#!/bin/sh
+
+CURR_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+CORE_DIR=$(realpath "${CURR_DIR}/../..")
+
+export BINSIZE_ROOT_DIR="${CORE_DIR}"
+binsize history -s ".flash" -s ".flash2" $@

core/tools/size/tree.sh

@@ -0,0 +1,10 @@
+#!/bin/sh
+
+CURR_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+CORE_DIR=$(realpath "${CURR_DIR}/../..")
+
+FIRMWARE_ELF="${CORE_DIR}/build/firmware/firmware.elf"
+MAP_FILE="${CORE_DIR}/build/firmware/firmware.map"
+
+export BINSIZE_ROOT_DIR="${CORE_DIR}"
+binsize tree "${FIRMWARE_ELF}" -m "${MAP_FILE}" -s ".flash" -s ".flash2"  $@