monorepo: add notes and creation tool

README-monorepo.md

@@ -0,0 +1,76 @@
+Monorepo notes
+==============
+
+
+Generating
+----------
+
+Use the [create-monorepo] script to regenerate from current master(s).
+
+[create-monorepo]: create-monorepo.py
+
+
+Structure
+---------
+
+This is a result of Git merge of several unrelated histories, each of which is
+moved to its own subdirectory during the merge.
+
+That means that this is actually all the original repos at the same time. You can
+check out any historical commit hash, or any historical tag.
+
+All tags from the previous history still exist, and in addition, each has a version
+named by its directory. I.e., for trezor-mcu tag `v1.6.3`, you can also check out
+`legacy/v1.6.3`.
+
+
+Merging pre-existing branches
+-----------------------------
+
+Because the repository shares all the histories, merging a branch or PR can be done
+with a simple `git merge`. It's often necessary to add hints to git by specifying a merge strategy - especially when some commits add new files.
+
+Use the following options: `-s subtree -X subtree=<destdir>`.
+
+Example for your local checkout:
+
+    $ git remote add core-local ~/git/trezor-core
+    $ git fetch core-local
+    $ git merge core-local/wip -s subtree -X subtree=core
+
+Same options should be used for `git rebase` of a pre-existing branch.
+
+
+Sub-repositories
+----------------
+
+The monorepo has two subdirectories that can be exported to separate repos:
+
+* **common** exports to https://github.com/trezor/trezor-common
+* **crypto** exports to https://github.com/trezor/trezor-crypto
+
+These exports are managed with [git-subrepo] tool. To export all commits that touch
+one of these directories, run the following command:
+
+    $ git subrepo push <dirname>
+
+You will need commit access to the respective GitHub repository.
+
+For installation instructions and detailed usage info, refer to the [git-subrepo] README.
+
+[git-subrepo]: https://github.com/ingydotnet/git-subrepo
+
+---
+
+Sketch of further details:
+
+What git-subrepo does under the hood is create and fetch a remote for the export,
+check out `parent` revision and replay all commits since `commit` using
+something along the lines of `git filter-branch --subdirectory-filter`.
+
+So basically a nicely tuned git-subtree.
+
+This can all be done manually if need be (or if you need more advanced usecases like
+importing changes from the repo commit-by-commit, because git-subrepo will squash
+on import). See [this nice article](https://medium.com/@porteneuve/mastering-git-subtrees-943d29a798ec)
+for hints.

create_monorepo.py

@@ -0,0 +1,137 @@
+#!/usr/bin/python3
+import glob
+import os
+import subprocess
+
+TREZOR_REPO = "https://github.com/trezor"
+
+NAME="monorepo"
+MAIN_REPO = "trezor-core"
+SUBREPOS = {
+    "trezor-common": "common",
+    "trezor-crypto": "crypto",
+    "trezor-mcu": "legacy",
+    "trezor-storage": "storage",
+    "python-trezor": "python",
+}
+PUBLISHED_SUBREPOS = ["trezor-common", "trezor-crypto"]
+
+KEEP_TAGS = ["trezor-core", "trezor-mcu", "python-trezor"]
+
+GITSUBREPO_TEMPLATE = """\
+; DO NOT EDIT (unless you know what you are doing)
+;
+; This subdirectory is a git "subrepo", and this file is maintained by the
+; git-subrepo command. See https://github.com/git-commands/git-subrepo#readme
+;
+[subrepo]
+	remote = git+ssh://git@github.com/trezor/{remote}
+	branch = master
+	commit = {remote_head}
+	parent = {current_head}
+	method = rebase
+	cmdver = 0.4.0
+"""
+
+
+def lines(s):
+    yield from s.strip().split("\n")
+
+
+def git(args):
+    print("+ git:", args)
+    return subprocess.check_output("git " + args, universal_newlines=True, shell=True)
+
+
+def move_to_subtree(remote, dst):
+    os.makedirs(dst, exist_ok=True)
+    for fn in lines(git(f"ls-tree --name-only remotes/{remote}/master")):
+        if fn == ".gitmodules":
+            continue
+        git(f"mv {fn} {dst}/{fn}")
+
+
+def rewrite_gitmodules(remote, dst):
+    master_gitmodules = git("show master:.gitmodules")
+    try:
+        gitmodules = git(f"show {remote}/master:.gitmodules")
+    except:
+        # no gitmodules
+        return
+    gitmodules = gitmodules.replace('submodule "', f'submodule "{dst}/')
+    with open(".gitmodules", "w") as f:
+        f.write(master_gitmodules + gitmodules)
+    git("add .gitmodules")
+
+
+def merge_remote(remote, dst):
+    git(f"remote add {remote} {TREZOR_REPO}/{remote}")
+    git(f"fetch {remote}")
+    try:
+        git(f"merge --no-commit --allow-unrelated-histories {remote}/master")
+    except:
+        # this might fail because of .gitmodules conflict
+        pass
+
+    rewrite_gitmodules(remote, dst)
+    move_to_subtree(remote, dst)
+
+
+def retag_remote(remote, dst):
+    for tagline in lines(git(f"ls-remote -t {remote}")):
+        commit, tagpath = tagline.split()
+        tagname = os.path.basename(tagpath)
+        git(f"tag {dst}/{tagname} {commit}")
+        git(f"tag -d {tagname}")
+
+
+def generate_subrepo_file(remote):
+    current_head = git("rev-parse master").strip()
+    remote_head = git(f"rev-parse {remote}/master").strip()
+    dst = SUBREPOS[remote]
+    with open(f"{dst}/.gitrepo", "w") as f:
+        f.write(GITSUBREPO_TEMPLATE.format(remote=remote, current_head=current_head, remote_head=remote_head))
+    git(f"add {dst}/.gitrepo")
+
+
+def main():
+    git(f"clone {TREZOR_REPO}/{MAIN_REPO} {NAME}")
+    os.chdir(NAME)
+    move_to_subtree("origin", "core")
+    git(f"commit -m 'MONOREPO CREATE FROM {MAIN_REPO}'")
+    retag_remote("origin", "core")
+
+    for remote, dst in SUBREPOS.items():
+        merge_remote(remote, dst)
+
+        if remote in PUBLISHED_SUBREPOS:
+            with open(f"{dst}/.gitmodules", "w") as f:
+                f.write(git(f"show {remote}/master:.gitmodules"))
+            git(f"add {dst}/.gitmodules")
+
+        git(f"commit -m 'MONOREPO MERGE {remote}'")
+
+        try:
+            retag_remote(remote, dst)
+        except:
+            pass
+
+    for submodule in glob.glob("*/vendor/*"):
+        modname = os.path.basename(submodule)
+        if modname not in SUBREPOS:
+            continue
+
+        git(f"rm {submodule}")
+        symlink_target = f"../../{SUBREPOS[modname]}"
+        os.symlink(symlink_target, submodule)
+        git(f"add {submodule}")
+
+    git(f"commit -m 'MONOREPO RELINK SUBMODULES'")
+
+    for remote in PUBLISHED_SUBREPOS:
+        generate_subrepo_file(remote)
+    git(f"commit -m 'MONOREPO SUBREPO FILES'")
+
+
+if __name__ == "__main__":
+    main()