diff --git a/README-monorepo.md b/README-monorepo.md new file mode 100644 index 000000000..d091096b1 --- /dev/null +++ b/README-monorepo.md @@ -0,0 +1,76 @@ +Monorepo notes +============== + + +Generating +---------- + +Use the [create-monorepo] script to regenerate from current master(s). + +[create-monorepo]: create-monorepo.py + + +Structure +--------- + +This is a result of Git merge of several unrelated histories, each of which is +moved to its own subdirectory during the merge. + +That means that this is actually all the original repos at the same time. You can +check out any historical commit hash, or any historical tag. + +All tags from the previous history still exist, and in addition, each has a version +named by its directory. I.e., for trezor-mcu tag `v1.6.3`, you can also check out +`legacy/v1.6.3`. + + +Merging pre-existing branches +----------------------------- + +Because the repository shares all the histories, merging a branch or PR can be done +with a simple `git merge`. It's often necessary to add hints to git by specifying a merge strategy - especially when some commits add new files. + +Use the following options: `-s subtree -X subtree=`. + +Example for your local checkout: + + $ git remote add core-local ~/git/trezor-core + $ git fetch core-local + $ git merge core-local/wip -s subtree -X subtree=core + +Same options should be used for `git rebase` of a pre-existing branch. + + +Sub-repositories +---------------- + +The monorepo has two subdirectories that can be exported to separate repos: + +* **common** exports to https://github.com/trezor/trezor-common +* **crypto** exports to https://github.com/trezor/trezor-crypto + +These exports are managed with [git-subrepo] tool. To export all commits that touch +one of these directories, run the following command: + + $ git subrepo push + +You will need commit access to the respective GitHub repository. + +For installation instructions and detailed usage info, refer to the [git-subrepo] README. + +[git-subrepo]: https://github.com/ingydotnet/git-subrepo + +--- + +Sketch of further details: + +What git-subrepo does under the hood is create and fetch a remote for the export, +check out `parent` revision and replay all commits since `commit` using +something along the lines of `git filter-branch --subdirectory-filter`. + +So basically a nicely tuned git-subtree. + +This can all be done manually if need be (or if you need more advanced usecases like +importing changes from the repo commit-by-commit, because git-subrepo will squash +on import). See [this nice article](https://medium.com/@porteneuve/mastering-git-subtrees-943d29a798ec) +for hints. diff --git a/create_monorepo.py b/create_monorepo.py new file mode 100644 index 000000000..434142ed7 --- /dev/null +++ b/create_monorepo.py @@ -0,0 +1,137 @@ +#!/usr/bin/python3 +import glob +import os +import subprocess + +TREZOR_REPO = "https://github.com/trezor" + +NAME="monorepo" +MAIN_REPO = "trezor-core" +SUBREPOS = { + "trezor-common": "common", + "trezor-crypto": "crypto", + "trezor-mcu": "legacy", + "trezor-storage": "storage", + "python-trezor": "python", +} +PUBLISHED_SUBREPOS = ["trezor-common", "trezor-crypto"] + +KEEP_TAGS = ["trezor-core", "trezor-mcu", "python-trezor"] + +GITSUBREPO_TEMPLATE = """\ +; DO NOT EDIT (unless you know what you are doing) +; +; This subdirectory is a git "subrepo", and this file is maintained by the +; git-subrepo command. See https://github.com/git-commands/git-subrepo#readme +; +[subrepo] + remote = git+ssh://git@github.com/trezor/{remote} + branch = master + commit = {remote_head} + parent = {current_head} + method = rebase + cmdver = 0.4.0 +""" + + +def lines(s): + yield from s.strip().split("\n") + + +def git(args): + print("+ git:", args) + return subprocess.check_output("git " + args, universal_newlines=True, shell=True) + + +def move_to_subtree(remote, dst): + os.makedirs(dst, exist_ok=True) + for fn in lines(git(f"ls-tree --name-only remotes/{remote}/master")): + if fn == ".gitmodules": + continue + git(f"mv {fn} {dst}/{fn}") + + +def rewrite_gitmodules(remote, dst): + master_gitmodules = git("show master:.gitmodules") + try: + gitmodules = git(f"show {remote}/master:.gitmodules") + except: + # no gitmodules + return + gitmodules = gitmodules.replace('submodule "', f'submodule "{dst}/') + with open(".gitmodules", "w") as f: + f.write(master_gitmodules + gitmodules) + git("add .gitmodules") + + +def merge_remote(remote, dst): + git(f"remote add {remote} {TREZOR_REPO}/{remote}") + git(f"fetch {remote}") + try: + git(f"merge --no-commit --allow-unrelated-histories {remote}/master") + except: + # this might fail because of .gitmodules conflict + pass + + rewrite_gitmodules(remote, dst) + move_to_subtree(remote, dst) + + +def retag_remote(remote, dst): + for tagline in lines(git(f"ls-remote -t {remote}")): + commit, tagpath = tagline.split() + tagname = os.path.basename(tagpath) + git(f"tag {dst}/{tagname} {commit}") + git(f"tag -d {tagname}") + + +def generate_subrepo_file(remote): + current_head = git("rev-parse master").strip() + remote_head = git(f"rev-parse {remote}/master").strip() + dst = SUBREPOS[remote] + with open(f"{dst}/.gitrepo", "w") as f: + f.write(GITSUBREPO_TEMPLATE.format(remote=remote, current_head=current_head, remote_head=remote_head)) + git(f"add {dst}/.gitrepo") + + +def main(): + git(f"clone {TREZOR_REPO}/{MAIN_REPO} {NAME}") + os.chdir(NAME) + move_to_subtree("origin", "core") + git(f"commit -m 'MONOREPO CREATE FROM {MAIN_REPO}'") + retag_remote("origin", "core") + + for remote, dst in SUBREPOS.items(): + merge_remote(remote, dst) + + if remote in PUBLISHED_SUBREPOS: + with open(f"{dst}/.gitmodules", "w") as f: + f.write(git(f"show {remote}/master:.gitmodules")) + git(f"add {dst}/.gitmodules") + + git(f"commit -m 'MONOREPO MERGE {remote}'") + + try: + retag_remote(remote, dst) + except: + pass + + for submodule in glob.glob("*/vendor/*"): + modname = os.path.basename(submodule) + if modname not in SUBREPOS: + continue + + git(f"rm {submodule}") + symlink_target = f"../../{SUBREPOS[modname]}" + os.symlink(symlink_target, submodule) + git(f"add {submodule}") + + git(f"commit -m 'MONOREPO RELINK SUBMODULES'") + + for remote in PUBLISHED_SUBREPOS: + generate_subrepo_file(remote) + git(f"commit -m 'MONOREPO SUBREPO FILES'") + + +if __name__ == "__main__": + main()