From 7958061705228cd147c959dc5780b4097d073bc3 Mon Sep 17 00:00:00 2001 From: matejcik Date: Fri, 3 Nov 2023 14:19:24 +0100 Subject: [PATCH] build(core): optimize qstr and module collection Run the preprocessor on each file separately. This allows parallelization and doesn't need to re-run for all files if just a small number is changed. Replace simple extractors with one-liners which are generally going to be faster. --- core/SConscript.firmware | 12 +++++++----- core/SConscript.unix | 14 ++++++++------ core/site_scons/site_tools/micropython/__init__.py | 14 ++++++++++---- .../site_tools/micropython/moduledefs.py | 13 ------------- core/site_scons/site_tools/micropython/qstrdefs.py | 14 -------------- 5 files changed, 25 insertions(+), 42 deletions(-) delete mode 100644 core/site_scons/site_tools/micropython/moduledefs.py delete mode 100644 core/site_scons/site_tools/micropython/qstrdefs.py diff --git a/core/SConscript.firmware b/core/SConscript.firmware index 85cd7f6c33..23d4959e72 100644 --- a/core/SConscript.firmware +++ b/core/SConscript.firmware @@ -489,8 +489,10 @@ qstr_protobuf = env.Command( qstr_micropython = 'vendor/micropython/py/qstrdefs.h' +micropy_defines = env.MicroPyDefines(source=SOURCE_QSTR) + qstr_collected = env.CollectQstr( - target='genhdr/qstrdefs.collected.h', source=SOURCE_QSTR) + target='genhdr/qstrdefs.collected.h', source=micropy_defines) qstr_preprocessed = env.PreprocessQstr( target='genhdr/qstrdefs.preprocessed.h', @@ -506,16 +508,16 @@ env.Ignore(qstr_collected, qstr_generated) # moduledefs_collected = env.CollectModules( - target='genhdr/moduledefs.collected.h', source=SOURCE_QSTR) + target='genhdr/moduledefs.collected.h', source=micropy_defines) hdr_moduledefs = env.Command( target='genhdr/moduledefs.h', source=moduledefs_collected, action='$MAKEMODULEDEFS $SOURCE > $TARGET', ) -env.Ignore(moduledefs_collected, moduledefs_collected) -env.Ignore(moduledefs_collected, qstr_generated) -env.Ignore(moduledefs_collected, hdr_moduledefs) +env.Ignore(micropy_defines, micropy_defines) +env.Ignore(micropy_defines, qstr_generated) +env.Ignore(micropy_defines, hdr_moduledefs) # # Micropython version diff --git a/core/SConscript.unix b/core/SConscript.unix index 94617ef1a0..1f50c9733d 100644 --- a/core/SConscript.unix +++ b/core/SConscript.unix @@ -564,8 +564,10 @@ qstr_protobuf = env.Command( qstr_micropython = 'vendor/micropython/py/qstrdefs.h' +micropy_defines = env.MicroPyDefines(source=SOURCE_QSTR) + qstr_collected = env.CollectQstr( - target='genhdr/qstrdefs.collected.h', source=SOURCE_QSTR) + target='genhdr/qstrdefs.collected.h', source=micropy_defines) qstr_preprocessed = env.PreprocessQstr( target='genhdr/qstrdefs.preprocessed.h', @@ -581,7 +583,7 @@ env.Ignore(qstr_collected, qstr_generated) # moduledefs_collected = env.CollectModules( - target='genhdr/moduledefs.collected.h', source=SOURCE_QSTR) + target='genhdr/moduledefs.collected.h', source=micropy_defines) hdr_moduledefs = env.Command( @@ -602,10 +604,10 @@ hdr_version = env.Command( source='', action='$MAKEVERSIONHDR $TARGET', ) -env.Ignore(hdr_moduledefs, hdr_moduledefs) -env.Ignore(hdr_moduledefs, qstr_collected) -env.Ignore(hdr_moduledefs, qstr_preprocessed) -env.Ignore(hdr_moduledefs, qstr_generated) +env.Ignore(micropy_defines, hdr_moduledefs) +env.Ignore(micropy_defines, qstr_collected) +env.Ignore(micropy_defines, qstr_preprocessed) +env.Ignore(micropy_defines, qstr_generated) # # Frozen modules diff --git a/core/site_scons/site_tools/micropython/__init__.py b/core/site_scons/site_tools/micropython/__init__.py index 80a03ca6b4..cf231a673a 100644 --- a/core/site_scons/site_tools/micropython/__init__.py +++ b/core/site_scons/site_tools/micropython/__init__.py @@ -8,9 +8,14 @@ def generate(env): MODULECOL="site_scons/site_tools/micropython/moduledefs.py", ) + env["BUILDERS"]["MicroPyDefines"] = SCons.Builder.Builder( + action="$CC -E $CCFLAGS_QSTR $CFLAGS $CCFLAGS $_CCCOMCOM $SOURCE > $TARGET", + suffix=".upydef", + single_source=True, + ) + env["BUILDERS"]["CollectQstr"] = SCons.Builder.Builder( - action="$CC -E $CCFLAGS_QSTR $CFLAGS $CCFLAGS $_CCCOMCOM $SOURCES" - " | $PYTHON $QSTRCOL > $TARGET" + action="cat $SOURCES | perl -nle 'print \"Q($1)\" while /MP_QSTR_(\\w+)/g' > $TARGET" ) env["BUILDERS"]["PreprocessQstr"] = SCons.Builder.Builder( @@ -25,8 +30,9 @@ def generate(env): ) env["BUILDERS"]["CollectModules"] = SCons.Builder.Builder( - action="$CC -E $CCFLAGS_QSTR $CFLAGS $CCFLAGS $_CCCOMCOM $SOURCES" - " | $PYTHON $MODULECOL > $TARGET" + action="grep ^MP_REGISTER_MODULE $SOURCES > $TARGET" + # action="$CC -E $CCFLAGS_QSTR $CFLAGS $CCFLAGS $_CCCOMCOM $SOURCES" + # " | $PYTHON $MODULECOL > $TARGET" ) def generate_frozen_module(source, target, env, for_signature): diff --git a/core/site_scons/site_tools/micropython/moduledefs.py b/core/site_scons/site_tools/micropython/moduledefs.py deleted file mode 100644 index d6efb3ca46..0000000000 --- a/core/site_scons/site_tools/micropython/moduledefs.py +++ /dev/null @@ -1,13 +0,0 @@ -import re -import sys - - -def process(source, target): - re_module = re.compile(r"MP_REGISTER_MODULE\(.*?,\s*.*?\);") - for line in source: - for match in re_module.findall(line): - target.write(f"{match}\n") - - -if __name__ == "__main__": - process(sys.stdin, sys.stdout) diff --git a/core/site_scons/site_tools/micropython/qstrdefs.py b/core/site_scons/site_tools/micropython/qstrdefs.py deleted file mode 100644 index 718935989a..0000000000 --- a/core/site_scons/site_tools/micropython/qstrdefs.py +++ /dev/null @@ -1,14 +0,0 @@ -import re -import sys - - -def process(source, target): - re_qstr = re.compile(r"MP_QSTR_[_a-zA-Z0-9]+") - for line in source: - for match in re_qstr.findall(line): - name = match.replace("MP_QSTR_", "") - target.write(f"Q({name})\n") - - -if __name__ == "__main__": - process(sys.stdin, sys.stdout)