From 607e4a905ae8654bd8450ae137ffa24367039277 Mon Sep 17 00:00:00 2001
From: Jeff Epler <jepler@gmail.com>
Date: Sun, 11 Oct 2020 15:19:55 -0500
Subject: build: parallelize the creation of qstr.i.last

Rather than simply invoking gcc in preprocessor mode with a list of files, use
a Python script with the (python3) ThreadPoolExecutor to invoke the
preprocessor in parallel.

The amount of concurrency is the number of system CPUs, not the makefile "-j"
parallelism setting, because there is no simple and correct way for a Python
program to correctly work together with make's idea of parallelism.

This reduces the build time of stm32f405 feather (a non-LTO build) from 16s to
12s on my 16-thread Ryzen machine.
---
 py/genlast.py | 38 ++++++++++++++++++++++++++++++++++++++
 py/mkrules.mk |  2 +-
 2 files changed, 39 insertions(+), 1 deletion(-)
 create mode 100644 py/genlast.py

diff --git a/py/genlast.py b/py/genlast.py
new file mode 100644
index 000000000..e14e7ef1e
--- /dev/null
+++ b/py/genlast.py
@@ -0,0 +1,38 @@
+#!/usr/bin/env python3
+
+import sys
+from concurrent.futures import ThreadPoolExecutor
+import multiprocessing
+import threading
+import subprocess
+
+
+def checkoutput1(args):
+    info = subprocess.run(args, check=True, stdout=subprocess.PIPE, input='')
+    return info.stdout
+
+idx1 = sys.argv.index('--')
+idx2 = sys.argv.index('--', idx1+1)
+check = sys.argv[1:idx1]
+always = sys.argv[idx1+1:idx2]
+command = sys.argv[idx2+1:]
+
+output_lock = threading.Lock()
+def preprocess(fn):
+    output = checkoutput1(command + [fn])
+    # Ensure our output doesn't interleave with others
+    # a threading.Lock is not a context manager object :(
+    try:
+        output_lock.acquire()
+        sys.stdout.buffer.write(output)
+    finally:
+        output_lock.release()
+
+def maybe_preprocess(fn):
+    if subprocess.call(["grep", "-lqE", "(MP_QSTR|translate)", fn]) == 0:
+        preprocess(fn)
+
+executor = ThreadPoolExecutor(max_workers=multiprocessing.cpu_count() + 1)
+executor.map(maybe_preprocess, check)
+executor.map(preprocess, always)
+executor.shutdown()
diff --git a/py/mkrules.mk b/py/mkrules.mk
index 0b3ecc0e1..84e16ec96 100644
--- a/py/mkrules.mk
+++ b/py/mkrules.mk
@@ -78,7 +78,7 @@ $(OBJ): | $(HEADER_BUILD)/qstrdefs.enum.h $(HEADER_BUILD)/mpversion.h
 # - else, process all source files ($^) [this covers "make -B" which can set $? to empty]
 $(HEADER_BUILD)/qstr.i.last: $(SRC_QSTR) $(SRC_QSTR_PREPROCESSOR) $(QSTR_GLOBAL_DEPENDENCIES) | $(HEADER_BUILD)/mpversion.h
 	$(STEPECHO) "GEN $@"
-	$(Q)grep -lE "(MP_QSTR|translate)" $(if $(filter $?,$(QSTR_GLOBAL_DEPENDENCIES)),$^,$(if $?,$?,$^)) | xargs $(CPP) $(QSTR_GEN_EXTRA_CFLAGS) $(CFLAGS) $(SRC_QSTR_PREPROCESSOR) >$(HEADER_BUILD)/qstr.i.last;
+	$(Q)$(PYTHON3) $(PY_SRC)/genlast.py $(if $(filter $?,$(QSTR_GLOBAL_DEPENDENCIES)),$^,$(if $?,$?,$^)) --  $(SRC_QSTR_PREPROCESSOR) -- $(CPP) $(QSTR_GEN_EXTRA_CFLAGS) $(CFLAGS) >$(HEADER_BUILD)/qstr.i.last;
 
 $(HEADER_BUILD)/qstr.split: $(HEADER_BUILD)/qstr.i.last $(PY_SRC)/makeqstrdefs.py
 	$(STEPECHO) "GEN $@"
-- 
cgit v1.2.3