Added ability to freeze multiple directories; freeze neopixel library in cpx build (#199)

Reworked frozen module support: clean up makefiles and handle multiple directories.
Modules to freeze are included as git submodules.

Add neopixel to circuitplayground express build.

Fixes #56 

7 files changed, 38 insertions, 22 deletions

diff --git a/.gitmodules b/.gitmodules
index 21af5f0a6..54c4d5a9c 100644
--- a/.gitmodules
+++ b/.gitmodules
@@ -17,3 +17,6 @@
 [submodule "tools/uf2"]
 	path = tools/uf2
 	url = https://github.com/Microsoft/uf2.git
+[submodule "atmel-samd/frozen/Adafruit_CircuitPython_NeoPixel"]
+	path = frozen/Adafruit_CircuitPython_NeoPixel
+	url = https://github.com/adafruit/Adafruit_CircuitPython_NeoPixel
diff --git a/atmel-samd/Makefile b/atmel-samd/Makefile
index 9e3a10bd8..913461261 100644
--- a/atmel-samd/Makefile
+++ b/atmel-samd/Makefile
@@ -145,9 +145,11 @@ CFLAGS += -DMICROPY_MODULE_FROZEN_STR
 CFLAGS += -Wno-error=lto-type-mismatch
 endif
 
-ifneq ($(FROZEN_MPY_DIR),)
 # To use frozen bytecode, put your .py files in a subdirectory (eg frozen/) and
-# then invoke make with FROZEN_MPY_DIR=frozen (be sure to build from scratch).
+# then invoke make with FROZEN_MPY_DIR=frozen or FROZEN_MPY_DIRS="dir1 dir2"
+# (be sure to build from scratch).
+
+ifneq ($(FROZEN_MPY_DIRS),)
 CFLAGS += -DMICROPY_QSTR_EXTRA_POOL=mp_qstr_frozen_const_pool
 CFLAGS += -DMICROPY_MODULE_FROZEN_MPY
 CFLAGS += -Wno-error=lto-type-mismatch
diff --git a/atmel-samd/boards/circuitplayground_express/mpconfigboard.mk b/atmel-samd/boards/circuitplayground_express/mpconfigboard.mk
index 54739ddb2..d773f2f60 100644
--- a/atmel-samd/boards/circuitplayground_express/mpconfigboard.mk
+++ b/atmel-samd/boards/circuitplayground_express/mpconfigboard.mk
@@ -5,3 +5,6 @@ USB_PID = 0x8019
 FLASH_IMPL = spi_flash.c
 
 CHIP_VARIANT = SAMD21G18A
+
+# Include these Python libraries in firmware.
+FROZEN_MPY_DIRS += $(TOP)/frozen/Adafruit_CircuitPython_NeoPixel
diff --git a/frozen/Adafruit_CircuitPython_NeoPixel b/frozen/Adafruit_CircuitPython_NeoPixel
new file mode 160000
+Subproject 1142f1c7fdc232a46d74dd4f1946a5f462ae255
diff --git a/py/mkenv.mk b/py/mkenv.mk
index e4381f1b2..5a118854a 100644
--- a/py/mkenv.mk
+++ b/py/mkenv.mk
@@ -42,12 +42,16 @@ endif
 PY_SRC ?= $(TOP)/py
 BUILD ?= build
 
-RM = rm
 ECHO = @echo
+
+CD = cd
 CP = cp
+FIND = find
 MKDIR = mkdir
-SED = sed
 PYTHON = python
+RM = rm
+RSYNC = rsync
+SED = sed
 
 AS = $(CROSS_COMPILE)as
 CC = $(CROSS_COMPILE)gcc
diff --git a/py/mkrules.mk b/py/mkrules.mk
index d61e1dda0..ab163e757 100644
--- a/py/mkrules.mk
+++ b/py/mkrules.mk
@@ -101,29 +101,30 @@ $(BUILD)/frozen.c: $(wildcard $(FROZEN_DIR)/*) $(HEADER_BUILD) $(FROZEN_EXTRA_DE
 	$(Q)$(MAKE_FROZEN) $(FROZEN_DIR) > $@
 endif
 
-ifneq ($(FROZEN_MPY_DIR),)
+ifneq ($(FROZEN_MPY_DIRS),)
 # to build the MicroPython cross compiler
-$(TOP)/mpy-cross/mpy-cross: $(TOP)/py/*.[ch] $(TOP)/mpy-cross/*.[ch] $(TOP)/windows/fmode.c
+# Currently not used, because the wrong mpy-cross may be left over from a previous build. Build by hand to make sure.
+$(MPY_CROSS): $(TOP)/py/*.[ch] $(TOP)/mpy-cross/*.[ch] $(TOP)/windows/fmode.c
 	$(Q)$(MAKE) -C $(TOP)/mpy-cross
 
-# make a list of all the .py files that need compiling and freezing
-BLAH := $(info $(shell pwd))
-
-FROZEN_MPY_PY_FILES := $(shell find -L $(FROZEN_MPY_DIR) -type f -name '*.py' | $(SED) -e 's=^$(FROZEN_MPY_DIR)/==')
-FROZEN_MPY_MPY_FILES := $(addprefix $(BUILD)/frozen_mpy/,$(FROZEN_MPY_PY_FILES:.py=.mpy))
-
-# to build .mpy files from .py files
-$(BUILD)/frozen_mpy/%.mpy: $(FROZEN_MPY_DIR)/%.py
-	@$(ECHO) "MPY $<"
-	$(Q)$(MKDIR) -p $(dir $@)
-	$(Q)$(MPY_CROSS) -o $@ -s $(<:$(FROZEN_MPY_DIR)/%=%) $(MPY_CROSS_FLAGS) $<
+# Copy all the modules and single python files to freeze to a common area, omitting top-level dirs (the repo names).
+# Remove any conf.py (sphinx config) and setup.py (module install info) files, which are not meant to be frozen.
+# Then compile .mpy files from all the .py files, placing them in the same directories as the .py files.
+$(BUILD)/frozen_mpy: $(FROZEN_MPY_DIRS)
+	$(ECHO) FREEZE $(FROZEN_MPY_DIRS)
+	$(Q)$(MKDIR) -p $@
+	$(Q)$(RSYNC) -rL --include="*/" --include='*.py' --exclude="*" $(addsuffix /*,$(FROZEN_MPY_DIRS)) $@
+	$(Q)$(RM) -f $@/conf.py $@/setup.py
+	$(Q)$(CD) $@ && \
+$(FIND) -L . -type f -name '*.py' | sed 's=^\./==' | \
+xargs -n1 $(abspath $(MPY_CROSS)) $(MPY_CROSS_FLAGS)
 
 # to build frozen_mpy.c from all .mpy files
 # You need to define MPY_TOOL_LONGINT_IMPL in mpconfigport.mk
 # if the default will not work (mpz is the default).
-$(BUILD)/frozen_mpy.c: $(FROZEN_MPY_MPY_FILES) $(BUILD)/genhdr/qstrdefs.generated.h
+$(BUILD)/frozen_mpy.c: $(BUILD)/frozen_mpy $(BUILD)/genhdr/qstrdefs.generated.h
 	$(STEPECHO) "Creating $@"
-	$(Q)$(PYTHON) $(MPY_TOOL) $(MPY_TOOL_LONGINT_IMPL) -f -q $(BUILD)/genhdr/qstrdefs.preprocessed.h $(FROZEN_MPY_MPY_FILES) > $@
+	$(Q)$(PYTHON) $(MPY_TOOL) $(MPY_TOOL_LONGINT_IMPL) -f -q $(BUILD)/genhdr/qstrdefs.preprocessed.h $(shell $(FIND) -L $(BUILD)/frozen_mpy -type f -name '*.mpy') > $@
 endif
 
 ifneq ($(PROG),)
diff --git a/py/py.mk b/py/py.mk
index 70891677d..22826b1f4 100644
--- a/py/py.mk
+++ b/py/py.mk
@@ -247,12 +247,15 @@ PY_O = $(addprefix $(PY_BUILD)/, $(PY_O_BASENAME))
 
 # object file for frozen files
 ifneq ($(FROZEN_DIR),)
-PY_O += $(BUILD)/$(BUILD)/frozen.o
+PY_O += $(BUILD)/frozen.o
 endif
 
+# Combine old singular FROZEN_MPY_DIR with new multiple value form.
+FROZEN_MPY_DIRS += $(FROZEN_MPY_DIR)
+
 # object file for frozen bytecode (frozen .mpy files)
-ifneq ($(FROZEN_MPY_DIR),)
-PY_O += $(BUILD)/$(BUILD)/frozen_mpy.o
+ifneq ($(FROZEN_MPY_DIRS),)
+PY_O += $(BUILD)/frozen_mpy.o
 endif
 
 # Sources that may contain qstrings