Added ability to freeze multiple directories; freeze neopixel library in cpx build (#199)

Reworked frozen module support: clean up makefiles and handle multiple directories.
Modules to freeze are included as git submodules.

Add neopixel to circuitplayground express build.

Fixes #56 

3 files changed, 28 insertions, 20 deletions

diff --git a/py/mkenv.mk b/py/mkenv.mk
index e4381f1b2..5a118854a 100644
--- a/py/mkenv.mk
+++ b/py/mkenv.mk
@@ -42,12 +42,16 @@ endif
 PY_SRC ?= $(TOP)/py
 BUILD ?= build
 
-RM = rm
 ECHO = @echo
+
+CD = cd
 CP = cp
+FIND = find
 MKDIR = mkdir
-SED = sed
 PYTHON = python
+RM = rm
+RSYNC = rsync
+SED = sed
 
 AS = $(CROSS_COMPILE)as
 CC = $(CROSS_COMPILE)gcc
diff --git a/py/mkrules.mk b/py/mkrules.mk
index d61e1dda0..ab163e757 100644
--- a/py/mkrules.mk
+++ b/py/mkrules.mk
@@ -101,29 +101,30 @@ $(BUILD)/frozen.c: $(wildcard $(FROZEN_DIR)/*) $(HEADER_BUILD) $(FROZEN_EXTRA_DE
 	$(Q)$(MAKE_FROZEN) $(FROZEN_DIR) > $@
 endif
 
-ifneq ($(FROZEN_MPY_DIR),)
+ifneq ($(FROZEN_MPY_DIRS),)
 # to build the MicroPython cross compiler
-$(TOP)/mpy-cross/mpy-cross: $(TOP)/py/*.[ch] $(TOP)/mpy-cross/*.[ch] $(TOP)/windows/fmode.c
+# Currently not used, because the wrong mpy-cross may be left over from a previous build. Build by hand to make sure.
+$(MPY_CROSS): $(TOP)/py/*.[ch] $(TOP)/mpy-cross/*.[ch] $(TOP)/windows/fmode.c
 	$(Q)$(MAKE) -C $(TOP)/mpy-cross
 
-# make a list of all the .py files that need compiling and freezing
-BLAH := $(info $(shell pwd))
-
-FROZEN_MPY_PY_FILES := $(shell find -L $(FROZEN_MPY_DIR) -type f -name '*.py' | $(SED) -e 's=^$(FROZEN_MPY_DIR)/==')
-FROZEN_MPY_MPY_FILES := $(addprefix $(BUILD)/frozen_mpy/,$(FROZEN_MPY_PY_FILES:.py=.mpy))
-
-# to build .mpy files from .py files
-$(BUILD)/frozen_mpy/%.mpy: $(FROZEN_MPY_DIR)/%.py
-	@$(ECHO) "MPY $<"
-	$(Q)$(MKDIR) -p $(dir $@)
-	$(Q)$(MPY_CROSS) -o $@ -s $(<:$(FROZEN_MPY_DIR)/%=%) $(MPY_CROSS_FLAGS) $<
+# Copy all the modules and single python files to freeze to a common area, omitting top-level dirs (the repo names).
+# Remove any conf.py (sphinx config) and setup.py (module install info) files, which are not meant to be frozen.
+# Then compile .mpy files from all the .py files, placing them in the same directories as the .py files.
+$(BUILD)/frozen_mpy: $(FROZEN_MPY_DIRS)
+	$(ECHO) FREEZE $(FROZEN_MPY_DIRS)
+	$(Q)$(MKDIR) -p $@
+	$(Q)$(RSYNC) -rL --include="*/" --include='*.py' --exclude="*" $(addsuffix /*,$(FROZEN_MPY_DIRS)) $@
+	$(Q)$(RM) -f $@/conf.py $@/setup.py
+	$(Q)$(CD) $@ && \
+$(FIND) -L . -type f -name '*.py' | sed 's=^\./==' | \
+xargs -n1 $(abspath $(MPY_CROSS)) $(MPY_CROSS_FLAGS)
 
 # to build frozen_mpy.c from all .mpy files
 # You need to define MPY_TOOL_LONGINT_IMPL in mpconfigport.mk
 # if the default will not work (mpz is the default).
-$(BUILD)/frozen_mpy.c: $(FROZEN_MPY_MPY_FILES) $(BUILD)/genhdr/qstrdefs.generated.h
+$(BUILD)/frozen_mpy.c: $(BUILD)/frozen_mpy $(BUILD)/genhdr/qstrdefs.generated.h
 	$(STEPECHO) "Creating $@"
-	$(Q)$(PYTHON) $(MPY_TOOL) $(MPY_TOOL_LONGINT_IMPL) -f -q $(BUILD)/genhdr/qstrdefs.preprocessed.h $(FROZEN_MPY_MPY_FILES) > $@
+	$(Q)$(PYTHON) $(MPY_TOOL) $(MPY_TOOL_LONGINT_IMPL) -f -q $(BUILD)/genhdr/qstrdefs.preprocessed.h $(shell $(FIND) -L $(BUILD)/frozen_mpy -type f -name '*.mpy') > $@
 endif
 
 ifneq ($(PROG),)
diff --git a/py/py.mk b/py/py.mk
index 70891677d..22826b1f4 100644
--- a/py/py.mk
+++ b/py/py.mk
@@ -247,12 +247,15 @@ PY_O = $(addprefix $(PY_BUILD)/, $(PY_O_BASENAME))
 
 # object file for frozen files
 ifneq ($(FROZEN_DIR),)
-PY_O += $(BUILD)/$(BUILD)/frozen.o
+PY_O += $(BUILD)/frozen.o
 endif
 
+# Combine old singular FROZEN_MPY_DIR with new multiple value form.
+FROZEN_MPY_DIRS += $(FROZEN_MPY_DIR)
+
 # object file for frozen bytecode (frozen .mpy files)
-ifneq ($(FROZEN_MPY_DIR),)
-PY_O += $(BUILD)/$(BUILD)/frozen_mpy.o
+ifneq ($(FROZEN_MPY_DIRS),)
+PY_O += $(BUILD)/frozen_mpy.o
 endif
 
 # Sources that may contain qstrings