Merge branch 'master' into new-pixelbuf-api

11 files changed, 175 insertions, 34 deletions

diff --git a/py/circuitpy_defns.mk b/py/circuitpy_defns.mk
index 211e85368..446620d27 100644
--- a/py/circuitpy_defns.mk
+++ b/py/circuitpy_defns.mk
@@ -117,6 +117,9 @@ endif
 ifeq ($(CIRCUITPY_AUDIOMIXER),1)
 SRC_PATTERNS += audiomixer/%
 endif
+ifeq ($(CIRCUITPY_AUDIOMP3),1)
+SRC_PATTERNS += audiomp3/%
+endif
 ifeq ($(CIRCUITPY_BITBANGIO),1)
 SRC_PATTERNS += bitbangio/%
 endif
@@ -319,6 +322,8 @@ SRC_SHARED_MODULE_ALL = \
 	audiomixer/__init__.c \
 	audiomixer/Mixer.c \
 	audiomixer/MixerVoice.c \
+	audiomp3/__init__.c \
+	audiomp3/MP3File.c \
 	bitbangio/I2C.c \
 	bitbangio/OneWire.c \
 	bitbangio/SPI.c \
@@ -371,6 +376,26 @@ SRC_SHARED_MODULE_ALL += \
 	touchio/TouchIn.c \
 	touchio/__init__.c
 endif
+ifeq ($(CIRCUITPY_AUDIOMP3),1)
+SRC_MOD += $(addprefix lib/mp3/src/, \
+	bitstream.c \
+	buffers.c \
+	dct32.c \
+	dequant.c \
+	dqchan.c \
+	huffman.c \
+	hufftabs.c \
+	imdct.c \
+	mp3dec.c \
+	mp3tabs.c \
+	polyphase.c \
+	scalfact.c \
+	stproc.c \
+	subband.c \
+	trigtabs.c \
+)
+$(BUILD)/lib/mp3/src/buffers.o: CFLAGS += -include "py/misc.h" -D'MPDEC_ALLOCATOR(x)=m_malloc(x,0)' -D'MPDEC_FREE(x)=m_free(x)'
+endif
 
 # All possible sources are listed here, and are filtered by SRC_PATTERNS.
 SRC_SHARED_MODULE_INTERNAL = \
diff --git a/py/circuitpy_mpconfig.h b/py/circuitpy_mpconfig.h
index b1e7bc05a..5ee5d52c0 100644
--- a/py/circuitpy_mpconfig.h
+++ b/py/circuitpy_mpconfig.h
@@ -28,11 +28,12 @@
 // sure that the same feature set and settings are used, such as in atmel-samd
 // and nrf.
 
-#include <stdint.h>
-
 #ifndef __INCLUDED_MPCONFIG_CIRCUITPY_H
 #define __INCLUDED_MPCONFIG_CIRCUITPY_H
 
+#include <stdint.h>
+#include <stdatomic.h>
+
 // This is CircuitPython.
 #define CIRCUITPY 1
 
@@ -251,6 +252,13 @@ extern const struct _mp_obj_module_t audiomixer_module;
 #define AUDIOMIXER_MODULE
 #endif
 
+#if CIRCUITPY_AUDIOMP3
+#define AUDIOMP3_MODULE         { MP_OBJ_NEW_QSTR(MP_QSTR_audiomp3), (mp_obj_t)&audiomp3_module },
+extern const struct _mp_obj_module_t audiomp3_module;
+#else
+#define AUDIOMP3_MODULE
+#endif
+
 #if CIRCUITPY_AUDIOPWMIO
 #define AUDIOPWMIO_MODULE         { MP_OBJ_NEW_QSTR(MP_QSTR_audiopwmio), (mp_obj_t)&audiopwmio_module },
 extern const struct _mp_obj_module_t audiopwmio_module;
@@ -581,6 +589,7 @@ extern const struct _mp_obj_module_t ustack_module;
     AUDIOCORE_MODULE \
     AUDIOIO_MODULE \
     AUDIOMIXER_MODULE \
+    AUDIOMP3_MODULE \
     AUDIOPWMIO_MODULE \
     BITBANGIO_MODULE \
     BLEIO_MODULE \
@@ -652,17 +661,19 @@ extern const struct _mp_obj_module_t ustack_module;
     FLASH_ROOT_POINTERS \
     NETWORK_ROOT_POINTERS \
 
-void run_background_tasks(void);
-#define RUN_BACKGROUND_TASKS (run_background_tasks())
+void supervisor_run_background_tasks_if_tick(void);
+#define RUN_BACKGROUND_TASKS (supervisor_run_background_tasks_if_tick())
 
 // TODO: Used in wiznet5k driver, but may not be needed in the long run.
 #define MICROPY_THREAD_YIELD()
 
-#define MICROPY_VM_HOOK_LOOP run_background_tasks();
-#define MICROPY_VM_HOOK_RETURN run_background_tasks();
+#define MICROPY_VM_HOOK_LOOP RUN_BACKGROUND_TASKS;
+#define MICROPY_VM_HOOK_RETURN RUN_BACKGROUND_TASKS;
 
 #define CIRCUITPY_AUTORELOAD_DELAY_MS 500
 #define CIRCUITPY_FILESYSTEM_FLUSH_INTERVAL_MS 1000
 #define CIRCUITPY_BOOT_OUTPUT_FILE "/boot_out.txt"
 
+#define CIRCUITPY_VERBOSE_BLE 0
+
 #endif  // __INCLUDED_MPCONFIG_CIRCUITPY_H
diff --git a/py/circuitpy_mpconfig.mk b/py/circuitpy_mpconfig.mk
index 241fa1017..c614ea06a 100644
--- a/py/circuitpy_mpconfig.mk
+++ b/py/circuitpy_mpconfig.mk
@@ -104,6 +104,15 @@ CIRCUITPY_AUDIOMIXER = $(CIRCUITPY_AUDIOIO)
 endif
 CFLAGS += -DCIRCUITPY_AUDIOMIXER=$(CIRCUITPY_AUDIOMIXER)
 
+ifndef CIRCUITPY_AUDIOMP3
+ifeq ($(CIRCUITPY_FULL_BUILD),1)
+CIRCUITPY_AUDIOMP3 = $(CIRCUITPY_AUDIOCORE)
+else
+CIRCUITPY_AUDIOMP3 = 0
+endif
+endif
+CFLAGS += -DCIRCUITPY_AUDIOMP3=$(CIRCUITPY_AUDIOMP3)
+
 ifndef CIRCUITPY_BITBANGIO
 CIRCUITPY_BITBANGIO = $(CIRCUITPY_FULL_BUILD)
 endif
diff --git a/py/makeqstrdata.py b/py/makeqstrdata.py
index 8e3835d86..0d667959d 100644
--- a/py/makeqstrdata.py
+++ b/py/makeqstrdata.py
@@ -103,14 +103,10 @@ def compute_huffman_coding(translations, qstrs, compression_filename):
     # go through each qstr and print it out
     for _, _, qstr in qstrs.values():
         all_strings.append(qstr)
-    all_strings_concat = "".join(all_strings).encode("utf-8")
+    all_strings_concat = "".join(all_strings)
     counts = collections.Counter(all_strings_concat)
-    # add other values
-    for i in range(256):
-        if i not in counts:
-            counts[i] = 0
     cb = huffman.codebook(counts.items())
-    values = bytearray()
+    values = []
     length_count = {}
     renumbered = 0
     last_l = None
@@ -124,26 +120,27 @@ def compute_huffman_coding(translations, qstrs, compression_filename):
         if last_l:
             renumbered <<= (l - last_l)
         canonical[ch] = '{0:0{width}b}'.format(renumbered, width=l)
-        if chr(ch) in C_ESCAPES:
-            s = C_ESCAPES[chr(ch)]
-        else:
-            s = chr(ch)
-        print("//", ch, s, counts[ch], canonical[ch], renumbered)
+        s = C_ESCAPES.get(ch, ch)
+        print("//", ord(ch), s, counts[ch], canonical[ch], renumbered)
         renumbered += 1
         last_l = l
     lengths = bytearray()
-    for i in range(1, max(length_count) + 1):
+    print("// length count", length_count)
+    for i in range(1, max(length_count) + 2):
         lengths.append(length_count.get(i, 0))
+    print("// values", values, "lengths", len(lengths), lengths)
+    print("// estimated total memory size", len(lengths) + 2*len(values) + sum(len(cb[u]) for u in all_strings_concat))
     print("//", values, lengths)
+    values_type = "uint16_t" if max(ord(u) for u in values) > 255 else "uint8_t"
     with open(compression_filename, "w") as f:
         f.write("const uint8_t lengths[] = {{ {} }};\n".format(", ".join(map(str, lengths))))
-        f.write("const uint8_t values[256] = {{ {} }};\n".format(", ".join(map(str, values))))
+        f.write("const {} values[] = {{ {} }};\n".format(values_type, ", ".join(str(ord(u)) for u in values)))
     return values, lengths
 
 def decompress(encoding_table, length, encoded):
     values, lengths = encoding_table
     #print(l, encoded)
-    dec = bytearray(length)
+    dec = []
     this_byte = 0
     this_bit = 7
     b = encoded[this_byte]
@@ -173,14 +170,14 @@ def decompress(encoding_table, length, encoded):
             searched_length += lengths[bit_length]
 
         v = values[searched_length + bits - max_code]
-        dec[i] = v
-    return dec
+        dec.append(v)
+    return ''.join(dec)
 
 def compress(encoding_table, decompressed):
-    if not isinstance(decompressed, bytes):
+    if not isinstance(decompressed, str):
         raise TypeError()
     values, lengths = encoding_table
-    enc = bytearray(len(decompressed) * 2)
+    enc = bytearray(len(decompressed) * 3)
     #print(decompressed)
     #print(lengths)
     current_bit = 7
@@ -228,7 +225,7 @@ def compress(encoding_table, decompressed):
     if current_bit != 7:
         current_byte += 1
     if current_byte > len(decompressed):
-        print("Note: compression increased length", repr(decompressed.decode('utf-8')), len(decompressed), current_byte, file=sys.stderr)
+        print("Note: compression increased length", repr(decompressed), len(decompressed), current_byte, file=sys.stderr)
     return enc[:current_byte]
 
 def qstr_escape(qst):
@@ -347,9 +344,9 @@ def print_qstr_data(encoding_table, qcfgs, qstrs, i18ns):
     total_text_compressed_size = 0
     for original, translation in i18ns:
         translation_encoded = translation.encode("utf-8")
-        compressed = compress(encoding_table, translation_encoded)
+        compressed = compress(encoding_table, translation)
         total_text_compressed_size += len(compressed)
-        decompressed = decompress(encoding_table, len(translation_encoded), compressed).decode("utf-8")
+        decompressed = decompress(encoding_table, len(translation_encoded), compressed)
         for c in C_ESCAPES:
             decompressed = decompressed.replace(c, C_ESCAPES[c])
         print("TRANSLATION(\"{}\", {}, {{ {} }}) // {}".format(original, len(translation_encoded)+1, ", ".join(["0x{:02x}".format(x) for x in compressed]), decompressed))
diff --git a/py/modio.c b/py/modio.c
index d7c1a58a8..17840a6d8 100644
--- a/py/modio.c
+++ b/py/modio.c
@@ -90,6 +90,7 @@ STATIC mp_uint_t iobase_ioctl(mp_obj_t obj, mp_uint_t request, uintptr_t arg, in
 }
 
 STATIC const mp_stream_p_t iobase_p = {
+    MP_PROTO_IMPLEMENT(MP_QSTR_protocol_stream)
     .read = iobase_read,
     .write = iobase_write,
     .ioctl = iobase_ioctl,
@@ -185,6 +186,7 @@ STATIC const mp_rom_map_elem_t bufwriter_locals_dict_table[] = {
 STATIC MP_DEFINE_CONST_DICT(bufwriter_locals_dict, bufwriter_locals_dict_table);
 
 STATIC const mp_stream_p_t bufwriter_stream_p = {
+    MP_PROTO_IMPLEMENT(MP_QSTR_protocol_stream)
     .write = bufwriter_write,
 };
 
diff --git a/py/objstringio.c b/py/objstringio.c
index d2ca6decd..178e6446c 100644
--- a/py/objstringio.c
+++ b/py/objstringio.c
@@ -240,6 +240,7 @@ STATIC const mp_rom_map_elem_t stringio_locals_dict_table[] = {
 STATIC MP_DEFINE_CONST_DICT(stringio_locals_dict, stringio_locals_dict_table);
 
 STATIC const mp_stream_p_t stringio_stream_p = {
+    MP_PROTO_IMPLEMENT(MP_QSTR_protocol_stream)
     .read = stringio_read,
     .write = stringio_write,
     .ioctl = stringio_ioctl,
@@ -247,6 +248,7 @@ STATIC const mp_stream_p_t stringio_stream_p = {
 };
 
 STATIC const mp_stream_p_t bytesio_stream_p = {
+    MP_PROTO_IMPLEMENT(MP_QSTR_protocol_stream)
     .read = stringio_read,
     .write = stringio_write,
     .ioctl = stringio_ioctl,
diff --git a/py/proto.c b/py/proto.c
new file mode 100644
index 000000000..e5053130b
--- /dev/null
+++ b/py/proto.c
@@ -0,0 +1,50 @@
+/*
+ * This file is part of the MicroPython project, http://micropython.org/
+ *
+ * The MIT License (MIT)
+ *
+ * Copyright (c) 2019 Jeff Epler for Adafruit Industries
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+#include "py/obj.h"
+#include "py/proto.h"
+#include "py/runtime.h"
+
+#ifndef MICROPY_UNSAFE_PROTO
+const void *mp_proto_get(uint16_t name, mp_const_obj_t obj) {
+    mp_obj_type_t *type = mp_obj_get_type(obj);
+    if (!type->protocol) return NULL;
+    uint16_t proto_name = *(const uint16_t*) type->protocol;
+    if (proto_name == name) {
+        return type->protocol;
+    }
+    return NULL;
+}
+#endif
+
+const void *mp_proto_get_or_throw(uint16_t name, mp_const_obj_t obj) {
+    const void *proto = mp_proto_get(name, obj);
+    if (proto) {
+        return proto;
+    }
+    mp_raise_TypeError_varg(translate("'%s' object does not support '%q'"),
+        mp_obj_get_type_str(obj), name);
+}
diff --git a/py/proto.h b/py/proto.h
new file mode 100644
index 000000000..2d4f80565
--- /dev/null
+++ b/py/proto.h
@@ -0,0 +1,43 @@
+/*
+ * This file is part of the MicroPython project, http://micropython.org/
+ *
+ * The MIT License (MIT)
+ *
+ * Copyright (c) 2019 Jeff Epler for Adafruit Industries
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+#ifndef MICROPY_INCLUDED_PY_PROTO_H
+#define MICROPY_INCLUDED_PY_PROTO_H
+
+#ifdef MICROPY_UNSAFE_PROTO
+#define MP_PROTOCOL_HEAD /* NOTHING */
+#define MP_PROTO_IMPLEMENT(name) /* NOTHING */
+static inline void *mp_proto_get(uint16_t name, mp_const_obj_type_t obj) { return mp_obj_get_type(obj)->protocol; }
+#else
+#define MP_PROTOCOL_HEAD \
+    uint16_t name; // The name of this protocol, a qstr
+#define MP_PROTO_IMPLEMENT(n) .name = n,
+const void *mp_proto_get(uint16_t name, mp_const_obj_t obj);
+const void *mp_proto_get_or_throw(uint16_t name, mp_const_obj_t obj);
+#endif
+
+#endif
+
diff --git a/py/py.mk b/py/py.mk
index 17cb79264..ae9b005b6 100644
--- a/py/py.mk
+++ b/py/py.mk
@@ -214,6 +214,7 @@ PY_CORE_O_BASENAME = $(addprefix py/,\
 	objtype.o \
 	objzip.o \
 	opmethods.o \
+	proto.o \
 	reload.o \
 	sequence.o \
 	stream.o \
diff --git a/py/stream.c b/py/stream.c
index 1d25f6470..2ff2b76a6 100644
--- a/py/stream.c
+++ b/py/stream.c
@@ -86,8 +86,7 @@ mp_uint_t mp_stream_rw(mp_obj_t stream, void *buf_, mp_uint_t size, int *errcode
 }
 
 const mp_stream_p_t *mp_get_stream_raise(mp_obj_t self_in, int flags) {
-    mp_obj_type_t *type = mp_obj_get_type(self_in);
-    const mp_stream_p_t *stream_p = type->protocol;
+    const mp_stream_p_t *stream_p = mp_proto_get(MP_QSTR_protocol_stream, self_in);
     if (stream_p == NULL
         || ((flags & MP_STREAM_OP_READ) && stream_p->read == NULL)
         || ((flags & MP_STREAM_OP_WRITE) && stream_p->write == NULL)
@@ -522,7 +521,7 @@ int mp_stream_errno;
 
 ssize_t mp_stream_posix_write(mp_obj_t stream, const void *buf, size_t len) {
     mp_obj_base_t* o = (mp_obj_base_t*)MP_OBJ_TO_PTR(stream);
-    const mp_stream_p_t *stream_p = o->type->protocol;
+    const mp_stream_p_t *stream_p = mp_get_stream(o);
     mp_uint_t out_sz = stream_p->write(stream, buf, len, &mp_stream_errno);
     if (out_sz == MP_STREAM_ERROR) {
         return -1;
@@ -533,7 +532,7 @@ ssize_t mp_stream_posix_write(mp_obj_t stream, const void *buf, size_t len) {
 
 ssize_t mp_stream_posix_read(mp_obj_t stream, void *buf, size_t len) {
     mp_obj_base_t* o = (mp_obj_base_t*)MP_OBJ_TO_PTR(stream);
-    const mp_stream_p_t *stream_p = o->type->protocol;
+    const mp_stream_p_t *stream_p = mp_get_stream(o);
     mp_uint_t out_sz = stream_p->read(stream, buf, len, &mp_stream_errno);
     if (out_sz == MP_STREAM_ERROR) {
         return -1;
@@ -544,7 +543,7 @@ ssize_t mp_stream_posix_read(mp_obj_t stream, void *buf, size_t len) {
 
 off_t mp_stream_posix_lseek(mp_obj_t stream, off_t offset, int whence) {
     const mp_obj_base_t* o = (mp_obj_base_t*)MP_OBJ_TO_PTR(stream);
-    const mp_stream_p_t *stream_p = o->type->protocol;
+    const mp_stream_p_t *stream_p = mp_get_stream(o);
     struct mp_stream_seek_t seek_s;
     seek_s.offset = offset;
     seek_s.whence = whence;
@@ -557,7 +556,7 @@ off_t mp_stream_posix_lseek(mp_obj_t stream, off_t offset, int whence) {
 
 int mp_stream_posix_fsync(mp_obj_t stream) {
     mp_obj_base_t* o = (mp_obj_base_t*)MP_OBJ_TO_PTR(stream);
-    const mp_stream_p_t *stream_p = o->type->protocol;
+    const mp_stream_p_t *stream_p = mp_get_stream(o);
     mp_uint_t res = stream_p->ioctl(stream, MP_STREAM_FLUSH, 0, &mp_stream_errno);
     if (res == MP_STREAM_ERROR) {
         return -1;
diff --git a/py/stream.h b/py/stream.h
index 03e24c746..543fe8c82 100644
--- a/py/stream.h
+++ b/py/stream.h
@@ -27,6 +27,7 @@
 #define MICROPY_INCLUDED_PY_STREAM_H
 
 #include "py/obj.h"
+#include "py/proto.h"
 #include "py/mperrno.h"
 
 #define MP_STREAM_ERROR ((mp_uint_t)-1)
@@ -64,6 +65,7 @@ struct mp_stream_seek_t {
 
 // Stream protocol
 typedef struct _mp_stream_p_t {
+    MP_PROTOCOL_HEAD
     // On error, functions should return MP_STREAM_ERROR and fill in *errcode (values
     // are implementation-dependent, but will be exposed to user, e.g. via exception).
     mp_uint_t (*read)(mp_obj_t obj, void *buf, mp_uint_t size, int *errcode);
@@ -93,7 +95,7 @@ MP_DECLARE_CONST_FUN_OBJ_VAR_BETWEEN(mp_stream_ioctl_obj);
 
 // Object is assumed to have a non-NULL stream protocol with valid r/w/ioctl methods
 static inline const mp_stream_p_t *mp_get_stream(mp_const_obj_t self) {
-    return (const mp_stream_p_t*)((const mp_obj_base_t*)MP_OBJ_TO_PTR(self))->type->protocol;
+    return mp_proto_get(MP_QSTR_protocol_stream, self); 
 }
 
 const mp_stream_p_t *mp_get_stream_raise(mp_obj_t self_in, int flags);