Merge pull request #1 from adafruit/main

Merged main to fork

42 files changed, 367 insertions, 192 deletions

diff --git a/tools/analyze_heap_dump.py b/tools/analyze_heap_dump.py
index b8f2bd011..887871db7 100755
--- a/tools/analyze_heap_dump.py
+++ b/tools/analyze_heap_dump.py
@@ -1,3 +1,7 @@
+# SPDX-FileCopyrightText: 2014 MicroPython & CircuitPython contributors (https://github.com/adafruit/circuitpython/graphs/contributors)
+#
+# SPDX-License-Identifier: MIT
+
 # This script renders a graph of the MicroPython heap at the given point it was dumped.
 # It takes three files, the binary dump of ram, the binary for CircuitPython and the linker map file.
 
diff --git a/tools/analyze_mpy.py b/tools/analyze_mpy.py
index 376207a24..a2f541d0f 100644
--- a/tools/analyze_mpy.py
+++ b/tools/analyze_mpy.py
@@ -1,3 +1,7 @@
+# SPDX-FileCopyrightText: 2014 MicroPython & CircuitPython contributors (https://github.com/adafruit/circuitpython/graphs/contributors)
+#
+# SPDX-License-Identifier: MIT
+
 import sys
 import binascii
 import io
diff --git a/tools/bootstrap_upip.sh b/tools/bootstrap_upip.sh
index 2891775d7..8b9d199fa 100755
--- a/tools/bootstrap_upip.sh
+++ b/tools/bootstrap_upip.sh
@@ -1,3 +1,7 @@
+# SPDX-FileCopyrightText: 2014 MicroPython & CircuitPython contributors (https://github.com/adafruit/circuitpython/graphs/contributors)
+#
+# SPDX-License-Identifier: MIT
+
 # This script performs bootstrap installation of upip package manager from PyPI
 # All the other packages can be installed using it.
 
diff --git a/tools/build-stm-latest.sh b/tools/build-stm-latest.sh
index 07cb168da..bfee1b192 100755
--- a/tools/build-stm-latest.sh
+++ b/tools/build-stm-latest.sh
@@ -1,5 +1,9 @@
 #!/bin/bash
 
+# SPDX-FileCopyrightText: 2014 MicroPython & CircuitPython contributors (https://github.com/adafruit/circuitpython/graphs/contributors)
+#
+# SPDX-License-Identifier: MIT
+
 # function for building firmware
 function do_build() {
     descr=$1
diff --git a/tools/build_board_info.py b/tools/build_board_info.py
index 6c670930f..6ed8b8c16 100644
--- a/tools/build_board_info.py
+++ b/tools/build_board_info.py
@@ -1,5 +1,9 @@
 #!/usr/bin/env python3
 
+# SPDX-FileCopyrightText: 2014 MicroPython & CircuitPython contributors (https://github.com/adafruit/circuitpython/graphs/contributors)
+#
+# SPDX-License-Identifier: MIT
+
 import json
 import os
 import subprocess
@@ -9,6 +13,9 @@ import base64
 from datetime import date
 from sh.contrib import git
 
+sys.path.append("../docs")
+import shared_bindings_matrix
+
 sys.path.append("adabot")
 import adabot.github_requests as github
 
@@ -150,7 +157,7 @@ def create_pr(changes, updated, git_info, user):
         info["id"] = id
         updated_list.append(info)
 
-    updated = json.dumps(updated_list, sort_keys=True, indent=4).encode("utf-8") + b"\n"
+    updated = json.dumps(updated_list, sort_keys=True, indent=1).encode("utf-8") + b"\n"
     #print(updated.decode("utf-8"))
     pr_title = "Automated website update for release {}".format(changes["new_release"])
     boards = ""
@@ -202,19 +209,6 @@ def create_pr(changes, updated, git_info, user):
     print(changes)
     print(pr_info)
 
-def update_downloads(boards, release):
-    response = github.get("/repos/adafruit/circuitpython/releases/tags/{}".format(release))
-    if not response.ok:
-        print(response.text)
-        raise RuntimeError("cannot get previous release info")
-
-    assets = response.json()["assets"]
-    for asset in assets:
-        board_name = asset["name"].split("-")[2]
-        if board_name not in boards:
-            continue
-        boards[board_name]["download_count"] += asset["download_count"]
-
 
 def print_active_user():
     response = github.get("/user")
@@ -246,6 +240,10 @@ def generate_download_info():
 
     languages = get_languages()
 
+    support_matrix = shared_bindings_matrix.support_matrix_by_board(
+        use_branded_name=False
+    )
+
     new_stable = "-" not in new_tag
 
     previous_releases = set()
@@ -262,9 +260,6 @@ def generate_download_info():
 
     board_mapping = get_board_mapping()
 
-    for release in previous_releases:
-        update_downloads(board_mapping, release)
-
     for port in SUPPORTED_PORTS:
         board_path = os.path.join("../ports", port, "boards")
         for board_path in os.scandir(board_path):
@@ -283,7 +278,10 @@ def generate_download_info():
                     new_version = {
                         "stable": new_stable,
                         "version": new_tag,
-                        "files": {}
+                        "modules": support_matrix.get(alias, "[]"),
+                        "files": {},
+                        "languages": languages,
+                        "extensions": board_info["extensions"]
                     }
                     for language in languages:
                         files = []
diff --git a/tools/build_memory_info.py b/tools/build_memory_info.py
index 808b70bd2..26697b974 100644
--- a/tools/build_memory_info.py
+++ b/tools/build_memory_info.py
@@ -1,28 +1,9 @@
 #!/usr/bin/env python3
+
+# SPDX-FileCopyrightText: Copyright (c) 2017 Scott Shawcroft for Adafruit Industries
+# SPDX-FileCopyrightText: 2014 MicroPython & CircuitPython contributors (https://github.com/adafruit/circuitpython/graphs/contributors)
 #
-# This file is part of the MicroPython project, http://micropython.org/
-#
-# The MIT License (MIT)
-#
-# Copyright (c) 2017 Scott Shawcroft for Adafruit Industries
-#
-# Permission is hereby granted, free of charge, to any person obtaining a copy
-# of this software and associated documentation files (the "Software"), to deal
-# in the Software without restriction, including without limitation the rights
-# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-# copies of the Software, and to permit persons to whom the Software is
-# furnished to do so, subject to the following conditions:
-#
-# The above copyright notice and this permission notice shall be included in
-# all copies or substantial portions of the Software.
-#
-# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-# THE SOFTWARE.
+# SPDX-License-Identifier: MIT
 
 import re
 import sys
diff --git a/tools/build_release_files.py b/tools/build_release_files.py
index 09133e51f..3563dd99f 100755
--- a/tools/build_release_files.py
+++ b/tools/build_release_files.py
@@ -1,5 +1,9 @@
 #! /usr/bin/env python3
 
+# SPDX-FileCopyrightText: 2014 MicroPython & CircuitPython contributors (https://github.com/adafruit/circuitpython/graphs/contributors)
+#
+# SPDX-License-Identifier: MIT
+
 import os
 import sys
 import subprocess
diff --git a/tools/chart_code_size.py b/tools/chart_code_size.py
index 0b55787fa..f75269130 100644
--- a/tools/chart_code_size.py
+++ b/tools/chart_code_size.py
@@ -1,3 +1,7 @@
+# SPDX-FileCopyrightText: 2014 MicroPython & CircuitPython contributors (https://github.com/adafruit/circuitpython/graphs/contributors)
+#
+# SPDX-License-Identifier: MIT
+
 # This script renders a graph of the CircuitPython rom image.
 # It takes the single elf file and uses objdump to get its contents.
 
diff --git a/tools/check_code_size.sh b/tools/check_code_size.sh
index 2925ff168..bdf1b5ad9 100755
--- a/tools/check_code_size.sh
+++ b/tools/check_code_size.sh
@@ -1,4 +1,9 @@
 #!/bin/bash
+
+# SPDX-FileCopyrightText: 2014 MicroPython & CircuitPython contributors (https://github.com/adafruit/circuitpython/graphs/contributors)
+#
+# SPDX-License-Identifier: MIT
+
 #
 # This script check that changes don't lead to code size regressions.
 # (Size of the language core (== minimal port should not grow)).
diff --git a/tools/check_translations.py b/tools/check_translations.py
index 7c008d3da..95776a544 100644
--- a/tools/check_translations.py
+++ b/tools/check_translations.py
@@ -1,3 +1,7 @@
+# SPDX-FileCopyrightText: 2014 MicroPython & CircuitPython contributors (https://github.com/adafruit/circuitpython/graphs/contributors)
+#
+# SPDX-License-Identifier: MIT
+
 # Validate that all entries in the .pot are in every .po. Only the .pot is updated so we can detect
 # if a translation was added to the source but isn't in a .po. This ensures translators can grab
 # complete files to work on.
diff --git a/tools/ci_new_boards_check.py b/tools/ci_new_boards_check.py
index 8bb8876fb..86010bad3 100644
--- a/tools/ci_new_boards_check.py
+++ b/tools/ci_new_boards_check.py
@@ -1,5 +1,9 @@
 #! /usr/bin/env python3
 
+# SPDX-FileCopyrightText: 2014 MicroPython & CircuitPython contributors (https://github.com/adafruit/circuitpython/graphs/contributors)
+#
+# SPDX-License-Identifier: MIT
+
 import sys
 import os
 import json
diff --git a/tools/codestats.sh b/tools/codestats.sh
index 5f7625c45..c0dd747dc 100755
--- a/tools/codestats.sh
+++ b/tools/codestats.sh
@@ -1,4 +1,9 @@
 #!/bin/sh
+
+# SPDX-FileCopyrightText: 2014 MicroPython & CircuitPython contributors (https://github.com/adafruit/circuitpython/graphs/contributors)
+#
+# SPDX-License-Identifier: MIT
+
 #
 # This script generates statistics (build size, speed) for successive
 # revisions of the code.  It checks out git commits one an a time, compiles
diff --git a/tools/convert_release_notes.py b/tools/convert_release_notes.py
index 649184102..797e71834 100644
--- a/tools/convert_release_notes.py
+++ b/tools/convert_release_notes.py
@@ -1,12 +1,17 @@
+# SPDX-FileCopyrightText: 2014 MicroPython & CircuitPython contributors (https://github.com/adafruit/circuitpython/graphs/contributors)
+#
+# SPDX-License-Identifier: MIT
+
 import sys
 import mistune
+import mistune.renderers
 
 print(sys.argv[1])
 
 with open(sys.argv[1], "r") as source_file:
     source = source_file.read()
 
-html = mistune.Markdown()
+html = mistune.create_markdown()
 
 print()
 print("HTML")
@@ -15,39 +20,39 @@ print("From the <a href=\"\">GitHub release page</a>:\n<blockquote>")
 print(html(source))
 print("</blockquote>")
 
-class AdafruitBBCodeRenderer:
-    def __init__(self, **kwargs):
-        self.options = kwargs
-
+class AdafruitBBCodeRenderer(mistune.renderers.BaseRenderer):
     def placeholder(self):
         return ''
 
     def paragraph(self, text):
         return text + "\n\n"
 
+    def block_text(self, text):
+        return text
+
     def text(self, text):
         return text
 
     def link(self, link, title, text):
-        return "[url={}]{}[/url]".format(link, text)
+        return "[url={}]{}[/url]".format(link, title)
 
     def autolink(self, link, is_email):
         if not is_email:
             return "[url={}]{}[/url]".format(link, link)
         return link
 
-    def header(self, text, level, raw):
+    def heading(self, text, level):
         return "[b][size=150]{}[/size][/b]\n".format(text)
 
     def codespan(self, text):
         return "[color=#E74C3C][size=95]{}[/size][/color]".format(text)
 
-    def list_item(self, text):
+    def list_item(self, text, level):
         return "[*]{}[/*]\n".format(text.strip())
 
-    def list(self, body, ordered=True):
+    def list(self, text, ordered, level, start=None):
         ordered_indicator = "=" if ordered else ""
-        return "[list{}]\n{}[/list]".format(ordered_indicator, body)
+        return "[list{}]\n{}[/list]".format(ordered_indicator, text)
 
     def double_emphasis(self, text):
         return "[b]{}[/b]".format(text)
@@ -55,7 +60,10 @@ class AdafruitBBCodeRenderer:
     def emphasis(self, text):
         return "[b]{}[/b]".format(text)
 
-bbcode = mistune.Markdown(renderer=AdafruitBBCodeRenderer())
+    def strong(self, text):
+        return "[i]{}[/i]".format(text)
+
+bbcode = mistune.create_markdown(renderer=AdafruitBBCodeRenderer())
 
 print()
 print("BBCode")
diff --git a/tools/cpboard.py b/tools/cpboard.py
index 7769cb4f4..464be4ba4 100644
--- a/tools/cpboard.py
+++ b/tools/cpboard.py
@@ -1,33 +1,12 @@
 #!/usr/bin/env python3
+
+# SPDX-FileCopyrightText: Copyright (c) 2014-2016 Damien P. George
+# SPDX-FileCopyrightText: Copyright (c) 2017 Paul Sokolovsky
+# SPDX-FileCopyrightText: Copyright (c) 2017 Scott Shawcroft for Adafruit Industries
+# SPDX-FileCopyrightText: Copyright (c) 2018 Noralf Trønnes
+# SPDX-FileCopyrightText: 2014 MicroPython & CircuitPython contributors (https://github.com/adafruit/circuitpython/graphs/contributors)
 #
-# This file is part of the MicroPython project, http://micropython.org/
-#
-# The MIT License (MIT)
-#
-# Copyright (c) 2017 Scott Shawcroft for Adafruit Industries
-# Copyright (c) 2018 Noralf Trønnes
-#
-# Parts taken from pyboard.py:
-# Copyright (c) 2014-2016 Damien P. George
-# Copyright (c) 2017 Paul Sokolovsky
-#
-# Permission is hereby granted, free of charge, to any person obtaining a copy
-# of this software and associated documentation files (the "Software"), to deal
-# in the Software without restriction, including without limitation the rights
-# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-# copies of the Software, and to permit persons to whom the Software is
-# furnished to do so, subject to the following conditions:
-#
-# The above copyright notice and this permission notice shall be included in
-# all copies or substantial portions of the Software.
-#
-# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-# THE SOFTWARE.
+# SPDX-License-Identifier: MIT
 
 import os
 import re
diff --git a/tools/dfu.py b/tools/dfu.py
index dd6019235..489465f0c 100644
--- a/tools/dfu.py
+++ b/tools/dfu.py
@@ -1,5 +1,9 @@
 #!/usr/bin/python
 
+# SPDX-FileCopyrightText: 2014 MicroPython & CircuitPython contributors (https://github.com/adafruit/circuitpython/graphs/contributors)
+#
+# SPDX-License-Identifier: MIT
+
 # Written by Antonio Galea - 2010/11/18
 # Updated for DFU 1.1 by Sean Cross - 2020/03/31
 # Distributed under Gnu LGPL 3.0
diff --git a/tools/extract_pyi.py b/tools/extract_pyi.py
index d749d202b..d29c3444f 100644
--- a/tools/extract_pyi.py
+++ b/tools/extract_pyi.py
@@ -1,21 +1,101 @@
+# SPDX-FileCopyrightText: 2014 MicroPython & CircuitPython contributors (https://github.com/adafruit/circuitpython/graphs/contributors)
+#
+# SPDX-License-Identifier: MIT
+
+# Run with 'python tools/extract_pyi.py shared-bindings/ path/to/stub/dir
+# You can also test a specific library in shared-bindings by putting the path
+# to that directory instead
+
+import ast
 import os
+import re
 import sys
-import astroid
 import traceback
 
-top_level = sys.argv[1].strip("/")
-stub_directory = sys.argv[2]
+import isort
+import black
+
+
+IMPORTS_IGNORE = frozenset({'int', 'float', 'bool', 'str', 'bytes', 'tuple', 'list', 'set', 'dict', 'bytearray', 'slice', 'file', 'buffer', 'range', 'array', 'struct_time'})
+IMPORTS_TYPING = frozenset({'Any', 'Optional', 'Union', 'Tuple', 'List', 'Sequence', 'Iterable', 'Iterator', 'overload'})
+IMPORTS_TYPESHED = frozenset({'ReadableBuffer', 'WriteableBuffer'})
+
+
+def is_any(node):
+    node_type = type(node)
+    if node is None:
+        return True
+    if node_type == ast.Name and node.id == "Any":
+        return True
+    if (node_type == ast.Attribute and type(node.value) == ast.Name
+            and node.value.id == "typing" and node.attr == "Any"):
+        return True
+    return False
+
+
+def report_missing_annotations(tree):
+    for node in ast.walk(tree):
+        node_type = type(node)
+        if node_type == ast.AnnAssign:
+            if is_any(node.annotation):
+                print(f"Missing attribute type on line {node.lineno}")
+        elif node_type == ast.arg:
+            if is_any(node.annotation) and node.arg != "self":
+                print(f"Missing argument type: {node.arg} on line {node.lineno}")
+        elif node_type == ast.FunctionDef:
+            if is_any(node.returns) and node.name != "__init__":
+                print(f"Missing return type: {node.name} on line {node.lineno}")
+
+
+def extract_imports(tree):
+    modules = set()
+    typing = set()
+    typeshed = set()
+
+    def collect_annotations(anno_tree):
+        if anno_tree is None:
+            return
+        for node in ast.walk(anno_tree):
+            node_type = type(node)
+            if node_type == ast.Name:
+                if node.id in IMPORTS_IGNORE:
+                    continue
+                elif node.id in IMPORTS_TYPING:
+                    typing.add(node.id)
+                elif node.id in IMPORTS_TYPESHED:
+                    typeshed.add(node.id)
+            if node_type == ast.Attribute:
+                if type(node.value) == ast.Name:
+                    modules.add(node.value.id)
+
+    for node in ast.walk(tree):
+        node_type = type(node)
+        if (node_type == ast.AnnAssign) or (node_type == ast.arg):
+            collect_annotations(node.annotation)
+        elif node_type == ast.FunctionDef:
+            collect_annotations(node.returns)
+            for deco in node.decorator_list:
+                if deco.id in IMPORTS_TYPING:
+                    typing.add(deco.id)
+
+    return {
+        "modules": sorted(modules),
+        "typing": sorted(typing),
+        "typeshed": sorted(typeshed),
+    }
+
 
 def convert_folder(top_level, stub_directory):
     ok = 0
     total = 0
     filenames = sorted(os.listdir(top_level))
     pyi_lines = []
+
     for filename in filenames:
         full_path = os.path.join(top_level, filename)
         file_lines = []
         if os.path.isdir(full_path):
-            mok, mtotal = convert_folder(full_path, os.path.join(stub_directory, filename))
+            (mok, mtotal) = convert_folder(full_path, os.path.join(stub_directory, filename))
             ok += mok
             total += mtotal
         elif filename.endswith(".c"):
@@ -40,44 +120,56 @@ def convert_folder(top_level, stub_directory):
             pyi_lines.extend(file_lines)
 
     if not pyi_lines:
-        return ok, total
+        return (ok, total)
 
     stub_filename = os.path.join(stub_directory, "__init__.pyi")
     print(stub_filename)
     stub_contents = "".join(pyi_lines)
-    os.makedirs(stub_directory, exist_ok=True)
-    with open(stub_filename, "w") as f:
-        f.write(stub_contents)
 
     # Validate that the module is a parseable stub.
     total += 1
     try:
-        tree = astroid.parse(stub_contents)
-        for i in tree.body:
-            if 'name' in i.__dict__:
-                print(i.__dict__['name'])
-                for j in i.body:
-                    if isinstance(j, astroid.scoped_nodes.FunctionDef):
-                        if None in j.args.__dict__['annotations']:
-                            print(f"Missing parameter type: {j.__dict__['name']} on line {j.__dict__['lineno']}\n")
-                        if j.returns:
-                            if 'Any' in j.returns.__dict__.values():
-                                print(f"Missing return type: {j.__dict__['name']} on line {j.__dict__['lineno']}")
-                    elif isinstance(j, astroid.node_classes.AnnAssign):
-                        if 'name' in j.__dict__['annotation'].__dict__:
-                            if j.__dict__['annotation'].__dict__['name'] == 'Any':
-                                print(f"missing attribute type on line {j.__dict__['lineno']}")
-
+        tree = ast.parse(stub_contents)
+        imports = extract_imports(tree)
+        report_missing_annotations(tree)
         ok += 1
-    except astroid.exceptions.AstroidSyntaxError as e:
-        e = e.__cause__
+    except SyntaxError as e:
         traceback.print_exception(type(e), e, e.__traceback__)
+        return (ok, total)
+
+    # Add import statements
+    import_lines = ["from __future__ import annotations"]
+    if imports["typing"]:
+        import_lines.append("from typing import " + ", ".join(imports["typing"]))
+    if imports["typeshed"]:
+        import_lines.append("from _typeshed import " + ", ".join(imports["typeshed"]))
+    import_lines.extend(f"import {m}" for m in imports["modules"])
+    import_body = "\n".join(import_lines)
+    m = re.match(r'(\s*""".*?""")', stub_contents, flags=re.DOTALL)
+    if m:
+        stub_contents = m.group(1) + "\n\n" + import_body + "\n\n" + stub_contents[m.end():]
+    else:
+        stub_contents = import_body + "\n\n" + stub_contents
+
+    # Code formatting
+    stub_contents = isort.code(stub_contents)
+    stub_contents = black.format_str(stub_contents, mode=black.FileMode(is_pyi=True))
+
+    os.makedirs(stub_directory, exist_ok=True)
+    with open(stub_filename, "w") as f:
+        f.write(stub_contents)
+
     print()
-    return ok, total
+    return (ok, total)
+
+
+if __name__ == "__main__":
+    top_level = sys.argv[1].strip("/")
+    stub_directory = sys.argv[2]
 
-ok, total = convert_folder(top_level, stub_directory)
+    (ok, total) = convert_folder(top_level, stub_directory)
 
-print(f"{ok} ok out of {total}")
+    print(f"Parsing .pyi files: {total - ok} failed, {ok} passed")
 
-if ok != total:
-    sys.exit(total - ok)
+    if ok != total:
+        sys.exit(total - ok)
diff --git a/tools/file2h.py b/tools/file2h.py
index 2a04ae22b..706dd4dd2 100644
--- a/tools/file2h.py
+++ b/tools/file2h.py
@@ -1,3 +1,7 @@
+# SPDX-FileCopyrightText: 2014 MicroPython & CircuitPython contributors (https://github.com/adafruit/circuitpython/graphs/contributors)
+#
+# SPDX-License-Identifier: MIT
+
 # Reads in a text file, and performs the necessary escapes so that it
 # can be #included as a static string like:
 #    static const char string_from_textfile[] =
diff --git a/tools/fixup_translations.py b/tools/fixup_translations.py
index 6db7f1cc5..0362923e8 100644
--- a/tools/fixup_translations.py
+++ b/tools/fixup_translations.py
@@ -1,3 +1,7 @@
+# SPDX-FileCopyrightText: 2014 MicroPython & CircuitPython contributors (https://github.com/adafruit/circuitpython/graphs/contributors)
+#
+# SPDX-License-Identifier: MIT
+
 # Validate that all entries in the .pot are in every .po. Only the .pot is updated so we can detect
 # if a translation was added to the source but isn't in a .po. This ensures translators can grab
 # complete files to work on.
diff --git a/tools/fonts/ter-u12n.bdf.license b/tools/fonts/ter-u12n.bdf.license
new file mode 100644
index 000000000..a40e8711d
--- /dev/null
+++ b/tools/fonts/ter-u12n.bdf.license
@@ -0,0 +1,3 @@
+SPDX-FileCopyrightText: 2018 Dimitar Toshkov Zhekov
+
+SPDX-License-Identifier: OFL-1.1
diff --git a/tools/gc_activity.py b/tools/gc_activity.py
index 8d172ea3a..cc9a21836 100644
--- a/tools/gc_activity.py
+++ b/tools/gc_activity.py
@@ -1,3 +1,7 @@
+# SPDX-FileCopyrightText: 2014 MicroPython & CircuitPython contributors (https://github.com/adafruit/circuitpython/graphs/contributors)
+#
+# SPDX-License-Identifier: MIT
+
 import sys
 import json
 
diff --git a/tools/gc_activity_between_collects.py b/tools/gc_activity_between_collects.py
index c02c479f9..f906cf5c7 100644
--- a/tools/gc_activity_between_collects.py
+++ b/tools/gc_activity_between_collects.py
@@ -1,3 +1,7 @@
+# SPDX-FileCopyrightText: 2014 MicroPython & CircuitPython contributors (https://github.com/adafruit/circuitpython/graphs/contributors)
+#
+# SPDX-License-Identifier: MIT
+
 import sys
 import json
 
diff --git a/tools/gen-changelog.sh b/tools/gen-changelog.sh
index 6eca1b4b1..b29606b0c 100755
--- a/tools/gen-changelog.sh
+++ b/tools/gen-changelog.sh
@@ -1,5 +1,9 @@
 #!/bin/sh
 
+# SPDX-FileCopyrightText: 2014 MicroPython & CircuitPython contributors (https://github.com/adafruit/circuitpython/graphs/contributors)
+#
+# SPDX-License-Identifier: MIT
+
 echo "MicroPython change log"
 
 for t in $(git tag | grep -v v1.0-rc1 | sort -rV); do
diff --git a/tools/gen_display_resources.py b/tools/gen_display_resources.py
index c08daf477..478a2f22f 100644
--- a/tools/gen_display_resources.py
+++ b/tools/gen_display_resources.py
@@ -1,3 +1,7 @@
+# SPDX-FileCopyrightText: 2014 MicroPython & CircuitPython contributors (https://github.com/adafruit/circuitpython/graphs/contributors)
+#
+# SPDX-License-Identifier: MIT
+
 import argparse
 
 import os
diff --git a/tools/gen_ld_files.py b/tools/gen_ld_files.py
index c2ac812bc..1fea1a7ef 100755
--- a/tools/gen_ld_files.py
+++ b/tools/gen_ld_files.py
@@ -1,4 +1,9 @@
 #! /usr/bin/env python3
+
+# SPDX-FileCopyrightText: 2014 MicroPython & CircuitPython contributors (https://github.com/adafruit/circuitpython/graphs/contributors)
+#
+# SPDX-License-Identifier: MIT
+
 import argparse
 
 import os
diff --git a/tools/gen_usb_descriptor.py b/tools/gen_usb_descriptor.py
index 5e25528f9..adec33100 100644
--- a/tools/gen_usb_descriptor.py
+++ b/tools/gen_usb_descriptor.py
@@ -1,3 +1,7 @@
+# SPDX-FileCopyrightText: 2014 MicroPython & CircuitPython contributors (https://github.com/adafruit/circuitpython/graphs/contributors)
+#
+# SPDX-License-Identifier: MIT
+
 import argparse
 
 import os
@@ -535,14 +539,14 @@ h_file.write("""\
 
 #include <stdint.h>
 
-const uint8_t usb_desc_dev[{device_length}];
+extern const uint8_t usb_desc_dev[{device_length}];
 // Make sure the control buffer is big enough to fit the descriptor.
 #define CFG_TUD_ENUM_BUFFER_SIZE {max_configuration_length}
-const uint8_t usb_desc_cfg[{configuration_length}];
-uint16_t usb_serial_number[{serial_number_length}];
-uint16_t const * const string_desc_arr [{string_descriptor_length}];
+extern const uint8_t usb_desc_cfg[{configuration_length}];
+extern uint16_t usb_serial_number[{serial_number_length}];
+extern uint16_t const * const string_desc_arr [{string_descriptor_length}];
 
-const uint8_t hid_report_descriptor[{hid_report_descriptor_length}];
+extern const uint8_t hid_report_descriptor[{hid_report_descriptor_length}];
 
 #define USB_HID_NUM_DEVICES {hid_num_devices}
 
diff --git a/tools/gendoc.py b/tools/gendoc.py
index 61844d203..85411cb41 100644
--- a/tools/gendoc.py
+++ b/tools/gendoc.py
@@ -1,3 +1,7 @@
+# SPDX-FileCopyrightText: 2014 MicroPython & CircuitPython contributors (https://github.com/adafruit/circuitpython/graphs/contributors)
+#
+# SPDX-License-Identifier: MIT
+
 """
 Generate documentation for pyboard API from C files.
 """
diff --git a/tools/git-checkout-latest-tag.sh b/tools/git-checkout-latest-tag.sh
index f6242a946..69fa2ced4 100755
--- a/tools/git-checkout-latest-tag.sh
+++ b/tools/git-checkout-latest-tag.sh
@@ -1,3 +1,8 @@
 #!/bin/bash
+
+# SPDX-FileCopyrightText: 2014 MicroPython & CircuitPython contributors (https://github.com/adafruit/circuitpython/graphs/contributors)
+#
+# SPDX-License-Identifier: MIT
+
 git fetch --tags
 git checkout $(git describe --tags `git rev-list --tags --max-count=1`)
diff --git a/tools/hid_report_descriptors.py b/tools/hid_report_descriptors.py
index 07ed26744..e13e0dbdd 100644
--- a/tools/hid_report_descriptors.py
+++ b/tools/hid_report_descriptors.py
@@ -1,24 +1,7 @@
-# The MIT License (MIT)
+# SPDX-FileCopyrightText: Copyright (c) 2018 Dan Halbert for Adafruit Industries
+# SPDX-FileCopyrightText: 2014 MicroPython & CircuitPython contributors (https://github.com/adafruit/circuitpython/graphs/contributors)
 #
-# Copyright (c) 2018 Dan Halbert for Adafruit Industries
-#
-# Permission is hereby granted, free of charge, to any person obtaining a copy
-# of this software and associated documentation files (the "Software"), to deal
-# in the Software without restriction, including without limitation the rights
-# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-# copies of the Software, and to permit persons to whom the Software is
-# furnished to do so, subject to the following conditions:
-#
-# The above copyright notice and this permission notice shall be included in
-# all copies or substantial portions of the Software.
-#
-# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-# THE SOFTWARE.
+# SPDX-License-Identifier: MIT
 
 import struct
 
diff --git a/tools/insert-usb-ids.py b/tools/insert-usb-ids.py
index cdccd3be9..bf74ceeab 100644
--- a/tools/insert-usb-ids.py
+++ b/tools/insert-usb-ids.py
@@ -1,3 +1,7 @@
+# SPDX-FileCopyrightText: 2014 MicroPython & CircuitPython contributors (https://github.com/adafruit/circuitpython/graphs/contributors)
+#
+# SPDX-License-Identifier: MIT
+
 # Reads the USB VID and PID from the file specified by sys.argv[1] and then
 # inserts those values into the template file specified by sys.argv[2],
 # printing the result to stdout
diff --git a/tools/join_bins.py b/tools/join_bins.py
index cb73aaabb..a370f86e4 100644
--- a/tools/join_bins.py
+++ b/tools/join_bins.py
@@ -1,3 +1,7 @@
+# SPDX-FileCopyrightText: 2014 MicroPython & CircuitPython contributors (https://github.com/adafruit/circuitpython/graphs/contributors)
+#
+# SPDX-License-Identifier: MIT
+
 import sys
 
 output_filename = sys.argv[1]
diff --git a/tools/make-frozen.py b/tools/make-frozen.py
index 1051b520e..ad23b9d59 100755
--- a/tools/make-frozen.py
+++ b/tools/make-frozen.py
@@ -1,4 +1,9 @@
 #!/usr/bin/env python
+
+# SPDX-FileCopyrightText: 2014 MicroPython & CircuitPython contributors (https://github.com/adafruit/circuitpython/graphs/contributors)
+#
+# SPDX-License-Identifier: MIT
+
 #
 # Create frozen modules structure for MicroPython.
 #
diff --git a/tools/mksdiodata.py b/tools/mksdiodata.py
new file mode 100755
index 000000000..0cce3819f
--- /dev/null
+++ b/tools/mksdiodata.py
@@ -0,0 +1,40 @@
+#!/usr/bin/python3
+
+def defines(name, function):
+    print(f'pin_function_t {name} [] = {{')
+    for instance in (0, 1):
+        for port in 'ABCD':
+            for idx in range(32):
+                pin = f'P{port}{idx:02d}'
+                pinmux = f'PINMUX_{pin}I_SDHC{instance}_{function}'
+                print(f'''\
+#if defined({pinmux}) && ! defined(IGNORE_PIN_{pin})
+    {{&pin_{pin}, {instance}, PIN_{pin}, {pinmux} & 0xffff}},
+#endif''')
+    print(f'{{NULL, 0, 0}}')
+    print(f'}};')
+    print()
+
+print('''\
+#include <stdint.h>
+#include "py/obj.h"
+#include "sam.h"
+#include "samd/pins.h"
+#include "mpconfigport.h"
+#include "atmel_start_pins.h"
+#include "hal/include/hal_gpio.h"
+
+typedef struct {
+    const mcu_pin_obj_t *obj;
+    uint8_t instance;
+    uint8_t pin;
+    uint16_t function;
+} pin_function_t;
+''')
+
+defines('sdio_ck', 'SDCK')
+defines('sdio_cmd', 'SDCMD')
+defines('sdio_dat0', 'SDDAT0')
+defines('sdio_dat1', 'SDDAT1')
+defines('sdio_dat2', 'SDDAT2')
+defines('sdio_dat3', 'SDDAT3')
diff --git a/tools/mpy-tool.py b/tools/mpy-tool.py
index 5ce24061b..95090466c 100755
--- a/tools/mpy-tool.py
+++ b/tools/mpy-tool.py
@@ -1,28 +1,9 @@
 #!/usr/bin/env python3
+
+# SPDX-FileCopyrightText: Copyright (c) 2016 Damien P. George
+# SPDX-FileCopyrightText: 2014 MicroPython & CircuitPython contributors (https://github.com/adafruit/circuitpython/graphs/contributors)
 #
-# This file is part of the MicroPython project, http://micropython.org/
-#
-# The MIT License (MIT)
-#
-# Copyright (c) 2016 Damien P. George
-#
-# Permission is hereby granted, free of charge, to any person obtaining a copy
-# of this software and associated documentation files (the "Software"), to deal
-# in the Software without restriction, including without limitation the rights
-# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-# copies of the Software, and to permit persons to whom the Software is
-# furnished to do so, subject to the following conditions:
-#
-# The above copyright notice and this permission notice shall be included in
-# all copies or substantial portions of the Software.
-#
-# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-# THE SOFTWARE.
+# SPDX-License-Identifier: MIT
 
 # Python 2/3 compatibility code
 from __future__ import print_function
diff --git a/tools/mpy_bin2res.py b/tools/mpy_bin2res.py
index 0c89e7e92..8d67b71ba 100755
--- a/tools/mpy_bin2res.py
+++ b/tools/mpy_bin2res.py
@@ -1,4 +1,9 @@
 #!/usr/bin/env python3
+
+# SPDX-FileCopyrightText: 2014 MicroPython & CircuitPython contributors (https://github.com/adafruit/circuitpython/graphs/contributors)
+#
+# SPDX-License-Identifier: MIT
+
 #
 # This tool converts binary resource files passed on the command line
 # into a Python source file containing data from these files, which can
diff --git a/tools/mpy_cross_all.py b/tools/mpy_cross_all.py
index 2bda71e9b..5d9f8bc0b 100755
--- a/tools/mpy_cross_all.py
+++ b/tools/mpy_cross_all.py
@@ -1,4 +1,9 @@
 #!/usr/bin/env python3
+
+# SPDX-FileCopyrightText: 2014 MicroPython & CircuitPython contributors (https://github.com/adafruit/circuitpython/graphs/contributors)
+#
+# SPDX-License-Identifier: MIT
+
 import argparse
 import os
 import os.path
diff --git a/tools/preprocess_frozen_modules.py b/tools/preprocess_frozen_modules.py
index 7ae20c4d6..b75a2e723 100755
--- a/tools/preprocess_frozen_modules.py
+++ b/tools/preprocess_frozen_modules.py
@@ -1,4 +1,9 @@
 #!/usr/bin/env python3
+
+# SPDX-FileCopyrightText: 2014 MicroPython & CircuitPython contributors (https://github.com/adafruit/circuitpython/graphs/contributors)
+#
+# SPDX-License-Identifier: MIT
+
 import argparse
 import os
 import os.path
diff --git a/tools/print_status.py b/tools/print_status.py
index ed563fd68..9a8b311dc 100755
--- a/tools/print_status.py
+++ b/tools/print_status.py
@@ -1,5 +1,9 @@
 #!/usr/bin/env python3
 
+# SPDX-FileCopyrightText: 2014 MicroPython & CircuitPython contributors (https://github.com/adafruit/circuitpython/graphs/contributors)
+#
+# SPDX-License-Identifier: MIT
+
 import sys
 if len(sys.argv) != 2:
     print("""\
diff --git a/tools/pyboard.py b/tools/pyboard.py
index bb5642bd7..ddf6bb6c4 100755
--- a/tools/pyboard.py
+++ b/tools/pyboard.py
@@ -1,29 +1,10 @@
 #!/usr/bin/env python
+
+# SPDX-FileCopyrightText: Copyright (c) 2014-2016 Damien P. George
+# SPDX-FileCopyrightText: Copyright (c) 2017 Paul Sokolovsky
+# SPDX-FileCopyrightText: 2014 MicroPython & CircuitPython contributors (https://github.com/adafruit/circuitpython/graphs/contributors)
 #
-# This file is part of the MicroPython project, http://micropython.org/
-#
-# The MIT License (MIT)
-#
-# Copyright (c) 2014-2016 Damien P. George
-# Copyright (c) 2017 Paul Sokolovsky
-#
-# Permission is hereby granted, free of charge, to any person obtaining a copy
-# of this software and associated documentation files (the "Software"), to deal
-# in the Software without restriction, including without limitation the rights
-# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-# copies of the Software, and to permit persons to whom the Software is
-# furnished to do so, subject to the following conditions:
-#
-# The above copyright notice and this permission notice shall be included in
-# all copies or substantial portions of the Software.
-#
-# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-# THE SOFTWARE.
+# SPDX-License-Identifier: MIT
 
 """
 pyboard interface
diff --git a/tools/pydfu.py b/tools/pydfu.py
index a7adda37c..a6210c603 100755
--- a/tools/pydfu.py
+++ b/tools/pydfu.py
@@ -1,8 +1,9 @@
 #!/usr/bin/env python
-# This file is part of the OpenMV project.
-# Copyright (c) 2013/2014 Ibrahim Abdelkader <i.abdalkader@gmail.com>
-# This work is licensed under the MIT license, see the file LICENSE for
-# details.
+
+# SPDX-FileCopyrightText: Copyright (c) 2013/2014 Ibrahim Abdelkader <i.abdalkader@gmail.com>
+# SPDX-FileCopyrightText: 2014 MicroPython & CircuitPython contributors (https://github.com/adafruit/circuitpython/graphs/contributors)
+#
+# SPDX-License-Identifier: MIT
 
 """This module implements enough functionality to program the STM32F4xx over
 DFU, without requiring dfu-util.
diff --git a/tools/tinytest-codegen.py b/tools/tinytest-codegen.py
index 7a48f8a9a..74b1878f9 100755
--- a/tools/tinytest-codegen.py
+++ b/tools/tinytest-codegen.py
@@ -1,5 +1,9 @@
 #!/usr/bin/env python3
 
+# SPDX-FileCopyrightText: 2014 MicroPython & CircuitPython contributors (https://github.com/adafruit/circuitpython/graphs/contributors)
+#
+# SPDX-License-Identifier: MIT
+
 import os, sys
 from glob import glob
 from re import sub
diff --git a/tools/upip.py b/tools/upip.py
index a400c3174..27fbae272 100644
--- a/tools/upip.py
+++ b/tools/upip.py
@@ -1,10 +1,10 @@
-#
 # upip - Package manager for MicroPython
+
+# SPDX-FileCopyrightText: Copyright (c) 2015-2018 Paul Sokolovsky
+# SPDX-FileCopyrightText: 2014 MicroPython & CircuitPython contributors (https://github.com/adafruit/circuitpython/graphs/contributors)
 #
-# Copyright (c) 2015-2018 Paul Sokolovsky
-#
-# Licensed under the MIT license.
-#
+# SPDX-License-Identifier: MIT
+
 import sys
 import gc
 import uos as os
diff --git a/tools/upip_utarfile.py b/tools/upip_utarfile.py
index 460ca2cd4..44d636452 100644
--- a/tools/upip_utarfile.py
+++ b/tools/upip_utarfile.py
@@ -1,3 +1,7 @@
+# SPDX-FileCopyrightText: 2014 MicroPython & CircuitPython contributors (https://github.com/adafruit/circuitpython/graphs/contributors)
+#
+# SPDX-License-Identifier: MIT
+
 import uctypes
 
 # http://www.gnu.org/software/tar/manual/html_node/Standard.html