Merge pull request #1985 from sommersoft/dynamic_support_matrix

Docs: Dynamically Build Support Matrix Table

8 files changed, 339 insertions, 78 deletions

diff --git a/.travis.yml b/.travis.yml
index dcf27bff2..8ca66d70d 100755
--- a/.travis.yml
+++ b/.travis.yml
@@ -79,10 +79,10 @@ before_script:
 
   # For coverage testing (upgrade is used to get latest urllib3 version)
   - sudo apt-get install -y python3-pip
-  - pip3 install --user sh click
+  - pip3 install --user sh click setuptools
   - ([[ -z "$TRAVIS_TESTS" ]] || sudo pip install --upgrade cpp-coveralls)
   - (! var_search "${TRAVIS_TESTS-}" docs || sudo apt-get install -y librsvg2-bin)
-  - (! var_search "${TRAVIS_TESTS-}" docs || pip install --user Sphinx sphinx-rtd-theme recommonmark sphinxcontrib-svg2pdfconverter)
+  - (! var_search "${TRAVIS_TESTS-}" docs || pip3 install --user Sphinx sphinx-rtd-theme recommonmark sphinxcontrib-svg2pdfconverter)
   - (! var_search "${TRAVIS_TESTS-}" translations || pip3 install --user polib)
 
   # Check if there's any board missing in TRAVIS_BOARDS
diff --git a/conf.py b/conf.py
index 59e19a513..56625e758 100644
--- a/conf.py
+++ b/conf.py
@@ -13,6 +13,7 @@
 # All configuration values have a default; values that are commented out
 # serve to show the default.
 
+import json
 import sys
 import os
 
@@ -24,8 +25,20 @@ from recommonmark.parser import CommonMarkParser
 sys.path.insert(0, os.path.abspath('docs'))
 sys.path.insert(0, os.path.abspath('.'))
 
+import shared_bindings_matrix
+
 master_doc = 'docs/index'
 
+# Grab the JSON values to use while building the module support matrix
+# in 'shared-bindings/index.rst'
+
+#modules_support_matrix = shared_bindings_matrix.support_matrix_excluded_boards()
+modules_support_matrix = shared_bindings_matrix.support_matrix_by_board()
+
+html_context = {
+    'support_matrix': modules_support_matrix
+}
+
 # -- General configuration ------------------------------------------------
 
 # If your documentation needs a minimal Sphinx version, state it here.
@@ -40,7 +53,9 @@ extensions = [
     'sphinxcontrib.rsvgconverter',
     'sphinx.ext.intersphinx',
     'sphinx.ext.todo',
-    'sphinx.ext.coverage'
+    'sphinx.ext.coverage',
+    'rstjinja',
+    'c2rst'
 ]
 
 # Add any paths that contain templates here, relative to this directory.
@@ -49,8 +64,7 @@ templates_path = ['templates']
 # The suffix of source filenames.
 source_suffix = ['.rst', '.md', '.c', '.h']
 
-source_parsers = {'.md': CommonMarkParser,
-                  '.c': "c2rst.CStrip", '.h': "c2rst.CStrip"}
+source_parsers = {'.md': CommonMarkParser}
 
 # The encoding of source files.
 #source_encoding = 'utf-8-sig'
diff --git a/docs/c2rst.py b/docs/c2rst.py
index 904fb7408..76489dca3 100644
--- a/docs/c2rst.py
+++ b/docs/c2rst.py
@@ -1,19 +1,31 @@
-import sphinx.parsers
+def c2rst(app, docname, source):
+    """ Pre-parse '.c' & '.h' files that contain rST source.
+    """
+    # Make sure we're outputting HTML
+    if app.builder.format != 'html':
+        return
 
-class CStrip(sphinx.parsers.Parser):
-    def __init__(self):
-        self.rst_parser = sphinx.parsers.RSTParser()
+    fname = app.env.doc2path(docname)
+    if (not fname.endswith(".c") and
+        not fname.endswith(".h")):
+            #print("skipping:", fname)
+            return
 
-    def parse(self, inputstring, document):
-        # This setting is missing starting with Sphinx 1.7.1 so we set it ourself.
-        document.settings.tab_width = 4
-        document.settings.character_level_inline_markup = False
-        stripped = []
-        for line in inputstring.split("\n"):
-            line = line.strip()
-            if line == "//|":
-                stripped.append("")
-            elif line.startswith("//| "):
-                stripped.append(line[len("//| "):])
-        stripped = "\r\n".join(stripped)
-        self.rst_parser.parse(stripped, document)
+    src = source[0]
+
+    stripped = []
+    for line in src.split("\n"):
+        line = line.strip()
+        if line == "//|":
+            stripped.append("")
+        elif line.startswith("//| "):
+            stripped.append(line[len("//| "):])
+    stripped = "\r\n".join(stripped)
+
+    rendered = app.builder.templates.render_string(
+        stripped, app.config.html_context
+    )
+    source[0] = rendered
+
+def setup(app):
+    app.connect("source-read", c2rst)
diff --git a/docs/library/network.rst b/docs/library/network.rst
index d25f9f388..cdcc5f323 100644
--- a/docs/library/network.rst
+++ b/docs/library/network.rst
@@ -5,6 +5,7 @@
 .. include:: ../templates/unsupported_in_circuitpython.inc
 
 .. module:: network
+   :noindex:
    :synopsis: network configuration
 
 This module provides network drivers and routing configuration. To use this
diff --git a/docs/rstjinja.py b/docs/rstjinja.py
new file mode 100644
index 000000000..3a08b2599
--- /dev/null
+++ b/docs/rstjinja.py
@@ -0,0 +1,24 @@
+# Derived from code on Eric Holscher's blog, found at:
+# https://www.ericholscher.com/blog/2016/jul/25/integrating-jinja-rst-sphinx/
+
+def rstjinja(app, docname, source):
+    """
+    Render our pages as a jinja template for fancy templating goodness.
+    """
+    # Make sure we're outputting HTML
+    if app.builder.format != 'html':
+        return
+
+    # we only want our one jinja template to run through this func
+    if "shared-bindings/support_matrix" not in docname:
+        return
+
+    src = source[0]
+    print(docname)
+    rendered = app.builder.templates.render_string(
+        src, app.config.html_context
+    )
+    source[0] = rendered
+
+def setup(app):
+    app.connect("source-read", rstjinja)
diff --git a/docs/shared_bindings_matrix.py b/docs/shared_bindings_matrix.py
new file mode 100644
index 000000000..ed88efc75
--- /dev/null
+++ b/docs/shared_bindings_matrix.py
@@ -0,0 +1,244 @@
+# The MIT License (MIT)
+#
+# Copyright (c) 2019 Michael Schroeder
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+#
+
+import json
+import os
+import re
+
+
+SUPPORTED_PORTS = ["atmel-samd", "nrf"]
+
+
+def parse_port_config(contents, chip_keyword=None):
+    """ Compile a dictionary of port-wide module configs, which may
+        be categorized by chipset.
+    """
+    chip_fam = "all"
+    ifeq_found = False
+    port_config_results = {"all": []}
+
+    chip_pattern = ""
+    if chip_keyword:
+        chip_pattern = (
+            re.compile("(?<=ifeq\s\(\$\({}\)\,)(\w+)".format(chip_keyword))
+        )
+
+    for line in contents:
+        if chip_keyword:
+            if not ifeq_found:
+                check_ifeq = chip_pattern.search(line)
+                if check_ifeq:
+                    ifeq_found = True
+                    chip_fam = check_ifeq.group(1)
+                    #print("found chip:", chip_fam)
+                else:
+                    ifeq_found = False
+                    chip_fam = "all"
+            else:
+                if "endif" in line:
+                    ifeq_found = False
+                    chip_fam = "all"
+
+        if "CIRCUITPY_" in line:
+            if chip_fam in port_config_results:
+                port_config_results[chip_fam].append(line.rstrip("\n"))
+            else:
+                port_config_results[chip_fam] = [line.rstrip("\n")]
+
+    #print(port_config_results)
+    return port_config_results
+
+def get_shared_bindings():
+    """ Get a list of modules in shared-bindings based on folder names
+    """
+    return [item for item in os.listdir("./shared-bindings")]
+
+
+def read_mpconfig():
+    """ Open 'circuitpy_mpconfig.mk' and return the contents.
+    """
+    configs = []
+    with open("py/circuitpy_mpconfig.mk") as mpconfig:
+        configs = mpconfig.read()
+
+    return configs
+
+
+def build_module_map():
+    """ Establish the base of the JSON file, based on the contents from
+        `configs`. Base will contain module names, if they're part of
+        the `FULL_BUILD`, or their default value (0 | 1).
+
+    """
+    base = dict()
+    modules = get_shared_bindings()
+    configs = read_mpconfig()
+    full_build = False
+    for module in modules:
+        full_name = module
+        search_name = module.lstrip("_")
+        re_pattern = "CIRCUITPY_{}\s=\s(.+)".format(search_name.upper())
+        find_config = re.search(re_pattern, configs)
+        #print(module, "|", find_config)
+        if not find_config:
+            continue
+        full_build = int("FULL_BUILD" in find_config.group(0))
+        #print(find_config[1])
+        if not full_build:
+            default_val = find_config.group(1)
+        else:
+            default_val = "None"
+        base[search_name] = {
+            "name": full_name,
+            "full_build": str(full_build),
+            "default_value": default_val,
+            "excluded": {}
+        }
+
+    return base
+
+
+def get_excluded_boards(base):
+    """ Cycles through each board's `mpconfigboard.mk` file to determine
+        if each module is included or not. Boards are selected by existence
+        in a port listed in `SUPPORTED_PORTS` (e.g. `/port/nrf/feather_52840`)
+
+        Boards are further categorized by their respective chipset (SAMD21,
+        SAMD51, nRF52840, etc.)
+    """
+    modules = list(base.keys())
+
+    re_board_chip = None
+    chip_keyword = None
+    for port in SUPPORTED_PORTS:
+        # each port appears to use its own define for the chipset
+        if port in ["atmel-samd"]:
+            re_board_chip = re.compile("CHIP_FAMILY\s=\s(\w+)")
+            chip_keyword = "CHIP_FAMILY"
+        elif port in ["nrf"]:
+            re_board_chip = re.compile("MCU_VARIANT\s=\s(\w+)")
+
+        port_dir = "ports/{}".format(port)
+
+        port_config_contents = ""
+        with open(os.path.join(port_dir, "mpconfigport.mk")) as port_config:
+            port_config_contents = port_config.readlines()
+        port_config = parse_port_config(port_config_contents, chip_keyword)
+
+        for entry in os.scandir(os.path.join(port_dir, "boards")):
+            if not entry.is_dir():
+                continue
+
+            contents = ""
+            board_dir = os.path.join(entry.path, "mpconfigboard.mk")
+            with open(board_dir) as board:
+                contents = board.read()
+
+            board_chip = re_board_chip.search(contents)
+            #print(entry.name, board_chip.group(1))
+            if not board_chip:
+                board_chip = "Unknown Chip"
+            else:
+                board_chip = board_chip.group(1)
+
+            # add port_config results to contents
+            contents += "\n" + "\n".join(port_config["all"])
+            if board_chip in port_config:
+                contents += "\n" + "\n".join(port_config[board_chip])
+
+            for module in modules:
+                board_is_excluded = False
+                # check if board uses `SMALL_BUILD`. if yes, and current
+                # module is marked as `FULL_BUILD`, board is excluded
+                small_build = re.search("CIRCUITPY_SMALL_BUILD = 1", contents)
+                if small_build and base[module]["full_build"] == "1":
+                    board_is_excluded = True
+                # check if module is specifically disabled for this board
+                re_pattern = "CIRCUITPY_{}\s=\s(\w)".format(module.upper())
+                find_module = re.search(re_pattern, contents)
+                if not find_module:
+                    # check if default inclusion is off ('0'). if the board doesn't
+                    # have it explicitly enabled, its excluded.
+                    if base[module]["default_value"] == "0":
+                        board_is_excluded = True
+                else:
+                    if (find_module.group(1) == "0" and
+                        find_module.group(1) != base[module]["default_value"]):
+                            board_is_excluded = True
+
+                if board_is_excluded:
+                    if board_chip in base[module]["excluded"]:
+                        base[module]["excluded"][board_chip].append(entry.name)
+                    else:
+                        base[module]["excluded"][board_chip] = [entry.name]
+    #print(json.dumps(base, indent=2))
+    return base
+
+
+def support_matrix_excluded_boards():
+    """ Compiles a list of available modules, and which board definitions
+        do not include them.
+    """
+    base = build_module_map()
+
+    return get_excluded_boards(base)
+
+def support_matrix_by_board():
+    """ Compiles a list of the available core modules available for each
+        board.
+    """
+    base = build_module_map()
+    base_with_exclusions = get_excluded_boards(base)
+
+    boards = dict()
+    for port in SUPPORTED_PORTS:
+        port_dir = "ports/{}/boards".format(port)
+        for entry in os.scandir(port_dir):
+            if not entry.is_dir():
+                continue
+            board_modules = []
+
+            board_name = entry.name
+            board_contents = ""
+            with open(os.path.join(entry.path, "mpconfigboard.h")) as get_name:
+                board_contents = get_name.read()
+            board_name_re = re.search("(?<=MICROPY_HW_BOARD_NAME)\s+(.+)",
+                                      board_contents)
+            if board_name_re:
+                board_name = board_name_re.group(1).strip('"')
+
+            for module in base_with_exclusions.keys():
+                #print(module)
+                board_has_module = True
+                if base_with_exclusions[module]["excluded"]:
+                    for port in base_with_exclusions[module]["excluded"].values():
+                        #print(port)
+                        if entry.name in port:
+                            board_has_module = False
+
+                if board_has_module:
+                    board_modules.append(base_with_exclusions[module]["name"])
+            boards[board_name] = sorted(board_modules)
+
+    #print(json.dumps(boards, indent=2))
+    return boards
diff --git a/shared-bindings/index.rst b/shared-bindings/index.rst
index b7690d292..cbffdb614 100644
--- a/shared-bindings/index.rst
+++ b/shared-bindings/index.rst
@@ -1,11 +1,11 @@
 Core Modules
 ========================================
 
-These core modules are intended on being consistent across ports. Currently
-they are only implemented in the SAMD21 and ESP8266 ports. A module may not exist
-in a port if no underlying hardware support is present or if flash space is
-limited. For example, a microcontroller without analog features will not have
-`analogio`.
+These core modules are intended on being consistent across ports and boards.
+A module may not exist on a port/board if no underlying hardware support is
+present or if flash space is limited. For example, a microcontroller without
+analog features will not have `analogio`. See the `support_matrix` page for
+a list of modules supported on each board.
 
 Modules
 ---------
@@ -14,56 +14,6 @@ Modules
     :glob:
     :maxdepth: 3
 
+    support_matrix
     */__init__
     help
-
-.. _module-support-matrix:
-
-Support Matrix
----------------
-NOTE 1: **All Supported** means the following ports are supported: SAMD21, SAMD21 Express,
-SAMD51, SAMD51 Express, and ESP8266.
-
-NOTE 2: **SAMD** and/or **SAMD Express** without additional numbers, means both SAMD21 & SAMD51 versions
-are supported.
-
-NOTE 3: The `pIRkey SAMD21 board <https://www.adafruit.com/product/3364>`_ is specialized and may not
-have modules as listed below.
-
-=================  ==============================
-Module             Supported Ports
-=================  ==============================
-`analogio`         **All Supported**
-`audiobusio`       **SAMD/SAMD Express**
-`audioio`          **SAMD Express**
-`audiocore`        **All with audioio**
-`binascii`         **ESP8266**
-`bitbangio`        **SAMD Express, ESP8266**
-`board`            **All Supported**
-`bleio`            **nRF**
-`busio`            **All Supported**
-`digitalio`        **All Supported**
-`frequencyio`      **SAMD51**
-`gamepad`          **SAMD Express, nRF**
-`hashlib`          **ESP8266**
-`i2cslave`         **SAMD Express**
-`math`             **All Supported**
-`microcontroller`  **All Supported**
-`multiterminal`    **ESP8266**
-`neopixel_write`   **All Supported**
-`nvm`              **SAMD Express**
-`os`               **All Supported**
-`pulseio`          **SAMD/SAMD Express**
-`ps2io`            **SAMD/SAMD Express**
-`random`           **All Supported**
-`rotaryio`         **SAMD51, SAMD Express**
-`storage`          **All Supported**
-`struct`           **All Supported**
-`supervisor`       **SAMD/SAMD Express**
-`time`             **All Supported**
-`touchio`          **SAMD/SAMD Express**
-`uheap`            **Debug (All)**
-`usb_hid`          **SAMD/SAMD Express**
-`_pixelbuf`        **SAMD Express**
-`_stage`           **SAMD/SAMD Express**
-=================  ==============================
diff --git a/shared-bindings/support_matrix.rst b/shared-bindings/support_matrix.rst
new file mode 100644
index 000000000..124f3a810
--- /dev/null
+++ b/shared-bindings/support_matrix.rst
@@ -0,0 +1,16 @@
+.. _module-support-matrix:
+
+Support Matrix
+===============
+
+The following table lists the available built-in modules for each CircuitPython
+capable board.
+
+.. csv-table::
+   :header-rows: 1
+   :widths: 7, 50
+
+   "Board", "Modules Available"
+   {% for key, value in support_matrix|dictsort -%}
+   "{{ key }}", "{{ '`' ~ value|join("`, `") ~ '`' }}"
+   {% endfor -%}