Add support for adding release info into adafruit/circuitpython-org

This also changes the build script to python with better output.

4 files changed, 313 insertions, 116 deletions

diff --git a/tools/adabot b/tools/adabot
new file mode 160000
+Subproject 393c2756714b5cccf028a82c23b873d36b2d9e8
diff --git a/tools/build_adafruit_bins.py b/tools/build_adafruit_bins.py
new file mode 100755
index 000000000..fd0282b55
--- /dev/null
+++ b/tools/build_adafruit_bins.py
@@ -0,0 +1,70 @@
+import os
+from sh import rm
+from sh import make
+import sys
+import subprocess
+import shutil
+import build_board_info as build_info
+import time
+
+for port in build_info.SUPPORTED_PORTS:
+    rm("-rf", "../ports/{}/build*".format(port))
+
+ROSIE_SETUPS = ["rosie-ci"]
+rosie_ok = {}
+for rosie in ROSIE_SETUPS:
+    rosie_ok[rosie] = True
+
+PARALLEL = "-j 5"
+travis = False
+if "TRAVIS" in os.environ and os.environ["TRAVIS"] == "true":
+    PARALLEL="-j 2"
+    travis = True
+
+all_boards = build_info.get_board_mapping()
+build_boards = list(all_boards.keys())
+if "TRAVIS_BOARDS" in os.environ:
+    build_boards = os.environ["TRAVIS_BOARDS"].split()
+
+sha, version = build_info.get_version_info()
+
+languages = build_info.get_languages()
+exit_status = 0
+for board in build_boards:
+    bin_directory = "../bin/{}/".format(board)
+    os.makedirs(bin_directory, exist_ok=True)
+    board_info = all_boards[board]
+
+    for language in languages:
+        start_time = time.monotonic()
+        make_result = subprocess.run("make -C ../ports/" + board_info["port"] + " TRANSLATION=" + language + " BOARD=" + board, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+        build_duration = time.monotonic() - start_time
+        success = "\033[32msucceeded\033[0m"
+        if make_result.returncode != 0:
+            exit_status = make_result.returncode
+            success = "\033[31mfailed\033[0m"
+
+        for extension in board_info["extensions"]:
+            temp_filename = "../ports/{port}/build-{board}/firmware.{extension}".format(port=board_info["port"], board=board, extension=extension)
+            final_filename = "adafruit-circuitpython-{board}-{language}-{version}.{extension}".format(board=board, language=language, version=version, extension=extension)
+            final_filename = os.path.join(bin_directory, final_filename)
+            shutil.copyfile(temp_filename, final_filename)
+
+        if travis:
+            print('travis_fold:start:adafruit-bins-{}-{}\\r'.format(language, board))
+        print("Build {} for {} took {:.2f}s and {}".format(board, language, build_duration, success))
+        print(len(make_result.stdout))
+        print(make_result.stdout.decode("utf-8"))
+        # Only upload to Rosie if its a pull request.
+        if travis:
+            for rosie in ROSIE_SETUPS:
+                if not rosie_ok[rosie]:
+                    break
+                print("Uploading to https://{rosie}.ngrok.io/upload/{sha}".format(rosie=rosie, sha=sha))
+                #curl -F "file=@$final_filename" https://$rosie.ngrok.io/upload/$sha
+        if travis:
+            print('travis_fold:end:adafruit-bins-{}-{}\\r'.format(language, board))
+
+        print()
+
+sys.exit(exit_status)
diff --git a/tools/build_adafruit_bins.sh b/tools/build_adafruit_bins.sh
deleted file mode 100755
index 4c672c6de..000000000
--- a/tools/build_adafruit_bins.sh
+++ /dev/null
@@ -1,116 +0,0 @@
-#!/usr/bin/env bash
-
-rm -rf ports/atmel-samd/build*
-rm -rf ports/esp8266/build*
-rm -rf ports/nrf/build*
-
-# Alphabetical.
-HW_BOARDS="\
-arduino_mkr1300 \
-arduino_zero \
-catwan_usbstick \
-circuitplayground_express \
-circuitplayground_express_crickit \
-feather_huzzah \
-feather_m0_adalogger \
-feather_m0_basic \
-feather_m0_express \
-feather_m0_express_crickit \
-feather_m0_rfm69 \
-feather_m0_rfm9x \
-feather_m4_express \
-feather_nrf52832 \
-feather_nrf52840_express \
-feather_radiofruit_zigbee \
-gemma_m0 \
-grandcentral_m4_express \
-hallowing_m0_express \
-itsybitsy_m0_express \
-itsybitsy_m4_express \
-makerdiary_nrf52840_mdk \
-metro_m0_express \
-metro_m4_express \
-meowmeow \
-pca10056 \
-pca10059 \
-pirkey_m0 \
-trellis_m4_express \
-trinket_m0 \
-"
-ROSIE_SETUPS="rosie-ci"
-
-PARALLEL="-j 5"
-if [[ "$TRAVIS" == "true" ]]; then
-    PARALLEL="-j 2"
-fi
-
-if [[ -z "$TRAVIS_BOARD" ]]; then
-    boards=$HW_BOARDS
-else
-    boards=$TRAVIS_BOARD
-fi
-
-version=`git describe --tags --exact-match`
-if [[ $? -ne 0 ]]; then
-    version=`date +%Y%m%d`-`git rev-parse --short HEAD`
-fi
-
-
-if [[ "$TRAVIS" == "true" ]]; then
-    sha=$TRAVIS_COMMIT
-    if [[ "$TRAVIS_PULL_REQUEST" != "false" ]]; then
-       version=`date +%Y%m%d`-`echo $TRAVIS_PULL_REQUEST_SHA | cut -c1-7`
-       sha=$TRAVIS_PULL_REQUEST_SHA
-    fi
-fi
-
-for board in $boards; do
-    mkdir -p bin/$board/
-    extension=uf2
-    for language_file in $(ls locale/*.po); do
-        language=$(basename -s .po $language_file)
-        echo "Building $board for $language"
-        # There is a bug in the Huzzah Makefile that causes it to fail occasionally with -j > 1.
-        if [[ $board == "feather_huzzah" ]]; then
-            make -C ports/esp8266 TRANSLATION=$language BOARD=$board
-            (( exit_status = exit_status || $? ))
-            temp_filename=ports/esp8266/build/firmware-combined.bin
-            extension=bin
-        elif [[ $board == "feather_nrf52832" ]]; then
-            make $PARALLEL -C ports/nrf TRANSLATION=$language BOARD=$board
-            (( exit_status = exit_status || $? ))
-            temp_filename=ports/nrf/build-$board-s132/firmware.bin
-            extension=bin
-        elif [[ $board == "feather_nrf52840_express" || $board == "pca10056" || $board == "pca10059" ]]; then
-            make $PARALLEL -C ports/nrf TRANSLATION=$language BOARD=$board SD=s140
-            (( exit_status = exit_status || $? ))
-            temp_filename=ports/nrf/build-$board-s140/firmware.uf2
-            extension=uf2
-        elif [[ $board == "makerdiary_nrf52840_mdk" ]]; then
-            make $PARALLEL -C ports/nrf TRANSLATION=$language BOARD=$board SD=s140
-            (( exit_status = exit_status || $? ))
-            temp_filename=ports/nrf/build-$board-s140/firmware.hex
-            extension=hex
-        else
-            time make $PARALLEL -C ports/atmel-samd TRANSLATION=$language BOARD=$board
-            (( exit_status = exit_status || $? ))
-            cp ports/atmel-samd/build-$board/firmware.bin bin/$board/adafruit-circuitpython-$board-$language-$version.bin
-            (( exit_status = exit_status || $? ))
-            temp_filename=ports/atmel-samd/build-$board/firmware.uf2
-            extension=uf2
-        fi
-        final_filename=bin/$board/adafruit-circuitpython-$board-$language-$version.$extension
-        cp $temp_filename $final_filename
-        (( exit_status = exit_status || $? ))
-        # Only upload to Rosie if its a pull request.
-        if [[ "$TRAVIS" == "true" ]]; then
-            for rosie in $ROSIE_SETUPS; do
-                echo "Uploading to https://$rosie.ngrok.io/upload/$sha"
-                curl -F "file=@$final_filename" https://$rosie.ngrok.io/upload/$sha
-            done
-        fi
-        echo
-    done
-done
-
-exit $exit_status
diff --git a/tools/build_board_info.py b/tools/build_board_info.py
new file mode 100644
index 000000000..f6b2d3b09
--- /dev/null
+++ b/tools/build_board_info.py
@@ -0,0 +1,243 @@
+#!/usr/bin/env python3
+
+import json
+import os
+import subprocess
+import sys
+import sh
+import base64
+from datetime import date
+from sh.contrib import git
+
+sys.path.append("adabot")
+import adabot.github_requests as github
+
+SUPPORTED_PORTS = ["nrf", "esp8266", "atmel-samd"]
+
+BIN = ('bin',)
+UF2 = ('uf2',)
+BIN_UF2 = ('bin', 'uf2')
+HEX = ('hex',)
+
+# Default extensions
+extension_by_port = {
+    "nrf": BIN,
+    "esp8266": BIN,
+    "atmel-samd": UF2,
+}
+
+# Per board overrides
+extension_by_board = {
+    "feather_nrf52832": BIN,
+    "arduino_zero": BIN,
+    "feather_m0_adalogger": BIN_UF2,
+    "feather_m0_basic": BIN_UF2,
+    "makerdiary_nrf52840_mdk": HEX
+}
+
+def get_languages():
+    languages = []
+    for f in os.scandir("../locale"):
+        if f.name.endswith(".po"):
+            languages.append(f.name[:-3])
+    return languages
+
+def get_board_mapping():
+    boards = {}
+    for port in SUPPORTED_PORTS:
+        board_path = os.path.join("../ports", port, "boards")
+        for board_path in os.scandir(board_path):
+            if board_path.is_dir():
+                board_files = os.listdir(board_path.path)
+                board_id = board_path.name
+                extensions = extension_by_port[port]
+                extensions = extension_by_board.get(board_path.name, extensions)
+                boards[board_id] = {"port": port, "extensions": extensions, "download_count": 0}
+    return boards
+
+def get_version_info():
+    version = None
+    sha = git("rev-parse", "--short", "HEAD").stdout.decode("utf-8")
+    try:
+        version = git("describe", "--tags", "--exact-match").stdout.decode("utf-8")
+    except sh.ErrorReturnCode_128:
+        # No exact match
+        pass
+
+    if "TRAVIS" in os.environ and os.environ["TRAVIS"] == "true":
+        sha = os.environ["TRAVIS_COMMIT"]
+        if os.environ["TRAVIS_PULL_REQUEST"] != "false":
+            sha = os.environ["TRAVIS_PULL_REQUEST_SHA"]
+
+    if not version:
+        version="{}-{}".format(date.today().strftime("%Y%m%d"), sha[:7])
+
+    return sha, version
+
+def get_current_info():
+    response = github.get("/repos/adafruit/circuitpython-org/git/refs/heads/master")
+    if not response.ok:
+        print(response.text)
+        raise RuntimeError("cannot get master sha")
+    commit_sha = response.json()["object"]["sha"]
+
+    response = github.get("/repos/adafruit/circuitpython-org/contents/_data/files.json?ref=" + commit_sha)
+    if not response.ok:
+        print(response.text)
+        raise RuntimeError("cannot get previous files.json")
+
+    response = response.json()
+
+    git_info = commit_sha, response["sha"]
+    return git_info, json.loads(base64.b64decode(response["content"]))
+
+def create_pr(changes, updated, git_info):
+    commit_sha, original_blob_sha = git_info
+    branch_name = "new_release_" + changes["new_release"]
+
+    updated = json.dumps(updated, sort_keys=True, indent=4).encode("utf-8") + b"\n"
+    print(updated.decode("utf-8"))
+    pr_title = "Automated website update for release {}".format(changes["new_release"])
+    boards = ""
+    if changes["new_boards"]:
+        boards = "New boards:\n* " + "\n* ".join(changes["new_boards"])
+    languages = ""
+    if changes["new_languages"]:
+        languages = "New languages:\n* " + "\n* ".join(changes["new_languages"])
+    message = "Automated website update for release {} by AdaBot.\n\n{}\n\n{}\n".format(
+        changes["new_release"],
+        boards,
+        languages
+    )
+
+    create_branch = {
+        "ref": "refs/heads/" + branch_name,
+        "sha": commit_sha
+    }
+    response = github.post("/repos/adafruit-adabot/circuitpython-org/git/refs", json=create_branch)
+    if not response.ok and response.json()["message"] != "Reference already exists":
+        print("unable to create branch")
+        print(response.text)
+        return
+
+    update_file = {
+        "message": message,
+        "content": base64.b64encode(updated).decode("utf-8"),
+        "sha": original_blob_sha,
+        "branch": branch_name
+    }
+
+    response = github.put("/repos/adafruit-adabot/circuitpython-org/contents/_data/files.json", json=update_file)
+    if not response.ok:
+        print("unable to post new file")
+        print(response.text)
+        return
+    pr_info = {
+        "title": pr_title,
+        "head": "adafruit-adabot:" + branch_name,
+        "base": "master",
+        "body": message,
+        "maintainer_can_modify": True
+    }
+    response = github.post("/repos/adafruit/circuitpython-org/pulls", json=pr_info)
+    if not response.ok:
+        print("unable to create pr")
+        print(response.text)
+        return
+    print(changes)
+    print(pr_info)
+
+def update_downloads(boards, release):
+    response = github.get("/repos/adafruit/circuitpython/releases/tags/{}".format(release))
+    if not response.ok:
+        print(response.text)
+        raise RuntimeError("cannot get previous release info")
+
+    assets = response.json()["assets"]
+    for asset in assets:
+        boards[asset["name"].split("-")[2]]["download_count"] += asset["download_count"]
+
+
+def print_active_user():
+    response = github.get("/user")
+    if response.ok:
+        print("Logged in as {}".format(response.json()["login"]))
+    else:
+        print("Not logged in")
+
+def generate_download_info():
+    boards = {}
+    errors = []
+
+    new_tag = os.environ.get("TRAVIS_TAG", "4.0.0-alpha.3")
+
+    changes = {
+        "new_release": new_tag,
+        "new_boards": [],
+        "new_languages": []
+    }
+
+    print_active_user()
+
+    sha, this_version = get_version_info()
+
+    git_info, current_info = get_current_info()
+
+    languages = get_languages()
+
+    new_stable = "-" not in new_tag
+
+    previous_releases = set()
+    previous_languages = set()
+
+    # Delete the release we are replacing
+    for board in current_info:
+        info = current_info[board]
+        for version in info["versions"]:
+            previous_releases.add(version["version"])
+            previous_languages.update(version["files"].keys())
+            if version["stable"] == new_stable:
+                info["versions"].remove(version)
+
+    board_mapping = get_board_mapping()
+
+    print(previous_releases)
+    for release in previous_releases:
+        update_downloads(board_mapping, release)
+
+    for port in SUPPORTED_PORTS:
+        board_path = os.path.join("../ports", port, "boards")
+        for board_path in os.scandir(board_path):
+            if board_path.is_dir():
+                board_files = os.listdir(board_path)
+                board_id = board_path.name
+                board_info = board_mapping[board_id]
+
+                if board_id not in current_info:
+                    changes["new_boards"].append(board_id)
+                    current_info[board_id] = {"downloads": 0,
+                                              "versions": []}
+
+                new_version = {
+                    "stable": new_stable,
+                    "version": new_tag,
+                    "files": {}
+                }
+                for language in languages:
+                    files = []
+                    new_version["files"][language] = files
+                    for extension in board_info["extensions"]:
+                        files.append("https://github.com/adafruit/circuitpython/releases/download/{tag}/adafruit-circuitpython-{board}-{language}-{tag}.{extension}".format(tag=new_tag, board=board_id, language=language, extension=extension))
+                current_info[board_id]["downloads"] = board_info["download_count"]
+                current_info[board_id]["versions"].append(new_version)
+
+    changes["new_languages"] = set(languages) - previous_languages
+
+    if changes["new_languages"]:
+        create_pr(changes, current_info, git_info)
+
+if __name__ == "__main__":
+    if "TRAVIS_TAG" in os.environ:
+        generate_download_info()
+    else:
+        print("skipping website update because this isn't a tag")