Add support for adding release info into adafruit/circuitpython-org

This also changes the build script to python with better output.

12 files changed, 353 insertions, 136 deletions

diff --git a/.gitmodules b/.gitmodules
index 1c1f8eb65..362cb55e5 100644
--- a/.gitmodules
+++ b/.gitmodules
@@ -83,3 +83,6 @@
 [submodule "tools/huffman"]
 	path = tools/huffman
 	url = https://github.com/tannewt/huffman.git
+[submodule "tools/adabot"]
+	path = tools/adabot
+	url = https://github.com/adafruit/adabot.git
diff --git a/.travis.yml b/.travis.yml
index 396e194b7..99515ccec 100755
--- a/.travis.yml
+++ b/.travis.yml
@@ -21,11 +21,11 @@ git:
 # that SDK is shortest and add it there.  In the case of major re-organizations,
 # just try to make the builds "about equal in run time"
 env:
-  - TRAVIS_TESTS="unix docs translations" TRAVIS_BOARDS="feather_huzzah circuitplayground_express pca10056 pca10059 feather_nrf52832 feather_nrf52840_express makerdiary_nrf52840_mdk" TRAVIS_SDK=arm:nrf:esp8266
+  - TRAVIS_TESTS="unix docs translations website" TRAVIS_BOARDS="feather_huzzah circuitplayground_express pca10056 pca10059 feather_nrf52832 feather_nrf52840_express makerdiary_nrf52840_mdk" TRAVIS_SDK=arm:nrf:esp8266
   - TRAVIS_BOARDS="metro_m0_express metro_m4_express pirkey_m0 trellis_m4_express trinket_m0" TRAVIS_SDK=arm
   - TRAVIS_BOARDS="feather_radiofruit_zigbee gemma_m0 hallowing_m0_express itsybitsy_m0_express itsybitsy_m4_express meowmeow" TRAVIS_SDK=arm
   - TRAVIS_BOARDS="feather_m0_express_crickit feather_m0_rfm69 feather_m0_rfm9x feather_m4_express arduino_zero arduino_mkr1300" TRAVIS_SDK=arm
-  - TRAVIS_BOARDS="circuitplayground_express_crickit feather_m0_adalogger feather_m0_basic feather_m0_express catwan_usbstick" TRAVIS_SDK=arm 
+  - TRAVIS_BOARDS="circuitplayground_express_crickit feather_m0_adalogger feather_m0_basic feather_m0_express catwan_usbstick" TRAVIS_SDK=arm
 
 addons:
   artifacts:
@@ -33,6 +33,17 @@ addons:
     - $(ls -d1 bin/*/* | tr "\n" ":")
     target_paths: /
 
+deploy:
+  provider: releases
+  api_key:
+    secure: "jdqVFw6itRY4qwQF4ReXo0uaymT+Mob6RhYX0lw8KWFNqBgHnLVuKmKKcGMEuRvBVMPkvxF7bMuOQzSBOunqwlHFse3oMzdWvQODv1zwV7pSRXGwTdIvTPbBjKWxnBG9uSNRf2R5AMflJFUxy2CbtBpgvNzr+4VLREZDrrjEu8C1iTtXGpSm5AQ5iIp2fkMAWD85FP7CQPpkqRoxhSIFZmTdurfYRmenq1OZ/4SeD5TESKcyvqJNzVT9z210B3cg3eAkP6ukvelW4qE2zgIANqUkGqvDEnAvEII9M89kuwhCMAekdfwnUSPrry+g77i1dUZHoRN1+MFj+waYtPaqxdYo2G1sysa6enxlu4jHMR5MfMk9eKHgaNgL3PiyANusYSS44amh8QIiVaX5nw82myZDCpQOZW7YqJKE6WX70Lbs4mS+wIs+ig4KIXO1B0p9kMb0OeVjHRl+KcXsWGRu/ECG/ExpqlVIssSPU407LohMXT2cJ37CY/R/EeK2XSDsQ2M3L3EAGUjCJdBGuwsOJ+2lG+HQpAVu9vAB4kq5jy9Ye+MG+8Xlkly3XZZ5+FkXyYxKnXb26/QVv0e5sIG5OmdJCPYFaH2J1QdKo7CdhEcBtrf6DMPWaimGMldShFqzLjOz3b3qLysRxFF0aGb7ipKPa57vawNzYHoPAViOcXQ="
+  file:
+  - $(ls -d1 bin/*/* | tr "\n" ":")
+  skip_cleanup: true
+  draft: true
+  on:
+    tags: true
+
 notifications:
   webhooks:
     urls:
@@ -46,12 +57,13 @@ notifications:
 before_script:
   # Expand the git tree back to 4.0.0-alpha.1 and then fetch the latest tag.
   - LAST_TAG=`git ls-remote --quiet --tags --sort=version:refname | egrep -o "refs/tags/[0-9]+.*\$" | tail -n 1`
-  - git fetch --shallow-exclude=4.0.0-alpha.1
+  - git fetch --shallow-exclude=4.0.0-alpha.1 || git fetch --unshallow
   - git fetch --depth 1 origin $LAST_TAG:$LAST_TAG
   - git describe --dirty --always --tags
   - function var_search () { case "$1" in *$2*) true;; *) false;; esac; }
   - sudo dpkg --add-architecture i386
 
+
   - (! var_search "${TRAVIS_SDK-}" arm || (wget https://s3.amazonaws.com/adafruit-circuit-python/gcc-arm-embedded_7-2018q2-1~xenial1_amd64.deb && sudo dpkg -i gcc-arm-embedded*_amd64.deb))
 
   # For nrf builds
@@ -62,10 +74,11 @@ before_script:
   - if var_search "${TRAVIS_SDK-}" esp8266 ; then PATH=$(readlink -f ../xtensa-lx106-elf/bin):$PATH; fi
 
   # For coverage testing (upgrade is used to get latest urllib3 version)
-  - ([[ -z "$TRAVIS_TESTS" ]] || sudo apt-get install -y python3-pip)
+  - sudo apt-get install -y python3-pip
+  - pip3 install --user sh click
   - ([[ -z "$TRAVIS_TESTS" ]] || sudo pip install --upgrade cpp-coveralls)
-  - (! var_search "${TRAVIS_TESTS-}" docs || sudo pip install 'Sphinx<1.8.0' sphinx-rtd-theme recommonmark)
-  - (! var_search "${TRAVIS_TESTS-}" translations || sudo pip3 install polib)
+  - (! var_search "${TRAVIS_TESTS-}" docs || pip install --user 'Sphinx<1.8.0' sphinx-rtd-theme recommonmark)
+  - (! var_search "${TRAVIS_TESTS-}" translations || pip3 install --user polib)
 
   # report some good version numbers to the build
   - gcc --version
@@ -79,9 +92,8 @@ script:
   - make -C mpy-cross -j2
   - echo -en 'travis_fold:end:mpy-cross\\r'
 
-  - echo 'Building Adafruit binaries' && echo -en 'travis_fold:start:adafruit-bins\\r'
-  - (for board in $TRAVIS_BOARDS; do TRAVIS_BOARD=$board tools/build_adafruit_bins.sh || exit $?; done)
-  - echo -en 'travis_fold:end:adafruit-bins\\r'
+  - cd tools && python3 build_adafruit_bins.py
+  - cd ..
 
   - echo 'Building unix' && echo -en 'travis_fold:start:unix\\r'
   - (! var_search "${TRAVIS_TESTS-}" unix || (make -C ports/unix deplibs -j2 && make -C ports/unix -j2 && make -C ports/unix coverage -j2))
@@ -119,5 +131,7 @@ script:
   # run coveralls coverage analysis (try to, even if some builds/tests failed)
   #- (cd ports/unix && coveralls --root ../.. --build-root . --gcov $(which gcov) --gcov-options '\-o build-coverage/' --include py --include extmod)
 
+  - (! var_search "${TRAVIS_TESTS-}" website || (cd tools && python3 build_board_info.py && cd ..))
+
 after_failure:
   - (cd tests && for exp in *.exp; do testbase=$(basename $exp .exp); echo -e "\nFAILURE $testbase"; diff -u $testbase.exp $testbase.out; done)
diff --git a/ports/esp8266/.gitignore b/ports/esp8266/.gitignore
new file mode 100644
index 000000000..414487d53
--- /dev/null
+++ b/ports/esp8266/.gitignore
@@ -0,0 +1 @@
+build-*/
diff --git a/ports/esp8266/Makefile b/ports/esp8266/Makefile
index 03d5a65cb..69226f442 100644
--- a/ports/esp8266/Makefile
+++ b/ports/esp8266/Makefile
@@ -6,7 +6,7 @@ $(error Invalid BOARD specified)
 endif
 
 # If the build directory is not given, make it reflect the board name.
-BUILD ?= build
+BUILD ?= build-$(BOARD)
 
 include ../../py/mkenv.mk
 
@@ -25,7 +25,7 @@ FROZEN_MPY_DIR ?= modules
 # include py core make definitions
 include $(TOP)/py/py.mk
 
-FWBIN = $(BUILD)/firmware-combined.bin
+FWBIN = $(BUILD)/firmware.bin
 PORT ?= /dev/ttyACM0
 BAUD ?= 115200
 FLASH_MODE ?= qio
@@ -232,7 +232,7 @@ FROZEN_EXTRA_DEPS = $(CONFVARS_FILE)
 
 .PHONY: deploy
 
-deploy: $(BUILD)/firmware-combined.bin
+deploy: $(FWBIN)
 	$(ECHO) "Writing $< to the board"
 	$(Q)$(ESPTOOL) --port $(PORT) --baud $(BAUD) write_flash --verify --flash_size=$(FLASH_SIZE) --flash_mode=$(FLASH_MODE) 0 $<
 
@@ -275,4 +275,4 @@ $(BUILD)/libaxtls.a:
 
 clean-modules:
 	git clean -f -d modules
-	rm -f build/frozen*.c
+	rm -f $(BUILD)/frozen*.c
diff --git a/ports/esp8266/esp8266_common.ld b/ports/esp8266/esp8266_common.ld
index c76b83bc4..c355105ff 100644
--- a/ports/esp8266/esp8266_common.ld
+++ b/ports/esp8266/esp8266_common.ld
@@ -142,7 +142,7 @@ SECTIONS
         *lib/utils/*.o*(.literal*, .text*)
         *drivers/bus/*.o(.literal* .text*)
 
-        build/main.o(.literal* .text*)
+        build*/main.o(.literal* .text*)
         *gccollect.o(.literal* .text*)
         *gchelper.o(.literal* .text*)
         *help.o(.literal* .text*)
@@ -178,7 +178,7 @@ SECTIONS
         */frozen.o(.rodata.mp_frozen_content) /* frozen modules */
 
         /* for -mforce-l32 */
-        build/*.o(.rodata*)
+        build*/*.o(.rodata*)
 
         _irom0_text_end = ABSOLUTE(.);
     } >irom0_0_seg :irom0_0_phdr
diff --git a/ports/nrf/Makefile b/ports/nrf/Makefile
index 558731a44..b8d186621 100755
--- a/ports/nrf/Makefile
+++ b/ports/nrf/Makefile
@@ -10,12 +10,14 @@ else
   endif
 endif
 
+CLI_SD := $(SD)
+
 include boards/$(BOARD)/mpconfigboard.mk
 
 SD_LOWER = $(shell echo $(SD) | tr '[:upper:]' '[:lower:]')
 
-# Build directory with SD
-BUILD ?= $(if $(SD),build-$(BOARD)-$(SD_LOWER),build-$(BOARD))
+# Build directory with SD if it's different from the default.
+BUILD ?= $(if $(CLI_SD),build-$(BOARD)-$(SD_LOWER),build-$(BOARD))
 
 include ../../py/mkenv.mk
 -include mpconfigport.mk
diff --git a/ports/unix/Makefile b/ports/unix/Makefile
index 1d36198ec..3b52507c9 100644
--- a/ports/unix/Makefile
+++ b/ports/unix/Makefile
@@ -74,7 +74,7 @@ else
 # Use gcc syntax for map file
 LDFLAGS_ARCH = -Wl,-Map=$@.map,--cref -Wl,--gc-sections
 endif
-LDFLAGS = $(LDFLAGS_MOD) $(LDFLAGS_ARCH) -lm $(LDFLAGS_EXTRA)
+LDFLAGS = -Lbuild $(LDFLAGS_MOD) $(LDFLAGS_ARCH) -lm $(LDFLAGS_EXTRA)
 
 ifeq ($(MICROPY_FORCE_32BIT),1)
 # Note: you may need to install i386 versions of dependency packages,
@@ -245,7 +245,7 @@ coverage:
 	$(MAKE) \
 	    COPT="-O0" CFLAGS_EXTRA='$(CFLAGS_EXTRA) -DMP_CONFIGFILE="<mpconfigport_coverage.h>" \
 	    -fprofile-arcs -ftest-coverage \
-	    -Wdouble-promotion -Wformat -Wmissing-declarations -Wmissing-prototypes -Wsign-compare \
+	    -Wformat -Wmissing-declarations -Wmissing-prototypes -Wsign-compare \
 	    -Wold-style-definition -Wpointer-arith -Wshadow -Wuninitialized -Wunused-parameter \
 	    -DMICROPY_UNIX_COVERAGE' \
 	    LDFLAGS_EXTRA='-fprofile-arcs -ftest-coverage' \
diff --git a/py/py.mk b/py/py.mk
index c640e6cff..69ad4e9b9 100644
--- a/py/py.mk
+++ b/py/py.mk
@@ -28,7 +28,7 @@ ifeq ($(MICROPY_PY_USSL),1)
 CFLAGS_MOD += -DMICROPY_PY_USSL=1
 ifeq ($(MICROPY_SSL_AXTLS),1)
 CFLAGS_MOD += -DMICROPY_SSL_AXTLS=1 -I$(TOP)/lib/axtls/ssl -I$(TOP)/lib/axtls/crypto -I$(TOP)/lib/axtls/config
-LDFLAGS_MOD += -Lbuild -laxtls
+LDFLAGS_MOD += -L$(BUILD) -laxtls
 else ifeq ($(MICROPY_SSL_MBEDTLS),1)
 # Can be overridden by ports which have "builtin" mbedTLS
 MICROPY_SSL_MBEDTLS_INCLUDE ?= $(TOP)/lib/mbedtls/include
diff --git a/tools/adabot b/tools/adabot
new file mode 160000
+Subproject 393c2756714b5cccf028a82c23b873d36b2d9e8
diff --git a/tools/build_adafruit_bins.py b/tools/build_adafruit_bins.py
new file mode 100755
index 000000000..fd0282b55
--- /dev/null
+++ b/tools/build_adafruit_bins.py
@@ -0,0 +1,70 @@
+import os
+from sh import rm
+from sh import make
+import sys
+import subprocess
+import shutil
+import build_board_info as build_info
+import time
+
+for port in build_info.SUPPORTED_PORTS:
+    rm("-rf", "../ports/{}/build*".format(port))
+
+ROSIE_SETUPS = ["rosie-ci"]
+rosie_ok = {}
+for rosie in ROSIE_SETUPS:
+    rosie_ok[rosie] = True
+
+PARALLEL = "-j 5"
+travis = False
+if "TRAVIS" in os.environ and os.environ["TRAVIS"] == "true":
+    PARALLEL="-j 2"
+    travis = True
+
+all_boards = build_info.get_board_mapping()
+build_boards = list(all_boards.keys())
+if "TRAVIS_BOARDS" in os.environ:
+    build_boards = os.environ["TRAVIS_BOARDS"].split()
+
+sha, version = build_info.get_version_info()
+
+languages = build_info.get_languages()
+exit_status = 0
+for board in build_boards:
+    bin_directory = "../bin/{}/".format(board)
+    os.makedirs(bin_directory, exist_ok=True)
+    board_info = all_boards[board]
+
+    for language in languages:
+        start_time = time.monotonic()
+        make_result = subprocess.run("make -C ../ports/" + board_info["port"] + " TRANSLATION=" + language + " BOARD=" + board, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+        build_duration = time.monotonic() - start_time
+        success = "\033[32msucceeded\033[0m"
+        if make_result.returncode != 0:
+            exit_status = make_result.returncode
+            success = "\033[31mfailed\033[0m"
+
+        for extension in board_info["extensions"]:
+            temp_filename = "../ports/{port}/build-{board}/firmware.{extension}".format(port=board_info["port"], board=board, extension=extension)
+            final_filename = "adafruit-circuitpython-{board}-{language}-{version}.{extension}".format(board=board, language=language, version=version, extension=extension)
+            final_filename = os.path.join(bin_directory, final_filename)
+            shutil.copyfile(temp_filename, final_filename)
+
+        if travis:
+            print('travis_fold:start:adafruit-bins-{}-{}\\r'.format(language, board))
+        print("Build {} for {} took {:.2f}s and {}".format(board, language, build_duration, success))
+        print(len(make_result.stdout))
+        print(make_result.stdout.decode("utf-8"))
+        # Only upload to Rosie if its a pull request.
+        if travis:
+            for rosie in ROSIE_SETUPS:
+                if not rosie_ok[rosie]:
+                    break
+                print("Uploading to https://{rosie}.ngrok.io/upload/{sha}".format(rosie=rosie, sha=sha))
+                #curl -F "file=@$final_filename" https://$rosie.ngrok.io/upload/$sha
+        if travis:
+            print('travis_fold:end:adafruit-bins-{}-{}\\r'.format(language, board))
+
+        print()
+
+sys.exit(exit_status)
diff --git a/tools/build_adafruit_bins.sh b/tools/build_adafruit_bins.sh
deleted file mode 100755
index 4c672c6de..000000000
--- a/tools/build_adafruit_bins.sh
+++ /dev/null
@@ -1,116 +0,0 @@
-#!/usr/bin/env bash
-
-rm -rf ports/atmel-samd/build*
-rm -rf ports/esp8266/build*
-rm -rf ports/nrf/build*
-
-# Alphabetical.
-HW_BOARDS="\
-arduino_mkr1300 \
-arduino_zero \
-catwan_usbstick \
-circuitplayground_express \
-circuitplayground_express_crickit \
-feather_huzzah \
-feather_m0_adalogger \
-feather_m0_basic \
-feather_m0_express \
-feather_m0_express_crickit \
-feather_m0_rfm69 \
-feather_m0_rfm9x \
-feather_m4_express \
-feather_nrf52832 \
-feather_nrf52840_express \
-feather_radiofruit_zigbee \
-gemma_m0 \
-grandcentral_m4_express \
-hallowing_m0_express \
-itsybitsy_m0_express \
-itsybitsy_m4_express \
-makerdiary_nrf52840_mdk \
-metro_m0_express \
-metro_m4_express \
-meowmeow \
-pca10056 \
-pca10059 \
-pirkey_m0 \
-trellis_m4_express \
-trinket_m0 \
-"
-ROSIE_SETUPS="rosie-ci"
-
-PARALLEL="-j 5"
-if [[ "$TRAVIS" == "true" ]]; then
-    PARALLEL="-j 2"
-fi
-
-if [[ -z "$TRAVIS_BOARD" ]]; then
-    boards=$HW_BOARDS
-else
-    boards=$TRAVIS_BOARD
-fi
-
-version=`git describe --tags --exact-match`
-if [[ $? -ne 0 ]]; then
-    version=`date +%Y%m%d`-`git rev-parse --short HEAD`
-fi
-
-
-if [[ "$TRAVIS" == "true" ]]; then
-    sha=$TRAVIS_COMMIT
-    if [[ "$TRAVIS_PULL_REQUEST" != "false" ]]; then
-       version=`date +%Y%m%d`-`echo $TRAVIS_PULL_REQUEST_SHA | cut -c1-7`
-       sha=$TRAVIS_PULL_REQUEST_SHA
-    fi
-fi
-
-for board in $boards; do
-    mkdir -p bin/$board/
-    extension=uf2
-    for language_file in $(ls locale/*.po); do
-        language=$(basename -s .po $language_file)
-        echo "Building $board for $language"
-        # There is a bug in the Huzzah Makefile that causes it to fail occasionally with -j > 1.
-        if [[ $board == "feather_huzzah" ]]; then
-            make -C ports/esp8266 TRANSLATION=$language BOARD=$board
-            (( exit_status = exit_status || $? ))
-            temp_filename=ports/esp8266/build/firmware-combined.bin
-            extension=bin
-        elif [[ $board == "feather_nrf52832" ]]; then
-            make $PARALLEL -C ports/nrf TRANSLATION=$language BOARD=$board
-            (( exit_status = exit_status || $? ))
-            temp_filename=ports/nrf/build-$board-s132/firmware.bin
-            extension=bin
-        elif [[ $board == "feather_nrf52840_express" || $board == "pca10056" || $board == "pca10059" ]]; then
-            make $PARALLEL -C ports/nrf TRANSLATION=$language BOARD=$board SD=s140
-            (( exit_status = exit_status || $? ))
-            temp_filename=ports/nrf/build-$board-s140/firmware.uf2
-            extension=uf2
-        elif [[ $board == "makerdiary_nrf52840_mdk" ]]; then
-            make $PARALLEL -C ports/nrf TRANSLATION=$language BOARD=$board SD=s140
-            (( exit_status = exit_status || $? ))
-            temp_filename=ports/nrf/build-$board-s140/firmware.hex
-            extension=hex
-        else
-            time make $PARALLEL -C ports/atmel-samd TRANSLATION=$language BOARD=$board
-            (( exit_status = exit_status || $? ))
-            cp ports/atmel-samd/build-$board/firmware.bin bin/$board/adafruit-circuitpython-$board-$language-$version.bin
-            (( exit_status = exit_status || $? ))
-            temp_filename=ports/atmel-samd/build-$board/firmware.uf2
-            extension=uf2
-        fi
-        final_filename=bin/$board/adafruit-circuitpython-$board-$language-$version.$extension
-        cp $temp_filename $final_filename
-        (( exit_status = exit_status || $? ))
-        # Only upload to Rosie if its a pull request.
-        if [[ "$TRAVIS" == "true" ]]; then
-            for rosie in $ROSIE_SETUPS; do
-                echo "Uploading to https://$rosie.ngrok.io/upload/$sha"
-                curl -F "file=@$final_filename" https://$rosie.ngrok.io/upload/$sha
-            done
-        fi
-        echo
-    done
-done
-
-exit $exit_status
diff --git a/tools/build_board_info.py b/tools/build_board_info.py
new file mode 100644
index 000000000..f6b2d3b09
--- /dev/null
+++ b/tools/build_board_info.py
@@ -0,0 +1,243 @@
+#!/usr/bin/env python3
+
+import json
+import os
+import subprocess
+import sys
+import sh
+import base64
+from datetime import date
+from sh.contrib import git
+
+sys.path.append("adabot")
+import adabot.github_requests as github
+
+SUPPORTED_PORTS = ["nrf", "esp8266", "atmel-samd"]
+
+BIN = ('bin',)
+UF2 = ('uf2',)
+BIN_UF2 = ('bin', 'uf2')
+HEX = ('hex',)
+
+# Default extensions
+extension_by_port = {
+    "nrf": BIN,
+    "esp8266": BIN,
+    "atmel-samd": UF2,
+}
+
+# Per board overrides
+extension_by_board = {
+    "feather_nrf52832": BIN,
+    "arduino_zero": BIN,
+    "feather_m0_adalogger": BIN_UF2,
+    "feather_m0_basic": BIN_UF2,
+    "makerdiary_nrf52840_mdk": HEX
+}
+
+def get_languages():
+    languages = []
+    for f in os.scandir("../locale"):
+        if f.name.endswith(".po"):
+            languages.append(f.name[:-3])
+    return languages
+
+def get_board_mapping():
+    boards = {}
+    for port in SUPPORTED_PORTS:
+        board_path = os.path.join("../ports", port, "boards")
+        for board_path in os.scandir(board_path):
+            if board_path.is_dir():
+                board_files = os.listdir(board_path.path)
+                board_id = board_path.name
+                extensions = extension_by_port[port]
+                extensions = extension_by_board.get(board_path.name, extensions)
+                boards[board_id] = {"port": port, "extensions": extensions, "download_count": 0}
+    return boards
+
+def get_version_info():
+    version = None
+    sha = git("rev-parse", "--short", "HEAD").stdout.decode("utf-8")
+    try:
+        version = git("describe", "--tags", "--exact-match").stdout.decode("utf-8")
+    except sh.ErrorReturnCode_128:
+        # No exact match
+        pass
+
+    if "TRAVIS" in os.environ and os.environ["TRAVIS"] == "true":
+        sha = os.environ["TRAVIS_COMMIT"]
+        if os.environ["TRAVIS_PULL_REQUEST"] != "false":
+            sha = os.environ["TRAVIS_PULL_REQUEST_SHA"]
+
+    if not version:
+        version="{}-{}".format(date.today().strftime("%Y%m%d"), sha[:7])
+
+    return sha, version
+
+def get_current_info():
+    response = github.get("/repos/adafruit/circuitpython-org/git/refs/heads/master")
+    if not response.ok:
+        print(response.text)
+        raise RuntimeError("cannot get master sha")
+    commit_sha = response.json()["object"]["sha"]
+
+    response = github.get("/repos/adafruit/circuitpython-org/contents/_data/files.json?ref=" + commit_sha)
+    if not response.ok:
+        print(response.text)
+        raise RuntimeError("cannot get previous files.json")
+
+    response = response.json()
+
+    git_info = commit_sha, response["sha"]
+    return git_info, json.loads(base64.b64decode(response["content"]))
+
+def create_pr(changes, updated, git_info):
+    commit_sha, original_blob_sha = git_info
+    branch_name = "new_release_" + changes["new_release"]
+
+    updated = json.dumps(updated, sort_keys=True, indent=4).encode("utf-8") + b"\n"
+    print(updated.decode("utf-8"))
+    pr_title = "Automated website update for release {}".format(changes["new_release"])
+    boards = ""
+    if changes["new_boards"]:
+        boards = "New boards:\n* " + "\n* ".join(changes["new_boards"])
+    languages = ""
+    if changes["new_languages"]:
+        languages = "New languages:\n* " + "\n* ".join(changes["new_languages"])
+    message = "Automated website update for release {} by AdaBot.\n\n{}\n\n{}\n".format(
+        changes["new_release"],
+        boards,
+        languages
+    )
+
+    create_branch = {
+        "ref": "refs/heads/" + branch_name,
+        "sha": commit_sha
+    }
+    response = github.post("/repos/adafruit-adabot/circuitpython-org/git/refs", json=create_branch)
+    if not response.ok and response.json()["message"] != "Reference already exists":
+        print("unable to create branch")
+        print(response.text)
+        return
+
+    update_file = {
+        "message": message,
+        "content": base64.b64encode(updated).decode("utf-8"),
+        "sha": original_blob_sha,
+        "branch": branch_name
+    }
+
+    response = github.put("/repos/adafruit-adabot/circuitpython-org/contents/_data/files.json", json=update_file)
+    if not response.ok:
+        print("unable to post new file")
+        print(response.text)
+        return
+    pr_info = {
+        "title": pr_title,
+        "head": "adafruit-adabot:" + branch_name,
+        "base": "master",
+        "body": message,
+        "maintainer_can_modify": True
+    }
+    response = github.post("/repos/adafruit/circuitpython-org/pulls", json=pr_info)
+    if not response.ok:
+        print("unable to create pr")
+        print(response.text)
+        return
+    print(changes)
+    print(pr_info)
+
+def update_downloads(boards, release):
+    response = github.get("/repos/adafruit/circuitpython/releases/tags/{}".format(release))
+    if not response.ok:
+        print(response.text)
+        raise RuntimeError("cannot get previous release info")
+
+    assets = response.json()["assets"]
+    for asset in assets:
+        boards[asset["name"].split("-")[2]]["download_count"] += asset["download_count"]
+
+
+def print_active_user():
+    response = github.get("/user")
+    if response.ok:
+        print("Logged in as {}".format(response.json()["login"]))
+    else:
+        print("Not logged in")
+
+def generate_download_info():
+    boards = {}
+    errors = []
+
+    new_tag = os.environ.get("TRAVIS_TAG", "4.0.0-alpha.3")
+
+    changes = {
+        "new_release": new_tag,
+        "new_boards": [],
+        "new_languages": []
+    }
+
+    print_active_user()
+
+    sha, this_version = get_version_info()
+
+    git_info, current_info = get_current_info()
+
+    languages = get_languages()
+
+    new_stable = "-" not in new_tag
+
+    previous_releases = set()
+    previous_languages = set()
+
+    # Delete the release we are replacing
+    for board in current_info:
+        info = current_info[board]
+        for version in info["versions"]:
+            previous_releases.add(version["version"])
+            previous_languages.update(version["files"].keys())
+            if version["stable"] == new_stable:
+                info["versions"].remove(version)
+
+    board_mapping = get_board_mapping()
+
+    print(previous_releases)
+    for release in previous_releases:
+        update_downloads(board_mapping, release)
+
+    for port in SUPPORTED_PORTS:
+        board_path = os.path.join("../ports", port, "boards")
+        for board_path in os.scandir(board_path):
+            if board_path.is_dir():
+                board_files = os.listdir(board_path)
+                board_id = board_path.name
+                board_info = board_mapping[board_id]
+
+                if board_id not in current_info:
+                    changes["new_boards"].append(board_id)
+                    current_info[board_id] = {"downloads": 0,
+                                              "versions": []}
+
+                new_version = {
+                    "stable": new_stable,
+                    "version": new_tag,
+                    "files": {}
+                }
+                for language in languages:
+                    files = []
+                    new_version["files"][language] = files
+                    for extension in board_info["extensions"]:
+                        files.append("https://github.com/adafruit/circuitpython/releases/download/{tag}/adafruit-circuitpython-{board}-{language}-{tag}.{extension}".format(tag=new_tag, board=board_id, language=language, extension=extension))
+                current_info[board_id]["downloads"] = board_info["download_count"]
+                current_info[board_id]["versions"].append(new_version)
+
+    changes["new_languages"] = set(languages) - previous_languages
+
+    if changes["new_languages"]:
+        create_pr(changes, current_info, git_info)
+
+if __name__ == "__main__":
+    if "TRAVIS_TAG" in os.environ:
+        generate_download_info()
+    else:
+        print("skipping website update because this isn't a tag")