diff options
Diffstat (limited to 'tools')
34 files changed, 2227 insertions, 1635 deletions
diff --git a/tools/analyze_heap_dump.py b/tools/analyze_heap_dump.py index 887871db7..d9b3dda59 100755 --- a/tools/analyze_heap_dump.py +++ b/tools/analyze_heap_dump.py @@ -38,42 +38,73 @@ MP_OBJ_SENTINEL = 4 READLINE_HIST_SIZE = 8 -SKIP_SYMBOLS = [".debug_ranges", ".debug_frame", ".debug_loc", ".comment", ".debug_str", ".debug_line", ".debug_abbrev", ".debug_info", "COMMON"] +SKIP_SYMBOLS = [ + ".debug_ranges", + ".debug_frame", + ".debug_loc", + ".comment", + ".debug_str", + ".debug_line", + ".debug_abbrev", + ".debug_info", + "COMMON", +] + @click.command() @click.argument("ram_filename") @click.argument("bin_filename") @click.argument("map_filename") -@click.option("--print_block_contents", default=False, - help="Prints the contents of each allocated block") -@click.option("--print_unknown_types", default=False, - help="Prints the micropython base type if we don't understand it.") -@click.option("--print_block_state", default=False, - help="Prints the heap block states (allocated or free)") -@click.option("--print_conflicting_symbols", default=False, - help="Prints conflicting symbols from the map") -@click.option("--print-heap-structure/--no-print-heap-structure", default=False, - help="Print heap structure") -@click.option("--output_directory", default="heapvis", - help="Destination for rendered output") -@click.option("--draw-heap-layout/--no-draw-heap-layout", default=True, - help="Draw the heap layout") -@click.option("--draw-heap-ownership/--no-draw-heap-ownership", default=False, - help="Draw the ownership graph of blocks on the heap") -@click.option("--analyze-snapshots", default="last", type=click.Choice(['all', 'last'])) -def do_all_the_things(ram_filename, bin_filename, map_filename, print_block_contents, - print_unknown_types, print_block_state, print_conflicting_symbols, - print_heap_structure, output_directory, draw_heap_layout, - draw_heap_ownership, analyze_snapshots): +@click.option( + "--print_block_contents", default=False, help="Prints the contents of each allocated block" +) +@click.option( + "--print_unknown_types", + default=False, + help="Prints the micropython base type if we don't understand it.", +) +@click.option( + "--print_block_state", default=False, help="Prints the heap block states (allocated or free)" +) +@click.option( + "--print_conflicting_symbols", default=False, help="Prints conflicting symbols from the map" +) +@click.option( + "--print-heap-structure/--no-print-heap-structure", default=False, help="Print heap structure" +) +@click.option("--output_directory", default="heapvis", help="Destination for rendered output") +@click.option( + "--draw-heap-layout/--no-draw-heap-layout", default=True, help="Draw the heap layout" +) +@click.option( + "--draw-heap-ownership/--no-draw-heap-ownership", + default=False, + help="Draw the ownership graph of blocks on the heap", +) +@click.option("--analyze-snapshots", default="last", type=click.Choice(["all", "last"])) +def do_all_the_things( + ram_filename, + bin_filename, + map_filename, + print_block_contents, + print_unknown_types, + print_block_state, + print_conflicting_symbols, + print_heap_structure, + output_directory, + draw_heap_layout, + draw_heap_ownership, + analyze_snapshots, +): with open(ram_filename, "rb") as f: ram_dump = f.read() with open(bin_filename, "rb") as f: rom = f.read() - symbols = {} # name -> address, size - symbol_lookup = {} # address -> name - manual_symbol_map = {} # autoname -> name + symbols = {} # name -> address, size + symbol_lookup = {} # address -> name + manual_symbol_map = {} # autoname -> name def add_symbol(name, address=None, size=None): if "lto_priv" in name: @@ -85,7 +116,11 @@ def do_all_the_things(ram_filename, bin_filename, map_filename, print_block_cont if name in symbols: if address and symbols[name][0] and symbols[name][0] != address: if print_conflicting_symbols: - print("Conflicting symbol: {} at addresses 0x{:08x} and 0x{:08x}".format(name, address, symbols[name][0])) + print( + "Conflicting symbol: {} at addresses 0x{:08x} and 0x{:08x}".format( + name, address, symbols[name][0] + ) + ) return if not address: address = symbols[name][0] @@ -118,21 +153,42 @@ def do_all_the_things(ram_filename, bin_filename, map_filename, print_block_cont add_symbol(parts[0], size=parts[1]) name = None else: - if len(parts) == 1 and parts[0].startswith((".text", ".rodata", ".bss")) and parts[0].count(".") > 1 and not parts[0].isnumeric() and ".str" not in parts[0]: + if ( + len(parts) == 1 + and parts[0].startswith((".text", ".rodata", ".bss")) + and parts[0].count(".") > 1 + and not parts[0].isnumeric() + and ".str" not in parts[0] + ): name = parts[0].split(".")[2] - if len(parts) == 3 and parts[0].startswith("0x") and parts[1].startswith("0x") and name: + if ( + len(parts) == 3 + and parts[0].startswith("0x") + and parts[1].startswith("0x") + and name + ): add_symbol(name, parts[0], parts[1]) name = None if len(parts) == 2 and parts[0].startswith("0x") and not parts[1].startswith("0x"): add_symbol(parts[1], parts[0]) - if len(parts) == 4 and parts[0] not in SKIP_SYMBOLS and parts[1].startswith("0x") and parts[2].startswith("0x"): + if ( + len(parts) == 4 + and parts[0] not in SKIP_SYMBOLS + and parts[1].startswith("0x") + and parts[2].startswith("0x") + ): name, address, size, source = parts if name.startswith((".text", ".rodata", ".bss")) and name.count(".") > 1: name = name.split(".")[-1] add_symbol(name, address, size) name = None # Linker symbols - if len(parts) >= 4 and parts[0].startswith("0x") and parts[2] == "=" and parts[1] != ".": + if ( + len(parts) >= 4 + and parts[0].startswith("0x") + and parts[2] == "=" + and parts[1] != "." + ): add_symbol(parts[1], parts[0]) rom_start = symbols["_sfixed"][0] @@ -143,13 +199,14 @@ def do_all_the_things(ram_filename, bin_filename, map_filename, print_block_cont # print(len(ram_dump) // ram_length, "snapshots") if analyze_snapshots == "all": snapshots = range(len(ram_dump) // ram_length - 1, -1, -1) - #snapshots = range(4576, -1, -1) + # snapshots = range(4576, -1, -1) elif analyze_snapshots == "last": snapshots = range(len(ram_dump) // ram_length - 1, len(ram_dump) // ram_length - 2, -1) for snapshot_num in snapshots: - ram = ram_dump[ram_length*snapshot_num:ram_length*(snapshot_num + 1)] + ram = ram_dump[ram_length * snapshot_num : ram_length * (snapshot_num + 1)] ownership_graph = pgv.AGraph(directed=True) + def load(address, size=4): if size is None: raise ValueError("You must provide a size") @@ -157,11 +214,11 @@ def do_all_the_things(ram_filename, bin_filename, map_filename, print_block_cont ram_address = address - ram_start if (ram_address + size) > len(ram): raise ValueError("Unable to read 0x{:08x} from ram.".format(address)) - return ram[ram_address:ram_address+size] + return ram[ram_address : ram_address + size] elif address < len(rom): if (address + size) > len(rom): raise ValueError("Unable to read 0x{:08x} from rom.".format(address)) - return rom[address:address+size] + return rom[address : address + size] def load_pointer(address): return struct.unpack("<I", load(address))[0] @@ -197,11 +254,15 @@ def do_all_the_things(ram_filename, bin_filename, map_filename, print_block_cont # These change every run so we load them from the symbol table mp_state_ctx = symbols["mp_state_ctx"][0] manual_symbol_map["mp_state_ctx+20"] = "mp_state_ctx.vm.last_pool" - last_pool = load_pointer(mp_state_ctx + 20) # (gdb) p &mp_state_ctx.vm.last_pool + last_pool = load_pointer(mp_state_ctx + 20) # (gdb) p &mp_state_ctx.vm.last_pool manual_symbol_map["mp_state_ctx+108"] = "mp_state_ctx.vm.dict_main.map.table" - dict_main_table = load_pointer(mp_state_ctx + 108) # (gdb) p &mp_state_ctx.vm.dict_main.map.table + dict_main_table = load_pointer( + mp_state_ctx + 108 + ) # (gdb) p &mp_state_ctx.vm.dict_main.map.table manual_symbol_map["mp_state_ctx+84"] = "mp_state_ctx.vm.mp_loaded_modules_dict.map.table" - imports_table = load_pointer(mp_state_ctx + 84) # (gdb) p &mp_state_ctx.vm.mp_loaded_modules_dict.map.table + imports_table = load_pointer( + mp_state_ctx + 84 + ) # (gdb) p &mp_state_ctx.vm.mp_loaded_modules_dict.map.table manual_symbol_map["mp_state_ctx+124"] = "mp_state_ctx.vm.mp_sys_path_obj.items" manual_symbol_map["mp_state_ctx+140"] = "mp_state_ctx.vm.mp_sys_argv_obj.items" @@ -209,7 +270,9 @@ def do_all_the_things(ram_filename, bin_filename, map_filename, print_block_cont manual_symbol_map["0x200015e0"] = "mp_state_ctx.vm.dict_main" for i in range(READLINE_HIST_SIZE): - manual_symbol_map["mp_state_ctx+{}".format(148 + i * 4)] = "mp_state_ctx.vm.readline_hist[{}]".format(i) + manual_symbol_map[ + "mp_state_ctx+{}".format(148 + i * 4) + ] = "mp_state_ctx.vm.readline_hist[{}]".format(i) tuple_type = symbols["mp_type_tuple"][0] type_type = symbols["mp_type_type"][0] @@ -217,12 +280,17 @@ def do_all_the_things(ram_filename, bin_filename, map_filename, print_block_cont dict_type = symbols["mp_type_dict"][0] property_type = symbols["mp_type_property"][0] str_type = symbols["mp_type_str"][0] - function_types = [symbols["mp_type_fun_" + x][0] for x in ["bc", "builtin_0", "builtin_1", "builtin_2", "builtin_3", "builtin_var"]] + function_types = [ + symbols["mp_type_fun_" + x][0] + for x in ["bc", "builtin_0", "builtin_1", "builtin_2", "builtin_3", "builtin_var"] + ] bytearray_type = symbols["mp_type_bytearray"][0] - dynamic_type = 0x40000000 # placeholder, doesn't match any memory + dynamic_type = 0x40000000 # placeholder, doesn't match any memory - long_lived_start = load_pointer(mp_state_ctx + 272) # (gdb) p &mp_state_ctx.mem.gc_lowest_long_lived_ptr + long_lived_start = load_pointer( + mp_state_ctx + 272 + ) # (gdb) p &mp_state_ctx.mem.gc_lowest_long_lived_ptr type_colors = { dict_type: "red", @@ -231,13 +299,23 @@ def do_all_the_things(ram_filename, bin_filename, map_filename, print_block_cont type_type: "orange", tuple_type: "skyblue", str_type: "pink", - bytearray_type: "purple" - } + bytearray_type: "purple", + } pool_shift = heap_start % BYTES_PER_BLOCK - atb_length = total_byte_len * BITS_PER_BYTE // (BITS_PER_BYTE + BITS_PER_BYTE * BLOCKS_PER_ATB // BLOCKS_PER_FTB + BITS_PER_BYTE * BLOCKS_PER_ATB * BYTES_PER_BLOCK) + atb_length = ( + total_byte_len + * BITS_PER_BYTE + // ( + BITS_PER_BYTE + + BITS_PER_BYTE * BLOCKS_PER_ATB // BLOCKS_PER_FTB + + BITS_PER_BYTE * BLOCKS_PER_ATB * BYTES_PER_BLOCK + ) + ) pool_length = atb_length * BLOCKS_PER_ATB * BYTES_PER_BLOCK - gc_finaliser_table_byte_len = (atb_length * BLOCKS_PER_ATB + BLOCKS_PER_FTB - 1) // BLOCKS_PER_FTB + gc_finaliser_table_byte_len = ( + atb_length * BLOCKS_PER_ATB + BLOCKS_PER_FTB - 1 + ) // BLOCKS_PER_FTB if print_heap_structure: print("mp_state_ctx at 0x{:08x} and length {}".format(*symbols["mp_state_ctx"])) @@ -247,7 +325,7 @@ def do_all_the_things(ram_filename, bin_filename, map_filename, print_block_cont print("FTB length:", gc_finaliser_table_byte_len) pool_start = heap_start + total_byte_len - pool_length - pool_shift - pool = heap[-pool_length-pool_shift:] + pool = heap[-pool_length - pool_shift :] total_height = 128 * 18 total_width = (pool_length // (128 * 16)) * 85 @@ -279,9 +357,11 @@ def do_all_the_things(ram_filename, bin_filename, map_filename, print_block_cont for k in range(current_allocation - 1): rows += "<tr>" for l in range(4): - rows += "<td port=\"{}\" height=\"18\" width=\"20\"></td>".format(4 * (k + 1) + l) + rows += '<td port="{}" height="18" width="20"></td>'.format(4 * (k + 1) + l) rows += "</tr>" - table = "<<table bgcolor=\"gray\" border=\"1\" cellpadding=\"0\" cellspacing=\"0\"><tr><td colspan=\"4\" port=\"0\" height=\"18\" width=\"80\">0x{:08x}</td></tr>{}</table>>".format(address, rows) + table = '<<table bgcolor="gray" border="1" cellpadding="0" cellspacing="0"><tr><td colspan="4" port="0" height="18" width="80">0x{:08x}</td></tr>{}</table>>'.format( + address, rows + ) ownership_graph.add_node(address, label=table, style="invisible", shape="plaintext") print("add 0x{:08x}".format(address)) @@ -299,7 +379,7 @@ def do_all_the_things(ram_filename, bin_filename, map_filename, print_block_cont potential_type = word bgcolor = "gray" if address in qstr_pools: - #print(address, len(data)) + # print(address, len(data)) bgcolor = "tomato" elif potential_type in function_types: bgcolor = "green" @@ -308,12 +388,13 @@ def do_all_the_things(ram_filename, bin_filename, map_filename, print_block_cont elif print_unknown_types: print("unknown type", hex(potential_type)) - node.attr["label"] = "<" + node.attr["label"].replace("\"gray\"", "\"" + bgcolor + "\"") + ">" + node.attr["label"] = ( + "<" + node.attr["label"].replace('"gray"', '"' + bgcolor + '"') + ">" + ) if potential_type == str_type and k == 3: string_blocks.append(word) - if potential_type == dict_type: if k == 3: map_element_blocks.append(word) @@ -322,7 +403,7 @@ def do_all_the_things(ram_filename, bin_filename, map_filename, print_block_cont port = k if k < 4: port = 0 - ownership_graph.add_edge(address, word, tailport=str(port)+":_") + ownership_graph.add_edge(address, word, tailport=str(port) + ":_") print(" 0x{:08x}".format(word)) if address in qstr_pools: if k > 0: @@ -330,7 +411,6 @@ def do_all_the_things(ram_filename, bin_filename, map_filename, print_block_cont if k == 0: potential_type = dynamic_type - if potential_type == dynamic_type: if k == 0: node.attr["fillcolor"] = "plum" @@ -341,7 +421,6 @@ def do_all_the_things(ram_filename, bin_filename, map_filename, print_block_cont if k == 2 and ram_start < word < ram_end: bytecode_blocks.append(word) - longest_free = 0 current_free = 0 current_allocation = 0 @@ -356,7 +435,9 @@ def do_all_the_things(ram_filename, bin_filename, map_filename, print_block_cont print("{} bytes free".format(current_free * BYTES_PER_BLOCK)) current_free = 0 if block_state != AT_TAIL and current_allocation > 0: - save_allocated_block((i * BLOCKS_PER_ATB + j) * BYTES_PER_BLOCK, current_allocation) + save_allocated_block( + (i * BLOCKS_PER_ATB + j) * BYTES_PER_BLOCK, current_allocation + ) current_allocation = 0 if block_state == AT_FREE: current_free += 1 @@ -368,13 +449,13 @@ def do_all_the_things(ram_filename, bin_filename, map_filename, print_block_cont # current_allocation > 0 ensures we only extend an allocation thats started. current_allocation += 1 longest_free = max(longest_free, current_free) - #if current_free > 0: + # if current_free > 0: # print("{} bytes free".format(current_free * BYTES_PER_BLOCK)) if current_allocation > 0: save_allocated_block(pool_length, current_allocation) def is_qstr(obj): - return obj & 0xff800007 == 0x00000006 + return obj & 0xFF800007 == 0x00000006 def find_qstr(qstr_index): pool_ptr = last_pool @@ -396,8 +477,10 @@ def do_all_the_things(ram_filename, bin_filename, map_filename, print_block_cont return "missing" else: rom_offset = pool_ptr - rom_start - prev, total_prev_len, alloc, length = struct.unpack_from("<IIII", rom[rom_offset:rom_offset+32]) - pool = rom[rom_offset:rom_offset + 32 + length * 4] + prev, total_prev_len, alloc, length = struct.unpack_from( + "<IIII", rom[rom_offset : rom_offset + 32] + ) + pool = rom[rom_offset : rom_offset + 32 + length * 4] if qstr_index >= total_prev_len: offset = (qstr_index - total_prev_len) * 4 + 16 @@ -406,14 +489,14 @@ def do_all_the_things(ram_filename, bin_filename, map_filename, print_block_cont start -= rom_start if start > len(rom): return "more than rom: {:x}".format(start + rom_start) - qstr_hash, qstr_len = struct.unpack("<BB", rom[start:start+2]) - return rom[start+2:start+2+qstr_len].decode("utf-8") + qstr_hash, qstr_len = struct.unpack("<BB", rom[start : start + 2]) + return rom[start + 2 : start + 2 + qstr_len].decode("utf-8") else: if start > heap_start + len(heap): return "out of range: {:x}".format(start) local = start - heap_start - qstr_hash, qstr_len = struct.unpack("<BB", heap[local:local+2]) - return heap[local+2:local+2+qstr_len].decode("utf-8") + qstr_hash, qstr_len = struct.unpack("<BB", heap[local : local + 2]) + return heap[local + 2 : local + 2 + qstr_len].decode("utf-8") pool_ptr = prev return "unknown" @@ -432,7 +515,11 @@ def do_all_the_things(ram_filename, bin_filename, map_filename, print_block_cont try: node = ownership_graph.get_node(block) except KeyError: - print("Unable to find memory block for 0x{:08x}. Is there something running?".format(block)) + print( + "Unable to find memory block for 0x{:08x}. Is there something running?".format( + block + ) + ) continue if block not in block_data: continue @@ -449,14 +536,16 @@ def do_all_the_things(ram_filename, bin_filename, map_filename, print_block_cont edge.attr["tailport"] = str(key) rows = "" for i in range(len(cells) // 2): - rows += "<tr><td port=\"{}\">{}</td><td port=\"{}\">{}</td></tr>".format( - cells[2*i][0], - cells[2*i][1], - cells[2*i+1][0], - cells[2*i+1][1]) + rows += '<tr><td port="{}">{}</td><td port="{}">{}</td></tr>'.format( + cells[2 * i][0], cells[2 * i][1], cells[2 * i + 1][0], cells[2 * i + 1][1] + ) node.attr["shape"] = "plaintext" node.attr["style"] = "invisible" - node.attr["label"] = "<<table bgcolor=\"gold\" border=\"1\" cellpadding=\"0\" cellspacing=\"0\"><tr><td colspan=\"2\">0x{:08x}</td></tr>{}</table>>".format(block, rows) + node.attr[ + "label" + ] = '<<table bgcolor="gold" border="1" cellpadding="0" cellspacing="0"><tr><td colspan="2">0x{:08x}</td></tr>{}</table>>'.format( + block, rows + ) for node, degree in ownership_graph.in_degree_iter(): print(node, degree) @@ -488,12 +577,12 @@ def do_all_the_things(ram_filename, bin_filename, map_filename, print_block_cont print("Unable to find memory block for string at 0x{:08x}.".format(block)) continue try: - raw_string = block_data[block].decode('utf-8') + raw_string = block_data[block].decode("utf-8") except: raw_string = str(block_data[block]) wrapped = [] for i in range(0, len(raw_string), 16): - wrapped.append(raw_string[i:i+16]) + wrapped.append(raw_string[i : i + 16]) node.attr["label"] = "\n".join(wrapped) node.attr["style"] = "filled" node.attr["fontname"] = "FiraCode-Medium" @@ -516,17 +605,29 @@ def do_all_the_things(ram_filename, bin_filename, map_filename, print_block_cont rows = "" remaining_bytecode = len(data) - 16 while code_info_size >= 16: - rows += "<tr><td colspan=\"16\" bgcolor=\"palegreen\" height=\"18\" width=\"80\"></td></tr>" + rows += ( + '<tr><td colspan="16" bgcolor="palegreen" height="18" width="80"></td></tr>' + ) code_info_size -= 16 remaining_bytecode -= 16 if code_info_size > 0: - rows += ("<tr><td colspan=\"{}\" bgcolor=\"palegreen\" height=\"18\" width=\"{}\"></td>" - "<td colspan=\"{}\" bgcolor=\"seagreen\" height=\"18\" width=\"{}\"></td></tr>" - ).format(code_info_size, code_info_size * (80 / 16), (16 - code_info_size), (80 / 16) * (16 - code_info_size)) + rows += ( + '<tr><td colspan="{}" bgcolor="palegreen" height="18" width="{}"></td>' + '<td colspan="{}" bgcolor="seagreen" height="18" width="{}"></td></tr>' + ).format( + code_info_size, + code_info_size * (80 / 16), + (16 - code_info_size), + (80 / 16) * (16 - code_info_size), + ) remaining_bytecode -= 16 for i in range(remaining_bytecode // 16): - rows += "<tr><td colspan=\"16\" bgcolor=\"seagreen\" height=\"18\" width=\"80\"></td></tr>" - node.attr["label"] = "<<table border=\"1\" cellspacing=\"0\"><tr><td colspan=\"16\" bgcolor=\"lightseagreen\" height=\"18\" width=\"80\">0x{:08x}</td></tr>{}</table>>".format(block, rows) + rows += '<tr><td colspan="16" bgcolor="seagreen" height="18" width="80"></td></tr>' + node.attr[ + "label" + ] = '<<table border="1" cellspacing="0"><tr><td colspan="16" bgcolor="lightseagreen" height="18" width="80">0x{:08x}</td></tr>{}</table>>'.format( + block, rows + ) for block in qstr_chunks: if block not in block_data: @@ -543,9 +644,11 @@ def do_all_the_things(ram_filename, bin_filename, map_filename, print_block_cont continue offset += 2 + qstr_len + 1 try: - qstrs_in_chunk += " " + data[offset - qstr_len - 1: offset - 1].decode("utf-8") + qstrs_in_chunk += " " + data[offset - qstr_len - 1 : offset - 1].decode( + "utf-8" + ) except UnicodeDecodeError: - qstrs_in_chunk += " " + "░"*qstr_len + qstrs_in_chunk += " " + "░" * qstr_len printable_qstrs = "" for i in range(len(qstrs_in_chunk)): c = qstrs_in_chunk[i] @@ -555,9 +658,13 @@ def do_all_the_things(ram_filename, bin_filename, map_filename, print_block_cont printable_qstrs += qstrs_in_chunk[i] wrapped = [] for i in range(0, len(printable_qstrs), 16): - wrapped.append(html.escape(printable_qstrs[i:i+16])) + wrapped.append(html.escape(printable_qstrs[i : i + 16])) node = ownership_graph.get_node(block) - node.attr["label"] = "<<table border=\"1\" cellspacing=\"0\" bgcolor=\"lightsalmon\" width=\"80\"><tr><td height=\"18\" >0x{:08x}</td></tr><tr><td height=\"{}\" >{}</td></tr></table>>".format(block, 18 * (len(wrapped) - 1), "<br/>".join(wrapped)) + node.attr[ + "label" + ] = '<<table border="1" cellspacing="0" bgcolor="lightsalmon" width="80"><tr><td height="18" >0x{:08x}</td></tr><tr><td height="{}" >{}</td></tr></table>>'.format( + block, 18 * (len(wrapped) - 1), "<br/>".join(wrapped) + ) node.attr["fontname"] = "FiraCode-Bold" if block >= long_lived_start: node.attr["fontcolor"] = "hotpink" @@ -598,10 +705,10 @@ def do_all_the_things(ram_filename, bin_filename, map_filename, print_block_cont height = float(node.attr["height"]) except: height = 0.25 - #print(hex(address), "height", height, y) - #if address in block_data: + # print(hex(address), "height", height, y) + # if address in block_data: # print(hex(address), block, len(block_data[address]), x, y, height) - node.attr["pos"] = "{},{}".format(x * 80, (y - (height - 0.25) * 2) * 18) # in inches + node.attr["pos"] = "{},{}".format(x * 80, (y - (height - 0.25) * 2) * 18) # in inches # Reformat block nodes so they are the correct size and do not have keys in them. for block in sorted(map_element_blocks): @@ -609,45 +716,58 @@ def do_all_the_things(ram_filename, bin_filename, map_filename, print_block_cont node = ownership_graph.get_node(block) except KeyError: if block != 0: - print("Unable to find memory block for 0x{:08x}. Is there something running?".format(block)) + print( + "Unable to find memory block for 0x{:08x}. Is there something running?".format( + block + ) + ) continue - #node.attr["fillcolor"] = "gold" + # node.attr["fillcolor"] = "gold" if block not in block_data: continue data = block_data[block] - #print("0x{:08x}".format(block)) + # print("0x{:08x}".format(block)) cells = [] for i in range(len(data) // 8): key, value = struct.unpack_from("<II", data, offset=(i * 8)) if key == MP_OBJ_NULL or key == MP_OBJ_SENTINEL: - #print(" <empty slot>") + # print(" <empty slot>") cells.append(("", " ")) else: - #print(" {}, {}".format(format(key), format(value))) + # print(" {}, {}".format(format(key), format(value))) cells.append((key, "")) # if value in block_data: # edge = ownership_graph.get_edge(block, value) # edge.attr["tailport"] = str(key) rows = "" for i in range(len(cells) // 2): - rows += "<tr><td port=\"{}\" height=\"18\" width=\"40\">{}</td><td port=\"{}\" height=\"18\" width=\"40\">{}</td></tr>".format( - cells[2*i][0], - cells[2*i][1], - cells[2*i+1][0], - cells[2*i+1][1]) - node.attr["label"] = "<<table bgcolor=\"gold\" border=\"1\" cellpadding=\"0\" cellspacing=\"0\">{}</table>>".format(rows) - - - ownership_graph.add_node("center", pos="{},{}".format(total_width // 2 - 40, total_height // 2), shape="plaintext", label=" ") - ownership_graph.graph_attr["viewport"] = "{},{},1,{}".format(total_width, total_height, "center") + rows += '<tr><td port="{}" height="18" width="40">{}</td><td port="{}" height="18" width="40">{}</td></tr>'.format( + cells[2 * i][0], cells[2 * i][1], cells[2 * i + 1][0], cells[2 * i + 1][1] + ) + node.attr[ + "label" + ] = '<<table bgcolor="gold" border="1" cellpadding="0" cellspacing="0">{}</table>>'.format( + rows + ) + + ownership_graph.add_node( + "center", + pos="{},{}".format(total_width // 2 - 40, total_height // 2), + shape="plaintext", + label=" ", + ) + ownership_graph.graph_attr["viewport"] = "{},{},1,{}".format( + total_width, total_height, "center" + ) ownership_graph.has_layout = True if draw_heap_layout: fn = os.path.join(output_directory, "heap_layout{:04d}.png".format(snapshot_num)) print(fn) - #ownership_graph.write(fn+".dot") + # ownership_graph.write(fn+".dot") ownership_graph.draw(fn) + if __name__ == "__main__": do_all_the_things() diff --git a/tools/analyze_mpy.py b/tools/analyze_mpy.py index 99c06f62a..8c2056e03 100755 --- a/tools/analyze_mpy.py +++ b/tools/analyze_mpy.py @@ -10,219 +10,129 @@ import io bytecode_format_sizes = { "MP_OPCODE_BYTE": 1, "MP_OPCODE_QSTR": 3, - "MP_OPCODE_VAR_UINT": None, # Unknown because uint encoding uses the top bit to indicate the end. + "MP_OPCODE_VAR_UINT": None, # Unknown because uint encoding uses the top bit to indicate the end. "MP_OPCODE_OFFSET": 3, "MP_OPCODE_BYTE_EXTRA": 2, "MP_OPCODE_VAR_UINT_EXTRA": None, - "MP_OPCODE_OFFSET_EXTRA": 4 + "MP_OPCODE_OFFSET_EXTRA": 4, } bytecodes = { - 0x00: {"name": "MP_BC_LOAD_FAST_MULTI", - "format": "MP_OPCODE_BYTE"}, - 0x10: {"name": "MP_BC_LOAD_CONST_FALSE", - "format": "MP_OPCODE_BYTE"}, - 0x11: {"name": "MP_BC_LOAD_CONST_NONE", - "format": "MP_OPCODE_BYTE"}, - 0x12: {"name": "MP_BC_LOAD_CONST_TRUE", - "format": "MP_OPCODE_BYTE"}, - 0x14: {"name": "MP_BC_LOAD_CONST_SMALL_INT", - "format": "MP_OPCODE_VAR_UINT"}, - 0x16: {"name": "MP_BC_LOAD_CONST_STRING", - "format": "MP_OPCODE_QSTR"}, - 0x17: {"name": "MP_BC_LOAD_CONST_OBJ", - "format": "MP_OPCODE_VAR_UINT"}, -#define MP_BC_LOAD_CONST_OBJ (0x17) // ptr - 0x18: {"name": "MP_BC_LOAD_NULL", - "format": "MP_OPCODE_BYTE"}, - -#define MP_BC_LOAD_FAST_N (0x19) // uint - 0x1a: {"name": "MP_BC_LOAD_DEREF", - "format": "MP_OPCODE_VAR_UINT"}, - 0x1b: {"name": "MP_BC_LOAD_NAME", - "format": "MP_OPCODE_QSTR"}, - 0x1c: {"name": "MP_BC_LOAD_GLOBAL", - "format": "MP_OPCODE_QSTR"}, - 0x1d: {"name": "MP_BC_LOAD_ATTR", - "format": "MP_OPCODE_QSTR"}, - 0x1e: {"name": "MP_BC_LOAD_METHOD", - "format": "MP_OPCODE_QSTR"}, - 0x1f: {"name": "MP_BC_LOAD_SUPER_METHOD", - "format": "MP_OPCODE_QSTR"}, - 0x20: {"name": "MP_BC_LOAD_BUILD_CLASS", - "format": "MP_OPCODE_BYTE"}, -#define MP_BC_LOAD_BUILD_CLASS (0x20) -#define MP_BC_LOAD_SUBSCR (0x21) - 0x21: {"name": "MP_BC_LOAD_SUBSCR", - "format": "MP_OPCODE_BYTE"}, - -#define MP_BC_STORE_FAST_N (0x22) // uint -#define MP_BC_STORE_DEREF (0x23) // uint -#define MP_BC_STORE_NAME (0x24) // qstr - 0x24: {"name": "MP_BC_STORE_NAME", - "format": "MP_OPCODE_QSTR"}, - 0x25: {"name": "MP_BC_STORE_GLOBAL", - "format": "MP_OPCODE_QSTR"}, - 0x26: {"name": "MP_BC_STORE_ATTR", - "format": "MP_OPCODE_QSTR"}, - 0x27: {"name": "MP_BC_LOAD_SUBSCR", - "format": "MP_OPCODE_BYTE"}, - - 0x28: {"name": "MP_BC_DELETE_FAST", - "format": "MP_OPCODE_VAR_UINT"}, -#define MP_BC_DELETE_FAST (0x28) // uint -#define MP_BC_DELETE_DEREF (0x29) // uint -#define MP_BC_DELETE_NAME (0x2a) // qstr -#define MP_BC_DELETE_GLOBAL (0x2b) // qstr - - 0x30: {"name": "MP_BC_DUP_TOP", - "format": "MP_OPCODE_BYTE"}, -#define MP_BC_DUP_TOP_TWO (0x31) - 0x32: {"name": "MP_BC_POP_TOP", - "format": "MP_OPCODE_BYTE"}, - 0x33: {"name": "MP_BC_ROT_TWO", - "format": "MP_OPCODE_BYTE"}, - 0x34: {"name": "MP_BC_ROT_THREE", - "format": "MP_OPCODE_BYTE"}, - - 0x35: {"name": "MP_BC_JUMP", - "format": "MP_OPCODE_OFFSET"}, - 0x36: {"name": "MP_BC_POP_JUMP_IF_TRUE", - "format": "MP_OPCODE_OFFSET"}, - 0x37: {"name": "MP_BC_POP_JUMP_IF_FALSE", - "format": "MP_OPCODE_OFFSET"}, -#define MP_BC_JUMP_IF_TRUE_OR_POP (0x38) // rel byte code offset, 16-bit signed, in excess -#define MP_BC_JUMP_IF_FALSE_OR_POP (0x39) // rel byte code offset, 16-bit signed, in excess -#define MP_BC_SETUP_WITH (0x3d) // rel byte code offset, 16-bit unsigned -#define MP_BC_WITH_CLEANUP (0x3e) -#define MP_BC_SETUP_EXCEPT (0x3f) // rel byte code offset, 16-bit unsigned -#define MP_BC_SETUP_FINALLY (0x40) // rel byte code offset, 16-bit unsigned -#define MP_BC_END_FINALLY (0x41) -#define MP_BC_GET_ITER (0x42) -#define MP_BC_FOR_ITER (0x43) // rel byte code offset, 16-bit unsigned - 0x43: {"name": "MP_BC_FOR_ITER", - "format": "MP_OPCODE_OFFSET"}, - - 0x44: {"name": "MP_BC_POP_BLOCK", - "format": "MP_OPCODE_BYTE"}, -#define MP_BC_POP_EXCEPT (0x45) -#define MP_BC_UNWIND_JUMP (0x46) // rel byte code offset, 16-bit signed, in excess; then a byte - 0x47: {"name": "MP_BC_GET_ITER_STACK", - "format": "MP_OPCODE_BYTE"}, - - - 0x50: {"name": "MP_BC_BUILD_TUPLE", - "format": "MP_OPCODE_VAR_UINT"}, - 0x51: {"name": "MP_BC_BUILD_LIST", - "format": "MP_OPCODE_VAR_UINT"}, - 0x53: {"name": "MP_BC_BUILD_MAP", - "format": "MP_OPCODE_VAR_UINT"}, - 0x54: {"name": "MP_BC_STORE_MAP", - "format": "MP_OPCODE_BYTE"}, -#define MP_BC_BUILD_SET (0x56) // uint -#define MP_BC_BUILD_SLICE (0x58) // uint -#define MP_BC_STORE_COMP (0x57) // uint - 0x57: {"name": "MP_BC_STORE_COMP", - "format": "MP_OPCODE_VAR_UINT"}, -#define MP_BC_UNPACK_SEQUENCE (0x59) // uint -#define MP_BC_UNPACK_EX (0x5a) // uint - - 0x5b: {"name": "MP_BC_RETURN_VALUE", - "format": "MP_OPCODE_BYTE"}, - 0x5c: {"name": "MP_BC_RAISE_VARARGS", - "format": "MP_OPCODE_BYTE_EXTRA"}, -#define MP_BC_YIELD_VALUE (0x5d) -#define MP_BC_YIELD_FROM (0x5e) - -#define MP_BC_MAKE_FUNCTION (0x60) // uint - 0x60: {"name": "MP_BC_MAKE_FUNCTION", - "format": "MP_OPCODE_VAR_UINT"}, - 0x61: {"name": "MP_BC_MAKE_FUNCTION_DEFARGS", - "format": "MP_OPCODE_VAR_UINT"}, - 0x62: {"name": "MP_BC_MAKE_CLOSURE", - "format": "MP_OPCODE_VAR_UINT_EXTRA"}, - 0x63: {"name": "MP_BC_MAKE_CLOSURE", - "format": "MP_OPCODE_VAR_UINT_EXTRA"}, - 0x64: {"name": "MP_BC_CALL_FUNCTION", - "format": "MP_OPCODE_VAR_UINT"}, - 0x65: {"name": "MP_BC_CALL_FUNCTION_VAR_KW", - "format": "MP_OPCODE_VAR_UINT"}, - 0x66: {"name": "MP_BC_CALL_METHOD", - "format": "MP_OPCODE_VAR_UINT"}, - 0x67: {"name": "MP_BC_CALL_METHOD_VAR_KW", - "format": "MP_OPCODE_VAR_UINT"}, - - 0x68: {"name": "MP_BC_IMPORT_NAME", - "format": "MP_OPCODE_QSTR"}, - 0x69: {"name": "MP_BC_IMPORT_FROM", - "format": "MP_OPCODE_QSTR"}, -#define MP_BC_IMPORT_FROM (0x69) // qstr -#define MP_BC_IMPORT_STAR (0x6a) - -#define MP_BC_LOAD_CONST_SMALL_INT_MULTI (0x70) // + N(64) - 0x7f: {"name": "MP_BC_LOAD_CONST_SMALL_INT_MULTI -1", - "format": "MP_OPCODE_BYTE"}, - 0x80: {"name": "MP_BC_LOAD_CONST_SMALL_INT_MULTI 0", - "format": "MP_OPCODE_BYTE"}, - 0x81: {"name": "MP_BC_LOAD_CONST_SMALL_INT_MULTI 1", - "format": "MP_OPCODE_BYTE"}, - 0x82: {"name": "MP_BC_LOAD_CONST_SMALL_INT_MULTI 2", - "format": "MP_OPCODE_BYTE"}, - 0x83: {"name": "MP_BC_LOAD_CONST_SMALL_INT_MULTI 3", - "format": "MP_OPCODE_BYTE"}, - 0x84: {"name": "MP_BC_LOAD_CONST_SMALL_INT_MULTI 4", - "format": "MP_OPCODE_BYTE"}, -#define MP_BC_LOAD_FAST_MULTI (0xb0) // + N(16) - 0xb0: {"name": "MP_BC_LOAD_FAST_MULTI 0", - "format": "MP_OPCODE_BYTE"}, - 0xb1: {"name": "MP_BC_LOAD_FAST_MULTI 1", - "format": "MP_OPCODE_BYTE"}, - 0xb2: {"name": "MP_BC_LOAD_FAST_MULTI 2", - "format": "MP_OPCODE_BYTE"}, - 0xb3: {"name": "MP_BC_LOAD_FAST_MULTI 3", - "format": "MP_OPCODE_BYTE"}, - 0xb4: {"name": "MP_BC_LOAD_FAST_MULTI 4", - "format": "MP_OPCODE_BYTE"}, - 0xb5: {"name": "MP_BC_LOAD_FAST_MULTI 5", - "format": "MP_OPCODE_BYTE"}, - 0xb6: {"name": "MP_BC_LOAD_FAST_MULTI 6", - "format": "MP_OPCODE_BYTE"}, - 0xb7: {"name": "MP_BC_LOAD_FAST_MULTI 7", - "format": "MP_OPCODE_BYTE"}, - 0xb8: {"name": "MP_BC_LOAD_FAST_MULTI 8", - "format": "MP_OPCODE_BYTE"}, -#define MP_BC_STORE_FAST_MULTI (0xc0) // + N(16) - 0xc0: {"name": "MP_BC_STORE_FAST_MULTI 0", - "format": "MP_OPCODE_BYTE"}, - 0xc1: {"name": "MP_BC_STORE_FAST_MULTI 1", - "format": "MP_OPCODE_BYTE"}, - 0xc2: {"name": "MP_BC_STORE_FAST_MULTI 2", - "format": "MP_OPCODE_BYTE"}, - 0xc3: {"name": "MP_BC_STORE_FAST_MULTI 3", - "format": "MP_OPCODE_BYTE"}, - 0xc4: {"name": "MP_BC_STORE_FAST_MULTI 4", - "format": "MP_OPCODE_BYTE"}, - 0xc5: {"name": "MP_BC_STORE_FAST_MULTI 5", - "format": "MP_OPCODE_BYTE"}, - 0xc6: {"name": "MP_BC_STORE_FAST_MULTI 6", - "format": "MP_OPCODE_BYTE"}, - 0xc7: {"name": "MP_BC_STORE_FAST_MULTI 7", - "format": "MP_OPCODE_BYTE"}, -#define MP_BC_UNARY_OP_MULTI (0xd0) // + op(<MP_UNARY_OP_NUM_BYTECODE) - + 0x00: {"name": "MP_BC_LOAD_FAST_MULTI", "format": "MP_OPCODE_BYTE"}, + 0x10: {"name": "MP_BC_LOAD_CONST_FALSE", "format": "MP_OPCODE_BYTE"}, + 0x11: {"name": "MP_BC_LOAD_CONST_NONE", "format": "MP_OPCODE_BYTE"}, + 0x12: {"name": "MP_BC_LOAD_CONST_TRUE", "format": "MP_OPCODE_BYTE"}, + 0x14: {"name": "MP_BC_LOAD_CONST_SMALL_INT", "format": "MP_OPCODE_VAR_UINT"}, + 0x16: {"name": "MP_BC_LOAD_CONST_STRING", "format": "MP_OPCODE_QSTR"}, + 0x17: {"name": "MP_BC_LOAD_CONST_OBJ", "format": "MP_OPCODE_VAR_UINT"}, + # define MP_BC_LOAD_CONST_OBJ (0x17) // ptr + 0x18: {"name": "MP_BC_LOAD_NULL", "format": "MP_OPCODE_BYTE"}, + # define MP_BC_LOAD_FAST_N (0x19) // uint + 0x1A: {"name": "MP_BC_LOAD_DEREF", "format": "MP_OPCODE_VAR_UINT"}, + 0x1B: {"name": "MP_BC_LOAD_NAME", "format": "MP_OPCODE_QSTR"}, + 0x1C: {"name": "MP_BC_LOAD_GLOBAL", "format": "MP_OPCODE_QSTR"}, + 0x1D: {"name": "MP_BC_LOAD_ATTR", "format": "MP_OPCODE_QSTR"}, + 0x1E: {"name": "MP_BC_LOAD_METHOD", "format": "MP_OPCODE_QSTR"}, + 0x1F: {"name": "MP_BC_LOAD_SUPER_METHOD", "format": "MP_OPCODE_QSTR"}, + 0x20: {"name": "MP_BC_LOAD_BUILD_CLASS", "format": "MP_OPCODE_BYTE"}, + # define MP_BC_LOAD_BUILD_CLASS (0x20) + # define MP_BC_LOAD_SUBSCR (0x21) + 0x21: {"name": "MP_BC_LOAD_SUBSCR", "format": "MP_OPCODE_BYTE"}, + # define MP_BC_STORE_FAST_N (0x22) // uint + # define MP_BC_STORE_DEREF (0x23) // uint + # define MP_BC_STORE_NAME (0x24) // qstr + 0x24: {"name": "MP_BC_STORE_NAME", "format": "MP_OPCODE_QSTR"}, + 0x25: {"name": "MP_BC_STORE_GLOBAL", "format": "MP_OPCODE_QSTR"}, + 0x26: {"name": "MP_BC_STORE_ATTR", "format": "MP_OPCODE_QSTR"}, + 0x27: {"name": "MP_BC_LOAD_SUBSCR", "format": "MP_OPCODE_BYTE"}, + 0x28: {"name": "MP_BC_DELETE_FAST", "format": "MP_OPCODE_VAR_UINT"}, + # define MP_BC_DELETE_FAST (0x28) // uint + # define MP_BC_DELETE_DEREF (0x29) // uint + # define MP_BC_DELETE_NAME (0x2a) // qstr + # define MP_BC_DELETE_GLOBAL (0x2b) // qstr + 0x30: {"name": "MP_BC_DUP_TOP", "format": "MP_OPCODE_BYTE"}, + # define MP_BC_DUP_TOP_TWO (0x31) + 0x32: {"name": "MP_BC_POP_TOP", "format": "MP_OPCODE_BYTE"}, + 0x33: {"name": "MP_BC_ROT_TWO", "format": "MP_OPCODE_BYTE"}, + 0x34: {"name": "MP_BC_ROT_THREE", "format": "MP_OPCODE_BYTE"}, + 0x35: {"name": "MP_BC_JUMP", "format": "MP_OPCODE_OFFSET"}, + 0x36: {"name": "MP_BC_POP_JUMP_IF_TRUE", "format": "MP_OPCODE_OFFSET"}, + 0x37: {"name": "MP_BC_POP_JUMP_IF_FALSE", "format": "MP_OPCODE_OFFSET"}, + # define MP_BC_JUMP_IF_TRUE_OR_POP (0x38) // rel byte code offset, 16-bit signed, in excess + # define MP_BC_JUMP_IF_FALSE_OR_POP (0x39) // rel byte code offset, 16-bit signed, in excess + # define MP_BC_SETUP_WITH (0x3d) // rel byte code offset, 16-bit unsigned + # define MP_BC_WITH_CLEANUP (0x3e) + # define MP_BC_SETUP_EXCEPT (0x3f) // rel byte code offset, 16-bit unsigned + # define MP_BC_SETUP_FINALLY (0x40) // rel byte code offset, 16-bit unsigned + # define MP_BC_END_FINALLY (0x41) + # define MP_BC_GET_ITER (0x42) + # define MP_BC_FOR_ITER (0x43) // rel byte code offset, 16-bit unsigned + 0x43: {"name": "MP_BC_FOR_ITER", "format": "MP_OPCODE_OFFSET"}, + 0x44: {"name": "MP_BC_POP_BLOCK", "format": "MP_OPCODE_BYTE"}, + # define MP_BC_POP_EXCEPT (0x45) + # define MP_BC_UNWIND_JUMP (0x46) // rel byte code offset, 16-bit signed, in excess; then a byte + 0x47: {"name": "MP_BC_GET_ITER_STACK", "format": "MP_OPCODE_BYTE"}, + 0x50: {"name": "MP_BC_BUILD_TUPLE", "format": "MP_OPCODE_VAR_UINT"}, + 0x51: {"name": "MP_BC_BUILD_LIST", "format": "MP_OPCODE_VAR_UINT"}, + 0x53: {"name": "MP_BC_BUILD_MAP", "format": "MP_OPCODE_VAR_UINT"}, + 0x54: {"name": "MP_BC_STORE_MAP", "format": "MP_OPCODE_BYTE"}, + # define MP_BC_BUILD_SET (0x56) // uint + # define MP_BC_BUILD_SLICE (0x58) // uint + # define MP_BC_STORE_COMP (0x57) // uint + 0x57: {"name": "MP_BC_STORE_COMP", "format": "MP_OPCODE_VAR_UINT"}, + # define MP_BC_UNPACK_SEQUENCE (0x59) // uint + # define MP_BC_UNPACK_EX (0x5a) // uint + 0x5B: {"name": "MP_BC_RETURN_VALUE", "format": "MP_OPCODE_BYTE"}, + 0x5C: {"name": "MP_BC_RAISE_VARARGS", "format": "MP_OPCODE_BYTE_EXTRA"}, + # define MP_BC_YIELD_VALUE (0x5d) + # define MP_BC_YIELD_FROM (0x5e) + # define MP_BC_MAKE_FUNCTION (0x60) // uint + 0x60: {"name": "MP_BC_MAKE_FUNCTION", "format": "MP_OPCODE_VAR_UINT"}, + 0x61: {"name": "MP_BC_MAKE_FUNCTION_DEFARGS", "format": "MP_OPCODE_VAR_UINT"}, + 0x62: {"name": "MP_BC_MAKE_CLOSURE", "format": "MP_OPCODE_VAR_UINT_EXTRA"}, + 0x63: {"name": "MP_BC_MAKE_CLOSURE", "format": "MP_OPCODE_VAR_UINT_EXTRA"}, + 0x64: {"name": "MP_BC_CALL_FUNCTION", "format": "MP_OPCODE_VAR_UINT"}, + 0x65: {"name": "MP_BC_CALL_FUNCTION_VAR_KW", "format": "MP_OPCODE_VAR_UINT"}, + 0x66: {"name": "MP_BC_CALL_METHOD", "format": "MP_OPCODE_VAR_UINT"}, + 0x67: {"name": "MP_BC_CALL_METHOD_VAR_KW", "format": "MP_OPCODE_VAR_UINT"}, + 0x68: {"name": "MP_BC_IMPORT_NAME", "format": "MP_OPCODE_QSTR"}, + 0x69: {"name": "MP_BC_IMPORT_FROM", "format": "MP_OPCODE_QSTR"}, + # define MP_BC_IMPORT_FROM (0x69) // qstr + # define MP_BC_IMPORT_STAR (0x6a) + # define MP_BC_LOAD_CONST_SMALL_INT_MULTI (0x70) // + N(64) + 0x7F: {"name": "MP_BC_LOAD_CONST_SMALL_INT_MULTI -1", "format": "MP_OPCODE_BYTE"}, + 0x80: {"name": "MP_BC_LOAD_CONST_SMALL_INT_MULTI 0", "format": "MP_OPCODE_BYTE"}, + 0x81: {"name": "MP_BC_LOAD_CONST_SMALL_INT_MULTI 1", "format": "MP_OPCODE_BYTE"}, + 0x82: {"name": "MP_BC_LOAD_CONST_SMALL_INT_MULTI 2", "format": "MP_OPCODE_BYTE"}, + 0x83: {"name": "MP_BC_LOAD_CONST_SMALL_INT_MULTI 3", "format": "MP_OPCODE_BYTE"}, + 0x84: {"name": "MP_BC_LOAD_CONST_SMALL_INT_MULTI 4", "format": "MP_OPCODE_BYTE"}, + # define MP_BC_LOAD_FAST_MULTI (0xb0) // + N(16) + 0xB0: {"name": "MP_BC_LOAD_FAST_MULTI 0", "format": "MP_OPCODE_BYTE"}, + 0xB1: {"name": "MP_BC_LOAD_FAST_MULTI 1", "format": "MP_OPCODE_BYTE"}, + 0xB2: {"name": "MP_BC_LOAD_FAST_MULTI 2", "format": "MP_OPCODE_BYTE"}, + 0xB3: {"name": "MP_BC_LOAD_FAST_MULTI 3", "format": "MP_OPCODE_BYTE"}, + 0xB4: {"name": "MP_BC_LOAD_FAST_MULTI 4", "format": "MP_OPCODE_BYTE"}, + 0xB5: {"name": "MP_BC_LOAD_FAST_MULTI 5", "format": "MP_OPCODE_BYTE"}, + 0xB6: {"name": "MP_BC_LOAD_FAST_MULTI 6", "format": "MP_OPCODE_BYTE"}, + 0xB7: {"name": "MP_BC_LOAD_FAST_MULTI 7", "format": "MP_OPCODE_BYTE"}, + 0xB8: {"name": "MP_BC_LOAD_FAST_MULTI 8", "format": "MP_OPCODE_BYTE"}, + # define MP_BC_STORE_FAST_MULTI (0xc0) // + N(16) + 0xC0: {"name": "MP_BC_STORE_FAST_MULTI 0", "format": "MP_OPCODE_BYTE"}, + 0xC1: {"name": "MP_BC_STORE_FAST_MULTI 1", "format": "MP_OPCODE_BYTE"}, + 0xC2: {"name": "MP_BC_STORE_FAST_MULTI 2", "format": "MP_OPCODE_BYTE"}, + 0xC3: {"name": "MP_BC_STORE_FAST_MULTI 3", "format": "MP_OPCODE_BYTE"}, + 0xC4: {"name": "MP_BC_STORE_FAST_MULTI 4", "format": "MP_OPCODE_BYTE"}, + 0xC5: {"name": "MP_BC_STORE_FAST_MULTI 5", "format": "MP_OPCODE_BYTE"}, + 0xC6: {"name": "MP_BC_STORE_FAST_MULTI 6", "format": "MP_OPCODE_BYTE"}, + 0xC7: {"name": "MP_BC_STORE_FAST_MULTI 7", "format": "MP_OPCODE_BYTE"}, + # define MP_BC_UNARY_OP_MULTI (0xd0) // + op(<MP_UNARY_OP_NUM_BYTECODE) # // 9 relational operations, should return a bool - 0xd7: {"name": "MP_BC_BINARY_OP_MULTI MP_BINARY_OP_LESS", - "format": "MP_OPCODE_BYTE"}, - 0xd8: {"name": "MP_BC_BINARY_OP_MULTI MP_BINARY_OP_MORE", - "format": "MP_OPCODE_BYTE"}, - 0xd9: {"name": "MP_BC_BINARY_OP_MULTI MP_BINARY_OP_EQUAL", - "format": "MP_OPCODE_BYTE"}, - 0xda: {"name": "MP_BC_BINARY_OP_MULTI MP_BINARY_OP_LESS_EQUAL", - "format": "MP_OPCODE_BYTE"}, - 0xdb: {"name": "MP_BC_BINARY_OP_MULTI MP_BINARY_OP_MORE_EQUAL", - "format": "MP_OPCODE_BYTE"}, - 0xdc: {"name": "MP_BC_BINARY_OP_MULTI MP_BINARY_OP_NOT_EQUAL", - "format": "MP_OPCODE_BYTE"}, + 0xD7: {"name": "MP_BC_BINARY_OP_MULTI MP_BINARY_OP_LESS", "format": "MP_OPCODE_BYTE"}, + 0xD8: {"name": "MP_BC_BINARY_OP_MULTI MP_BINARY_OP_MORE", "format": "MP_OPCODE_BYTE"}, + 0xD9: {"name": "MP_BC_BINARY_OP_MULTI MP_BINARY_OP_EQUAL", "format": "MP_OPCODE_BYTE"}, + 0xDA: {"name": "MP_BC_BINARY_OP_MULTI MP_BINARY_OP_LESS_EQUAL", "format": "MP_OPCODE_BYTE"}, + 0xDB: {"name": "MP_BC_BINARY_OP_MULTI MP_BINARY_OP_MORE_EQUAL", "format": "MP_OPCODE_BYTE"}, + 0xDC: {"name": "MP_BC_BINARY_OP_MULTI MP_BINARY_OP_NOT_EQUAL", "format": "MP_OPCODE_BYTE"}, # dc: MP_BINARY_OP_NOT_EQUAL, # dd: MP_BINARY_OP_IN, # de: MP_BINARY_OP_IS, @@ -235,10 +145,11 @@ bytecodes = { # e3: MP_BINARY_OP_INPLACE_LSHIFT, # e4: MP_BINARY_OP_INPLACE_RSHIFT, # e5: MP_BINARY_OP_INPLACE_ADD, - 0xe5: {"name": "MP_BC_BINARY_OP_MULTI MP_BINARY_OP_INPLACE_ADD", - "format": "MP_OPCODE_BYTE"}, - 0xe6: {"name": "MP_BC_BINARY_OP_MULTI MP_BINARY_OP_INPLACE_SUBTRACT", - "format": "MP_OPCODE_BYTE"}, + 0xE5: {"name": "MP_BC_BINARY_OP_MULTI MP_BINARY_OP_INPLACE_ADD", "format": "MP_OPCODE_BYTE"}, + 0xE6: { + "name": "MP_BC_BINARY_OP_MULTI MP_BINARY_OP_INPLACE_SUBTRACT", + "format": "MP_OPCODE_BYTE", + }, # e7: MP_BINARY_OP_INPLACE_MULTIPLY, # e8: MP_BINARY_OP_INPLACE_FLOOR_DIVIDE, # e9: MP_BINARY_OP_INPLACE_TRUE_DIVIDE, @@ -251,23 +162,17 @@ bytecodes = { # ee: MP_BINARY_OP_AND, # ef: MP_BINARY_OP_LSHIFT, # f0: MP_BINARY_OP_RSHIFT, -#define MP_BC_BINARY_OP_MULTI (0xd7) // + op(<MP_BINARY_OP_NUM_BYTECODE) - 0xf1: {"name": "MP_BC_BINARY_OP_MULTI MP_BINARY_OP_ADD", - "format": "MP_OPCODE_BYTE"}, - 0xf2: {"name": "MP_BC_BINARY_OP_MULTI MP_BINARY_OP_SUBTRACT", - "format": "MP_OPCODE_BYTE"}, - 0xf3: {"name": "MP_BC_BINARY_OP_MULTI MP_BINARY_OP_MULTIPLY", - "format": "MP_OPCODE_BYTE"}, - 0xf4: {"name": "MP_BC_BINARY_OP_MULTI MP_BINARY_OP_FLOOR_DIVIDE", - "format": "MP_OPCODE_BYTE"}, - 0xf5: {"name": "MP_BC_BINARY_OP_MULTI MP_BINARY_OP_TRUE_DIVIDE", - "format": "MP_OPCODE_BYTE"}, - 0xf6: {"name": "MP_BC_BINARY_OP_MULTI MP_BINARY_OP_MODULO", - "format": "MP_OPCODE_BYTE"}, - 0xf7: {"name": "MP_BC_BINARY_OP_MULTI MP_BINARY_OP_POWER", - "format": "MP_OPCODE_BYTE"}, + # define MP_BC_BINARY_OP_MULTI (0xd7) // + op(<MP_BINARY_OP_NUM_BYTECODE) + 0xF1: {"name": "MP_BC_BINARY_OP_MULTI MP_BINARY_OP_ADD", "format": "MP_OPCODE_BYTE"}, + 0xF2: {"name": "MP_BC_BINARY_OP_MULTI MP_BINARY_OP_SUBTRACT", "format": "MP_OPCODE_BYTE"}, + 0xF3: {"name": "MP_BC_BINARY_OP_MULTI MP_BINARY_OP_MULTIPLY", "format": "MP_OPCODE_BYTE"}, + 0xF4: {"name": "MP_BC_BINARY_OP_MULTI MP_BINARY_OP_FLOOR_DIVIDE", "format": "MP_OPCODE_BYTE"}, + 0xF5: {"name": "MP_BC_BINARY_OP_MULTI MP_BINARY_OP_TRUE_DIVIDE", "format": "MP_OPCODE_BYTE"}, + 0xF6: {"name": "MP_BC_BINARY_OP_MULTI MP_BINARY_OP_MODULO", "format": "MP_OPCODE_BYTE"}, + 0xF7: {"name": "MP_BC_BINARY_OP_MULTI MP_BINARY_OP_POWER", "format": "MP_OPCODE_BYTE"}, } + def read_uint(encoded_uint, peek=False): unum = 0 i = 0 @@ -276,21 +181,23 @@ def read_uint(encoded_uint, peek=False): b = encoded_uint.peek()[i] else: b = encoded_uint.read(1)[0] - unum = (unum << 7) | (b & 0x7f) + unum = (unum << 7) | (b & 0x7F) if (b & 0x80) == 0: break i += 1 return unum + class Prelude: def __init__(self, encoded_prelude): - self.n_state = read_uint(encoded_prelude) - self.n_exc_stack = read_uint(encoded_prelude) - self.scope_flags = encoded_prelude.read(1)[0] - self.n_pos_args = encoded_prelude.read(1)[0] - self.n_kwonly_args = encoded_prelude.read(1)[0] - self.n_def_pos_args = encoded_prelude.read(1)[0] - self.code_info_size = read_uint(encoded_prelude, peek=True) + self.n_state = read_uint(encoded_prelude) + self.n_exc_stack = read_uint(encoded_prelude) + self.scope_flags = encoded_prelude.read(1)[0] + self.n_pos_args = encoded_prelude.read(1)[0] + self.n_kwonly_args = encoded_prelude.read(1)[0] + self.n_def_pos_args = encoded_prelude.read(1)[0] + self.code_info_size = read_uint(encoded_prelude, peek=True) + class RawCode: # mp_raw_code_kind_t kind : 3; @@ -319,20 +226,20 @@ class RawCode: prelude = Prelude(bc) encoded_code_info = bc.read(prelude.code_info_size) bc.read(1) - while bc.peek(1)[0] == 0xff: + while bc.peek(1)[0] == 0xFF: bc.read(1) bc = bytearray(bc.read()) - #print(encoded_code_info, bc) + # print(encoded_code_info, bc) self.qstrs = [] self.simple_name = self._load_qstr(encoded_raw_code) self.source_file = self._load_qstr(encoded_raw_code) # the simple name and source file qstr indexes get written back into the byte code somehow - #print(bc) + # print(bc) self._load_bytecode_qstrs(encoded_raw_code, bc) - #print(encoded_raw_code.peek(20)[:20]) + # print(encoded_raw_code.peek(20)[:20]) n_obj = read_uint(encoded_raw_code) n_raw_code = read_uint(encoded_raw_code) @@ -349,7 +256,7 @@ class RawCode: self.const_table.append(RawCode(encoded_raw_code)) print(self.qstrs[self.simple_name], self.qstrs[self.source_file]) - #print(binascii.hexlify(encoded_raw_code.peek(20)[:20])) + # print(binascii.hexlify(encoded_raw_code.peek(20)[:20])) def _load_qstr(self, encoded_qstr): string_len = read_uint(encoded_qstr) @@ -363,20 +270,20 @@ class RawCode: def _load_obj(self, encoded_obj): obj_type = encoded_obj.read(1) - if obj_type == b'e': + if obj_type == b"e": return "..." else: str_len = read_uint(encoded_obj) s = encoded_obj.read(str_len) - if obj_type == b's': + if obj_type == b"s": return s.decode("utf-8") - elif obj_type == b'b': + elif obj_type == b"b": return s - elif obj_type == b'i': + elif obj_type == b"i": return int(s) - elif obj_type == b'f': + elif obj_type == b"f": return float(s) - elif obj_type == b'c': + elif obj_type == b"c": return float(s) raise RuntimeError("Unknown object type {}".format(obj_type)) @@ -393,8 +300,8 @@ class RawCode: opcode_size = bytecode_format_sizes[bc["format"]] if bc["format"] == "MP_OPCODE_QSTR": qstr_index = self._load_qstr(encoded_raw_code) - bytecode[i+1] = qstr_index - bytecode[i+2] = qstr_index >> 8 + bytecode[i + 1] = qstr_index + bytecode[i + 2] = qstr_index >> 8 if not opcode_size: i += 2 while (bytecode[i] & 0x80) != 0: @@ -404,17 +311,19 @@ class RawCode: else: i += opcode_size + class mpyFile: def __init__(self, encoded_mpy): # this matches mp-raw_code_save in py/persistentcode.c first_byte = encoded_mpy.read(1) - if first_byte != b'M': + if first_byte != b"M": raise ValueError("Not a valid first byte. Should be 'M' but is {}".format(first_byte)) self.version = encoded_mpy.read(1)[0] self.feature_flags = encoded_mpy.read(1)[0] self.small_int_bits = encoded_mpy.read(1)[0] self.raw_code = RawCode(encoded_mpy) + if __name__ == "__main__": with open(sys.argv[1], "rb") as f: mpy = mpyFile(f) diff --git a/tools/build_board_info.py b/tools/build_board_info.py index 70d5e2bc7..30246bd17 100644 --- a/tools/build_board_info.py +++ b/tools/build_board_info.py @@ -125,9 +125,7 @@ def get_version_info(): version = None sha = git("rev-parse", "--short", "HEAD").stdout.decode("utf-8") try: - version = ( - git("describe", "--tags", "--exact-match").stdout.decode("utf-8").strip() - ) + version = git("describe", "--tags", "--exact-match").stdout.decode("utf-8").strip() except sh.ErrorReturnCode_128: # No exact match pass @@ -191,9 +189,7 @@ def create_pr(changes, updated, git_info, user): ) create_branch = {"ref": "refs/heads/" + branch_name, "sha": commit_sha} - response = github.post( - "/repos/{}/circuitpython-org/git/refs".format(user), json=create_branch - ) + response = github.post("/repos/{}/circuitpython-org/git/refs".format(user), json=create_branch) if not response.ok and response.json()["message"] != "Reference already exists": print("unable to create branch") print(response.text) @@ -207,8 +203,7 @@ def create_pr(changes, updated, git_info, user): } response = github.put( - "/repos/{}/circuitpython-org/contents/_data/files.json".format(user), - json=update_file, + "/repos/{}/circuitpython-org/contents/_data/files.json".format(user), json=update_file ) if not response.ok: print("unable to post new file") @@ -257,9 +252,7 @@ def generate_download_info(): languages = get_languages() - support_matrix = shared_bindings_matrix.support_matrix_by_board( - use_branded_name=False - ) + support_matrix = shared_bindings_matrix.support_matrix_by_board(use_branded_name=False) new_stable = "-" not in new_tag diff --git a/tools/build_memory_info.py b/tools/build_memory_info.py index 89aedf0f4..722c548a1 100644 --- a/tools/build_memory_info.py +++ b/tools/build_memory_info.py @@ -9,11 +9,11 @@ import re import sys # Handle size constants with K or M suffixes (allowed in .ld but not in Python). -K_PATTERN = re.compile(r'([0-9]+)[kK]') -K_REPLACE = r'(\1*1024)' +K_PATTERN = re.compile(r"([0-9]+)[kK]") +K_REPLACE = r"(\1*1024)" -M_PATTERN = re.compile(r'([0-9]+)[mM]') -M_REPLACE = r'(\1*1024*1024)' +M_PATTERN = re.compile(r"([0-9]+)[mM]") +M_REPLACE = r"(\1*1024*1024)" print() @@ -51,8 +51,16 @@ used_flash = data + text free_flash = firmware_region - used_flash used_ram = data + bss free_ram = ram_region - used_ram -print("{} bytes used, {} bytes free in flash firmware space out of {} bytes ({}kB).".format(used_flash, free_flash, firmware_region, firmware_region / 1024)) -print("{} bytes used, {} bytes free in ram for stack and heap out of {} bytes ({}kB).".format(used_ram, free_ram, ram_region, ram_region / 1024)) +print( + "{} bytes used, {} bytes free in flash firmware space out of {} bytes ({}kB).".format( + used_flash, free_flash, firmware_region, firmware_region / 1024 + ) +) +print( + "{} bytes used, {} bytes free in ram for stack and heap out of {} bytes ({}kB).".format( + used_ram, free_ram, ram_region, ram_region / 1024 + ) +) print() # Check that we have free flash space. GCC doesn't fail when the text + data diff --git a/tools/build_release_files.py b/tools/build_release_files.py index dc6c094c0..4e22e9725 100755 --- a/tools/build_release_files.py +++ b/tools/build_release_files.py @@ -17,7 +17,7 @@ for port in build_info.SUPPORTED_PORTS: PARALLEL = "-j 5" if "GITHUB_ACTION" in os.environ: - PARALLEL="-j 2" + PARALLEL = "-j 2" all_boards = build_info.get_board_mapping() build_boards = list(all_boards.keys()) @@ -27,12 +27,27 @@ if "BOARDS" in os.environ: sha, version = build_info.get_version_info() languages = build_info.get_languages() -language_allow_list = ['ID', 'de_DE', 'en_US', 'en_x_pirate', 'es', 'fil', 'fr', 'it_IT', 'ja', 'nl', 'pl', 'pt_BR', 'sv', 'zh_Latn_pinyin'] -print('Note: Not building languages', set(languages) - set(language_allow_list)) +language_allow_list = [ + "ID", + "de_DE", + "en_US", + "en_x_pirate", + "es", + "fil", + "fr", + "it_IT", + "ja", + "nl", + "pl", + "pt_BR", + "sv", + "zh_Latn_pinyin", +] +print("Note: Not building languages", set(languages) - set(language_allow_list)) exit_status = 0 cores = multiprocessing.cpu_count() -print('building boards with parallelism {}'.format(cores)) +print("building boards with parallelism {}".format(cores)) for board in build_boards: bin_directory = "../bin/{}/".format(board) os.makedirs(bin_directory, exist_ok=True) @@ -48,8 +63,12 @@ for board in build_boards: # CFLAGS_INLINE_LIMIT is set for a particular language to make it fit. clean_build_check_result = subprocess.run( "make -C ../ports/{port} TRANSLATION={language} BOARD={board} check-release-needs-clean-build -j {cores} | fgrep 'RELEASE_NEEDS_CLEAN_BUILD = 1'".format( - port = board_info["port"], language=language, board=board, cores=cores), - shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + port=board_info["port"], language=language, board=board, cores=cores + ), + shell=True, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + ) clean_build = clean_build_check_result.returncode == 0 build_dir = "build-{board}".format(board=board) @@ -58,8 +77,16 @@ for board in build_boards: make_result = subprocess.run( "make -C ../ports/{port} TRANSLATION={language} BOARD={board} BUILD={build} -j {cores}".format( - port = board_info["port"], language=language, board=board, build=build_dir, cores=cores), - shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + port=board_info["port"], + language=language, + board=board, + build=build_dir, + cores=cores, + ), + shell=True, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + ) build_duration = time.monotonic() - start_time success = "\033[32msucceeded\033[0m" @@ -71,13 +98,14 @@ for board in build_boards: for extension in board_info["extensions"]: temp_filename = "../ports/{port}/{build}/firmware.{extension}".format( - port=board_info["port"], build=build_dir, extension=extension) + port=board_info["port"], build=build_dir, extension=extension + ) for alias in board_info["aliases"] + [board]: - bin_directory = "../bin/{alias}/{language}".format( - alias=alias, language=language) + bin_directory = "../bin/{alias}/{language}".format(alias=alias, language=language) os.makedirs(bin_directory, exist_ok=True) final_filename = "adafruit-circuitpython-{alias}-{language}-{version}.{extension}".format( - alias=alias, language=language, version=version, extension=extension) + alias=alias, language=language, version=version, extension=extension + ) final_filename = os.path.join(bin_directory, final_filename) try: shutil.copyfile(temp_filename, final_filename) @@ -86,9 +114,15 @@ for board in build_boards: if exit_status == 0: exit_status = 1 - print("Build {board} for {language}{clean_build} took {build_duration:.2f}s and {success}".format( - board=board, language=language, clean_build=(" (clean_build)" if clean_build else ""), - build_duration=build_duration, success=success)) + print( + "Build {board} for {language}{clean_build} took {build_duration:.2f}s and {success}".format( + board=board, + language=language, + clean_build=(" (clean_build)" if clean_build else ""), + build_duration=build_duration, + success=success, + ) + ) print(make_result.stdout.decode("utf-8")) print(other_output) diff --git a/tools/chart_code_size.py b/tools/chart_code_size.py index f75269130..2328edd72 100644 --- a/tools/chart_code_size.py +++ b/tools/chart_code_size.py @@ -12,15 +12,15 @@ import sh # Replace dashes with underscores objdump = sh.arm_none_eabi_objdump + def parse_hex(h): return int("0x" + h, 0) + BAD_JUMPS = ["UNPREDICTABLE", "_etext"] -SPECIAL_NODE_COLORS = { - "main": "pink", - "exception_table": "green" -} +SPECIAL_NODE_COLORS = {"main": "pink", "exception_table": "green"} + @click.command() @click.argument("elf_filename") @@ -60,7 +60,17 @@ def do_all_the_things(elf_filename): pass else: symbols_by_memory_address[symbol["start_address"]] = symbol - elif symbol["debug_type"] in ["DW_TAG_member", "DW_TAG_label", "DW_TAG_typedef", "DW_TAG_enumerator", "DW_TAG_enumeration_type", "DW_TAG_base_type", "DW_TAG_structure_type", "DW_TAG_compile_unit", "DW_TAG_union_type"]: + elif symbol["debug_type"] in [ + "DW_TAG_member", + "DW_TAG_label", + "DW_TAG_typedef", + "DW_TAG_enumerator", + "DW_TAG_enumeration_type", + "DW_TAG_base_type", + "DW_TAG_structure_type", + "DW_TAG_compile_unit", + "DW_TAG_union_type", + ]: # skip symbols that don't end up in memory. the type info is available through the debug address map pass else: @@ -71,7 +81,11 @@ def do_all_the_things(elf_filename): # print() pass all_symbols[symbol["name"]] = symbol - elif symbol and symbol["debug_type"] == "DW_TAG_GNU_call_site_parameter" and "call_site_value" in symbol: + elif ( + symbol + and symbol["debug_type"] == "DW_TAG_GNU_call_site_parameter" + and "call_site_value" in symbol + ): parent = -1 while symbol_stack[parent]["debug_type"] != "DW_TAG_subprogram": parent -= 1 @@ -144,10 +158,10 @@ def do_all_the_things(elf_filename): symbol["call_site_value"] = parse_hex(parts[-1].strip(")")) else: symbol["other"].append(line) - #print(parts) + # print(parts) pass else: - #print(line) + # print(line) pass MEMORY_NONE = 0 @@ -162,10 +176,16 @@ def do_all_the_things(elf_filename): def get_pointer_map(t, depth=0): if t["debug_type"] == "DW_TAG_pointer_type": return {0: MEMORY_POINTER} - elif t["debug_type"] in ["DW_TAG_const_type", "DW_TAG_typedef", "DW_TAG_member", "DW_TAG_subrange_type", "DW_TAG_volatile_type"]: + elif t["debug_type"] in [ + "DW_TAG_const_type", + "DW_TAG_typedef", + "DW_TAG_member", + "DW_TAG_subrange_type", + "DW_TAG_volatile_type", + ]: if "name" in t and t["name"] == "mp_rom_obj_t": return {0: MEMORY_PY_OBJECT} - return get_pointer_map(symbols_by_debug_address[t["type"]], depth+1) + return get_pointer_map(symbols_by_debug_address[t["type"]], depth + 1) elif t["debug_type"] in ["DW_TAG_base_type", "DW_TAG_enumeration_type"]: return {} elif t["debug_type"] == "DW_TAG_union_type": @@ -181,7 +201,7 @@ def do_all_the_things(elf_filename): return combined_map elif "subtype" in t: subtype = symbols_by_debug_address[t["type"]] - pmap = get_pointer_map(subtype, depth+1) + pmap = get_pointer_map(subtype, depth + 1) size = get_size(subtype) expanded_map = {} for i in range(t["maxlen"]): @@ -216,11 +236,11 @@ def do_all_the_things(elf_filename): elif t_symbol["debug_type"] == "DW_TAG_volatile_type": type_string.append("volatile") else: - #print(" ", t_symbol) + # print(" ", t_symbol) pass type_string.reverse() symbol["type_string"] = " ".join(type_string) - #print(symbol_name, symbol["debug_type"], symbol.get("type_string", "")) + # print(symbol_name, symbol["debug_type"], symbol.get("type_string", "")) # print() # print() @@ -263,13 +283,17 @@ def do_all_the_things(elf_filename): elif symbol_name not in all_symbols: if symbol_name == "nlr_push_tail_var": fake_type = all_symbols["mp_obj_get_type"]["type"] - symbol = {"debug_type": "DW_TAG_variable", "name": symbol_name, "type": fake_type} + symbol = { + "debug_type": "DW_TAG_variable", + "name": symbol_name, + "type": fake_type, + } else: print(line) print(symbol_name, symbol_address) symbol = {"debug_type": "DW_TAG_subprogram", "name": symbol_name} all_symbols[symbol_name] = symbol - #raise RuntimeError() + # raise RuntimeError() symbol = all_symbols[symbol_name] symbol["start_address"] = symbol_address @@ -292,13 +316,13 @@ def do_all_the_things(elf_filename): offset = last_address - symbol["start_address"] if "pointer_map" in symbol: if offset not in symbol["pointer_map"]: - #print(offset, symbol) + # print(offset, symbol) pass else: ref = parse_hex(parts[1]) pointer_style = symbol["pointer_map"][offset] if pointer_style == MEMORY_POINTER: - symbol["outgoing_pointers"].add(ref & 0xfffffffe) + symbol["outgoing_pointers"].add(ref & 0xFFFFFFFE) elif pointer_style == MEMORY_PY_OBJECT and ref & 0x3 == 0: symbol["outgoing_pointers"].add(ref) if len(parts[1]) == 8 and parts[1][0] == "0": @@ -315,7 +339,7 @@ def do_all_the_things(elf_filename): print(symbol) if jump_to != symbol["name"] and jump_to not in BAD_JUMPS: symbol["outgoing_jumps"].add(jump_to) - #print(symbol_name, jump_to) + # print(symbol_name, jump_to) if jump_to == "_etext": print(line) elif "UNDEFINED" in line: @@ -325,7 +349,7 @@ def do_all_the_things(elf_filename): else: print(line) else: - #print(line) + # print(line) pass # print() @@ -344,7 +368,7 @@ def do_all_the_things(elf_filename): for outgoing in symbol["outgoing_pointers"]: if outgoing in symbols_by_memory_address: outgoing = symbols_by_memory_address[outgoing] - #print(outgoing) + # print(outgoing) if outgoing["debug_type"] in ["DW_TAG_GNU_call_site", "DW_TAG_lexical_block"]: continue if outgoing["name"] == "audioio_wavefile_type": @@ -359,25 +383,25 @@ def do_all_the_things(elf_filename): if "outgoing_jumps" in symbol: for outgoing in symbol["outgoing_jumps"]: if outgoing not in all_symbols: - #print(outgoing, symbol_name) + # print(outgoing, symbol_name) continue - #print(all_symbols[outgoing], symbol_name) + # print(all_symbols[outgoing], symbol_name) referenced_symbol = all_symbols[outgoing] if "incoming_jumps" not in referenced_symbol: - #print(symbol_name, "->", outgoing) + # print(symbol_name, "->", outgoing) referenced_symbol["incoming_jumps"] = set() referenced_symbol["incoming_jumps"].add(symbol_name) if "outgoing_pointers" in symbol: for outgoing in symbol["outgoing_pointers"]: if outgoing not in all_symbols: - #print(outgoing, symbol_name) + # print(outgoing, symbol_name) continue - #print(all_symbols[outgoing], symbol_name) + # print(all_symbols[outgoing], symbol_name) referenced_symbol = all_symbols[outgoing] if "incoming_pointers" not in referenced_symbol: - #print(symbol_name, "->", outgoing) + # print(symbol_name, "->", outgoing) referenced_symbol["incoming_pointers"] = set() referenced_symbol["incoming_pointers"].add(symbol_name) @@ -395,8 +419,10 @@ def do_all_the_things(elf_filename): # print(" ", len(symbol["outgoing_pointers"]), "ptrs") # if i > 3000: # break - if ("incoming_jumps" not in symbol or len(symbol["incoming_jumps"]) == 0) and ("incoming_pointers" not in symbol or len(symbol["incoming_pointers"]) == 0): - #print(symbol_name) + if ("incoming_jumps" not in symbol or len(symbol["incoming_jumps"]) == 0) and ( + "incoming_pointers" not in symbol or len(symbol["incoming_pointers"]) == 0 + ): + # print(symbol_name) continue if "start_address" not in symbol: continue @@ -407,7 +433,7 @@ def do_all_the_things(elf_filename): if "outgoing_pointers" in symbol: for outgoing in symbol["outgoing_pointers"]: callgraph.add_edge(symbol_name, outgoing, color="red") - #print(symbol_name, symbol) + # print(symbol_name, symbol) # Style all of the nodes print("styling") @@ -454,5 +480,6 @@ def do_all_the_things(elf_filename): print(fn) callgraph.draw(fn) + if __name__ == "__main__": do_all_the_things() diff --git a/tools/ci_check_duplicate_usb_vid_pid.py b/tools/ci_check_duplicate_usb_vid_pid.py index cb4efc5f1..33260eb39 100644 --- a/tools/ci_check_duplicate_usb_vid_pid.py +++ b/tools/ci_check_duplicate_usb_vid_pid.py @@ -46,7 +46,7 @@ DEFAULT_IGNORELIST = [ "cp32-m4", "metro_m4_express", "unexpectedmaker_feathers2", - "unexpectedmaker_feathers2_prerelease" + "unexpectedmaker_feathers2_prerelease", ] cli_parser = argparse.ArgumentParser(description="USB VID/PID Duplicate Checker") @@ -60,27 +60,28 @@ cli_parser.add_argument( "Board names to ignore duplicate VID/PID combinations. Pass an empty " "string to disable all duplicate ignoring. Defaults are: " f"{', '.join(DEFAULT_IGNORELIST)}" - ) + ), ) + def configboard_files(): - """ A pathlib glob search for all ports/*/boards/*/mpconfigboard.mk file - paths. + """A pathlib glob search for all ports/*/boards/*/mpconfigboard.mk file + paths. - :returns: A ``pathlib.Path.glob()`` genarator object + :returns: A ``pathlib.Path.glob()`` genarator object """ working_dir = pathlib.Path().resolve() if not working_dir.name.startswith("circuitpython"): raise RuntimeError( - "Please run USB VID/PID duplicate verification at the " - "top-level directory." + "Please run USB VID/PID duplicate verification at the " "top-level directory." ) return working_dir.glob("ports/**/boards/**/mpconfigboard.mk") + def check_vid_pid(files, ignorelist): - """ Compiles a list of USB VID & PID values for all boards, and checks - for duplicates. Exits with ``sys.exit()`` (non-zero exit code) - if duplicates are found, and lists the duplicates. + """Compiles a list of USB VID & PID values for all boards, and checks + for duplicates. Exits with ``sys.exit()`` (non-zero exit code) + if duplicates are found, and lists the duplicates. """ duplicates_found = False @@ -106,24 +107,18 @@ def check_vid_pid(files, ignorelist): if usb_vid and usb_pid: id_group = f"{usb_vid.group(1)}:{usb_pid.group(1)}" if id_group not in usb_ids: - usb_ids[id_group] = { - "boards": [board_name], - "duplicate": False - } + usb_ids[id_group] = {"boards": [board_name], "duplicate": False} else: - usb_ids[id_group]['boards'].append(board_name) + usb_ids[id_group]["boards"].append(board_name) if not board_ignorelisted: - usb_ids[id_group]['duplicate'] = True + usb_ids[id_group]["duplicate"] = True duplicates_found = True if duplicates_found: duplicates = "" for key, value in usb_ids.items(): if value["duplicate"]: - duplicates += ( - f"- VID/PID: {key}\n" - f" Boards: {', '.join(value['boards'])}\n" - ) + duplicates += f"- VID/PID: {key}\n" f" Boards: {', '.join(value['boards'])}\n" duplicate_message = ( f"Duplicate VID/PID usage found!\n{duplicates}\n" @@ -139,10 +134,7 @@ if __name__ == "__main__": arguments = cli_parser.parse_args() print("Running USB VID/PID Duplicate Checker...") - print( - f"Ignoring the following boards: {', '.join(arguments.ignorelist)}", - end="\n\n" - ) + print(f"Ignoring the following boards: {', '.join(arguments.ignorelist)}", end="\n\n") board_files = configboard_files() check_vid_pid(board_files, arguments.ignorelist) diff --git a/tools/ci_new_boards_check.py b/tools/ci_new_boards_check.py index 86010bad3..3b41c8d67 100644 --- a/tools/ci_new_boards_check.py +++ b/tools/ci_new_boards_check.py @@ -11,17 +11,19 @@ import yaml import build_board_info -workflow_file = '.github/workflows/build.yml' +workflow_file = ".github/workflows/build.yml" # Get boards in json format boards_info_json = build_board_info.get_board_mapping() # Get all the boards out of the json format -info_boards = [board for board in boards_info_json.keys() if not boards_info_json[board].get("alias", False)] +info_boards = [ + board for board in boards_info_json.keys() if not boards_info_json[board].get("alias", False) +] # We need to know the path of the workflow file base_path = os.path.dirname(__file__) -yml_path = os.path.abspath(os.path.join(base_path, '..', workflow_file)) +yml_path = os.path.abspath(os.path.join(base_path, "..", workflow_file)) # Loading board list based on build jobs in the workflow file. ci_boards = [] @@ -34,8 +36,8 @@ for job in workflow["jobs"]: continue job_boards = workflow["jobs"][job]["strategy"]["matrix"]["board"] if job_boards != sorted(job_boards): - print("Boards for job \"{}\" not sorted. Must be:".format(job)) - print(" - \"" + "\"\n - \"".join(sorted(job_boards)) + "\"") + print('Boards for job "{}" not sorted. Must be:'.format(job)) + print(' - "' + '"\n - "'.join(sorted(job_boards)) + '"') ok = False ci_boards.extend(job_boards) @@ -47,7 +49,7 @@ missing_boards = set(info_boards) - set(ci_boards) if missing_boards: ok = False - print('Boards missing in {}:'.format(workflow_file)) + print("Boards missing in {}:".format(workflow_file)) for board in missing_boards: print(board) diff --git a/tools/codeformat.py b/tools/codeformat.py index 42b43b661..389bda877 100644 --- a/tools/codeformat.py +++ b/tools/codeformat.py @@ -71,10 +71,7 @@ TOP = os.path.abspath(os.path.join(os.path.dirname(__file__), "..")) UNCRUSTIFY_CFG = os.path.join(TOP, "tools/uncrustify.cfg") -C_EXTS = ( - ".c", - ".h", -) +C_EXTS = (".c", ".h") PY_EXTS = (".py",) diff --git a/tools/convert_release_notes.py b/tools/convert_release_notes.py index 797e71834..964a7fc37 100644 --- a/tools/convert_release_notes.py +++ b/tools/convert_release_notes.py @@ -16,13 +16,14 @@ html = mistune.create_markdown() print() print("HTML") print("=====================================") -print("From the <a href=\"\">GitHub release page</a>:\n<blockquote>") +print('From the <a href="">GitHub release page</a>:\n<blockquote>') print(html(source)) print("</blockquote>") + class AdafruitBBCodeRenderer(mistune.renderers.BaseRenderer): def placeholder(self): - return '' + return "" def paragraph(self, text): return text + "\n\n" @@ -63,6 +64,7 @@ class AdafruitBBCodeRenderer(mistune.renderers.BaseRenderer): def strong(self, text): return "[i]{}[/i]".format(text) + bbcode = mistune.create_markdown(renderer=AdafruitBBCodeRenderer()) print() diff --git a/tools/cpboard.py b/tools/cpboard.py index 464be4ba4..647a3ddc5 100644 --- a/tools/cpboard.py +++ b/tools/cpboard.py @@ -100,9 +100,7 @@ class REPL: # Use read() since serial.reset_input_buffer() fails with termios.error now and then self.read() self.session = b"" - self.write( - b"\r" + REPL.CHAR_CTRL_C + REPL.CHAR_CTRL_C - ) # interrupt any running program + self.write(b"\r" + REPL.CHAR_CTRL_C + REPL.CHAR_CTRL_C) # interrupt any running program self.write(b"\r" + REPL.CHAR_CTRL_B) # enter or reset friendly repl data = self.read_until(b">>> ") @@ -158,11 +156,7 @@ class Disk: self.mountpoint = None with open("/etc/mtab", "r") as f: mtab = f.read() - mount = [ - mount.split(" ") - for mount in mtab.splitlines() - if mount.startswith(self.dev) - ] + mount = [mount.split(" ") for mount in mtab.splitlines() if mount.startswith(self.dev)] if mount: self._path = mount[0][1] else: @@ -268,9 +262,7 @@ class CPboard: vendor, _, product = name.partition(":") if vendor and product: - return CPboard.from_usb( - **kwargs, idVendor=int(vendor, 16), idProduct=int(product, 16) - ) + return CPboard.from_usb(**kwargs, idVendor=int(vendor, 16), idProduct=int(product, 16)) return CPboard(name, **kwargs) @@ -363,11 +355,7 @@ class CPboard: except: pass else: - serials = [ - serial - for serial in os.listdir("/dev/serial/by-path") - if portstr in serial - ] + serials = [serial for serial in os.listdir("/dev/serial/by-path") if portstr in serial] if len(serials) != 1: raise RuntimeError("Can't find excatly one matching usb serial device") self.device = os.path.realpath("/dev/serial/by-path/" + serials[0]) @@ -461,9 +449,7 @@ class CPboard: % mode ) try: - self.exec( - "import microcontroller;microcontroller.reset()", wait_for_response=False - ) + self.exec("import microcontroller;microcontroller.reset()", wait_for_response=False) except OSError: pass @@ -512,9 +498,7 @@ class CPboard: if not serial: raise RuntimeError("Serial number not found for: " + self.device) return [ - "/dev/disk/by-id/" + disk - for disk in os.listdir("/dev/disk/by-id") - if serial in disk + "/dev/disk/by-id/" + disk for disk in os.listdir("/dev/disk/by-id") if serial in disk ] @property @@ -562,9 +546,7 @@ PyboardError = CPboardError class Pyboard: - def __init__( - self, device, baudrate=115200, user="micro", password="python", wait=0 - ): + def __init__(self, device, baudrate=115200, user="micro", password="python", wait=0): self.board = CPboard.from_try_all(device, baudrate=baudrate, wait=wait) with self.board.disk as disk: disk.copy("skip_if.py") @@ -616,9 +598,7 @@ def upload(args): if not board.bootloader: print_verbose(args, "Reset to bootloader...", end="") - board.reset_to_bootloader( - repl=True - ) # Feather M0 Express doesn't respond to 1200 baud + board.reset_to_bootloader(repl=True) # Feather M0 Express doesn't respond to 1200 baud time.sleep(5) print_verbose(args, "done") @@ -655,9 +635,7 @@ def main(): cmd_parser.add_argument("-f", "--firmware", help="upload UF2 firmware file") cmd_parser.add_argument("-c", "--command", help="program passed in as string") cmd_parser.add_argument("--tty", action="store_true", help="print tty") - cmd_parser.add_argument( - "--verbose", "-v", action="count", default=0, help="be verbose" - ) + cmd_parser.add_argument("--verbose", "-v", action="count", default=0, help="be verbose") cmd_parser.add_argument("-q", "--quiet", action="store_true", help="be quiet") cmd_parser.add_argument("--debug", action="store_true", help="raise exceptions") args = cmd_parser.parse_args() @@ -702,9 +680,7 @@ def main(): with board as b: print("Device: ", end="") if b.usb_dev: - print( - "%04x:%04x on " % (b.usb_dev.idVendor, b.usb_dev.idProduct), end="" - ) + print("%04x:%04x on " % (b.usb_dev.idVendor, b.usb_dev.idProduct), end="") print(b.device) print("Serial number:", b.serial_number) uname = os_uname(b) diff --git a/tools/dfu.py b/tools/dfu.py index 489465f0c..4dcb4fdca 100644 --- a/tools/dfu.py +++ b/tools/dfu.py @@ -9,91 +9,122 @@ # Distributed under Gnu LGPL 3.0 # see http://www.gnu.org/licenses/lgpl-3.0.txt -import sys,struct,zlib,os +import sys, struct, zlib, os from optparse import OptionParser -DEFAULT_DEVICE="0x0483:0xdf11" +DEFAULT_DEVICE = "0x0483:0xdf11" + + +def named(tuple, names): + return dict(zip(names.split(), tuple)) + + +def consume(fmt, data, names): + n = struct.calcsize(fmt) + return named(struct.unpack(fmt, data[:n]), names), data[n:] + -def named(tuple,names): - return dict(zip(names.split(),tuple)) -def consume(fmt,data,names): - n = struct.calcsize(fmt) - return named(struct.unpack(fmt,data[:n]),names),data[n:] def cstring(string): - return string.split('\0',1)[0] + return string.split("\0", 1)[0] + + def compute_crc(data): - return 0xFFFFFFFF & -zlib.crc32(data) -1 - -def parse(file,dump_images=False): - print ('File: "%s"' % file) - data = open(file,'rb').read() - crc = compute_crc(data[:-4]) - data = data[len(data)-16:] - suffix = named(struct.unpack('<4H3sBI',data[:16]),'device product vendor dfu ufd len crc') - print ('usb: %(vendor)04x:%(product)04x, device: 0x%(device)04x, dfu: 0x%(dfu)04x, %(ufd)s, %(len)d, 0x%(crc)08x' % suffix) - if crc != suffix['crc']: - print ("CRC ERROR: computed crc32 is 0x%08x" % crc) - data = data[16:] - if data: - print ("PARSE ERROR") - -def build(file,data,device=DEFAULT_DEVICE): - # Parse the VID and PID from the `device` argument - v,d=map(lambda x: int(x,0) & 0xFFFF, device.split(':',1)) - - # Generate the DFU suffix, consisting of these fields: - # Field name | Length | Description - # ================+=========+================================ - # bcdDevice | 2 | The release number of this firmware (0xffff - don't care) - # idProduct | 2 | PID of this device - # idVendor | 2 | VID of this device - # bcdDFU | 2 | Version of this DFU spec (0x01 0x00) - # ucDfuSignature | 3 | The characters 'DFU', printed in reverse order - # bLength | 1 | The length of this suffix (16 bytes) - # dwCRC | 4 | A CRC32 of the data, including this suffix - data += struct.pack('<4H3sB',0xffff,d,v,0x0100,b'UFD',16) - crc = compute_crc(data) - # Append the CRC32 of the entire block - data += struct.pack('<I',crc) - open(file,'wb').write(data) - -if __name__=="__main__": - usage = """ + return 0xFFFFFFFF & -zlib.crc32(data) - 1 + + +def parse(file, dump_images=False): + print('File: "%s"' % file) + data = open(file, "rb").read() + crc = compute_crc(data[:-4]) + data = data[len(data) - 16 :] + suffix = named(struct.unpack("<4H3sBI", data[:16]), "device product vendor dfu ufd len crc") + print( + "usb: %(vendor)04x:%(product)04x, device: 0x%(device)04x, dfu: 0x%(dfu)04x, %(ufd)s, %(len)d, 0x%(crc)08x" + % suffix + ) + if crc != suffix["crc"]: + print("CRC ERROR: computed crc32 is 0x%08x" % crc) + data = data[16:] + if data: + print("PARSE ERROR") + + +def build(file, data, device=DEFAULT_DEVICE): + # Parse the VID and PID from the `device` argument + v, d = map(lambda x: int(x, 0) & 0xFFFF, device.split(":", 1)) + + # Generate the DFU suffix, consisting of these fields: + # Field name | Length | Description + # ================+=========+================================ + # bcdDevice | 2 | The release number of this firmware (0xffff - don't care) + # idProduct | 2 | PID of this device + # idVendor | 2 | VID of this device + # bcdDFU | 2 | Version of this DFU spec (0x01 0x00) + # ucDfuSignature | 3 | The characters 'DFU', printed in reverse order + # bLength | 1 | The length of this suffix (16 bytes) + # dwCRC | 4 | A CRC32 of the data, including this suffix + data += struct.pack("<4H3sB", 0xFFFF, d, v, 0x0100, b"UFD", 16) + crc = compute_crc(data) + # Append the CRC32 of the entire block + data += struct.pack("<I", crc) + open(file, "wb").write(data) + + +if __name__ == "__main__": + usage = """ %prog [-d|--dump] infile.dfu %prog {-b|--build} file.bin [{-D|--device}=vendor:device] outfile.dfu""" - parser = OptionParser(usage=usage) - parser.add_option("-b", "--build", action="store", dest="binfile", - help="build a DFU file from given BINFILE", metavar="BINFILE") - parser.add_option("-D", "--device", action="store", dest="device", - help="build for DEVICE, defaults to %s" % DEFAULT_DEVICE, metavar="DEVICE") - parser.add_option("-d", "--dump", action="store_true", dest="dump_images", - default=False, help="dump contained images to current directory") - (options, args) = parser.parse_args() - - if options.binfile and len(args)==1: - binfile = options.binfile - if not os.path.isfile(binfile): - print ("Unreadable file '%s'." % binfile) - sys.exit(1) - target = open(binfile,'rb').read() - outfile = args[0] - device = DEFAULT_DEVICE - # If a device is specified, parse the pair into a VID:PID pair - # in order to validate them. - if options.device: - device=options.device - try: - v,d=map(lambda x: int(x,0) & 0xFFFF, device.split(':',1)) - except: - print ("Invalid device '%s'." % device) - sys.exit(1) - build(outfile,target,device) - elif len(args)==1: - infile = args[0] - if not os.path.isfile(infile): - print ("Unreadable file '%s'." % infile) - sys.exit(1) - parse(infile, dump_images=options.dump_images) - else: - parser.print_help() - sys.exit(1) + parser = OptionParser(usage=usage) + parser.add_option( + "-b", + "--build", + action="store", + dest="binfile", + help="build a DFU file from given BINFILE", + metavar="BINFILE", + ) + parser.add_option( + "-D", + "--device", + action="store", + dest="device", + help="build for DEVICE, defaults to %s" % DEFAULT_DEVICE, + metavar="DEVICE", + ) + parser.add_option( + "-d", + "--dump", + action="store_true", + dest="dump_images", + default=False, + help="dump contained images to current directory", + ) + (options, args) = parser.parse_args() + + if options.binfile and len(args) == 1: + binfile = options.binfile + if not os.path.isfile(binfile): + print("Unreadable file '%s'." % binfile) + sys.exit(1) + target = open(binfile, "rb").read() + outfile = args[0] + device = DEFAULT_DEVICE + # If a device is specified, parse the pair into a VID:PID pair + # in order to validate them. + if options.device: + device = options.device + try: + v, d = map(lambda x: int(x, 0) & 0xFFFF, device.split(":", 1)) + except: + print("Invalid device '%s'." % device) + sys.exit(1) + build(outfile, target, device) + elif len(args) == 1: + infile = args[0] + if not os.path.isfile(infile): + print("Unreadable file '%s'." % infile) + sys.exit(1) + parse(infile, dump_images=options.dump_images) + else: + parser.print_help() + sys.exit(1) diff --git a/tools/extract_pyi.py b/tools/extract_pyi.py index 2730102ac..7937b95c6 100644 --- a/tools/extract_pyi.py +++ b/tools/extract_pyi.py @@ -16,10 +16,47 @@ import isort import black -IMPORTS_IGNORE = frozenset({'int', 'float', 'bool', 'str', 'bytes', 'tuple', 'list', 'set', 'dict', 'bytearray', 'slice', 'file', 'buffer', 'range', 'array', 'struct_time'}) -IMPORTS_TYPING = frozenset({'Any', 'Optional', 'Union', 'Tuple', 'List', 'Sequence', 'NamedTuple', 'Iterable', 'Iterator', 'Callable', 'AnyStr', 'overload', 'Type'}) -IMPORTS_TYPES = frozenset({'TracebackType'}) -CPY_TYPING = frozenset({'ReadableBuffer', 'WriteableBuffer', 'AudioSample', 'FrameBuffer', 'Alarm'}) +IMPORTS_IGNORE = frozenset( + { + "int", + "float", + "bool", + "str", + "bytes", + "tuple", + "list", + "set", + "dict", + "bytearray", + "slice", + "file", + "buffer", + "range", + "array", + "struct_time", + } +) +IMPORTS_TYPING = frozenset( + { + "Any", + "Optional", + "Union", + "Tuple", + "List", + "Sequence", + "NamedTuple", + "Iterable", + "Iterator", + "Callable", + "AnyStr", + "overload", + "Type", + } +) +IMPORTS_TYPES = frozenset({"TracebackType"}) +CPY_TYPING = frozenset( + {"ReadableBuffer", "WriteableBuffer", "AudioSample", "FrameBuffer", "Alarm"} +) def is_typed(node, allow_any=False): @@ -29,8 +66,12 @@ def is_typed(node, allow_any=False): return True elif isinstance(node, ast.Name) and node.id == "Any": return False - elif isinstance(node, ast.Attribute) and type(node.value) == ast.Name \ - and node.value.id == "typing" and node.attr == "Any": + elif ( + isinstance(node, ast.Attribute) + and type(node.value) == ast.Name + and node.value.id == "typing" + and node.attr == "Any" + ): return False return True @@ -50,12 +91,23 @@ def find_stub_issues(tree): if sys.version_info >= (3, 8): allargs.extend(node.posonlyargs) for arg_node in allargs: - if not is_typed(arg_node.annotation) and (arg_node.arg != "self" and arg_node.arg != "cls"): - yield ("WARN", f"Missing argument type: {arg_node.arg} on line {arg_node.lineno}") + if not is_typed(arg_node.annotation) and ( + arg_node.arg != "self" and arg_node.arg != "cls" + ): + yield ( + "WARN", + f"Missing argument type: {arg_node.arg} on line {arg_node.lineno}", + ) if node.vararg and not is_typed(node.vararg.annotation, allow_any=True): - yield ("WARN", f"Missing argument type: *{node.vararg.arg} on line {node.vararg.lineno}") + yield ( + "WARN", + f"Missing argument type: *{node.vararg.arg} on line {node.vararg.lineno}", + ) if node.kwarg and not is_typed(node.kwarg.annotation, allow_any=True): - yield ("WARN", f"Missing argument type: **{node.kwarg.arg} on line {node.kwarg.lineno}") + yield ( + "WARN", + f"Missing argument type: **{node.kwarg.arg} on line {node.kwarg.lineno}", + ) elif isinstance(node, ast.FunctionDef): if not is_typed(node.returns): yield ("WARN", f"Missing return type: {node.name} on line {node.lineno}") @@ -196,7 +248,7 @@ def convert_folder(top_level, stub_directory): import_body = "\n".join(import_lines) m = re.match(r'(\s*""".*?""")', stub_contents, flags=re.DOTALL) if m: - stub_contents = m.group(1) + "\n\n" + import_body + "\n\n" + stub_contents[m.end():] + stub_contents = m.group(1) + "\n\n" + import_body + "\n\n" + stub_contents[m.end() :] else: stub_contents = import_body + "\n\n" + stub_contents diff --git a/tools/file2h.py b/tools/file2h.py index 706dd4dd2..df9cc02fd 100644 --- a/tools/file2h.py +++ b/tools/file2h.py @@ -16,19 +16,19 @@ import sys # Can either be set explicitly, or left blank to auto-detect # Except auto-detect doesn't work because the file has been passed # through Python text processing, which makes all EOL a \n -line_end = '\\r\\n' +line_end = "\\r\\n" if __name__ == "__main__": filename = sys.argv[1] - for line in open(filename, 'r').readlines(): + for line in open(filename, "r").readlines(): if not line_end: - for ending in ('\r\n', '\r', '\n'): + for ending in ("\r\n", "\r", "\n"): if line.endswith(ending): - line_end = ending.replace('\r', '\\r').replace('\n', '\\n') + line_end = ending.replace("\r", "\\r").replace("\n", "\\n") break if not line_end: raise Exception("Couldn't auto-detect line-ending of %s" % filename) - line = line.rstrip('\r\n') - line = line.replace('\\', '\\\\') + line = line.rstrip("\r\n") + line = line.replace("\\", "\\\\") line = line.replace('"', '\\"') print('"%s%s"' % (line, line_end)) diff --git a/tools/fixup_translations.py b/tools/fixup_translations.py index 0362923e8..4c6f89b26 100644 --- a/tools/fixup_translations.py +++ b/tools/fixup_translations.py @@ -45,7 +45,9 @@ for po_filename in po_filenames: entry.merge(newer_entry) print(entry) elif newer_entry and entry.msgstr != newer_entry.msgstr: - if newer_entry.msgstr != "" and (newer_entry.msgstr != entry.msgid or entry.msgid in NO_TRANSLATION_WHITELIST): + if newer_entry.msgstr != "" and ( + newer_entry.msgstr != entry.msgid or entry.msgid in NO_TRANSLATION_WHITELIST + ): entry.merge(newer_entry) entry.msgstr = newer_entry.msgstr if "fuzzy" not in newer_entry.flags and "fuzzy" in entry.flags: @@ -55,7 +57,7 @@ for po_filename in po_filenames: bad_commits[commit] = set() bad_commits[commit].add(po_filename) fixed_ids.add(entry.msgid) - #print(entry.msgid, "\"" + entry.msgstr + "\"", "\"" + newer_entry.msgstr + "\"",) + # print(entry.msgid, "\"" + entry.msgstr + "\"", "\"" + newer_entry.msgstr + "\"",) elif newer_entry and newer_entry.flags != entry.flags: entry.flags = newer_entry.flags elif newer_entry and newer_entry.obsolete != entry.obsolete: @@ -86,4 +88,4 @@ for commit in bad_commits: files = bad_commits[commit] print(commit) for file in files: - print("\t",file) + print("\t", file) diff --git a/tools/gc_activity.py b/tools/gc_activity.py index cc9a21836..6a1772faa 100644 --- a/tools/gc_activity.py +++ b/tools/gc_activity.py @@ -10,18 +10,22 @@ current_heap = {} allocation_history = [] root = {} + def change_root(trace, size): level = root for frame in reversed(trace): file_location = frame[1] if file_location not in level: - level[file_location] = {"blocks": 0, - "file": file_location, - "function": frame[2], - "subcalls": {}} + level[file_location] = { + "blocks": 0, + "file": file_location, + "function": frame[2], + "subcalls": {}, + } level[file_location]["blocks"] += size level = level[file_location]["subcalls"] + total_actions = 0 with open(sys.argv[1], "r") as f: for line in f: @@ -31,13 +35,13 @@ with open(sys.argv[1], "r") as f: action = None if line.startswith("Breakpoint 2"): break - next(f) # throw away breakpoint code line - next(f) # first frame + next(f) # throw away breakpoint code line + next(f) # first frame block = 0 size = 0 trace = [] for line in f: - #print(line.strip()) + # print(line.strip()) if line[0] == "#": frame = line.strip().split() if frame[1].startswith("0x"): @@ -52,7 +56,12 @@ with open(sys.argv[1], "r") as f: action = "unknown" if block not in current_heap: - current_heap[block] = {"start_block": block, "size": size, "start_trace": trace, "start_time": total_actions} + current_heap[block] = { + "start_block": block, + "size": size, + "start_trace": trace, + "start_time": total_actions, + } action = "alloc" change_root(trace, size) else: @@ -62,7 +71,12 @@ with open(sys.argv[1], "r") as f: change_root(alloc["start_trace"], -1 * alloc["size"]) if size > 0: action = "realloc" - current_heap[block] = {"start_block": block, "size": size, "start_trace": trace, "start_time": total_actions} + current_heap[block] = { + "start_block": block, + "size": size, + "start_trace": trace, + "start_time": total_actions, + } change_root(trace, size) else: action = "free" @@ -81,13 +95,19 @@ for alloc in current_heap.values(): alloc["end_time"] = total_actions allocation_history.append(alloc) + def print_frame(frame, indent=0): for key in sorted(frame): - if not frame[key]["blocks"] or key.startswith("../py/malloc.c") or key.startswith("../py/gc.c"): + if ( + not frame[key]["blocks"] + or key.startswith("../py/malloc.c") + or key.startswith("../py/gc.c") + ): continue print(" " * (indent - 1), key, frame[key]["function"], frame[key]["blocks"], "blocks") print_frame(frame[key]["subcalls"], indent + 2) + print_frame(root) total_blocks = 0 for key in sorted(root): diff --git a/tools/gc_activity_between_collects.py b/tools/gc_activity_between_collects.py index f906cf5c7..f1afede70 100644 --- a/tools/gc_activity_between_collects.py +++ b/tools/gc_activity_between_collects.py @@ -10,18 +10,22 @@ current_heap = {} allocation_history = [] root = {} + def change_root(trace, size): level = root for frame in reversed(trace): file_location = frame[1] if file_location not in level: - level[file_location] = {"blocks": 0, - "file": file_location, - "function": frame[2], - "subcalls": {}} + level[file_location] = { + "blocks": 0, + "file": file_location, + "function": frame[2], + "subcalls": {}, + } level[file_location]["blocks"] += size level = level[file_location]["subcalls"] + total_actions = 0 non_single_block_streak = 0 max_nsbs = 0 @@ -41,7 +45,7 @@ with open(sys.argv[1], "r") as f: action = None if line.startswith("Breakpoint 2"): break - next(f) # throw away breakpoint code line + next(f) # throw away breakpoint code line # print(next(f)) # first frame block = 0 size = 0 @@ -55,7 +59,7 @@ with open(sys.argv[1], "r") as f: else: trace.append(("0x0", frame[-1], frame[1])) elif line[0] == "$": - #print(line.strip().split()[-1]) + # print(line.strip().split()[-1]) block = int(line.strip().split()[-1][2:], 16) next_line = next(f) size = int(next_line.strip().split()[-1][2:], 16) @@ -66,14 +70,19 @@ with open(sys.argv[1], "r") as f: action = "unknown" if block not in current_heap: - current_heap[block] = {"start_block": block, "size": size, "start_trace": trace, "start_time": total_actions} + current_heap[block] = { + "start_block": block, + "size": size, + "start_trace": trace, + "start_time": total_actions, + } action = "alloc" if size == 1: max_nsbs = max(max_nsbs, non_single_block_streak) non_single_block_streak = 0 else: non_single_block_streak += 1 - #change_root(trace, size) + # change_root(trace, size) if size not in block_sizes: block_sizes[size] = 0 source = trace[-1][-1] @@ -89,15 +98,27 @@ with open(sys.argv[1], "r") as f: change_root(alloc["start_trace"], -1 * alloc["size"]) if size > 0: action = "realloc" - current_heap[block] = {"start_block": block, "size": size, "start_trace": trace, "start_time": total_actions} - #change_root(trace, size) + current_heap[block] = { + "start_block": block, + "size": size, + "start_trace": trace, + "start_time": total_actions, + } + # change_root(trace, size) else: action = "free" if trace[0][2] == "gc_sweep": action = "sweep" non_single_block_streak = 0 - if (trace[3][2] == "py_gc_collect" or (trace[3][2] == "gc_deinit" and count > 1)) and last_action != "sweep": - print(ticks_ms - last_ticks_ms, total_actions - last_total_actions, "gc.collect", max_nsbs) + if ( + trace[3][2] == "py_gc_collect" or (trace[3][2] == "gc_deinit" and count > 1) + ) and last_action != "sweep": + print( + ticks_ms - last_ticks_ms, + total_actions - last_total_actions, + "gc.collect", + max_nsbs, + ) print(actions) print(block_sizes) print(allocation_sources) @@ -117,7 +138,7 @@ with open(sys.argv[1], "r") as f: actions[action] = 0 actions[action] += 1 last_action = action - #print(total_actions, non_single_block_streak, action, block, size) + # print(total_actions, non_single_block_streak, action, block, size) total_actions += 1 print(actions) print(max_nsbs) @@ -128,13 +149,19 @@ for alloc in current_heap.values(): alloc["end_time"] = total_actions allocation_history.append(alloc) + def print_frame(frame, indent=0): for key in sorted(frame): - if not frame[key]["blocks"] or key.startswith("../py/malloc.c") or key.startswith("../py/gc.c"): + if ( + not frame[key]["blocks"] + or key.startswith("../py/malloc.c") + or key.startswith("../py/gc.c") + ): continue print(" " * (indent - 1), key, frame[key]["function"], frame[key]["blocks"], "blocks") print_frame(frame[key]["subcalls"], indent + 2) + # print_frame(root) # total_blocks = 0 # for key in sorted(root): diff --git a/tools/gen_display_resources.py b/tools/gen_display_resources.py index 141e6ee2a..69ad50178 100644 --- a/tools/gen_display_resources.py +++ b/tools/gen_display_resources.py @@ -13,25 +13,28 @@ sys.path.insert(0, "../../tools/bitmap_font") from adafruit_bitmap_font import bitmap_font -parser = argparse.ArgumentParser(description='Generate USB descriptors.') -parser.add_argument('--font', type=str, - help='Font path', required=True) -parser.add_argument('--extra_characters', type=str, - help='Unicode string of extra characters') -parser.add_argument('--sample_file', type=argparse.FileType('r'), - help='Text file that includes strings to support.') -parser.add_argument('--output_c_file', type=argparse.FileType('w'), required=True) +parser = argparse.ArgumentParser(description="Generate USB descriptors.") +parser.add_argument("--font", type=str, help="Font path", required=True) +parser.add_argument("--extra_characters", type=str, help="Unicode string of extra characters") +parser.add_argument( + "--sample_file", + type=argparse.FileType("r"), + help="Text file that includes strings to support.", +) +parser.add_argument("--output_c_file", type=argparse.FileType("w"), required=True) args = parser.parse_args() + class BitmapStub: def __init__(self, width, height, color_depth): self.width = width - self.rows = [b''] * height + self.rows = [b""] * height def _load_row(self, y, row): self.rows[y] = bytes(row) + f = bitmap_font.load_font(args.font, BitmapStub) # Load extra characters from the sample file. @@ -45,7 +48,7 @@ if args.sample_file: sample_characters.add(c) # Merge visible ascii, sample characters and extra characters. -visible_ascii = bytes(range(0x20, 0x7f)).decode("utf-8") +visible_ascii = bytes(range(0x20, 0x7F)).decode("utf-8") all_characters = visible_ascii for c in sample_characters: if c not in all_characters: @@ -87,7 +90,7 @@ for x, c in enumerate(filtered_characters): for i in range(g["bounds"][0]): byte = i // 8 bit = i % 8 - if row[byte] & (1 << (7-bit)) != 0: + if row[byte] & (1 << (7 - bit)) != 0: overall_bit = start_bit + (start_y + y) * bytes_per_row * 8 + i b[overall_bit // 8] |= 1 << (7 - (overall_bit % 8)) @@ -99,14 +102,17 @@ for c in filtered_characters: c_file = args.output_c_file -c_file.write("""\ +c_file.write( + """\ #include "shared-bindings/displayio/Palette.h" #include "supervisor/shared/display.h" -""") +""" +) -c_file.write("""\ +c_file.write( + """\ _displayio_color_t terminal_colors[2] = { { .rgb888 = 0x000000, @@ -128,9 +134,11 @@ displayio_palette_t supervisor_terminal_color = { .color_count = 2, .needs_refresh = false }; -""") +""" +) -c_file.write("""\ +c_file.write( + """\ displayio_tilegrid_t supervisor_terminal_text_grid = {{ .base = {{ .type = &displayio_tilegrid_type }}, .bitmap = (displayio_bitmap_t*) &supervisor_terminal_font_bitmap, @@ -154,22 +162,32 @@ displayio_tilegrid_t supervisor_terminal_text_grid = {{ .inline_tiles = false, .in_group = true }}; -""".format(len(all_characters), tile_x, tile_y)) +""".format( + len(all_characters), tile_x, tile_y + ) +) -c_file.write("""\ +c_file.write( + """\ const uint32_t font_bitmap_data[{}] = {{ -""".format(bytes_per_row * tile_y // 4)) +""".format( + bytes_per_row * tile_y // 4 + ) +) for i, word in enumerate(struct.iter_unpack(">I", b)): c_file.write("0x{:08x}, ".format(word[0])) if (i + 1) % (bytes_per_row // 4) == 0: c_file.write("\n") -c_file.write("""\ +c_file.write( + """\ }; -""") +""" +) -c_file.write("""\ +c_file.write( + """\ displayio_bitmap_t supervisor_terminal_font_bitmap = {{ .base = {{.type = &displayio_bitmap_type }}, .width = {}, @@ -182,10 +200,14 @@ displayio_bitmap_t supervisor_terminal_font_bitmap = {{ .bitmask = 0x1, .read_only = true }}; -""".format(len(all_characters) * tile_x, tile_y, bytes_per_row / 4)) +""".format( + len(all_characters) * tile_x, tile_y, bytes_per_row / 4 + ) +) -c_file.write("""\ +c_file.write( + """\ const fontio_builtinfont_t supervisor_terminal_font = {{ .base = {{.type = &fontio_builtinfont_type }}, .bitmap = &supervisor_terminal_font_bitmap, @@ -194,9 +216,13 @@ const fontio_builtinfont_t supervisor_terminal_font = {{ .unicode_characters = (const uint8_t*) "{}", .unicode_characters_len = {} }}; -""".format(tile_x, tile_y, extra_characters, len(extra_characters.encode("utf-8")))) +""".format( + tile_x, tile_y, extra_characters, len(extra_characters.encode("utf-8")) + ) +) -c_file.write("""\ +c_file.write( + """\ terminalio_terminal_obj_t supervisor_terminal = { .base = { .type = &terminalio_terminal_type }, .font = &supervisor_terminal_font, @@ -204,4 +230,5 @@ terminalio_terminal_obj_t supervisor_terminal = { .cursor_y = 0, .tilegrid = &supervisor_terminal_text_grid }; -""") +""" +) diff --git a/tools/gen_ld_files.py b/tools/gen_ld_files.py index 1fea1a7ef..28f73d9be 100755 --- a/tools/gen_ld_files.py +++ b/tools/gen_ld_files.py @@ -12,31 +12,39 @@ import sys import re from string import Template -parser = argparse.ArgumentParser(description='Apply #define values to .template.ld file.') -parser.add_argument('template_files', metavar='TEMPLATE_FILE', type=argparse.FileType('r'), - nargs='+', help="template filename: <something>.template.ld") -parser.add_argument('--defines', type=argparse.FileType('r'), required=True) -parser.add_argument('--out_dir', required=True) +parser = argparse.ArgumentParser(description="Apply #define values to .template.ld file.") +parser.add_argument( + "template_files", + metavar="TEMPLATE_FILE", + type=argparse.FileType("r"), + nargs="+", + help="template filename: <something>.template.ld", +) +parser.add_argument("--defines", type=argparse.FileType("r"), required=True) +parser.add_argument("--out_dir", required=True) args = parser.parse_args() defines = {} # -REMOVE_UL_RE = re.compile('([0-9]+)UL') +REMOVE_UL_RE = re.compile("([0-9]+)UL") + + def remove_UL(s): - return REMOVE_UL_RE.sub(r'\1', s) + return REMOVE_UL_RE.sub(r"\1", s) + # We skip all lines before # // START_LD_DEFINES # Then we look for lines like this: # /*NAME_OF_VALUE=*/ NAME_OF_VALUE; -VALUE_LINE_RE = re.compile(r'^/\*\s*(\w+)\s*=\*/\s*(.*);\s*$') +VALUE_LINE_RE = re.compile(r"^/\*\s*(\w+)\s*=\*/\s*(.*);\s*$") start_processing = False for line in args.defines: line = line.strip() - if line == '// START_LD_DEFINES': + if line == "// START_LD_DEFINES": start_processing = True continue if start_processing: @@ -50,10 +58,10 @@ fail = False for template_file in args.template_files: ld_template_basename = os.path.basename(template_file.name) - ld_pathname = os.path.join(args.out_dir, ld_template_basename.replace('.template.ld', '.ld')) - with open(ld_pathname, 'w') as output: - for k,v in defines.items(): - print('/*', k, '=', v, '*/', file=output) + ld_pathname = os.path.join(args.out_dir, ld_template_basename.replace(".template.ld", ".ld")) + with open(ld_pathname, "w") as output: + for k, v in defines.items(): + print("/*", k, "=", v, "*/", file=output) print(file=output) try: output.write(Template(template_file.read()).substitute(defines)) diff --git a/tools/gen_usb_descriptor.py b/tools/gen_usb_descriptor.py index b21cfd920..89bb8a817 100644 --- a/tools/gen_usb_descriptor.py +++ b/tools/gen_usb_descriptor.py @@ -25,16 +25,11 @@ ALL_HID_DEVICES_SET = frozenset(ALL_HID_DEVICES.split()) DEFAULT_HID_DEVICES = "KEYBOARD MOUSE CONSUMER GAMEPAD" # In the following URL, don't include the https:// because that prefix gets added automatically -DEFAULT_WEBUSB_URL = ( - "circuitpython.org" # In the future, this may become a specific landing page -) +DEFAULT_WEBUSB_URL = "circuitpython.org" # In the future, this may become a specific landing page parser = argparse.ArgumentParser(description="Generate USB descriptors.") parser.add_argument( - "--highspeed", - default=False, - action="store_true", - help="descriptor for highspeed device", + "--highspeed", default=False, action="store_true", help="descriptor for highspeed device" ) parser.add_argument("--manufacturer", type=str, help="manufacturer of the device") parser.add_argument("--product", type=str, help="product name of the device") @@ -71,16 +66,10 @@ parser.add_argument( help="use to not renumber endpoint", ) parser.add_argument( - "--cdc_ep_num_notification", - type=int, - default=0, - help="endpoint number of CDC NOTIFICATION", + "--cdc_ep_num_notification", type=int, default=0, help="endpoint number of CDC NOTIFICATION" ) parser.add_argument( - "--cdc2_ep_num_notification", - type=int, - default=0, - help="endpoint number of CDC2 NOTIFICATION", + "--cdc2_ep_num_notification", type=int, default=0, help="endpoint number of CDC2 NOTIFICATION" ) parser.add_argument( "--cdc_ep_num_data_out", type=int, default=0, help="endpoint number of CDC DATA OUT" @@ -89,35 +78,18 @@ parser.add_argument( "--cdc_ep_num_data_in", type=int, default=0, help="endpoint number of CDC DATA IN" ) parser.add_argument( - "--cdc2_ep_num_data_out", - type=int, - default=0, - help="endpoint number of CDC2 DATA OUT", + "--cdc2_ep_num_data_out", type=int, default=0, help="endpoint number of CDC2 DATA OUT" ) parser.add_argument( "--cdc2_ep_num_data_in", type=int, default=0, help="endpoint number of CDC2 DATA IN" ) -parser.add_argument( - "--msc_ep_num_out", type=int, default=0, help="endpoint number of MSC OUT" -) -parser.add_argument( - "--msc_ep_num_in", type=int, default=0, help="endpoint number of MSC IN" -) -parser.add_argument( - "--hid_ep_num_out", type=int, default=0, help="endpoint number of HID OUT" -) -parser.add_argument( - "--hid_ep_num_in", type=int, default=0, help="endpoint number of HID IN" -) -parser.add_argument( - "--midi_ep_num_out", type=int, default=0, help="endpoint number of MIDI OUT" -) -parser.add_argument( - "--midi_ep_num_in", type=int, default=0, help="endpoint number of MIDI IN" -) -parser.add_argument( - "--max_ep", type=int, default=0, help="total number of endpoints available" -) +parser.add_argument("--msc_ep_num_out", type=int, default=0, help="endpoint number of MSC OUT") +parser.add_argument("--msc_ep_num_in", type=int, default=0, help="endpoint number of MSC IN") +parser.add_argument("--hid_ep_num_out", type=int, default=0, help="endpoint number of HID OUT") +parser.add_argument("--hid_ep_num_in", type=int, default=0, help="endpoint number of HID IN") +parser.add_argument("--midi_ep_num_out", type=int, default=0, help="endpoint number of MIDI OUT") +parser.add_argument("--midi_ep_num_in", type=int, default=0, help="endpoint number of MIDI IN") +parser.add_argument("--max_ep", type=int, default=0, help="total number of endpoints available") parser.add_argument( "--webusb_url", type=str, @@ -127,9 +99,7 @@ parser.add_argument( parser.add_argument( "--vendor_ep_num_out", type=int, default=0, help="endpoint number of VENDOR OUT" ) -parser.add_argument( - "--vendor_ep_num_in", type=int, default=0, help="endpoint number of VENDOR IN" -) +parser.add_argument("--vendor_ep_num_in", type=int, default=0, help="endpoint number of VENDOR IN") parser.add_argument( "--output_c_file", type=argparse.FileType("w", encoding="UTF-8"), required=True ) @@ -261,9 +231,7 @@ def make_cdc_call_management(name): ) -def make_cdc_comm_interface( - name, cdc_union, cdc_call_management, cdc_ep_num_notification -): +def make_cdc_comm_interface(name, cdc_union, cdc_call_management, cdc_ep_num_notification): return standard.InterfaceDescriptor( description="{} comm".format(name), bInterfaceClass=cdc.CDC_CLASS_COMM, # Communications Device Class @@ -273,9 +241,7 @@ def make_cdc_comm_interface( subdescriptors=[ cdc.Header(description="{} comm".format(name), bcdCDC=0x0110), cdc_call_management, - cdc.AbstractControlManagement( - description="{} comm".format(name), bmCapabilities=0x02 - ), + cdc.AbstractControlManagement(description="{} comm".format(name), bmCapabilities=0x02), cdc_union, standard.EndpointDescriptor( description="{} comm in".format(name), @@ -297,16 +263,14 @@ def make_cdc_data_interface(name, cdc_ep_num_data_in, cdc_ep_num_data_out): subdescriptors=[ standard.EndpointDescriptor( description="{} data out".format(name), - bEndpointAddress=cdc_ep_num_data_out - | standard.EndpointDescriptor.DIRECTION_OUT, + bEndpointAddress=cdc_ep_num_data_out | standard.EndpointDescriptor.DIRECTION_OUT, bmAttributes=standard.EndpointDescriptor.TYPE_BULK, bInterval=0, wMaxPacketSize=512 if args.highspeed else 64, ), standard.EndpointDescriptor( description="{} data in".format(name), - bEndpointAddress=cdc_ep_num_data_in - | standard.EndpointDescriptor.DIRECTION_IN, + bEndpointAddress=cdc_ep_num_data_in | standard.EndpointDescriptor.DIRECTION_IN, bmAttributes=standard.EndpointDescriptor.TYPE_BULK, bInterval=0, wMaxPacketSize=512 if args.highspeed else 64, @@ -350,8 +314,7 @@ if include_msc: subdescriptors=[ standard.EndpointDescriptor( description="MSC in", - bEndpointAddress=args.msc_ep_num_in - | standard.EndpointDescriptor.DIRECTION_IN, + bEndpointAddress=args.msc_ep_num_in | standard.EndpointDescriptor.DIRECTION_IN, bmAttributes=standard.EndpointDescriptor.TYPE_BULK, bInterval=0, wMaxPacketSize=512 if args.highspeed else 64, @@ -383,9 +346,7 @@ if include_hid: name = args.hid_devices[0] combined_hid_report_descriptor = hid.ReportDescriptor( description=name, - report_descriptor=bytes( - hid_report_descriptors.REPORT_DESCRIPTOR_FUNCTIONS[name](0) - ), + report_descriptor=bytes(hid_report_descriptors.REPORT_DESCRIPTOR_FUNCTIONS[name](0)), ) report_ids[name] = 0 else: @@ -393,9 +354,7 @@ if include_hid: concatenated_descriptors = bytearray() for name in args.hid_devices: concatenated_descriptors.extend( - bytes( - hid_report_descriptors.REPORT_DESCRIPTOR_FUNCTIONS[name](report_id) - ) + bytes(hid_report_descriptors.REPORT_DESCRIPTOR_FUNCTIONS[name](report_id)) ) report_ids[name] = report_id report_id += 1 @@ -414,8 +373,7 @@ if include_hid: hid_endpoint_out_descriptor = standard.EndpointDescriptor( description="HID out", - bEndpointAddress=args.hid_ep_num_out - | standard.EndpointDescriptor.DIRECTION_OUT, + bEndpointAddress=args.hid_ep_num_out | standard.EndpointDescriptor.DIRECTION_OUT, bmAttributes=standard.EndpointDescriptor.TYPE_INTERRUPT, bInterval=8, ) @@ -429,13 +387,12 @@ if include_hid: iInterface=StringIndex.index("{} HID".format(args.interface_name)), subdescriptors=[ hid.HIDDescriptor( - description="HID", - wDescriptorLength=len(bytes(combined_hid_report_descriptor)), + description="HID", wDescriptorLength=len(bytes(combined_hid_report_descriptor)) ), hid_endpoint_in_descriptor, hid_endpoint_out_descriptor, ], - ), + ) ] if include_audio: @@ -457,9 +414,7 @@ if include_audio: # USB IN <- midi_out_jack_emb <- midi_in_jack_ext <- CircuitPython midi_in_jack_ext = midi.InJackDescriptor( - description="MIDI data in from user code.", - bJackType=midi.JACK_TYPE_EXTERNAL, - iJack=0, + description="MIDI data in from user code.", bJackType=midi.JACK_TYPE_EXTERNAL, iJack=0 ) midi_out_jack_emb = midi.OutJackDescriptor( description="MIDI PC <- {}".format(args.interface_name), @@ -481,12 +436,11 @@ if include_audio: midi_in_jack_ext, midi_out_jack_emb, midi_out_jack_ext, - ], + ] ), standard.EndpointDescriptor( description="MIDI data out to {}".format(args.interface_name), - bEndpointAddress=args.midi_ep_num_out - | standard.EndpointDescriptor.DIRECTION_OUT, + bEndpointAddress=args.midi_ep_num_out | standard.EndpointDescriptor.DIRECTION_OUT, bmAttributes=standard.EndpointDescriptor.TYPE_BULK, bInterval=0, wMaxPacketSize=512 if args.highspeed else 64, @@ -494,8 +448,7 @@ if include_audio: midi.DataEndpointDescriptor(baAssocJack=[midi_in_jack_emb]), standard.EndpointDescriptor( description="MIDI data in from {}".format(args.interface_name), - bEndpointAddress=args.midi_ep_num_in - | standard.EndpointDescriptor.DIRECTION_IN, + bEndpointAddress=args.midi_ep_num_in | standard.EndpointDescriptor.DIRECTION_IN, bmAttributes=standard.EndpointDescriptor.TYPE_BULK, bInterval=0x0, wMaxPacketSize=512 if args.highspeed else 64, @@ -516,9 +469,7 @@ if include_audio: bInterfaceSubClass=audio.AUDIO_SUBCLASS_CONTROL, bInterfaceProtocol=audio.AUDIO_PROTOCOL_V1, iInterface=StringIndex.index("{} Audio".format(args.interface_name)), - subdescriptors=[ - cs_ac_interface, - ], + subdescriptors=[cs_ac_interface], ) # Audio streaming interfaces must occur before MIDI ones. @@ -532,16 +483,14 @@ if include_vendor: # Vendor-specific interface, for example WebUSB vendor_endpoint_in_descriptor = standard.EndpointDescriptor( description="VENDOR in", - bEndpointAddress=args.vendor_ep_num_in - | standard.EndpointDescriptor.DIRECTION_IN, + bEndpointAddress=args.vendor_ep_num_in | standard.EndpointDescriptor.DIRECTION_IN, bmAttributes=standard.EndpointDescriptor.TYPE_BULK, bInterval=16, ) vendor_endpoint_out_descriptor = standard.EndpointDescriptor( description="VENDOR out", - bEndpointAddress=args.vendor_ep_num_out - | standard.EndpointDescriptor.DIRECTION_OUT, + bEndpointAddress=args.vendor_ep_num_out | standard.EndpointDescriptor.DIRECTION_OUT, bmAttributes=standard.EndpointDescriptor.TYPE_BULK, bInterval=16, ) @@ -552,10 +501,7 @@ if include_vendor: bInterfaceSubClass=0x00, bInterfaceProtocol=0x00, iInterface=StringIndex.index("{} VENDOR".format(args.interface_name)), - subdescriptors=[ - vendor_endpoint_in_descriptor, - vendor_endpoint_out_descriptor, - ], + subdescriptors=[vendor_endpoint_in_descriptor, vendor_endpoint_out_descriptor], ) vendor_interfaces = [vendor_interface] @@ -583,9 +529,7 @@ if include_vendor: # util.join_interfaces() will renumber the endpoints to make them unique across descriptors, # and renumber the interfaces in order. But we still need to fix up certain # interface cross-references. -interfaces = util.join_interfaces( - interfaces_to_join, renumber_endpoints=args.renumber_endpoints -) +interfaces = util.join_interfaces(interfaces_to_join, renumber_endpoints=args.renumber_endpoints) if args.max_ep != 0: for interface in interfaces: @@ -597,10 +541,7 @@ if args.max_ep != 0: % (endpoint_address & 0x7F, interface.description, args.max_ep) ) else: - print( - "Unable to check whether maximum number of endpoints is respected", - file=sys.stderr, - ) + print("Unable to check whether maximum number of endpoints is respected", file=sys.stderr) # Now adjust the CDC interface cross-references. @@ -668,8 +609,7 @@ if include_vendor: configuration = standard.ConfigurationDescriptor( description="Composite configuration", wTotalLength=( - standard.ConfigurationDescriptor.bLength - + sum([len(bytes(x)) for x in descriptor_list]) + standard.ConfigurationDescriptor.bLength + sum([len(bytes(x)) for x in descriptor_list]) ), bNumInterfaces=len(interfaces), ) @@ -783,7 +723,9 @@ for idx, descriptor in enumerate(string_descriptors): const = "const " if variable_name == "usb_serial_number": length = len(b) - c_file.write(" uint16_t {NAME}[{length}];\n".format(NAME=variable_name, length=length//2)) + c_file.write( + " uint16_t {NAME}[{length}];\n".format(NAME=variable_name, length=length // 2) + ) else: c_file.write( """\ @@ -804,7 +746,7 @@ for idx, descriptor in enumerate(string_descriptors): """\ }; """ - ) + ) c_file.write( """\ @@ -932,9 +874,7 @@ const uint8_t hid_report_descriptor[{HID_DESCRIPTOR_LENGTH}] = {{ static uint8_t {name}_report_buffer[{report_length}]; """.format( name=name.lower(), - report_length=hid_report_descriptors.HID_DEVICE_DATA[ - name - ].report_length, + report_length=hid_report_descriptors.HID_DEVICE_DATA[name].report_length, ) ) @@ -944,9 +884,7 @@ static uint8_t {name}_report_buffer[{report_length}]; static uint8_t {name}_out_report_buffer[{report_length}]; """.format( name=name.lower(), - report_length=hid_report_descriptors.HID_DEVICE_DATA[ - name - ].out_report_length, + report_length=hid_report_descriptors.HID_DEVICE_DATA[name].out_report_length, ) ) @@ -1113,7 +1051,6 @@ TU_VERIFY_STATIC(sizeof(desc_ms_os_20) == MS_OS_20_DESC_LEN, "Incorrect size"); // End of section about desc_ms_os_20 """.format( - webusb_url=args.webusb_url, - webusb_interface=vendor_interface.bInterfaceNumber, + webusb_url=args.webusb_url, webusb_interface=vendor_interface.bInterfaceNumber ) ) diff --git a/tools/gendoc.py b/tools/gendoc.py index 85411cb41..bf79b3cf5 100644 --- a/tools/gendoc.py +++ b/tools/gendoc.py @@ -19,10 +19,12 @@ def re_match_first(regexs, line): return name, match return None, None + def makedirs(d): if not os.path.isdir(d): os.makedirs(d) + class Lexer: class LexerError(Exception): pass @@ -35,15 +37,15 @@ class Lexer: def __init__(self, file): self.filename = file - with open(file, 'rt') as f: + with open(file, "rt") as f: line_num = 0 lines = [] for line in f: line_num += 1 line = line.strip() - if line == '///': - lines.append((line_num, '')) - elif line.startswith('/// '): + if line == "///": + lines.append((line_num, "")) + elif line.startswith("/// "): lines.append((line_num, line[4:])) elif len(lines) > 0 and lines[-1][1] is not None: lines.append((line_num, None)) @@ -68,9 +70,10 @@ class Lexer: return l[1] def error(self, msg): - print('({}:{}) {}'.format(self.filename, self.cur_line, msg)) + print("({}:{}) {}".format(self.filename, self.cur_line, msg)) raise Lexer.LexerError + class MarkdownWriter: def __init__(self): pass @@ -79,52 +82,53 @@ class MarkdownWriter: self.lines = [] def end(self): - return '\n'.join(self.lines) + return "\n".join(self.lines) def heading(self, level, text): if len(self.lines) > 0: - self.lines.append('') - self.lines.append(level * '#' + ' ' + text) - self.lines.append('') + self.lines.append("") + self.lines.append(level * "#" + " " + text) + self.lines.append("") def para(self, text): - if len(self.lines) > 0 and self.lines[-1] != '': - self.lines.append('') + if len(self.lines) > 0 and self.lines[-1] != "": + self.lines.append("") if isinstance(text, list): self.lines.extend(text) elif isinstance(text, str): self.lines.append(text) else: assert False - self.lines.append('') + self.lines.append("") def single_line(self, text): self.lines.append(text) def module(self, name, short_descr, descr): - self.heading(1, 'module {}'.format(name)) + self.heading(1, "module {}".format(name)) self.para(descr) def function(self, ctx, name, args, descr): - proto = '{}.{}{}'.format(ctx, self.name, self.args) - self.heading(3, '`' + proto + '`') + proto = "{}.{}{}".format(ctx, self.name, self.args) + self.heading(3, "`" + proto + "`") self.para(descr) def method(self, ctx, name, args, descr): - if name == '\\constructor': - proto = '{}{}'.format(ctx, args) - elif name == '\\call': - proto = '{}{}'.format(ctx, args) + if name == "\\constructor": + proto = "{}{}".format(ctx, args) + elif name == "\\call": + proto = "{}{}".format(ctx, args) else: - proto = '{}.{}{}'.format(ctx, name, args) - self.heading(3, '`' + proto + '`') + proto = "{}.{}{}".format(ctx, name, args) + self.heading(3, "`" + proto + "`") self.para(descr) def constant(self, ctx, name, descr): - self.single_line('`{}.{}` - {}'.format(ctx, name, descr)) + self.single_line("`{}.{}` - {}".format(ctx, name, descr)) + class ReStructuredTextWriter: - head_chars = {1:'=', 2:'-', 3:'.'} + head_chars = {1: "=", 2: "-", 3: "."} def __init__(self): pass @@ -133,23 +137,23 @@ class ReStructuredTextWriter: self.lines = [] def end(self): - return '\n'.join(self.lines) + return "\n".join(self.lines) def _convert(self, text): - return text.replace('`', '``').replace('*', '\\*') + return text.replace("`", "``").replace("*", "\\*") def heading(self, level, text, convert=True): if len(self.lines) > 0: - self.lines.append('') + self.lines.append("") if convert: text = self._convert(text) self.lines.append(text) self.lines.append(len(text) * self.head_chars[level]) - self.lines.append('') + self.lines.append("") - def para(self, text, indent=''): - if len(self.lines) > 0 and self.lines[-1] != '': - self.lines.append('') + def para(self, text, indent=""): + if len(self.lines) > 0 and self.lines[-1] != "": + self.lines.append("") if isinstance(text, list): for t in text: self.lines.append(indent + self._convert(t)) @@ -157,39 +161,41 @@ class ReStructuredTextWriter: self.lines.append(indent + self._convert(text)) else: assert False - self.lines.append('') + self.lines.append("") def single_line(self, text): self.lines.append(self._convert(text)) def module(self, name, short_descr, descr): - self.heading(1, ':mod:`{}` --- {}'.format(name, self._convert(short_descr)), convert=False) - self.lines.append('.. module:: {}'.format(name)) - self.lines.append(' :synopsis: {}'.format(short_descr)) + self.heading(1, ":mod:`{}` --- {}".format(name, self._convert(short_descr)), convert=False) + self.lines.append(".. module:: {}".format(name)) + self.lines.append(" :synopsis: {}".format(short_descr)) self.para(descr) def function(self, ctx, name, args, descr): args = self._convert(args) - self.lines.append('.. function:: ' + name + args) - self.para(descr, indent=' ') + self.lines.append(".. function:: " + name + args) + self.para(descr, indent=" ") def method(self, ctx, name, args, descr): args = self._convert(args) - if name == '\\constructor': - self.lines.append('.. class:: ' + ctx + args) - elif name == '\\call': - self.lines.append('.. method:: ' + ctx + args) + if name == "\\constructor": + self.lines.append(".. class:: " + ctx + args) + elif name == "\\call": + self.lines.append(".. method:: " + ctx + args) else: - self.lines.append('.. method:: ' + ctx + '.' + name + args) - self.para(descr, indent=' ') + self.lines.append(".. method:: " + ctx + "." + name + args) + self.para(descr, indent=" ") def constant(self, ctx, name, descr): - self.lines.append('.. data:: ' + name) - self.para(descr, indent=' ') + self.lines.append(".. data:: " + name) + self.para(descr, indent=" ") + class DocValidateError(Exception): pass + class DocItem: def __init__(self): self.doc = [] @@ -206,6 +212,7 @@ class DocItem: def dump(self, writer): writer.para(self.doc) + class DocConstant(DocItem): def __init__(self, name, descr): super().__init__() @@ -215,6 +222,7 @@ class DocConstant(DocItem): def dump(self, ctx, writer): writer.constant(ctx, self.name, self.descr) + class DocFunction(DocItem): def __init__(self, name, args): super().__init__() @@ -224,6 +232,7 @@ class DocFunction(DocItem): def dump(self, ctx, writer): writer.function(ctx, self.name, self.args, self.doc) + class DocMethod(DocItem): def __init__(self, name, args): super().__init__() @@ -233,6 +242,7 @@ class DocMethod(DocItem): def dump(self, ctx, writer): writer.method(ctx, self.name, self.args, self.doc) + class DocClass(DocItem): def __init__(self, name, descr): super().__init__() @@ -244,51 +254,52 @@ class DocClass(DocItem): self.constants = {} def process_classmethod(self, lex, d): - name = d['id'] - if name == '\\constructor': + name = d["id"] + if name == "\\constructor": dict_ = self.constructors else: dict_ = self.classmethods if name in dict_: lex.error("multiple definition of method '{}'".format(name)) - method = dict_[name] = DocMethod(name, d['args']) + method = dict_[name] = DocMethod(name, d["args"]) method.add_doc(lex) def process_method(self, lex, d): - name = d['id'] + name = d["id"] dict_ = self.methods if name in dict_: lex.error("multiple definition of method '{}'".format(name)) - method = dict_[name] = DocMethod(name, d['args']) + method = dict_[name] = DocMethod(name, d["args"]) method.add_doc(lex) def process_constant(self, lex, d): - name = d['id'] + name = d["id"] if name in self.constants: lex.error("multiple definition of constant '{}'".format(name)) - self.constants[name] = DocConstant(name, d['descr']) + self.constants[name] = DocConstant(name, d["descr"]) lex.opt_break() def dump(self, writer): - writer.heading(1, 'class {}'.format(self.name)) + writer.heading(1, "class {}".format(self.name)) super().dump(writer) if len(self.constructors) > 0: - writer.heading(2, 'Constructors') - for f in sorted(self.constructors.values(), key=lambda x:x.name): + writer.heading(2, "Constructors") + for f in sorted(self.constructors.values(), key=lambda x: x.name): f.dump(self.name, writer) if len(self.classmethods) > 0: - writer.heading(2, 'Class methods') - for f in sorted(self.classmethods.values(), key=lambda x:x.name): + writer.heading(2, "Class methods") + for f in sorted(self.classmethods.values(), key=lambda x: x.name): f.dump(self.name, writer) if len(self.methods) > 0: - writer.heading(2, 'Methods') - for f in sorted(self.methods.values(), key=lambda x:x.name): + writer.heading(2, "Methods") + for f in sorted(self.methods.values(), key=lambda x: x.name): f.dump(self.name.lower(), writer) if len(self.constants) > 0: - writer.heading(2, 'Constants') - for c in sorted(self.constants.values(), key=lambda x:x.name): + writer.heading(2, "Constants") + for c in sorted(self.constants.values(), key=lambda x: x.name): c.dump(self.name, writer) + class DocModule(DocItem): def __init__(self, name, descr): super().__init__() @@ -303,22 +314,22 @@ class DocModule(DocItem): self.cur_class = None def process_function(self, lex, d): - name = d['id'] + name = d["id"] if name in self.functions: lex.error("multiple definition of function '{}'".format(name)) - function = self.functions[name] = DocFunction(name, d['args']) + function = self.functions[name] = DocFunction(name, d["args"]) function.add_doc(lex) - #def process_classref(self, lex, d): + # def process_classref(self, lex, d): # name = d['id'] # self.classes[name] = name # lex.opt_break() def process_class(self, lex, d): - name = d['id'] + name = d["id"] if name in self.classes: lex.error("multiple definition of class '{}'".format(name)) - self.cur_class = self.classes[name] = DocClass(name, d['descr']) + self.cur_class = self.classes[name] = DocClass(name, d["descr"]) self.cur_class.add_doc(lex) def process_classmethod(self, lex, d): @@ -330,10 +341,10 @@ class DocModule(DocItem): def process_constant(self, lex, d): if self.cur_class is None: # a module-level constant - name = d['id'] + name = d["id"] if name in self.constants: lex.error("multiple definition of constant '{}'".format(name)) - self.constants[name] = DocConstant(name, d['descr']) + self.constants[name] = DocConstant(name, d["descr"]) lex.opt_break() else: # a class-level constant @@ -341,50 +352,51 @@ class DocModule(DocItem): def validate(self): if self.descr is None: - raise DocValidateError('module {} referenced but never defined'.format(self.name)) + raise DocValidateError("module {} referenced but never defined".format(self.name)) def dump(self, writer): writer.module(self.name, self.descr, self.doc) if self.functions: - writer.heading(2, 'Functions') - for f in sorted(self.functions.values(), key=lambda x:x.name): + writer.heading(2, "Functions") + for f in sorted(self.functions.values(), key=lambda x: x.name): f.dump(self.name, writer) if self.constants: - writer.heading(2, 'Constants') - for c in sorted(self.constants.values(), key=lambda x:x.name): + writer.heading(2, "Constants") + for c in sorted(self.constants.values(), key=lambda x: x.name): c.dump(self.name, writer) if self.classes: - writer.heading(2, 'Classes') - for c in sorted(self.classes.values(), key=lambda x:x.name): - writer.para('[`{}.{}`]({}) - {}'.format(self.name, c.name, c.name, c.descr)) + writer.heading(2, "Classes") + for c in sorted(self.classes.values(), key=lambda x: x.name): + writer.para("[`{}.{}`]({}) - {}".format(self.name, c.name, c.name, c.descr)) def write_html(self, dir): md_writer = MarkdownWriter() md_writer.start() self.dump(md_writer) - with open(os.path.join(dir, 'index.html'), 'wt') as f: + with open(os.path.join(dir, "index.html"), "wt") as f: f.write(markdown.markdown(md_writer.end())) for c in self.classes.values(): class_dir = os.path.join(dir, c.name) makedirs(class_dir) md_writer.start() - md_writer.para('part of the [{} module](./)'.format(self.name)) + md_writer.para("part of the [{} module](./)".format(self.name)) c.dump(md_writer) - with open(os.path.join(class_dir, 'index.html'), 'wt') as f: + with open(os.path.join(class_dir, "index.html"), "wt") as f: f.write(markdown.markdown(md_writer.end())) def write_rst(self, dir): rst_writer = ReStructuredTextWriter() rst_writer.start() self.dump(rst_writer) - with open(dir + '/' + self.name + '.rst', 'wt') as f: + with open(dir + "/" + self.name + ".rst", "wt") as f: f.write(rst_writer.end()) for c in self.classes.values(): rst_writer.start() c.dump(rst_writer) - with open(dir + '/' + self.name + '.' + c.name + '.rst', 'wt') as f: + with open(dir + "/" + self.name + "." + c.name + ".rst", "wt") as f: f.write(rst_writer.end()) + class Doc: def __init__(self): self.modules = {} @@ -397,20 +409,20 @@ class Doc: def check_module(self, lex): if self.cur_module is None: - lex.error('module not defined') + lex.error("module not defined") def process_module(self, lex, d): - name = d['id'] + name = d["id"] if name not in self.modules: self.modules[name] = DocModule(name, None) self.cur_module = self.modules[name] if self.cur_module.descr is not None: lex.error("multiple definition of module '{}'".format(name)) - self.cur_module.descr = d['descr'] + self.cur_module.descr = d["descr"] self.cur_module.add_doc(lex) def process_moduleref(self, lex, d): - name = d['id'] + name = d["id"] if name not in self.modules: self.modules[name] = DocModule(name, None) self.cur_module = self.modules[name] @@ -441,41 +453,46 @@ class Doc: m.validate() def dump(self, writer): - writer.heading(1, 'Modules') - writer.para('These are the Python modules that are implemented.') - for m in sorted(self.modules.values(), key=lambda x:x.name): - writer.para('[`{}`]({}/) - {}'.format(m.name, m.name, m.descr)) + writer.heading(1, "Modules") + writer.para("These are the Python modules that are implemented.") + for m in sorted(self.modules.values(), key=lambda x: x.name): + writer.para("[`{}`]({}/) - {}".format(m.name, m.name, m.descr)) def write_html(self, dir): md_writer = MarkdownWriter() - with open(os.path.join(dir, 'module', 'index.html'), 'wt') as f: + with open(os.path.join(dir, "module", "index.html"), "wt") as f: md_writer.start() self.dump(md_writer) f.write(markdown.markdown(md_writer.end())) for m in self.modules.values(): - mod_dir = os.path.join(dir, 'module', m.name) + mod_dir = os.path.join(dir, "module", m.name) makedirs(mod_dir) m.write_html(mod_dir) def write_rst(self, dir): - #with open(os.path.join(dir, 'module', 'index.html'), 'wt') as f: + # with open(os.path.join(dir, 'module', 'index.html'), 'wt') as f: # f.write(markdown.markdown(self.dump())) for m in self.modules.values(): m.write_rst(dir) -regex_descr = r'(?P<descr>.*)' + +regex_descr = r"(?P<descr>.*)" doc_regexs = ( - (Doc.process_module, re.compile(r'\\module (?P<id>[a-z][a-z0-9]*) - ' + regex_descr + r'$')), - (Doc.process_moduleref, re.compile(r'\\moduleref (?P<id>[a-z]+)$')), - (Doc.process_function, re.compile(r'\\function (?P<id>[a-z0-9_]+)(?P<args>\(.*\))$')), - (Doc.process_classmethod, re.compile(r'\\classmethod (?P<id>\\?[a-z0-9_]+)(?P<args>\(.*\))$')), - (Doc.process_method, re.compile(r'\\method (?P<id>\\?[a-z0-9_]+)(?P<args>\(.*\))$')), - (Doc.process_constant, re.compile(r'\\constant (?P<id>[A-Za-z0-9_]+) - ' + regex_descr + r'$')), - #(Doc.process_classref, re.compile(r'\\classref (?P<id>[A-Za-z0-9_]+)$')), - (Doc.process_class, re.compile(r'\\class (?P<id>[A-Za-z0-9_]+) - ' + regex_descr + r'$')), + (Doc.process_module, re.compile(r"\\module (?P<id>[a-z][a-z0-9]*) - " + regex_descr + r"$")), + (Doc.process_moduleref, re.compile(r"\\moduleref (?P<id>[a-z]+)$")), + (Doc.process_function, re.compile(r"\\function (?P<id>[a-z0-9_]+)(?P<args>\(.*\))$")), + (Doc.process_classmethod, re.compile(r"\\classmethod (?P<id>\\?[a-z0-9_]+)(?P<args>\(.*\))$")), + (Doc.process_method, re.compile(r"\\method (?P<id>\\?[a-z0-9_]+)(?P<args>\(.*\))$")), + ( + Doc.process_constant, + re.compile(r"\\constant (?P<id>[A-Za-z0-9_]+) - " + regex_descr + r"$"), + ), + # (Doc.process_classref, re.compile(r'\\classref (?P<id>[A-Za-z0-9_]+)$')), + (Doc.process_class, re.compile(r"\\class (?P<id>[A-Za-z0-9_]+) - " + regex_descr + r"$")), ) + def process_file(file, doc): lex = Lexer(file) doc.new_file() @@ -485,11 +502,11 @@ def process_file(file, doc): line = lex.next() fun, match = re_match_first(doc_regexs, line) if fun == None: - lex.error('unknown line format: {}'.format(line)) + lex.error("unknown line format: {}".format(line)) fun(doc, lex, match.groupdict()) except Lexer.Break: - lex.error('unexpected break') + lex.error("unexpected break") except Lexer.EOF: pass @@ -499,16 +516,21 @@ def process_file(file, doc): return True + def main(): - cmd_parser = argparse.ArgumentParser(description='Generate documentation for pyboard API from C files.') - cmd_parser.add_argument('--outdir', metavar='<output dir>', default='gendoc-out', help='ouput directory') - cmd_parser.add_argument('--format', default='html', help='output format: html or rst') - cmd_parser.add_argument('files', nargs='+', help='input files') + cmd_parser = argparse.ArgumentParser( + description="Generate documentation for pyboard API from C files." + ) + cmd_parser.add_argument( + "--outdir", metavar="<output dir>", default="gendoc-out", help="ouput directory" + ) + cmd_parser.add_argument("--format", default="html", help="output format: html or rst") + cmd_parser.add_argument("files", nargs="+", help="input files") args = cmd_parser.parse_args() doc = Doc() for file in args.files: - print('processing', file) + print("processing", file) if not process_file(file, doc): return try: @@ -518,15 +540,16 @@ def main(): makedirs(args.outdir) - if args.format == 'html': + if args.format == "html": doc.write_html(args.outdir) - elif args.format == 'rst': + elif args.format == "rst": doc.write_rst(args.outdir) else: - print('unknown format:', args.format) + print("unknown format:", args.format) return - print('written to', args.outdir) + print("written to", args.outdir) + if __name__ == "__main__": main() diff --git a/tools/hid_report_descriptors.py b/tools/hid_report_descriptors.py index aaa5b18b1..827af3a3f 100644 --- a/tools/hid_report_descriptors.py +++ b/tools/hid_report_descriptors.py @@ -18,17 +18,36 @@ from adafruit_usb_descriptor import hid # Information about each kind of device # report_length does not include report ID in first byte, if present when sent. -DeviceData = namedtuple('DeviceData', ('report_length', 'out_report_length', 'usage_page', 'usage')) +DeviceData = namedtuple( + "DeviceData", ("report_length", "out_report_length", "usage_page", "usage") +) HID_DEVICE_DATA = { - "KEYBOARD" : DeviceData(report_length=8, out_report_length=1, usage_page=0x01, usage=0x06), # Generic Desktop, Keyboard - "MOUSE" : DeviceData(report_length=4, out_report_length=0, usage_page=0x01, usage=0x02), # Generic Desktop, Mouse - "CONSUMER" : DeviceData(report_length=2, out_report_length=0, usage_page=0x0C, usage=0x01), # Consumer, Consumer Control - "SYS_CONTROL" : DeviceData(report_length=1, out_report_length=0, usage_page=0x01, usage=0x80), # Generic Desktop, Sys Control - "GAMEPAD" : DeviceData(report_length=6, out_report_length=0, usage_page=0x01, usage=0x05), # Generic Desktop, Game Pad - "DIGITIZER" : DeviceData(report_length=5, out_report_length=0, usage_page=0x0D, usage=0x02), # Digitizers, Pen - "XAC_COMPATIBLE_GAMEPAD" : DeviceData(report_length=3, out_report_length=0, usage_page=0x01, usage=0x05), # Generic Desktop, Game Pad - "RAW" : DeviceData(report_length=64, out_report_length=0, usage_page=0xFFAF, usage=0xAF), # Vendor 0xFFAF "Adafruit", 0xAF - } + "KEYBOARD": DeviceData( + report_length=8, out_report_length=1, usage_page=0x01, usage=0x06 + ), # Generic Desktop, Keyboard + "MOUSE": DeviceData( + report_length=4, out_report_length=0, usage_page=0x01, usage=0x02 + ), # Generic Desktop, Mouse + "CONSUMER": DeviceData( + report_length=2, out_report_length=0, usage_page=0x0C, usage=0x01 + ), # Consumer, Consumer Control + "SYS_CONTROL": DeviceData( + report_length=1, out_report_length=0, usage_page=0x01, usage=0x80 + ), # Generic Desktop, Sys Control + "GAMEPAD": DeviceData( + report_length=6, out_report_length=0, usage_page=0x01, usage=0x05 + ), # Generic Desktop, Game Pad + "DIGITIZER": DeviceData( + report_length=5, out_report_length=0, usage_page=0x0D, usage=0x02 + ), # Digitizers, Pen + "XAC_COMPATIBLE_GAMEPAD": DeviceData( + report_length=3, out_report_length=0, usage_page=0x01, usage=0x05 + ), # Generic Desktop, Game Pad + "RAW": DeviceData( + report_length=64, out_report_length=0, usage_page=0xFFAF, usage=0xAF + ), # Vendor 0xFFAF "Adafruit", 0xAF +} + def keyboard_hid_descriptor(report_id): data = HID_DEVICE_DATA["KEYBOARD"] @@ -36,39 +55,73 @@ def keyboard_hid_descriptor(report_id): description="KEYBOARD", report_descriptor=bytes( # Regular keyboard - (0x05, data.usage_page, # Usage Page (Generic Desktop) - 0x09, data.usage, # Usage (Keyboard) - 0xA1, 0x01, # Collection (Application) - ) + - ((0x85, report_id) if report_id != 0 else ()) + - (0x05, 0x07, # Usage Page (Keyboard) - 0x19, 224, # Usage Minimum (224) - 0x29, 231, # Usage Maximum (231) - 0x15, 0x00, # Logical Minimum (0) - 0x25, 0x01, # Logical Maximum (1) - 0x75, 0x01, # Report Size (1) - 0x95, 0x08, # Report Count (8) - 0x81, 0x02, # Input (Data, Variable, Absolute) - 0x81, 0x01, # Input (Constant) - 0x19, 0x00, # Usage Minimum (0) - 0x29, 0xDD, # Usage Maximum (221) - 0x15, 0x00, # Logical Minimum (0) - 0x25, 0xDD, # Logical Maximum (221) - 0x75, 0x08, # Report Size (8) - 0x95, 0x06, # Report Count (6) - 0x81, 0x00, # Input (Data, Array) - 0x05, 0x08, # Usage Page (LED) - 0x19, 0x01, # Usage Minimum (1) - 0x29, 0x05, # Usage Maximum (5) - 0x15, 0x00, # Logical Minimum (0) - 0x25, 0x01, # Logical Maximum (1) - 0x75, 0x01, # Report Size (1) - 0x95, 0x05, # Report Count (5) - 0x91, 0x02, # Output (Data, Variable, Absolute) - 0x95, 0x03, # Report Count (3) - 0x91, 0x01, # Output (Constant) - 0xC0, # End Collection - ))) + ( + 0x05, + data.usage_page, # Usage Page (Generic Desktop) + 0x09, + data.usage, # Usage (Keyboard) + 0xA1, + 0x01, # Collection (Application) + ) + + ((0x85, report_id) if report_id != 0 else ()) + + ( + 0x05, + 0x07, # Usage Page (Keyboard) + 0x19, + 224, # Usage Minimum (224) + 0x29, + 231, # Usage Maximum (231) + 0x15, + 0x00, # Logical Minimum (0) + 0x25, + 0x01, # Logical Maximum (1) + 0x75, + 0x01, # Report Size (1) + 0x95, + 0x08, # Report Count (8) + 0x81, + 0x02, # Input (Data, Variable, Absolute) + 0x81, + 0x01, # Input (Constant) + 0x19, + 0x00, # Usage Minimum (0) + 0x29, + 0xDD, # Usage Maximum (221) + 0x15, + 0x00, # Logical Minimum (0) + 0x25, + 0xDD, # Logical Maximum (221) + 0x75, + 0x08, # Report Size (8) + 0x95, + 0x06, # Report Count (6) + 0x81, + 0x00, # Input (Data, Array) + 0x05, + 0x08, # Usage Page (LED) + 0x19, + 0x01, # Usage Minimum (1) + 0x29, + 0x05, # Usage Maximum (5) + 0x15, + 0x00, # Logical Minimum (0) + 0x25, + 0x01, # Logical Maximum (1) + 0x75, + 0x01, # Report Size (1) + 0x95, + 0x05, # Report Count (5) + 0x91, + 0x02, # Output (Data, Variable, Absolute) + 0x95, + 0x03, # Report Count (3) + 0x91, + 0x01, # Output (Constant) + 0xC0, # End Collection + ) + ), + ) + def mouse_hid_descriptor(report_id): data = HID_DEVICE_DATA["MOUSE"] @@ -76,41 +129,76 @@ def mouse_hid_descriptor(report_id): description="MOUSE", report_descriptor=bytes( # Regular mouse - (0x05, data.usage_page, # Usage Page (Generic Desktop) - 0x09, data.usage, # Usage (Mouse) - 0xA1, 0x01, # Collection (Application) - 0x09, 0x01, # Usage (Pointer) - 0xA1, 0x00, # Collection (Physical) - ) + - ((0x85, report_id) if report_id != 0 else ()) + - (0x05, 0x09, # Usage Page (Button) - 0x19, 0x01, # Usage Minimum (0x01) - 0x29, 0x05, # Usage Maximum (0x05) - 0x15, 0x00, # Logical Minimum (0) - 0x25, 0x01, # Logical Maximum (1) - 0x95, 0x05, # Report Count (5) - 0x75, 0x01, # Report Size (1) - 0x81, 0x02, # Input (Data,Var,Abs,No Wrap,Linear,Preferred State,No Null Position) - 0x95, 0x01, # Report Count (1) - 0x75, 0x03, # Report Size (3) - 0x81, 0x01, # Input (Const,Array,Abs,No Wrap,Linear,Preferred State,No Null Position) - 0x05, 0x01, # Usage Page (Generic Desktop Ctrls) - 0x09, 0x30, # Usage (X) - 0x09, 0x31, # Usage (Y) - 0x15, 0x81, # Logical Minimum (-127) - 0x25, 0x7F, # Logical Maximum (127) - 0x75, 0x08, # Report Size (8) - 0x95, 0x02, # Report Count (2) - 0x81, 0x06, # Input (Data,Var,Rel,No Wrap,Linear,Preferred State,No Null Position) - 0x09, 0x38, # Usage (Wheel) - 0x15, 0x81, # Logical Minimum (-127) - 0x25, 0x7F, # Logical Maximum (127) - 0x75, 0x08, # Report Size (8) - 0x95, 0x01, # Report Count (1) - 0x81, 0x06, # Input (Data,Var,Rel,No Wrap,Linear,Preferred State,No Null Position) - 0xC0, # End Collection - 0xC0, # End Collection - ))) + ( + 0x05, + data.usage_page, # Usage Page (Generic Desktop) + 0x09, + data.usage, # Usage (Mouse) + 0xA1, + 0x01, # Collection (Application) + 0x09, + 0x01, # Usage (Pointer) + 0xA1, + 0x00, # Collection (Physical) + ) + + ((0x85, report_id) if report_id != 0 else ()) + + ( + 0x05, + 0x09, # Usage Page (Button) + 0x19, + 0x01, # Usage Minimum (0x01) + 0x29, + 0x05, # Usage Maximum (0x05) + 0x15, + 0x00, # Logical Minimum (0) + 0x25, + 0x01, # Logical Maximum (1) + 0x95, + 0x05, # Report Count (5) + 0x75, + 0x01, # Report Size (1) + 0x81, + 0x02, # Input (Data,Var,Abs,No Wrap,Linear,Preferred State,No Null Position) + 0x95, + 0x01, # Report Count (1) + 0x75, + 0x03, # Report Size (3) + 0x81, + 0x01, # Input (Const,Array,Abs,No Wrap,Linear,Preferred State,No Null Position) + 0x05, + 0x01, # Usage Page (Generic Desktop Ctrls) + 0x09, + 0x30, # Usage (X) + 0x09, + 0x31, # Usage (Y) + 0x15, + 0x81, # Logical Minimum (-127) + 0x25, + 0x7F, # Logical Maximum (127) + 0x75, + 0x08, # Report Size (8) + 0x95, + 0x02, # Report Count (2) + 0x81, + 0x06, # Input (Data,Var,Rel,No Wrap,Linear,Preferred State,No Null Position) + 0x09, + 0x38, # Usage (Wheel) + 0x15, + 0x81, # Logical Minimum (-127) + 0x25, + 0x7F, # Logical Maximum (127) + 0x75, + 0x08, # Report Size (8) + 0x95, + 0x01, # Report Count (1) + 0x81, + 0x06, # Input (Data,Var,Rel,No Wrap,Linear,Preferred State,No Null Position) + 0xC0, # End Collection + 0xC0, # End Collection + ) + ), + ) + def consumer_hid_descriptor(report_id): data = HID_DEVICE_DATA["CONSUMER"] @@ -118,20 +206,37 @@ def consumer_hid_descriptor(report_id): description="CONSUMER", report_descriptor=bytes( # Consumer ("multimedia") keys - (0x05, data.usage_page, # Usage Page (Consumer) - 0x09, data.usage, # Usage (Consumer Control) - 0xA1, 0x01, # Collection (Application) - ) + - ((0x85, report_id) if report_id != 0 else ()) + - (0x75, 0x10, # Report Size (16) - 0x95, 0x01, # Report Count (1) - 0x15, 0x01, # Logical Minimum (1) - 0x26, 0x8C, 0x02, # Logical Maximum (652) - 0x19, 0x01, # Usage Minimum (Consumer Control) - 0x2A, 0x8C, 0x02, # Usage Maximum (AC Send) - 0x81, 0x00, # Input (Data,Array,Abs,No Wrap,Linear,Preferred State,No Null Position) - 0xC0, # End Collection - ))) + ( + 0x05, + data.usage_page, # Usage Page (Consumer) + 0x09, + data.usage, # Usage (Consumer Control) + 0xA1, + 0x01, # Collection (Application) + ) + + ((0x85, report_id) if report_id != 0 else ()) + + ( + 0x75, + 0x10, # Report Size (16) + 0x95, + 0x01, # Report Count (1) + 0x15, + 0x01, # Logical Minimum (1) + 0x26, + 0x8C, + 0x02, # Logical Maximum (652) + 0x19, + 0x01, # Usage Minimum (Consumer Control) + 0x2A, + 0x8C, + 0x02, # Usage Maximum (AC Send) + 0x81, + 0x00, # Input (Data,Array,Abs,No Wrap,Linear,Preferred State,No Null Position) + 0xC0, # End Collection + ) + ), + ) + def sys_control_hid_descriptor(report_id): data = HID_DEVICE_DATA["SYS_CONTROL"] @@ -139,23 +244,41 @@ def sys_control_hid_descriptor(report_id): description="SYS_CONTROL", report_descriptor=bytes( # Power controls - (0x05, data.usage_page, # Usage Page (Generic Desktop Ctrls) - 0x09, data.usage, # Usage (Sys Control) - 0xA1, 0x01, # Collection (Application) - ) + - ((0x85, report_id) if report_id != 0 else ()) + - (0x75, 0x02, # Report Size (2) - 0x95, 0x01, # Report Count (1) - 0x15, 0x01, # Logical Minimum (1) - 0x25, 0x03, # Logical Maximum (3) - 0x09, 0x82, # Usage (Sys Sleep) - 0x09, 0x81, # Usage (Sys Power Down) - 0x09, 0x83, # Usage (Sys Wake Up) - 0x81, 0x60, # Input (Data,Array,Abs,No Wrap,Linear,No Preferred State,Null State) - 0x75, 0x06, # Report Size (6) - 0x81, 0x03, # Input (Const,Var,Abs,No Wrap,Linear,Preferred State,No Null Position) - 0xC0, # End Collection - ))) + ( + 0x05, + data.usage_page, # Usage Page (Generic Desktop Ctrls) + 0x09, + data.usage, # Usage (Sys Control) + 0xA1, + 0x01, # Collection (Application) + ) + + ((0x85, report_id) if report_id != 0 else ()) + + ( + 0x75, + 0x02, # Report Size (2) + 0x95, + 0x01, # Report Count (1) + 0x15, + 0x01, # Logical Minimum (1) + 0x25, + 0x03, # Logical Maximum (3) + 0x09, + 0x82, # Usage (Sys Sleep) + 0x09, + 0x81, # Usage (Sys Power Down) + 0x09, + 0x83, # Usage (Sys Wake Up) + 0x81, + 0x60, # Input (Data,Array,Abs,No Wrap,Linear,No Preferred State,Null State) + 0x75, + 0x06, # Report Size (6) + 0x81, + 0x03, # Input (Const,Var,Abs,No Wrap,Linear,Preferred State,No Null Position) + 0xC0, # End Collection + ) + ), + ) + def gamepad_hid_descriptor(report_id): data = HID_DEVICE_DATA["GAMEPAD"] @@ -163,31 +286,57 @@ def gamepad_hid_descriptor(report_id): description="GAMEPAD", report_descriptor=bytes( # Gamepad with 16 buttons and two joysticks - (0x05, data.usage_page, # Usage Page (Generic Desktop Ctrls) - 0x09, data.usage, # Usage (Game Pad) - 0xA1, 0x01, # Collection (Application) - ) + - ((0x85, report_id) if report_id != 0 else ()) + - (0x05, 0x09, # Usage Page (Button) - 0x19, 0x01, # Usage Minimum (Button 1) - 0x29, 0x10, # Usage Maximum (Button 16) - 0x15, 0x00, # Logical Minimum (0) - 0x25, 0x01, # Logical Maximum (1) - 0x75, 0x01, # Report Size (1) - 0x95, 0x10, # Report Count (16) - 0x81, 0x02, # Input (Data,Var,Abs,No Wrap,Linear,Preferred State,No Null Position) - 0x05, 0x01, # Usage Page (Generic Desktop Ctrls) - 0x15, 0x81, # Logical Minimum (-127) - 0x25, 0x7F, # Logical Maximum (127) - 0x09, 0x30, # Usage (X) - 0x09, 0x31, # Usage (Y) - 0x09, 0x32, # Usage (Z) - 0x09, 0x35, # Usage (Rz) - 0x75, 0x08, # Report Size (8) - 0x95, 0x04, # Report Count (4) - 0x81, 0x02, # Input (Data,Var,Abs,No Wrap,Linear,Preferred State,No Null Position) - 0xC0, # End Collection - ))) + ( + 0x05, + data.usage_page, # Usage Page (Generic Desktop Ctrls) + 0x09, + data.usage, # Usage (Game Pad) + 0xA1, + 0x01, # Collection (Application) + ) + + ((0x85, report_id) if report_id != 0 else ()) + + ( + 0x05, + 0x09, # Usage Page (Button) + 0x19, + 0x01, # Usage Minimum (Button 1) + 0x29, + 0x10, # Usage Maximum (Button 16) + 0x15, + 0x00, # Logical Minimum (0) + 0x25, + 0x01, # Logical Maximum (1) + 0x75, + 0x01, # Report Size (1) + 0x95, + 0x10, # Report Count (16) + 0x81, + 0x02, # Input (Data,Var,Abs,No Wrap,Linear,Preferred State,No Null Position) + 0x05, + 0x01, # Usage Page (Generic Desktop Ctrls) + 0x15, + 0x81, # Logical Minimum (-127) + 0x25, + 0x7F, # Logical Maximum (127) + 0x09, + 0x30, # Usage (X) + 0x09, + 0x31, # Usage (Y) + 0x09, + 0x32, # Usage (Z) + 0x09, + 0x35, # Usage (Rz) + 0x75, + 0x08, # Report Size (8) + 0x95, + 0x04, # Report Count (4) + 0x81, + 0x02, # Input (Data,Var,Abs,No Wrap,Linear,Preferred State,No Null Position) + 0xC0, # End Collection + ) + ), + ) + def digitizer_hid_descriptor(report_id): data = HID_DEVICE_DATA["DIGITIZER"] @@ -195,36 +344,67 @@ def digitizer_hid_descriptor(report_id): description="DIGITIZER", report_descriptor=bytes( # Digitizer (used as an absolute pointer) - (0x05, data.usage_page, # Usage Page (Digitizers) - 0x09, data.usage, # Usage (Pen) - 0xA1, 0x01, # Collection (Application) - ) + - ((0x85, report_id) if report_id != 0 else ()) + - (0x09, 0x01, # Usage (Stylus) - 0xA1, 0x00, # Collection (Physical) - 0x09, 0x32, # Usage (In-Range) - 0x09, 0x42, # Usage (Tip Switch) - 0x09, 0x44, # Usage (Barrel Switch) - 0x09, 0x45, # Usage (Eraser Switch) - 0x15, 0x00, # Logical Minimum (0) - 0x25, 0x01, # Logical Maximum (1) - 0x75, 0x01, # Report Size (1) - 0x95, 0x04, # Report Count (4) - 0x81, 0x02, # Input (Data,Var,Abs) - 0x75, 0x04, # Report Size (4) -- Filler - 0x95, 0x01, # Report Count (1) -- Filler - 0x81, 0x01, # Input (Const,Array,Abs,No Wrap,Linear,Preferred State,No Null Position) - 0x05, 0x01, # Usage Page (Generic Desktop Ctrls) - 0x15, 0x00, # Logical Minimum (0) - 0x26, 0xff, 0x7f, # Logical Maximum (32767) - 0x09, 0x30, # Usage (X) - 0x09, 0x31, # Usage (Y) - 0x75, 0x10, # Report Size (16) - 0x95, 0x02, # Report Count (2) - 0x81, 0x02, # Input (Data,Var,Abs) - 0xC0, # End Collection - 0xC0, # End Collection - ))) + ( + 0x05, + data.usage_page, # Usage Page (Digitizers) + 0x09, + data.usage, # Usage (Pen) + 0xA1, + 0x01, # Collection (Application) + ) + + ((0x85, report_id) if report_id != 0 else ()) + + ( + 0x09, + 0x01, # Usage (Stylus) + 0xA1, + 0x00, # Collection (Physical) + 0x09, + 0x32, # Usage (In-Range) + 0x09, + 0x42, # Usage (Tip Switch) + 0x09, + 0x44, # Usage (Barrel Switch) + 0x09, + 0x45, # Usage (Eraser Switch) + 0x15, + 0x00, # Logical Minimum (0) + 0x25, + 0x01, # Logical Maximum (1) + 0x75, + 0x01, # Report Size (1) + 0x95, + 0x04, # Report Count (4) + 0x81, + 0x02, # Input (Data,Var,Abs) + 0x75, + 0x04, # Report Size (4) -- Filler + 0x95, + 0x01, # Report Count (1) -- Filler + 0x81, + 0x01, # Input (Const,Array,Abs,No Wrap,Linear,Preferred State,No Null Position) + 0x05, + 0x01, # Usage Page (Generic Desktop Ctrls) + 0x15, + 0x00, # Logical Minimum (0) + 0x26, + 0xFF, + 0x7F, # Logical Maximum (32767) + 0x09, + 0x30, # Usage (X) + 0x09, + 0x31, # Usage (Y) + 0x75, + 0x10, # Report Size (16) + 0x95, + 0x02, # Report Count (2) + 0x81, + 0x02, # Input (Data,Var,Abs) + 0xC0, # End Collection + 0xC0, # End Collection + ) + ), + ) + def xac_compatible_gamepad_hid_descriptor(report_id): data = HID_DEVICE_DATA["XAC_COMPATIBLE_GAMEPAD"] @@ -232,31 +412,59 @@ def xac_compatible_gamepad_hid_descriptor(report_id): description="XAC", report_descriptor=bytes( # This descriptor mimics the simple joystick from PDP that the XBox likes - (0x05, data.usage_page, # Usage Page (Desktop), - 0x09, data.usage, # Usage (Gamepad), - 0xA1, 0x01, # Collection (Application), - ) + - ((0x85, report_id) if report_id != 0 else ()) + - (0x15, 0x00, # Logical Minimum (0), - 0x25, 0x01, # Logical Maximum (1), - 0x35, 0x00, # Physical Minimum (0), - 0x45, 0x01, # Physical Maximum (1), - 0x75, 0x01, # Report Size (1), - 0x95, 0x08, # Report Count (8), - 0x05, 0x09, # Usage Page (Button), - 0x19, 0x01, # Usage Minimum (01h), - 0x29, 0x08, # Usage Maximum (08h), - 0x81, 0x02, # Input (Variable), - 0x05, 0x01, # Usage Page (Desktop), - 0x26, 0xFF, 0x00, # Logical Maximum (255), - 0x46, 0xFF, 0x00, # Physical Maximum (255), - 0x09, 0x30, # Usage (X), - 0x09, 0x31, # Usage (Y), - 0x75, 0x08, # Report Size (8), - 0x95, 0x02, # Report Count (2), - 0x81, 0x02, # Input (Variable), - 0xC0 # End Collection - ))) + ( + 0x05, + data.usage_page, # Usage Page (Desktop), + 0x09, + data.usage, # Usage (Gamepad), + 0xA1, + 0x01, # Collection (Application), + ) + + ((0x85, report_id) if report_id != 0 else ()) + + ( + 0x15, + 0x00, # Logical Minimum (0), + 0x25, + 0x01, # Logical Maximum (1), + 0x35, + 0x00, # Physical Minimum (0), + 0x45, + 0x01, # Physical Maximum (1), + 0x75, + 0x01, # Report Size (1), + 0x95, + 0x08, # Report Count (8), + 0x05, + 0x09, # Usage Page (Button), + 0x19, + 0x01, # Usage Minimum (01h), + 0x29, + 0x08, # Usage Maximum (08h), + 0x81, + 0x02, # Input (Variable), + 0x05, + 0x01, # Usage Page (Desktop), + 0x26, + 0xFF, + 0x00, # Logical Maximum (255), + 0x46, + 0xFF, + 0x00, # Physical Maximum (255), + 0x09, + 0x30, # Usage (X), + 0x09, + 0x31, # Usage (Y), + 0x75, + 0x08, # Report Size (8), + 0x95, + 0x02, # Report Count (2), + 0x81, + 0x02, # Input (Variable), + 0xC0, # End Collection + ) + ), + ) + def raw_hid_descriptor(report_id): if report_id != 0: @@ -267,29 +475,47 @@ def raw_hid_descriptor(report_id): report_descriptor=bytes( # Provide vendor-defined # This is a two-byte page value. - (0x06, data.usage_page & 0xff, (data.usage_page >> 8) & 0xff, # Usage Page (Vendor 0xFFAF "Adafruit"), - 0x09, data.usage, # Usage (AF), - 0xA1, 0x01, # Collection (Application), - 0x75, 0x08, # Report Size (8), - 0x15, 0x00, # Logical Minimum (0), - 0x26, 0xFF, 0x00, # Logical Maximum (255), - 0x95, 0x08, # Report Count (8), - 0x09, 0x01, # Usage(xxx) - 0x81, 0x02, # Input (Variable) - 0x95, 0x08, # Report Count (8), - 0x09, 0x02, # Usage(xxx) - 0x91, 0x02, # Input (Variable) - 0xC0 # End Collection - ))) + ( + 0x06, + data.usage_page & 0xFF, + (data.usage_page >> 8) & 0xFF, # Usage Page (Vendor 0xFFAF "Adafruit"), + 0x09, + data.usage, # Usage (AF), + 0xA1, + 0x01, # Collection (Application), + 0x75, + 0x08, # Report Size (8), + 0x15, + 0x00, # Logical Minimum (0), + 0x26, + 0xFF, + 0x00, # Logical Maximum (255), + 0x95, + 0x08, # Report Count (8), + 0x09, + 0x01, # Usage(xxx) + 0x81, + 0x02, # Input (Variable) + 0x95, + 0x08, # Report Count (8), + 0x09, + 0x02, # Usage(xxx) + 0x91, + 0x02, # Input (Variable) + 0xC0, # End Collection + ) + ), + ) + # Function to call for each kind of HID descriptor. REPORT_DESCRIPTOR_FUNCTIONS = { - "KEYBOARD" : keyboard_hid_descriptor, - "MOUSE" : mouse_hid_descriptor, - "CONSUMER" : consumer_hid_descriptor, - "SYS_CONTROL" : sys_control_hid_descriptor, - "GAMEPAD" : gamepad_hid_descriptor, - "DIGITIZER" : digitizer_hid_descriptor, - "XAC_COMPATIBLE_GAMEPAD" : xac_compatible_gamepad_hid_descriptor, - "RAW" : raw_hid_descriptor, + "KEYBOARD": keyboard_hid_descriptor, + "MOUSE": mouse_hid_descriptor, + "CONSUMER": consumer_hid_descriptor, + "SYS_CONTROL": sys_control_hid_descriptor, + "GAMEPAD": gamepad_hid_descriptor, + "DIGITIZER": digitizer_hid_descriptor, + "XAC_COMPATIBLE_GAMEPAD": xac_compatible_gamepad_hid_descriptor, + "RAW": raw_hid_descriptor, } diff --git a/tools/insert-usb-ids.py b/tools/insert-usb-ids.py index bf74ceeab..f63059ca1 100644 --- a/tools/insert-usb-ids.py +++ b/tools/insert-usb-ids.py @@ -12,15 +12,16 @@ import sys import re import string -needed_keys = ('USB_PID_CDC_MSC', 'USB_PID_CDC_HID', 'USB_PID_CDC', 'USB_VID') +needed_keys = ("USB_PID_CDC_MSC", "USB_PID_CDC_HID", "USB_PID_CDC", "USB_VID") + def parse_usb_ids(filename): rv = dict() for line in open(filename).readlines(): - line = line.rstrip('\r\n') - match = re.match('^#define\s+(\w+)\s+\(0x([0-9A-Fa-f]+)\)$', line) - if match and match.group(1).startswith('USBD_'): - key = match.group(1).replace('USBD', 'USB') + line = line.rstrip("\r\n") + match = re.match("^#define\s+(\w+)\s+\(0x([0-9A-Fa-f]+)\)$", line) + if match and match.group(1).startswith("USBD_"): + key = match.group(1).replace("USBD", "USB") val = match.group(2) print("key =", key, "val =", val) if key in needed_keys: @@ -30,9 +31,10 @@ def parse_usb_ids(filename): raise Exception("Unable to parse %s from %s" % (k, filename)) return rv + if __name__ == "__main__": usb_ids_file = sys.argv[1] template_file = sys.argv[2] replacements = parse_usb_ids(usb_ids_file) - for line in open(template_file, 'r').readlines(): - print(string.Template(line).safe_substitute(replacements), end='') + for line in open(template_file, "r").readlines(): + print(string.Template(line).safe_substitute(replacements), end="") diff --git a/tools/make-frozen.py b/tools/make-frozen.py index ad23b9d59..f9ffa7642 100755 --- a/tools/make-frozen.py +++ b/tools/make-frozen.py @@ -30,6 +30,7 @@ import os def module_name(f): return f + modules = [] root = sys.argv[1].rstrip("/") @@ -39,7 +40,7 @@ for dirpath, dirnames, filenames in os.walk(root): for f in filenames: fullpath = dirpath + "/" + f st = os.stat(fullpath) - modules.append((fullpath[root_len + 1:], st)) + modules.append((fullpath[root_len + 1 :], st)) print("#include <stdint.h>") print("const char mp_frozen_str_names[] = {") @@ -66,8 +67,8 @@ for f, st in modules: # data. We could just encode all characters as hex digits but it's nice # to be able to read the resulting C code as ASCII when possible. - data = bytearray(data) # so Python2 extracts each byte as an integer - esc_dict = {ord('\n'): '\\n', ord('\r'): '\\r', ord('"'): '\\"', ord('\\'): '\\\\'} + data = bytearray(data) # so Python2 extracts each byte as an integer + esc_dict = {ord("\n"): "\\n", ord("\r"): "\\r", ord('"'): '\\"', ord("\\"): "\\\\"} chrs = ['"'] break_str = False for c in data: @@ -80,9 +81,9 @@ for f, st in modules: break_str = False chrs.append(chr(c)) else: - chrs.append('\\x%02x' % c) + chrs.append("\\x%02x" % c) break_str = True chrs.append('\\0"') - print(''.join(chrs)) + print("".join(chrs)) print("};") diff --git a/tools/mpconfig_category_reader.py b/tools/mpconfig_category_reader.py index 2f813931e..7c2aede5b 100644 --- a/tools/mpconfig_category_reader.py +++ b/tools/mpconfig_category_reader.py @@ -1,4 +1,4 @@ -filepath = '../py/circuitpy_mpconfig.mk' +filepath = "../py/circuitpy_mpconfig.mk" with open(filepath) as fp: line = fp.readline() cnt = 1 diff --git a/tools/mpy-tool.py b/tools/mpy-tool.py index 95090466c..c989b6300 100755 --- a/tools/mpy-tool.py +++ b/tools/mpy-tool.py @@ -8,7 +8,8 @@ # Python 2/3 compatibility code from __future__ import print_function import platform -if platform.python_version_tuple()[0] == '2': + +if platform.python_version_tuple()[0] == "2": str_cons = lambda val, enc=None: val bytes_cons = lambda val, enc=None: bytearray(val) is_str_type = lambda o: type(o) is str @@ -26,22 +27,26 @@ import sys import struct from collections import namedtuple -sys.path.append(sys.path[0] + '/../py') +sys.path.append(sys.path[0] + "/../py") import makeqstrdata as qstrutil + class FreezeError(Exception): def __init__(self, rawcode, msg): self.rawcode = rawcode self.msg = msg def __str__(self): - return 'error while freezing %s: %s' % (self.rawcode.source_file, self.msg) + return "error while freezing %s: %s" % (self.rawcode.source_file, self.msg) + class Config: MPY_VERSION = 3 MICROPY_LONGINT_IMPL_NONE = 0 MICROPY_LONGINT_IMPL_LONGLONG = 1 MICROPY_LONGINT_IMPL_MPZ = 2 + + config = Config() MP_OPCODE_BYTE = 0 @@ -52,11 +57,11 @@ MP_OPCODE_OFFSET = 3 # extra bytes: MP_BC_MAKE_CLOSURE = 0x62 MP_BC_MAKE_CLOSURE_DEFARGS = 0x63 -MP_BC_RAISE_VARARGS = 0x5c +MP_BC_RAISE_VARARGS = 0x5C # extra byte if caching enabled: -MP_BC_LOAD_NAME = 0x1c -MP_BC_LOAD_GLOBAL = 0x1d -MP_BC_LOAD_ATTR = 0x1e +MP_BC_LOAD_NAME = 0x1C +MP_BC_LOAD_GLOBAL = 0x1D +MP_BC_LOAD_ATTR = 0x1E MP_BC_STORE_ATTR = 0x26 # load opcode names @@ -71,87 +76,86 @@ with open("../../py/bc0.h") as f: opcode = int(value.strip("()"), 0) opcode_names[opcode] = name + def make_opcode_format(): def OC4(a, b, c, d): return a | (b << 2) | (c << 4) | (d << 6) + U = 0 B = 0 Q = 1 V = 2 O = 3 - return bytes_cons(( - # this table is taken verbatim from py/bc.c - OC4(U, U, U, U), # 0x00-0x03 - OC4(U, U, U, U), # 0x04-0x07 - OC4(U, U, U, U), # 0x08-0x0b - OC4(U, U, U, U), # 0x0c-0x0f - OC4(B, B, B, U), # 0x10-0x13 - OC4(V, U, Q, V), # 0x14-0x17 - OC4(B, V, V, Q), # 0x18-0x1b - OC4(Q, Q, Q, Q), # 0x1c-0x1f - OC4(B, B, V, V), # 0x20-0x23 - OC4(Q, Q, Q, B), # 0x24-0x27 - OC4(V, V, Q, Q), # 0x28-0x2b - OC4(U, U, U, U), # 0x2c-0x2f - OC4(B, B, B, B), # 0x30-0x33 - OC4(B, O, O, O), # 0x34-0x37 - OC4(O, O, U, U), # 0x38-0x3b - OC4(U, O, B, O), # 0x3c-0x3f - OC4(O, B, B, O), # 0x40-0x43 - OC4(B, B, O, B), # 0x44-0x47 - OC4(U, U, U, U), # 0x48-0x4b - OC4(U, U, U, U), # 0x4c-0x4f - OC4(V, V, U, V), # 0x50-0x53 - OC4(B, U, V, V), # 0x54-0x57 - OC4(V, V, V, B), # 0x58-0x5b - OC4(B, B, B, U), # 0x5c-0x5f - OC4(V, V, V, V), # 0x60-0x63 - OC4(V, V, V, V), # 0x64-0x67 - OC4(Q, Q, B, U), # 0x68-0x6b - OC4(U, U, U, U), # 0x6c-0x6f - - OC4(B, B, B, B), # 0x70-0x73 - OC4(B, B, B, B), # 0x74-0x77 - OC4(B, B, B, B), # 0x78-0x7b - OC4(B, B, B, B), # 0x7c-0x7f - OC4(B, B, B, B), # 0x80-0x83 - OC4(B, B, B, B), # 0x84-0x87 - OC4(B, B, B, B), # 0x88-0x8b - OC4(B, B, B, B), # 0x8c-0x8f - OC4(B, B, B, B), # 0x90-0x93 - OC4(B, B, B, B), # 0x94-0x97 - OC4(B, B, B, B), # 0x98-0x9b - OC4(B, B, B, B), # 0x9c-0x9f - OC4(B, B, B, B), # 0xa0-0xa3 - OC4(B, B, B, B), # 0xa4-0xa7 - OC4(B, B, B, B), # 0xa8-0xab - OC4(B, B, B, B), # 0xac-0xaf - - OC4(B, B, B, B), # 0xb0-0xb3 - OC4(B, B, B, B), # 0xb4-0xb7 - OC4(B, B, B, B), # 0xb8-0xbb - OC4(B, B, B, B), # 0xbc-0xbf - - OC4(B, B, B, B), # 0xc0-0xc3 - OC4(B, B, B, B), # 0xc4-0xc7 - OC4(B, B, B, B), # 0xc8-0xcb - OC4(B, B, B, B), # 0xcc-0xcf - - OC4(B, B, B, B), # 0xd0-0xd3 - OC4(U, U, U, B), # 0xd4-0xd7 - OC4(B, B, B, B), # 0xd8-0xdb - OC4(B, B, B, B), # 0xdc-0xdf - - OC4(B, B, B, B), # 0xe0-0xe3 - OC4(B, B, B, B), # 0xe4-0xe7 - OC4(B, B, B, B), # 0xe8-0xeb - OC4(B, B, B, B), # 0xec-0xef - - OC4(B, B, B, B), # 0xf0-0xf3 - OC4(B, B, B, B), # 0xf4-0xf7 - OC4(U, U, U, U), # 0xf8-0xfb - OC4(U, U, U, U), # 0xfc-0xff - )) + return bytes_cons( + ( + # this table is taken verbatim from py/bc.c + OC4(U, U, U, U), # 0x00-0x03 + OC4(U, U, U, U), # 0x04-0x07 + OC4(U, U, U, U), # 0x08-0x0b + OC4(U, U, U, U), # 0x0c-0x0f + OC4(B, B, B, U), # 0x10-0x13 + OC4(V, U, Q, V), # 0x14-0x17 + OC4(B, V, V, Q), # 0x18-0x1b + OC4(Q, Q, Q, Q), # 0x1c-0x1f + OC4(B, B, V, V), # 0x20-0x23 + OC4(Q, Q, Q, B), # 0x24-0x27 + OC4(V, V, Q, Q), # 0x28-0x2b + OC4(U, U, U, U), # 0x2c-0x2f + OC4(B, B, B, B), # 0x30-0x33 + OC4(B, O, O, O), # 0x34-0x37 + OC4(O, O, U, U), # 0x38-0x3b + OC4(U, O, B, O), # 0x3c-0x3f + OC4(O, B, B, O), # 0x40-0x43 + OC4(B, B, O, B), # 0x44-0x47 + OC4(U, U, U, U), # 0x48-0x4b + OC4(U, U, U, U), # 0x4c-0x4f + OC4(V, V, U, V), # 0x50-0x53 + OC4(B, U, V, V), # 0x54-0x57 + OC4(V, V, V, B), # 0x58-0x5b + OC4(B, B, B, U), # 0x5c-0x5f + OC4(V, V, V, V), # 0x60-0x63 + OC4(V, V, V, V), # 0x64-0x67 + OC4(Q, Q, B, U), # 0x68-0x6b + OC4(U, U, U, U), # 0x6c-0x6f + OC4(B, B, B, B), # 0x70-0x73 + OC4(B, B, B, B), # 0x74-0x77 + OC4(B, B, B, B), # 0x78-0x7b + OC4(B, B, B, B), # 0x7c-0x7f + OC4(B, B, B, B), # 0x80-0x83 + OC4(B, B, B, B), # 0x84-0x87 + OC4(B, B, B, B), # 0x88-0x8b + OC4(B, B, B, B), # 0x8c-0x8f + OC4(B, B, B, B), # 0x90-0x93 + OC4(B, B, B, B), # 0x94-0x97 + OC4(B, B, B, B), # 0x98-0x9b + OC4(B, B, B, B), # 0x9c-0x9f + OC4(B, B, B, B), # 0xa0-0xa3 + OC4(B, B, B, B), # 0xa4-0xa7 + OC4(B, B, B, B), # 0xa8-0xab + OC4(B, B, B, B), # 0xac-0xaf + OC4(B, B, B, B), # 0xb0-0xb3 + OC4(B, B, B, B), # 0xb4-0xb7 + OC4(B, B, B, B), # 0xb8-0xbb + OC4(B, B, B, B), # 0xbc-0xbf + OC4(B, B, B, B), # 0xc0-0xc3 + OC4(B, B, B, B), # 0xc4-0xc7 + OC4(B, B, B, B), # 0xc8-0xcb + OC4(B, B, B, B), # 0xcc-0xcf + OC4(B, B, B, B), # 0xd0-0xd3 + OC4(U, U, U, B), # 0xd4-0xd7 + OC4(B, B, B, B), # 0xd8-0xdb + OC4(B, B, B, B), # 0xdc-0xdf + OC4(B, B, B, B), # 0xe0-0xe3 + OC4(B, B, B, B), # 0xe4-0xe7 + OC4(B, B, B, B), # 0xe8-0xeb + OC4(B, B, B, B), # 0xec-0xef + OC4(B, B, B, B), # 0xf0-0xf3 + OC4(B, B, B, B), # 0xf4-0xf7 + OC4(U, U, U, U), # 0xf8-0xfb + OC4(U, U, U, U), # 0xfc-0xff + ) + ) + # this function mirrors that in py/bc.c def mp_opcode_format(bytecode, ip, opcode_format=make_opcode_format()): @@ -165,7 +169,8 @@ def mp_opcode_format(bytecode, ip, opcode_format=make_opcode_format()): opcode == MP_BC_RAISE_VARARGS or opcode == MP_BC_MAKE_CLOSURE or opcode == MP_BC_MAKE_CLOSURE_DEFARGS - or config.MICROPY_OPT_CACHE_MAP_LOOKUP_IN_BYTECODE and ( + or config.MICROPY_OPT_CACHE_MAP_LOOKUP_IN_BYTECODE + and ( opcode == MP_BC_LOAD_NAME or opcode == MP_BC_LOAD_GLOBAL or opcode == MP_BC_LOAD_ATTR @@ -182,32 +187,51 @@ def mp_opcode_format(bytecode, ip, opcode_format=make_opcode_format()): ip += extra_byte return f, ip - ip_start + def decode_uint(bytecode, ip): unum = 0 while True: val = bytecode[ip] ip += 1 - unum = (unum << 7) | (val & 0x7f) + unum = (unum << 7) | (val & 0x7F) if not (val & 0x80): break return ip, unum + def extract_prelude(bytecode): ip = 0 ip, n_state = decode_uint(bytecode, ip) ip, n_exc_stack = decode_uint(bytecode, ip) - scope_flags = bytecode[ip]; ip += 1 - n_pos_args = bytecode[ip]; ip += 1 - n_kwonly_args = bytecode[ip]; ip += 1 - n_def_pos_args = bytecode[ip]; ip += 1 + scope_flags = bytecode[ip] + ip += 1 + n_pos_args = bytecode[ip] + ip += 1 + n_kwonly_args = bytecode[ip] + ip += 1 + n_def_pos_args = bytecode[ip] + ip += 1 ip2, code_info_size = decode_uint(bytecode, ip) ip += code_info_size - while bytecode[ip] != 0xff: + while bytecode[ip] != 0xFF: ip += 1 ip += 1 # ip now points to first opcode # ip2 points to simple_name qstr - return ip, ip2, (n_state, n_exc_stack, scope_flags, n_pos_args, n_kwonly_args, n_def_pos_args, code_info_size) + return ( + ip, + ip2, + ( + n_state, + n_exc_stack, + scope_flags, + n_pos_args, + n_kwonly_args, + n_def_pos_args, + code_info_size, + ), + ) + class RawCode: # a set of all escaped names, to make sure they are unique @@ -232,7 +256,7 @@ class RawCode: def dump(self): # dump children first for rc in self.raw_codes: - rc.freeze('') + rc.freeze("") # TODO def freeze(self, parent_name): @@ -245,35 +269,44 @@ class RawCode: i += 1 RawCode.escaped_names.add(self.escaped_name) - sizes = {"bytecode": 0, "strings": 0, "raw_code_overhead": 0, "const_table_overhead": 0, "string_overhead": 0, "number_overhead": 0} + sizes = { + "bytecode": 0, + "strings": 0, + "raw_code_overhead": 0, + "const_table_overhead": 0, + "string_overhead": 0, + "number_overhead": 0, + } # emit children first for rc in self.raw_codes: - subsize = rc.freeze(self.escaped_name + '_') + subsize = rc.freeze(self.escaped_name + "_") for k in sizes: sizes[k] += subsize[k] - # generate bytecode data print() - print('// frozen bytecode for file %s, scope %s%s' % (self.source_file.str, parent_name, self.simple_name.str)) + print( + "// frozen bytecode for file %s, scope %s%s" + % (self.source_file.str, parent_name, self.simple_name.str) + ) print("// bytecode size", len(self.bytecode)) - print('STATIC ', end='') + print("STATIC ", end="") if not config.MICROPY_OPT_CACHE_MAP_LOOKUP_IN_BYTECODE: - print('const ', end='') - print('byte bytecode_data_%s[%u] = {' % (self.escaped_name, len(self.bytecode))) + print("const ", end="") + print("byte bytecode_data_%s[%u] = {" % (self.escaped_name, len(self.bytecode))) sizes["bytecode"] += len(self.bytecode) - print(' ', end='') + print(" ", end="") for i in range(self.ip2): - print(' 0x%02x,' % self.bytecode[i], end='') + print(" 0x%02x," % self.bytecode[i], end="") print() print(" // simple name") - print(' ', self.simple_name.qstr_id, '& 0xff,', self.simple_name.qstr_id, '>> 8,') + print(" ", self.simple_name.qstr_id, "& 0xff,", self.simple_name.qstr_id, ">> 8,") print(" // source file") - print(' ', self.source_file.qstr_id, '& 0xff,', self.source_file.qstr_id, '>> 8,') + print(" ", self.source_file.qstr_id, "& 0xff,", self.source_file.qstr_id, ">> 8,") print(" // code info") - print(' ', end='') + print(" ", end="") for i in range(self.ip2 + 4, self.ip): - print(' 0x%02x,' % self.bytecode[i], end='') + print(" 0x%02x," % self.bytecode[i], end="") print() print(" // bytecode") ip = self.ip @@ -283,39 +316,51 @@ class RawCode: if opcode in opcode_names: opcode = opcode_names[opcode] else: - opcode = '0x%02x' % opcode + opcode = "0x%02x" % opcode if f == 1: qst = self._unpack_qstr(ip + 1).qstr_id - print(' {}, {} & 0xff, {} >> 8,'.format(opcode, qst, qst)) + print(" {}, {} & 0xff, {} >> 8,".format(opcode, qst, qst)) else: - print(' {},{}'.format(opcode, ''.join(' 0x%02x,' % self.bytecode[ip + i] for i in range(1, sz)))) + print( + " {},{}".format( + opcode, "".join(" 0x%02x," % self.bytecode[ip + i] for i in range(1, sz)) + ) + ) ip += sz - print('};') + print("};") # generate constant objects for i, obj in enumerate(self.objs): - obj_name = 'const_obj_%s_%u' % (self.escaped_name, i) + obj_name = "const_obj_%s_%u" % (self.escaped_name, i) if obj is Ellipsis: - print('#define %s mp_const_ellipsis_obj' % obj_name) + print("#define %s mp_const_ellipsis_obj" % obj_name) elif is_str_type(obj) or is_bytes_type(obj): if is_str_type(obj): - obj = bytes_cons(obj, 'utf8') - obj_type = 'mp_type_str' + obj = bytes_cons(obj, "utf8") + obj_type = "mp_type_str" else: - obj_type = 'mp_type_bytes' - print('STATIC const mp_obj_str_t %s = {{&%s}, %u, %u, (const byte*)"%s"}; // %s' - % (obj_name, obj_type, qstrutil.compute_hash(obj, config.MICROPY_QSTR_BYTES_IN_HASH), - len(obj), ''.join(('\\x%02x' % b) for b in obj), obj)) + obj_type = "mp_type_bytes" + print( + 'STATIC const mp_obj_str_t %s = {{&%s}, %u, %u, (const byte*)"%s"}; // %s' + % ( + obj_name, + obj_type, + qstrutil.compute_hash(obj, config.MICROPY_QSTR_BYTES_IN_HASH), + len(obj), + "".join(("\\x%02x" % b) for b in obj), + obj, + ) + ) sizes["strings"] += len(obj) sizes["string_overhead"] += 16 elif is_int_type(obj): if config.MICROPY_LONGINT_IMPL == config.MICROPY_LONGINT_IMPL_NONE: # TODO check if we can actually fit this long-int into a small-int - raise FreezeError(self, 'target does not support long int') + raise FreezeError(self, "target does not support long int") elif config.MICROPY_LONGINT_IMPL == config.MICROPY_LONGINT_IMPL_LONGLONG: # TODO - raise FreezeError(self, 'freezing int to long-long is not implemented') + raise FreezeError(self, "freezing int to long-long is not implemented") elif config.MICROPY_LONGINT_IMPL == config.MICROPY_LONGINT_IMPL_MPZ: neg = 0 if obj < 0: @@ -328,117 +373,137 @@ class RawCode: digs.append(z & ((1 << bits_per_dig) - 1)) z >>= bits_per_dig ndigs = len(digs) - digs = ','.join(('%#x' % d) for d in digs) - print('STATIC const mp_obj_int_t %s = {{&mp_type_int}, ' - '{.neg=%u, .fixed_dig=1, .alloc=%u, .len=%u, .dig=(uint%u_t[]){%s}}};' - % (obj_name, neg, ndigs, ndigs, bits_per_dig, digs)) + digs = ",".join(("%#x" % d) for d in digs) + print( + "STATIC const mp_obj_int_t %s = {{&mp_type_int}, " + "{.neg=%u, .fixed_dig=1, .alloc=%u, .len=%u, .dig=(uint%u_t[]){%s}}};" + % (obj_name, neg, ndigs, ndigs, bits_per_dig, digs) + ) sizes["number_overhead"] += 16 elif type(obj) is float: - print('#if MICROPY_OBJ_REPR == MICROPY_OBJ_REPR_A || MICROPY_OBJ_REPR == MICROPY_OBJ_REPR_B') - print('STATIC const mp_obj_float_t %s = {{&mp_type_float}, %.16g};' - % (obj_name, obj)) - print('#endif') + print( + "#if MICROPY_OBJ_REPR == MICROPY_OBJ_REPR_A || MICROPY_OBJ_REPR == MICROPY_OBJ_REPR_B" + ) + print( + "STATIC const mp_obj_float_t %s = {{&mp_type_float}, %.16g};" % (obj_name, obj) + ) + print("#endif") sizes["number_overhead"] += 8 elif type(obj) is complex: - print('STATIC const mp_obj_complex_t %s = {{&mp_type_complex}, %.16g, %.16g};' - % (obj_name, obj.real, obj.imag)) + print( + "STATIC const mp_obj_complex_t %s = {{&mp_type_complex}, %.16g, %.16g};" + % (obj_name, obj.real, obj.imag) + ) sizes["number_overhead"] += 12 else: - raise FreezeError(self, 'freezing of object %r is not implemented' % (obj,)) + raise FreezeError(self, "freezing of object %r is not implemented" % (obj,)) # generate constant table, if it has any entries const_table_len = len(self.qstrs) + len(self.objs) + len(self.raw_codes) if const_table_len: - print('STATIC const mp_rom_obj_t const_table_data_%s[%u] = {' - % (self.escaped_name, const_table_len)) + print( + "STATIC const mp_rom_obj_t const_table_data_%s[%u] = {" + % (self.escaped_name, const_table_len) + ) for qst in self.qstrs: sizes["const_table_overhead"] += 4 - print(' MP_ROM_QSTR(%s),' % global_qstrs[qst].qstr_id) + print(" MP_ROM_QSTR(%s)," % global_qstrs[qst].qstr_id) for i in range(len(self.objs)): sizes["const_table_overhead"] += 4 if type(self.objs[i]) is float: - print('#if MICROPY_OBJ_REPR == MICROPY_OBJ_REPR_A || MICROPY_OBJ_REPR == MICROPY_OBJ_REPR_B') - print(' MP_ROM_PTR(&const_obj_%s_%u),' % (self.escaped_name, i)) - print('#elif MICROPY_OBJ_REPR == MICROPY_OBJ_REPR_C') - n = struct.unpack('<I', struct.pack('<f', self.objs[i]))[0] + print( + "#if MICROPY_OBJ_REPR == MICROPY_OBJ_REPR_A || MICROPY_OBJ_REPR == MICROPY_OBJ_REPR_B" + ) + print(" MP_ROM_PTR(&const_obj_%s_%u)," % (self.escaped_name, i)) + print("#elif MICROPY_OBJ_REPR == MICROPY_OBJ_REPR_C") + n = struct.unpack("<I", struct.pack("<f", self.objs[i]))[0] n = ((n & ~0x3) | 2) + 0x80800000 - print(' (mp_rom_obj_t)(0x%08x),' % (n,)) - print('#else') - print('#error "MICROPY_OBJ_REPR_D not supported with floats in frozen mpy files"') - print('#endif') + print(" (mp_rom_obj_t)(0x%08x)," % (n,)) + print("#else") + print( + '#error "MICROPY_OBJ_REPR_D not supported with floats in frozen mpy files"' + ) + print("#endif") else: - print(' MP_ROM_PTR(&const_obj_%s_%u),' % (self.escaped_name, i)) + print(" MP_ROM_PTR(&const_obj_%s_%u)," % (self.escaped_name, i)) for rc in self.raw_codes: sizes["const_table_overhead"] += 4 - print(' MP_ROM_PTR(&raw_code_%s),' % rc.escaped_name) - print('};') + print(" MP_ROM_PTR(&raw_code_%s)," % rc.escaped_name) + print("};") # generate module - if self.simple_name.str != '<module>': - print('STATIC ', end='') - print('const mp_raw_code_t raw_code_%s = {' % self.escaped_name) - print(' .kind = MP_CODE_BYTECODE,') - print(' .scope_flags = 0x%02x,' % self.prelude[2]) - print(' .n_pos_args = %u,' % self.prelude[3]) - print(' .data.u_byte = {') - print(' .bytecode = bytecode_data_%s,' % self.escaped_name) + if self.simple_name.str != "<module>": + print("STATIC ", end="") + print("const mp_raw_code_t raw_code_%s = {" % self.escaped_name) + print(" .kind = MP_CODE_BYTECODE,") + print(" .scope_flags = 0x%02x," % self.prelude[2]) + print(" .n_pos_args = %u," % self.prelude[3]) + print(" .data.u_byte = {") + print(" .bytecode = bytecode_data_%s," % self.escaped_name) if const_table_len: - print(' .const_table = (mp_uint_t*)const_table_data_%s,' % self.escaped_name) + print(" .const_table = (mp_uint_t*)const_table_data_%s," % self.escaped_name) else: - print(' .const_table = NULL,') - print(' #if MICROPY_PERSISTENT_CODE_SAVE') - print(' .bc_len = %u,' % len(self.bytecode)) - print(' .n_obj = %u,' % len(self.objs)) - print(' .n_raw_code = %u,' % len(self.raw_codes)) - print(' #endif') - print(' },') - print('};') + print(" .const_table = NULL,") + print(" #if MICROPY_PERSISTENT_CODE_SAVE") + print(" .bc_len = %u," % len(self.bytecode)) + print(" .n_obj = %u," % len(self.objs)) + print(" .n_raw_code = %u," % len(self.raw_codes)) + print(" #endif") + print(" },") + print("};") sizes["raw_code_overhead"] += 16 return sizes + def read_uint(f): i = 0 while True: b = bytes_cons(f.read(1))[0] - i = (i << 7) | (b & 0x7f) + i = (i << 7) | (b & 0x7F) if b & 0x80 == 0: break return i + global_qstrs = [] -qstr_type = namedtuple('qstr', ('str', 'qstr_esc', 'qstr_id')) +qstr_type = namedtuple("qstr", ("str", "qstr_esc", "qstr_id")) + + def read_qstr(f): ln = read_uint(f) - data = str_cons(f.read(ln), 'utf8') + data = str_cons(f.read(ln), "utf8") qstr_esc = qstrutil.qstr_escape(data) - global_qstrs.append(qstr_type(data, qstr_esc, 'MP_QSTR_' + qstr_esc)) + global_qstrs.append(qstr_type(data, qstr_esc, "MP_QSTR_" + qstr_esc)) return len(global_qstrs) - 1 + def read_obj(f): obj_type = f.read(1) - if obj_type == b'e': + if obj_type == b"e": return Ellipsis else: buf = f.read(read_uint(f)) - if obj_type == b's': - return str_cons(buf, 'utf8') - elif obj_type == b'b': + if obj_type == b"s": + return str_cons(buf, "utf8") + elif obj_type == b"b": return bytes_cons(buf) - elif obj_type == b'i': - return int(str_cons(buf, 'ascii'), 10) - elif obj_type == b'f': - return float(str_cons(buf, 'ascii')) - elif obj_type == b'c': - return complex(str_cons(buf, 'ascii')) + elif obj_type == b"i": + return int(str_cons(buf, "ascii"), 10) + elif obj_type == b"f": + return float(str_cons(buf, "ascii")) + elif obj_type == b"c": + return complex(str_cons(buf, "ascii")) else: assert 0 + def read_qstr_and_pack(f, bytecode, ip): qst = read_qstr(f) - bytecode[ip] = qst & 0xff + bytecode[ip] = qst & 0xFF bytecode[ip + 1] = qst >> 8 + def read_bytecode_qstrs(file, bytecode, ip): while ip < len(bytecode): f, sz = mp_opcode_format(bytecode, ip) @@ -446,12 +511,13 @@ def read_bytecode_qstrs(file, bytecode, ip): read_qstr_and_pack(file, bytecode, ip + 1) ip += sz + def read_raw_code(f): bc_len = read_uint(f) bytecode = bytearray(f.read(bc_len)) ip, ip2, prelude = extract_prelude(bytecode) - read_qstr_and_pack(f, bytecode, ip2) # simple_name - read_qstr_and_pack(f, bytecode, ip2 + 2) # source_file + read_qstr_and_pack(f, bytecode, ip2) # simple_name + read_qstr_and_pack(f, bytecode, ip2 + 2) # source_file read_bytecode_qstrs(f, bytecode, ip) n_obj = read_uint(f) n_raw_code = read_uint(f) @@ -460,23 +526,26 @@ def read_raw_code(f): raw_codes = [read_raw_code(f) for _ in range(n_raw_code)] return RawCode(bytecode, qstrs, objs, raw_codes) + def read_mpy(filename): - with open(filename, 'rb') as f: + with open(filename, "rb") as f: header = bytes_cons(f.read(4)) - if header[0] != ord('M'): - raise Exception('not a valid .mpy file') + if header[0] != ord("M"): + raise Exception("not a valid .mpy file") if header[1] != config.MPY_VERSION: - raise Exception('incompatible .mpy version') + raise Exception("incompatible .mpy version") feature_flags = header[2] config.MICROPY_OPT_CACHE_MAP_LOOKUP_IN_BYTECODE = (feature_flags & 1) != 0 config.MICROPY_PY_BUILTINS_STR_UNICODE = (feature_flags & 2) != 0 config.mp_small_int_bits = header[3] return read_raw_code(f) + def dump_mpy(raw_codes): for rc in raw_codes: rc.dump() + def freeze_mpy(base_qstrs, raw_codes): # add to qstrs new = {} @@ -494,71 +563,79 @@ def freeze_mpy(base_qstrs, raw_codes): print('#include "py/emitglue.h"') print() - print('#if MICROPY_OPT_CACHE_MAP_LOOKUP_IN_BYTECODE != %u' % config.MICROPY_OPT_CACHE_MAP_LOOKUP_IN_BYTECODE) + print( + "#if MICROPY_OPT_CACHE_MAP_LOOKUP_IN_BYTECODE != %u" + % config.MICROPY_OPT_CACHE_MAP_LOOKUP_IN_BYTECODE + ) print('#error "incompatible MICROPY_OPT_CACHE_MAP_LOOKUP_IN_BYTECODE"') - print('#endif') + print("#endif") print() - print('#if MICROPY_LONGINT_IMPL != %u' % config.MICROPY_LONGINT_IMPL) + print("#if MICROPY_LONGINT_IMPL != %u" % config.MICROPY_LONGINT_IMPL) print('#error "incompatible MICROPY_LONGINT_IMPL"') - print('#endif') + print("#endif") print() if config.MICROPY_LONGINT_IMPL == config.MICROPY_LONGINT_IMPL_MPZ: - print('#if MPZ_DIG_SIZE != %u' % config.MPZ_DIG_SIZE) + print("#if MPZ_DIG_SIZE != %u" % config.MPZ_DIG_SIZE) print('#error "incompatible MPZ_DIG_SIZE"') - print('#endif') + print("#endif") print() - - print('#if MICROPY_PY_BUILTINS_FLOAT') - print('typedef struct _mp_obj_float_t {') - print(' mp_obj_base_t base;') - print(' mp_float_t value;') - print('} mp_obj_float_t;') - print('#endif') + print("#if MICROPY_PY_BUILTINS_FLOAT") + print("typedef struct _mp_obj_float_t {") + print(" mp_obj_base_t base;") + print(" mp_float_t value;") + print("} mp_obj_float_t;") + print("#endif") print() - print('#if MICROPY_PY_BUILTINS_COMPLEX') - print('typedef struct _mp_obj_complex_t {') - print(' mp_obj_base_t base;') - print(' mp_float_t real;') - print(' mp_float_t imag;') - print('} mp_obj_complex_t;') - print('#endif') + print("#if MICROPY_PY_BUILTINS_COMPLEX") + print("typedef struct _mp_obj_complex_t {") + print(" mp_obj_base_t base;") + print(" mp_float_t real;") + print(" mp_float_t imag;") + print("} mp_obj_complex_t;") + print("#endif") print() - print('enum {') + print("enum {") for i in range(len(new)): if i == 0: - print(' MP_QSTR_%s = MP_QSTRnumber_of,' % new[i][1]) + print(" MP_QSTR_%s = MP_QSTRnumber_of," % new[i][1]) else: - print(' MP_QSTR_%s,' % new[i][1]) - print('};') + print(" MP_QSTR_%s," % new[i][1]) + print("};") print() - print('extern const qstr_pool_t mp_qstr_const_pool;'); - print('const qstr_pool_t mp_qstr_frozen_const_pool = {') - print(' (qstr_pool_t*)&mp_qstr_const_pool, // previous pool') - print(' MP_QSTRnumber_of, // previous pool size') - print(' %u, // allocated entries' % len(new)) - print(' %u, // used entries' % len(new)) - print(' {') + print("extern const qstr_pool_t mp_qstr_const_pool;") + print("const qstr_pool_t mp_qstr_frozen_const_pool = {") + print(" (qstr_pool_t*)&mp_qstr_const_pool, // previous pool") + print(" MP_QSTRnumber_of, // previous pool size") + print(" %u, // allocated entries" % len(new)) + print(" %u, // used entries" % len(new)) + print(" {") qstr_size = {"metadata": 0, "data": 0} for _, _, qstr in new: - qstr_size["metadata"] += config.MICROPY_QSTR_BYTES_IN_LEN + config.MICROPY_QSTR_BYTES_IN_HASH + qstr_size["metadata"] += ( + config.MICROPY_QSTR_BYTES_IN_LEN + config.MICROPY_QSTR_BYTES_IN_HASH + ) qstr_size["data"] += len(qstr) - print(' %s,' - % qstrutil.make_bytes(config.MICROPY_QSTR_BYTES_IN_LEN, config.MICROPY_QSTR_BYTES_IN_HASH, qstr)) - print(' },') - print('};') + print( + " %s," + % qstrutil.make_bytes( + config.MICROPY_QSTR_BYTES_IN_LEN, config.MICROPY_QSTR_BYTES_IN_HASH, qstr + ) + ) + print(" },") + print("};") sizes = {} for rc in raw_codes: - sizes[rc.source_file.str] = rc.freeze(rc.source_file.str.replace('/', '_')[:-3] + '_') + sizes[rc.source_file.str] = rc.freeze(rc.source_file.str.replace("/", "_")[:-3] + "_") print() - print('const char mp_frozen_mpy_names[] = {') + print("const char mp_frozen_mpy_names[] = {") qstr_size["filenames"] = 1 for rc in raw_codes: module_name = rc.source_file.str @@ -566,53 +643,62 @@ def freeze_mpy(base_qstrs, raw_codes): qstr_size["filenames"] += len(module_name) + 1 print('"\\0"};') - print('const mp_raw_code_t *const mp_frozen_mpy_content[] = {') + print("const mp_raw_code_t *const mp_frozen_mpy_content[] = {") for rc in raw_codes: - print(' &raw_code_%s,' % rc.escaped_name) + print(" &raw_code_%s," % rc.escaped_name) size = sizes[rc.source_file.str] - print(' // Total size:', sum(size.values())) + print(" // Total size:", sum(size.values())) for k in size: print(" // {} {}".format(k, size[k])) - print('};') + print("};") print() - print('// Total size:', sum([sum(x.values()) for x in sizes.values()]) + sum(qstr_size.values())) + print( + "// Total size:", sum([sum(x.values()) for x in sizes.values()]) + sum(qstr_size.values()) + ) for k in size: total = sum([x[k] for x in sizes.values()]) print("// {} {}".format(k, total)) for k in qstr_size: print("// qstr {} {}".format(k, qstr_size[k])) + def main(): import argparse - cmd_parser = argparse.ArgumentParser(description='A tool to work with MicroPython .mpy files.') - cmd_parser.add_argument('-d', '--dump', action='store_true', - help='dump contents of files') - cmd_parser.add_argument('-f', '--freeze', action='store_true', - help='freeze files') - cmd_parser.add_argument('-q', '--qstr-header', - help='qstr header file to freeze against') - cmd_parser.add_argument('-mlongint-impl', choices=['none', 'longlong', 'mpz'], default='mpz', - help='long-int implementation used by target (default mpz)') - cmd_parser.add_argument('-mmpz-dig-size', metavar='N', type=int, default=16, - help='mpz digit size used by target (default 16)') - cmd_parser.add_argument('files', nargs='+', - help='input .mpy files') + + cmd_parser = argparse.ArgumentParser(description="A tool to work with MicroPython .mpy files.") + cmd_parser.add_argument("-d", "--dump", action="store_true", help="dump contents of files") + cmd_parser.add_argument("-f", "--freeze", action="store_true", help="freeze files") + cmd_parser.add_argument("-q", "--qstr-header", help="qstr header file to freeze against") + cmd_parser.add_argument( + "-mlongint-impl", + choices=["none", "longlong", "mpz"], + default="mpz", + help="long-int implementation used by target (default mpz)", + ) + cmd_parser.add_argument( + "-mmpz-dig-size", + metavar="N", + type=int, + default=16, + help="mpz digit size used by target (default 16)", + ) + cmd_parser.add_argument("files", nargs="+", help="input .mpy files") args = cmd_parser.parse_args() # set config values relevant to target machine config.MICROPY_LONGINT_IMPL = { - 'none':config.MICROPY_LONGINT_IMPL_NONE, - 'longlong':config.MICROPY_LONGINT_IMPL_LONGLONG, - 'mpz':config.MICROPY_LONGINT_IMPL_MPZ, + "none": config.MICROPY_LONGINT_IMPL_NONE, + "longlong": config.MICROPY_LONGINT_IMPL_LONGLONG, + "mpz": config.MICROPY_LONGINT_IMPL_MPZ, }[args.mlongint_impl] config.MPZ_DIG_SIZE = args.mmpz_dig_size # set config values for qstrs, and get the existing base set of qstrs if args.qstr_header: qcfgs, base_qstrs, _ = qstrutil.parse_input_headers([args.qstr_header]) - config.MICROPY_QSTR_BYTES_IN_LEN = int(qcfgs['BYTES_IN_LEN']) - config.MICROPY_QSTR_BYTES_IN_HASH = int(qcfgs['BYTES_IN_HASH']) + config.MICROPY_QSTR_BYTES_IN_LEN = int(qcfgs["BYTES_IN_LEN"]) + config.MICROPY_QSTR_BYTES_IN_HASH = int(qcfgs["BYTES_IN_HASH"]) else: config.MICROPY_QSTR_BYTES_IN_LEN = 1 config.MICROPY_QSTR_BYTES_IN_HASH = 1 @@ -629,5 +715,6 @@ def main(): print(er, file=sys.stderr) sys.exit(1) -if __name__ == '__main__': + +if __name__ == "__main__": main() diff --git a/tools/mpy_cross_all.py b/tools/mpy_cross_all.py index 5d9f8bc0b..9c217400f 100755 --- a/tools/mpy_cross_all.py +++ b/tools/mpy_cross_all.py @@ -11,14 +11,13 @@ import os.path argparser = argparse.ArgumentParser(description="Compile all .py files to .mpy recursively") argparser.add_argument("-o", "--out", help="output directory (default: input dir)") argparser.add_argument("--target", help="select MicroPython target config") -argparser.add_argument("-mcache-lookup-bc", action="store_true", help="cache map lookups in the bytecode") +argparser.add_argument( + "-mcache-lookup-bc", action="store_true", help="cache map lookups in the bytecode" +) argparser.add_argument("dir", help="input directory") args = argparser.parse_args() -TARGET_OPTS = { - "unix": "-mcache-lookup-bc", - "baremetal": "", -} +TARGET_OPTS = {"unix": "-mcache-lookup-bc", "baremetal": ""} args.dir = args.dir.rstrip("/") @@ -31,13 +30,17 @@ for path, subdirs, files in os.walk(args.dir): for f in files: if f.endswith(".py"): fpath = path + "/" + f - #print(fpath) + # print(fpath) out_fpath = args.out + "/" + fpath[path_prefix_len:-3] + ".mpy" out_dir = os.path.dirname(out_fpath) if not os.path.isdir(out_dir): os.makedirs(out_dir) - cmd = "mpy-cross -v -v %s -s %s %s -o %s" % (TARGET_OPTS.get(args.target, ""), - fpath[path_prefix_len:], fpath, out_fpath) - #print(cmd) + cmd = "mpy-cross -v -v %s -s %s %s -o %s" % ( + TARGET_OPTS.get(args.target, ""), + fpath[path_prefix_len:], + fpath, + out_fpath, + ) + # print(cmd) res = os.system(cmd) assert res == 0 diff --git a/tools/preprocess_frozen_modules.py b/tools/preprocess_frozen_modules.py index b75a2e723..294df317b 100755 --- a/tools/preprocess_frozen_modules.py +++ b/tools/preprocess_frozen_modules.py @@ -13,25 +13,35 @@ import subprocess # Compatible with Python 3.4 due to travis using trusty as default. + def version_string(path=None, *, valid_semver=False): version = None try: - tag = subprocess.check_output('git describe --tags --exact-match', shell=True, cwd=path) + tag = subprocess.check_output("git describe --tags --exact-match", shell=True, cwd=path) version = tag.strip().decode("utf-8", "strict") except subprocess.CalledProcessError: describe = subprocess.check_output("git describe --tags", shell=True, cwd=path) - tag, additional_commits, commitish = describe.strip().decode("utf-8", "strict").rsplit("-", maxsplit=2) + tag, additional_commits, commitish = ( + describe.strip().decode("utf-8", "strict").rsplit("-", maxsplit=2) + ) commitish = commitish[1:] if valid_semver: version_info = semver.parse_version_info(tag) if not version_info.prerelease: - version = semver.bump_patch(tag) + "-alpha.0.plus." + additional_commits + "+" + commitish + version = ( + semver.bump_patch(tag) + + "-alpha.0.plus." + + additional_commits + + "+" + + commitish + ) else: version = tag + ".plus." + additional_commits + "+" + commitish else: version = commitish return version + # Visit all the .py files in topdir. Replace any __version__ = "0.0.0-auto.0" type of info # with actual version info derived from git. def copy_and_process(in_dir, out_dir): @@ -39,14 +49,14 @@ def copy_and_process(in_dir, out_dir): # Skip library examples directory and subfolders. relative_path_parts = Path(root).relative_to(in_dir).parts - if relative_path_parts and relative_path_parts[0] in ['examples', 'docs', 'tests']: + if relative_path_parts and relative_path_parts[0] in ["examples", "docs", "tests"]: del subdirs[:] continue for file in files: # Skip top-level setup.py (module install info) and conf.py (sphinx config), # which are not part of the library - if (root == in_dir) and file in ('conf.py', 'setup.py'): + if (root == in_dir) and file in ("conf.py", "setup.py"): continue input_file_path = Path(root, file) @@ -62,15 +72,22 @@ def copy_and_process(in_dir, out_dir): line = line.replace("0.0.0-auto.0", module_version) output.write(line) -if __name__ == '__main__': - argparser = argparse.ArgumentParser(description="""\ + +if __name__ == "__main__": + argparser = argparse.ArgumentParser( + description="""\ Copy and pre-process .py files into output directory, before freezing. 1. Remove top-level repo directory. 2. Update __version__ info. 3. Remove examples. - 4. Remove non-library setup.py and conf.py""") - argparser.add_argument("in_dirs", metavar="input-dir", nargs="+", - help="top-level code dirs (may be git repo dirs)") + 4. Remove non-library setup.py and conf.py""" + ) + argparser.add_argument( + "in_dirs", + metavar="input-dir", + nargs="+", + help="top-level code dirs (may be git repo dirs)", + ) argparser.add_argument("-o", "--out_dir", help="output directory") args = argparser.parse_args() diff --git a/tools/print_status.py b/tools/print_status.py index 9a8b311dc..50535502e 100755 --- a/tools/print_status.py +++ b/tools/print_status.py @@ -5,15 +5,19 @@ # SPDX-License-Identifier: MIT import sys + if len(sys.argv) != 2: - print("""\ + print( + """\ Usage: print_status.py STATUS_FILENAME STATUS_FILENAME contains one line with an integer status.""" ) sys.exit(1) -with open(sys.argv[1], 'r') as status_in: +with open(sys.argv[1], "r") as status_in: status = int(status_in.readline()) -print('{} with status {}'.format( - "\033[32msucceeded\033[0m" if status == 0 else "\033[31mfailed\033[0m", - status)) +print( + "{} with status {}".format( + "\033[32msucceeded\033[0m" if status == 0 else "\033[31mfailed\033[0m", status + ) +) diff --git a/tools/pyboard.py b/tools/pyboard.py index ddf6bb6c4..b8f1c381e 100755 --- a/tools/pyboard.py +++ b/tools/pyboard.py @@ -57,34 +57,41 @@ except AttributeError: # Python2 doesn't have buffer attr stdout = sys.stdout + def stdout_write_bytes(b): b = b.replace(b"\x04", b"") stdout.write(b) stdout.flush() + class PyboardError(BaseException): pass + class TelnetToSerial: def __init__(self, ip, user, password, read_timeout=None): import telnetlib + self.tn = telnetlib.Telnet(ip, timeout=15) self.read_timeout = read_timeout - if b'Login as:' in self.tn.read_until(b'Login as:', timeout=read_timeout): - self.tn.write(bytes(user, 'ascii') + b"\r\n") + if b"Login as:" in self.tn.read_until(b"Login as:", timeout=read_timeout): + self.tn.write(bytes(user, "ascii") + b"\r\n") - if b'Password:' in self.tn.read_until(b'Password:', timeout=read_timeout): + if b"Password:" in self.tn.read_until(b"Password:", timeout=read_timeout): # needed because of internal implementation details of the telnet server time.sleep(0.2) - self.tn.write(bytes(password, 'ascii') + b"\r\n") + self.tn.write(bytes(password, "ascii") + b"\r\n") - if b'for more information.' in self.tn.read_until(b'Type "help()" for more information.', timeout=read_timeout): + if b"for more information." in self.tn.read_until( + b'Type "help()" for more information.', timeout=read_timeout + ): # login successful from collections import deque + self.fifo = deque() return - raise PyboardError('Failed to establish a telnet connection with the board') + raise PyboardError("Failed to establish a telnet connection with the board") def __del__(self): self.close() @@ -109,7 +116,7 @@ class TelnetToSerial: break timeout_count += 1 - data = b'' + data = b"" while len(data) < size and len(self.fifo) > 0: data += bytes([self.fifo.popleft()]) return data @@ -133,24 +140,33 @@ class ProcessToSerial: def __init__(self, cmd): import subprocess - self.subp = subprocess.Popen(cmd.split(), bufsize=0, shell=True, preexec_fn=os.setsid, - stdin=subprocess.PIPE, stdout=subprocess.PIPE) + + self.subp = subprocess.Popen( + cmd.split(), + bufsize=0, + shell=True, + preexec_fn=os.setsid, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + ) # Initially was implemented with selectors, but that adds Python3 # dependency. However, there can be race conditions communicating # with a particular child process (like QEMU), and selectors may # still work better in that case, so left inplace for now. # - #import selectors - #self.sel = selectors.DefaultSelector() - #self.sel.register(self.subp.stdout, selectors.EVENT_READ) + # import selectors + # self.sel = selectors.DefaultSelector() + # self.sel.register(self.subp.stdout, selectors.EVENT_READ) import select + self.poll = select.poll() self.poll.register(self.subp.stdout.fileno()) def close(self): import signal + os.killpg(os.getpgid(self.subp.pid), signal.SIGTERM) def read(self, size=1): @@ -164,7 +180,7 @@ class ProcessToSerial: return len(data) def inWaiting(self): - #res = self.sel.select(0) + # res = self.sel.select(0) res = self.poll.poll(0) if res: return 1 @@ -180,8 +196,16 @@ class ProcessPtyToTerminal: import subprocess import re import serial - self.subp = subprocess.Popen(cmd.split(), bufsize=0, shell=False, preexec_fn=os.setsid, - stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + + self.subp = subprocess.Popen( + cmd.split(), + bufsize=0, + shell=False, + preexec_fn=os.setsid, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) pty_line = self.subp.stderr.readline().decode("utf-8") m = re.search(r"/dev/pts/[0-9]+", pty_line) if not m: @@ -195,6 +219,7 @@ class ProcessPtyToTerminal: def close(self): import signal + os.killpg(os.getpgid(self.subp.pid), signal.SIGTERM) def read(self, size=1): @@ -208,36 +233,37 @@ class ProcessPtyToTerminal: class Pyboard: - def __init__(self, device, baudrate=115200, user='micro', password='python', wait=0): + def __init__(self, device, baudrate=115200, user="micro", password="python", wait=0): if device.startswith("exec:"): - self.serial = ProcessToSerial(device[len("exec:"):]) + self.serial = ProcessToSerial(device[len("exec:") :]) elif device.startswith("execpty:"): - self.serial = ProcessPtyToTerminal(device[len("qemupty:"):]) - elif device and device[0].isdigit() and device[-1].isdigit() and device.count('.') == 3: + self.serial = ProcessPtyToTerminal(device[len("qemupty:") :]) + elif device and device[0].isdigit() and device[-1].isdigit() and device.count(".") == 3: # device looks like an IP address self.serial = TelnetToSerial(device, user, password, read_timeout=10) else: import serial + delayed = False for attempt in range(wait + 1): try: self.serial = serial.Serial(device, baudrate=baudrate, interCharTimeout=1) break - except (OSError, IOError): # Py2 and Py3 have different errors + except (OSError, IOError): # Py2 and Py3 have different errors if wait == 0: continue if attempt == 0: - sys.stdout.write('Waiting {} seconds for pyboard '.format(wait)) + sys.stdout.write("Waiting {} seconds for pyboard ".format(wait)) delayed = True time.sleep(1) - sys.stdout.write('.') + sys.stdout.write(".") sys.stdout.flush() else: if delayed: - print('') - raise PyboardError('failed to access ' + device) + print("") + raise PyboardError("failed to access " + device) if delayed: - print('') + print("") def close(self): self.serial.close() @@ -264,7 +290,7 @@ class Pyboard: return data def enter_raw_repl(self): - self.serial.write(b'\r\x03\x03') # ctrl-C twice: interrupt any running program + self.serial.write(b"\r\x03\x03") # ctrl-C twice: interrupt any running program # flush input (without relying on serial.flushInput()) n = self.serial.inWaiting() @@ -272,38 +298,38 @@ class Pyboard: self.serial.read(n) n = self.serial.inWaiting() - self.serial.write(b'\r\x01') # ctrl-A: enter raw REPL - data = self.read_until(1, b'raw REPL; CTRL-B to exit\r\n>') - if not data.endswith(b'raw REPL; CTRL-B to exit\r\n>'): + self.serial.write(b"\r\x01") # ctrl-A: enter raw REPL + data = self.read_until(1, b"raw REPL; CTRL-B to exit\r\n>") + if not data.endswith(b"raw REPL; CTRL-B to exit\r\n>"): print(data) - raise PyboardError('could not enter raw repl') + raise PyboardError("could not enter raw repl") - self.serial.write(b'\x04') # ctrl-D: soft reset - data = self.read_until(1, b'soft reboot\r\n') - if not data.endswith(b'soft reboot\r\n'): + self.serial.write(b"\x04") # ctrl-D: soft reset + data = self.read_until(1, b"soft reboot\r\n") + if not data.endswith(b"soft reboot\r\n"): print(data) - raise PyboardError('could not enter raw repl') + raise PyboardError("could not enter raw repl") # By splitting this into 2 reads, it allows boot.py to print stuff, # which will show up after the soft reboot and before the raw REPL. - data = self.read_until(1, b'raw REPL; CTRL-B to exit\r\n') - if not data.endswith(b'raw REPL; CTRL-B to exit\r\n'): + data = self.read_until(1, b"raw REPL; CTRL-B to exit\r\n") + if not data.endswith(b"raw REPL; CTRL-B to exit\r\n"): print(data) - raise PyboardError('could not enter raw repl') + raise PyboardError("could not enter raw repl") def exit_raw_repl(self): - self.serial.write(b'\r\x02') # ctrl-B: enter friendly REPL + self.serial.write(b"\r\x02") # ctrl-B: enter friendly REPL def follow(self, timeout, data_consumer=None): # wait for normal output - data = self.read_until(1, b'\x04', timeout=timeout, data_consumer=data_consumer) - if not data.endswith(b'\x04'): - raise PyboardError('timeout waiting for first EOF reception') + data = self.read_until(1, b"\x04", timeout=timeout, data_consumer=data_consumer) + if not data.endswith(b"\x04"): + raise PyboardError("timeout waiting for first EOF reception") data = data[:-1] # wait for error output - data_err = self.read_until(1, b'\x04', timeout=timeout) - if not data_err.endswith(b'\x04'): - raise PyboardError('timeout waiting for second EOF reception') + data_err = self.read_until(1, b"\x04", timeout=timeout) + if not data_err.endswith(b"\x04"): + raise PyboardError("timeout waiting for second EOF reception") data_err = data_err[:-1] # return normal and error output @@ -313,53 +339,55 @@ class Pyboard: if isinstance(command, bytes): command_bytes = command else: - command_bytes = bytes(command, encoding='utf8') + command_bytes = bytes(command, encoding="utf8") # check we have a prompt - data = self.read_until(1, b'>') - if not data.endswith(b'>'): - raise PyboardError('could not enter raw repl') + data = self.read_until(1, b">") + if not data.endswith(b">"): + raise PyboardError("could not enter raw repl") # write command for i in range(0, len(command_bytes), 256): - self.serial.write(command_bytes[i:min(i + 256, len(command_bytes))]) + self.serial.write(command_bytes[i : min(i + 256, len(command_bytes))]) time.sleep(0.01) - self.serial.write(b'\x04') + self.serial.write(b"\x04") # check if we could exec command data = self.serial.read(2) - if data != b'OK': - raise PyboardError('could not exec command (response: %r)' % data) + if data != b"OK": + raise PyboardError("could not exec command (response: %r)" % data) def exec_raw(self, command, timeout=10, data_consumer=None): - self.exec_raw_no_follow(command); + self.exec_raw_no_follow(command) return self.follow(timeout, data_consumer) def eval(self, expression): - ret = self.exec_('print({})'.format(expression)) + ret = self.exec_("print({})".format(expression)) ret = ret.strip() return ret def exec_(self, command): ret, ret_err = self.exec_raw(command) if ret_err: - raise PyboardError('exception', ret, ret_err) + raise PyboardError("exception", ret, ret_err) return ret def execfile(self, filename): - with open(filename, 'rb') as f: + with open(filename, "rb") as f: pyfile = f.read() return self.exec_(pyfile) def get_time(self): - t = str(self.eval('pyb.RTC().datetime()'), encoding='utf8')[1:-1].split(', ') + t = str(self.eval("pyb.RTC().datetime()"), encoding="utf8")[1:-1].split(", ") return int(t[4]) * 3600 + int(t[5]) * 60 + int(t[6]) + # in Python2 exec is a keyword so one must use "exec_" # but for Python3 we want to provide the nicer version "exec" setattr(Pyboard, "exec", Pyboard.exec_) -def execfile(filename, device='/dev/ttyACM0', baudrate=115200, user='micro', password='python'): + +def execfile(filename, device="/dev/ttyACM0", baudrate=115200, user="micro", password="python"): pyb = Pyboard(device, baudrate, user, password) pyb.enter_raw_repl() output = pyb.execfile(filename) @@ -367,17 +395,35 @@ def execfile(filename, device='/dev/ttyACM0', baudrate=115200, user='micro', pas pyb.exit_raw_repl() pyb.close() + def main(): import argparse - cmd_parser = argparse.ArgumentParser(description='Run scripts on the pyboard.') - cmd_parser.add_argument('--device', default='/dev/ttyACM0', help='the serial device or the IP address of the pyboard') - cmd_parser.add_argument('-b', '--baudrate', default=115200, help='the baud rate of the serial device') - cmd_parser.add_argument('-u', '--user', default='micro', help='the telnet login username') - cmd_parser.add_argument('-p', '--password', default='python', help='the telnet login password') - cmd_parser.add_argument('-c', '--command', help='program passed in as string') - cmd_parser.add_argument('-w', '--wait', default=0, type=int, help='seconds to wait for USB connected board to become available') - cmd_parser.add_argument('--follow', action='store_true', help='follow the output after running the scripts [default if no scripts given]') - cmd_parser.add_argument('files', nargs='*', help='input files') + + cmd_parser = argparse.ArgumentParser(description="Run scripts on the pyboard.") + cmd_parser.add_argument( + "--device", + default="/dev/ttyACM0", + help="the serial device or the IP address of the pyboard", + ) + cmd_parser.add_argument( + "-b", "--baudrate", default=115200, help="the baud rate of the serial device" + ) + cmd_parser.add_argument("-u", "--user", default="micro", help="the telnet login username") + cmd_parser.add_argument("-p", "--password", default="python", help="the telnet login password") + cmd_parser.add_argument("-c", "--command", help="program passed in as string") + cmd_parser.add_argument( + "-w", + "--wait", + default=0, + type=int, + help="seconds to wait for USB connected board to become available", + ) + cmd_parser.add_argument( + "--follow", + action="store_true", + help="follow the output after running the scripts [default if no scripts given]", + ) + cmd_parser.add_argument("files", nargs="*", help="input files") args = cmd_parser.parse_args() # open the connection to the pyboard @@ -415,11 +461,11 @@ def main(): # run the command, if given if args.command is not None: - execbuffer(args.command.encode('utf-8')) + execbuffer(args.command.encode("utf-8")) # run any files for filename in args.files: - with open(filename, 'rb') as f: + with open(filename, "rb") as f: pyfile = f.read() execbuffer(pyfile) @@ -443,5 +489,6 @@ def main(): # close the connection to the pyboard pyb.close() + if __name__ == "__main__": main() diff --git a/tools/pydfu.py b/tools/pydfu.py index a6210c603..5ee942c17 100755 --- a/tools/pydfu.py +++ b/tools/pydfu.py @@ -24,34 +24,34 @@ import zlib # VID/PID __VID = 0x0483 -__PID = 0xdf11 +__PID = 0xDF11 # USB request __TIMEOUT __TIMEOUT = 4000 # DFU commands -__DFU_DETACH = 0 -__DFU_DNLOAD = 1 -__DFU_UPLOAD = 2 +__DFU_DETACH = 0 +__DFU_DNLOAD = 1 +__DFU_UPLOAD = 2 __DFU_GETSTATUS = 3 __DFU_CLRSTATUS = 4 -__DFU_GETSTATE = 5 -__DFU_ABORT = 6 +__DFU_GETSTATE = 5 +__DFU_ABORT = 6 # DFU status -__DFU_STATE_APP_IDLE = 0x00 -__DFU_STATE_APP_DETACH = 0x01 -__DFU_STATE_DFU_IDLE = 0x02 -__DFU_STATE_DFU_DOWNLOAD_SYNC = 0x03 -__DFU_STATE_DFU_DOWNLOAD_BUSY = 0x04 -__DFU_STATE_DFU_DOWNLOAD_IDLE = 0x05 -__DFU_STATE_DFU_MANIFEST_SYNC = 0x06 -__DFU_STATE_DFU_MANIFEST = 0x07 -__DFU_STATE_DFU_MANIFEST_WAIT_RESET = 0x08 -__DFU_STATE_DFU_UPLOAD_IDLE = 0x09 -__DFU_STATE_DFU_ERROR = 0x0a +__DFU_STATE_APP_IDLE = 0x00 +__DFU_STATE_APP_DETACH = 0x01 +__DFU_STATE_DFU_IDLE = 0x02 +__DFU_STATE_DFU_DOWNLOAD_SYNC = 0x03 +__DFU_STATE_DFU_DOWNLOAD_BUSY = 0x04 +__DFU_STATE_DFU_DOWNLOAD_IDLE = 0x05 +__DFU_STATE_DFU_MANIFEST_SYNC = 0x06 +__DFU_STATE_DFU_MANIFEST = 0x07 +__DFU_STATE_DFU_MANIFEST_WAIT_RESET = 0x08 +__DFU_STATE_DFU_UPLOAD_IDLE = 0x09 +__DFU_STATE_DFU_ERROR = 0x0A -_DFU_DESCRIPTOR_TYPE = 0x21 +_DFU_DESCRIPTOR_TYPE = 0x21 # USB device handle @@ -63,10 +63,13 @@ __verbose = None __DFU_INTERFACE = 0 import inspect -if 'length' in inspect.getargspec(usb.util.get_string).args: + +if "length" in inspect.getargspec(usb.util.get_string).args: # PyUSB 1.0.0.b1 has the length argument def get_string(dev, index): return usb.util.get_string(dev, 255, index) + + else: # PyUSB 1.0.0.b2 dropped the length argument def get_string(dev, index): @@ -78,7 +81,7 @@ def init(): global __dev devices = get_dfu_devices(idVendor=__VID, idProduct=__PID) if not devices: - raise ValueError('No DFU device found') + raise ValueError("No DFU device found") if len(devices) > 1: raise ValueError("Multiple DFU devices found") __dev = devices[0] @@ -93,14 +96,12 @@ def init(): def clr_status(): """Clears any error status (perhaps left over from a previous session).""" - __dev.ctrl_transfer(0x21, __DFU_CLRSTATUS, 0, __DFU_INTERFACE, - None, __TIMEOUT) + __dev.ctrl_transfer(0x21, __DFU_CLRSTATUS, 0, __DFU_INTERFACE, None, __TIMEOUT) def get_status(): """Get the status of the last operation.""" - stat = __dev.ctrl_transfer(0xA1, __DFU_GETSTATUS, 0, __DFU_INTERFACE, - 6, 20000) + stat = __dev.ctrl_transfer(0xA1, __DFU_GETSTATUS, 0, __DFU_INTERFACE, 6, 20000) # print (__DFU_STAT[stat[4]], stat) return stat[4] @@ -108,8 +109,7 @@ def get_status(): def mass_erase(): """Performs a MASS erase (i.e. erases the entire device.""" # Send DNLOAD with first byte=0x41 - __dev.ctrl_transfer(0x21, __DFU_DNLOAD, 0, __DFU_INTERFACE, - "\x41", __TIMEOUT) + __dev.ctrl_transfer(0x21, __DFU_DNLOAD, 0, __DFU_INTERFACE, "\x41", __TIMEOUT) # Execute last command if get_status() != __DFU_STATE_DFU_DOWNLOAD_BUSY: @@ -166,22 +166,23 @@ def write_memory(addr, buf, progress=None, progress_addr=0, progress_size=0): while xfer_bytes < xfer_total: if __verbose and xfer_count % 512 == 0: - print ("Addr 0x%x %dKBs/%dKBs..." % (xfer_base + xfer_bytes, - xfer_bytes // 1024, - xfer_total // 1024)) + print( + "Addr 0x%x %dKBs/%dKBs..." + % (xfer_base + xfer_bytes, xfer_bytes // 1024, xfer_total // 1024) + ) if progress and xfer_count % 2 == 0: - progress(progress_addr, xfer_base + xfer_bytes - progress_addr, - progress_size) + progress(progress_addr, xfer_base + xfer_bytes - progress_addr, progress_size) # Set mem write address - set_address(xfer_base+xfer_bytes) + set_address(xfer_base + xfer_bytes) # Send DNLOAD with fw data # the "2048" is the DFU transfer size supported by the ST DFU bootloader # TODO: this number should be extracted from the USB config descriptor - chunk = min(2048, xfer_total-xfer_bytes) - __dev.ctrl_transfer(0x21, __DFU_DNLOAD, 2, __DFU_INTERFACE, - buf[xfer_bytes:xfer_bytes + chunk], __TIMEOUT) + chunk = min(2048, xfer_total - xfer_bytes) + __dev.ctrl_transfer( + 0x21, __DFU_DNLOAD, 2, __DFU_INTERFACE, buf[xfer_bytes : xfer_bytes + chunk], __TIMEOUT + ) # Execute last command if get_status() != __DFU_STATE_DFU_DOWNLOAD_BUSY: @@ -203,7 +204,7 @@ def write_page(buf, xfer_offset): xfer_base = 0x08000000 # Set mem write address - set_address(xfer_base+xfer_offset) + set_address(xfer_base + xfer_offset) # Send DNLOAD with fw data __dev.ctrl_transfer(0x21, __DFU_DNLOAD, 2, __DFU_INTERFACE, buf, __TIMEOUT) @@ -217,7 +218,7 @@ def write_page(buf, xfer_offset): raise Exception("DFU: write memory failed") if __verbose: - print ("Write: 0x%x " % (xfer_base + xfer_offset)) + print("Write: 0x%x " % (xfer_base + xfer_offset)) def exit_dfu(): @@ -227,8 +228,7 @@ def exit_dfu(): set_address(0x08000000) # Send DNLOAD with 0 length to exit DFU - __dev.ctrl_transfer(0x21, __DFU_DNLOAD, 0, __DFU_INTERFACE, - None, __TIMEOUT) + __dev.ctrl_transfer(0x21, __DFU_DNLOAD, 0, __DFU_INTERFACE, None, __TIMEOUT) try: # Execute last command @@ -256,7 +256,7 @@ def consume(fmt, data, names): def cstring(string): """Extracts a null-terminated string from a byte array.""" - return string.decode('utf-8').split('\0', 1)[0] + return string.decode("utf-8").split("\0", 1)[0] def compute_crc(data): @@ -276,7 +276,7 @@ def read_dfu_file(filename): """ print("File: {}".format(filename)) - with open(filename, 'rb') as fin: + with open(filename, "rb") as fin: data = fin.read() crc = compute_crc(data[:-4]) elements = [] @@ -289,11 +289,12 @@ def read_dfu_file(filename): # B uint8_t version 1 # I uint32_t size Size of the DFU file (not including suffix) # B uint8_t targets Number of targets - dfu_prefix, data = consume('<5sBIB', data, - 'signature version size targets') - print (" %(signature)s v%(version)d, image size: %(size)d, " - "targets: %(targets)d" % dfu_prefix) - for target_idx in range(dfu_prefix['targets']): + dfu_prefix, data = consume("<5sBIB", data, "signature version size targets") + print( + " %(signature)s v%(version)d, image size: %(size)d, " + "targets: %(targets)d" % dfu_prefix + ) + for target_idx in range(dfu_prefix["targets"]): # Decode the Image Prefix # # <6sBI255s2I @@ -304,33 +305,33 @@ def read_dfu_file(filename): # 255s char[255] name name of the target # I uint32_t size size of image (not incl prefix) # I uint32_t elements Number of elements in the image - img_prefix, data = consume('<6sBI255s2I', data, - 'signature altsetting named name ' - 'size elements') - img_prefix['num'] = target_idx - if img_prefix['named']: - img_prefix['name'] = cstring(img_prefix['name']) + img_prefix, data = consume( + "<6sBI255s2I", data, "signature altsetting named name " "size elements" + ) + img_prefix["num"] = target_idx + if img_prefix["named"]: + img_prefix["name"] = cstring(img_prefix["name"]) else: - img_prefix['name'] = '' - print(' %(signature)s %(num)d, alt setting: %(altsetting)s, ' - 'name: "%(name)s", size: %(size)d, elements: %(elements)d' - % img_prefix) + img_prefix["name"] = "" + print( + " %(signature)s %(num)d, alt setting: %(altsetting)s, " + 'name: "%(name)s", size: %(size)d, elements: %(elements)d' % img_prefix + ) - target_size = img_prefix['size'] + target_size = img_prefix["size"] target_data, data = data[:target_size], data[target_size:] - for elem_idx in range(img_prefix['elements']): + for elem_idx in range(img_prefix["elements"]): # Decode target prefix # < little endian # I uint32_t element address # I uint32_t element size - elem_prefix, target_data = consume('<2I', target_data, 'addr size') - elem_prefix['num'] = elem_idx - print(' %(num)d, address: 0x%(addr)08x, size: %(size)d' - % elem_prefix) - elem_size = elem_prefix['size'] + elem_prefix, target_data = consume("<2I", target_data, "addr size") + elem_prefix["num"] = elem_idx + print(" %(num)d, address: 0x%(addr)08x, size: %(size)d" % elem_prefix) + elem_size = elem_prefix["size"] elem_data = target_data[:elem_size] target_data = target_data[elem_size:] - elem_prefix['data'] = elem_data + elem_prefix["data"] = elem_data elements.append(elem_prefix) if len(target_data): @@ -345,11 +346,14 @@ def read_dfu_file(filename): # 3s char[3] ufd 'UFD' # B uint8_t len 16 # I uint32_t crc32 - dfu_suffix = named(struct.unpack('<4H3sBI', data[:16]), - 'device product vendor dfu ufd len crc') - print (' usb: %(vendor)04x:%(product)04x, device: 0x%(device)04x, ' - 'dfu: 0x%(dfu)04x, %(ufd)s, %(len)d, 0x%(crc)08x' % dfu_suffix) - if crc != dfu_suffix['crc']: + dfu_suffix = named( + struct.unpack("<4H3sBI", data[:16]), "device product vendor dfu ufd len crc" + ) + print( + " usb: %(vendor)04x:%(product)04x, device: 0x%(device)04x, " + "dfu: 0x%(dfu)04x, %(ufd)s, %(len)d, 0x%(crc)08x" % dfu_suffix + ) + if crc != dfu_suffix["crc"]: print("CRC ERROR: computed crc32 is 0x%08x" % crc) return data = data[16:] @@ -368,8 +372,7 @@ class FilterDFU(object): def __call__(self, device): for cfg in device: for intf in cfg: - return (intf.bInterfaceClass == 0xFE and - intf.bInterfaceSubClass == 1) + return intf.bInterfaceClass == 0xFE and intf.bInterfaceSubClass == 1 def get_dfu_devices(*args, **kwargs): @@ -378,8 +381,7 @@ def get_dfu_devices(*args, **kwargs): refine the search. """ # convert to list for compatibility with newer pyusb - return list(usb.core.find(*args, find_all=True, - custom_match=FilterDFU(), **kwargs)) + return list(usb.core.find(*args, find_all=True, custom_match=FilterDFU(), **kwargs)) def get_memory_layout(device): @@ -394,25 +396,29 @@ def get_memory_layout(device): cfg = device[0] intf = cfg[(0, 0)] mem_layout_str = get_string(device, intf.iInterface) - mem_layout = mem_layout_str.split('/') + mem_layout = mem_layout_str.split("/") result = [] for mem_layout_index in range(1, len(mem_layout), 2): addr = int(mem_layout[mem_layout_index], 0) - segments = mem_layout[mem_layout_index + 1].split(',') - seg_re = re.compile(r'(\d+)\*(\d+)(.)(.)') + segments = mem_layout[mem_layout_index + 1].split(",") + seg_re = re.compile(r"(\d+)\*(\d+)(.)(.)") for segment in segments: seg_match = seg_re.match(segment) num_pages = int(seg_match.groups()[0], 10) page_size = int(seg_match.groups()[1], 10) multiplier = seg_match.groups()[2] - if multiplier == 'K': + if multiplier == "K": page_size *= 1024 - if multiplier == 'M': + if multiplier == "M": page_size *= 1024 * 1024 size = num_pages * page_size last_addr = addr + size - 1 - result.append(named((addr, last_addr, size, num_pages, page_size), - "addr last_addr size num_pages page_size")) + result.append( + named( + (addr, last_addr, size, num_pages, page_size), + "addr last_addr size num_pages page_size", + ) + ) addr += size return result @@ -424,15 +430,19 @@ def list_dfu_devices(*args, **kwargs): print("No DFU capable devices found") return for device in devices: - print("Bus {} Device {:03d}: ID {:04x}:{:04x}" - .format(device.bus, device.address, - device.idVendor, device.idProduct)) + print( + "Bus {} Device {:03d}: ID {:04x}:{:04x}".format( + device.bus, device.address, device.idVendor, device.idProduct + ) + ) layout = get_memory_layout(device) print("Memory Layout") for entry in layout: - print(" 0x{:x} {:2d} pages of {:3d}K bytes" - .format(entry['addr'], entry['num_pages'], - entry['page_size'] // 1024)) + print( + " 0x{:x} {:2d} pages of {:3d}K bytes".format( + entry["addr"], entry["num_pages"], entry["page_size"] // 1024 + ) + ) def write_elements(elements, mass_erase_used, progress=None): @@ -442,9 +452,9 @@ def write_elements(elements, mass_erase_used, progress=None): mem_layout = get_memory_layout(__dev) for elem in elements: - addr = elem['addr'] - size = elem['size'] - data = elem['data'] + addr = elem["addr"] + size = elem["size"] + data = elem["data"] elem_size = size elem_addr = addr if progress: @@ -453,18 +463,16 @@ def write_elements(elements, mass_erase_used, progress=None): write_size = size if not mass_erase_used: for segment in mem_layout: - if addr >= segment['addr'] and \ - addr <= segment['last_addr']: + if addr >= segment["addr"] and addr <= segment["last_addr"]: # We found the page containing the address we want to # write, erase it - page_size = segment['page_size'] + page_size = segment["page_size"] page_addr = addr & ~(page_size - 1) if addr + write_size > page_addr + page_size: write_size = page_addr + page_size - addr page_erase(page_addr) break - write_memory(addr, data[:write_size], progress, - elem_addr, elem_size) + write_memory(addr, data[:write_size], progress, elem_addr, elem_size) data = data[write_size:] addr += write_size size -= write_size @@ -476,9 +484,12 @@ def cli_progress(addr, offset, size): """Prints a progress report suitable for use on the command line.""" width = 25 done = offset * width // size - print("\r0x{:08x} {:7d} [{}{}] {:3d}% " - .format(addr, size, '=' * done, ' ' * (width - done), - offset * 100 // size), end="") + print( + "\r0x{:08x} {:7d} [{}{}] {:3d}% ".format( + addr, size, "=" * done, " " * (width - done), offset * 100 // size + ), + end="", + ) sys.stdout.flush() if offset == size: print("") @@ -488,31 +499,19 @@ def main(): """Test program for verifying this files functionality.""" global __verbose # Parse CMD args - parser = argparse.ArgumentParser(description='DFU Python Util') - #parser.add_argument("path", help="file path") + parser = argparse.ArgumentParser(description="DFU Python Util") + # parser.add_argument("path", help="file path") parser.add_argument( - "-l", "--list", - help="list available DFU devices", - action="store_true", - default=False + "-l", "--list", help="list available DFU devices", action="store_true", default=False ) parser.add_argument( - "-m", "--mass-erase", - help="mass erase device", - action="store_true", - default=False + "-m", "--mass-erase", help="mass erase device", action="store_true", default=False ) parser.add_argument( - "-u", "--upload", - help="read file from DFU device", - dest="path", - default=False + "-u", "--upload", help="read file from DFU device", dest="path", default=False ) parser.add_argument( - "-v", "--verbose", - help="increase output verbosity", - action="store_true", - default=False + "-v", "--verbose", help="increase output verbosity", action="store_true", default=False ) args = parser.parse_args() @@ -525,7 +524,7 @@ def main(): init() if args.mass_erase: - print ("Mass erase...") + print("Mass erase...") mass_erase() if args.path: @@ -541,5 +540,6 @@ def main(): print("No command specified") -if __name__ == '__main__': + +if __name__ == "__main__": main() diff --git a/tools/tinytest-codegen.py b/tools/tinytest-codegen.py index 74b1878f9..0e7d2ee60 100755 --- a/tools/tinytest-codegen.py +++ b/tools/tinytest-codegen.py @@ -12,18 +12,13 @@ import argparse def escape(s): s = s.decode() - lookup = { - '\0': '\\0', - '\t': '\\t', - '\n': '\\n\"\n\"', - '\r': '\\r', - '\\': '\\\\', - '\"': '\\\"', - } - return "\"\"\n\"{}\"".format(''.join([lookup[x] if x in lookup else x for x in s])) + lookup = {"\0": "\\0", "\t": "\\t", "\n": '\\n"\n"', "\r": "\\r", "\\": "\\\\", '"': '\\"'} + return '""\n"{}"'.format("".join([lookup[x] if x in lookup else x for x in s])) + def chew_filename(t): - return { 'func': "test_{}_fn".format(sub(r'/|\.|-', '_', t)), 'desc': t } + return {"func": "test_{}_fn".format(sub(r"/|\.|-", "_", t)), "desc": t} + def script_to_map(test_file): r = {"name": chew_filename(test_file)["func"]} @@ -32,24 +27,25 @@ def script_to_map(test_file): # Test for import skip_if and inject it into the test as needed. if "import skip_if\n" in script: - index = script.index("import skip_if\n") - script.pop(index) - script.insert(index, "class skip_if:\n") - with open("../tests/skip_if.py") as skip_if: - total_lines = 1 - for line in skip_if: - stripped = line.strip() - if not stripped or stripped.startswith(("#", "\"\"\"")): - continue - script.insert(index + total_lines, "\t" + line) - total_lines += 1 - r['script'] = escape(b''.join(script)) + index = script.index("import skip_if\n") + script.pop(index) + script.insert(index, "class skip_if:\n") + with open("../tests/skip_if.py") as skip_if: + total_lines = 1 + for line in skip_if: + stripped = line.strip() + if not stripped or stripped.startswith(("#", '"""')): + continue + script.insert(index + total_lines, "\t" + line) + total_lines += 1 + r["script"] = escape(b"".join(script)) with open(test_file + ".exp", "rb") as f: r["output"] = escape(f.read()) return r + test_function = ( "void {name}(void* data) {{\n" " static const char pystr[] = {script};\n" @@ -59,73 +55,72 @@ test_function = ( "}}" ) -testcase_struct = ( - "struct testcase_t {name}_tests[] = {{\n{body}\n END_OF_TESTCASES\n}};" -) -testcase_member = ( - " {{ \"{desc}\", {func}, TT_ENABLED_, 0, 0 }}," -) +testcase_struct = "struct testcase_t {name}_tests[] = {{\n{body}\n END_OF_TESTCASES\n}};" +testcase_member = ' {{ "{desc}", {func}, TT_ENABLED_, 0, 0 }},' -testgroup_struct = ( - "struct testgroup_t groups[] = {{\n{body}\n END_OF_GROUPS\n}};" -) -testgroup_member = ( - " {{ \"{name}\", {name}_tests }}," -) +testgroup_struct = "struct testgroup_t groups[] = {{\n{body}\n END_OF_GROUPS\n}};" +testgroup_member = ' {{ "{name}", {name}_tests }},' ## XXX: may be we could have `--without <groups>` argument... # currently these tests are selected because they pass on qemu-arm -test_dirs = ('basics', 'micropython', 'float', 'extmod', 'inlineasm') # 'import', 'io', 'misc') +test_dirs = ("basics", "micropython", "float", "extmod", "inlineasm") # 'import', 'io', 'misc') exclude_tests = ( # pattern matching in .exp - 'basics/bytes_compare3.py', - 'extmod/ticks_diff.py', - 'extmod/time_ms_us.py', - 'extmod/uheapq_timeq.py', + "basics/bytes_compare3.py", + "extmod/ticks_diff.py", + "extmod/time_ms_us.py", + "extmod/uheapq_timeq.py", # unicode char issue - 'extmod/ujson_loads.py', + "extmod/ujson_loads.py", # doesn't output to python stdout - 'extmod/ure_debug.py', - 'extmod/vfs_basic.py', - 'extmod/vfs_fat_ramdisk.py', 'extmod/vfs_fat_fileio.py', - 'extmod/vfs_fat_fsusermount.py', 'extmod/vfs_fat_oldproto.py', + "extmod/ure_debug.py", + "extmod/vfs_basic.py", + "extmod/vfs_fat_ramdisk.py", + "extmod/vfs_fat_fileio.py", + "extmod/vfs_fat_fsusermount.py", + "extmod/vfs_fat_oldproto.py", # rounding issues - 'float/float_divmod.py', + "float/float_divmod.py", # requires double precision floating point to work - 'float/float2int_doubleprec_intbig.py', - 'float/float_parse_doubleprec.py', + "float/float2int_doubleprec_intbig.py", + "float/float_parse_doubleprec.py", # inline asm FP tests (require Cortex-M4) - 'inlineasm/asmfpaddsub.py', 'inlineasm/asmfpcmp.py', 'inlineasm/asmfpldrstr.py', - 'inlineasm/asmfpmuldiv.py','inlineasm/asmfpsqrt.py', + "inlineasm/asmfpaddsub.py", + "inlineasm/asmfpcmp.py", + "inlineasm/asmfpldrstr.py", + "inlineasm/asmfpmuldiv.py", + "inlineasm/asmfpsqrt.py", # different filename in output - 'micropython/emg_exc.py', - 'micropython/heapalloc_traceback.py', + "micropython/emg_exc.py", + "micropython/heapalloc_traceback.py", # pattern matching in .exp - 'micropython/meminfo.py', + "micropython/meminfo.py", ) output = [] tests = [] -argparser = argparse.ArgumentParser(description='Convert native MicroPython tests to tinytest/upytesthelper C code') -argparser.add_argument('--stdin', action="store_true", help='read list of tests from stdin') +argparser = argparse.ArgumentParser( + description="Convert native MicroPython tests to tinytest/upytesthelper C code" +) +argparser.add_argument("--stdin", action="store_true", help="read list of tests from stdin") args = argparser.parse_args() if not args.stdin: for group in test_dirs: - tests += [test for test in glob('{}/*.py'.format(group)) if test not in exclude_tests] + tests += [test for test in glob("{}/*.py".format(group)) if test not in exclude_tests] else: for l in sys.stdin: tests.append(l.rstrip()) output.extend([test_function.format(**script_to_map(test)) for test in tests]) testcase_members = [testcase_member.format(**chew_filename(test)) for test in tests] -output.append(testcase_struct.format(name="", body='\n'.join(testcase_members))) +output.append(testcase_struct.format(name="", body="\n".join(testcase_members))) testgroup_members = [testgroup_member.format(name=group) for group in [""]] -output.append(testgroup_struct.format(body='\n'.join(testgroup_members))) +output.append(testgroup_struct.format(body="\n".join(testgroup_members))) ## XXX: may be we could have `--output <filename>` argument... # Don't depend on what system locale is set, use utf8 encoding. -sys.stdout.buffer.write('\n\n'.join(output).encode('utf8')) +sys.stdout.buffer.write("\n\n".join(output).encode("utf8")) diff --git a/tools/upip.py b/tools/upip.py index 27fbae272..a7187e700 100644 --- a/tools/upip.py +++ b/tools/upip.py @@ -12,6 +12,7 @@ import uerrno as errno import ujson as json import uzlib import upip_utarfile as tarfile + gc.collect() @@ -22,9 +23,11 @@ gzdict_sz = 16 + 15 file_buf = bytearray(512) + class NotFoundError(Exception): pass + def op_split(path): if path == "": return ("", "") @@ -36,9 +39,11 @@ def op_split(path): head = "/" return (head, r[1]) + def op_basename(path): return op_split(path)[1] + # Expects *file* name def _makedirs(name, mode=0o777): ret = False @@ -69,26 +74,27 @@ def save_file(fname, subf): break outf.write(file_buf, sz) + def install_tar(f, prefix): meta = {} for info in f: - #print(info) + # print(info) fname = info.name try: - fname = fname[fname.index("/") + 1:] + fname = fname[fname.index("/") + 1 :] except ValueError: fname = "" save = True for p in ("setup.", "PKG-INFO", "README"): - #print(fname, p) - if fname.startswith(p) or ".egg-info" in fname: - if fname.endswith("/requires.txt"): - meta["deps"] = f.extractfile(info).read() - save = False - if debug: - print("Skipping", fname) - break + # print(fname, p) + if fname.startswith(p) or ".egg-info" in fname: + if fname.endswith("/requires.txt"): + meta["deps"] = f.extractfile(info).read() + save = False + if debug: + print("Skipping", fname) + break if save: outfname = prefix + fname @@ -100,32 +106,37 @@ def install_tar(f, prefix): save_file(outfname, subf) return meta + def expandhome(s): if "~/" in s: h = os.getenv("HOME") s = s.replace("~/", h + "/") return s + import ussl import usocket + warn_ussl = True + + def url_open(url): global warn_ussl if debug: print(url) - proto, _, host, urlpath = url.split('/', 3) + proto, _, host, urlpath = url.split("/", 3) try: ai = usocket.getaddrinfo(host, 443, 0, usocket.SOCK_STREAM) except OSError as e: fatal("Unable to resolve %s (no Internet?)" % host, e) - #print("Address infos:", ai) + # print("Address infos:", ai) ai = ai[0] s = usocket.socket(ai[0], ai[1], ai[2]) try: - #print("Connect address:", addr) + # print("Connect address:", addr) s.connect(ai[-1]) if proto == "https:": @@ -146,7 +157,7 @@ def url_open(url): l = s.readline() if not l: raise ValueError("Unexpected EOF in HTTP headers") - if l == b'\r\n': + if l == b"\r\n": break except Exception as e: s.close() @@ -169,6 +180,7 @@ def fatal(msg, exc=None): raise exc sys.exit(1) + def install_pkg(pkg_spec, install_path): data = get_pkg_metadata(pkg_spec) @@ -192,6 +204,7 @@ def install_pkg(pkg_spec, install_path): gc.collect() return meta + def install(to_install, install_path=None): # Calculate gzip dictionary size to use global gzdict_sz @@ -224,9 +237,11 @@ def install(to_install, install_path=None): deps = deps.decode("utf-8").split("\n") to_install.extend(deps) except Exception as e: - print("Error installing '{}': {}, packages may be partially installed".format( - pkg_spec, e), - file=sys.stderr) + print( + "Error installing '{}': {}, packages may be partially installed".format(pkg_spec, e), + file=sys.stderr, + ) + def get_install_path(): global install_path @@ -236,6 +251,7 @@ def get_install_path(): install_path = expandhome(install_path) return install_path + def cleanup(): for fname in cleanup_files: try: @@ -243,21 +259,27 @@ def cleanup(): except OSError: print("Warning: Cannot delete " + fname) + def help(): - print("""\ + print( + """\ upip - Simple PyPI package manager for MicroPython Usage: micropython -m upip install [-p <path>] <package>... | -r <requirements.txt> import upip; upip.install(package_or_list, [<path>]) If <path> is not given, packages will be installed into sys.path[1] (can be set from MICROPYPATH environment variable, if current system -supports that).""") +supports that).""" + ) print("Current value of sys.path[1]:", sys.path[1]) - print("""\ + print( + """\ Note: only MicroPython packages (usually, named micropython-*) are supported for installation, upip does not support arbitrary code in setup.py. -""") +""" + ) + def main(): global debug diff --git a/tools/upip_utarfile.py b/tools/upip_utarfile.py index 44d636452..3e527fad8 100644 --- a/tools/upip_utarfile.py +++ b/tools/upip_utarfile.py @@ -13,11 +13,12 @@ TAR_HEADER = { DIRTYPE = "dir" REGTYPE = "file" + def roundup(val, align): return (val + align - 1) & ~(align - 1) -class FileSection: +class FileSection: def __init__(self, f, content_len, aligned_len): self.f = f self.content_len = content_len @@ -37,7 +38,7 @@ class FileSection: if self.content_len == 0: return 0 if len(buf) > self.content_len: - buf = memoryview(buf)[:self.content_len] + buf = memoryview(buf)[: self.content_len] sz = self.f.readinto(buf) self.content_len -= sz return sz @@ -51,13 +52,13 @@ class FileSection: self.f.readinto(buf, s) sz -= s -class TarInfo: +class TarInfo: def __str__(self): return "TarInfo(%r, %s, %d)" % (self.name, self.type, self.size) -class TarFile: +class TarFile: def __init__(self, name=None, fileobj=None): if fileobj: self.f = fileobj @@ -66,24 +67,24 @@ class TarFile: self.subf = None def next(self): - if self.subf: - self.subf.skip() - buf = self.f.read(512) - if not buf: - return None - - h = uctypes.struct(uctypes.addressof(buf), TAR_HEADER, uctypes.LITTLE_ENDIAN) - - # Empty block means end of archive - if h.name[0] == 0: - return None - - d = TarInfo() - d.name = str(h.name, "utf-8").rstrip("\0") - d.size = int(bytes(h.size), 8) - d.type = [REGTYPE, DIRTYPE][d.name[-1] == "/"] - self.subf = d.subf = FileSection(self.f, d.size, roundup(d.size, 512)) - return d + if self.subf: + self.subf.skip() + buf = self.f.read(512) + if not buf: + return None + + h = uctypes.struct(uctypes.addressof(buf), TAR_HEADER, uctypes.LITTLE_ENDIAN) + + # Empty block means end of archive + if h.name[0] == 0: + return None + + d = TarInfo() + d.name = str(h.name, "utf-8").rstrip("\0") + d.size = int(bytes(h.size), 8) + d.type = [REGTYPE, DIRTYPE][d.name[-1] == "/"] + self.subf = d.subf = FileSection(self.f, d.size, roundup(d.size, 512)) + return d def __iter__(self): return self |