From a16edbc45ca00b3181bfe68a30a3fd7681393a7f Mon Sep 17 00:00:00 2001 From: dherrada Date: Thu, 14 May 2020 18:22:07 -0400 Subject: First semi-functional version of extract_types.py --- tools/extract_types.py | 80 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 80 insertions(+) create mode 100644 tools/extract_types.py (limited to 'tools') diff --git a/tools/extract_types.py b/tools/extract_types.py new file mode 100644 index 000000000..2e9b0a496 --- /dev/null +++ b/tools/extract_types.py @@ -0,0 +1,80 @@ +import os +import sys +import astroid +import traceback + +top_level = sys.argv[1].strip("/") + +if top_level.count("/") == 1: + top_level, module = top_level.split("/") + modules = [module] +else: + modules = os.listdir(top_level) + modules = sorted(modules) + +ok = 0 +total = 0 +for module in modules: + module_path = os.path.join(top_level, module) + if not os.path.isdir(module_path): + continue + pyi_lines = [] + classes = os.listdir(module_path) + classes = [x for x in sorted(classes) if x.endswith(".c")] + if classes and classes[-1] == "__init__.c": + classes.insert(0, classes.pop()) + for class_file in classes: + class_path = os.path.join(module_path, class_file) + with open(class_path, "r") as f: + for line in f: + if line.startswith("//|"): + if line[3] == " ": + line = line[4:] + elif line[3] == "\n": + line = line[3:] + else: + continue + pyi_lines.append(line) + + raw_stubs = [x for x in sorted(classes) if x.endswith(".pyi")] + if raw_stubs and raw_stubs[-1] == "__init__.pyi": + raw_stubs.insert(0, raw_stubs.pop()) + for raw_stub in raw_stubs: + raw_stub_path = os.path.join(module_path, raw_stub) + with open(raw_stub_path, "r") as f: + pyi_lines.extend(f.readlines()) + stub_contents = "".join(pyi_lines) + + # Validate that the module is a parseable stub. + total += 1 + try: + tree = astroid.parse(stub_contents) + #print(tree.repr_tree()) + for i in tree.body: + for j in i.body: + if isinstance(j, astroid.scoped_nodes.FunctionDef): + argdict = j.args.__dict__ + a = argdict.pop('lineno') + a = argdict.pop('col_offset') + a = argdict.pop('parent') + print(argdict) + if j.returns: + returndict = j.returns.__dict__ + a = returndict.pop('lineno') + a = returndict.pop('col_offset') + a = returndict.pop('parent') + print(returndict) + print('\n') + #print(tree.body[0].body[0]) + else: + print(type(j)) + ok += 1 + except astroid.exceptions.AstroidSyntaxError as e: + e = e.__cause__ + traceback.print_exception(type(e), e, e.__traceback__) + print() + +print(f"{ok} ok out of {total}") + +if ok != total: + sys.exit(total - ok) -- cgit v1.2.3 From 9613cdd184758421fba558f3d07e58f0b5cb15d1 Mon Sep 17 00:00:00 2001 From: dherrada Date: Thu, 14 May 2020 18:58:28 -0400 Subject: First fully working version --- tools/extract_types.py | 38 ++++++++++++++++++++++---------------- 1 file changed, 22 insertions(+), 16 deletions(-) (limited to 'tools') diff --git a/tools/extract_types.py b/tools/extract_types.py index 2e9b0a496..502c69455 100644 --- a/tools/extract_types.py +++ b/tools/extract_types.py @@ -47,34 +47,40 @@ for module in modules: # Validate that the module is a parseable stub. total += 1 + missing_parameter_type = 0 + total_1 = 0 + missing_return_type = 0 + total_2 = 0 + missing_attribute_type = 0 + total_3 = 0 try: tree = astroid.parse(stub_contents) - #print(tree.repr_tree()) for i in tree.body: for j in i.body: if isinstance(j, astroid.scoped_nodes.FunctionDef): - argdict = j.args.__dict__ - a = argdict.pop('lineno') - a = argdict.pop('col_offset') - a = argdict.pop('parent') - print(argdict) + if None in j.args.__dict__['annotations']: + missing_parameter_type += 1 + total_1 += 1 if j.returns: - returndict = j.returns.__dict__ - a = returndict.pop('lineno') - a = returndict.pop('col_offset') - a = returndict.pop('parent') - print(returndict) - print('\n') - #print(tree.body[0].body[0]) - else: - print(type(j)) + if 'Any' in j.returns.__dict__.values(): + missing_return_type += 1 + total_2 += 1 + elif isinstance(j, astroid.node_classes.AnnAssign): + if 'Any' == j.__dict__['annotation'].__dict__['name']: + missing_attribute_type += 1 + total_3 += 1 + + ok += 1 except astroid.exceptions.AstroidSyntaxError as e: e = e.__cause__ traceback.print_exception(type(e), e, e.__traceback__) print() -print(f"{ok} ok out of {total}") +print(f"{missing_parameter_type} of {total_1} are missing the parameter type") +print(f"{missing_return_type} of {total_2} are missing the return type") +print(f"{missing_attribute_type} of {total_3} are missing the attribute type") + if ok != total: sys.exit(total - ok) -- cgit v1.2.3 From 49cd9ac36e605d85ef393c8c009b90c418305ebe Mon Sep 17 00:00:00 2001 From: dherrada Date: Fri, 15 May 2020 13:29:41 -0400 Subject: Made extract_types return a more useful output --- tools/extract_types.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'tools') diff --git a/tools/extract_types.py b/tools/extract_types.py index 502c69455..f5b9e8613 100644 --- a/tools/extract_types.py +++ b/tools/extract_types.py @@ -60,14 +60,17 @@ for module in modules: if isinstance(j, astroid.scoped_nodes.FunctionDef): if None in j.args.__dict__['annotations']: missing_parameter_type += 1 + print(f"Parameter: {j.__dict__['name']} on line {j.__dict__['lineno']}") total_1 += 1 if j.returns: if 'Any' in j.returns.__dict__.values(): - missing_return_type += 1 + print(f"Return: {j.__dict__['name']} on line {j.__dict__['lineno']}") + missing_return_type += 1 total_2 += 1 elif isinstance(j, astroid.node_classes.AnnAssign): if 'Any' == j.__dict__['annotation'].__dict__['name']: missing_attribute_type += 1 + print(f"attribute on line {j.__dict__['lineno']}") total_3 += 1 -- cgit v1.2.3 From 416da442c073397112b54c8cc10ef6acc8af3197 Mon Sep 17 00:00:00 2001 From: dherrada Date: Fri, 15 May 2020 13:33:20 -0400 Subject: Now outputs class name --- tools/extract_types.py | 2 ++ 1 file changed, 2 insertions(+) (limited to 'tools') diff --git a/tools/extract_types.py b/tools/extract_types.py index f5b9e8613..d8de1c193 100644 --- a/tools/extract_types.py +++ b/tools/extract_types.py @@ -56,6 +56,7 @@ for module in modules: try: tree = astroid.parse(stub_contents) for i in tree.body: + print(i.__dict__['name']) for j in i.body: if isinstance(j, astroid.scoped_nodes.FunctionDef): if None in j.args.__dict__['annotations']: @@ -72,6 +73,7 @@ for module in modules: missing_attribute_type += 1 print(f"attribute on line {j.__dict__['lineno']}") total_3 += 1 + print('\n') ok += 1 -- cgit v1.2.3 From 0e39d4398c95412edd05c7202b7051fe7bd1e2b9 Mon Sep 17 00:00:00 2001 From: dherrada Date: Fri, 15 May 2020 13:55:46 -0400 Subject: Merged extract_types into extract_pyi --- tools/extract_pyi.py | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) (limited to 'tools') diff --git a/tools/extract_pyi.py b/tools/extract_pyi.py index 95370f761..d6309212a 100644 --- a/tools/extract_pyi.py +++ b/tools/extract_pyi.py @@ -53,12 +53,29 @@ for module in modules: # Validate that the module is a parseable stub. total += 1 try: - astroid.parse(stub_contents) + tree = astroid.parse(stub_contents) + for i in tree.body: + print(i.__dict__['name']) + for j in i.body: + if isinstance(j, astroid.scoped_nodes.FunctionDef): + a = '' + if None in j.args.__dict__['annotations']: + a += f"Missing parameter type: {j.__dict__['name']} on line {j.__dict__['lineno']}\n" + if j.returns: + if 'Any' in j.returns.__dict__.values(): + a += f"Missing return type: {j.__dict__['name']} on line {j.__dict__['lineno']}" + if a: + raise TypeError(a) + elif isinstance(j, astroid.node_classes.AnnAssign): + if 'Any' == j.__dict__['annotation'].__dict__['name']: + raise TypeError(f"missing attribute type on line {j.__dict__['lineno']}") + ok += 1 except astroid.exceptions.AstroidSyntaxError as e: e = e.__cause__ traceback.print_exception(type(e), e, e.__traceback__) - print() + except TypeError as err: + print(err) print(f"{ok} ok out of {total}") -- cgit v1.2.3 From acf88d7c0089c732222740a26a1d73563d016a13 Mon Sep 17 00:00:00 2001 From: dherrada Date: Fri, 15 May 2020 13:57:13 -0400 Subject: Removed extract_types.py --- tools/extract_types.py | 91 -------------------------------------------------- 1 file changed, 91 deletions(-) delete mode 100644 tools/extract_types.py (limited to 'tools') diff --git a/tools/extract_types.py b/tools/extract_types.py deleted file mode 100644 index d8de1c193..000000000 --- a/tools/extract_types.py +++ /dev/null @@ -1,91 +0,0 @@ -import os -import sys -import astroid -import traceback - -top_level = sys.argv[1].strip("/") - -if top_level.count("/") == 1: - top_level, module = top_level.split("/") - modules = [module] -else: - modules = os.listdir(top_level) - modules = sorted(modules) - -ok = 0 -total = 0 -for module in modules: - module_path = os.path.join(top_level, module) - if not os.path.isdir(module_path): - continue - pyi_lines = [] - classes = os.listdir(module_path) - classes = [x for x in sorted(classes) if x.endswith(".c")] - if classes and classes[-1] == "__init__.c": - classes.insert(0, classes.pop()) - for class_file in classes: - class_path = os.path.join(module_path, class_file) - with open(class_path, "r") as f: - for line in f: - if line.startswith("//|"): - if line[3] == " ": - line = line[4:] - elif line[3] == "\n": - line = line[3:] - else: - continue - pyi_lines.append(line) - - raw_stubs = [x for x in sorted(classes) if x.endswith(".pyi")] - if raw_stubs and raw_stubs[-1] == "__init__.pyi": - raw_stubs.insert(0, raw_stubs.pop()) - for raw_stub in raw_stubs: - raw_stub_path = os.path.join(module_path, raw_stub) - with open(raw_stub_path, "r") as f: - pyi_lines.extend(f.readlines()) - stub_contents = "".join(pyi_lines) - - # Validate that the module is a parseable stub. - total += 1 - missing_parameter_type = 0 - total_1 = 0 - missing_return_type = 0 - total_2 = 0 - missing_attribute_type = 0 - total_3 = 0 - try: - tree = astroid.parse(stub_contents) - for i in tree.body: - print(i.__dict__['name']) - for j in i.body: - if isinstance(j, astroid.scoped_nodes.FunctionDef): - if None in j.args.__dict__['annotations']: - missing_parameter_type += 1 - print(f"Parameter: {j.__dict__['name']} on line {j.__dict__['lineno']}") - total_1 += 1 - if j.returns: - if 'Any' in j.returns.__dict__.values(): - print(f"Return: {j.__dict__['name']} on line {j.__dict__['lineno']}") - missing_return_type += 1 - total_2 += 1 - elif isinstance(j, astroid.node_classes.AnnAssign): - if 'Any' == j.__dict__['annotation'].__dict__['name']: - missing_attribute_type += 1 - print(f"attribute on line {j.__dict__['lineno']}") - total_3 += 1 - print('\n') - - - ok += 1 - except astroid.exceptions.AstroidSyntaxError as e: - e = e.__cause__ - traceback.print_exception(type(e), e, e.__traceback__) - print() - -print(f"{missing_parameter_type} of {total_1} are missing the parameter type") -print(f"{missing_return_type} of {total_2} are missing the return type") -print(f"{missing_attribute_type} of {total_3} are missing the attribute type") - - -if ok != total: - sys.exit(total - ok) -- cgit v1.2.3 From cf524cb6b19dc971610452986f1ce8d1001d9d7e Mon Sep 17 00:00:00 2001 From: dherrada <=> Date: Mon, 18 May 2020 18:59:14 -0400 Subject: extract_pyi no longer raises a TypeError for missing types --- tools/extract_pyi.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) (limited to 'tools') diff --git a/tools/extract_pyi.py b/tools/extract_pyi.py index d6309212a..7c664d965 100644 --- a/tools/extract_pyi.py +++ b/tools/extract_pyi.py @@ -64,18 +64,16 @@ for module in modules: if j.returns: if 'Any' in j.returns.__dict__.values(): a += f"Missing return type: {j.__dict__['name']} on line {j.__dict__['lineno']}" - if a: - raise TypeError(a) elif isinstance(j, astroid.node_classes.AnnAssign): if 'Any' == j.__dict__['annotation'].__dict__['name']: - raise TypeError(f"missing attribute type on line {j.__dict__['lineno']}") + a = f"missing attribute type on line {j.__dict__['lineno']}" + if a: + print(a) ok += 1 except astroid.exceptions.AstroidSyntaxError as e: e = e.__cause__ traceback.print_exception(type(e), e, e.__traceback__) - except TypeError as err: - print(err) print(f"{ok} ok out of {total}") -- cgit v1.2.3 From 58b07ecb43047ca252a031b7ba673ba8d4e0de4e Mon Sep 17 00:00:00 2001 From: dherrada <=> Date: Tue, 19 May 2020 14:50:47 -0400 Subject: Removed a --- tools/extract_pyi.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) (limited to 'tools') diff --git a/tools/extract_pyi.py b/tools/extract_pyi.py index 3b30e1ac3..b61e86e4b 100644 --- a/tools/extract_pyi.py +++ b/tools/extract_pyi.py @@ -57,17 +57,14 @@ def convert_folder(top_level, stub_directory): print(i.__dict__['name']) for j in i.body: if isinstance(j, astroid.scoped_nodes.FunctionDef): - a = '' if None in j.args.__dict__['annotations']: - a += f"Missing parameter type: {j.__dict__['name']} on line {j.__dict__['lineno']}\n" + print(f"Missing parameter type: {j.__dict__['name']} on line {j.__dict__['lineno']}\n") if j.returns: if 'Any' in j.returns.__dict__.values(): - a += f"Missing return type: {j.__dict__['name']} on line {j.__dict__['lineno']}" + print(f"Missing return type: {j.__dict__['name']} on line {j.__dict__['lineno']}") elif isinstance(j, astroid.node_classes.AnnAssign): if 'Any' == j.__dict__['annotation'].__dict__['name']: - a = f"missing attribute type on line {j.__dict__['lineno']}" - if a: - print(a) + print(f"missing attribute type on line {j.__dict__['lineno']}") ok += 1 except astroid.exceptions.AstroidSyntaxError as e: -- cgit v1.2.3 From 67cb48acbf46b8f9508a6493578e2895c2b75463 Mon Sep 17 00:00:00 2001 From: dherrada <=> Date: Thu, 21 May 2020 18:21:32 -0400 Subject: Added another except --- tools/extract_pyi.py | 2 ++ 1 file changed, 2 insertions(+) (limited to 'tools') diff --git a/tools/extract_pyi.py b/tools/extract_pyi.py index b61e86e4b..f0c45dab6 100644 --- a/tools/extract_pyi.py +++ b/tools/extract_pyi.py @@ -70,6 +70,8 @@ def convert_folder(top_level, stub_directory): except astroid.exceptions.AstroidSyntaxError as e: e = e.__cause__ traceback.print_exception(type(e), e, e.__traceback__) + except KeyError: + print("Function does not have a key: Name") print() return ok, total -- cgit v1.2.3 From 4e22b9a3464bbace0d0cd6d51bf2afaf40cd5722 Mon Sep 17 00:00:00 2001 From: dherrada <=> Date: Wed, 27 May 2020 11:30:51 -0400 Subject: Better keyerror handling --- tools/extract_pyi.py | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) (limited to 'tools') diff --git a/tools/extract_pyi.py b/tools/extract_pyi.py index f0c45dab6..d749d202b 100644 --- a/tools/extract_pyi.py +++ b/tools/extract_pyi.py @@ -54,24 +54,24 @@ def convert_folder(top_level, stub_directory): try: tree = astroid.parse(stub_contents) for i in tree.body: - print(i.__dict__['name']) - for j in i.body: - if isinstance(j, astroid.scoped_nodes.FunctionDef): - if None in j.args.__dict__['annotations']: - print(f"Missing parameter type: {j.__dict__['name']} on line {j.__dict__['lineno']}\n") - if j.returns: - if 'Any' in j.returns.__dict__.values(): - print(f"Missing return type: {j.__dict__['name']} on line {j.__dict__['lineno']}") - elif isinstance(j, astroid.node_classes.AnnAssign): - if 'Any' == j.__dict__['annotation'].__dict__['name']: - print(f"missing attribute type on line {j.__dict__['lineno']}") + if 'name' in i.__dict__: + print(i.__dict__['name']) + for j in i.body: + if isinstance(j, astroid.scoped_nodes.FunctionDef): + if None in j.args.__dict__['annotations']: + print(f"Missing parameter type: {j.__dict__['name']} on line {j.__dict__['lineno']}\n") + if j.returns: + if 'Any' in j.returns.__dict__.values(): + print(f"Missing return type: {j.__dict__['name']} on line {j.__dict__['lineno']}") + elif isinstance(j, astroid.node_classes.AnnAssign): + if 'name' in j.__dict__['annotation'].__dict__: + if j.__dict__['annotation'].__dict__['name'] == 'Any': + print(f"missing attribute type on line {j.__dict__['lineno']}") ok += 1 except astroid.exceptions.AstroidSyntaxError as e: e = e.__cause__ traceback.print_exception(type(e), e, e.__traceback__) - except KeyError: - print("Function does not have a key: Name") print() return ok, total -- cgit v1.2.3