1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
|
"""
This script processes the output from the C preprocessor and extracts all
qstr. Each qstr is transformed into a qstr definition of the form 'Q(...)'.
This script works with Python 2.6, 2.7, 3.3 and 3.4.
"""
import re
import argparse
import os
# Python 2/3 compatibility:
# - iterating through bytes is different
# - codepoint2name lives in a different module
import platform
if platform.python_version_tuple()[0] == '2':
bytes_cons = lambda val, enc=None: bytearray(val)
from htmlentitydefs import name2codepoint
elif platform.python_version_tuple()[0] == '3':
bytes_cons = bytes
from html.entities import name2codepoint
unichr = chr
# end compatibility code
# Blacklist of qstrings that are specially handled in further
# processing and should be ignored
QSTRING_BLACK_LIST = set(['NULL', 'number_of'])
# add some custom names to map characters that aren't in HTML
name2codepoint['hyphen'] = ord('-')
name2codepoint['space'] = ord(' ')
name2codepoint['squot'] = ord('\'')
name2codepoint['comma'] = ord(',')
name2codepoint['dot'] = ord('.')
name2codepoint['colon'] = ord(':')
name2codepoint['semicolon'] = ord(';')
name2codepoint['slash'] = ord('/')
name2codepoint['percent'] = ord('%')
name2codepoint['hash'] = ord('#')
name2codepoint['paren_open'] = ord('(')
name2codepoint['paren_close'] = ord(')')
name2codepoint['bracket_open'] = ord('[')
name2codepoint['bracket_close'] = ord(']')
name2codepoint['brace_open'] = ord('{')
name2codepoint['brace_close'] = ord('}')
name2codepoint['star'] = ord('*')
name2codepoint['bang'] = ord('!')
name2codepoint['backslash'] = ord('\\')
name2codepoint['plus'] = ord('+')
name2codepoint['dollar'] = ord('$')
name2codepoint['equals'] = ord('=')
name2codepoint['question'] = ord('?')
name2codepoint['at_sign'] = ord('@')
name2codepoint['caret'] = ord('^')
name2codepoint['pipe'] = ord('|')
name2codepoint['tilde'] = ord('~')
def write_out(fname, output):
if output:
for m, r in [("/", "__"), ("\\", "__"), (":", "@"), ("..", "@@")]:
fname = fname.replace(m, r)
with open(args.output_dir + "/" + fname + ".qstr", "w") as f:
f.write("\n".join(output) + "\n")
def qstr_unescape(qstr):
for name in name2codepoint:
if "__" + name + "__" in qstr:
continue
if "_" + name + "_" in qstr:
qstr = qstr.replace("_" + name + "_", str(unichr(name2codepoint[name])))
return qstr
def process_file(f):
output = []
last_fname = None
for line in f:
# match gcc-like output (# n "file") and msvc-like output (#line n "file")
if line and (line[0:2] == "# " or line[0:5] == "#line"):
m = re.match(r"#[line]*\s\d+\s\"([^\"]+)\"", line)
assert m is not None
fname = m.group(1)
if not fname.endswith(".c"):
continue
if fname != last_fname:
write_out(last_fname, output)
output = []
last_fname = fname
continue
for match in re.findall(r'MP_QSTR_[_a-zA-Z0-9]+', line):
name = match.replace('MP_QSTR_', '')
if name not in QSTRING_BLACK_LIST:
output.append('Q(' + qstr_unescape(name) + ')')
write_out(last_fname, output)
return ""
def cat_together():
import glob
import hashlib
hasher = hashlib.md5()
all_lines = []
outf = open(args.output_dir + "/out", "wb")
for fname in glob.glob(args.output_dir + "/*.qstr"):
with open(fname, "rb") as f:
lines = f.readlines()
all_lines += lines
all_lines.sort()
all_lines = b"\n".join(all_lines)
outf.write(all_lines)
outf.close()
hasher.update(all_lines)
new_hash = hasher.hexdigest()
#print(new_hash)
old_hash = None
try:
with open(args.output_file + ".hash") as f:
old_hash = f.read()
except IOError:
pass
if old_hash != new_hash:
print("QSTR updated")
try:
# rename below might fail if file exists
os.remove(args.output_file)
except:
pass
os.rename(args.output_dir + "/out", args.output_file)
with open(args.output_file + ".hash", "w") as f:
f.write(new_hash)
else:
print("QSTR not updated")
if __name__ == "__main__":
parser = argparse.ArgumentParser(description='Generates qstr definitions from a specified source')
parser.add_argument('command',
help='Command (split/cat)')
parser.add_argument('input_filename',
help='Name of the input file (when not specified, the script reads standard input)')
parser.add_argument('output_dir',
help='Output directory to store individual qstr files')
parser.add_argument('output_file',
help='Name of the output file with collected qstrs')
args = parser.parse_args()
try:
os.makedirs(args.output_dir)
except OSError:
pass
if args.command == "split":
with open(args.input_filename) as infile:
process_file(infile)
if args.command == "cat":
cat_together()
|
