516 строки
16 KiB
Python
516 строки
16 KiB
Python
# -*- encoding: utf8 -*-
|
|
|
|
'''
|
|
This tool extracts information about structs and defines from the C headers.
|
|
You can pass either the raw header files or JSON files to this script.
|
|
|
|
The JSON input format is as follows:
|
|
[
|
|
{
|
|
'file': 'some/header.h',
|
|
'structs': {
|
|
'struct_name': [
|
|
'field1',
|
|
'field2',
|
|
'field3',
|
|
{
|
|
'field4': [
|
|
'nested1',
|
|
'nested2',
|
|
{
|
|
'nested3': [
|
|
'deep_nested1',
|
|
...
|
|
]
|
|
}
|
|
...
|
|
]
|
|
},
|
|
'field5'
|
|
],
|
|
'other_struct': [
|
|
'field1',
|
|
'field2',
|
|
...
|
|
]
|
|
},
|
|
'defines': [
|
|
'DEFINE_1',
|
|
'DEFINE_2',
|
|
['f', 'FLOAT_DEFINE'],
|
|
'DEFINE_3',
|
|
...
|
|
]
|
|
},
|
|
{
|
|
'file': 'some/other/header.h',
|
|
...
|
|
}
|
|
]
|
|
|
|
Please note that the 'f' for 'FLOAT_DEFINE' is just the format passed to printf(), you can put anything printf() understands.
|
|
If you call this script with the flag "-f" and pass a header file, it will create an automated boilerplate for you.
|
|
|
|
The JSON output format is based on the return value of Runtime.generateStructInfo().
|
|
{
|
|
'structs': {
|
|
'struct_name': {
|
|
'__size__': <the struct's size>,
|
|
'field1': <field1's offset>,
|
|
'field2': <field2's offset>,
|
|
'field3': <field3's offset>,
|
|
'field4': {
|
|
'__size__': <field4's size>,
|
|
'nested1': <nested1's offset>,
|
|
...
|
|
},
|
|
...
|
|
}
|
|
},
|
|
'defines': {
|
|
'DEFINE_1': <DEFINE_1's value>,
|
|
...
|
|
}
|
|
}
|
|
|
|
'''
|
|
|
|
import sys, os, re, json, argparse, tempfile, subprocess
|
|
import shared
|
|
|
|
DEBUG = os.environ.get('EMCC_DEBUG')
|
|
if DEBUG == "0":
|
|
DEBUG = None
|
|
|
|
QUIET = (__name__ != '__main__')
|
|
|
|
def show(msg):
|
|
global QUIET, DEBUG
|
|
if DEBUG or not QUIET:
|
|
sys.stderr.write('gen_struct_info: ' + msg + '\n')
|
|
|
|
# Try to load pycparser.
|
|
try:
|
|
import pycparser
|
|
except ImportError:
|
|
# The import failed
|
|
|
|
def parse_header(path, cpp_opts):
|
|
# Tell the user how to get pycparser, if he or she tries to parse a C file.
|
|
sys.stderr.write('ERR: I need pycparser to process C files. \n')
|
|
sys.stderr.write(' Use "pip install pycparser" to install or download it from "https://github.com/eliben/pycparser".\n')
|
|
sys.exit(1)
|
|
else:
|
|
# We successfully imported pycparser, the script will be completely functional.
|
|
|
|
class DelayedRef(object):
|
|
def __init__(self, dest):
|
|
self.dest = dest
|
|
|
|
def __str__(self):
|
|
return self.dest
|
|
|
|
# For a list of node types and their fields, look here: https://github.com/eliben/pycparser/blob/master/pycparser/_c_ast.cfg
|
|
class FieldVisitor(pycparser.c_ast.NodeVisitor):
|
|
def __init__(self):
|
|
self._name = None
|
|
self.structs = {}
|
|
self.named_structs = {}
|
|
|
|
def visit_Struct(self, node):
|
|
if node.decls == None:
|
|
self.named_structs[self._name] = DelayedRef(node.name)
|
|
return
|
|
|
|
|
|
fields = []
|
|
for decl in node.decls:
|
|
if decl.name == None:
|
|
# Well, this field doesn't have a name.
|
|
continue
|
|
|
|
if decl.type != None and isinstance(decl.type, pycparser.c_ast.PtrDecl):
|
|
# This field is a pointer, there's no point in looking for nested structs.
|
|
fields.append(decl.name)
|
|
else:
|
|
# Look for nested structs.
|
|
subwalk = FieldVisitor()
|
|
subwalk.visit(decl)
|
|
|
|
if subwalk.named_structs:
|
|
# Store the nested fields.
|
|
fields.append(subwalk.named_structs)
|
|
else:
|
|
# Just store the field name.
|
|
fields.append(decl.name)
|
|
|
|
if node.name != None:
|
|
self.structs[node.name] = fields
|
|
|
|
self.named_structs[self._name] = fields
|
|
|
|
def visit_Union(self, node):
|
|
self.visit_Struct(node)
|
|
|
|
def visit_TypeDecl(self, node):
|
|
# Remember the name of this typedef, so we can access it later in visit_Struct().
|
|
old_name = self._name
|
|
self._name = node.declname
|
|
self.generic_visit(node)
|
|
self._name = old_name
|
|
|
|
# The first parameter is a structure, the second is a path (a list containing all the keys, needed to reach the destination).
|
|
# The last parameter is an item to look for. This function will try to follow the path into the given object and then look there for this key.
|
|
# As long as the nested object doesn't have the given key, it will descent into the next higher object till it finds the given key.
|
|
#
|
|
# Example:
|
|
#
|
|
# res = look_through({
|
|
# 'la1': {
|
|
# 'lb1': {
|
|
# 'lc1': 99,
|
|
# 'lc2': { 'ld1': 11 }
|
|
# 'lc2': 200
|
|
# },
|
|
# 'nice': 100
|
|
# },
|
|
# 'nice': 300
|
|
# }, ['la1', 'lb1', 'lc2'], 'nice')
|
|
#
|
|
# print(res) # Prints 100 .
|
|
#
|
|
# In this case the function looked inside obj['la1']['lb1']['lc2']['nice'], then obj['la1']['lb1']['nice'] and found the value
|
|
# in obj['la1']['nice']. As soon as it finds a value it returns it and stops looking.
|
|
def look_through(obj, path, name):
|
|
cur_level = obj
|
|
path = path[:]
|
|
for i, p in enumerate(path):
|
|
cur_level = cur_level[p]
|
|
path[i] = cur_level
|
|
|
|
path = [ obj ] + path
|
|
|
|
while len(path) > 0:
|
|
if name in path[-1]:
|
|
return path[-1][name]
|
|
else:
|
|
path.pop()
|
|
|
|
return None
|
|
|
|
# Use the above function to resolve all DelayedRef() inside a list or dict recursively.
|
|
def resolve_delayed(item, root=None, path=[]):
|
|
if root == None:
|
|
root = item
|
|
|
|
if isinstance(item, DelayedRef):
|
|
if item.dest in path:
|
|
show('WARN: Circular reference found! Field "' + path[-1] + '" references "' + item.dest + '"! (Path = ' + '/'.join([str(part) for part in path]) + ')')
|
|
return { '__ref__': item.dest }
|
|
else:
|
|
return look_through(root, path[:-1], item.dest)
|
|
elif isinstance(item, dict):
|
|
for name, val in item.items():
|
|
item[name] = resolve_delayed(val, root, path + [ name ])
|
|
elif isinstance(item, list):
|
|
for i, val in enumerate(item):
|
|
item[i] = resolve_delayed(val, root, path + [ i ])
|
|
|
|
return item
|
|
|
|
def parse_header(path, cpp_opts):
|
|
show('Parsing header "' + path + '"...')
|
|
|
|
# Use clang -E as the preprocessor for pycparser.
|
|
ast = pycparser.parse_file(path, True, cpp_path=shared.CLANG_CC, cpp_args=['-E'] + cpp_opts)
|
|
|
|
# Walk the parsed AST and filter out all the declared structs and their fields.
|
|
walker = FieldVisitor()
|
|
walker.visit(ast)
|
|
|
|
walker.structs = resolve_delayed(walker.structs)
|
|
with open(path, 'r') as stream:
|
|
defines = re.findall(r'(?:^|\n)\s*#define\s+([A-Z|_|0-9]+)\s.*', stream.read())
|
|
|
|
return {
|
|
'file': path,
|
|
'defines': defines,
|
|
'structs': walker.structs
|
|
}
|
|
|
|
# The following three functions generate C code. The output of the compiled code will be
|
|
# parsed later on and then put back together into a dict structure by parse_c_output().
|
|
#
|
|
# Example:
|
|
# c_descent('test1', code)
|
|
# c_set('item', 'i%i', '111', code)
|
|
# c_set('item2', 'i%i', '9', code)
|
|
# c_set('item3', 's%s', '"Hello"', code)
|
|
# c_ascent(code)
|
|
# c_set('outer', 'f%f', '0.999', code)
|
|
#
|
|
# Will result in:
|
|
# {
|
|
# 'test1': {
|
|
# 'item': 111,
|
|
# 'item2': 9,
|
|
# 'item3': 'Hello',
|
|
# },
|
|
# 'outer': 0.999
|
|
# }
|
|
def c_set(name, type_, value, code):
|
|
code.append('printf("K' + name + '\\n");')
|
|
code.append('printf("V' + type_ + '\\n", ' + value + ');')
|
|
|
|
def c_descent(name, code):
|
|
code.append('printf("D' + name + '\\n");')
|
|
|
|
def c_ascent(code):
|
|
code.append('printf("A\\n");')
|
|
|
|
def parse_c_output(lines):
|
|
result = {}
|
|
cur_level = result
|
|
parent = []
|
|
key = None
|
|
|
|
for line in lines:
|
|
arg = line[1:].strip()
|
|
if line[0] == 'K':
|
|
# This is a key
|
|
key = arg
|
|
elif line[0] == 'V':
|
|
# A value
|
|
if arg[0] == 'i':
|
|
arg = int(arg[1:])
|
|
elif arg[0] == 'f':
|
|
arg = float(arg[1:])
|
|
elif arg[0] == 's':
|
|
arg = arg[1:]
|
|
|
|
cur_level[key] = arg
|
|
elif line[0] == 'D':
|
|
# Remember the current level as the last parent.
|
|
parent.append(cur_level)
|
|
|
|
# We descend one level.
|
|
cur_level[arg] = {}
|
|
cur_level = cur_level[arg]
|
|
elif line[0] == 'A':
|
|
# We return to the parent dict. (One level up.)
|
|
cur_level = parent.pop()
|
|
|
|
return result
|
|
|
|
def gen_inspect_code(path, struct, code):
|
|
if path[0][-1] == '#':
|
|
path[0] = path[0][:-1]
|
|
prefix = ''
|
|
else:
|
|
prefix = 'struct '
|
|
|
|
c_descent(path[-1], code)
|
|
|
|
if len(path) == 1:
|
|
c_set('__size__', 'i%u', 'sizeof (' + prefix + path[0] + ')', code)
|
|
else:
|
|
c_set('__size__', 'i%u', 'sizeof ((' + prefix + path[0] + ' *)0)->' + '.'.join(path[1:]), code)
|
|
#c_set('__offset__', 'i%u', 'offsetof(' + prefix + path[0] + ', ' + '.'.join(path[1:]) + ')', code)
|
|
|
|
for field in struct:
|
|
if isinstance(field, dict):
|
|
# We have to recurse to inspect the nested dict.
|
|
fname = field.keys()[0]
|
|
gen_inspect_code(path + [fname], field[fname], code)
|
|
else:
|
|
c_set(field, 'i%u', 'offsetof(' + prefix + path[0] + ', ' + '.'.join(path[1:] + [field]) + ')', code)
|
|
|
|
c_ascent(code)
|
|
|
|
def inspect_code(headers, cpp_opts, structs, defines):
|
|
show('Generating C code...')
|
|
|
|
code = ['#include <stdio.h>', '#include <stddef.h>']
|
|
# Include all the needed headers.
|
|
for path in headers:
|
|
code.append('#include "' + path + '"')
|
|
|
|
code.append('int main() {')
|
|
c_descent('structs', code)
|
|
for name, struct in structs.items():
|
|
gen_inspect_code([name], struct, code)
|
|
|
|
c_ascent(code)
|
|
c_descent('defines', code)
|
|
for name, type_ in defines.items():
|
|
# Add the necessary python type, if missing.
|
|
if '%' not in type_:
|
|
if type_[-1] in ('d', 'i', 'u'):
|
|
# integer
|
|
type_ = 'i%' + type_
|
|
elif type_[-1] in ('f', 'F', 'e', 'E', 'g', 'G'):
|
|
# float
|
|
type_ = 'f%' + type_
|
|
elif type_[-1] in ('x', 'X', 'a', 'A', 'c', 's'):
|
|
# hexadecimal or string
|
|
type_ = 's%' + type_
|
|
|
|
c_set(name, type_, name, code)
|
|
|
|
code.append('return 0;')
|
|
code.append('}')
|
|
|
|
# Write the source code to a temporary file.
|
|
src_file = tempfile.mkstemp('.c')
|
|
js_file = tempfile.mkstemp('.js')
|
|
|
|
os.write(src_file[0], '\n'.join(code))
|
|
|
|
# Close all unneeded FDs.
|
|
os.close(src_file[0])
|
|
os.close(js_file[0])
|
|
|
|
# Remove dangerous env modifications
|
|
safe_env = os.environ.copy()
|
|
for opt in ['EMCC_FORCE_STDLIBS', 'EMCC_ONLY_FORCED_STDLIBS']:
|
|
if opt in safe_env:
|
|
del safe_env[opt]
|
|
|
|
info = []
|
|
try:
|
|
# Compile the program.
|
|
show('Compiling generated code...')
|
|
subprocess.check_call([shared.PYTHON, shared.EMCC] + cpp_opts + ['-o', js_file[1], src_file[1], '-s', 'BOOTSTRAPPING_STRUCT_INFO=1', '-s', 'WARN_ON_UNDEFINED_SYMBOLS=0', '-Oz', '--js-opts', '0', '--memory-init-file', '0'], env=safe_env) # -Oz optimizes enough to avoid warnings on code size/num locals
|
|
# Run the compiled program.
|
|
show('Calling generated program...')
|
|
info = shared.run_js(js_file[1]).splitlines()
|
|
except subprocess.CalledProcessError:
|
|
if os.path.isfile(js_file[1]):
|
|
sys.stderr.write('FAIL: Running the generated program failed!\n')
|
|
else:
|
|
sys.stderr.write('FAIL: Compilation failed!\n')
|
|
|
|
sys.exit(1)
|
|
finally:
|
|
# Remove all temporary files.
|
|
os.unlink(src_file[1])
|
|
|
|
if os.path.exists(js_file[1]):
|
|
os.unlink(js_file[1])
|
|
|
|
# Parse the output of the program into a dict.
|
|
return parse_c_output(info)
|
|
|
|
def parse_json(path, header_files, structs, defines):
|
|
with open(path, 'r') as stream:
|
|
# Remove comments before loading the JSON.
|
|
data = json.loads(re.sub(r'//.*\n', '', stream.read()))
|
|
|
|
if not isinstance(data, list):
|
|
data = [ data ]
|
|
|
|
for item in data:
|
|
header_files.append(item['file'])
|
|
for name, data in item['structs'].items():
|
|
if name in structs:
|
|
show('WARN: Description of struct "' + name + '" in file "' + item['file'] + '" replaces an existing description!')
|
|
|
|
structs[name] = data
|
|
|
|
for part in item['defines']:
|
|
if not isinstance(part, list):
|
|
# If no type is specified, assume integer.
|
|
part = ['i', part]
|
|
|
|
if part[1] in defines:
|
|
show('WARN: Description of define "' + part[1] + '" in file "' + item['file'] + '" replaces an existing description!')
|
|
|
|
defines[part[1]] = part[0]
|
|
|
|
def output_json(obj, compressed=True, stream=None):
|
|
if stream == None:
|
|
stream = sys.stdout
|
|
elif isinstance(stream, str):
|
|
stream = open(stream, 'w')
|
|
|
|
if compressed:
|
|
json.dump(obj, stream, separators=(',', ':'))
|
|
else:
|
|
json.dump(obj, stream, indent=4, sort_keys=True)
|
|
|
|
stream.close()
|
|
|
|
def filter_opts(opts):
|
|
# Only apply compiler options regarding syntax, includes and defines.
|
|
# We have to compile for the current system, we aren't compiling to bitcode after all.
|
|
out = []
|
|
for flag in opts:
|
|
if flag[:2] in ('-f', '-I', '-i', '-D', '-U'):
|
|
out.append(flag)
|
|
|
|
return out
|
|
|
|
def main(args):
|
|
global QUIET
|
|
|
|
parser = argparse.ArgumentParser(description='Generate JSON infos for structs.')
|
|
parser.add_argument('headers', nargs='+', help='A header (.h) file or a JSON file with a list of structs and their fields')
|
|
parser.add_argument('-q', dest='quiet', action='store_true', default=False, help='Don\'t output anything besides error messages.')
|
|
parser.add_argument('-f', dest='list_fields', action='store_true', default=False, help='Output a list of structs and fields for the given headers.')
|
|
parser.add_argument('-p', dest='pretty_print', action='store_true', default=False, help='Pretty print the outputted JSON.')
|
|
parser.add_argument('-o', dest='output', metavar='path', default=None, help='Path to the JSON file that will be written. If omitted, the generated data will be printed to stdout.')
|
|
parser.add_argument('-I', dest='includes', metavar='dir', action='append', default=[], help='Add directory to include search path')
|
|
parser.add_argument('-D', dest='defines', metavar='define', action='append', default=[], help='Pass a define to the preprocessor')
|
|
parser.add_argument('-U', dest='undefines', metavar='undefine', action='append', default=[], help='Pass an undefine to the preprocessor')
|
|
args = parser.parse_args(args)
|
|
|
|
QUIET = args.quiet
|
|
|
|
# Avoid parsing problems due to gcc specifc syntax.
|
|
cpp_opts = ['-D_GNU_SOURCE']
|
|
|
|
# Add the user options to the list as well.
|
|
for path in args.includes:
|
|
cpp_opts.append('-I' + path)
|
|
|
|
for arg in args.defines:
|
|
cpp_opts.append('-D' + arg)
|
|
|
|
for arg in args.undefines:
|
|
cpp_opts.append('-U' + arg)
|
|
|
|
if args.list_fields:
|
|
# Just parse the given headers and output the result.
|
|
data = []
|
|
for path in args.headers:
|
|
if path[-5:] == '.json':
|
|
show('WARN: Skipping "' + path + '" because it\'s already a JSON file!')
|
|
else:
|
|
data.append(parse_header(path, cpp_opts))
|
|
|
|
output_json(data, not args.pretty_print, args.output)
|
|
sys.exit(0)
|
|
|
|
# Look for structs in all passed headers.
|
|
header_files = []
|
|
structs = {}
|
|
defines = {}
|
|
|
|
for header in args.headers:
|
|
if header[-5:] == '.json':
|
|
# This is a JSON file, parse it.
|
|
parse_json(header, header_files, structs, defines)
|
|
else:
|
|
# If the passed file isn't a JSON file, assume it's a header.
|
|
header_files.append(header)
|
|
data = parse_header(header, cpp_opts)
|
|
structs.update(data['structs'])
|
|
defines.extend(data['defines'])
|
|
|
|
# Inspect all collected structs.
|
|
struct_info = inspect_code(header_files, cpp_opts, structs, defines)
|
|
output_json(struct_info, not args.pretty_print, args.output)
|
|
|
|
if __name__ == '__main__':
|
|
main(sys.argv[1:])
|