Merged PR 4615: Support for Includes
Added support to multiple files, will now recursively go looking for structs/classes that are required for functions that are on included files
This commit is contained in:
Teo Magnino Chaban
Родитель
a5ea64f005
Коммит
6996e9721c
|
|
@ -43,6 +43,7 @@ def _GetObjectType(node, SimpleVarType, FullVarType):
|
|||
object_type['simple_var_types'] = []
|
||||
object_type['definition_line'] = node.location.line
|
||||
object_type['constructor_list'] = []
|
||||
object_type['file'] = os.path.realpath(node.location.file.name)
|
||||
|
||||
# The way to see if this is a definition or not, is to see if 'node' has any children.
|
||||
is_def = True
|
||||
|
|
@ -182,6 +183,21 @@ def ObtainFunctions(
|
|||
os.remove(input_filename)
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
pattern_words = re.compile(r"[\w']+")
|
||||
def TestAndVerify(types):
|
||||
'''
|
||||
This is an early version of TestAndVerify that checks if a type should be accepted or not.
|
||||
It will find all words in the type and check them against a policy. This will be adapted as we
|
||||
get more information about what is supported and what is not.
|
||||
'''
|
||||
|
||||
type_list = re.findall(pattern_words, types)
|
||||
|
||||
for var_type in type_list:
|
||||
if not policy(var_type):
|
||||
return False
|
||||
return True
|
||||
# ----------------------------------------------------------------------
|
||||
|
||||
with callOnExit.CallOnExit(DeleteFile):
|
||||
index = cindex.Index.create()
|
||||
|
|
@ -294,8 +310,6 @@ def ObtainFunctions(
|
|||
# validate exclude_regexes
|
||||
# use the correct name in on_unsupported
|
||||
|
||||
funcs_list = {}
|
||||
|
||||
translation_unit = index.parse(filename, args=args + ['-std=c++17'])
|
||||
|
||||
diagnostics = list(translation_unit.diagnostics)
|
||||
|
|
@ -337,7 +351,6 @@ def ObtainFunctions(
|
|||
)
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
pattern_words = re.compile(r"[\w']+")
|
||||
|
||||
def FullVarType(types):
|
||||
'''
|
||||
|
|
@ -352,44 +365,37 @@ def ObtainFunctions(
|
|||
return types
|
||||
# ----------------------------------------------------------------------
|
||||
|
||||
def TestAndVerify(types):
|
||||
'''
|
||||
This is an early version of TestAndVerify that checks if a type should be accepted or not.
|
||||
It will find all words in the type and check them against a policy. This will be adapted as we
|
||||
get more information about what is supported and what is not.
|
||||
'''
|
||||
type_list = re.findall(pattern_words, types)
|
||||
|
||||
for var_type in type_list:
|
||||
if not policy(var_type):
|
||||
return False
|
||||
return True
|
||||
|
||||
object_type_list = []
|
||||
funcs_list = {}
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
def Enumerate(node):
|
||||
if node.location.file.name == filename:
|
||||
pass
|
||||
def EnumerateObjType(node):
|
||||
if node.kind == cindex.CursorKind.NAMESPACE:
|
||||
for child in node.get_children():
|
||||
Enumerate(child)
|
||||
EnumerateObjType(child)
|
||||
|
||||
if (node.kind == cindex.CursorKind.STRUCT_DECL or node.kind == cindex.CursorKind.CLASS_DECL) and node.location.file.name == filename:
|
||||
if node.kind == cindex.CursorKind.STRUCT_DECL or node.kind == cindex.CursorKind.CLASS_DECL:
|
||||
obj_type = _GetObjectType(node, SimpleVarType, FullVarType)
|
||||
|
||||
if obj_type:
|
||||
object_type_list.append(obj_type)
|
||||
elif obj_type is None:
|
||||
elif obj_type is None and node.location.file.name == filename:
|
||||
# If None was returned, there was a problem with the ObjectType and it can't be processed
|
||||
on_unsupported_func(_FullName(node), filename if (not is_temp_file or filename != input_filename) else None, node.location.line)
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
|
||||
def EnumerateFuncs(node):
|
||||
if node.kind == cindex.CursorKind.NAMESPACE:
|
||||
for child in node.get_children():
|
||||
EnumerateFuncs(child)
|
||||
|
||||
if node.kind == cindex.CursorKind.CONSTRUCTOR:
|
||||
'''
|
||||
TODO: This will support a constructor outside a struct/class. This functionality
|
||||
will be implemented when multiple files are supported (next PR).
|
||||
[EDIT]: This is proving to be much harder than expected. This will get a PR for itself later.
|
||||
'''
|
||||
pass
|
||||
|
||||
if node.kind == cindex.CursorKind.FUNCTION_DECL and node.location.file.name == filename:
|
||||
ret_type = FullVarType(node.result_type.spelling)
|
||||
|
|
@ -413,71 +419,41 @@ def ObtainFunctions(
|
|||
|
||||
# ----------------------------------------------------------------------
|
||||
|
||||
# EnumerateObjType needs to be separated from EnumerateFuncs because of constructors that might be out
|
||||
# of the function.
|
||||
for child in cursor.get_children():
|
||||
Enumerate(child)
|
||||
EnumerateObjType(child)
|
||||
|
||||
function_list = [func.ToObject(key["declaration_line"], key["definition_line"]) for func, key in funcs_list.items()]
|
||||
for child in cursor.get_children():
|
||||
EnumerateFuncs(child)
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
def GetIncludeList():
|
||||
def GetIncludeList(filename):
|
||||
include_list = []
|
||||
for child in translation_unit.get_includes():
|
||||
if child.location.file.name == filename:
|
||||
include_list.append(os.path.realpath(os.path.join(filename, str(child.include))))
|
||||
include_list.append((os.path.realpath(str(child.location.file.name)) if (not is_temp_file or child.location.file.name != input_filename) else None, os.path.realpath(os.path.join(filename, str(child.include)))))
|
||||
return include_list
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
def GetAcceptedObjList():
|
||||
accepted_obj_list = []
|
||||
|
||||
for obj in object_type_list:
|
||||
for curr_obj in object_type_list:
|
||||
if curr_obj in accepted_obj_list:
|
||||
break
|
||||
valid_obj = True
|
||||
for var_type in curr_obj['simple_var_types']:
|
||||
if not TestAndVerify(var_type):
|
||||
valid_obj = False
|
||||
break
|
||||
if valid_obj:
|
||||
accepted_obj_list.append(curr_obj)
|
||||
return accepted_obj_list
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
def GetAcceptedFuncList():
|
||||
accepted_func_list = []
|
||||
|
||||
for func in function_list:
|
||||
valid_func = True
|
||||
for var_type in func['simple_var_types']:
|
||||
if not TestAndVerify(var_type):
|
||||
valid_func = False
|
||||
if not TestAndVerify(func['simple_return_type']):
|
||||
valid_func = False
|
||||
if valid_func:
|
||||
accepted_func_list.append(func)
|
||||
return accepted_func_list
|
||||
# ----------------------------------------------------------------------
|
||||
|
||||
include_list = GetIncludeList()
|
||||
accepted_obj_list = GetAcceptedObjList()
|
||||
accepted_func_list = GetAcceptedFuncList()
|
||||
include_list = GetIncludeList(filename)
|
||||
function_list = [func.ToObject(key["declaration_line"], key["definition_line"]) for func, key in funcs_list.items()]
|
||||
|
||||
for obj in object_type_list:
|
||||
if obj not in accepted_obj_list:
|
||||
on_unsupported_func(obj['name'], filename if (not is_temp_file or filename != input_filename) else None, obj['definition_line'])
|
||||
for func in function_list:
|
||||
if func not in accepted_func_list:
|
||||
on_unsupported_func(func['func_name'], filename if (not is_temp_file or filename != input_filename) else None, func['definition_line'])
|
||||
|
||||
|
||||
# TODO: Needs to expose structs/classes that are on other files. For now, it will just say that its an invalid
|
||||
# function/struct/class.
|
||||
|
||||
return {"function_list": accepted_func_list, "object_type_list": accepted_obj_list, "include_list": include_list }
|
||||
return {"function_list": function_list, "object_type_list": object_type_list, "include_list": include_list}
|
||||
# ----------------------------------------------------------------------
|
||||
|
||||
all_results = OrderedDict()
|
||||
clean_results = OrderedDict()
|
||||
|
||||
def InitializeCleanResults(filename, raw_includes):
|
||||
clean_results[filename] = {}
|
||||
clean_results[filename]["function_list"] = []
|
||||
clean_results[filename]["object_type_list"] = []
|
||||
clean_results[filename]["include_list"] = []
|
||||
for include_tuple in raw_includes:
|
||||
name, include = include_tuple
|
||||
if name == filename:
|
||||
clean_results[filename]["include_list"].append(include)
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
|
||||
while parse_queue:
|
||||
filename = parse_queue.pop(0)
|
||||
|
|
@ -487,9 +463,82 @@ def ObtainFunctions(
|
|||
|
||||
# If the original file was a temp file, make the key None rather than
|
||||
# the name of the temporary file used.
|
||||
if not all_results and is_temp_file:
|
||||
if not clean_results and is_temp_file:
|
||||
filename = None
|
||||
|
||||
all_results[filename] = these_results
|
||||
if not clean_results:
|
||||
InitializeCleanResults(filename, these_results["include_list"])
|
||||
|
||||
return all_results
|
||||
needed_obj_type_list = []
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
def getObjType(obj_type_name):
|
||||
'''
|
||||
Get ObjType from its name.
|
||||
'''
|
||||
for obj_type in these_results["object_type_list"]:
|
||||
if obj_type["name"] == obj_type_name:
|
||||
return obj_type
|
||||
return None
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
def isValidObjType(obj_type):
|
||||
'''
|
||||
Check all var types in this ObjType, this Obj is valid if they are all valid. There is an assumption that
|
||||
this function will only be called for an ObjType that is required. If this Obj depends on another ObjType,
|
||||
that means that the other ObjType is also required.
|
||||
'''
|
||||
if obj_type is None:
|
||||
return False
|
||||
if obj_type in needed_obj_type_list:
|
||||
return True
|
||||
for var_type in obj_type["simple_var_types"]:
|
||||
if not TestAndVerify(var_type) and getObjType(var_type) != obj_type and not isValidObjType(getObjType(var_type)):
|
||||
return False
|
||||
for constructor in obj_type["constructor_list"]:
|
||||
for arg_type in constructor["simple_arg_types"]:
|
||||
if not TestAndVerify(arg_type) and getObjType(arg_type) != obj_type and not isValidObjType(getObjType(arg_type)):
|
||||
return False
|
||||
|
||||
needed_obj_type_list.append(obj_type)
|
||||
return True
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
|
||||
# All ObjTypes in my file are required.
|
||||
for obj_type in these_results["object_type_list"]:
|
||||
if filename is None or obj_type["file"] == filename:
|
||||
if not isValidObjType(obj_type):
|
||||
on_unsupported_func(obj_type['name'], filename, obj_type['definition_line'])
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
def isValidFunc(func):
|
||||
'''
|
||||
A function is valid if all var types are valid.
|
||||
'''
|
||||
for var_type in func["simple_var_types"]:
|
||||
if not TestAndVerify(var_type) and not isValidObjType(getObjType(var_type)):
|
||||
return False
|
||||
if not TestAndVerify(func["simple_return_type"]) and not isValidObjType(getObjType(func["simple_return_type"])):
|
||||
return False
|
||||
return True
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
|
||||
for func in these_results["function_list"]:
|
||||
if isValidFunc(func):
|
||||
clean_results[filename]["function_list"].append(func)
|
||||
else:
|
||||
on_unsupported_func(func['func_name'], filename, func['definition_line'])
|
||||
|
||||
# Add required ObjType to the clean_results list.
|
||||
for obj in needed_obj_type_list:
|
||||
this_file_name = obj.pop('file')
|
||||
if is_temp_file:
|
||||
this_file_name = None
|
||||
if this_file_name not in clean_results:
|
||||
InitializeCleanResults(this_file_name, these_results["include_list"])
|
||||
if obj not in clean_results[this_file_name]["object_type_list"]:
|
||||
clean_results[this_file_name]["object_type_list"].append(obj)
|
||||
|
||||
return clean_results
|
||||
|
|
|
|||
|
|
@ -155,6 +155,43 @@ class FileTest(unittest.TestCase):
|
|||
|
||||
self.assertEqual(len(include_list), 1)
|
||||
|
||||
def test_multiple_includes(self):
|
||||
filename = os.path.join(_script_dir, "includes.cpp")
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
def Policy(var_type):
|
||||
accepted_list = ['double', 'int32_t', 'int64_t','uint32_t','uint64_t','int', 'bool', 'float', 'char', 'vector', 'map', 'pair', 'tuple', 'string', 'void']
|
||||
ignored_list = ['const', 'signed', 'unsigned', 'std']
|
||||
|
||||
if var_type not in accepted_list and var_type not in ignored_list:
|
||||
return False
|
||||
return True
|
||||
# ----------------------------------------------------------------------
|
||||
|
||||
all_results = CppToJson.ObtainFunctions(filename, None, Policy)
|
||||
|
||||
self.assertEqual(len(all_results), 3)
|
||||
|
||||
self.assertEqual(filename, list(all_results.keys())[0])
|
||||
self.assertEqual(all_results[filename]["function_list"][0], {'func_name': 'gox', 'raw_return_type': 'void', 'simple_return_type': 'void', 'var_names': ['x'], 'raw_var_types': ['go'], 'simple_var_types': ['go'], 'declaration_line': None, 'definition_line': 4})
|
||||
self.assertEqual(all_results[filename]["function_list"][1], {'func_name': 'main', 'raw_return_type': 'int', 'simple_return_type': 'int', 'var_names': [], 'raw_var_types': [], 'simple_var_types': [], 'declaration_line': None, 'definition_line': 8})
|
||||
self.assertEqual(all_results[filename]["object_type_list"], [])
|
||||
self.assertEqual(len(all_results[filename]["include_list"]), 1)
|
||||
|
||||
header2 = os.path.realpath(os.path.join(_script_dir, "header2.hpp"))
|
||||
self.assertEqual(header2, list(all_results.keys())[1])
|
||||
self.assertEqual(all_results[header2]["function_list"], [])
|
||||
self.assertEqual(len(all_results[header2]["object_type_list"]), 1)
|
||||
self.assertEqual(all_results[header2]["object_type_list"][0], {'name': 'go2', 'var_names': ['a', 'b'], 'raw_var_types': ['int', 'int'], 'simple_var_types': ['int', 'int'], 'definition_line': 5, 'constructor_list': [{'arg_names': ['other'], 'raw_arg_types': ['go2 &&'], 'simple_arg_types': ['go2'], 'definition_line': 7}]})
|
||||
self.assertEqual(len(all_results[header2]["include_list"]), 1)
|
||||
|
||||
header1 = os.path.realpath(os.path.join(_script_dir, "header1.hpp"))
|
||||
self.assertEqual(header1, list(all_results.keys())[2])
|
||||
self.assertEqual(all_results[header1]["function_list"], [])
|
||||
self.assertEqual(len(all_results[header2]["object_type_list"]), 1)
|
||||
self.assertEqual(all_results[header1]["object_type_list"][0], {'name': 'go', 'var_names': ['a', 'b', 'x'], 'raw_var_types': ['int', 'int', 'go2'], 'simple_var_types': ['int', 'int', 'go2'], 'definition_line': 5, 'constructor_list': [{'arg_names': ['other'], 'raw_arg_types': ['go &&'], 'simple_arg_types': ['go'], 'definition_line': 8}]})
|
||||
self.assertEqual(len(all_results[header1]["include_list"]), 1)
|
||||
|
||||
def _GetFuncList(self, filename, results):
|
||||
self.assertEqual(len(results), 1)
|
||||
self.assertEqual(filename, list(results.keys())[0])
|
||||
|
|
|
|||
|
|
@ -0,0 +1,20 @@
|
|||
#ifndef HEADER1_H
|
||||
#define HEADER1_H
|
||||
#include "header2.hpp"
|
||||
|
||||
struct go{
|
||||
int a, b;
|
||||
go2 x;
|
||||
go(struct go &&other): a(std::move(other.a)), b(std::move(other.b)), x(std::move(other.x)){}
|
||||
};
|
||||
|
||||
struct oth{
|
||||
int x;
|
||||
oth(struct oth &&other): x(std::move(other.x)){}
|
||||
};
|
||||
|
||||
int notCounting(std::vector<float> v){
|
||||
return v.size();
|
||||
}
|
||||
|
||||
#endif
|
||||
|
|
@ -0,0 +1,16 @@
|
|||
#ifndef HEADER2_H
|
||||
#define HEADER2_H
|
||||
#include <vector>
|
||||
|
||||
struct go2{
|
||||
int a, b;
|
||||
go2(struct go2 &&other): a(std::move(other.a)), b(std::move(other.b)){}
|
||||
};
|
||||
|
||||
|
||||
int AlsoNotCounting(std::vector<int> v){
|
||||
return v.size();
|
||||
}
|
||||
|
||||
|
||||
#endif
|
||||
|
|
@ -0,0 +1,11 @@
|
|||
#include "header1.hpp"
|
||||
|
||||
|
||||
void gox(struct go x){
|
||||
|
||||
}
|
||||
|
||||
int main(){
|
||||
|
||||
return 0;
|
||||
}
|
||||
Загрузка…
Ссылка в новой задаче