diff --git a/Libraries/Python/DataPipelines/v1.0/DataPipelines/CppToJson.py b/Libraries/Python/DataPipelines/v1.0/DataPipelines/CppToJson.py index c021e7c..9a1079b 100644 --- a/Libraries/Python/DataPipelines/v1.0/DataPipelines/CppToJson.py +++ b/Libraries/Python/DataPipelines/v1.0/DataPipelines/CppToJson.py @@ -1,16 +1,111 @@ import os import re +import textwrap from collections import OrderedDict import six import clang.cindex as cindex + import CommonEnvironment.FileSystem as fileSystem import CommonEnvironment.CallOnExit as callOnExit from CommonEnvironment.Shell.All import CurrentShell import CommonEnvironment.Shell as CE_Shell +# ---------------------------------------------------------------------- + +def _FullName(node): + ''' + This function will make the name of the function complete to include its namespace. + ''' + name = node.spelling + parent = node.semantic_parent + while parent.kind != cindex.CursorKind.TRANSLATION_UNIT: + name = parent.spelling + "::" + name + parent = parent.semantic_parent + return name + +# ---------------------------------------------------------------------- + +def _GetObjectType(node, SimpleVarType, FullVarType): + ''' + This function will return the Object Type that this node refers to. It will return None if there were + errors. + ''' + + valid_object_type = True + has_move_constructor = False + object_type = {} + object_type['name'] = _FullName(node) + object_type['var_names'] = [] + object_type['raw_var_types'] = [] + object_type['simple_var_types'] = [] + object_type['definition_line'] = node.location.line + object_type['constructor_list'] = [] + + # The way to see if this is a definition or not, is to see if 'node' has any children. + is_def = True + + for child in node.get_children(): + is_def = False + if child.kind == cindex.CursorKind.CONSTRUCTOR: + constructor = {} + if child.is_move_constructor() and child.access_specifier == cindex.AccessSpecifier.PUBLIC: + has_move_constructor = True + elif child.is_copy_constructor() and child.access_specifier == cindex.AccessSpecifier.PUBLIC: + token_list = [] + # If this is a public copy constructor, it needs to be deleted with "=delete" + for token in child.get_tokens(): + token_list.append(token.spelling) + if len(token_list) < 2 or token_list[-1] != 'delete' or token_list[-2] != '=': + valid_object_type = False + + constructor['arg_names'] = [] + constructor['raw_arg_types'] = [] + constructor['simple_arg_types'] = [] + constructor['definition_line'] = child.location.line + for arg in child.get_arguments(): + constructor['arg_names'].append(arg.spelling) + arg_type = FullVarType(arg.type.spelling) + constructor['raw_arg_types'].append(arg_type) + constructor['simple_arg_types'].append(SimpleVarType(arg_type)) + object_type['constructor_list'].append(constructor) + + elif child.kind == cindex.CursorKind.FIELD_DECL: + object_type['var_names'].append(child.spelling) + var_type = FullVarType(child.type.spelling) + object_type['raw_var_types'].append(var_type) + object_type['simple_var_types'].append(SimpleVarType(var_type)) + if child.access_specifier != cindex.AccessSpecifier.PUBLIC: + valid_object_type = False + elif child.kind == cindex.CursorKind.CXX_METHOD: + ''' + TODO: This will change at some point, we will want to support 'operator=' as long as it is public and + a move operator. This is a small draft of what it will look like. For now, all functions are not + supported. + + move_operator_arg_type = FullVarType(node.spelling) + " &&" + if child.spelling == "operator=" and child.access_specifier == cindex.AccessSpecifier.PUBLIC: + for arg in child.get_arguments(): + if FullVarType(arg.type.spelling) != move_operator_arg_type: + valid_object_type = False + else: + valid_object_type = False + ''' + + valid_object_type = False + + elif child.kind != cindex.CursorKind.CXX_ACCESS_SPEC_DECL: + valid_object_type = False + + if not is_def and (not valid_object_type or not has_move_constructor): + return None + elif not is_def: + return object_type + return {} + +# ---------------------------------------------------------------------- def ObtainFunctions( input_filename, @@ -21,14 +116,14 @@ def ObtainFunctions( exclude_regexes=None, ): ''' - This function will extract return value, name and parameters for every - function given. input_filename can be a file name or a string that is the code - itself. - Return value: - Returns a list of functions, every item in this list is a dictionary that - has information about the function. + This function will extract return value, name and parameters for every + function given. input_filename can be a file name or a string that is the code + itself. + Return value: + Returns a list of functions, every item in this list is a dictionary that + has information about the function. ''' - + # ---------------------------------------------------------------------- def ProcessRegexes(regexes): if regexes is None: @@ -59,13 +154,13 @@ def ObtainFunctions( file_pointer.write(file_content) # ---------------------------------------------------------------------- - def DeleteFile(): + def _DeleteFile(): if is_temp_file: os.remove(input_filename) # ---------------------------------------------------------------------- - with callOnExit.CallOnExit(DeleteFile): + with callOnExit.CallOnExit(_DeleteFile): index = cindex.Index.create() args = [] @@ -94,14 +189,18 @@ def ObtainFunctions( # ---------------------------------------------------------------------- def SimpleVarType(name): + ''' + Remove 'const', '*' and '&' + ''' name = re.sub(pattern_const, "", name) name = re.sub(pattern_star, "", name) name = re.sub(pattern_amper, "", name) return name # ---------------------------------------------------------------------- + class Funcs: - ''' + ''' This class will hold a function's information, it provides __hash__ and __eq__ functions. It is needed so that its possible to have a dictionary using this class as a key, to keep track of the declaration and implementation lines and have fast lookup. @@ -113,7 +212,7 @@ def ObtainFunctions( self._var_names = [] self._raw_var_types = [] self._simple_var_types = [] - + def AddVar(self, var_name, raw_var_type, simple_var_type): self._var_names.append(var_name) self._raw_var_types.append(raw_var_type) @@ -174,79 +273,169 @@ def ObtainFunctions( cursor = translation_unit.cursor + def GetAlias(): + ''' + This function will process all 'typedef' and 'using' and it will map the underlying type to + its definition. + ''' + alias = {} + for child in cursor.get_children(): + if (child.kind == cindex.CursorKind.TYPEDEF_DECL or child.kind == cindex.CursorKind.TYPE_ALIAS_DECL) and child.location.file.name == input_filename: + alias[child.spelling] = child.underlying_typedef_type.spelling + return alias + + alias = GetAlias() + + alias_regex = re.compile( + textwrap.dedent( + r'''(?# + Not a letter)(?{})(?# + Not a letter)(?!\w)(?# + )''' + ).format("|".join([re.escape(key) for key in alias.keys()])) + ) + + struct_class_pattern = re.compile( + textwrap.dedent( + r'''(?# + Not a letter)(?struct\s|class\s)(?# + )''' + ) + ) + + # ---------------------------------------------------------------------- + pattern_words = re.compile(r"[\w']+") + + def FullVarType(types): + ''' + This will undo all 'typedef' and 'using' by looking for the items in the 'alias' dict and substituting + the corresponding definitions. It will also remove all occurences of the words 'struct' and 'class'. + ''' + num_subs = True + while num_subs and alias: + types, num_subs = re.subn(alias_regex, lambda k: alias[k.group(1)], types) + + types = struct_class_pattern.sub(r'', types) + return types + # ---------------------------------------------------------------------- + + def TestAndVerify(types): + ''' + This is an early version of TestAndVerify that checks if a type should be accepted or not. + It will find all words in the type and check them against a policy. This will be adapted as we + get more information about what is supported and what is not. + ''' + type_list = re.findall(pattern_words, types) + + for var_type in type_list: + if not policy(var_type): + return False + return True + + object_type_list = [] + # ---------------------------------------------------------------------- def Enumerate(node): - if node.kind == cindex.CursorKind.NAMESPACE: for child in node.get_children(): Enumerate(child) - # ---------------------------------------------------------------------- + if (node.kind == cindex.CursorKind.STRUCT_DECL or node.kind == cindex.CursorKind.CLASS_DECL) and node.location.file.name == filename: + obj_type = _GetObjectType(node, SimpleVarType, FullVarType) - pattern_words = re.compile(r"[\w']+") + if obj_type: + object_type_list.append(obj_type) + elif obj_type is None: + # If None was returned, there was a problem with the ObjectType and it can't be processed + on_unsupported_func(_FullName(node), filename if (not is_temp_file or filename != input_filename) else None, node.location.line) - def TestAndVerify(types): - ''' - This is an early version of TestAndVerify that checks if a type should be accepted or not. - It will find all words in the type and check them against a policy. This will be adapted as we - get more information about what is supported and what is not. - ''' - type_list = re.findall(pattern_words, types) - - for var_type in type_list: - if not policy(var_type): - return False - return True - - # ---------------------------------------------------------------------- if node.kind == cindex.CursorKind.FUNCTION_DECL and node.location.file.name == filename: - valid_func = True - - # ---------------------------------------------------------------------- - def FullName(node): - ''' - This function will make the name of the function complete to include its namespace. - ''' - name = node.spelling - parent = node.semantic_parent - while parent.kind != cindex.CursorKind.TRANSLATION_UNIT: - name = parent.spelling + "::" + name - parent = parent.semantic_parent - return name - # ---------------------------------------------------------------------- - func = Funcs(FullName(node), node.result_type.spelling, SimpleVarType(node.result_type.spelling)) - - if not TestAndVerify(node.result_type.spelling): - valid_func = False + ret_type = FullVarType(node.result_type.spelling) + func = Funcs(_FullName(node), ret_type, SimpleVarType(ret_type)) for arg in node.get_arguments(): - func.AddVar(arg.displayname, arg.type.spelling, SimpleVarType(arg.type.spelling)) + arg_type = FullVarType(arg.type.spelling) + func.AddVar(arg.displayname, arg_type, SimpleVarType(arg_type)) - if not TestAndVerify(arg.type.spelling): - valid_func = False + if func not in funcs_list.keys(): + funcs_list[func] = {'definition_line': None, 'declaration_line': None} + is_def = False + for child in node.get_children(): + if child.kind == cindex.CursorKind.COMPOUND_STMT: + is_def = True - if not valid_func: - on_unsupported_func(FullName(node), filename if (not is_temp_file or filename != input_filename) else None, node.location.line) + if is_def: + funcs_list[func]['definition_line'] = node.location.line else: - if func not in funcs_list.keys(): - funcs_list[func] = {'definition_line': None, 'declaration_line': None} - is_def = False - for child in node.get_children(): - if child.kind == cindex.CursorKind.COMPOUND_STMT: - is_def = True - - if is_def: - funcs_list[func]['definition_line'] = node.location.line - else: - funcs_list[func]['declaration_line'] = node.location.line + funcs_list[func]['declaration_line'] = node.location.line # ---------------------------------------------------------------------- + for child in cursor.get_children(): Enumerate(child) + + function_list = [func.ToObject(key["declaration_line"], key["definition_line"]) for func, key in funcs_list.items()] - return [func.ToObject(key["declaration_line"], key["definition_line"]) for func, key in funcs_list.items()] + # ---------------------------------------------------------------------- + def GetIncludeList(): + include_list = [] + for child in translation_unit.get_includes(): + if child.location.file.name == filename: + include_list.append(os.path.realpath(os.path.join(filename, str(child.include)))) + return include_list + # ---------------------------------------------------------------------- + def GetAcceptedObjList(): + accepted_obj_list = [] + + for obj in object_type_list: + for curr_obj in object_type_list: + if curr_obj in accepted_obj_list: + break + valid_obj = True + for var_type in curr_obj['simple_var_types']: + if not TestAndVerify(var_type): + valid_obj = False + break + if valid_obj: + accepted_obj_list.append(curr_obj) + return accepted_obj_list + + # ---------------------------------------------------------------------- + def GetAcceptedFuncList(): + accepted_func_list = [] + + for func in function_list: + valid_func = True + for var_type in func['simple_var_types']: + if not TestAndVerify(var_type): + valid_func = False + if not TestAndVerify(func['simple_return_type']): + valid_func = False + if valid_func: + accepted_func_list.append(func) + return accepted_func_list + # ---------------------------------------------------------------------- + + include_list = GetIncludeList() + accepted_obj_list = GetAcceptedObjList() + accepted_func_list = GetAcceptedFuncList() + + for obj in object_type_list: + if obj not in accepted_obj_list: + on_unsupported_func(obj['name'], filename if (not is_temp_file or filename != input_filename) else None, obj['definition_line']) + for func in function_list: + if func not in accepted_func_list: + on_unsupported_func(func['func_name'], filename if (not is_temp_file or filename != input_filename) else None, func['definition_line']) + + + # TODO: Needs to expose structs/classes that are on other files. For now, it will just say that its an invalid + # function/struct/class. + + return {"function_list": accepted_func_list, "object_type_list": accepted_obj_list, "include_list": include_list } # ---------------------------------------------------------------------- all_results = OrderedDict() @@ -263,5 +452,5 @@ def ObtainFunctions( filename = None all_results[filename] = these_results - + return all_results diff --git a/Libraries/Python/DataPipelines/v1.0/DataPipelines/IntegrationTests/CppToJson_IntegrationTest.py b/Libraries/Python/DataPipelines/v1.0/DataPipelines/IntegrationTests/CppToJson_IntegrationTest.py index 7ba341e..0ae5d31 100644 --- a/Libraries/Python/DataPipelines/v1.0/DataPipelines/IntegrationTests/CppToJson_IntegrationTest.py +++ b/Libraries/Python/DataPipelines/v1.0/DataPipelines/IntegrationTests/CppToJson_IntegrationTest.py @@ -23,6 +23,8 @@ class FileTest(unittest.TestCase): def test_basic_file(self): filename = os.path.join(_script_dir, "basicFunc.cpp") func_list = self._GetFuncList(filename, CppToJson.ObtainFunctions(filename, None, lambda type: True)) + obj_type_list = self._GetObjList(filename, CppToJson.ObtainFunctions(filename, None, lambda type: True)) + include_list = self._GetIncludeList(filename, CppToJson.ObtainFunctions(filename, None, lambda type: True)) self.assertEqual(func_list[0], {'func_name': 'add', 'raw_return_type': 'int', 'simple_return_type': 'int', 'var_names': ['a', 'b'], 'raw_var_types': ['int', 'int'], 'simple_var_types': ['int', 'int'], 'definition_line': 6, 'declaration_line': None}) self.assertEqual(func_list[1], {'func_name': 'sub', 'raw_return_type': 'float', 'simple_return_type': 'float', 'var_names': ['a', 'b'], 'raw_var_types': ['float', 'float'], 'simple_var_types': ['float', 'float'], 'definition_line': 10, 'declaration_line': None}) @@ -31,9 +33,15 @@ class FileTest(unittest.TestCase): self.assertEqual(func_list[4], {'func_name': 'nothing', 'raw_return_type': 'void', 'simple_return_type': 'void', 'var_names': [], 'raw_var_types': [], 'simple_var_types': [], 'definition_line': 22, 'declaration_line': None}) self.assertEqual(func_list[5], {'func_name': 'main', 'raw_return_type': 'int', 'simple_return_type': 'int', 'var_names': [], 'raw_var_types': [], 'simple_var_types': [], 'definition_line': 27, 'declaration_line': None}) + self.assertEqual(obj_type_list, []) + self.assertEqual(len(include_list), 1) + + def test_medium_file(self): filename = os.path.join(_script_dir, "mediumFunc.cpp") func_list = self._GetFuncList(filename, CppToJson.ObtainFunctions(filename, None, lambda type: True)) + obj_type_list = self._GetObjList(filename, CppToJson.ObtainFunctions(filename, None, lambda type: True)) + include_list = self._GetIncludeList(filename, CppToJson.ObtainFunctions(filename, None, lambda type: True)) self.assertEqual(func_list[0], {'func_name': 'add', 'raw_return_type': 'int', 'simple_return_type': 'int', 'var_names': ['a', 'b'], 'raw_var_types': ['float', 'int'], 'simple_var_types': ['float', 'int'], 'definition_line': 5, 'declaration_line': None}) self.assertEqual(func_list[1], {'func_name': 'mult', 'raw_return_type': 'float', 'simple_return_type': 'float', 'var_names': ['a', 'b', 'signal'], 'raw_var_types': ['int', 'float', 'bool'], 'simple_var_types': ['int', 'float', 'bool'], 'definition_line': 9, 'declaration_line': None}) @@ -41,10 +49,14 @@ class FileTest(unittest.TestCase): self.assertEqual(func_list[3], {'func_name': 'fat', 'raw_return_type': 'int', 'simple_return_type': 'int', 'var_names': ['curr', 'at'], 'raw_var_types': ['int', 'int'], 'simple_var_types': ['int', 'int'], 'definition_line': 19, 'declaration_line': None}) self.assertEqual(func_list[4], {'func_name': 'main', 'raw_return_type': 'int', 'simple_return_type': 'int', 'var_names': [], 'raw_var_types': [], 'simple_var_types': [], 'definition_line': 24, 'declaration_line': None}) + self.assertEqual(obj_type_list, []) + self.assertEqual(len(include_list), 1) def test_hard_file(self): filename = os.path.join(_script_dir, "hardFunc.cpp") func_list = self._GetFuncList(filename, CppToJson.ObtainFunctions(filename, None, lambda type: True)) + obj_type_list = self._GetObjList(filename, CppToJson.ObtainFunctions(filename, None, lambda type: True)) + include_list = self._GetIncludeList(filename, CppToJson.ObtainFunctions(filename, None, lambda type: True)) self.assertEqual(func_list[0], {'func_name': 'add', 'raw_return_type': 'int', 'simple_return_type': 'int', 'var_names': ['a'], 'raw_var_types': ['int'], 'simple_var_types': ['int'], 'definition_line': 10, 'declaration_line': None}) self.assertEqual(func_list[1], {'func_name': 'main', 'raw_return_type': 'int', 'simple_return_type': 'int', 'var_names': [], 'raw_var_types': [], 'simple_var_types': [], 'definition_line': 17, 'declaration_line': None}) @@ -53,10 +65,14 @@ class FileTest(unittest.TestCase): self.assertEqual(func_list[4], {'func_name': 'keys', 'raw_return_type': 'vector', 'simple_return_type': 'vector', 'var_names': ['mp'], 'raw_var_types': ['map'], 'simple_var_types': ['map'], 'definition_line': 35, 'declaration_line': None}) self.assertEqual(func_list[5], {'func_name': 'goCount', 'raw_return_type': 'map', 'simple_return_type': 'map', 'var_names': ['v', 'signal'], 'raw_var_types': ['vector', 'bool'], 'simple_var_types': ['vector', 'bool'], 'definition_line': 43, 'declaration_line': None}) + self.assertEqual(obj_type_list, []) + self.assertEqual(len(include_list), 3) def test_convoluted_file(self): filename = os.path.join(_script_dir, "convolutedFunc.cpp") func_list = self._GetFuncList(filename, CppToJson.ObtainFunctions(filename, None, lambda type: True)) + obj_type_list = self._GetObjList(filename, CppToJson.ObtainFunctions(filename, None, lambda type: True)) + include_list = self._GetIncludeList(filename, CppToJson.ObtainFunctions(filename, None, lambda type: True)) self.assertEqual(func_list[0], {'func_name': 'matrix', 'raw_return_type': 'vector >', 'simple_return_type': 'vector >', 'var_names': ['n'], 'raw_var_types': ['int'], 'simple_var_types': ['int'], 'definition_line': 9, 'declaration_line': None}) self.assertEqual(func_list[1], {'func_name': 'nonsense', 'raw_return_type': 'map >, vector >', 'simple_return_type': 'map >, vector >', 'var_names': ['n'], 'raw_var_types': ['int'], 'simple_var_types': ['int'], 'definition_line': 19, 'declaration_line': None}) @@ -64,9 +80,14 @@ class FileTest(unittest.TestCase): self.assertEqual(func_list[3], {'func_name': 'countVector', 'raw_return_type': 'map', 'simple_return_type': 'map', 'var_names': ['v'], 'raw_var_types': ['vector >'], 'simple_var_types': ['vector >'], 'definition_line': 42, 'declaration_line': None}) self.assertEqual(func_list[4], {'func_name': 'main', 'raw_return_type': 'int', 'simple_return_type': 'int', 'var_names': [], 'raw_var_types': [], 'simple_var_types': [], 'definition_line': 50, 'declaration_line': None}) + self.assertEqual(obj_type_list, []) + self.assertEqual(len(include_list), 3) + def test_mix_file(self): filename = os.path.join(_script_dir, "mixFunc.cpp") func_list = self._GetFuncList(filename, CppToJson.ObtainFunctions(filename, None, lambda type: True)) + obj_type_list = self._GetObjList(filename, CppToJson.ObtainFunctions(filename, None, lambda type: True)) + include_list = self._GetIncludeList(filename, CppToJson.ObtainFunctions(filename, None, lambda type: True)) self.assertEqual(func_list[0], {'func_name': 'nonsense', 'raw_return_type': 'vector > *', 'simple_return_type': 'vector >', 'var_names': ['v', 'mp'], 'raw_var_types': ['vector &', 'map *'], 'simple_var_types': ['vector', 'map'], 'definition_line': 6, 'declaration_line': None}) self.assertEqual(func_list[1], {'func_name': 'address', 'raw_return_type': 'vector &', 'simple_return_type': 'vector', 'var_names': ['v'], 'raw_var_types': ['vector &'], 'simple_var_types': ['vector'], 'definition_line': 11, 'declaration_line': None}) @@ -75,6 +96,9 @@ class FileTest(unittest.TestCase): self.assertEqual(func_list[4], {'func_name': 'constDereference', 'raw_return_type': 'const int **********', 'simple_return_type': 'int', 'var_names': ['ref'], 'raw_var_types': ['const int ***********'], 'simple_var_types': ['int'], 'definition_line': 26, 'declaration_line': None}) self.assertEqual(func_list[5], {'func_name': 'main', 'raw_return_type': 'int', 'simple_return_type': 'int', 'var_names': [], 'raw_var_types': [], 'simple_var_types': [], 'definition_line': 31, 'declaration_line': None}) + self.assertEqual(obj_type_list, []) + self.assertEqual(len(include_list), 2) + def test_class_file_unsupported(self): filename = os.path.join(_script_dir, "classFunc.cpp") @@ -84,19 +108,25 @@ class FileTest(unittest.TestCase): def onUnsupportedFunc(func, this_filename, line): nonlocal called_count called_count += 1 - self.assertTrue([func, this_filename, line] in [['operator+', filename, 15], ['sum', filename, 22], ['go', filename, 26], ['main', filename, 34]]) + self.assertTrue([func, this_filename, line] in [['Point', filename, 5], ['operator+', filename, 15], ['sum', filename, 22], ['go', filename, 26], ['main', filename, 34]]) # ---------------------------------------------------------------------- func_list = self._GetFuncList(filename, CppToJson.ObtainFunctions(filename, onUnsupportedFunc, lambda type: False)) - - self.assertEqual(called_count, 4) + obj_type_list = self._GetObjList(filename, CppToJson.ObtainFunctions(filename, onUnsupportedFunc, lambda type: False)) + include_list = self._GetIncludeList(filename, CppToJson.ObtainFunctions(filename, onUnsupportedFunc, lambda type: False)) + + self.assertEqual(called_count, 15) self.assertEqual(func_list, []) + self.assertEqual(obj_type_list, []) + self.assertEqual(len(include_list), 1) def test_namespace_file(self): filename = os.path.join(_script_dir, "arithmetic.cpp") func_list = self._GetFuncList(filename, CppToJson.ObtainFunctions(filename, None, lambda type: True)) + obj_type_list = self._GetObjList(filename, CppToJson.ObtainFunctions(filename, None, lambda type: True)) + include_list = self._GetIncludeList(filename, CppToJson.ObtainFunctions(filename, None, lambda type: True)) self.assertEqual(func_list[0], {'func_name': 'DataPipelines::Arithmetic::Add', 'raw_return_type': 'int64_t', 'simple_return_type': 'int64_t', 'var_names': ['a', 'b'], 'raw_var_types': ['const int64_t', 'const int64_t'], 'simple_var_types': ['int64_t', 'int64_t'], 'definition_line': 12, 'declaration_line': None}) self.assertEqual(func_list[1], {'func_name': 'DataPipelines::Arithmetic::Add', 'raw_return_type': 'uint64_t', 'simple_return_type': 'uint64_t', 'var_names': ['a', 'b'], 'raw_var_types': ['const uint64_t', 'const uint64_t'], 'simple_var_types': ['uint64_t', 'uint64_t'], 'definition_line': 13, 'declaration_line': None}) @@ -109,11 +139,39 @@ class FileTest(unittest.TestCase): self.assertEqual(func_list[8], {'func_name': 'Addi64', 'raw_return_type': 'int64_t', 'simple_return_type': 'int64_t', 'var_names': ['a', 'b'], 'raw_var_types': ['const int64_t', 'const int64_t'], 'simple_var_types': ['int64_t', 'int64_t'], 'definition_line': 44, 'declaration_line': None}) self.assertEqual(func_list[9], {'func_name': 'Addu32', 'raw_return_type': 'uint32_t', 'simple_return_type': 'uint32_t', 'var_names': ['a', 'b'], 'raw_var_types': ['const uint32_t', 'const uint32_t'], 'simple_var_types': ['uint32_t', 'uint32_t'], 'definition_line': 48, 'declaration_line': None}) + self.assertEqual(obj_type_list, []) + self.assertEqual(len(include_list), 2) + + def test_supported_struct(self): + filename = os.path.join(_script_dir, "supportedStruct.cpp") + func_list = self._GetFuncList(filename, CppToJson.ObtainFunctions(filename, None, lambda type: True)) + obj_type_list = self._GetObjList(filename, CppToJson.ObtainFunctions(filename, None, lambda type: True)) + include_list = self._GetIncludeList(filename, CppToJson.ObtainFunctions(filename, None, lambda type: True)) + + self.assertEqual(func_list[0], {'func_name': 'go', 'raw_return_type': 'int', 'simple_return_type': 'int', 'var_names': ['ya'], 'raw_var_types': ['x'], 'simple_var_types': ['x'], 'declaration_line': None, 'definition_line': 12}) + self.assertEqual(func_list[1], {'func_name': 'main', 'raw_return_type': 'int', 'simple_return_type': 'int', 'var_names': [], 'raw_var_types': [], 'simple_var_types': [], 'declaration_line': None, 'definition_line': 16}) + + self.assertEqual(obj_type_list[0], {'name': 'x', 'var_names': ['a', 'b'], 'raw_var_types': ['int', 'int'], 'simple_var_types': ['int', 'int'], 'definition_line': 3, 'constructor_list': [{'arg_names': ['other'], 'raw_arg_types': ['x &&'], 'simple_arg_types': ['x'], 'definition_line': 5}, {'arg_names': ['xa', 'xb'], 'raw_arg_types': ['int', 'int'], 'simple_arg_types': ['int', 'int'], 'definition_line': 6}]}) + + self.assertEqual(len(include_list), 1) + def _GetFuncList(self, filename, results): self.assertEqual(len(results), 1) self.assertEqual(filename, list(results.keys())[0]) - return results[filename] + return results[filename]['function_list'] + + def _GetObjList(self, filename, results): + self.assertEqual(len(results), 1) + self.assertEqual(filename, list(results.keys())[0]) + + return results[filename]['object_type_list'] + + def _GetIncludeList(self, filename, results): + self.assertEqual(len(results), 1) + self.assertEqual(filename, list(results.keys())[0]) + + return results[filename]['include_list'] if __name__ == '__main__': unittest.main() diff --git a/Libraries/Python/DataPipelines/v1.0/DataPipelines/IntegrationTests/basicFunc.cpp b/Libraries/Python/DataPipelines/v1.0/DataPipelines/IntegrationTests/basicFunc.cpp index 0760cc0..e2f5b7a 100644 --- a/Libraries/Python/DataPipelines/v1.0/DataPipelines/IntegrationTests/basicFunc.cpp +++ b/Libraries/Python/DataPipelines/v1.0/DataPipelines/IntegrationTests/basicFunc.cpp @@ -1,5 +1,5 @@ #include -#include + using namespace std; diff --git a/Libraries/Python/DataPipelines/v1.0/DataPipelines/IntegrationTests/convolutedFunc.cpp b/Libraries/Python/DataPipelines/v1.0/DataPipelines/IntegrationTests/convolutedFunc.cpp index 5562ad0..8cc6fdb 100644 --- a/Libraries/Python/DataPipelines/v1.0/DataPipelines/IntegrationTests/convolutedFunc.cpp +++ b/Libraries/Python/DataPipelines/v1.0/DataPipelines/IntegrationTests/convolutedFunc.cpp @@ -1,9 +1,9 @@ #include -#include -#include #include #include + + using namespace std; vector> matrix(int n){ diff --git a/Libraries/Python/DataPipelines/v1.0/DataPipelines/IntegrationTests/hardFunc.cpp b/Libraries/Python/DataPipelines/v1.0/DataPipelines/IntegrationTests/hardFunc.cpp index 9845e57..0160d3b 100644 --- a/Libraries/Python/DataPipelines/v1.0/DataPipelines/IntegrationTests/hardFunc.cpp +++ b/Libraries/Python/DataPipelines/v1.0/DataPipelines/IntegrationTests/hardFunc.cpp @@ -1,10 +1,10 @@ #include -#include -#include #include #include + + using namespace std; int add(int a){ diff --git a/Libraries/Python/DataPipelines/v1.0/DataPipelines/IntegrationTests/mediumFunc.cpp b/Libraries/Python/DataPipelines/v1.0/DataPipelines/IntegrationTests/mediumFunc.cpp index d47fa3f..c7e60c3 100644 --- a/Libraries/Python/DataPipelines/v1.0/DataPipelines/IntegrationTests/mediumFunc.cpp +++ b/Libraries/Python/DataPipelines/v1.0/DataPipelines/IntegrationTests/mediumFunc.cpp @@ -1,6 +1,6 @@ #include -#include -#include + + int add(float a, int b){ return a + b; diff --git a/Libraries/Python/DataPipelines/v1.0/DataPipelines/IntegrationTests/supportedStruct.cpp b/Libraries/Python/DataPipelines/v1.0/DataPipelines/IntegrationTests/supportedStruct.cpp new file mode 100644 index 0000000..ac1f4bc --- /dev/null +++ b/Libraries/Python/DataPipelines/v1.0/DataPipelines/IntegrationTests/supportedStruct.cpp @@ -0,0 +1,18 @@ +#include + +struct x{ + int a, b; + x(struct x &&other): a(std::move(other.a)), b(std::move(other.b)){} + x(int xa, int xb){ + a=xa; + b=xb; + } +}; + +int go(struct x ya){ + return 2; +} + +int main(){ + return 0; +} \ No newline at end of file diff --git a/Libraries/Python/DataPipelines/v1.0/DataPipelines/UnitTests/CppToJson_UnitTest.py b/Libraries/Python/DataPipelines/v1.0/DataPipelines/UnitTests/CppToJson_UnitTest.py index 86e8ac5..b703c83 100644 --- a/Libraries/Python/DataPipelines/v1.0/DataPipelines/UnitTests/CppToJson_UnitTest.py +++ b/Libraries/Python/DataPipelines/v1.0/DataPipelines/UnitTests/CppToJson_UnitTest.py @@ -19,6 +19,8 @@ class FuncTest(unittest.TestCase): } ''') func_list = self._GetFuncList(CppToJson.ObtainFunctions(s, None, lambda type: True)) + obj_type_list = self._GetObjList(CppToJson.ObtainFunctions(s, None, lambda type: True)) + include_list = self._GetIncludeList(CppToJson.ObtainFunctions(s, None, lambda type: True)) self.assertEqual(func_list[0]['func_name'], 'add') self.assertEqual(func_list[0]['raw_return_type'], 'int') @@ -38,6 +40,10 @@ class FuncTest(unittest.TestCase): self.assertEqual(func_list[1]['definition_line'], 4) self.assertEqual(func_list[1]['declaration_line'], None) + self.assertEqual(obj_type_list, []) + self.assertEqual(include_list, []) + + def test_vector_func(self): s = textwrap.dedent('''\ #include @@ -54,6 +60,8 @@ class FuncTest(unittest.TestCase): } ''') func_list = self._GetFuncList(CppToJson.ObtainFunctions(s, None, lambda type: True)) + obj_type_list = self._GetObjList(CppToJson.ObtainFunctions(s, None, lambda type: True)) + include_list = self._GetIncludeList(CppToJson.ObtainFunctions(s, None, lambda type: True)) self.assertEqual(func_list[0]['func_name'], 'counting') self.assertEqual(func_list[0]['raw_return_type'], 'std::vector') @@ -73,6 +81,9 @@ class FuncTest(unittest.TestCase): self.assertEqual(func_list[1]['definition_line'], 10) self.assertEqual(func_list[1]['declaration_line'], None) + self.assertEqual(obj_type_list, []) + self.assertEqual(len(include_list), 1) + def test_vector_using_std_func(self): s = textwrap.dedent('''\ @@ -90,6 +101,8 @@ class FuncTest(unittest.TestCase): } ''') func_list = self._GetFuncList(CppToJson.ObtainFunctions(s, None, lambda type: True)) + obj_type_list = self._GetObjList(CppToJson.ObtainFunctions(s, None, lambda type: True)) + include_list = self._GetIncludeList(CppToJson.ObtainFunctions(s, None, lambda type: True)) self.assertEqual(func_list[0]['func_name'], 'counting') self.assertEqual(func_list[0]['raw_return_type'], 'vector') @@ -109,6 +122,9 @@ class FuncTest(unittest.TestCase): self.assertEqual(func_list[1]['definition_line'], 10) self.assertEqual(func_list[1]['declaration_line'], None) + self.assertEqual(obj_type_list, []) + self.assertEqual(len(include_list), 1) + def test_vector_map(self): s = textwrap.dedent('''\ #include @@ -126,6 +142,8 @@ class FuncTest(unittest.TestCase): } ''') func_list = self._GetFuncList(CppToJson.ObtainFunctions(s, None, lambda type: True)) + obj_type_list = self._GetObjList(CppToJson.ObtainFunctions(s, None, lambda type: True)) + include_list = self._GetIncludeList(CppToJson.ObtainFunctions(s, None, lambda type: True)) self.assertEqual(func_list[0]['func_name'], 'counting') self.assertEqual(func_list[0]['raw_return_type'], 'vector >') @@ -145,6 +163,9 @@ class FuncTest(unittest.TestCase): self.assertEqual(func_list[1]['definition_line'], 11) self.assertEqual(func_list[1]['declaration_line'], None) + self.assertEqual(obj_type_list, []) + self.assertEqual(len(include_list), 2) + def test_many_arguments(self): s = textwrap.dedent('''\ int two(int a, int b, int c, int d, int e, int f, int g, int h, int i, int j, @@ -158,6 +179,9 @@ class FuncTest(unittest.TestCase): } ''') func_list = self._GetFuncList(CppToJson.ObtainFunctions(s, None, lambda type: True)) + obj_type_list = self._GetObjList(CppToJson.ObtainFunctions(s, None, lambda type: True)) + include_list = self._GetIncludeList(CppToJson.ObtainFunctions(s, None, lambda type: True)) + self.assertEqual(func_list[0]['func_name'], 'two') self.assertEqual(func_list[0]['raw_return_type'], 'int') self.assertEqual(func_list[0]['simple_return_type'], 'int') @@ -176,6 +200,9 @@ class FuncTest(unittest.TestCase): self.assertEqual(func_list[1]['definition_line'], 7) self.assertEqual(func_list[1]['declaration_line'], None) + self.assertEqual(obj_type_list, []) + self.assertEqual(include_list, []) + def test_many_arguments_ref(self): s = textwrap.dedent('''\ int two(int **a, int **b, int **c, int **d, int **e, int **f, int **g, int **h, @@ -189,6 +216,9 @@ class FuncTest(unittest.TestCase): } ''') func_list = self._GetFuncList(CppToJson.ObtainFunctions(s, None, lambda type: True)) + obj_type_list = self._GetObjList(CppToJson.ObtainFunctions(s, None, lambda type: True)) + include_list = self._GetIncludeList(CppToJson.ObtainFunctions(s, None, lambda type: True)) + self.assertEqual(func_list[0]['func_name'], 'two') self.assertEqual(func_list[0]['raw_return_type'], 'int') self.assertEqual(func_list[0]['simple_return_type'], 'int') @@ -207,6 +237,9 @@ class FuncTest(unittest.TestCase): self.assertEqual(func_list[1]['definition_line'], 7) self.assertEqual(func_list[1]['declaration_line'], None) + self.assertEqual(obj_type_list, []) + self.assertEqual(include_list, []) + def test_const_ret(self): s = textwrap.dedent('''\ const float square(float x){ @@ -218,6 +251,9 @@ class FuncTest(unittest.TestCase): } ''') func_list = self._GetFuncList(CppToJson.ObtainFunctions(s, None, lambda type: True)) + obj_type_list = self._GetObjList(CppToJson.ObtainFunctions(s, None, lambda type: True)) + include_list = self._GetIncludeList(CppToJson.ObtainFunctions(s, None, lambda type: True)) + self.assertEqual(func_list[0]['func_name'], 'square') self.assertEqual(func_list[0]['raw_return_type'], 'const float') self.assertEqual(func_list[0]['simple_return_type'], 'float') @@ -236,6 +272,9 @@ class FuncTest(unittest.TestCase): self.assertEqual(func_list[1]['definition_line'], 5) self.assertEqual(func_list[1]['declaration_line'], None) + self.assertEqual(obj_type_list, []) + self.assertEqual(include_list, []) + def test_const_arg(self): s = textwrap.dedent('''\ float sum(const float x, const int y){ @@ -246,6 +285,9 @@ class FuncTest(unittest.TestCase): } ''') func_list = self._GetFuncList(CppToJson.ObtainFunctions(s, None, lambda type: True)) + obj_type_list = self._GetObjList(CppToJson.ObtainFunctions(s, None, lambda type: True)) + include_list = self._GetIncludeList(CppToJson.ObtainFunctions(s, None, lambda type: True)) + self.assertEqual(func_list[0]['func_name'], 'sum') self.assertEqual(func_list[0]['raw_return_type'], 'float') self.assertEqual(func_list[0]['simple_return_type'], 'float') @@ -264,6 +306,9 @@ class FuncTest(unittest.TestCase): self.assertEqual(func_list[1]['definition_line'], 4) self.assertEqual(func_list[1]['declaration_line'], None) + self.assertEqual(obj_type_list, []) + self.assertEqual(include_list, []) + def test_map_vec_ref(self): s = textwrap.dedent( '''\ #include @@ -281,6 +326,9 @@ class FuncTest(unittest.TestCase): } ''') func_list = self._GetFuncList(CppToJson.ObtainFunctions(s, None, lambda type: True)) + obj_type_list = self._GetObjList(CppToJson.ObtainFunctions(s, None, lambda type: True)) + include_list = self._GetIncludeList(CppToJson.ObtainFunctions(s, None, lambda type: True)) + self.assertEqual(func_list[0]['func_name'], 'nonsense') self.assertEqual(func_list[0]['raw_return_type'], 'vector > *') self.assertEqual(func_list[0]['simple_return_type'], 'vector >') @@ -299,12 +347,17 @@ class FuncTest(unittest.TestCase): self.assertEqual(func_list[1]['definition_line'], 11) self.assertEqual(func_list[1]['declaration_line'], None) + self.assertEqual(obj_type_list, []) + self.assertEqual(len(include_list), 2) + def test_struct_unsupported(self): was_called = False s = textwrap.dedent( '''\ + #include struct x{ int a, b; + x(struct x &&other): a(std::move(other.a)), b(std::move(other.b)){} }; int go(struct x ya){ @@ -319,15 +372,74 @@ class FuncTest(unittest.TestCase): def onUnsupportedFunc(func, filename, line): nonlocal was_called was_called = True - self.assertTrue([func, filename, line] in [['go', None, 5], ['main', None, 9]]) + self.assertTrue([func, filename, line] in [['x', None, 2], ['go', None, 7], ['main', None, 11]]) # ---------------------------------------------------------------------- func_list = self._GetFuncList(CppToJson.ObtainFunctions(s, onUnsupportedFunc, lambda type: False)) + obj_type_list = self._GetObjList(CppToJson.ObtainFunctions(s, onUnsupportedFunc, lambda type: False)) + include_list = self._GetIncludeList(CppToJson.ObtainFunctions(s, onUnsupportedFunc, lambda type: False)) + self.assertEqual(was_called, True) self.assertEqual(func_list, []) + self.assertEqual(obj_type_list, []) + self.assertEqual(len(include_list), 1) + + def test_struct_supported(self): + + s = textwrap.dedent( '''\ + #include + struct x{ + int a, b; + x(struct x &&other): a(std::move(other.a)), b(std::move(other.b)){} + }; + + int go(struct x ya){ + return 2; + } + + int main(){ + return 0; + } + ''') + + func_list = self._GetFuncList(CppToJson.ObtainFunctions(s, None, lambda type: True)) + obj_type_list = self._GetObjList(CppToJson.ObtainFunctions(s, None, lambda type: True)) + include_list = self._GetIncludeList(CppToJson.ObtainFunctions(s, None, lambda type: True)) + + self.assertEqual(func_list[0]['func_name'], 'go') + self.assertEqual(func_list[0]['raw_return_type'], 'int') + self.assertEqual(func_list[0]['simple_return_type'], 'int') + self.assertEqual(func_list[0]['var_names'], ['ya']) + self.assertEqual(func_list[0]['raw_var_types'], ['x']) + self.assertEqual(func_list[0]['simple_var_types'], ['x']) + self.assertEqual(func_list[0]['definition_line'], 7) + self.assertEqual(func_list[0]['declaration_line'], None) + + self.assertEqual(func_list[1]['func_name'], 'main') + self.assertEqual(func_list[1]['raw_return_type'], 'int') + self.assertEqual(func_list[1]['simple_return_type'], 'int') + self.assertEqual(func_list[1]['var_names'], []) + self.assertEqual(func_list[1]['raw_var_types'], []) + self.assertEqual(func_list[1]['simple_var_types'], []) + self.assertEqual(func_list[1]['definition_line'], 11) + self.assertEqual(func_list[1]['declaration_line'], None) + + self.assertEqual(obj_type_list[0]['name'], 'x') + self.assertEqual(obj_type_list[0]['var_names'], ['a', 'b']) + self.assertEqual(obj_type_list[0]['raw_var_types'], ['int', 'int']) + self.assertEqual(obj_type_list[0]['simple_var_types'], ['int', 'int']) + self.assertEqual(obj_type_list[0]['definition_line'], 2) + + self.assertEqual(len(obj_type_list[0]['constructor_list']), 1) + self.assertEqual(obj_type_list[0]['constructor_list'][0]['arg_names'], ['other']) + self.assertEqual(obj_type_list[0]['constructor_list'][0]['raw_arg_types'], ['x &&']) + self.assertEqual(obj_type_list[0]['constructor_list'][0]['simple_arg_types'], ['x']) + self.assertEqual(obj_type_list[0]['constructor_list'][0]['definition_line'], 4) + + self.assertEqual(len(include_list), 1) def test_class_unsupported(self): @@ -359,14 +471,18 @@ class FuncTest(unittest.TestCase): def onUnsupportedFunc(func, filename, line): nonlocal was_called was_called = True - self.assertTrue([func, filename, line] in [['operator+', None, 11], ['main', None, 18]]) + self.assertTrue([func, filename, line] in [['Point', None, 1], ['operator+', None, 11], ['main', None, 18]]) # ---------------------------------------------------------------------- func_list = self._GetFuncList(CppToJson.ObtainFunctions(s, onUnsupportedFunc, lambda type: False)) + obj_type_list = self._GetObjList(CppToJson.ObtainFunctions(s, onUnsupportedFunc, lambda type: False)) + include_list = self._GetIncludeList(CppToJson.ObtainFunctions(s, onUnsupportedFunc, lambda type: False)) self.assertEqual(was_called, True) self.assertEqual(func_list, []) + self.assertEqual(obj_type_list, []) + self.assertEqual(include_list, []) def test_declaration(self): s = textwrap.dedent('''\ @@ -382,6 +498,9 @@ class FuncTest(unittest.TestCase): func_list = self._GetFuncList(CppToJson.ObtainFunctions(s, None, lambda type: True)) + obj_type_list = self._GetObjList(CppToJson.ObtainFunctions(s, None, lambda type: True)) + include_list = self._GetIncludeList(CppToJson.ObtainFunctions(s, None, lambda type: True)) + self.assertEqual(func_list[0]['func_name'], 'add') self.assertEqual(func_list[0]['raw_return_type'], 'int') self.assertEqual(func_list[0]['simple_return_type'], 'int') @@ -400,6 +519,9 @@ class FuncTest(unittest.TestCase): self.assertEqual(func_list[1]['definition_line'], 6) self.assertEqual(func_list[1]['declaration_line'], None) + self.assertEqual(obj_type_list, []) + self.assertEqual(include_list, []) + def test_namespace_declaration(self): s = textwrap.dedent('''\ namespace DataPipelines { @@ -419,6 +541,9 @@ class FuncTest(unittest.TestCase): func_list = self._GetFuncList(CppToJson.ObtainFunctions(s, None, lambda type: True)) + obj_type_list = self._GetObjList(CppToJson.ObtainFunctions(s, None, lambda type: True)) + include_list = self._GetIncludeList(CppToJson.ObtainFunctions(s, None, lambda type: True)) + self.assertEqual(func_list[0]['func_name'], 'DataPipelines::Arithmetic::thisGuy') self.assertEqual(func_list[0]['raw_return_type'], 'void') self.assertEqual(func_list[0]['simple_return_type'], 'void') @@ -437,6 +562,9 @@ class FuncTest(unittest.TestCase): self.assertEqual(func_list[1]['definition_line'], 11) self.assertEqual(func_list[1]['declaration_line'], None) + self.assertEqual(obj_type_list, []) + self.assertEqual(include_list, []) + def test_namespace_func(self): s = textwrap.dedent('''\ #include @@ -451,6 +579,9 @@ class FuncTest(unittest.TestCase): func_list = self._GetFuncList(CppToJson.ObtainFunctions(s, None, lambda type: True)) + obj_type_list = self._GetObjList(CppToJson.ObtainFunctions(s, None, lambda type: True)) + include_list = self._GetIncludeList(CppToJson.ObtainFunctions(s, None, lambda type: True)) + self.assertEqual(func_list[0]['func_name'], 'DataPipelines::Arithmetic::Add') self.assertEqual(func_list[0]['raw_return_type'], 'int64_t') self.assertEqual(func_list[0]['simple_return_type'], 'int64_t') @@ -460,6 +591,9 @@ class FuncTest(unittest.TestCase): self.assertEqual(func_list[0]['definition_line'], 6) self.assertEqual(func_list[0]['declaration_line'], None) + self.assertEqual(obj_type_list, []) + self.assertEqual(len(include_list), 1) + def test_namespace_unsupported(self): was_called = False @@ -486,23 +620,35 @@ class FuncTest(unittest.TestCase): def onUnsupportedFunc(func, filename, line): nonlocal was_called was_called = True - self.assertEqual(func, "DataPipelines::Arithmetic::Add") - self.assertEqual(filename, None) - self.assertEqual(line, 11) + self.assertTrue([func, filename, line] in [["DataPipelines::Arithmetic::MyStruct", None, 6], ["DataPipelines::Arithmetic::Add", None, 11]]) # ---------------------------------------------------------------------- func_list = self._GetFuncList(CppToJson.ObtainFunctions(s, onUnsupportedFunc, lambda type: False)) + obj_type_list = self._GetObjList(CppToJson.ObtainFunctions(s, onUnsupportedFunc, lambda type: False)) + include_list = self._GetIncludeList(CppToJson.ObtainFunctions(s, onUnsupportedFunc, lambda type: False)) self.assertEqual(was_called, True) self.assertEqual(func_list, []) + self.assertEqual(obj_type_list, []) + self.assertEqual(len(include_list), 1) def _GetFuncList(self, results): self.assertEqual(len(results), 1) self.assertEqual(None, list(results.keys())[0]) - return results[None] + return results[None]['function_list'] + def _GetObjList(self, results): + self.assertEqual(len(results), 1) + self.assertEqual(None, list(results.keys())[0]) + return results[None]['object_type_list'] + + def _GetIncludeList(self, results): + self.assertEqual(len(results), 1) + self.assertEqual(None, list(results.keys())[0]) + + return results[None]['include_list'] if __name__ == '__main__': unittest.main()