Merged PR 4814: General Improvements

Fixed some naming conventions.
Added more UnitTests.
This commit is contained in:
Teo Magnino Chaban 2019-07-24 21:38:42 +00:00
Родитель 212b948d9e
Коммит 10294a6334
3 изменённых файлов: 474 добавлений и 26 удалений

Просмотреть файл

@ -1,7 +1,7 @@
import re
import textwrap
def Policy(var_type, verifyStruct):
def Policy(var_type, verify_struct_func):
# Basic Types that don't require recursion
int_types = ["std::int8_t", "std::int16_t", "std::int32_t", "std::int64_t","std::uint8_t", "std::uint16_t", "std::uint32_t", "std::uint64_t"]
number_types = ["float16", "float32", "float64", "complex64", "complex128", "bfloat16"]
@ -43,9 +43,9 @@ def Policy(var_type, verifyStruct):
pattern_words = re.compile(
textwrap.dedent(
r"""(?#
Keyword)[[\w:\[0-9\]]+(?#
Or closing brackets)|\)(?#
Or opening brackets)|\((?#
Keyword)[[\w:]+(?#
Or closing parenthesis)|\)(?#
Or opening parenthesis)|\((?#
)"""
)
)
@ -127,7 +127,7 @@ def Policy(var_type, verifyStruct):
return True, index + 1
# Check to see if this is a valid Struct.
if verifyStruct(type_list[index]):
if verify_struct_func(type_list[index]):
return True, index + 1
return False, index + 1

Просмотреть файл

@ -6,7 +6,6 @@ from collections import OrderedDict
import clang.cindex as cindex
import CommonEnvironment
import CommonEnvironment.FileSystem as fileSystem
import CommonEnvironment.CallOnExit as callOnExit
from CommonEnvironment.Shell.All import CurrentShell
@ -20,7 +19,7 @@ from DataPipelines.CppToJson.Impl.Function import Function
def ObtainFunctions(
input_filename,
on_unsupported,
policy,
policy_func,
):
"""
This function will extract return value, name and parameters for every
@ -47,8 +46,8 @@ def ObtainFunctions(
os.remove(input_filename)
# ----------------------------------------------------------------------
def TestAndVerify(types, verifyStruct):
return policy(types, verifyStruct)
def TestAndVerify(types, verify_struct_func):
return policy_func(types, verify_struct_func)
# ----------------------------------------------------------------------
@ -111,30 +110,47 @@ def ObtainFunctions(
cursor = translation_unit.cursor
# ----------------------------------------------------------------------
def GetAlias():
# ----------------------------------------------------------------------
def GetAlias(node, prefix):
"""
This function will process all 'typedef' and 'using' and it will map the underlying type to
its definition.
"""
alias = {}
for child in cursor.get_children():
if (child.kind == cindex.CursorKind.TYPEDEF_DECL or child.kind == cindex.CursorKind.TYPE_ALIAS_DECL) and child.location.file.name == input_filename:
alias[child.spelling] = child.underlying_typedef_type.spelling
if node.kind == cindex.CursorKind.NAMESPACE:
for child in node.get_children():
ret = GetAlias(child, prefix + node.spelling + "::")
for a, b in ret.items():
assert prefix + a not in alias.keys()
alias[prefix + a] = b
if (node.kind == cindex.CursorKind.TYPEDEF_DECL or node.kind == cindex.CursorKind.TYPE_ALIAS_DECL):
alias[prefix + node.spelling] = node.underlying_typedef_type.spelling
return alias
# ----------------------------------------------------------------------
alias = GetAlias()
alias = {}
for node in cursor.get_children():
for before_type, after_type in GetAlias(node, "").items():
assert before_type not in alias.keys()
'''
If TestAndVerify with no structs (lambda function is always False) is True, then it means
that before_type is already an accepted type, so we don't want to extend it any further.
'''
if not TestAndVerify(before_type, lambda type: False):
alias[before_type] = after_type
alias_regex = re.compile(
textwrap.dedent(
r"""(?#
Not a letter)(?<!\w)(?#
Not a letter or a ':')(?<![:\w])(?#
Keyword)(?P<keyword>{})(?#
Not a letter)(?!\w)(?#
Not a letter or a ':')(?![:\w])(?#
)"""
).format("|".join([re.escape(key) for key in alias.keys()]))
).format("|".join([re.escape(key) for key in alias]))
)
struct_pattern = re.compile(
@ -414,7 +430,7 @@ def _FullName(node):
# ----------------------------------------------------------------------
def _GetStruct(node, SimpleVarType, FullVarType):
def _GetStruct(node, simple_var_type_func, full_var_type_func):
"""
This function will return the Object Type that this node refers to. It will return None if there were
errors.
@ -447,8 +463,8 @@ def _GetStruct(node, SimpleVarType, FullVarType):
struct_vars = []
for child in node.get_children():
if child.kind == cindex.CursorKind.FIELD_DECL:
var_type = FullVarType(child.type.spelling)
struct_vars.append((child.spelling, var_type, SimpleVarType(var_type)))
var_type = full_var_type_func(child.type.spelling)
struct_vars.append((child.spelling, var_type, simple_var_type_func(var_type)))
this_struct = Struct(_FullName(node), node.location.line, os.path.realpath(node.location.file.name), struct_vars)
@ -462,8 +478,8 @@ def _GetStruct(node, SimpleVarType, FullVarType):
constructor_args = []
for arg in child.get_arguments():
arg_type = FullVarType(arg.type.spelling)
constructor_args.append((arg.spelling, arg_type, SimpleVarType(arg_type)))
arg_type = full_var_type_func(arg.type.spelling)
constructor_args.append((arg.spelling, arg_type, simple_var_type_func(arg_type)))
constructor = Constructor(child.location.line, constructor_args)
@ -490,11 +506,11 @@ def _GetStruct(node, SimpleVarType, FullVarType):
continue
# 'operator=' is supported as long as it is public and a move operator.
move_operator_arg_type = FullVarType(node.spelling) + " &&"
move_operator_arg_type = full_var_type_func(node.spelling) + " &&"
if child.spelling == "operator=" and child.access_specifier == cindex.AccessSpecifier.PUBLIC:
for arg in child.get_arguments():
# Check the arguments to verify that this is a move operator.
if FullVarType(arg.type.spelling) != move_operator_arg_type:
if full_var_type_func(arg.type.spelling) != move_operator_arg_type:
this_struct.has_other = True
else:
# No other functions besides move operators are allowed.

Просмотреть файл

@ -1336,7 +1336,439 @@ class FuncTest(unittest.TestCase):
self.assertEqual(func_list, [])
self.assertEqual(struct_list, [])
self.assertEqual(include_list, [])
def test_alias(self):
s = textwrap.dedent('''\
#include <cstdint>
using int_t = std::int32_t;
typedef std::int32_t i_t;
int_t add(i_t a, int_t b){
return a+b;
}
i_t main(){
return 0;
}
''')
# ----------------------------------------------------------------------
def Policy(var_type, verifyStruct):
if var_type == "std::int32_t" or verifyStruct(var_type):
return True
return False
# ----------------------------------------------------------------------
result = CppToJson.ObtainFunctions(s, None, Policy)
func_list = self._GetFuncList(result)
struct_list = self._GetStructList(result)
include_list = self._GetIncludeList(result)
self.assertEqual(func_list[0]['name'], 'add')
self.assertEqual(func_list[0]['raw_return_type'], 'std::int32_t')
self.assertEqual(func_list[0]['simple_return_type'], 'std::int32_t')
self.assertEqual(func_list[0]['var_names'], ['a', 'b'])
self.assertEqual(func_list[0]['raw_var_types'], ['std::int32_t', 'std::int32_t'])
self.assertEqual(func_list[0]['simple_var_types'], ['std::int32_t', 'std::int32_t'])
self.assertEqual(func_list[0]['definition_line'], 5)
self.assertEqual(func_list[0]['declaration_line'], 5)
self.assertEqual(func_list[1]['name'], 'main')
self.assertEqual(func_list[1]['raw_return_type'], 'std::int32_t')
self.assertEqual(func_list[1]['simple_return_type'], 'std::int32_t')
self.assertEqual(func_list[1]['var_names'], [])
self.assertEqual(func_list[1]['raw_var_types'], [])
self.assertEqual(func_list[1]['simple_var_types'], [])
self.assertEqual(func_list[1]['definition_line'], 8)
self.assertEqual(func_list[1]['declaration_line'], 8)
self.assertEqual(struct_list, [])
self.assertEqual(len(include_list), 1)
def test_alias_namespace(self):
s = textwrap.dedent('''\
#include <cstdint>
#include <utility>
namespace DataPipelines {
namespace Arithmetic {
struct R{
std::int32_t a;
R(){}
R(R && other): a(std::move(other.a)){}
};
R thisGuy(std::int32_t x);
}
}
using Rx = DataPipelines::Arithmetic::R;
Rx DataPipelines::Arithmetic::thisGuy(std::int32_t x){
Rx y;
return y;
}
std::int32_t main(){
return 0;
}
''')
# ----------------------------------------------------------------------
def Policy(var_type, verifyStruct):
if var_type == "std::int32_t" or verifyStruct(var_type):
return True
return False
# ----------------------------------------------------------------------
result = CppToJson.ObtainFunctions(s, None, Policy)
func_list = self._GetFuncList(result)
struct_list = self._GetStructList(result)
include_list = self._GetIncludeList(result)
self.assertEqual(func_list[0]['name'], 'DataPipelines::Arithmetic::thisGuy')
self.assertEqual(func_list[0]['raw_return_type'], 'DataPipelines::Arithmetic::R')
self.assertEqual(func_list[0]['simple_return_type'], 'DataPipelines::Arithmetic::R')
self.assertEqual(func_list[0]['var_names'], ['x'])
self.assertEqual(func_list[0]['raw_var_types'], ['std::int32_t'])
self.assertEqual(func_list[0]['simple_var_types'], ['std::int32_t'])
self.assertEqual(func_list[0]['definition_line'], 16)
self.assertEqual(func_list[0]['declaration_line'], 11)
self.assertEqual(func_list[1]['name'], 'main')
self.assertEqual(func_list[1]['raw_return_type'], 'std::int32_t')
self.assertEqual(func_list[1]['simple_return_type'], 'std::int32_t')
self.assertEqual(func_list[1]['var_names'], [])
self.assertEqual(func_list[1]['raw_var_types'], [])
self.assertEqual(func_list[1]['simple_var_types'], [])
self.assertEqual(func_list[1]['definition_line'], 21)
self.assertEqual(func_list[1]['declaration_line'], 21)
self.assertEqual(struct_list[0]['name'], 'DataPipelines::Arithmetic::R')
self.assertEqual(struct_list[0]['var_names'], ['a'])
self.assertEqual(struct_list[0]['raw_var_types'], ['std::int32_t'])
self.assertEqual(struct_list[0]['simple_var_types'], ['std::int32_t'])
self.assertEqual(struct_list[0]['definition_line'], 6)
self.assertEqual(struct_list[0]['base_structs'], [])
self.assertEqual(len(struct_list[0]['constructor_list']), 2)
self.assertEqual(struct_list[0]['constructor_list'][0]['var_names'], [])
self.assertEqual(struct_list[0]['constructor_list'][0]['raw_var_types'], [])
self.assertEqual(struct_list[0]['constructor_list'][0]['simple_var_types'], [])
self.assertEqual(struct_list[0]['constructor_list'][0]['definition_line'], 8)
self.assertEqual(struct_list[0]['constructor_list'][1]['var_names'], ['other'])
self.assertEqual(struct_list[0]['constructor_list'][1]['raw_var_types'], ['DataPipelines::Arithmetic::R &&'])
self.assertEqual(struct_list[0]['constructor_list'][1]['simple_var_types'], ['DataPipelines::Arithmetic::R'])
self.assertEqual(struct_list[0]['constructor_list'][1]['definition_line'], 9)
self.assertEqual(len(include_list), 2)
def test_copy_constructor(self):
times_called = 0
s = textwrap.dedent( '''\
#include <utility>
#include <cstdio>
#include <cstdint>
struct x{
std::int32_t a, b;
x(){}
x(x &&other): a(std::move(other.a)), b(std::move(other.b)){}
// Copy constructor
x(x& other){}
};
x go(std::int32_t y){
x ret = x();
return ret;
}
std::int32_t main(){
return 0;
}
struct y{
std::int32_t a, b;
y(y &&other): a(std::move(other.a)), b(std::move(other.b)){}
};
''')
# ----------------------------------------------------------------------
def onUnsupportedFunc(error_desc, filename, line):
nonlocal times_called
times_called = times_called + 1
unsupported_list = [
[textwrap.dedent("""\
The struct x is not supported:
\t- Struct has a copy constructor.
"""), None, 5],
[textwrap.dedent("""\
The function go is not supported:
\t- Invalid return type x.
"""), None, 13]
]
self.assertTrue([error_desc, filename, line] in unsupported_list)
# ----------------------------------------------------------------------
def Policy(var_type, verifyStruct):
if var_type == "std::int32_t" or verifyStruct(var_type):
return True
return False
# ----------------------------------------------------------------------
result = CppToJson.ObtainFunctions(s, onUnsupportedFunc, Policy)
func_list = self._GetFuncList(result)
struct_list = self._GetStructList(result)
include_list = self._GetIncludeList(result)
self.assertEqual(func_list[0]['name'], 'main')
self.assertEqual(func_list[0]['raw_return_type'], 'std::int32_t')
self.assertEqual(func_list[0]['simple_return_type'], 'std::int32_t')
self.assertEqual(func_list[0]['var_names'], [])
self.assertEqual(func_list[0]['raw_var_types'], [])
self.assertEqual(func_list[0]['simple_var_types'], [])
self.assertEqual(func_list[0]['definition_line'], 18)
self.assertEqual(func_list[0]['declaration_line'], 18)
self.assertEqual(len(include_list), 3)
self.assertEqual(times_called, 2)
def test_copy_constructor_deleted(self):
s = textwrap.dedent( '''\
#include <utility>
#include <cstdio>
#include <cstdint>
struct x{
std::int32_t a, b;
x(){}
x(x &&other): a(std::move(other.a)), b(std::move(other.b)){}
// Copy constructor
x(x& other)=delete;
};
x go(std::int32_t y){
x ret = x();
return ret;
}
std::int32_t main(){
return 0;
}
struct y{
std::int32_t a, b;
y(y &&other): a(std::move(other.a)), b(std::move(other.b)){}
};
''')
# ----------------------------------------------------------------------
def Policy(var_type, verifyStruct):
if var_type == "std::int32_t" or verifyStruct(var_type):
return True
return False
# ----------------------------------------------------------------------
result = CppToJson.ObtainFunctions(s, None, Policy)
func_list = self._GetFuncList(result)
struct_list = self._GetStructList(result)
include_list = self._GetIncludeList(result)
self.assertEqual(func_list[0]['name'], 'go')
self.assertEqual(func_list[0]['raw_return_type'], 'x')
self.assertEqual(func_list[0]['simple_return_type'], 'x')
self.assertEqual(func_list[0]['var_names'], ['y'])
self.assertEqual(func_list[0]['raw_var_types'], ['std::int32_t'])
self.assertEqual(func_list[0]['simple_var_types'], ['std::int32_t'])
self.assertEqual(func_list[0]['definition_line'], 13)
self.assertEqual(func_list[0]['declaration_line'], 13)
self.assertEqual(func_list[1]['name'], 'main')
self.assertEqual(func_list[1]['raw_return_type'], 'std::int32_t')
self.assertEqual(func_list[1]['simple_return_type'], 'std::int32_t')
self.assertEqual(func_list[1]['var_names'], [])
self.assertEqual(func_list[1]['raw_var_types'], [])
self.assertEqual(func_list[1]['simple_var_types'], [])
self.assertEqual(func_list[1]['definition_line'], 18)
self.assertEqual(func_list[1]['declaration_line'], 18)
self.assertEqual(struct_list[0]['name'], 'x')
self.assertEqual(struct_list[0]['var_names'], ['a', 'b'])
self.assertEqual(struct_list[0]['raw_var_types'], ['std::int32_t', 'std::int32_t'])
self.assertEqual(struct_list[0]['simple_var_types'], ['std::int32_t', 'std::int32_t'])
self.assertEqual(struct_list[0]['definition_line'], 5)
self.assertEqual(len(struct_list[0]['constructor_list']), 2)
self.assertEqual(struct_list[0]['constructor_list'][0]['var_names'], [])
self.assertEqual(struct_list[0]['constructor_list'][0]['raw_var_types'], [])
self.assertEqual(struct_list[0]['constructor_list'][0]['simple_var_types'], [])
self.assertEqual(struct_list[0]['constructor_list'][0]['definition_line'], 7)
self.assertEqual(struct_list[0]['constructor_list'][1]['var_names'], ['other'])
self.assertEqual(struct_list[0]['constructor_list'][1]['raw_var_types'], ['x &&'])
self.assertEqual(struct_list[0]['constructor_list'][1]['simple_var_types'], ['x'])
self.assertEqual(struct_list[0]['constructor_list'][1]['definition_line'], 8)
self.assertEqual(len(include_list), 3)
def test_operator_equal(self):
s = textwrap.dedent( '''\
#include <utility>
#include <cstdio>
#include <cstdint>
struct x{
std::int32_t a, b;
x(){}
x(x &&other): a(std::move(other.a)), b(std::move(other.b)){}
x& operator=(x&& y){
return *this;
}
};
x go(std::int32_t y){
x ret = x();
return ret;
}
std::int32_t main(){
return 0;
}
struct y{
std::int32_t a, b;
y(y &&other): a(std::move(other.a)), b(std::move(other.b)){}
};
''')
# ----------------------------------------------------------------------
def Policy(var_type, verifyStruct):
if var_type == "std::int32_t" or verifyStruct(var_type):
return True
return False
# ----------------------------------------------------------------------
result = CppToJson.ObtainFunctions(s, None, Policy)
func_list = self._GetFuncList(result)
struct_list = self._GetStructList(result)
include_list = self._GetIncludeList(result)
self.assertEqual(func_list[0]['name'], 'go')
self.assertEqual(func_list[0]['raw_return_type'], 'x')
self.assertEqual(func_list[0]['simple_return_type'], 'x')
self.assertEqual(func_list[0]['var_names'], ['y'])
self.assertEqual(func_list[0]['raw_var_types'], ['std::int32_t'])
self.assertEqual(func_list[0]['simple_var_types'], ['std::int32_t'])
self.assertEqual(func_list[0]['definition_line'], 14)
self.assertEqual(func_list[0]['declaration_line'], 14)
self.assertEqual(func_list[1]['name'], 'main')
self.assertEqual(func_list[1]['raw_return_type'], 'std::int32_t')
self.assertEqual(func_list[1]['simple_return_type'], 'std::int32_t')
self.assertEqual(func_list[1]['var_names'], [])
self.assertEqual(func_list[1]['raw_var_types'], [])
self.assertEqual(func_list[1]['simple_var_types'], [])
self.assertEqual(func_list[1]['definition_line'], 19)
self.assertEqual(func_list[1]['declaration_line'], 19)
self.assertEqual(struct_list[0]['name'], 'x')
self.assertEqual(struct_list[0]['var_names'], ['a', 'b'])
self.assertEqual(struct_list[0]['raw_var_types'], ['std::int32_t', 'std::int32_t'])
self.assertEqual(struct_list[0]['simple_var_types'], ['std::int32_t', 'std::int32_t'])
self.assertEqual(struct_list[0]['definition_line'], 5)
self.assertEqual(len(struct_list[0]['constructor_list']), 2)
self.assertEqual(struct_list[0]['constructor_list'][0]['var_names'], [])
self.assertEqual(struct_list[0]['constructor_list'][0]['raw_var_types'], [])
self.assertEqual(struct_list[0]['constructor_list'][0]['simple_var_types'], [])
self.assertEqual(struct_list[0]['constructor_list'][0]['definition_line'], 7)
self.assertEqual(struct_list[0]['constructor_list'][1]['var_names'], ['other'])
self.assertEqual(struct_list[0]['constructor_list'][1]['raw_var_types'], ['x &&'])
self.assertEqual(struct_list[0]['constructor_list'][1]['simple_var_types'], ['x'])
self.assertEqual(struct_list[0]['constructor_list'][1]['definition_line'], 8)
self.assertEqual(len(include_list), 3)
def test_namespace_alias(self):
s = textwrap.dedent('''\
#include <stdint.h>
#include <cstdint>
/* Defined in enclosed namespace */
namespace Foo{
using Bar = std::int32_t;
Bar func1(){
return 2;
}
}
namespace Baz{
using Bar = bool;
Bar func2(){
return false;
}
}
namespace Foo{
Bar func3(){
return 2;
}
}
''')
# ----------------------------------------------------------------------
def Policy(var_type, verifyStruct):
if var_type == "std::int32_t" or var_type == "bool" or verifyStruct(var_type):
return True
return False
# ----------------------------------------------------------------------
result = CppToJson.ObtainFunctions(s, None, Policy)
func_list = self._GetFuncList(result)
struct_list = self._GetStructList(result)
include_list = self._GetIncludeList(result)
self.assertEqual(func_list[0]['name'], 'Foo::func1')
self.assertEqual(func_list[0]['raw_return_type'], 'std::int32_t')
self.assertEqual(func_list[0]['simple_return_type'], 'std::int32_t')
self.assertEqual(func_list[0]['var_names'], [])
self.assertEqual(func_list[0]['raw_var_types'], [])
self.assertEqual(func_list[0]['simple_var_types'], [])
self.assertEqual(func_list[0]['definition_line'], 7)
self.assertEqual(func_list[0]['declaration_line'], 7)
self.assertEqual(func_list[1]['name'], 'Baz::func2')
self.assertEqual(func_list[1]['raw_return_type'], 'bool')
self.assertEqual(func_list[1]['simple_return_type'], 'bool')
self.assertEqual(func_list[1]['var_names'], [])
self.assertEqual(func_list[1]['raw_var_types'], [])
self.assertEqual(func_list[1]['simple_var_types'], [])
self.assertEqual(func_list[1]['definition_line'], 13)
self.assertEqual(func_list[1]['declaration_line'], 13)
self.assertEqual(func_list[2]['name'], 'Foo::func3')
self.assertEqual(func_list[2]['raw_return_type'], 'std::int32_t')
self.assertEqual(func_list[2]['simple_return_type'], 'std::int32_t')
self.assertEqual(func_list[2]['var_names'], [])
self.assertEqual(func_list[2]['raw_var_types'], [])
self.assertEqual(func_list[2]['simple_var_types'], [])
self.assertEqual(func_list[2]['definition_line'], 19)
self.assertEqual(func_list[2]['declaration_line'], 19)
self.assertEqual(struct_list, [])
self.assertEqual(len(include_list), 2)
def _GetFuncList(self, results):