Add python3 support
- Conditionally compile bond::blob convert/extract routines based on Python version with PyString_Type or PyBytes_Type - Update Python unit tests to be compatible with Python3 - Generalize python/boost vars in Config.cmake to allow Python3 - Update python docs for blob
This commit is contained in:
Fred Park
Родитель
1c9e96f034
Коммит
73f98653f0
|
|
@ -19,8 +19,17 @@ if (WIN32)
|
|||
"${CMAKE_CURRENT_SOURCE_DIR}/cs/test/compat/bin/retail")
|
||||
endif()
|
||||
|
||||
find_package (PythonLibs 2.7)
|
||||
# find python interpreter, library and boost python library.
|
||||
# to specify a different version, invoke cmake with:
|
||||
# -DPYTHON_EXECUTABLE=/path/to/python
|
||||
# -DPYTHON_LIBRARY=/path/to/libpython.so
|
||||
# -DBoost_PYTHON_LIBRARY_RELEASE=/path/to/libboost-python.so
|
||||
# (or Boost_PYTHON_LIBRARY_DEBUG if CMAKE_BUILD_TYPE=Debug)
|
||||
# and optionally with:
|
||||
# -DPython_ADDITIONAL_VERSIONS=Major.Minor
|
||||
# if your python version is not implicitly supported by cmake
|
||||
find_package (PythonInterp 2.7)
|
||||
find_package (PythonLibs 2.7)
|
||||
|
||||
find_package (Boost 1.53.0
|
||||
OPTIONAL_COMPONENTS
|
||||
|
|
@ -30,6 +39,8 @@ find_package (Boost 1.53.0
|
|||
unit_test_framework
|
||||
python)
|
||||
|
||||
message(STATUS "Boost Python Library: ${Boost_PYTHON_LIBRARY}")
|
||||
|
||||
# disable Boost auto-linking
|
||||
add_definitions (-DBOOST_ALL_NO_LIB)
|
||||
|
||||
|
|
|
|||
|
|
@ -157,9 +157,10 @@ instead objects with two methods: `key()` and `data()`.
|
|||
Blob
|
||||
----
|
||||
|
||||
Bond `blob` is represented as Python string object. Initializing a `blob` from
|
||||
a Python string does not involve memory copy, it just increases reference count
|
||||
on the underlying Python object.
|
||||
Bond `blob` is represented as either a string object in Python2 or a bytes
|
||||
object in Python3. Initializing a `blob` from a Python object does not involve
|
||||
a memory copy; instead, the reference count on the underlying Python object is
|
||||
increased.
|
||||
|
||||
Nullable and `nothing`
|
||||
----------------------
|
||||
|
|
|
|||
|
|
@ -1,3 +1,5 @@
|
|||
from __future__ import print_function
|
||||
import hashlib
|
||||
import python_extension as extension
|
||||
|
||||
def person(first, last, eyes):
|
||||
|
|
@ -6,6 +8,9 @@ def person(first, last, eyes):
|
|||
obj.first_name = first
|
||||
obj.last_name = last
|
||||
obj.eyes = eyes
|
||||
m = hashlib.md5()
|
||||
m.update('{}::{}::{}'.format(first, last, eyes).encode('utf8'))
|
||||
obj.hash = m.digest()
|
||||
return obj
|
||||
|
||||
def main():
|
||||
|
|
@ -16,6 +21,9 @@ def main():
|
|||
obj.people = {i: person(first, last, eyes) for i, first, last, eyes in \
|
||||
[(1, "Carlos", "Danger", extension.Color.Red)]}
|
||||
|
||||
# Print JSON representation
|
||||
print(extension.Serialize(obj, extension.ProtocolType.SIMPLE_JSON_PROTOCOL))
|
||||
|
||||
# Serialize and deserialize using default protocol (Compact Binary)
|
||||
data = extension.Serialize(obj)
|
||||
new = extension.Example()
|
||||
|
|
@ -23,7 +31,7 @@ def main():
|
|||
assert(new == obj)
|
||||
|
||||
# Print JSON representation
|
||||
print extension.Serialize(obj, extension.ProtocolType.SIMPLE_JSON_PROTOCOL)
|
||||
print(extension.Serialize(new, extension.ProtocolType.SIMPLE_JSON_PROTOCOL))
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
|
|
|
|||
|
|
@ -15,6 +15,7 @@ struct Person
|
|||
0: string first_name;
|
||||
1: string last_name;
|
||||
2: Color eyes = Brown;
|
||||
3: blob hash;
|
||||
}
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -34,7 +34,7 @@ namespace python
|
|||
// Convert Bond type name to a valid python identifier
|
||||
class pythonic_name
|
||||
{
|
||||
public:
|
||||
public:
|
||||
template <typename T>
|
||||
pythonic_name()
|
||||
: _name(bond::detail::type<T>::name())
|
||||
|
|
@ -181,12 +181,19 @@ extend_map(Map& map, const boost::python::object& obj)
|
|||
typedef typename Map::value_type value_type;
|
||||
|
||||
BOOST_ASSERT(PyDict_Check(obj.ptr()));
|
||||
|
||||
|
||||
dict dict(obj);
|
||||
|
||||
|
||||
const auto keys =
|
||||
#if PY_VERSION_HEX >= 0x03000000
|
||||
dict.keys();
|
||||
#else
|
||||
dict.iterkeys();
|
||||
#endif
|
||||
|
||||
BOOST_FOREACH(object k,
|
||||
std::make_pair(
|
||||
stl_input_iterator<object>(dict.iterkeys()),
|
||||
stl_input_iterator<object>(keys),
|
||||
stl_input_iterator<object>()
|
||||
))
|
||||
{
|
||||
|
|
@ -211,7 +218,7 @@ extend_map(Map& map, const boost::python::object& obj)
|
|||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
PyErr_SetString(PyExc_TypeError, "Incompatible Data Type");
|
||||
throw_error_already_set();
|
||||
}
|
||||
|
|
@ -247,7 +254,7 @@ extend_set(T& set, const boost::python::object& obj)
|
|||
{
|
||||
using namespace boost::python;
|
||||
typedef typename T::value_type data_type;
|
||||
|
||||
|
||||
BOOST_FOREACH(object elem,
|
||||
std::make_pair(
|
||||
stl_input_iterator<object>(obj),
|
||||
|
|
@ -274,7 +281,7 @@ extend_set(T& set, const boost::python::object& obj)
|
|||
throw_error_already_set();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -303,26 +310,51 @@ struct rvalue_set_container_from_python
|
|||
|
||||
// Conversion policy for bond::blob
|
||||
struct blob_converter
|
||||
#if PY_VERSION_HEX >= 0x03000000
|
||||
: boost::python::converter::wrap_pytype<&PyBytes_Type>
|
||||
#else
|
||||
: boost::python::converter::wrap_pytype<&PyString_Type>
|
||||
#endif
|
||||
{
|
||||
// Conversion from Python string to bond::blob
|
||||
// Conversion from Python2 string or Python3 bytes to bond::blob
|
||||
static unaryfunc* convertible(PyObject* obj)
|
||||
{
|
||||
#if PY_VERSION_HEX >= 0x03000000
|
||||
return (PyBytes_Check(obj)) ? &py_object_identity : 0;
|
||||
#else
|
||||
return (PyString_Check(obj)) ? &obj->ob_type->tp_str : 0;
|
||||
#endif
|
||||
}
|
||||
|
||||
static void extract(bond::blob& dst, const boost::python::object& src)
|
||||
{
|
||||
#if PY_VERSION_HEX >= 0x03000000
|
||||
boost::shared_ptr<void> hold_convertible_ref_count(
|
||||
(void*)0,
|
||||
boost::python::converter::shared_ptr_deleter(
|
||||
boost::python::handle<>(boost::python::borrowed(src.ptr()))));
|
||||
boost::shared_ptr<char[]> bytes(
|
||||
hold_convertible_ref_count,
|
||||
PyBytes_AsString(src.ptr()));
|
||||
dst.assign(
|
||||
boost::python::extract<boost::shared_ptr<char> >(src)(),
|
||||
bytes,
|
||||
static_cast<uint32_t>(PyBytes_Size(src.ptr())));
|
||||
#else
|
||||
dst.assign(
|
||||
boost::python::extract<boost::shared_ptr<char> >(src)(),
|
||||
static_cast<uint32_t>(PyString_Size(src.ptr())));
|
||||
#endif
|
||||
}
|
||||
|
||||
// Conversion from bond::blob to Python string
|
||||
// Conversion from bond::blob to Python2 string or Python3 bytes
|
||||
static PyObject* convert(const bond::blob& blob)
|
||||
{
|
||||
return boost::python::incref(
|
||||
#if PY_VERSION_HEX >= 0x03000000
|
||||
PyBytes_FromStringAndSize(blob.content(), blob.length()));
|
||||
#else
|
||||
boost::python::str(blob.content(), blob.length()).ptr());
|
||||
#endif
|
||||
}
|
||||
};
|
||||
|
||||
|
|
@ -338,13 +370,13 @@ struct nullable_maybe_converter
|
|||
{
|
||||
if (obj == Py_None)
|
||||
return &py_object_identity;
|
||||
|
||||
|
||||
const auto* reg = boost::python::converter::registry::query
|
||||
(typeid(typename T::value_type));
|
||||
|
||||
if (reg && reg->m_class_object == Py_TYPE(obj))
|
||||
return &py_object_identity;
|
||||
|
||||
|
||||
if (reg && reg->rvalue_chain)
|
||||
return static_cast<unaryfunc*>(reg->rvalue_chain->convertible(obj));
|
||||
|
||||
|
|
@ -363,7 +395,7 @@ struct nullable_maybe_converter
|
|||
else
|
||||
{
|
||||
boost::python::extract<typename T::value_type> value(src);
|
||||
|
||||
|
||||
if (value.check())
|
||||
{
|
||||
dst = T(static_cast<typename T::value_type>(value));
|
||||
|
|
@ -396,7 +428,7 @@ void register_builtin_convereters()
|
|||
using namespace boost::python;
|
||||
|
||||
rvalue_from_python<
|
||||
bond::blob,
|
||||
bond::blob,
|
||||
blob_converter
|
||||
>();
|
||||
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
from __future__ import print_function
|
||||
import bond_python_compatibility_test as test
|
||||
import sys, getopt
|
||||
|
||||
|
|
@ -20,13 +21,13 @@ def compat(input_, output, test_case):
|
|||
output.write(test.Serialize(obj, protocol))
|
||||
|
||||
def usage():
|
||||
print ''
|
||||
print 'compat.py -d INPUT_FILE -s OUTPUT_FILE TEST'
|
||||
print ''
|
||||
print ' -? --help show this help text'
|
||||
print ' TEST compact | json | schema'
|
||||
print ' -d --deserialize=INPUT_FILE deserialize object from specified file'
|
||||
print ' -s --serialize=OUTPUT_FILE serialize object to specified file'
|
||||
print('')
|
||||
print('compat.py -d INPUT_FILE -s OUTPUT_FILE TEST')
|
||||
print('')
|
||||
print(' -? --help show this help text')
|
||||
print(' TEST compact | json | schema')
|
||||
print(' -d --deserialize=INPUT_FILE deserialize object from specified file')
|
||||
print(' -s --serialize=OUTPUT_FILE serialize object to specified file')
|
||||
|
||||
def main(argv):
|
||||
inputfile = ''
|
||||
|
|
|
|||
|
|
@ -88,7 +88,8 @@ struct Nullable
|
|||
3: nullable<SimpleStruct> nullable_struct;
|
||||
4: nullable<map<int8, int8>> nullable_map;
|
||||
5: nullable<string> nullable_string;
|
||||
10: nullable<nullable<uint32>> nullable_nullable_uint32;
|
||||
6: nullable<blob> nullable_blob;
|
||||
10: nullable<nullable<uint32>> nullable_nullable_uint32;
|
||||
};
|
||||
|
||||
struct Nothing
|
||||
|
|
|
|||
|
|
@ -2,12 +2,22 @@ import unittest
|
|||
import random
|
||||
import string
|
||||
import functools
|
||||
import sys
|
||||
from bond_python_unit_test import Serialize, Deserialize, Marshal, Unmarshal, GetRuntimeSchema
|
||||
import bond_python_unit_test as test
|
||||
|
||||
def atleast_python3():
|
||||
return sys.version_info[0] >= 3
|
||||
|
||||
def random_string():
|
||||
return ''.join(random.sample(string.ascii_lowercase*16, 16))
|
||||
|
||||
def random_blob():
|
||||
if atleast_python3():
|
||||
return bytes(random_string(), 'ascii')
|
||||
else:
|
||||
return random_string()
|
||||
|
||||
def random_uint(bits):
|
||||
return random.randint(0, (1 << bits) - 1)
|
||||
|
||||
|
|
@ -52,10 +62,10 @@ def marshal_unmarshal(obj):
|
|||
new_obj = obj_type()
|
||||
|
||||
Unmarshal(data, new_obj)
|
||||
|
||||
|
||||
data2 = Marshal(new_obj)
|
||||
new_obj2 = obj_type()
|
||||
|
||||
|
||||
Unmarshal(data2, new_obj2, GetRuntimeSchema(obj))
|
||||
return new_obj2
|
||||
|
||||
|
|
@ -81,7 +91,7 @@ class BondTest(unittest.TestCase):
|
|||
test.EnumType1.EnumValue3, \
|
||||
test.EnumType1.EnumValue4, \
|
||||
test.EnumType1.EnumValue5])
|
||||
obj.m_blob = random_string();
|
||||
obj.m_blob = random_blob()
|
||||
|
||||
def randomSimpleStruct(self):
|
||||
obj = test.SimpleStruct()
|
||||
|
|
@ -93,7 +103,7 @@ class BondTest(unittest.TestCase):
|
|||
# initSimpleStruct will set values of derived, not of SimpleStruct.
|
||||
# Below we set those fields for the base. Ideally it should be the
|
||||
# other way around, initSimpleStruct should set fields of base and here
|
||||
# we would set the derived overrides. Need to figure out hot to achieve
|
||||
# we would set the derived overrides. Need to figure out how to achieve
|
||||
# this in Python...
|
||||
self.initSimpleStruct(obj)
|
||||
test.SimpleStruct.m_int32.__set__(obj, random_int(32))
|
||||
|
|
@ -131,11 +141,13 @@ class BondTest(unittest.TestCase):
|
|||
obj.nullable_struct = test.SimpleStruct()
|
||||
obj.nullable_map = random_map(random_int8, random_int8)
|
||||
obj.nullable_string = random_string()
|
||||
obj.nullable_blob = random_blob()
|
||||
obj.nullable_nullable_uint32 = random_uint(32)
|
||||
self.assertNotEqual(None, obj.nullable_list)
|
||||
self.assertNotEqual(None, obj.nullable_struct)
|
||||
self.assertNotEqual(None, obj.nullable_map)
|
||||
self.assertNotEqual(None, obj.nullable_string)
|
||||
self.assertNotEqual(None, obj.nullable_blob)
|
||||
self.assertNotEqual(None, obj.nullable_nullable_uint32)
|
||||
|
||||
def initNestedContainers(self, obj):
|
||||
|
|
@ -164,7 +176,7 @@ class BondTest(unittest.TestCase):
|
|||
obj_type = type(obj)
|
||||
for i in range(0, 50):
|
||||
init(obj)
|
||||
new_obj = obj_type()
|
||||
new_obj = obj_type()
|
||||
self.assertFalse(obj == new_obj)
|
||||
new_obj = serialize_deserialize(obj)
|
||||
self.assertTrue(obj == new_obj)
|
||||
|
|
@ -173,7 +185,7 @@ class BondTest(unittest.TestCase):
|
|||
obj_type = type(obj)
|
||||
for i in range(0, 50):
|
||||
init(obj)
|
||||
new_obj = obj_type()
|
||||
new_obj = obj_type()
|
||||
self.assertFalse(obj == new_obj)
|
||||
new_obj = marshal_unmarshal(obj)
|
||||
self.assertTrue(obj == new_obj)
|
||||
|
|
@ -186,7 +198,7 @@ class BondTest(unittest.TestCase):
|
|||
del a[-1]
|
||||
self.assertTrue(len(a)==len(b) and all(a[i] == b[i] for i in range(0, len(a))))
|
||||
a.extend(b)
|
||||
self.assertTrue(all(a[i] == a[i+len(a)/2] for i in range(0, len(a)/2)))
|
||||
self.assertTrue(all(a[i] == a[i+len(a)//2] for i in range(0, len(a)//2)))
|
||||
s1 = set(a)
|
||||
s2 = set(b)
|
||||
self.assertTrue(s1 - s2 == set())
|
||||
|
|
@ -196,7 +208,7 @@ class BondTest(unittest.TestCase):
|
|||
a[:] = [b[0]]*len(a)
|
||||
self.assertEqual(len(a), len(b))
|
||||
self.assertTrue(all(a[i] == b[0] for i in range(0, len(a))))
|
||||
a[0:len(a)/2] = b[0]
|
||||
a[0:len(a)//2] = b[0]
|
||||
self.assertEqual(a[0], b[0])
|
||||
x = a[-1]
|
||||
del a[:-1]
|
||||
|
|
@ -265,6 +277,7 @@ class BondTest(unittest.TestCase):
|
|||
self.assertEqual(None, obj.nullable_struct)
|
||||
self.assertEqual(None, obj.nullable_map)
|
||||
self.assertEqual(None, obj.nullable_string)
|
||||
self.assertEqual(None, obj.nullable_blob)
|
||||
self.assertEqual(None, obj.nullable_nullable_uint32)
|
||||
self.serialization(obj, self.initNullable)
|
||||
self.marshaling(obj, self.initNullable)
|
||||
|
|
@ -276,6 +289,8 @@ class BondTest(unittest.TestCase):
|
|||
obj.nullable_map = 0
|
||||
with self.assertRaises(TypeError):
|
||||
obj.nullable_string = 1
|
||||
with self.assertRaises(TypeError):
|
||||
obj.nullable_blob = 0
|
||||
with self.assertRaises(TypeError):
|
||||
obj.nullable_nullable_uint32 = 3.14
|
||||
with self.assertRaises(OverflowError):
|
||||
|
|
@ -355,7 +370,7 @@ class BondTest(unittest.TestCase):
|
|||
obj3 = test.SimpleWithBase()
|
||||
bonded.Deserialize(obj3)
|
||||
self.assertTrue(obj3, src2.n2)
|
||||
|
||||
|
||||
def test_Polymorphism(self):
|
||||
src = test.Bonded()
|
||||
obj = self.randomSimpleWithBase()
|
||||
|
|
|
|||
Загрузка…
Ссылка в новой задаче