From 5a246f5cef7e6238f6372ba8746fce391e5d5b9d Mon Sep 17 00:00:00 2001 From: Taku Kudo Date: Tue, 19 Jun 2018 23:29:12 +0900 Subject: [PATCH] Avoids copy in Python2 Unicode mode. --- python/sentencepiece.i | 62 +- python/sentencepiece.py | 30 - python/sentencepiece_wrap.cxx | 1067 ++++++--------------------------- src/sentencepiece_processor.h | 30 +- 4 files changed, 234 insertions(+), 955 deletions(-) diff --git a/python/sentencepiece.i b/python/sentencepiece.i index 26afdad..23eaada 100644 --- a/python/sentencepiece.i +++ b/python/sentencepiece.i @@ -7,59 +7,63 @@ #include namespace { -PyObject* kStringInput = reinterpret_cast(0x1); -PyObject* kUnicodeInput = reinterpret_cast(0x2); +PyObject* kUnicodeInput = reinterpret_cast(0x1); + +inline void ReleaseResultObject(PyObject *obj) { + if (obj != nullptr && obj != kUnicodeInput) + Py_XDECREF(obj); +} class PyInputString { public: explicit PyInputString(PyObject* obj) { #if PY_VERSION_HEX >= 0x03000000 if (PyUnicode_Check(obj)) { + // Python3, Unicode str_ = const_cast(PyUnicode_AsUTF8AndSize(obj, &size_)); input_type_ = kUnicodeInput; } else if (PyBytes_Check(obj)) { + // Python3, Bytes PyBytes_AsStringAndSize(obj, &str_, &size_); - input_type_ = kStringInput; + input_type_ = nullptr; } #else if (PyUnicode_Check(obj)) { - utf8_obj_ = PyUnicode_AsUTF8String(obj); - PyString_AsStringAndSize(utf8_obj_, &str_, &size_); - input_type_ = kUnicodeInput; + // Python2, Unicode + PyObject *utf8_obj = PyUnicode_AsUTF8String(obj); + PyString_AsStringAndSize(utf8_obj, &str_, &size_); + input_type_ = utf8_obj; } else if (PyString_Check(obj)) { + // Python2, Bytes, PyString_AsStringAndSize(obj, &str_, &size_); - input_type_ = kStringInput; + input_type_ = nullptr; } #endif else { str_ = nullptr; } } - virtual ~PyInputString() { - Py_XDECREF(utf8_obj_); - } const char* data() const { return str_; } Py_ssize_t size() const { return size_; } bool IsAvalable() const { return str_ != nullptr; } - bool IsCopy() const { return utf8_obj_ != nullptr; } PyObject *input_type() const { return input_type_; } private: - PyObject* utf8_obj_ = nullptr; PyObject* input_type_ = nullptr; char* str_ = nullptr; Py_ssize_t size_ = 0; }; -PyObject* MakePyOutputString(const std::string& output, PyObject *resultobj) { +PyObject* MakePyOutputString(const std::string& output, + PyObject *resultobj) { #if PY_VERSION_HEX >= 0x03000000 - return resultobj == kStringInput ? + return resultobj != nullptr ? PyBytes_FromStringAndSize(output.data(), output.size()) : PyUnicode_FromStringAndSize(output.data(), output.size()); #else - return resultobj == kUnicodeInput ? - PyUnicode_FromStringAndSize(output.data(), output.size()) : - PyString_FromStringAndSize(output.data(), output.size()); + return resultobj == nullptr ? + PyString_FromStringAndSize(output.data(), output.size()) : + PyUnicode_FromStringAndSize(output.data(), output.size()); #endif } @@ -80,7 +84,10 @@ int ToSwigError(sentencepiece::util::error::Code code) { %} %exception { - try { $action } + try { + $action + ReleaseResultObject(resultobj); + } catch (const sentencepiece::util::Status &status) { SWIG_exception(ToSwigError(status.code()), status.ToString().c_str()); } @@ -88,6 +95,7 @@ int ToSwigError(sentencepiece::util::error::Code code) { %ignore sentencepiece::util::Status; %ignore sentencepiece::util::error::Code; +%ignore sentencepiece::util::min_string_view; %ignore sentencepiece::SentencePieceText; %ignore sentencepiece::NormalizerSpec; %ignore sentencepiece::TrainerSpec; @@ -277,7 +285,7 @@ int ToSwigError(sentencepiece::util::error::Code code) { const PyInputString ustring($input); if (!ustring.IsAvalable()) { PyErr_SetString(PyExc_TypeError, "not a string"); - return nullptr; + SWIG_fail; } resultobj = ustring.input_type(); $1 = new std::string(ustring.data(), ustring.size()); @@ -289,14 +297,10 @@ int ToSwigError(sentencepiece::util::error::Code code) { const PyInputString ustring($input); if (!ustring.IsAvalable()) { PyErr_SetString(PyExc_TypeError, "not a string"); - return nullptr; + SWIG_fail; } resultobj = ustring.input_type(); - if (ustring.IsCopy()) { - $1.copy(ustring.data(), ustring.size()); - } else { - $1.assign(ustring.data(), ustring.size()); - } + $1 = sentencepiece::util::min_string_view(ustring.data(), ustring.size()); } @@ -311,13 +315,13 @@ int ToSwigError(sentencepiece::util::error::Code code) { (*out)[i] = std::string(ustring.data(), ustring.size()); } else { PyErr_SetString(PyExc_TypeError, "list must contain strings"); - return nullptr; + SWIG_fail; } resultobj = ustring.input_type(); } } else { PyErr_SetString(PyExc_TypeError, "not a list"); - return nullptr; + SWIG_fail; } $1 = out; } @@ -333,12 +337,12 @@ int ToSwigError(sentencepiece::util::error::Code code) { (*out)[i] = static_cast(PyInt_AsLong(o)); } else { PyErr_SetString(PyExc_TypeError,"list must contain integers"); - return nullptr; + SWIG_fail; } } } else { PyErr_SetString(PyExc_TypeError,"not a list"); - return nullptr; + SWIG_fail; } $1 = out; } diff --git a/python/sentencepiece.py b/python/sentencepiece.py index 2c69c61..5b15e57 100644 --- a/python/sentencepiece.py +++ b/python/sentencepiece.py @@ -98,36 +98,6 @@ except __builtin__.Exception: pass _newclass = 0 -class min_string_view(_object): - __swig_setmethods__ = {} - __setattr__ = lambda self, name, value: _swig_setattr(self, min_string_view, name, value) - __swig_getmethods__ = {} - __getattr__ = lambda self, name: _swig_getattr(self, min_string_view, name) - __repr__ = _swig_repr - - def data(self): - return _sentencepiece.min_string_view_data(self) - - def size(self): - return _sentencepiece.min_string_view_size(self) - - def assign(self, data, len): - return _sentencepiece.min_string_view_assign(self, data, len) - - def copy(self, data, len): - return _sentencepiece.min_string_view_copy(self, data, len) - - def __init__(self, *args): - this = _sentencepiece.new_min_string_view(*args) - try: - self.this.append(this) - except __builtin__.Exception: - self.this = this - __swig_destroy__ = _sentencepiece.delete_min_string_view - __del__ = lambda self: None -min_string_view_swigregister = _sentencepiece.min_string_view_swigregister -min_string_view_swigregister(min_string_view) - class SentencePieceProcessor(_object): __swig_setmethods__ = {} __setattr__ = lambda self, name, value: _swig_setattr(self, SentencePieceProcessor, name, value) diff --git a/python/sentencepiece_wrap.cxx b/python/sentencepiece_wrap.cxx index 260c479..4997356 100644 --- a/python/sentencepiece_wrap.cxx +++ b/python/sentencepiece_wrap.cxx @@ -3010,12 +3010,10 @@ SWIG_Python_NonDynamicSetAttr(PyObject *obj, PyObject *name, PyObject *value) { #define SWIGTYPE_p_char swig_types[0] #define SWIGTYPE_p_sentencepiece__SentencePieceProcessor swig_types[1] #define SWIGTYPE_p_sentencepiece__SentencePieceTrainer swig_types[2] -#define SWIGTYPE_p_sentencepiece__util__min_string_view swig_types[3] -#define SWIGTYPE_p_std__string swig_types[4] -#define SWIGTYPE_p_std__vectorT_int_t swig_types[5] -#define SWIGTYPE_p_std__vectorT_std__string_t swig_types[6] -static swig_type_info *swig_types[8]; -static swig_module_info swig_module = {swig_types, 7, 0, 0, 0, 0}; +#define SWIGTYPE_p_std__vectorT_int_t swig_types[3] +#define SWIGTYPE_p_std__vectorT_std__string_t swig_types[4] +static swig_type_info *swig_types[6]; +static swig_module_info swig_module = {swig_types, 5, 0, 0, 0, 0}; #define SWIG_TypeQuery(name) SWIG_TypeQueryModule(&swig_module, &swig_module, name) #define SWIG_MangledTypeQuery(name) SWIG_MangledTypeQueryModule(&swig_module, &swig_module, name) @@ -3125,59 +3123,63 @@ namespace swig { #include namespace { -PyObject* kStringInput = reinterpret_cast(0x1); -PyObject* kUnicodeInput = reinterpret_cast(0x2); +PyObject* kUnicodeInput = reinterpret_cast(0x1); + +inline void ReleaseResultObject(PyObject *obj) { + if (obj != nullptr && obj != kUnicodeInput) + Py_XDECREF(obj); +} class PyInputString { public: explicit PyInputString(PyObject* obj) { #if PY_VERSION_HEX >= 0x03000000 if (PyUnicode_Check(obj)) { + // Python3, Unicode str_ = const_cast(PyUnicode_AsUTF8AndSize(obj, &size_)); input_type_ = kUnicodeInput; } else if (PyBytes_Check(obj)) { + // Python3, Bytes PyBytes_AsStringAndSize(obj, &str_, &size_); - input_type_ = kStringInput; + input_type_ = nullptr; } #else if (PyUnicode_Check(obj)) { - utf8_obj_ = PyUnicode_AsUTF8String(obj); - PyString_AsStringAndSize(utf8_obj_, &str_, &size_); - input_type_ = kUnicodeInput; + // Python2, Unicode + PyObject *utf8_obj = PyUnicode_AsUTF8String(obj); + PyString_AsStringAndSize(utf8_obj, &str_, &size_); + input_type_ = utf8_obj; } else if (PyString_Check(obj)) { + // Python2, Bytes, PyString_AsStringAndSize(obj, &str_, &size_); - input_type_ = kStringInput; + input_type_ = nullptr; } #endif else { str_ = nullptr; } } - virtual ~PyInputString() { - Py_XDECREF(utf8_obj_); - } const char* data() const { return str_; } Py_ssize_t size() const { return size_; } bool IsAvalable() const { return str_ != nullptr; } - bool IsCopy() const { return utf8_obj_ != nullptr; } PyObject *input_type() const { return input_type_; } private: - PyObject* utf8_obj_ = nullptr; PyObject* input_type_ = nullptr; char* str_ = nullptr; Py_ssize_t size_ = 0; }; -PyObject* MakePyOutputString(const std::string& output, PyObject *resultobj) { +PyObject* MakePyOutputString(const std::string& output, + PyObject *resultobj) { #if PY_VERSION_HEX >= 0x03000000 - return resultobj == kStringInput ? + return resultobj != nullptr ? PyBytes_FromStringAndSize(output.data(), output.size()) : PyUnicode_FromStringAndSize(output.data(), output.size()); #else - return resultobj == kUnicodeInput ? - PyUnicode_FromStringAndSize(output.data(), output.size()) : - PyString_FromStringAndSize(output.data(), output.size()); + return resultobj == nullptr ? + PyString_FromStringAndSize(output.data(), output.size()) : + PyUnicode_FromStringAndSize(output.data(), output.size()); #endif } @@ -3326,6 +3328,16 @@ SWIG_AsCharPtrAndSize(PyObject *obj, char** cptr, size_t* psize, int *alloc) +#include +#if !defined(SWIG_NO_LLONG_MAX) +# if !defined(LLONG_MAX) && defined(__GNUC__) && defined (__LONG_LONG_MAX__) +# define LLONG_MAX __LONG_LONG_MAX__ +# define LLONG_MIN (-LLONG_MAX - 1LL) +# define ULLONG_MAX (LLONG_MAX * 2ULL + 1ULL) +# endif +#endif + + SWIGINTERN int SWIG_AsVal_double (PyObject *obj, double *val) { @@ -3408,202 +3420,6 @@ SWIG_CanCastAsInteger(double *d, double min, double max) { } -SWIGINTERN int -SWIG_AsVal_unsigned_SS_long (PyObject *obj, unsigned long *val) -{ -#if PY_VERSION_HEX < 0x03000000 - if (PyInt_Check(obj)) { - long v = PyInt_AsLong(obj); - if (v >= 0) { - if (val) *val = v; - return SWIG_OK; - } else { - return SWIG_OverflowError; - } - } else -#endif - if (PyLong_Check(obj)) { - unsigned long v = PyLong_AsUnsignedLong(obj); - if (!PyErr_Occurred()) { - if (val) *val = v; - return SWIG_OK; - } else { - PyErr_Clear(); - return SWIG_OverflowError; - } - } -#ifdef SWIG_PYTHON_CAST_MODE - { - int dispatch = 0; - unsigned long v = PyLong_AsUnsignedLong(obj); - if (!PyErr_Occurred()) { - if (val) *val = v; - return SWIG_AddCast(SWIG_OK); - } else { - PyErr_Clear(); - } - if (!dispatch) { - double d; - int res = SWIG_AddCast(SWIG_AsVal_double (obj,&d)); - if (SWIG_IsOK(res) && SWIG_CanCastAsInteger(&d, 0, ULONG_MAX)) { - if (val) *val = (unsigned long)(d); - return res; - } - } - } -#endif - return SWIG_TypeError; -} - - -#include -#if !defined(SWIG_NO_LLONG_MAX) -# if !defined(LLONG_MAX) && defined(__GNUC__) && defined (__LONG_LONG_MAX__) -# define LLONG_MAX __LONG_LONG_MAX__ -# define LLONG_MIN (-LLONG_MAX - 1LL) -# define ULLONG_MAX (LLONG_MAX * 2ULL + 1ULL) -# endif -#endif - - -#if defined(LLONG_MAX) && !defined(SWIG_LONG_LONG_AVAILABLE) -# define SWIG_LONG_LONG_AVAILABLE -#endif - - -#ifdef SWIG_LONG_LONG_AVAILABLE -SWIGINTERN int -SWIG_AsVal_unsigned_SS_long_SS_long (PyObject *obj, unsigned long long *val) -{ - int res = SWIG_TypeError; - if (PyLong_Check(obj)) { - unsigned long long v = PyLong_AsUnsignedLongLong(obj); - if (!PyErr_Occurred()) { - if (val) *val = v; - return SWIG_OK; - } else { - PyErr_Clear(); - res = SWIG_OverflowError; - } - } else { - unsigned long v; - res = SWIG_AsVal_unsigned_SS_long (obj,&v); - if (SWIG_IsOK(res)) { - if (val) *val = v; - return res; - } - } -#ifdef SWIG_PYTHON_CAST_MODE - { - const double mant_max = 1LL << DBL_MANT_DIG; - double d; - res = SWIG_AsVal_double (obj,&d); - if (SWIG_IsOK(res) && SWIG_CanCastAsInteger(&d, 0, mant_max)) { - if (val) *val = (unsigned long long)(d); - return SWIG_AddCast(res); - } - res = SWIG_TypeError; - } -#endif - return res; -} -#endif - - -SWIGINTERNINLINE int -SWIG_AsVal_size_t (PyObject * obj, size_t *val) -{ - int res = SWIG_TypeError; -#ifdef SWIG_LONG_LONG_AVAILABLE - if (sizeof(size_t) <= sizeof(unsigned long)) { -#endif - unsigned long v; - res = SWIG_AsVal_unsigned_SS_long (obj, val ? &v : 0); - if (SWIG_IsOK(res) && val) *val = static_cast< size_t >(v); -#ifdef SWIG_LONG_LONG_AVAILABLE - } else if (sizeof(size_t) <= sizeof(unsigned long long)) { - unsigned long long v; - res = SWIG_AsVal_unsigned_SS_long_SS_long (obj, val ? &v : 0); - if (SWIG_IsOK(res) && val) *val = static_cast< size_t >(v); - } -#endif - return res; -} - - -SWIGINTERNINLINE PyObject * -SWIG_FromCharPtrAndSize(const char* carray, size_t size) -{ - if (carray) { - if (size > INT_MAX) { - swig_type_info* pchar_descriptor = SWIG_pchar_descriptor(); - return pchar_descriptor ? - SWIG_InternalNewPointerObj(const_cast< char * >(carray), pchar_descriptor, 0) : SWIG_Py_Void(); - } else { -#if PY_VERSION_HEX >= 0x03000000 -#if defined(SWIG_PYTHON_STRICT_BYTE_CHAR) - return PyBytes_FromStringAndSize(carray, static_cast< Py_ssize_t >(size)); -#else -#if PY_VERSION_HEX >= 0x03010000 - return PyUnicode_DecodeUTF8(carray, static_cast< Py_ssize_t >(size), "surrogateescape"); -#else - return PyUnicode_FromStringAndSize(carray, static_cast< Py_ssize_t >(size)); -#endif -#endif -#else - return PyString_FromStringAndSize(carray, static_cast< Py_ssize_t >(size)); -#endif - } - } else { - return SWIG_Py_Void(); - } -} - - -SWIGINTERNINLINE PyObject * -SWIG_FromCharPtr(const char *cptr) -{ - return SWIG_FromCharPtrAndSize(cptr, (cptr ? strlen(cptr) : 0)); -} - - - #define SWIG_From_long PyInt_FromLong - - -SWIGINTERNINLINE PyObject* -SWIG_From_unsigned_SS_long (unsigned long value) -{ - return (value > LONG_MAX) ? - PyLong_FromUnsignedLong(value) : PyInt_FromLong(static_cast< long >(value)); -} - - -#ifdef SWIG_LONG_LONG_AVAILABLE -SWIGINTERNINLINE PyObject* -SWIG_From_unsigned_SS_long_SS_long (unsigned long long value) -{ - return (value > LONG_MAX) ? - PyLong_FromUnsignedLongLong(value) : PyInt_FromLong(static_cast< long >(value)); -} -#endif - - -SWIGINTERNINLINE PyObject * -SWIG_From_size_t (size_t value) -{ -#ifdef SWIG_LONG_LONG_AVAILABLE - if (sizeof(size_t) <= sizeof(unsigned long)) { -#endif - return SWIG_From_unsigned_SS_long (static_cast< unsigned long >(value)); -#ifdef SWIG_LONG_LONG_AVAILABLE - } else { - /* assume sizeof(size_t) <= sizeof(unsigned long long) */ - return SWIG_From_unsigned_SS_long_SS_long (static_cast< unsigned long long >(value)); - } -#endif -} - - SWIGINTERN int SWIG_AsVal_long (PyObject *obj, long* val) { @@ -3801,425 +3617,6 @@ SWIGINTERN sentencepiece::util::Status sentencepiece_SentencePieceTrainer_train( #ifdef __cplusplus extern "C" { #endif -SWIGINTERN PyObject *_wrap_new_min_string_view__SWIG_0(PyObject *SWIGUNUSEDPARM(self), PyObject *args) { - PyObject *resultobj = 0; - sentencepiece::util::min_string_view *result = 0 ; - - if (!PyArg_ParseTuple(args,(char *)":new_min_string_view")) SWIG_fail; - { - try { - result = (sentencepiece::util::min_string_view *)new sentencepiece::util::min_string_view(); - } - catch (const sentencepiece::util::Status &status) { - SWIG_exception(ToSwigError(status.code()), status.ToString().c_str()); - } - } - resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(result), SWIGTYPE_p_sentencepiece__util__min_string_view, SWIG_POINTER_NEW | 0 ); - return resultobj; -fail: - return NULL; -} - - -SWIGINTERN PyObject *_wrap_new_min_string_view__SWIG_1(PyObject *SWIGUNUSEDPARM(self), PyObject *args) { - PyObject *resultobj = 0; - std::string *arg1 = 0 ; - PyObject * obj0 = 0 ; - sentencepiece::util::min_string_view *result = 0 ; - - if (!PyArg_ParseTuple(args,(char *)"O:new_min_string_view",&obj0)) SWIG_fail; - { - const PyInputString ustring(obj0); - if (!ustring.IsAvalable()) { - PyErr_SetString(PyExc_TypeError, "not a string"); - return nullptr; - } - resultobj = ustring.input_type(); - arg1 = new std::string(ustring.data(), ustring.size()); - } - { - try { - result = (sentencepiece::util::min_string_view *)new sentencepiece::util::min_string_view((std::string const &)*arg1); - } - catch (const sentencepiece::util::Status &status) { - SWIG_exception(ToSwigError(status.code()), status.ToString().c_str()); - } - } - resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(result), SWIGTYPE_p_sentencepiece__util__min_string_view, SWIG_POINTER_NEW | 0 ); - { - delete arg1; - } - return resultobj; -fail: - { - delete arg1; - } - return NULL; -} - - -SWIGINTERN PyObject *_wrap_new_min_string_view__SWIG_2(PyObject *SWIGUNUSEDPARM(self), PyObject *args) { - PyObject *resultobj = 0; - char *arg1 = (char *) 0 ; - int res1 ; - char *buf1 = 0 ; - int alloc1 = 0 ; - PyObject * obj0 = 0 ; - sentencepiece::util::min_string_view *result = 0 ; - - if (!PyArg_ParseTuple(args,(char *)"O:new_min_string_view",&obj0)) SWIG_fail; - res1 = SWIG_AsCharPtrAndSize(obj0, &buf1, NULL, &alloc1); - if (!SWIG_IsOK(res1)) { - SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "new_min_string_view" "', argument " "1"" of type '" "char const *""'"); - } - arg1 = reinterpret_cast< char * >(buf1); - { - try { - result = (sentencepiece::util::min_string_view *)new sentencepiece::util::min_string_view((char const *)arg1); - } - catch (const sentencepiece::util::Status &status) { - SWIG_exception(ToSwigError(status.code()), status.ToString().c_str()); - } - } - resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(result), SWIGTYPE_p_sentencepiece__util__min_string_view, SWIG_POINTER_NEW | 0 ); - if (alloc1 == SWIG_NEWOBJ) delete[] buf1; - return resultobj; -fail: - if (alloc1 == SWIG_NEWOBJ) delete[] buf1; - return NULL; -} - - -SWIGINTERN PyObject *_wrap_new_min_string_view__SWIG_3(PyObject *SWIGUNUSEDPARM(self), PyObject *args) { - PyObject *resultobj = 0; - char *arg1 = (char *) 0 ; - size_t arg2 ; - int res1 ; - char *buf1 = 0 ; - int alloc1 = 0 ; - size_t val2 ; - int ecode2 = 0 ; - PyObject * obj0 = 0 ; - PyObject * obj1 = 0 ; - sentencepiece::util::min_string_view *result = 0 ; - - if (!PyArg_ParseTuple(args,(char *)"OO:new_min_string_view",&obj0,&obj1)) SWIG_fail; - res1 = SWIG_AsCharPtrAndSize(obj0, &buf1, NULL, &alloc1); - if (!SWIG_IsOK(res1)) { - SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "new_min_string_view" "', argument " "1"" of type '" "char const *""'"); - } - arg1 = reinterpret_cast< char * >(buf1); - ecode2 = SWIG_AsVal_size_t(obj1, &val2); - if (!SWIG_IsOK(ecode2)) { - SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "new_min_string_view" "', argument " "2"" of type '" "size_t""'"); - } - arg2 = static_cast< size_t >(val2); - { - try { - result = (sentencepiece::util::min_string_view *)new sentencepiece::util::min_string_view((char const *)arg1,arg2); - } - catch (const sentencepiece::util::Status &status) { - SWIG_exception(ToSwigError(status.code()), status.ToString().c_str()); - } - } - resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(result), SWIGTYPE_p_sentencepiece__util__min_string_view, SWIG_POINTER_NEW | 0 ); - if (alloc1 == SWIG_NEWOBJ) delete[] buf1; - return resultobj; -fail: - if (alloc1 == SWIG_NEWOBJ) delete[] buf1; - return NULL; -} - - -SWIGINTERN PyObject *_wrap_min_string_view_data(PyObject *SWIGUNUSEDPARM(self), PyObject *args) { - PyObject *resultobj = 0; - sentencepiece::util::min_string_view *arg1 = (sentencepiece::util::min_string_view *) 0 ; - void *argp1 = 0 ; - int res1 = 0 ; - PyObject * obj0 = 0 ; - char *result = 0 ; - - if (!PyArg_ParseTuple(args,(char *)"O:min_string_view_data",&obj0)) SWIG_fail; - res1 = SWIG_ConvertPtr(obj0, &argp1,SWIGTYPE_p_sentencepiece__util__min_string_view, 0 | 0 ); - if (!SWIG_IsOK(res1)) { - SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "min_string_view_data" "', argument " "1"" of type '" "sentencepiece::util::min_string_view const *""'"); - } - arg1 = reinterpret_cast< sentencepiece::util::min_string_view * >(argp1); - { - try { - result = (char *)((sentencepiece::util::min_string_view const *)arg1)->data(); - } - catch (const sentencepiece::util::Status &status) { - SWIG_exception(ToSwigError(status.code()), status.ToString().c_str()); - } - } - resultobj = SWIG_FromCharPtr((const char *)result); - return resultobj; -fail: - return NULL; -} - - -SWIGINTERN PyObject *_wrap_min_string_view_size(PyObject *SWIGUNUSEDPARM(self), PyObject *args) { - PyObject *resultobj = 0; - sentencepiece::util::min_string_view *arg1 = (sentencepiece::util::min_string_view *) 0 ; - void *argp1 = 0 ; - int res1 = 0 ; - PyObject * obj0 = 0 ; - size_t result; - - if (!PyArg_ParseTuple(args,(char *)"O:min_string_view_size",&obj0)) SWIG_fail; - res1 = SWIG_ConvertPtr(obj0, &argp1,SWIGTYPE_p_sentencepiece__util__min_string_view, 0 | 0 ); - if (!SWIG_IsOK(res1)) { - SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "min_string_view_size" "', argument " "1"" of type '" "sentencepiece::util::min_string_view const *""'"); - } - arg1 = reinterpret_cast< sentencepiece::util::min_string_view * >(argp1); - { - try { - result = ((sentencepiece::util::min_string_view const *)arg1)->size(); - } - catch (const sentencepiece::util::Status &status) { - SWIG_exception(ToSwigError(status.code()), status.ToString().c_str()); - } - } - resultobj = SWIG_From_size_t(static_cast< size_t >(result)); - return resultobj; -fail: - return NULL; -} - - -SWIGINTERN PyObject *_wrap_min_string_view_assign(PyObject *SWIGUNUSEDPARM(self), PyObject *args) { - PyObject *resultobj = 0; - sentencepiece::util::min_string_view *arg1 = (sentencepiece::util::min_string_view *) 0 ; - char *arg2 = (char *) 0 ; - size_t arg3 ; - void *argp1 = 0 ; - int res1 = 0 ; - int res2 ; - char *buf2 = 0 ; - int alloc2 = 0 ; - size_t val3 ; - int ecode3 = 0 ; - PyObject * obj0 = 0 ; - PyObject * obj1 = 0 ; - PyObject * obj2 = 0 ; - - if (!PyArg_ParseTuple(args,(char *)"OOO:min_string_view_assign",&obj0,&obj1,&obj2)) SWIG_fail; - res1 = SWIG_ConvertPtr(obj0, &argp1,SWIGTYPE_p_sentencepiece__util__min_string_view, 0 | 0 ); - if (!SWIG_IsOK(res1)) { - SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "min_string_view_assign" "', argument " "1"" of type '" "sentencepiece::util::min_string_view *""'"); - } - arg1 = reinterpret_cast< sentencepiece::util::min_string_view * >(argp1); - res2 = SWIG_AsCharPtrAndSize(obj1, &buf2, NULL, &alloc2); - if (!SWIG_IsOK(res2)) { - SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "min_string_view_assign" "', argument " "2"" of type '" "char const *""'"); - } - arg2 = reinterpret_cast< char * >(buf2); - ecode3 = SWIG_AsVal_size_t(obj2, &val3); - if (!SWIG_IsOK(ecode3)) { - SWIG_exception_fail(SWIG_ArgError(ecode3), "in method '" "min_string_view_assign" "', argument " "3"" of type '" "size_t""'"); - } - arg3 = static_cast< size_t >(val3); - { - try { - (arg1)->assign((char const *)arg2,arg3); - } - catch (const sentencepiece::util::Status &status) { - SWIG_exception(ToSwigError(status.code()), status.ToString().c_str()); - } - } - resultobj = SWIG_Py_Void(); - if (alloc2 == SWIG_NEWOBJ) delete[] buf2; - return resultobj; -fail: - if (alloc2 == SWIG_NEWOBJ) delete[] buf2; - return NULL; -} - - -SWIGINTERN PyObject *_wrap_min_string_view_copy(PyObject *SWIGUNUSEDPARM(self), PyObject *args) { - PyObject *resultobj = 0; - sentencepiece::util::min_string_view *arg1 = (sentencepiece::util::min_string_view *) 0 ; - char *arg2 = (char *) 0 ; - size_t arg3 ; - void *argp1 = 0 ; - int res1 = 0 ; - int res2 ; - char *buf2 = 0 ; - int alloc2 = 0 ; - size_t val3 ; - int ecode3 = 0 ; - PyObject * obj0 = 0 ; - PyObject * obj1 = 0 ; - PyObject * obj2 = 0 ; - - if (!PyArg_ParseTuple(args,(char *)"OOO:min_string_view_copy",&obj0,&obj1,&obj2)) SWIG_fail; - res1 = SWIG_ConvertPtr(obj0, &argp1,SWIGTYPE_p_sentencepiece__util__min_string_view, 0 | 0 ); - if (!SWIG_IsOK(res1)) { - SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "min_string_view_copy" "', argument " "1"" of type '" "sentencepiece::util::min_string_view *""'"); - } - arg1 = reinterpret_cast< sentencepiece::util::min_string_view * >(argp1); - res2 = SWIG_AsCharPtrAndSize(obj1, &buf2, NULL, &alloc2); - if (!SWIG_IsOK(res2)) { - SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "min_string_view_copy" "', argument " "2"" of type '" "char const *""'"); - } - arg2 = reinterpret_cast< char * >(buf2); - ecode3 = SWIG_AsVal_size_t(obj2, &val3); - if (!SWIG_IsOK(ecode3)) { - SWIG_exception_fail(SWIG_ArgError(ecode3), "in method '" "min_string_view_copy" "', argument " "3"" of type '" "size_t""'"); - } - arg3 = static_cast< size_t >(val3); - { - try { - (arg1)->copy((char const *)arg2,arg3); - } - catch (const sentencepiece::util::Status &status) { - SWIG_exception(ToSwigError(status.code()), status.ToString().c_str()); - } - } - resultobj = SWIG_Py_Void(); - if (alloc2 == SWIG_NEWOBJ) delete[] buf2; - return resultobj; -fail: - if (alloc2 == SWIG_NEWOBJ) delete[] buf2; - return NULL; -} - - -SWIGINTERN PyObject *_wrap_new_min_string_view__SWIG_4(PyObject *SWIGUNUSEDPARM(self), PyObject *args) { - PyObject *resultobj = 0; - sentencepiece::util::min_string_view *arg1 = 0 ; - void *argp1 = 0 ; - int res1 = 0 ; - PyObject * obj0 = 0 ; - sentencepiece::util::min_string_view *result = 0 ; - - if (!PyArg_ParseTuple(args,(char *)"O:new_min_string_view",&obj0)) SWIG_fail; - res1 = SWIG_ConvertPtr(obj0, &argp1, SWIGTYPE_p_sentencepiece__util__min_string_view, 0 | 0); - if (!SWIG_IsOK(res1)) { - SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "new_min_string_view" "', argument " "1"" of type '" "sentencepiece::util::min_string_view const &""'"); - } - if (!argp1) { - SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "new_min_string_view" "', argument " "1"" of type '" "sentencepiece::util::min_string_view const &""'"); - } - arg1 = reinterpret_cast< sentencepiece::util::min_string_view * >(argp1); - { - try { - result = (sentencepiece::util::min_string_view *)new sentencepiece::util::min_string_view((sentencepiece::util::min_string_view const &)*arg1); - } - catch (const sentencepiece::util::Status &status) { - SWIG_exception(ToSwigError(status.code()), status.ToString().c_str()); - } - } - resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(result), SWIGTYPE_p_sentencepiece__util__min_string_view, SWIG_POINTER_NEW | 0 ); - return resultobj; -fail: - return NULL; -} - - -SWIGINTERN PyObject *_wrap_new_min_string_view(PyObject *self, PyObject *args) { - Py_ssize_t argc; - PyObject *argv[3] = { - 0 - }; - Py_ssize_t ii; - - if (!PyTuple_Check(args)) SWIG_fail; - argc = args ? PyObject_Length(args) : 0; - for (ii = 0; (ii < 2) && (ii < argc); ii++) { - argv[ii] = PyTuple_GET_ITEM(args,ii); - } - if (argc == 0) { - return _wrap_new_min_string_view__SWIG_0(self, args); - } - if (argc == 1) { - int _v; - int res = SWIG_ConvertPtr(argv[0], 0, SWIGTYPE_p_std__string, 0); - _v = SWIG_CheckState(res); - if (_v) { - return _wrap_new_min_string_view__SWIG_1(self, args); - } - } - if (argc == 1) { - int _v; - int res = SWIG_ConvertPtr(argv[0], 0, SWIGTYPE_p_sentencepiece__util__min_string_view, 0); - _v = SWIG_CheckState(res); - if (_v) { - return _wrap_new_min_string_view__SWIG_4(self, args); - } - } - if (argc == 1) { - int _v; - int res = SWIG_AsCharPtrAndSize(argv[0], 0, NULL, 0); - _v = SWIG_CheckState(res); - if (_v) { - return _wrap_new_min_string_view__SWIG_2(self, args); - } - } - if (argc == 2) { - int _v; - int res = SWIG_AsCharPtrAndSize(argv[0], 0, NULL, 0); - _v = SWIG_CheckState(res); - if (_v) { - { - int res = SWIG_AsVal_size_t(argv[1], NULL); - _v = SWIG_CheckState(res); - } - if (_v) { - return _wrap_new_min_string_view__SWIG_3(self, args); - } - } - } - -fail: - SWIG_SetErrorMsg(PyExc_NotImplementedError,"Wrong number or type of arguments for overloaded function 'new_min_string_view'.\n" - " Possible C/C++ prototypes are:\n" - " sentencepiece::util::min_string_view::min_string_view()\n" - " sentencepiece::util::min_string_view::min_string_view(std::string const &)\n" - " sentencepiece::util::min_string_view::min_string_view(char const *)\n" - " sentencepiece::util::min_string_view::min_string_view(char const *,size_t)\n" - " sentencepiece::util::min_string_view::min_string_view(sentencepiece::util::min_string_view const &)\n"); - return 0; -} - - -SWIGINTERN PyObject *_wrap_delete_min_string_view(PyObject *SWIGUNUSEDPARM(self), PyObject *args) { - PyObject *resultobj = 0; - sentencepiece::util::min_string_view *arg1 = (sentencepiece::util::min_string_view *) 0 ; - void *argp1 = 0 ; - int res1 = 0 ; - PyObject * obj0 = 0 ; - - if (!PyArg_ParseTuple(args,(char *)"O:delete_min_string_view",&obj0)) SWIG_fail; - res1 = SWIG_ConvertPtr(obj0, &argp1,SWIGTYPE_p_sentencepiece__util__min_string_view, SWIG_POINTER_DISOWN | 0 ); - if (!SWIG_IsOK(res1)) { - SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "delete_min_string_view" "', argument " "1"" of type '" "sentencepiece::util::min_string_view *""'"); - } - arg1 = reinterpret_cast< sentencepiece::util::min_string_view * >(argp1); - { - try { - delete arg1; - } - catch (const sentencepiece::util::Status &status) { - SWIG_exception(ToSwigError(status.code()), status.ToString().c_str()); - } - } - resultobj = SWIG_Py_Void(); - return resultobj; -fail: - return NULL; -} - - -SWIGINTERN PyObject *min_string_view_swigregister(PyObject *SWIGUNUSEDPARM(self), PyObject *args) { - PyObject *obj; - if (!PyArg_ParseTuple(args,(char*)"O:swigregister", &obj)) return NULL; - SWIG_TypeNewClientData(SWIGTYPE_p_sentencepiece__util__min_string_view, SWIG_NewClientData(obj)); - return SWIG_Py_Void(); -} - SWIGINTERN PyObject *_wrap_new_SentencePieceProcessor(PyObject *SWIGUNUSEDPARM(self), PyObject *args) { PyObject *resultobj = 0; sentencepiece::SentencePieceProcessor *result = 0 ; @@ -4227,7 +3624,8 @@ SWIGINTERN PyObject *_wrap_new_SentencePieceProcessor(PyObject *SWIGUNUSEDPARM(s if (!PyArg_ParseTuple(args,(char *)":new_SentencePieceProcessor")) SWIG_fail; { try { - result = (sentencepiece::SentencePieceProcessor *)new sentencepiece::SentencePieceProcessor(); + result = (sentencepiece::SentencePieceProcessor *)new sentencepiece::SentencePieceProcessor(); + ReleaseResultObject(resultobj); } catch (const sentencepiece::util::Status &status) { SWIG_exception(ToSwigError(status.code()), status.ToString().c_str()); @@ -4255,7 +3653,8 @@ SWIGINTERN PyObject *_wrap_delete_SentencePieceProcessor(PyObject *SWIGUNUSEDPAR arg1 = reinterpret_cast< sentencepiece::SentencePieceProcessor * >(argp1); { try { - delete arg1; + delete arg1; + ReleaseResultObject(resultobj); } catch (const sentencepiece::util::Status &status) { SWIG_exception(ToSwigError(status.code()), status.ToString().c_str()); @@ -4288,18 +3687,15 @@ SWIGINTERN PyObject *_wrap_SentencePieceProcessor_Load(PyObject *SWIGUNUSEDPARM( const PyInputString ustring(obj1); if (!ustring.IsAvalable()) { PyErr_SetString(PyExc_TypeError, "not a string"); - return nullptr; + SWIG_fail; } resultobj = ustring.input_type(); - if (ustring.IsCopy()) { - (&arg2)->copy(ustring.data(), ustring.size()); - } else { - (&arg2)->assign(ustring.data(), ustring.size()); - } + arg2 = sentencepiece::util::min_string_view(ustring.data(), ustring.size()); } { try { - result = (arg1)->Load(arg2); + result = (arg1)->Load(arg2); + ReleaseResultObject(resultobj); } catch (const sentencepiece::util::Status &status) { SWIG_exception(ToSwigError(status.code()), status.ToString().c_str()); @@ -4336,18 +3732,15 @@ SWIGINTERN PyObject *_wrap_SentencePieceProcessor_LoadOrDie(PyObject *SWIGUNUSED const PyInputString ustring(obj1); if (!ustring.IsAvalable()) { PyErr_SetString(PyExc_TypeError, "not a string"); - return nullptr; + SWIG_fail; } resultobj = ustring.input_type(); - if (ustring.IsCopy()) { - (&arg2)->copy(ustring.data(), ustring.size()); - } else { - (&arg2)->assign(ustring.data(), ustring.size()); - } + arg2 = sentencepiece::util::min_string_view(ustring.data(), ustring.size()); } { try { - (arg1)->LoadOrDie(arg2); + (arg1)->LoadOrDie(arg2); + ReleaseResultObject(resultobj); } catch (const sentencepiece::util::Status &status) { SWIG_exception(ToSwigError(status.code()), status.ToString().c_str()); @@ -4380,18 +3773,15 @@ SWIGINTERN PyObject *_wrap_SentencePieceProcessor_SetEncodeExtraOptions(PyObject const PyInputString ustring(obj1); if (!ustring.IsAvalable()) { PyErr_SetString(PyExc_TypeError, "not a string"); - return nullptr; + SWIG_fail; } resultobj = ustring.input_type(); - if (ustring.IsCopy()) { - (&arg2)->copy(ustring.data(), ustring.size()); - } else { - (&arg2)->assign(ustring.data(), ustring.size()); - } + arg2 = sentencepiece::util::min_string_view(ustring.data(), ustring.size()); } { try { - result = (arg1)->SetEncodeExtraOptions(arg2); + result = (arg1)->SetEncodeExtraOptions(arg2); + ReleaseResultObject(resultobj); } catch (const sentencepiece::util::Status &status) { SWIG_exception(ToSwigError(status.code()), status.ToString().c_str()); @@ -4429,18 +3819,15 @@ SWIGINTERN PyObject *_wrap_SentencePieceProcessor_SetDecodeExtraOptions(PyObject const PyInputString ustring(obj1); if (!ustring.IsAvalable()) { PyErr_SetString(PyExc_TypeError, "not a string"); - return nullptr; + SWIG_fail; } resultobj = ustring.input_type(); - if (ustring.IsCopy()) { - (&arg2)->copy(ustring.data(), ustring.size()); - } else { - (&arg2)->assign(ustring.data(), ustring.size()); - } + arg2 = sentencepiece::util::min_string_view(ustring.data(), ustring.size()); } { try { - result = (arg1)->SetDecodeExtraOptions(arg2); + result = (arg1)->SetDecodeExtraOptions(arg2); + ReleaseResultObject(resultobj); } catch (const sentencepiece::util::Status &status) { SWIG_exception(ToSwigError(status.code()), status.ToString().c_str()); @@ -4485,19 +3872,20 @@ SWIGINTERN PyObject *_wrap_SentencePieceProcessor_SetVocabulary(PyObject *SWIGUN (*out)[i] = std::string(ustring.data(), ustring.size()); } else { PyErr_SetString(PyExc_TypeError, "list must contain strings"); - return nullptr; + SWIG_fail; } resultobj = ustring.input_type(); } } else { PyErr_SetString(PyExc_TypeError, "not a list"); - return nullptr; + SWIG_fail; } arg2 = out; } { try { - result = (arg1)->SetVocabulary((std::vector< std::string > const &)*arg2); + result = (arg1)->SetVocabulary((std::vector< std::string > const &)*arg2); + ReleaseResultObject(resultobj); } catch (const sentencepiece::util::Status &status) { SWIG_exception(ToSwigError(status.code()), status.ToString().c_str()); @@ -4537,7 +3925,8 @@ SWIGINTERN PyObject *_wrap_SentencePieceProcessor_ResetVocabulary(PyObject *SWIG arg1 = reinterpret_cast< sentencepiece::SentencePieceProcessor * >(argp1); { try { - result = (arg1)->ResetVocabulary(); + result = (arg1)->ResetVocabulary(); + ReleaseResultObject(resultobj); } catch (const sentencepiece::util::Status &status) { SWIG_exception(ToSwigError(status.code()), status.ToString().c_str()); @@ -4579,14 +3968,10 @@ SWIGINTERN PyObject *_wrap_SentencePieceProcessor_LoadVocabulary(PyObject *SWIGU const PyInputString ustring(obj1); if (!ustring.IsAvalable()) { PyErr_SetString(PyExc_TypeError, "not a string"); - return nullptr; + SWIG_fail; } resultobj = ustring.input_type(); - if (ustring.IsCopy()) { - (&arg2)->copy(ustring.data(), ustring.size()); - } else { - (&arg2)->assign(ustring.data(), ustring.size()); - } + arg2 = sentencepiece::util::min_string_view(ustring.data(), ustring.size()); } ecode3 = SWIG_AsVal_int(obj2, &val3); if (!SWIG_IsOK(ecode3)) { @@ -4595,7 +3980,8 @@ SWIGINTERN PyObject *_wrap_SentencePieceProcessor_LoadVocabulary(PyObject *SWIGU arg3 = static_cast< int >(val3); { try { - result = (arg1)->LoadVocabulary(arg2,arg3); + result = (arg1)->LoadVocabulary(arg2,arg3); + ReleaseResultObject(resultobj); } catch (const sentencepiece::util::Status &status) { SWIG_exception(ToSwigError(status.code()), status.ToString().c_str()); @@ -4633,18 +4019,15 @@ SWIGINTERN PyObject *_wrap_SentencePieceProcessor_EncodeAsPieces(PyObject *SWIGU const PyInputString ustring(obj1); if (!ustring.IsAvalable()) { PyErr_SetString(PyExc_TypeError, "not a string"); - return nullptr; + SWIG_fail; } resultobj = ustring.input_type(); - if (ustring.IsCopy()) { - (&arg2)->copy(ustring.data(), ustring.size()); - } else { - (&arg2)->assign(ustring.data(), ustring.size()); - } + arg2 = sentencepiece::util::min_string_view(ustring.data(), ustring.size()); } { try { - result = ((sentencepiece::SentencePieceProcessor const *)arg1)->EncodeAsPieces(arg2); + result = ((sentencepiece::SentencePieceProcessor const *)arg1)->EncodeAsPieces(arg2); + ReleaseResultObject(resultobj); } catch (const sentencepiece::util::Status &status) { SWIG_exception(ToSwigError(status.code()), status.ToString().c_str()); @@ -4683,18 +4066,15 @@ SWIGINTERN PyObject *_wrap_SentencePieceProcessor_EncodeAsIds(PyObject *SWIGUNUS const PyInputString ustring(obj1); if (!ustring.IsAvalable()) { PyErr_SetString(PyExc_TypeError, "not a string"); - return nullptr; + SWIG_fail; } resultobj = ustring.input_type(); - if (ustring.IsCopy()) { - (&arg2)->copy(ustring.data(), ustring.size()); - } else { - (&arg2)->assign(ustring.data(), ustring.size()); - } + arg2 = sentencepiece::util::min_string_view(ustring.data(), ustring.size()); } { try { - result = ((sentencepiece::SentencePieceProcessor const *)arg1)->EncodeAsIds(arg2); + result = ((sentencepiece::SentencePieceProcessor const *)arg1)->EncodeAsIds(arg2); + ReleaseResultObject(resultobj); } catch (const sentencepiece::util::Status &status) { SWIG_exception(ToSwigError(status.code()), status.ToString().c_str()); @@ -4736,14 +4116,10 @@ SWIGINTERN PyObject *_wrap_SentencePieceProcessor_NBestEncodeAsPieces(PyObject * const PyInputString ustring(obj1); if (!ustring.IsAvalable()) { PyErr_SetString(PyExc_TypeError, "not a string"); - return nullptr; + SWIG_fail; } resultobj = ustring.input_type(); - if (ustring.IsCopy()) { - (&arg2)->copy(ustring.data(), ustring.size()); - } else { - (&arg2)->assign(ustring.data(), ustring.size()); - } + arg2 = sentencepiece::util::min_string_view(ustring.data(), ustring.size()); } ecode3 = SWIG_AsVal_int(obj2, &val3); if (!SWIG_IsOK(ecode3)) { @@ -4752,7 +4128,8 @@ SWIGINTERN PyObject *_wrap_SentencePieceProcessor_NBestEncodeAsPieces(PyObject * arg3 = static_cast< int >(val3); { try { - result = ((sentencepiece::SentencePieceProcessor const *)arg1)->NBestEncodeAsPieces(arg2,arg3); + result = ((sentencepiece::SentencePieceProcessor const *)arg1)->NBestEncodeAsPieces(arg2,arg3); + ReleaseResultObject(resultobj); } catch (const sentencepiece::util::Status &status) { SWIG_exception(ToSwigError(status.code()), status.ToString().c_str()); @@ -4799,14 +4176,10 @@ SWIGINTERN PyObject *_wrap_SentencePieceProcessor_NBestEncodeAsIds(PyObject *SWI const PyInputString ustring(obj1); if (!ustring.IsAvalable()) { PyErr_SetString(PyExc_TypeError, "not a string"); - return nullptr; + SWIG_fail; } resultobj = ustring.input_type(); - if (ustring.IsCopy()) { - (&arg2)->copy(ustring.data(), ustring.size()); - } else { - (&arg2)->assign(ustring.data(), ustring.size()); - } + arg2 = sentencepiece::util::min_string_view(ustring.data(), ustring.size()); } ecode3 = SWIG_AsVal_int(obj2, &val3); if (!SWIG_IsOK(ecode3)) { @@ -4815,7 +4188,8 @@ SWIGINTERN PyObject *_wrap_SentencePieceProcessor_NBestEncodeAsIds(PyObject *SWI arg3 = static_cast< int >(val3); { try { - result = ((sentencepiece::SentencePieceProcessor const *)arg1)->NBestEncodeAsIds(arg2,arg3); + result = ((sentencepiece::SentencePieceProcessor const *)arg1)->NBestEncodeAsIds(arg2,arg3); + ReleaseResultObject(resultobj); } catch (const sentencepiece::util::Status &status) { SWIG_exception(ToSwigError(status.code()), status.ToString().c_str()); @@ -4865,14 +4239,10 @@ SWIGINTERN PyObject *_wrap_SentencePieceProcessor_SampleEncodeAsPieces(PyObject const PyInputString ustring(obj1); if (!ustring.IsAvalable()) { PyErr_SetString(PyExc_TypeError, "not a string"); - return nullptr; + SWIG_fail; } resultobj = ustring.input_type(); - if (ustring.IsCopy()) { - (&arg2)->copy(ustring.data(), ustring.size()); - } else { - (&arg2)->assign(ustring.data(), ustring.size()); - } + arg2 = sentencepiece::util::min_string_view(ustring.data(), ustring.size()); } ecode3 = SWIG_AsVal_int(obj2, &val3); if (!SWIG_IsOK(ecode3)) { @@ -4886,7 +4256,8 @@ SWIGINTERN PyObject *_wrap_SentencePieceProcessor_SampleEncodeAsPieces(PyObject arg4 = static_cast< float >(val4); { try { - result = ((sentencepiece::SentencePieceProcessor const *)arg1)->SampleEncodeAsPieces(arg2,arg3,arg4); + result = ((sentencepiece::SentencePieceProcessor const *)arg1)->SampleEncodeAsPieces(arg2,arg3,arg4); + ReleaseResultObject(resultobj); } catch (const sentencepiece::util::Status &status) { SWIG_exception(ToSwigError(status.code()), status.ToString().c_str()); @@ -4933,14 +4304,10 @@ SWIGINTERN PyObject *_wrap_SentencePieceProcessor_SampleEncodeAsIds(PyObject *SW const PyInputString ustring(obj1); if (!ustring.IsAvalable()) { PyErr_SetString(PyExc_TypeError, "not a string"); - return nullptr; + SWIG_fail; } resultobj = ustring.input_type(); - if (ustring.IsCopy()) { - (&arg2)->copy(ustring.data(), ustring.size()); - } else { - (&arg2)->assign(ustring.data(), ustring.size()); - } + arg2 = sentencepiece::util::min_string_view(ustring.data(), ustring.size()); } ecode3 = SWIG_AsVal_int(obj2, &val3); if (!SWIG_IsOK(ecode3)) { @@ -4954,7 +4321,8 @@ SWIGINTERN PyObject *_wrap_SentencePieceProcessor_SampleEncodeAsIds(PyObject *SW arg4 = static_cast< float >(val4); { try { - result = ((sentencepiece::SentencePieceProcessor const *)arg1)->SampleEncodeAsIds(arg2,arg3,arg4); + result = ((sentencepiece::SentencePieceProcessor const *)arg1)->SampleEncodeAsIds(arg2,arg3,arg4); + ReleaseResultObject(resultobj); } catch (const sentencepiece::util::Status &status) { SWIG_exception(ToSwigError(status.code()), status.ToString().c_str()); @@ -4999,19 +4367,20 @@ SWIGINTERN PyObject *_wrap_SentencePieceProcessor_DecodePieces(PyObject *SWIGUNU (*out)[i] = std::string(ustring.data(), ustring.size()); } else { PyErr_SetString(PyExc_TypeError, "list must contain strings"); - return nullptr; + SWIG_fail; } resultobj = ustring.input_type(); } } else { PyErr_SetString(PyExc_TypeError, "not a list"); - return nullptr; + SWIG_fail; } arg2 = out; } { try { - result = ((sentencepiece::SentencePieceProcessor const *)arg1)->DecodePieces((std::vector< std::string > const &)*arg2); + result = ((sentencepiece::SentencePieceProcessor const *)arg1)->DecodePieces((std::vector< std::string > const &)*arg2); + ReleaseResultObject(resultobj); } catch (const sentencepiece::util::Status &status) { SWIG_exception(ToSwigError(status.code()), status.ToString().c_str()); @@ -5060,18 +4429,19 @@ SWIGINTERN PyObject *_wrap_SentencePieceProcessor_DecodeIds(PyObject *SWIGUNUSED (*out)[i] = static_cast(PyInt_AsLong(o)); } else { PyErr_SetString(PyExc_TypeError,"list must contain integers"); - return nullptr; + SWIG_fail; } } } else { PyErr_SetString(PyExc_TypeError,"not a list"); - return nullptr; + SWIG_fail; } arg2 = out; } { try { - result = ((sentencepiece::SentencePieceProcessor const *)arg1)->DecodeIds((std::vector< int > const &)*arg2); + result = ((sentencepiece::SentencePieceProcessor const *)arg1)->DecodeIds((std::vector< int > const &)*arg2); + ReleaseResultObject(resultobj); } catch (const sentencepiece::util::Status &status) { SWIG_exception(ToSwigError(status.code()), status.ToString().c_str()); @@ -5109,7 +4479,8 @@ SWIGINTERN PyObject *_wrap_SentencePieceProcessor_GetPieceSize(PyObject *SWIGUNU arg1 = reinterpret_cast< sentencepiece::SentencePieceProcessor * >(argp1); { try { - result = (int)((sentencepiece::SentencePieceProcessor const *)arg1)->GetPieceSize(); + result = (int)((sentencepiece::SentencePieceProcessor const *)arg1)->GetPieceSize(); + ReleaseResultObject(resultobj); } catch (const sentencepiece::util::Status &status) { SWIG_exception(ToSwigError(status.code()), status.ToString().c_str()); @@ -5142,18 +4513,15 @@ SWIGINTERN PyObject *_wrap_SentencePieceProcessor_PieceToId(PyObject *SWIGUNUSED const PyInputString ustring(obj1); if (!ustring.IsAvalable()) { PyErr_SetString(PyExc_TypeError, "not a string"); - return nullptr; + SWIG_fail; } resultobj = ustring.input_type(); - if (ustring.IsCopy()) { - (&arg2)->copy(ustring.data(), ustring.size()); - } else { - (&arg2)->assign(ustring.data(), ustring.size()); - } + arg2 = sentencepiece::util::min_string_view(ustring.data(), ustring.size()); } { try { - result = (int)((sentencepiece::SentencePieceProcessor const *)arg1)->PieceToId(arg2); + result = (int)((sentencepiece::SentencePieceProcessor const *)arg1)->PieceToId(arg2); + ReleaseResultObject(resultobj); } catch (const sentencepiece::util::Status &status) { SWIG_exception(ToSwigError(status.code()), status.ToString().c_str()); @@ -5191,7 +4559,8 @@ SWIGINTERN PyObject *_wrap_SentencePieceProcessor_IdToPiece(PyObject *SWIGUNUSED arg2 = static_cast< int >(val2); { try { - result = ((sentencepiece::SentencePieceProcessor const *)arg1)->IdToPiece(arg2); + result = ((sentencepiece::SentencePieceProcessor const *)arg1)->IdToPiece(arg2); + ReleaseResultObject(resultobj); } catch (const sentencepiece::util::Status &status) { SWIG_exception(ToSwigError(status.code()), status.ToString().c_str()); @@ -5232,7 +4601,8 @@ SWIGINTERN PyObject *_wrap_SentencePieceProcessor_GetScore(PyObject *SWIGUNUSEDP arg2 = static_cast< int >(val2); { try { - result = (float)((sentencepiece::SentencePieceProcessor const *)arg1)->GetScore(arg2); + result = (float)((sentencepiece::SentencePieceProcessor const *)arg1)->GetScore(arg2); + ReleaseResultObject(resultobj); } catch (const sentencepiece::util::Status &status) { SWIG_exception(ToSwigError(status.code()), status.ToString().c_str()); @@ -5270,7 +4640,8 @@ SWIGINTERN PyObject *_wrap_SentencePieceProcessor_IsUnknown(PyObject *SWIGUNUSED arg2 = static_cast< int >(val2); { try { - result = (bool)((sentencepiece::SentencePieceProcessor const *)arg1)->IsUnknown(arg2); + result = (bool)((sentencepiece::SentencePieceProcessor const *)arg1)->IsUnknown(arg2); + ReleaseResultObject(resultobj); } catch (const sentencepiece::util::Status &status) { SWIG_exception(ToSwigError(status.code()), status.ToString().c_str()); @@ -5308,7 +4679,8 @@ SWIGINTERN PyObject *_wrap_SentencePieceProcessor_IsControl(PyObject *SWIGUNUSED arg2 = static_cast< int >(val2); { try { - result = (bool)((sentencepiece::SentencePieceProcessor const *)arg1)->IsControl(arg2); + result = (bool)((sentencepiece::SentencePieceProcessor const *)arg1)->IsControl(arg2); + ReleaseResultObject(resultobj); } catch (const sentencepiece::util::Status &status) { SWIG_exception(ToSwigError(status.code()), status.ToString().c_str()); @@ -5346,7 +4718,8 @@ SWIGINTERN PyObject *_wrap_SentencePieceProcessor_IsUnused(PyObject *SWIGUNUSEDP arg2 = static_cast< int >(val2); { try { - result = (bool)((sentencepiece::SentencePieceProcessor const *)arg1)->IsUnused(arg2); + result = (bool)((sentencepiece::SentencePieceProcessor const *)arg1)->IsUnused(arg2); + ReleaseResultObject(resultobj); } catch (const sentencepiece::util::Status &status) { SWIG_exception(ToSwigError(status.code()), status.ToString().c_str()); @@ -5379,18 +4752,15 @@ SWIGINTERN PyObject *_wrap_SentencePieceProcessor_load(PyObject *SWIGUNUSEDPARM( const PyInputString ustring(obj1); if (!ustring.IsAvalable()) { PyErr_SetString(PyExc_TypeError, "not a string"); - return nullptr; + SWIG_fail; } resultobj = ustring.input_type(); - if (ustring.IsCopy()) { - (&arg2)->copy(ustring.data(), ustring.size()); - } else { - (&arg2)->assign(ustring.data(), ustring.size()); - } + arg2 = sentencepiece::util::min_string_view(ustring.data(), ustring.size()); } { try { - result = sentencepiece_SentencePieceProcessor_load(arg1,arg2); + result = sentencepiece_SentencePieceProcessor_load(arg1,arg2); + ReleaseResultObject(resultobj); } catch (const sentencepiece::util::Status &status) { SWIG_exception(ToSwigError(status.code()), status.ToString().c_str()); @@ -5428,18 +4798,15 @@ SWIGINTERN PyObject *_wrap_SentencePieceProcessor_set_encode_extra_options(PyObj const PyInputString ustring(obj1); if (!ustring.IsAvalable()) { PyErr_SetString(PyExc_TypeError, "not a string"); - return nullptr; + SWIG_fail; } resultobj = ustring.input_type(); - if (ustring.IsCopy()) { - (&arg2)->copy(ustring.data(), ustring.size()); - } else { - (&arg2)->assign(ustring.data(), ustring.size()); - } + arg2 = sentencepiece::util::min_string_view(ustring.data(), ustring.size()); } { try { - result = sentencepiece_SentencePieceProcessor_set_encode_extra_options(arg1,arg2); + result = sentencepiece_SentencePieceProcessor_set_encode_extra_options(arg1,arg2); + ReleaseResultObject(resultobj); } catch (const sentencepiece::util::Status &status) { SWIG_exception(ToSwigError(status.code()), status.ToString().c_str()); @@ -5477,18 +4844,15 @@ SWIGINTERN PyObject *_wrap_SentencePieceProcessor_set_decode_extra_options(PyObj const PyInputString ustring(obj1); if (!ustring.IsAvalable()) { PyErr_SetString(PyExc_TypeError, "not a string"); - return nullptr; + SWIG_fail; } resultobj = ustring.input_type(); - if (ustring.IsCopy()) { - (&arg2)->copy(ustring.data(), ustring.size()); - } else { - (&arg2)->assign(ustring.data(), ustring.size()); - } + arg2 = sentencepiece::util::min_string_view(ustring.data(), ustring.size()); } { try { - result = sentencepiece_SentencePieceProcessor_set_decode_extra_options(arg1,arg2); + result = sentencepiece_SentencePieceProcessor_set_decode_extra_options(arg1,arg2); + ReleaseResultObject(resultobj); } catch (const sentencepiece::util::Status &status) { SWIG_exception(ToSwigError(status.code()), status.ToString().c_str()); @@ -5533,19 +4897,20 @@ SWIGINTERN PyObject *_wrap_SentencePieceProcessor_set_vocabulary(PyObject *SWIGU (*out)[i] = std::string(ustring.data(), ustring.size()); } else { PyErr_SetString(PyExc_TypeError, "list must contain strings"); - return nullptr; + SWIG_fail; } resultobj = ustring.input_type(); } } else { PyErr_SetString(PyExc_TypeError, "not a list"); - return nullptr; + SWIG_fail; } arg2 = out; } { try { - result = sentencepiece_SentencePieceProcessor_set_vocabulary(arg1,(std::vector< std::string > const &)*arg2); + result = sentencepiece_SentencePieceProcessor_set_vocabulary(arg1,(std::vector< std::string > const &)*arg2); + ReleaseResultObject(resultobj); } catch (const sentencepiece::util::Status &status) { SWIG_exception(ToSwigError(status.code()), status.ToString().c_str()); @@ -5585,7 +4950,8 @@ SWIGINTERN PyObject *_wrap_SentencePieceProcessor_reset_vocabulary(PyObject *SWI arg1 = reinterpret_cast< sentencepiece::SentencePieceProcessor * >(argp1); { try { - result = sentencepiece_SentencePieceProcessor_reset_vocabulary(arg1); + result = sentencepiece_SentencePieceProcessor_reset_vocabulary(arg1); + ReleaseResultObject(resultobj); } catch (const sentencepiece::util::Status &status) { SWIG_exception(ToSwigError(status.code()), status.ToString().c_str()); @@ -5627,14 +4993,10 @@ SWIGINTERN PyObject *_wrap_SentencePieceProcessor_load_vocabulary(PyObject *SWIG const PyInputString ustring(obj1); if (!ustring.IsAvalable()) { PyErr_SetString(PyExc_TypeError, "not a string"); - return nullptr; + SWIG_fail; } resultobj = ustring.input_type(); - if (ustring.IsCopy()) { - (&arg2)->copy(ustring.data(), ustring.size()); - } else { - (&arg2)->assign(ustring.data(), ustring.size()); - } + arg2 = sentencepiece::util::min_string_view(ustring.data(), ustring.size()); } ecode3 = SWIG_AsVal_int(obj2, &val3); if (!SWIG_IsOK(ecode3)) { @@ -5643,7 +5005,8 @@ SWIGINTERN PyObject *_wrap_SentencePieceProcessor_load_vocabulary(PyObject *SWIG arg3 = static_cast< int >(val3); { try { - result = sentencepiece_SentencePieceProcessor_load_vocabulary(arg1,arg2,arg3); + result = sentencepiece_SentencePieceProcessor_load_vocabulary(arg1,arg2,arg3); + ReleaseResultObject(resultobj); } catch (const sentencepiece::util::Status &status) { SWIG_exception(ToSwigError(status.code()), status.ToString().c_str()); @@ -5681,18 +5044,15 @@ SWIGINTERN PyObject *_wrap_SentencePieceProcessor_encode_as_pieces(PyObject *SWI const PyInputString ustring(obj1); if (!ustring.IsAvalable()) { PyErr_SetString(PyExc_TypeError, "not a string"); - return nullptr; + SWIG_fail; } resultobj = ustring.input_type(); - if (ustring.IsCopy()) { - (&arg2)->copy(ustring.data(), ustring.size()); - } else { - (&arg2)->assign(ustring.data(), ustring.size()); - } + arg2 = sentencepiece::util::min_string_view(ustring.data(), ustring.size()); } { try { - result = sentencepiece_SentencePieceProcessor_encode_as_pieces((sentencepiece::SentencePieceProcessor const *)arg1,arg2); + result = sentencepiece_SentencePieceProcessor_encode_as_pieces((sentencepiece::SentencePieceProcessor const *)arg1,arg2); + ReleaseResultObject(resultobj); } catch (const sentencepiece::util::Status &status) { SWIG_exception(ToSwigError(status.code()), status.ToString().c_str()); @@ -5731,18 +5091,15 @@ SWIGINTERN PyObject *_wrap_SentencePieceProcessor_encode_as_ids(PyObject *SWIGUN const PyInputString ustring(obj1); if (!ustring.IsAvalable()) { PyErr_SetString(PyExc_TypeError, "not a string"); - return nullptr; + SWIG_fail; } resultobj = ustring.input_type(); - if (ustring.IsCopy()) { - (&arg2)->copy(ustring.data(), ustring.size()); - } else { - (&arg2)->assign(ustring.data(), ustring.size()); - } + arg2 = sentencepiece::util::min_string_view(ustring.data(), ustring.size()); } { try { - result = sentencepiece_SentencePieceProcessor_encode_as_ids((sentencepiece::SentencePieceProcessor const *)arg1,arg2); + result = sentencepiece_SentencePieceProcessor_encode_as_ids((sentencepiece::SentencePieceProcessor const *)arg1,arg2); + ReleaseResultObject(resultobj); } catch (const sentencepiece::util::Status &status) { SWIG_exception(ToSwigError(status.code()), status.ToString().c_str()); @@ -5784,14 +5141,10 @@ SWIGINTERN PyObject *_wrap_SentencePieceProcessor_nbest_encode_as_pieces(PyObjec const PyInputString ustring(obj1); if (!ustring.IsAvalable()) { PyErr_SetString(PyExc_TypeError, "not a string"); - return nullptr; + SWIG_fail; } resultobj = ustring.input_type(); - if (ustring.IsCopy()) { - (&arg2)->copy(ustring.data(), ustring.size()); - } else { - (&arg2)->assign(ustring.data(), ustring.size()); - } + arg2 = sentencepiece::util::min_string_view(ustring.data(), ustring.size()); } ecode3 = SWIG_AsVal_int(obj2, &val3); if (!SWIG_IsOK(ecode3)) { @@ -5800,7 +5153,8 @@ SWIGINTERN PyObject *_wrap_SentencePieceProcessor_nbest_encode_as_pieces(PyObjec arg3 = static_cast< int >(val3); { try { - result = sentencepiece_SentencePieceProcessor_nbest_encode_as_pieces((sentencepiece::SentencePieceProcessor const *)arg1,arg2,arg3); + result = sentencepiece_SentencePieceProcessor_nbest_encode_as_pieces((sentencepiece::SentencePieceProcessor const *)arg1,arg2,arg3); + ReleaseResultObject(resultobj); } catch (const sentencepiece::util::Status &status) { SWIG_exception(ToSwigError(status.code()), status.ToString().c_str()); @@ -5847,14 +5201,10 @@ SWIGINTERN PyObject *_wrap_SentencePieceProcessor_nbest_encode_as_ids(PyObject * const PyInputString ustring(obj1); if (!ustring.IsAvalable()) { PyErr_SetString(PyExc_TypeError, "not a string"); - return nullptr; + SWIG_fail; } resultobj = ustring.input_type(); - if (ustring.IsCopy()) { - (&arg2)->copy(ustring.data(), ustring.size()); - } else { - (&arg2)->assign(ustring.data(), ustring.size()); - } + arg2 = sentencepiece::util::min_string_view(ustring.data(), ustring.size()); } ecode3 = SWIG_AsVal_int(obj2, &val3); if (!SWIG_IsOK(ecode3)) { @@ -5863,7 +5213,8 @@ SWIGINTERN PyObject *_wrap_SentencePieceProcessor_nbest_encode_as_ids(PyObject * arg3 = static_cast< int >(val3); { try { - result = sentencepiece_SentencePieceProcessor_nbest_encode_as_ids((sentencepiece::SentencePieceProcessor const *)arg1,arg2,arg3); + result = sentencepiece_SentencePieceProcessor_nbest_encode_as_ids((sentencepiece::SentencePieceProcessor const *)arg1,arg2,arg3); + ReleaseResultObject(resultobj); } catch (const sentencepiece::util::Status &status) { SWIG_exception(ToSwigError(status.code()), status.ToString().c_str()); @@ -5913,14 +5264,10 @@ SWIGINTERN PyObject *_wrap_SentencePieceProcessor_sample_encode_as_pieces(PyObje const PyInputString ustring(obj1); if (!ustring.IsAvalable()) { PyErr_SetString(PyExc_TypeError, "not a string"); - return nullptr; + SWIG_fail; } resultobj = ustring.input_type(); - if (ustring.IsCopy()) { - (&arg2)->copy(ustring.data(), ustring.size()); - } else { - (&arg2)->assign(ustring.data(), ustring.size()); - } + arg2 = sentencepiece::util::min_string_view(ustring.data(), ustring.size()); } ecode3 = SWIG_AsVal_int(obj2, &val3); if (!SWIG_IsOK(ecode3)) { @@ -5934,7 +5281,8 @@ SWIGINTERN PyObject *_wrap_SentencePieceProcessor_sample_encode_as_pieces(PyObje arg4 = static_cast< float >(val4); { try { - result = sentencepiece_SentencePieceProcessor_sample_encode_as_pieces((sentencepiece::SentencePieceProcessor const *)arg1,arg2,arg3,arg4); + result = sentencepiece_SentencePieceProcessor_sample_encode_as_pieces((sentencepiece::SentencePieceProcessor const *)arg1,arg2,arg3,arg4); + ReleaseResultObject(resultobj); } catch (const sentencepiece::util::Status &status) { SWIG_exception(ToSwigError(status.code()), status.ToString().c_str()); @@ -5981,14 +5329,10 @@ SWIGINTERN PyObject *_wrap_SentencePieceProcessor_sample_encode_as_ids(PyObject const PyInputString ustring(obj1); if (!ustring.IsAvalable()) { PyErr_SetString(PyExc_TypeError, "not a string"); - return nullptr; + SWIG_fail; } resultobj = ustring.input_type(); - if (ustring.IsCopy()) { - (&arg2)->copy(ustring.data(), ustring.size()); - } else { - (&arg2)->assign(ustring.data(), ustring.size()); - } + arg2 = sentencepiece::util::min_string_view(ustring.data(), ustring.size()); } ecode3 = SWIG_AsVal_int(obj2, &val3); if (!SWIG_IsOK(ecode3)) { @@ -6002,7 +5346,8 @@ SWIGINTERN PyObject *_wrap_SentencePieceProcessor_sample_encode_as_ids(PyObject arg4 = static_cast< float >(val4); { try { - result = sentencepiece_SentencePieceProcessor_sample_encode_as_ids((sentencepiece::SentencePieceProcessor const *)arg1,arg2,arg3,arg4); + result = sentencepiece_SentencePieceProcessor_sample_encode_as_ids((sentencepiece::SentencePieceProcessor const *)arg1,arg2,arg3,arg4); + ReleaseResultObject(resultobj); } catch (const sentencepiece::util::Status &status) { SWIG_exception(ToSwigError(status.code()), status.ToString().c_str()); @@ -6047,19 +5392,20 @@ SWIGINTERN PyObject *_wrap_SentencePieceProcessor_decode_pieces(PyObject *SWIGUN (*out)[i] = std::string(ustring.data(), ustring.size()); } else { PyErr_SetString(PyExc_TypeError, "list must contain strings"); - return nullptr; + SWIG_fail; } resultobj = ustring.input_type(); } } else { PyErr_SetString(PyExc_TypeError, "not a list"); - return nullptr; + SWIG_fail; } arg2 = out; } { try { - result = sentencepiece_SentencePieceProcessor_decode_pieces((sentencepiece::SentencePieceProcessor const *)arg1,(std::vector< std::string > const &)*arg2); + result = sentencepiece_SentencePieceProcessor_decode_pieces((sentencepiece::SentencePieceProcessor const *)arg1,(std::vector< std::string > const &)*arg2); + ReleaseResultObject(resultobj); } catch (const sentencepiece::util::Status &status) { SWIG_exception(ToSwigError(status.code()), status.ToString().c_str()); @@ -6108,18 +5454,19 @@ SWIGINTERN PyObject *_wrap_SentencePieceProcessor_decode_ids(PyObject *SWIGUNUSE (*out)[i] = static_cast(PyInt_AsLong(o)); } else { PyErr_SetString(PyExc_TypeError,"list must contain integers"); - return nullptr; + SWIG_fail; } } } else { PyErr_SetString(PyExc_TypeError,"not a list"); - return nullptr; + SWIG_fail; } arg2 = out; } { try { - result = sentencepiece_SentencePieceProcessor_decode_ids((sentencepiece::SentencePieceProcessor const *)arg1,(std::vector< int > const &)*arg2); + result = sentencepiece_SentencePieceProcessor_decode_ids((sentencepiece::SentencePieceProcessor const *)arg1,(std::vector< int > const &)*arg2); + ReleaseResultObject(resultobj); } catch (const sentencepiece::util::Status &status) { SWIG_exception(ToSwigError(status.code()), status.ToString().c_str()); @@ -6157,7 +5504,8 @@ SWIGINTERN PyObject *_wrap_SentencePieceProcessor_get_piece_size(PyObject *SWIGU arg1 = reinterpret_cast< sentencepiece::SentencePieceProcessor * >(argp1); { try { - result = (int)sentencepiece_SentencePieceProcessor_get_piece_size((sentencepiece::SentencePieceProcessor const *)arg1); + result = (int)sentencepiece_SentencePieceProcessor_get_piece_size((sentencepiece::SentencePieceProcessor const *)arg1); + ReleaseResultObject(resultobj); } catch (const sentencepiece::util::Status &status) { SWIG_exception(ToSwigError(status.code()), status.ToString().c_str()); @@ -6190,18 +5538,15 @@ SWIGINTERN PyObject *_wrap_SentencePieceProcessor_piece_to_id(PyObject *SWIGUNUS const PyInputString ustring(obj1); if (!ustring.IsAvalable()) { PyErr_SetString(PyExc_TypeError, "not a string"); - return nullptr; + SWIG_fail; } resultobj = ustring.input_type(); - if (ustring.IsCopy()) { - (&arg2)->copy(ustring.data(), ustring.size()); - } else { - (&arg2)->assign(ustring.data(), ustring.size()); - } + arg2 = sentencepiece::util::min_string_view(ustring.data(), ustring.size()); } { try { - result = (int)sentencepiece_SentencePieceProcessor_piece_to_id((sentencepiece::SentencePieceProcessor const *)arg1,arg2); + result = (int)sentencepiece_SentencePieceProcessor_piece_to_id((sentencepiece::SentencePieceProcessor const *)arg1,arg2); + ReleaseResultObject(resultobj); } catch (const sentencepiece::util::Status &status) { SWIG_exception(ToSwigError(status.code()), status.ToString().c_str()); @@ -6239,7 +5584,8 @@ SWIGINTERN PyObject *_wrap_SentencePieceProcessor_id_to_piece(PyObject *SWIGUNUS arg2 = static_cast< int >(val2); { try { - result = sentencepiece_SentencePieceProcessor_id_to_piece((sentencepiece::SentencePieceProcessor const *)arg1,arg2); + result = sentencepiece_SentencePieceProcessor_id_to_piece((sentencepiece::SentencePieceProcessor const *)arg1,arg2); + ReleaseResultObject(resultobj); } catch (const sentencepiece::util::Status &status) { SWIG_exception(ToSwigError(status.code()), status.ToString().c_str()); @@ -6280,7 +5626,8 @@ SWIGINTERN PyObject *_wrap_SentencePieceProcessor_get_score(PyObject *SWIGUNUSED arg2 = static_cast< int >(val2); { try { - result = (float)sentencepiece_SentencePieceProcessor_get_score((sentencepiece::SentencePieceProcessor const *)arg1,arg2); + result = (float)sentencepiece_SentencePieceProcessor_get_score((sentencepiece::SentencePieceProcessor const *)arg1,arg2); + ReleaseResultObject(resultobj); } catch (const sentencepiece::util::Status &status) { SWIG_exception(ToSwigError(status.code()), status.ToString().c_str()); @@ -6318,7 +5665,8 @@ SWIGINTERN PyObject *_wrap_SentencePieceProcessor_is_unknown(PyObject *SWIGUNUSE arg2 = static_cast< int >(val2); { try { - result = (bool)sentencepiece_SentencePieceProcessor_is_unknown((sentencepiece::SentencePieceProcessor const *)arg1,arg2); + result = (bool)sentencepiece_SentencePieceProcessor_is_unknown((sentencepiece::SentencePieceProcessor const *)arg1,arg2); + ReleaseResultObject(resultobj); } catch (const sentencepiece::util::Status &status) { SWIG_exception(ToSwigError(status.code()), status.ToString().c_str()); @@ -6356,7 +5704,8 @@ SWIGINTERN PyObject *_wrap_SentencePieceProcessor_is_control(PyObject *SWIGUNUSE arg2 = static_cast< int >(val2); { try { - result = (bool)sentencepiece_SentencePieceProcessor_is_control((sentencepiece::SentencePieceProcessor const *)arg1,arg2); + result = (bool)sentencepiece_SentencePieceProcessor_is_control((sentencepiece::SentencePieceProcessor const *)arg1,arg2); + ReleaseResultObject(resultobj); } catch (const sentencepiece::util::Status &status) { SWIG_exception(ToSwigError(status.code()), status.ToString().c_str()); @@ -6394,7 +5743,8 @@ SWIGINTERN PyObject *_wrap_SentencePieceProcessor_is_unused(PyObject *SWIGUNUSED arg2 = static_cast< int >(val2); { try { - result = (bool)sentencepiece_SentencePieceProcessor_is_unused((sentencepiece::SentencePieceProcessor const *)arg1,arg2); + result = (bool)sentencepiece_SentencePieceProcessor_is_unused((sentencepiece::SentencePieceProcessor const *)arg1,arg2); + ReleaseResultObject(resultobj); } catch (const sentencepiece::util::Status &status) { SWIG_exception(ToSwigError(status.code()), status.ToString().c_str()); @@ -6423,7 +5773,8 @@ SWIGINTERN PyObject *_wrap_SentencePieceProcessor___len__(PyObject *SWIGUNUSEDPA arg1 = reinterpret_cast< sentencepiece::SentencePieceProcessor * >(argp1); { try { - result = (int)sentencepiece_SentencePieceProcessor___len__(arg1); + result = (int)sentencepiece_SentencePieceProcessor___len__(arg1); + ReleaseResultObject(resultobj); } catch (const sentencepiece::util::Status &status) { SWIG_exception(ToSwigError(status.code()), status.ToString().c_str()); @@ -6456,18 +5807,15 @@ SWIGINTERN PyObject *_wrap_SentencePieceProcessor___getitem__(PyObject *SWIGUNUS const PyInputString ustring(obj1); if (!ustring.IsAvalable()) { PyErr_SetString(PyExc_TypeError, "not a string"); - return nullptr; + SWIG_fail; } resultobj = ustring.input_type(); - if (ustring.IsCopy()) { - (&arg2)->copy(ustring.data(), ustring.size()); - } else { - (&arg2)->assign(ustring.data(), ustring.size()); - } + arg2 = sentencepiece::util::min_string_view(ustring.data(), ustring.size()); } { try { - result = (int)sentencepiece_SentencePieceProcessor___getitem__((sentencepiece::SentencePieceProcessor const *)arg1,arg2); + result = (int)sentencepiece_SentencePieceProcessor___getitem__((sentencepiece::SentencePieceProcessor const *)arg1,arg2); + ReleaseResultObject(resultobj); } catch (const sentencepiece::util::Status &status) { SWIG_exception(ToSwigError(status.code()), status.ToString().c_str()); @@ -6498,18 +5846,15 @@ SWIGINTERN PyObject *_wrap_SentencePieceTrainer_Train(PyObject *SWIGUNUSEDPARM(s const PyInputString ustring(obj0); if (!ustring.IsAvalable()) { PyErr_SetString(PyExc_TypeError, "not a string"); - return nullptr; + SWIG_fail; } resultobj = ustring.input_type(); - if (ustring.IsCopy()) { - (&arg1)->copy(ustring.data(), ustring.size()); - } else { - (&arg1)->assign(ustring.data(), ustring.size()); - } + arg1 = sentencepiece::util::min_string_view(ustring.data(), ustring.size()); } { try { - result = sentencepiece::SentencePieceTrainer::Train(arg1); + result = sentencepiece::SentencePieceTrainer::Train(arg1); + ReleaseResultObject(resultobj); } catch (const sentencepiece::util::Status &status) { SWIG_exception(ToSwigError(status.code()), status.ToString().c_str()); @@ -6538,18 +5883,15 @@ SWIGINTERN PyObject *_wrap_SentencePieceTrainer_train(PyObject *SWIGUNUSEDPARM(s const PyInputString ustring(obj0); if (!ustring.IsAvalable()) { PyErr_SetString(PyExc_TypeError, "not a string"); - return nullptr; + SWIG_fail; } resultobj = ustring.input_type(); - if (ustring.IsCopy()) { - (&arg1)->copy(ustring.data(), ustring.size()); - } else { - (&arg1)->assign(ustring.data(), ustring.size()); - } + arg1 = sentencepiece::util::min_string_view(ustring.data(), ustring.size()); } { try { - result = sentencepiece_SentencePieceTrainer_train(arg1); + result = sentencepiece_SentencePieceTrainer_train(arg1); + ReleaseResultObject(resultobj); } catch (const sentencepiece::util::Status &status) { SWIG_exception(ToSwigError(status.code()), status.ToString().c_str()); @@ -6576,13 +5918,6 @@ SWIGINTERN PyObject *SentencePieceTrainer_swigregister(PyObject *SWIGUNUSEDPARM( static PyMethodDef SwigMethods[] = { { (char *)"SWIG_PyInstanceMethod_New", (PyCFunction)SWIG_PyInstanceMethod_New, METH_O, NULL}, - { (char *)"min_string_view_data", _wrap_min_string_view_data, METH_VARARGS, NULL}, - { (char *)"min_string_view_size", _wrap_min_string_view_size, METH_VARARGS, NULL}, - { (char *)"min_string_view_assign", _wrap_min_string_view_assign, METH_VARARGS, NULL}, - { (char *)"min_string_view_copy", _wrap_min_string_view_copy, METH_VARARGS, NULL}, - { (char *)"new_min_string_view", _wrap_new_min_string_view, METH_VARARGS, NULL}, - { (char *)"delete_min_string_view", _wrap_delete_min_string_view, METH_VARARGS, NULL}, - { (char *)"min_string_view_swigregister", min_string_view_swigregister, METH_VARARGS, NULL}, { (char *)"new_SentencePieceProcessor", _wrap_new_SentencePieceProcessor, METH_VARARGS, NULL}, { (char *)"delete_SentencePieceProcessor", _wrap_delete_SentencePieceProcessor, METH_VARARGS, NULL}, { (char *)"SentencePieceProcessor_Load", _wrap_SentencePieceProcessor_Load, METH_VARARGS, NULL}, @@ -6643,8 +5978,6 @@ static PyMethodDef SwigMethods[] = { static swig_type_info _swigt__p_char = {"_p_char", "char *", 0, 0, (void*)0, 0}; static swig_type_info _swigt__p_sentencepiece__SentencePieceProcessor = {"_p_sentencepiece__SentencePieceProcessor", "sentencepiece::SentencePieceProcessor *", 0, 0, (void*)0, 0}; static swig_type_info _swigt__p_sentencepiece__SentencePieceTrainer = {"_p_sentencepiece__SentencePieceTrainer", "sentencepiece::SentencePieceTrainer *", 0, 0, (void*)0, 0}; -static swig_type_info _swigt__p_sentencepiece__util__min_string_view = {"_p_sentencepiece__util__min_string_view", "sentencepiece::util::min_string_view *", 0, 0, (void*)0, 0}; -static swig_type_info _swigt__p_std__string = {"_p_std__string", "std::string *", 0, 0, (void*)0, 0}; static swig_type_info _swigt__p_std__vectorT_int_t = {"_p_std__vectorT_int_t", "std::vector< int > *", 0, 0, (void*)0, 0}; static swig_type_info _swigt__p_std__vectorT_std__string_t = {"_p_std__vectorT_std__string_t", "std::vector< std::string > *", 0, 0, (void*)0, 0}; @@ -6652,8 +5985,6 @@ static swig_type_info *swig_type_initial[] = { &_swigt__p_char, &_swigt__p_sentencepiece__SentencePieceProcessor, &_swigt__p_sentencepiece__SentencePieceTrainer, - &_swigt__p_sentencepiece__util__min_string_view, - &_swigt__p_std__string, &_swigt__p_std__vectorT_int_t, &_swigt__p_std__vectorT_std__string_t, }; @@ -6661,8 +5992,6 @@ static swig_type_info *swig_type_initial[] = { static swig_cast_info _swigc__p_char[] = { {&_swigt__p_char, 0, 0, 0},{0, 0, 0, 0}}; static swig_cast_info _swigc__p_sentencepiece__SentencePieceProcessor[] = { {&_swigt__p_sentencepiece__SentencePieceProcessor, 0, 0, 0},{0, 0, 0, 0}}; static swig_cast_info _swigc__p_sentencepiece__SentencePieceTrainer[] = { {&_swigt__p_sentencepiece__SentencePieceTrainer, 0, 0, 0},{0, 0, 0, 0}}; -static swig_cast_info _swigc__p_sentencepiece__util__min_string_view[] = { {&_swigt__p_sentencepiece__util__min_string_view, 0, 0, 0},{0, 0, 0, 0}}; -static swig_cast_info _swigc__p_std__string[] = { {&_swigt__p_std__string, 0, 0, 0},{0, 0, 0, 0}}; static swig_cast_info _swigc__p_std__vectorT_int_t[] = { {&_swigt__p_std__vectorT_int_t, 0, 0, 0},{0, 0, 0, 0}}; static swig_cast_info _swigc__p_std__vectorT_std__string_t[] = { {&_swigt__p_std__vectorT_std__string_t, 0, 0, 0},{0, 0, 0, 0}}; @@ -6670,8 +5999,6 @@ static swig_cast_info *swig_cast_initial[] = { _swigc__p_char, _swigc__p_sentencepiece__SentencePieceProcessor, _swigc__p_sentencepiece__SentencePieceTrainer, - _swigc__p_sentencepiece__util__min_string_view, - _swigc__p_std__string, _swigc__p_std__vectorT_int_t, _swigc__p_std__vectorT_std__string_t, }; diff --git a/src/sentencepiece_processor.h b/src/sentencepiece_processor.h index 07f113c..e0a591f 100644 --- a/src/sentencepiece_processor.h +++ b/src/sentencepiece_processor.h @@ -145,40 +145,18 @@ class Status { // the argument of public APIs. class min_string_view { public: - min_string_view() noexcept : ptr_(nullptr), length_(0) {} + min_string_view() : ptr_(nullptr), length_(0) {} min_string_view(const std::string &str) : ptr_(str.data()), length_(str.size()) {} - min_string_view(const char *str) - : ptr_(str), length_(std::strlen(str)) {} - min_string_view(const char *data, size_t len) - : ptr_(data), length_(len) {} + min_string_view(const char *str) : ptr_(str), length_(std::strlen(str)) {} + min_string_view(const char *data, size_t len) : ptr_(data), length_(len) {} + const char *data() const { return ptr_; } size_t size() const { return length_; } - void assign(const char *data, size_t len) { - ptr_ = data; - length_ = len; - } - - void copy(const char *data, size_t len) { - rep_.reset(new std::string(data, len)); - ptr_ = rep_->data(); - length_ = rep_->size(); - } - - min_string_view(const min_string_view &s) { - if (s.rep_) { - copy(s.rep_->data(), s.rep_->size()); - } else { - ptr_ = s.data(); - length_ = s.size(); - } - } - private: const char *ptr_ = nullptr; size_t length_ = 0; - std::unique_ptr rep_; }; } // namespace util