Merge pull request #174 from google/sr

Added make_py_wheel_mac.py

.travis.yml

@@ -43,6 +43,11 @@ matrix:
     - os: osx
       osx_image: xcode9.2
       env: IMAGE=native COMMAND=build_osx
+    - os: osx
+      osx_image: xcode9.4
+      env: IMAGE=native COMMAND=make_py_wheel_mac
+      script:
+        - $TRAVIS_BUILD_DIR/python/make_py_wheel_mac.sh
 script:
   - $TRAVIS_BUILD_DIR/test.sh ${IMAGE} ${COMMAND}
 env:

CMakeLists.txt

@@ -24,6 +24,7 @@ option(SPM_COVERAGE "Runs gcov to test coverage." OFF)
 option(SPM_ENABLE_TENSORFLOW_SHARED "Makes a tensorflow compatible shared file." OFF)
 option(SPM_ENABLE_TCMALLOC "Enable TCMalloc if available." ON)
 option(SPM_TCMALLOC_STATIC "Link static library of TCMALLOC." OFF)
+option(SPM_NO_THREADLOCAL "Disable thread_local operator" OFF)
 
 set(CMAKE_CXX_STANDARD 11)
 set(CMAKE_CXX_STANDARD_REQUIRED ON)

python/make_py_wheel_mac.sh

@@ -0,0 +1,94 @@
+#!/bin/bash
+# Copyright 2018 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.!
+
+set -e  # exit immediately on error
+set -x  # display all commands
+
+PROTOBUF_VERSION=3.6.1
+
+build_python() {
+  VERSION=$1
+  URL=$2
+  INSTALL_PATH="/Library/Frameworks/Python.framework/Versions/${VERSION}/bin"
+  CURRENT_PATH=${PATH}
+
+  curl -L -o python.pkg ${URL}
+  sudo installer -pkg python.pkg -target /
+
+  if [ -f "${INSTALL_PATH}/python3" ]; then
+    ln -s ${INSTALL_PATH}/python3        ${INSTALL_PATH}/python
+    ln -s ${INSTALL_PATH}/python3-config ${INSTALL_PATH}/python-config
+    ln -s ${INSTALL_PATH}/pip3           ${INSTALL_PATH}/pip
+  fi
+
+  export PATH="${INSTALL_PATH}:${CURRENT_PATH}"
+  ls -l ${INSTALL_PATH}
+  which python
+  which pip
+  python --version
+  sudo python get-pip.py --no-setuptools --no-wheel --ignore-installed
+  pip install --upgrade setuptools
+  pip install wheel
+  pip install delocate
+  python setup.py bdist_wheel --plat-name=macosx_10_6_x86_64
+  python setup.py test
+  delocate-listdeps dist/*.whl
+  delocate-wheel -w dist/delocated_wheel dist/*.whl
+  export PATH="${CURRENT_PATH}"
+
+  ls -l dist/delocated_wheel
+  rm -fr build
+  rm -fr *.so
+  rm -fr dist/*.whl
+  rm -fr python.pkg
+}
+
+build() {
+  cd python
+  rm -fr build
+  mkdir -p build
+  cd build
+
+  # Install protobuf
+  curl -L -O https://github.com/google/protobuf/releases/download/v${PROTOBUF_VERSION}/protobuf-cpp-${PROTOBUF_VERSION}.tar.gz
+  tar zxfv protobuf-cpp-${PROTOBUF_VERSION}.tar.gz
+  cd protobuf-${PROTOBUF_VERSION}
+  ./configure --disable-shared --with-pic
+  make CXXFLAGS+="-std=c++11 -O3 -DGOOGLE_PROTOBUF_NO_THREADLOCAL=1" \
+    CFLAGS+="-std=c++11 -O3 -DGOOGLE_PROTOBUF_NO_THREADLOCAL=1" -j4
+  make install || true
+  cd ..
+
+  # Install sentencepiece
+  cmake ../.. -DSPM_ENABLE_SHARED=OFF -DSPM_NO_THREADLOCAL=ON
+  make -j4 VERBOSE=1
+  make install
+  cd ..
+
+  mkdir -p dist/delocated_wheel
+  curl -L -O https://bootstrap.pypa.io/get-pip.py
+
+  build_python 2.7 https://www.python.org/ftp/python/2.7.15/python-2.7.15-macosx10.6.pkg
+  build_python 3.4 https://www.python.org/ftp/python/3.4.4/python-3.4.4-macosx10.6.pkg
+  build_python 3.5 https://www.python.org/ftp/python/3.5.4/python-3.5.4-macosx10.6.pkg
+  build_python 3.6 https://www.python.org/ftp/python/3.6.6/python-3.6.6-macosx10.6.pkg
+  build_python 3.7 https://www.python.org/ftp/python/3.7.0/python-3.7.0-macosx10.6.pkg
+
+  cd ..
+
+  rm -fr build
+}
+
+build

src/CMakeLists.txt

@@ -160,7 +160,7 @@ endif()
 set_target_properties(sentencepiece-static PROPERTIES OUTPUT_NAME "sentencepiece")
 set_target_properties(sentencepiece_train-static PROPERTIES OUTPUT_NAME "sentencepiece_train")
 
-if ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang" OR "${CMAKE_CXX_COMPILER_ID}" STREQUAL "GNU")
+if (NOT MSVC)
   if (SPM_COVERAGE)
     set(CMAKE_CXX_FLAGS "-O0 -Wall -fPIC -coverage ${CMAKE_CXX_FLAGS}")
   else()
@@ -169,6 +169,9 @@ if ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang" OR "${CMAKE_CXX_COMPILER_ID}" ST
   if (SPM_ENABLE_TENSORFLOW_SHARED)
     add_definitions(-D_GLIBCXX_USE_CXX11_ABI=0)
   endif()
+  if (SPM_NO_THREADLOCAL)
+    add_definitions(-DSPM_NO_THREADLOCAL=1)
+  endif()
   set_source_files_properties(
     sentencepiece.pb.cc sentencepiece_model.pb.cc
     PROPERTIES COMPILE_FLAGS "-Wno-misleading-indentation")

src/sentencepiece_processor.cc

@@ -15,7 +15,6 @@
 #include "sentencepiece_processor.h"
 
 #include <map>
-#include <random>
 #include <set>
 #include <utility>
 
@@ -412,10 +411,10 @@ util::Status SentencePieceProcessor::SampleEncode(
       probs[i] = std::exp(alpha * nbests[i].second);
     }
 
-    thread_local static std::mt19937 mt(std::random_device{}());
+    auto *mt = random::GetRandomGenerator();
     std::discrete_distribution<int> dist(probs.begin(), probs.end());
     RETURN_IF_ERROR(PopulateSentencePieceText(input, normalized, norm_to_orig,
-                                              nbests[dist(mt)].first, spt));
+                                              nbests[dist(*mt)].first, spt));
 
   } else if (nbest_size < 0) {
     const auto result = model_->SampleEncode(normalized, alpha);

src/unigram_model.cc

@@ -19,7 +19,6 @@
 #include <cmath>
 #include <map>
 #include <queue>
-#include <random>
 #include <string>
 #include <utility>
 #include <vector>
@@ -345,7 +344,7 @@ std::vector<Lattice::Node *> Lattice::Sample(float theta) {
     }
   }
 
-  thread_local static std::mt19937 mt(std::random_device{}());
+  auto *mt = random::GetRandomGenerator();
 
   std::vector<Node *> results;
   std::vector<float> probs;
@@ -358,7 +357,7 @@ std::vector<Lattice::Node *> Lattice::Sample(float theta) {
       probs.push_back(exp(alpha[lnode->node_id] + theta * lnode->score - Z));
     }
     std::discrete_distribution<int> dist(probs.begin(), probs.end());
-    node = end_nodes_[node->pos][dist(mt)];
+    node = end_nodes_[node->pos][dist(*mt)];
     if (node == bos_node()) break;
 
     Z = alpha[node->node_id];

src/util.cc

@@ -264,6 +264,43 @@ bool OutputBuffer::WriteLine(absl::string_view text) {
 }
 }  // namespace io
 
+namespace random {
+#ifdef SPM_NO_THREADLOCAL
+namespace {
+class RandomGeneratorStorage {
+ public:
+  RandomGeneratorStorage() {
+    pthread_key_create(&key_, &RandomGeneratorStorage::Delete);
+  }
+  virtual ~RandomGeneratorStorage() { pthread_key_delete(key_); }
+
+  std::mt19937 *Get() {
+    auto *result = static_cast<std::mt19937 *>(pthread_getspecific(key_));
+    if (result == nullptr) {
+      result = new std::mt19937(std::random_device{}());
+      pthread_setspecific(key_, result);
+    }
+    return result;
+  }
+
+ private:
+  static void Delete(void *value) { delete static_cast<std::mt19937 *>(value); }
+  pthread_key_t key_;
+};
+}  // namespace
+
+std::mt19937 *GetRandomGenerator() {
+  static RandomGeneratorStorage *storage = new RandomGeneratorStorage;
+  return storage->Get();
+}
+#else
+std::mt19937 *GetRandomGenerator() {
+  thread_local static std::mt19937 mt(std::random_device{}());
+  return &mt;
+}
+#endif
+}  // namespace random
+
 namespace util {
 
 std::string StrError(int errnum) {

src/util.h

@@ -21,6 +21,7 @@
 #include <algorithm>
 #include <fstream>
 #include <memory>
+#include <random>
 #include <sstream>
 #include <string>
 #include <utility>
@@ -29,6 +30,10 @@
 #include "sentencepiece_processor.h"
 #include "third_party/absl/strings/string_view.h"
 
+#ifdef SPM_NO_THREADLOCAL
+#include <pthread.h>
+#endif
+
 namespace sentencepiece {
 
 template <typename T>
@@ -423,6 +428,12 @@ void STLDeleteElements(std::vector<T *> *vec) {
 }
 }  // namespace port
 
+namespace random {
+
+std::mt19937 *GetRandomGenerator();
+
+}  // namespace random
+
 namespace util {
 
 inline std::string JoinPath(absl::string_view path) {