Update scripts for latest grpc, 0.12.

Note we now use the gRPC protobuf submodule for install,
to be sure its version matches gRPC.

bootstrap.sh

@@ -56,24 +56,8 @@ else
   touch $zk_dist/.build_finished
 fi
 
-# install protoc and proto python libraries
-protobuf_dist=$VTROOT/dist/protobuf
-if [ $SKIP_ROOT_INSTALLS == "True" ]; then
-  echo "skipping protobuf build, as root version was already installed."
-elif [ -f $protobuf_dist/.build_finished ]; then
-  echo "skipping protobuf build. remove $protobuf_dist to force rebuild."
-else
-  rm -rf $protobuf_dist
-  mkdir -p $protobuf_dist/lib/python2.7/site-packages
-  # The directory may not have existed yet, so it may not have been
-  # picked up by dev.env yet, but the install needs it to exist first,
-  # and be in PYTHONPATH.
-  export PYTHONPATH=$(prepend_path $PYTHONPATH $protobuf_dist/lib/python2.7/site-packages)
-  ./travis/install_protobuf.sh $protobuf_dist || fail "protobuf build failed"
-  touch $protobuf_dist/.build_finished
-fi
-
-# install gRPC C++ base, so we can install the python adapters
+# install gRPC C++ base, so we can install the python adapters.
+# this also installs protobufs
 grpc_dist=$VTROOT/dist/grpc
 if [ $SKIP_ROOT_INSTALLS == "True" ]; then
   echo "skipping grpc build, as root version was already installed."
@@ -81,7 +65,14 @@ elif [ -f $grpc_dist/.build_finished ]; then
   echo "skipping gRPC build. remove $grpc_dist to force rebuild."
 else
   rm -rf $grpc_dist
-  mkdir -p $grpc_dist
+  mkdir -p $grpc_dist/usr/local/bin
+  mkdir -p $grpc_dist/usr/local/lib/python2.7/dist-packages
+  # The directory may not have existed yet, so it may not have been
+  # picked up by dev.env yet, but the install needs it to be in
+  # PYTHONPATH.
+  export PYTHONPATH=$(prepend_path $PYTHONPATH $grpc_dist/usr/local/lib/python2.7/dist-packages)
+  export PATH=$(prepend_path $PATH $grpc_dist/usr/local/bin)
+  export LD_LIBRARY_PATH=$(prepend_path $LD_LIBRARY_PATH $grpc_dist/usr/local/lib)
   ./travis/install_grpc.sh $grpc_dist || fail "gRPC build failed"
   touch $grpc_dist/.build_finished
 fi

dev.env

@@ -30,7 +30,7 @@ function prepend_path()
   fi
 }
 
-for pypath in $(find $VTROOT/dist -name site-packages)
+for pypath in $(find $VTROOT/dist -name site-packages -or -name dist-packages | grep -v src/python/grpcio/.tox/py27/lib/python2.7/site-packages)
 do
   export PYTHONPATH=$(prepend_path $PYTHONPATH $pypath)
 done
@@ -102,7 +102,7 @@ export LD_LIBRARY_PATH=$(prepend_path $LD_LIBRARY_PATH $VTROOT/dist/vt-zookeeper
 # needed to correctly import grpc if it's not installed globally
 grpc_dist=$VTROOT/dist/grpc
 if [ -f $grpc_dist/.build_finished ]; then
-  export LD_LIBRARY_PATH=$(prepend_path $LD_LIBRARY_PATH $grpc_dist/lib)
+  export LD_LIBRARY_PATH=$(prepend_path $LD_LIBRARY_PATH $grpc_dist/usr/local/lib)
 fi
 
 export GOPATH=$(prepend_path $GOPATH $VTROOT)

travis/install_grpc.sh

@@ -11,19 +11,55 @@ if [ "$grpc_dist" != "" ]; then
   cd $grpc_dist
 fi
 
+# for python, we'll need the latest virtualenv and tox
+if [ "$grpc_dist" != "" ]; then
+  pip install --upgrade --root $grpc_dist virtualenv tox
+else
+  pip install --upgrade virtualenv tox
+fi
+
+# clone the repository, setup the submodules
 git clone https://github.com/grpc/grpc.git
 cd grpc
-git checkout 4831d02cc2341ec2233ff9d9ef66fb9a86138fb7 # Beta Release 0.11.1
+git checkout release-0_12_0
 git submodule update --init
+
+# build everything
 make
+
+# install protobuf side (it was already built by the 'make' earlier)
+cd third_party/protobuf
 if [ "$grpc_dist" != "" ]; then
-  make install prefix=$grpc_dist
+  make install prefix=$grpc_dist/usr/local
 else
   make install
 fi
-CONFIG=opt ./tools/run_tests/build_python.sh
+
+# build and install python protobuf side
+cd python
 if [ "$grpc_dist" != "" ]; then
-  CFLAGS=-I$grpc_dist/include LDFLAGS=-L$grpc_dist/lib pip install src/python/grpcio -t $grpc_dist/lib/python2.7/site-packages
+  python setup.py build --cpp_implementation
+  python setup.py install --cpp_implementation --root=$grpc_dist
+else
+  python setup.py build --cpp_implementation
+  python setup.py install --cpp_implementation
+fi
+
+# now install grpc itself
+cd ../../..
+if [ "$grpc_dist" != "" ]; then
+  make install prefix=$grpc_dist/usr/local
+else
+  make install
+fi
+
+# and now build and install gRPC python libraries
+# Note: running this twice as the first run exists
+# with 'build_data' not found error. Seems the python
+# libraries still work though.
+CONFIG=opt ./tools/run_tests/build_python.sh || CONFIG=opt ./tools/run_tests/build_python.sh
+if [ "$grpc_dist" != "" ]; then
+  CFLAGS=-I$grpc_dist/include LDFLAGS=-L$grpc_dist/lib pip install src/python/grpcio --root $grpc_dist
 else
   pip install src/python/grpcio
 fi