Travis build matrix to do parallel builds for make and CMake; CUDA-less

and CUDA-ful. Move bits of functionality into scripts under scripts/travis for readability. Only generate CUDA compute_50 for perfomance.
2014-08-17 03:11:59 -07:00 · 2014-08-17 03:11:59 -07:00 · a909f916e2
--- a/.travis.yml
+++ b/.travis.yml
@ -1,76 +1,34 @@
+# Use a build matrix to do two builds in parallel:
+# one using CMake, and one using make.
+env:
+  matrix:
+    - WITH_CUDA=false WITH_CMAKE=false
+    - WITH_CUDA=false WITH_CMAKE=true
+    - WITH_CUDA=true WITH_CMAKE=false
+    - WITH_CUDA=true WITH_CMAKE=true
+
 language: cpp

 # Cache Ubuntu apt packages.
 cache: apt

-compiler:
-  - gcc
-# Disable clang build: doesn't seem to work on Linux.
-# (@jeffdonahue: Travis buildbot's failure behavior is similar to what I see
-# building on Linux.)
-#   - clang
+compiler: gcc

 before_install:
-  - echo $LANG
-  - echo $LC_ALL
  - export NUM_THREADS=4
  - alias make="make --jobs=$NUM_THREADS"
-  - sudo add-apt-repository ppa:ubuntu-sdk-team/ppa -y
-  - sudo apt-get -y update
-  - sudo apt-get -y install wget git curl python-dev python-numpy libprotobuf-dev libleveldb-dev libsnappy-dev libopencv-dev libboost-dev libboost-system-dev libboost-python-dev libhdf5-serial-dev protobuf-compiler libatlas-dev libatlas-base-dev bc cmake
+  - export SCRIPTS=./scripts/travis
+  - set -e  # fail when a script fails

 install:
-  - wget https://google-glog.googlecode.com/files/glog-0.3.3.tar.gz -O /tmp/glog-0.3.3.tar.gz && tar -C /tmp -xzvf /tmp/glog-0.3.3.tar.gz && rm /tmp/glog-0.3.3.tar.gz
-  - cd /tmp/glog-0.3.3 && ./configure && make && sudo make install && cd -
-  - wget https://github.com/schuhschuh/gflags/archive/master.zip -O /tmp/gflags-master.zip && pushd /tmp/ && unzip gflags-master.zip && cd gflags-master && mkdir build && cd build && export CXXFLAGS="-fPIC" && cmake .. && make VERBOSE=1 && sudo make install && popd
-  - curl http://developer.download.nvidia.com/compute/cuda/repos/ubuntu1204/x86_64/cuda-repo-ubuntu1204_6.0-37_amd64.deb -o /tmp/cuda_install.deb && sudo dpkg -i /tmp/cuda_install.deb && rm /tmp/cuda_install.deb
-  - sudo apt-get -y update
-  # Install the minimal CUDA subpackages required to test Caffe build.
-  # For a full CUDA installation, add 'cuda' to the list of packages.
-  - sudo apt-get -y install cuda-core-6-0 cuda-extra-libs-6-0
-  # Create CUDA symlink at /usr/local/cuda
-  # (This would normally be created by the CUDA installer, but we create it
-  # manually since we did a partial installation.)
-  - sudo ln -s /usr/local/cuda-6.0 /usr/local/cuda
-  - curl https://gitorious.org/mdb/mdb/archive/7f038d0f15bec57b4c07aa3f31cd5564c88a1897.tar.gz -o /tmp/mdb.tar.gz && tar -C /tmp -xzvf /tmp/mdb.tar.gz && rm /tmp/mdb.tar.gz
-  - cd /tmp/mdb-mdb/libraries/liblmdb/ && make && sudo make install && cd -
+  - source $SCRIPTS/travis_install.sh

 before_script:
-  - mv Makefile.config.example Makefile.config
  - export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib
+  - if ! $WITH_CMAKE; then source $SCRIPTS/travis_setup_makefile_config.sh; fi

 script:
-  # CMake build
-  - mkdir build
-  - cd build
-  ## CPU-GPU: build only
-  - cmake -DBUILD_PYTHON=ON -DBUILD_EXAMPLES=ON -DCMAKE_BUILD_TYPE=Release ..
-  - make --keep-going
-  - make clean && rm -rf *
-   ## CPU-only: comprehensive
-  - cmake -DBUILD_PYTHON=ON -DBUILD_EXAMPLES=ON -DCMAKE_BUILD_TYPE=Release -DCPU_ONLY=ON ..
-  - make --keep-going
-  - make runtest
-  - make lint
-  - make clean
-    ## Cleanup CMake build
-  - cd ..
-  - rm -rf build
-
-  # Make build
-  ## CPU-GPU: build only
-  - export CPU_ONLY=0
-  - make --keep-going all
-  - make clean
-  ## CPU-only: comprehensive
-  - export CPU_ONLY=1
-  - make --keep-going all test warn lint
-  - make runtest
-  - make all
-  - make test
-  - make warn
-  - make lint
-  - make pycaffe
+  - if $WITH_CUDA; then source $SCRIPTS/travis_build.sh; else source $SCRIPTS/travis_build_and_test.sh; fi

 notifications:
 # Emails are sent to the committer's git-configured email address by default,
--- a/scripts/travis/travis_build.sh
+++ b/scripts/travis/travis_build.sh
@ -0,0 +1,18 @@
+#!/bin/bash
+# Script called by Travis to do a full build of Caffe,
+# including CUDA functionality.
+
+if $WITH_CMAKE; then
+  mkdir build
+  cd build
+  cmake -DBUILD_PYTHON=ON -DBUILD_EXAMPLES=ON -DCMAKE_BUILD_TYPE=Release ..
+  make --keep-going
+  cd -
+else
+  export CPU_ONLY=0
+  make --keep-going all test pycaffe warn
+  make all
+  make test
+  make pycaffe
+  make warn
+fi
--- a/scripts/travis/travis_build_and_test.sh
+++ b/scripts/travis/travis_build_and_test.sh
@ -0,0 +1,22 @@
+#!/bin/bash
+# Script called by Travis to do a CPU-only build of and test Caffe.
+
+if $WITH_CMAKE; then
+  mkdir build
+  cd build
+  cmake -DBUILD_PYTHON=ON -DBUILD_EXAMPLES=ON -DCMAKE_BUILD_TYPE=Release -DCPU_ONLY=ON ..
+  make --keep-going
+  make runtest
+  make lint
+  make clean
+  cd -
+else
+  export CPU_ONLY=1
+  make --keep-going all test pycaffe warn lint
+  make runtest
+  make all
+  make test
+  make pycaffe
+  make warn
+  make lint
+fi
--- a/scripts/travis/travis_install.sh
+++ b/scripts/travis/travis_install.sh
@ -0,0 +1,79 @@
+#!/bin/bash
+
+# Install apt packages where the Ubuntu 12.04 default works for Caffe
+sudo apt-get -y update
+sudo apt-get install \
+    wget git curl \
+    python-dev python-numpy \
+    libleveldb-dev libsnappy-dev libopencv-dev \
+    libboost-dev libboost-system-dev libboost-python-dev \
+    libprotobuf-dev protobuf-compiler \
+    libatlas-dev libatlas-base-dev \
+    libhdf5-serial-dev \
+    bc
+
+# Add a special apt-repository to install CMake 2.8.9 for CMake Caffe build,
+# if needed.  By default, Aptitude in Ubuntu 12.04 installs CMake 2.8.7, but
+# Caffe requires a minimum CMake version of 2.8.8.
+if $WITH_CMAKE; then
+    sudo add-apt-repository -y ppa:ubuntu-sdk-team/ppa
+    sudo apt-get -y update
+    sudo apt-get -y install cmake || (echo "CMake install failed"; exit 1)
+fi
+
+# Install glog
+GLOG_URL=https://google-glog.googlecode.com/files/glog-0.3.3.tar.gz
+GLOG_FILE=/tmp/glog-0.3.3.tar.gz
+pushd .
+wget $GLOG_URL -O $GLOG_FILE && \
+    tar -C /tmp -xzvf $GLOG_FILE && \
+    rm $GLOG_FILE && \
+    cd /tmp/glog-0.3.3 && \
+    ./configure && make && sudo make install || \
+    (echo "glog install failed"; exit 1)
+popd
+
+# Install gflags
+GFLAGS_URL=https://github.com/schuhschuh/gflags/archive/master.zip
+GFLAGS_FILE=/tmp/gflags-master.zip
+pushd .
+wget $GFLAGS_URL -O $GFLAGS_FILE && \
+    cd /tmp/ && unzip gflags-master.zip && \
+    cd gflags-master && \
+    mkdir build && \
+    cd build && \
+    export CXXFLAGS="-fPIC" && \
+    cmake .. && make VERBOSE=1 && sudo make install || \
+    (echo "gflags install failed"; exit 1)
+popd
+
+# Install CUDA, if needed
+CUDA_URL=http://developer.download.nvidia.com/compute/cuda/repos/ubuntu1204/x86_64/cuda-repo-ubuntu1204_6.0-37_amd64.deb
+CUDA_FILE=/tmp/cuda_install.deb
+if $WITH_CUDA; then
+    curl $CUDA_URL -o $CUDA_FILE && \
+        sudo dpkg -i $CUDA_FILE ||
+        (echo "CUDA install failed"; exit 1)
+    rm -f $CUDA_FILE
+    sudo apt-get -y update
+    # Install the minimal CUDA subpackages required to test Caffe build.
+    # For a full CUDA installation, add 'cuda' to the list of packages.
+    sudo apt-get -y install cuda-core-6-0 cuda-extra-libs-6-0
+    # Create CUDA symlink at /usr/local/cuda
+    # (This would normally be created by the CUDA installer, but we create it
+    # manually since we did a partial installation.)
+    sudo ln -s /usr/local/cuda-6.0 /usr/local/cuda ||
+        (echo "CUDA symlink creation failed"; exit 1)
+fi
+
+# Install LMDB
+LMDB_URL=https://gitorious.org/mdb/mdb/archive/7f038d0f15bec57b4c07aa3f31cd5564c88a1897.tar.gz
+LMDB_FILE=/tmp/mdb.tar.gz
+pushd .
+curl $LMDB_URL -o $LMDB_FILE && \
+    tar -C /tmp -xzvf $LMDB_FILE && \
+    cd /tmp/mdb-mdb/libraries/liblmdb/ && \
+    make && sudo make install || \
+    (echo "LMDB install failed"; exit 1)
+popd
+rm -f $LMDB_FILE
--- a/scripts/travis/travis_setup_makefile_config.sh
+++ b/scripts/travis/travis_setup_makefile_config.sh
@ -0,0 +1,12 @@
+#!/bin/bash
+
+mv Makefile.config.example Makefile.config
+
+if $WITH_CUDA; then
+  # Remove default gencode set; only generate compute_50.
+  sed -i 's/-gencode arch=.*\\//' Makefile.config
+  sed -i 's/CUDA_ARCH :=//' Makefile.config
+  GENCODE="-gencode arch=compute_50,code=sm_50"
+  GENCODE="$GENCODE -gencode arch=compute_50,code=compute_50"
+  echo "CUDA_ARCH := $GENCODE" >> Makefile.config
+fi