From f3a933a620b8b089a7fe29ba76ec27f5408ff68d Mon Sep 17 00:00:00 2001 From: Tea Date: Sat, 27 Jun 2015 11:44:56 +0800 Subject: [PATCH] Separate IO dependencies OpenCV, LMDB, LevelDB and Snappy are made optional via switches (USE_OPENCV, USE_LMDB, USE_LEVELDB) available for Make and CMake builds. Since Snappy is a LevelDB dependency, its use is determined by USE_LEVELDB. HDF5 is left bundled because it is used for serializing weights and solverstates. --- .travis.yml | 11 ++--- CMakeLists.txt | 7 +++- Makefile | 28 +++++++++++-- Makefile.config.example | 5 +++ cmake/ConfigGen.cmake | 12 ++++++ cmake/Dependencies.cmake | 41 ++++++++++++------- cmake/Summary.cmake | 18 +++++--- cmake/Templates/CaffeConfig.cmake.in | 26 ++++++------ cmake/Templates/caffe_config.h.in | 5 +++ docs/installation.md | 9 ++-- .../cpp_classification/classification.cpp | 8 ++++ examples/mnist/convert_mnist_data.cpp | 12 ++++++ .../siamese/convert_mnist_siamese_data.cpp | 9 +++- include/caffe/data_layers.hpp | 3 +- include/caffe/data_transformer.hpp | 5 ++- include/caffe/util/db_leveldb.hpp | 2 + include/caffe/util/db_lmdb.hpp | 2 + include/caffe/util/io.hpp | 2 + python/caffe/test/test_layer_type_list.py | 1 + scripts/travis/travis_build_and_test.sh | 14 ++++++- .../travis/travis_setup_makefile_config.sh | 6 +++ src/caffe/data_transformer.cpp | 16 +++++++- src/caffe/layers/data_layer.cpp | 3 +- src/caffe/layers/image_data_layer.cpp | 2 + src/caffe/layers/memory_data_layer.cpp | 4 ++ src/caffe/layers/window_data_layer.cpp | 2 + src/caffe/test/test_data_layer.cpp | 6 +++ src/caffe/test/test_data_transformer.cpp | 2 + src/caffe/test/test_db.cpp | 2 + src/caffe/test/test_image_data_layer.cpp | 2 + src/caffe/test/test_io.cpp | 2 + src/caffe/test/test_layer_factory.cpp | 4 ++ src/caffe/test/test_memory_data_layer.cpp | 5 ++- src/caffe/test/test_upgrade_proto.cpp | 12 +++++- src/caffe/util/db.cpp | 16 ++++++-- src/caffe/util/db_leveldb.cpp | 2 + src/caffe/util/db_lmdb.cpp | 2 + src/caffe/util/io.cpp | 10 ++++- tools/compute_image_mean.cpp | 4 ++ tools/convert_imageset.cpp | 4 ++ 40 files changed, 265 insertions(+), 61 deletions(-) diff --git a/.travis.yml b/.travis.yml index b920a935..4dc7ed72 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,11 +2,12 @@ # one using CMake, and one using make. env: matrix: - - WITH_CUDA=false WITH_CMAKE=false - - WITH_CUDA=false WITH_CMAKE=true - - WITH_CUDA=true WITH_CMAKE=false - - WITH_CUDA=true WITH_CMAKE=true - - WITH_CUDA=false WITH_CMAKE=true PYTHON_VERSION=3 + - WITH_CUDA=false WITH_CMAKE=false WITH_IO=true + - WITH_CUDA=false WITH_CMAKE=true WITH_IO=true PYTHON_VERSION=3 + - WITH_CUDA=true WITH_CMAKE=false WITH_IO=true + - WITH_CUDA=true WITH_CMAKE=true WITH_IO=true + - WITH_CUDA=false WITH_CMAKE=false WITH_IO=false + - WITH_CUDA=false WITH_CMAKE=true WITH_IO=false PYTHON_VERSION=3 language: cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index ef599b68..838723be 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -16,13 +16,16 @@ include(cmake/ConfigGen.cmake) # ---[ Options caffe_option(CPU_ONLY "Build Caffe without CUDA support" OFF) # TODO: rename to USE_CUDA -caffe_option(USE_CUDNN "Build Caffe with cuDNN libary support" ON IF NOT CPU_ONLY) +caffe_option(USE_CUDNN "Build Caffe with cuDNN library support" ON IF NOT CPU_ONLY) caffe_option(BUILD_SHARED_LIBS "Build shared libraries" ON) caffe_option(BUILD_python "Build Python wrapper" ON) set(python_version "2" CACHE STRING "Specify which python version to use") caffe_option(BUILD_matlab "Build Matlab wrapper" OFF IF UNIX OR APPLE) caffe_option(BUILD_docs "Build documentation" ON IF UNIX OR APPLE) -caffe_option(BUILD_python_layer "Build the Caffe python layer" ON) +caffe_option(BUILD_python_layer "Build the caffe python layer" ON) +caffe_option(USE_LMDB "Build with lmdb" ON) +caffe_option(USE_LEVELDB "Build with levelDB" ON) +caffe_option(USE_OPENCV "Build with OpenCV support" ON) # ---[ Dependencies include(cmake/Dependencies.cmake) diff --git a/Makefile b/Makefile index 80bc3737..ddaed59b 100644 --- a/Makefile +++ b/Makefile @@ -169,9 +169,18 @@ ifneq ($(CPU_ONLY), 1) LIBRARY_DIRS += $(CUDA_LIB_DIR) LIBRARIES := cudart cublas curand endif -LIBRARIES += glog gflags protobuf leveldb snappy \ - lmdb boost_system hdf5_hl hdf5 m \ - opencv_core opencv_highgui opencv_imgproc + +LIBRARIES += glog gflags protobuf boost_system m hdf5_hl hdf5 + +ifeq ($(USE_LEVELDB), 1) + LIBRARIES += leveldb snappy +endif +ifeq ($(USE_LMDB), 1) + LIBRARIES += lmdb +endif +ifeq ($(USE_OPENCV), 1) + LIBRARIES += opencv_core opencv_highgui opencv_imgproc +endif PYTHON_LIBRARIES := boost_python python2.7 WARNINGS := -Wall -Wno-sign-compare @@ -290,6 +299,17 @@ ifeq ($(USE_CUDNN), 1) COMMON_FLAGS += -DUSE_CUDNN endif +# i/o libraries configuration +ifeq ($(USE_OPENCV), 1) + COMMON_FLAGS += -DUSE_OPENCV +endif +ifeq ($(USE_LEVELDB), 1) + COMMON_FLAGS += -DUSE_LEVELDB +endif +ifeq ($(USE_LMDB), 1) + COMMON_FLAGS += -DUSE_LMDB +endif + # CPU-only configuration ifeq ($(CPU_ONLY), 1) OBJS := $(PROTO_OBJS) $(CXX_OBJS) @@ -472,7 +492,7 @@ runtest: $(TEST_ALL_BIN) pytest: py cd python; python -m unittest discover -s caffe/test - + mattest: mat cd matlab; $(MATLAB_DIR)/bin/matlab -nodisplay -r 'caffe.run_tests(), exit()' diff --git a/Makefile.config.example b/Makefile.config.example index a8735025..32e67ee4 100644 --- a/Makefile.config.example +++ b/Makefile.config.example @@ -7,6 +7,11 @@ # CPU-only switch (uncomment to build without GPU support). # CPU_ONLY := 1 +# comment out to disable IO dependencies +USE_LEVELDB := 1 +USE_LMDB := 1 +USE_OPENCV := 1 + # To customize your choice of compiler, uncomment and set the following. # N.B. the default for Linux is g++ and the default for OSX is clang++ # CUSTOM_CXX := g++ diff --git a/cmake/ConfigGen.cmake b/cmake/ConfigGen.cmake index 566d6ca0..8b259965 100644 --- a/cmake/ConfigGen.cmake +++ b/cmake/ConfigGen.cmake @@ -56,6 +56,18 @@ function(caffe_generate_export_configs) list(APPEND Caffe_DEFINITIONS -DCPU_ONLY) endif() + if(USE_OPENCV) + list(APPEND Caffe_DEFINITIONS -DUSE_OPENCV) + endif() + + if(USE_LMDB) + list(APPEND Caffe_DEFINITIONS -DUSE_LMDB) + endif() + + if(USE_LEVELDB) + list(APPEND Caffe_DEFINITIONS -DUSE_LEVELDB) + endif() + if(NOT HAVE_CUDNN) set(HAVE_CUDNN FALSE) else() diff --git a/cmake/Dependencies.cmake b/cmake/Dependencies.cmake index 7c86dd55..d68d7bfb 100644 --- a/cmake/Dependencies.cmake +++ b/cmake/Dependencies.cmake @@ -29,19 +29,27 @@ include_directories(SYSTEM ${HDF5_INCLUDE_DIRS} ${HDF5_HL_INCLUDE_DIR}) list(APPEND Caffe_LINKER_LIBS ${HDF5_LIBRARIES}) # ---[ LMDB -find_package(LMDB REQUIRED) -include_directories(SYSTEM ${LMDB_INCLUDE_DIR}) -list(APPEND Caffe_LINKER_LIBS ${LMDB_LIBRARIES}) +if(USE_LMDB) + find_package(LMDB REQUIRED) + include_directories(SYSTEM ${LMDB_INCLUDE_DIR}) + list(APPEND Caffe_LINKER_LIBS ${LMDB_LIBRARIES}) + add_definitions(-DUSE_LMDB) +endif() # ---[ LevelDB -find_package(LevelDB REQUIRED) -include_directories(SYSTEM ${LevelDB_INCLUDE}) -list(APPEND Caffe_LINKER_LIBS ${LevelDB_LIBRARIES}) +if(USE_LEVELDB) + find_package(LevelDB REQUIRED) + include_directories(SYSTEM ${LevelDB_INCLUDE}) + list(APPEND Caffe_LINKER_LIBS ${LevelDB_LIBRARIES}) + add_definitions(-DUSE_LEVELDB) +endif() # ---[ Snappy -find_package(Snappy REQUIRED) -include_directories(SYSTEM ${Snappy_INCLUDE_DIR}) -list(APPEND Caffe_LINKER_LIBS ${Snappy_LIBRARIES}) +if(USE_LEVELDB) + find_package(Snappy REQUIRED) + include_directories(SYSTEM ${Snappy_INCLUDE_DIR}) + list(APPEND Caffe_LINKER_LIBS ${Snappy_LIBRARIES}) +endif() # ---[ CUDA include(cmake/Cuda.cmake) @@ -57,13 +65,16 @@ if(NOT HAVE_CUDA) endif() # ---[ OpenCV -find_package(OpenCV QUIET COMPONENTS core highgui imgproc imgcodecs) -if(NOT OpenCV_FOUND) # if not OpenCV 3.x, then imgcodecs are not found - find_package(OpenCV REQUIRED COMPONENTS core highgui imgproc) +if(USE_OPENCV) + find_package(OpenCV QUIET COMPONENTS core highgui imgproc imgcodecs) + if(NOT OpenCV_FOUND) # if not OpenCV 3.x, then imgcodecs are not found + find_package(OpenCV REQUIRED COMPONENTS core highgui imgproc) + endif() + include_directories(SYSTEM ${OpenCV_INCLUDE_DIRS}) + list(APPEND Caffe_LINKER_LIBS ${OpenCV_LIBS}) + message(STATUS "OpenCV found (${OpenCV_CONFIG_PATH})") + add_definitions(-DUSE_OPENCV) endif() -include_directories(SYSTEM ${OpenCV_INCLUDE_DIRS}) -list(APPEND Caffe_LINKER_LIBS ${OpenCV_LIBS}) -message(STATUS "OpenCV found (${OpenCV_CONFIG_PATH})") # ---[ BLAS if(NOT APPLE) diff --git a/cmake/Summary.cmake b/cmake/Summary.cmake index e094ac00..3d12e81a 100644 --- a/cmake/Summary.cmake +++ b/cmake/Summary.cmake @@ -114,6 +114,9 @@ function(caffe_print_configuration_summary) caffe_status(" BUILD_matlab : ${BUILD_matlab}") caffe_status(" BUILD_docs : ${BUILD_docs}") caffe_status(" CPU_ONLY : ${CPU_ONLY}") + caffe_status(" USE_LMDB : ${USE_LMDB}") + caffe_status(" USE_LEVELDB : ${USE_LEVELDB}") + caffe_status(" USE_OPENCV : ${USE_OPENCV}") caffe_status("") caffe_status("Dependencies:") caffe_status(" BLAS : " APPLE THEN "Yes (vecLib)" ELSE "Yes (${BLAS})") @@ -121,10 +124,16 @@ function(caffe_print_configuration_summary) caffe_status(" glog : Yes") caffe_status(" gflags : Yes") caffe_status(" protobuf : " PROTOBUF_FOUND THEN "Yes (ver. ${PROTOBUF_VERSION})" ELSE "No" ) - caffe_status(" lmdb : " LMDB_FOUND THEN "Yes (ver. ${LMDB_VERSION})" ELSE "No") - caffe_status(" Snappy : " SNAPPY_FOUND THEN "Yes (ver. ${Snappy_VERSION})" ELSE "No" ) - caffe_status(" LevelDB : " LEVELDB_FOUND THEN "Yes (ver. ${LEVELDB_VERSION})" ELSE "No") - caffe_status(" OpenCV : Yes (ver. ${OpenCV_VERSION})") + if(USE_LMDB) + caffe_status(" lmdb : " LMDB_FOUND THEN "Yes (ver. ${LMDB_VERSION})" ELSE "No") + endif() + if(USE_LEVELDB) + caffe_status(" LevelDB : " LEVELDB_FOUND THEN "Yes (ver. ${LEVELDB_VERSION})" ELSE "No") + caffe_status(" Snappy : " SNAPPY_FOUND THEN "Yes (ver. ${Snappy_VERSION})" ELSE "No" ) + endif() + if(USE_OPENCV) + caffe_status(" OpenCV : Yes (ver. ${OpenCV_VERSION})") + endif() caffe_status(" CUDA : " HAVE_CUDA THEN "Yes (ver. ${CUDA_VERSION})" ELSE "No" ) caffe_status("") if(HAVE_CUDA) @@ -165,4 +174,3 @@ function(caffe_print_configuration_summary) caffe_status(" Install path : ${CMAKE_INSTALL_PREFIX}") caffe_status("") endfunction() - diff --git a/cmake/Templates/CaffeConfig.cmake.in b/cmake/Templates/CaffeConfig.cmake.in index 8f23742e..73f57ac2 100644 --- a/cmake/Templates/CaffeConfig.cmake.in +++ b/cmake/Templates/CaffeConfig.cmake.in @@ -17,22 +17,24 @@ # Caffe_HAVE_CUDNN - signals about cuDNN support -# OpenCV dependency +# OpenCV dependency (optional) -if(NOT OpenCV_FOUND) - set(Caffe_OpenCV_CONFIG_PATH "@OpenCV_CONFIG_PATH@") - if(Caffe_OpenCV_CONFIG_PATH) - get_filename_component(Caffe_OpenCV_CONFIG_PATH ${Caffe_OpenCV_CONFIG_PATH} ABSOLUTE) +if(@USE_OPENCV@) + if(NOT OpenCV_FOUND) + set(Caffe_OpenCV_CONFIG_PATH "@OpenCV_CONFIG_PATH@") + if(Caffe_OpenCV_CONFIG_PATH) + get_filename_component(Caffe_OpenCV_CONFIG_PATH ${Caffe_OpenCV_CONFIG_PATH} ABSOLUTE) - if(EXISTS ${Caffe_OpenCV_CONFIG_PATH} AND NOT TARGET opencv_core) - message(STATUS "Caffe: using OpenCV config from ${Caffe_OpenCV_CONFIG_PATH}") - include(${Caffe_OpenCV_CONFIG_PATH}/OpenCVModules.cmake) + if(EXISTS ${Caffe_OpenCV_CONFIG_PATH} AND NOT TARGET opencv_core) + message(STATUS "Caffe: using OpenCV config from ${Caffe_OpenCV_CONFIG_PATH}") + include(${Caffe_OpenCV_CONFIG_PATH}/OpenCVModules.cmake) + endif() + + else() + find_package(OpenCV REQUIRED) endif() - - else() - find_package(OpenCV REQUIRED) + unset(Caffe_OpenCV_CONFIG_PATH) endif() - unset(Caffe_OpenCV_CONFIG_PATH) endif() # Compute paths diff --git a/cmake/Templates/caffe_config.h.in b/cmake/Templates/caffe_config.h.in index 6039e8f6..9302022d 100644 --- a/cmake/Templates/caffe_config.h.in +++ b/cmake/Templates/caffe_config.h.in @@ -30,3 +30,8 @@ /* Matlab */ #cmakedefine HAVE_MATLAB + +/* IO libraries */ +#cmakedefine USE_OPENCV +#cmakedefine USE_LMDB +#cmakedefine USE_LEVELDB diff --git a/docs/installation.md b/docs/installation.md index d535c6d0..89a8c71c 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -17,16 +17,19 @@ When updating Caffe, it's best to `make clean` before re-compiling. ## Prerequisites -Caffe has several dependencies. +Caffe has several dependencies: * [CUDA](https://developer.nvidia.com/cuda-zone) is required for GPU mode. * library version 7.0 and the latest driver version are recommended, but 6.* is fine too * 5.5, and 5.0 are compatible but considered legacy * [BLAS](http://en.wikipedia.org/wiki/Basic_Linear_Algebra_Subprograms) via ATLAS, MKL, or OpenBLAS. * [Boost](http://www.boost.org/) >= 1.55 +* `protobuf`, `glog`, `gflags`, `hdf5` + +Optional dependencies: + * [OpenCV](http://opencv.org/) >= 2.4 including 3.0 -* `protobuf`, `glog`, `gflags` -* IO libraries `hdf5`, `leveldb`, `snappy`, `lmdb` +* IO libraries: `lmdb`, `leveldb` (note: leveldb requires `snappy`) Pycaffe and Matcaffe interfaces have their own natural needs. diff --git a/examples/cpp_classification/classification.cpp b/examples/cpp_classification/classification.cpp index dc8b863f..de48fb69 100644 --- a/examples/cpp_classification/classification.cpp +++ b/examples/cpp_classification/classification.cpp @@ -1,7 +1,9 @@ #include +#ifdef USE_OPENCV #include #include #include +#endif // USE_OPENCV #include #include #include @@ -9,6 +11,7 @@ #include #include +#ifdef USE_OPENCV using namespace caffe; // NOLINT(build/namespaces) using std::string; @@ -255,3 +258,8 @@ int main(int argc, char** argv) { << p.first << "\"" << std::endl; } } +#else +int main(int argc, char** argv) { + LOG(FATAL) << "This example requires OpenCV; compile with USE_OPENCV."; +} +#endif // USE_OPENCV diff --git a/examples/mnist/convert_mnist_data.cpp b/examples/mnist/convert_mnist_data.cpp index 54443f11..8f29bafd 100644 --- a/examples/mnist/convert_mnist_data.cpp +++ b/examples/mnist/convert_mnist_data.cpp @@ -9,9 +9,13 @@ #include #include #include + +#if defined(USE_LEVELDB) && defined(USE_LMDB) #include #include #include +#endif + #include #include @@ -20,6 +24,8 @@ #include "caffe/proto/caffe.pb.h" +#if defined(USE_LEVELDB) && defined(USE_LMDB) + using namespace caffe; // NOLINT(build/namespaces) using std::string; @@ -196,3 +202,9 @@ int main(int argc, char** argv) { } return 0; } +#else +int main(int argc, char** argv) { + LOG(FATAL) << "This example requires LevelDB and LMDB; " << + "compile with USE_LEVELDB and USE_LMDB."; +} +#endif // USE_LEVELDB and USE_LMDB diff --git a/examples/siamese/convert_mnist_siamese_data.cpp b/examples/siamese/convert_mnist_siamese_data.cpp index 8008b443..ad08036f 100644 --- a/examples/siamese/convert_mnist_siamese_data.cpp +++ b/examples/siamese/convert_mnist_siamese_data.cpp @@ -10,12 +10,14 @@ #include "glog/logging.h" #include "google/protobuf/text_format.h" -#include "leveldb/db.h" #include "stdint.h" #include "caffe/proto/caffe.pb.h" #include "caffe/util/math_functions.hpp" +#ifdef USE_LEVELDB +#include "leveldb/db.h" + uint32_t swap_endian(uint32_t val) { val = ((val << 8) & 0xFF00FF00) | ((val >> 8) & 0xFF00FF); return (val << 16) | (val >> 16); @@ -121,3 +123,8 @@ int main(int argc, char** argv) { } return 0; } +#else +int main(int argc, char** argv) { + LOG(FATAL) << "This example requires LevelDB; compile with USE_LEVELDB."; +} +#endif // USE_LEVELDB diff --git a/include/caffe/data_layers.hpp b/include/caffe/data_layers.hpp index 552d8141..90fd0d19 100644 --- a/include/caffe/data_layers.hpp +++ b/include/caffe/data_layers.hpp @@ -4,7 +4,6 @@ #include #include #include - #include "hdf5.h" #include "caffe/blob.hpp" @@ -275,8 +274,10 @@ class MemoryDataLayer : public BaseDataLayer { virtual inline int ExactNumTopBlobs() const { return 2; } virtual void AddDatumVector(const vector& datum_vector); +#ifdef USE_OPENCV virtual void AddMatVector(const vector& mat_vector, const vector& labels); +#endif // USE_OPENCV // Reset should accept const pointers, but can't, because the memory // will be given to Blob, which is mutable diff --git a/include/caffe/data_transformer.hpp b/include/caffe/data_transformer.hpp index 0ad68c80..97b4ee6a 100644 --- a/include/caffe/data_transformer.hpp +++ b/include/caffe/data_transformer.hpp @@ -50,6 +50,7 @@ class DataTransformer { void Transform(const vector & datum_vector, Blob* transformed_blob); +#ifdef USE_OPENCV /** * @brief Applies the transformation defined in the data layer's * transform_param block to a vector of Mat. @@ -74,6 +75,7 @@ class DataTransformer { * set_cpu_data() is used. See image_data_layer.cpp for an example. */ void Transform(const cv::Mat& cv_img, Blob* transformed_blob); +#endif // USE_OPENCV /** * @brief Applies the same transformation defined in the data layer's @@ -113,6 +115,7 @@ class DataTransformer { * @param mat_vector * A vector of Mat containing the data to be transformed. */ +#ifdef USE_OPENCV vector InferBlobShape(const vector & mat_vector); /** * @brief Infers the shape of transformed_blob will have when @@ -122,6 +125,7 @@ class DataTransformer { * cv::Mat containing the data to be transformed. */ vector InferBlobShape(const cv::Mat& cv_img); +#endif // USE_OPENCV protected: /** @@ -148,4 +152,3 @@ class DataTransformer { } // namespace caffe #endif // CAFFE_DATA_TRANSFORMER_HPP_ - diff --git a/include/caffe/util/db_leveldb.hpp b/include/caffe/util/db_leveldb.hpp index 10623554..e9fa0d32 100644 --- a/include/caffe/util/db_leveldb.hpp +++ b/include/caffe/util/db_leveldb.hpp @@ -1,3 +1,4 @@ +#ifdef USE_LEVELDB #ifndef CAFFE_UTIL_DB_LEVELDB_HPP #define CAFFE_UTIL_DB_LEVELDB_HPP @@ -71,3 +72,4 @@ class LevelDB : public DB { } // namespace caffe #endif // CAFFE_UTIL_DB_LEVELDB_HPP +#endif // USE_LEVELDB diff --git a/include/caffe/util/db_lmdb.hpp b/include/caffe/util/db_lmdb.hpp index cc7c90af..4e1568ac 100644 --- a/include/caffe/util/db_lmdb.hpp +++ b/include/caffe/util/db_lmdb.hpp @@ -1,3 +1,4 @@ +#ifdef USE_LMDB #ifndef CAFFE_UTIL_DB_LMDB_HPP #define CAFFE_UTIL_DB_LMDB_HPP @@ -89,3 +90,4 @@ class LMDB : public DB { } // namespace caffe #endif // CAFFE_UTIL_DB_LMDB_HPP +#endif // USE_LMDB diff --git a/include/caffe/util/io.hpp b/include/caffe/util/io.hpp index c0938ad0..6070b4c7 100644 --- a/include/caffe/util/io.hpp +++ b/include/caffe/util/io.hpp @@ -120,6 +120,7 @@ inline bool ReadImageToDatum(const string& filename, const int label, bool DecodeDatumNative(Datum* datum); bool DecodeDatum(Datum* datum, bool is_color); +#ifdef USE_OPENCV cv::Mat ReadImageToCVMat(const string& filename, const int height, const int width, const bool is_color); @@ -135,6 +136,7 @@ cv::Mat DecodeDatumToCVMatNative(const Datum& datum); cv::Mat DecodeDatumToCVMat(const Datum& datum, bool is_color); void CVMatToDatum(const cv::Mat& cv_img, Datum* datum); +#endif // USE_OPENCV } // namespace caffe diff --git a/python/caffe/test/test_layer_type_list.py b/python/caffe/test/test_layer_type_list.py index 7edc80df..47f4cf6d 100644 --- a/python/caffe/test/test_layer_type_list.py +++ b/python/caffe/test/test_layer_type_list.py @@ -5,6 +5,7 @@ import caffe class TestLayerTypeList(unittest.TestCase): def test_standard_types(self): + #removing 'Data' from list for type_name in ['Data', 'Convolution', 'InnerProduct']: self.assertIn(type_name, caffe.layer_type_list(), '%s not in layer_type_list()' % type_name) diff --git a/scripts/travis/travis_build_and_test.sh b/scripts/travis/travis_build_and_test.sh index 9ba737e2..bbc82133 100755 --- a/scripts/travis/travis_build_and_test.sh +++ b/scripts/travis/travis_build_and_test.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Script called by Travis to do a CPU-only build of and test Caffe. +# Script called by Travis to build and test Caffe. set -e MAKE="make --jobs=$NUM_THREADS --keep-going" @@ -15,7 +15,12 @@ if $WITH_CMAKE; then if [ "$PYTHON_VERSION" = "3" ]; then PYTHON_ARGS="$PYTHON_ARGS -Dpython_version=3 -DBOOST_LIBRARYDIR=$CONDA_DIR/lib/" fi - cmake -DBUILD_python=ON -DCMAKE_BUILD_TYPE=Release $CPU_ONLY $PYTHON_ARGS -DCMAKE_INCLUDE_PATH="$CONDA_DIR/include/" -DCMAKE_LIBRARY_PATH="$CONDA_DIR/lib/" .. + if $WITH_IO; then + IO_ARGS="-DUSE_OPENCV=ON -DUSE_LMDB=ON -DUSE_LEVELDB=ON" + else + IO_ARGS="-DUSE_OPENCV=OFF -DUSE_LMDB=OFF -DUSE_LEVELDB=OFF" + fi + cmake -DBUILD_python=ON -DCMAKE_BUILD_TYPE=Release $CPU_ONLY $PYTHON_ARGS -DCMAKE_INCLUDE_PATH="$CONDA_DIR/include/" -DCMAKE_LIBRARY_PATH="$CONDA_DIR/lib/" $IO_ARGS .. $MAKE $MAKE pytest if ! $WITH_CUDA; then @@ -28,6 +33,11 @@ else if ! $WITH_CUDA; then export CPU_ONLY=1 fi + if $WITH_IO; then + export USE_LMDB=1 + export USE_LEVELDB=1 + export USE_OPENCV=1 + fi $MAKE all test pycaffe warn lint || true if ! $WITH_CUDA; then $MAKE runtest diff --git a/scripts/travis/travis_setup_makefile_config.sh b/scripts/travis/travis_setup_makefile_config.sh index 1440be2a..83aacf11 100755 --- a/scripts/travis/travis_setup_makefile_config.sh +++ b/scripts/travis/travis_setup_makefile_config.sh @@ -11,6 +11,12 @@ if $WITH_CUDA; then echo "CUDA_ARCH := $GENCODE" >> Makefile.config fi +# Remove IO library settings from Makefile.config +# to avoid conflicts with CI configuration +sed -i -e '/USE_LMDB/d' Makefile.config +sed -i -e '/USE_LEVELDB/d' Makefile.config +sed -i -e '/USE_OPENCV/d' Makefile.config + cat << 'EOF' >> Makefile.config # Travis' nvcc doesn't like newer boost versions NVCCFLAGS := -Xcudafe --diag_suppress=cc_clobber_ignored -Xcudafe --diag_suppress=useless_using_declaration -Xcudafe --diag_suppress=set_but_not_used diff --git a/src/caffe/data_transformer.cpp b/src/caffe/data_transformer.cpp index 4666d9bd..7189d67e 100644 --- a/src/caffe/data_transformer.cpp +++ b/src/caffe/data_transformer.cpp @@ -1,4 +1,6 @@ +#ifdef USE_OPENCV #include +#endif // USE_OPENCV #include #include @@ -124,11 +126,13 @@ void DataTransformer::Transform(const Datum& datum, } } + template void DataTransformer::Transform(const Datum& datum, Blob* transformed_blob) { // If datum is encoded, decoded and transform the cv::image. if (datum.encoded()) { +#ifdef USE_OPENCV CHECK(!(param_.force_color() && param_.force_gray())) << "cannot set both force_color and force_gray"; cv::Mat cv_img; @@ -140,6 +144,9 @@ void DataTransformer::Transform(const Datum& datum, } // Transform the cv::image into blob. return Transform(cv_img, transformed_blob); +#else + LOG(FATAL) << "Encoded datum requires OpenCV; compile with USE_OPENCV."; +#endif // USE_OPENCV } else { if (param_.force_color() || param_.force_gray()) { LOG(ERROR) << "force_color and force_gray only for encoded datum"; @@ -194,6 +201,7 @@ void DataTransformer::Transform(const vector & datum_vector, } } +#ifdef USE_OPENCV template void DataTransformer::Transform(const vector & mat_vector, Blob* transformed_blob) { @@ -315,6 +323,7 @@ void DataTransformer::Transform(const cv::Mat& cv_img, } } } +#endif // USE_OPENCV template void DataTransformer::Transform(Blob* input_blob, @@ -432,6 +441,7 @@ void DataTransformer::Transform(Blob* input_blob, template vector DataTransformer::InferBlobShape(const Datum& datum) { if (datum.encoded()) { +#ifdef USE_OPENCV CHECK(!(param_.force_color() && param_.force_gray())) << "cannot set both force_color and force_gray"; cv::Mat cv_img; @@ -443,8 +453,10 @@ vector DataTransformer::InferBlobShape(const Datum& datum) { } // InferBlobShape using the cv::image. return InferBlobShape(cv_img); +#else + LOG(FATAL) << "Encoded datum requires OpenCV; compile with USE_OPENCV."; +#endif // USE_OPENCV } - const int crop_size = param_.crop_size(); const int datum_channels = datum.channels(); const int datum_height = datum.height(); @@ -474,6 +486,7 @@ vector DataTransformer::InferBlobShape( return shape; } +#ifdef USE_OPENCV template vector DataTransformer::InferBlobShape(const cv::Mat& cv_img) { const int crop_size = param_.crop_size(); @@ -504,6 +517,7 @@ vector DataTransformer::InferBlobShape( shape[0] = num; return shape; } +#endif // USE_OPENCV template void DataTransformer::InitRand() { diff --git a/src/caffe/layers/data_layer.cpp b/src/caffe/layers/data_layer.cpp index 0932d9fe..71f8cb09 100644 --- a/src/caffe/layers/data_layer.cpp +++ b/src/caffe/layers/data_layer.cpp @@ -1,5 +1,6 @@ +#ifdef USE_OPENCV #include - +#endif // USE_OPENCV #include #include diff --git a/src/caffe/layers/image_data_layer.cpp b/src/caffe/layers/image_data_layer.cpp index 223ba3a7..3d2190f8 100644 --- a/src/caffe/layers/image_data_layer.cpp +++ b/src/caffe/layers/image_data_layer.cpp @@ -1,3 +1,4 @@ +#ifdef USE_OPENCV #include #include // NOLINT(readability/streams) @@ -164,3 +165,4 @@ INSTANTIATE_CLASS(ImageDataLayer); REGISTER_LAYER_CLASS(ImageData); } // namespace caffe +#endif // USE_OPENCV diff --git a/src/caffe/layers/memory_data_layer.cpp b/src/caffe/layers/memory_data_layer.cpp index 42de4198..2370aa04 100644 --- a/src/caffe/layers/memory_data_layer.cpp +++ b/src/caffe/layers/memory_data_layer.cpp @@ -1,4 +1,6 @@ +#ifdef USE_OPENCV #include +#endif // USE_OPENCV #include @@ -53,6 +55,7 @@ void MemoryDataLayer::AddDatumVector(const vector& datum_vector) { has_new_data_ = true; } +#ifdef USE_OPENCV template void MemoryDataLayer::AddMatVector(const vector& mat_vector, const vector& labels) { @@ -76,6 +79,7 @@ void MemoryDataLayer::AddMatVector(const vector& mat_vector, Reset(top_data, top_label, num); has_new_data_ = true; } +#endif // USE_OPENCV template void MemoryDataLayer::Reset(Dtype* data, Dtype* labels, int n) { diff --git a/src/caffe/layers/window_data_layer.cpp b/src/caffe/layers/window_data_layer.cpp index f637f2ec..f8db61c9 100644 --- a/src/caffe/layers/window_data_layer.cpp +++ b/src/caffe/layers/window_data_layer.cpp @@ -1,3 +1,4 @@ +#ifdef USE_OPENCV #include #include @@ -468,3 +469,4 @@ INSTANTIATE_CLASS(WindowDataLayer); REGISTER_LAYER_CLASS(WindowData); } // namespace caffe +#endif // USE_OPENCV diff --git a/src/caffe/test/test_data_layer.cpp b/src/caffe/test/test_data_layer.cpp index afe2a40d..9e03954a 100644 --- a/src/caffe/test/test_data_layer.cpp +++ b/src/caffe/test/test_data_layer.cpp @@ -1,3 +1,4 @@ +#ifdef USE_OPENCV #include #include @@ -348,6 +349,7 @@ class DataLayerTest : public MultiDeviceTest { TYPED_TEST_CASE(DataLayerTest, TestDtypesAndDevices); +#ifdef USE_LEVELDB TYPED_TEST(DataLayerTest, TestReadLevelDB) { const bool unique_pixels = false; // all pixels the same; images different this->Fill(unique_pixels, DataParameter_DB_LEVELDB); @@ -385,7 +387,9 @@ TYPED_TEST(DataLayerTest, TestReadCropTestLevelDB) { this->Fill(unique_pixels, DataParameter_DB_LEVELDB); this->TestReadCrop(TEST); } +#endif // USE_LEVELDB +#ifdef USE_LMDB TYPED_TEST(DataLayerTest, TestReadLMDB) { const bool unique_pixels = false; // all pixels the same; images different this->Fill(unique_pixels, DataParameter_DB_LMDB); @@ -424,4 +428,6 @@ TYPED_TEST(DataLayerTest, TestReadCropTestLMDB) { this->TestReadCrop(TEST); } +#endif // USE_LMDB } // namespace caffe +#endif // USE_OPENCV diff --git a/src/caffe/test/test_data_transformer.cpp b/src/caffe/test/test_data_transformer.cpp index 16570e20..8a101374 100644 --- a/src/caffe/test/test_data_transformer.cpp +++ b/src/caffe/test/test_data_transformer.cpp @@ -1,3 +1,4 @@ +#ifdef USE_OPENCV #include #include @@ -353,3 +354,4 @@ TYPED_TEST(DataTransformTest, TestMeanFile) { } } // namespace caffe +#endif // USE_OPENCV diff --git a/src/caffe/test/test_db.cpp b/src/caffe/test/test_db.cpp index 5b2ac230..1b487b14 100644 --- a/src/caffe/test/test_db.cpp +++ b/src/caffe/test/test_db.cpp @@ -1,3 +1,4 @@ +#if defined(USE_LEVELDB) && defined(USE_LMDB) && defined(USE_OPENCV) #include #include "boost/scoped_ptr.hpp" @@ -132,3 +133,4 @@ TYPED_TEST(DBTest, TestWrite) { } } // namespace caffe +#endif // USE_LEVELDB, USE_LMDB and USE_OPENCV diff --git a/src/caffe/test/test_image_data_layer.cpp b/src/caffe/test/test_image_data_layer.cpp index 931a5ebf..481fcef7 100644 --- a/src/caffe/test/test_image_data_layer.cpp +++ b/src/caffe/test/test_image_data_layer.cpp @@ -1,3 +1,4 @@ +#ifdef USE_OPENCV #include #include #include @@ -177,3 +178,4 @@ TYPED_TEST(ImageDataLayerTest, TestShuffle) { } } // namespace caffe +#endif // USE_OPENCV diff --git a/src/caffe/test/test_io.cpp b/src/caffe/test/test_io.cpp index 4ab96311..c2c919e9 100644 --- a/src/caffe/test/test_io.cpp +++ b/src/caffe/test/test_io.cpp @@ -1,3 +1,4 @@ +#ifdef USE_OPENCV #include #include #include @@ -420,3 +421,4 @@ TEST_F(IOTest, TestDecodeDatumToCVMatContentNative) { } } // namespace caffe +#endif // USE_OPENCV diff --git a/src/caffe/test/test_layer_factory.cpp b/src/caffe/test/test_layer_factory.cpp index c86fafd0..7d5d39d8 100644 --- a/src/caffe/test/test_layer_factory.cpp +++ b/src/caffe/test/test_layer_factory.cpp @@ -31,12 +31,16 @@ TYPED_TEST(LayerFactoryTest, TestCreateLayer) { LayerParameter layer_param; // Data layers expect a DB if (iter->first == "Data") { +#ifdef USE_LEVELDB string tmp; MakeTempDir(&tmp); boost::scoped_ptr db(db::GetDB(DataParameter_DB_LEVELDB)); db->Open(tmp, db::NEW); db->Close(); layer_param.mutable_data_param()->set_source(tmp); +#else + continue; +#endif // USE_LEVELDB } layer_param.set_type(iter->first); layer = LayerRegistry::CreateLayer(layer_param); diff --git a/src/caffe/test/test_memory_data_layer.cpp b/src/caffe/test/test_memory_data_layer.cpp index a79033f5..7269a4d4 100644 --- a/src/caffe/test/test_memory_data_layer.cpp +++ b/src/caffe/test/test_memory_data_layer.cpp @@ -1,4 +1,6 @@ +#ifdef USE_OPENCV #include +#endif // USE_OPENCV #include #include @@ -113,6 +115,7 @@ TYPED_TEST(MemoryDataLayerTest, TestForward) { } } +#ifdef USE_OPENCV TYPED_TEST(MemoryDataLayerTest, AddDatumVectorDefaultTransform) { typedef typename TypeParam::Dtype Dtype; @@ -292,5 +295,5 @@ TYPED_TEST(MemoryDataLayerTest, TestSetBatchSize) { } } } - +#endif // USE_OPENCV } // namespace caffe diff --git a/src/caffe/test/test_upgrade_proto.cpp b/src/caffe/test/test_upgrade_proto.cpp index 00672023..ee05b151 100644 --- a/src/caffe/test/test_upgrade_proto.cpp +++ b/src/caffe/test/test_upgrade_proto.cpp @@ -2892,6 +2892,7 @@ TEST_F(NetUpgradeTest, TestImageNet) { this->RunV1UpgradeTest(expected_v1_proto, expected_v2_proto); } // NOLINT(readability/fn_size) +#ifdef USE_OPENCV TEST_F(NetUpgradeTest, TestUpgradeV1LayerType) { LayerParameter layer_param; shared_ptr > layer; @@ -2906,16 +2907,25 @@ TEST_F(NetUpgradeTest, TestUpgradeV1LayerType) { layer_param.set_type(v2_layer_type); // Data layers expect a DB if (v2_layer_type == "Data") { + #ifdef USE_LEVELDB string tmp; MakeTempDir(&tmp); boost::scoped_ptr db(db::GetDB(DataParameter_DB_LEVELDB)); db->Open(tmp, db::NEW); db->Close(); layer_param.mutable_data_param()->set_source(tmp); + #else + continue; + #endif // USE_LEVELDB } + #ifndef USE_OPENCV + if (v2_layer_type == "ImageData" || v2_layer_type == "WindowData") { + continue; + } + #endif // !USE_OPENCV layer = LayerRegistry::CreateLayer(layer_param); EXPECT_EQ(v2_layer_type, layer->type()); } } - +#endif // USE_OPENCV } // NOLINT(readability/fn_size) // namespace caffe diff --git a/src/caffe/util/db.cpp b/src/caffe/util/db.cpp index f55420e9..ccda054d 100644 --- a/src/caffe/util/db.cpp +++ b/src/caffe/util/db.cpp @@ -8,23 +8,31 @@ namespace caffe { namespace db { DB* GetDB(DataParameter::DB backend) { switch (backend) { +#ifdef USE_LEVELDB case DataParameter_DB_LEVELDB: return new LevelDB(); +#endif // USE_LEVELDB +#ifdef USE_LMDB case DataParameter_DB_LMDB: return new LMDB(); +#endif // USE_LMDB default: LOG(FATAL) << "Unknown database backend"; } } DB* GetDB(const string& backend) { +#ifdef USE_LEVELDB if (backend == "leveldb") { return new LevelDB(); - } else if (backend == "lmdb") { - return new LMDB(); - } else { - LOG(FATAL) << "Unknown database backend"; } +#endif // USE_LEVELDB +#ifdef USE_LMDB + if (backend == "lmdb") { + return new LMDB(); + } +#endif // USE_LMDB + LOG(FATAL) << "Unknown database backend"; } } // namespace db diff --git a/src/caffe/util/db_leveldb.cpp b/src/caffe/util/db_leveldb.cpp index 06c46627..f5c4d8a6 100644 --- a/src/caffe/util/db_leveldb.cpp +++ b/src/caffe/util/db_leveldb.cpp @@ -1,3 +1,4 @@ +#ifdef USE_LEVELDB #include "caffe/util/db_leveldb.hpp" #include @@ -19,3 +20,4 @@ void LevelDB::Open(const string& source, Mode mode) { } // namespace db } // namespace caffe +#endif // USE_LEVELDB diff --git a/src/caffe/util/db_lmdb.cpp b/src/caffe/util/db_lmdb.cpp index a054b796..78dd880a 100644 --- a/src/caffe/util/db_lmdb.cpp +++ b/src/caffe/util/db_lmdb.cpp @@ -1,3 +1,4 @@ +#ifdef USE_LMDB #include "caffe/util/db_lmdb.hpp" #include @@ -49,3 +50,4 @@ void LMDBTransaction::Put(const string& key, const string& value) { } // namespace db } // namespace caffe +#endif // USE_LMDB diff --git a/src/caffe/util/io.cpp b/src/caffe/util/io.cpp index 6f033142..f2b1dd98 100644 --- a/src/caffe/util/io.cpp +++ b/src/caffe/util/io.cpp @@ -3,9 +3,11 @@ #include #include #include +#ifdef USE_OPENCV #include #include #include +#endif // USE_OPENCV #include #include @@ -67,6 +69,7 @@ void WriteProtoToBinaryFile(const Message& proto, const char* filename) { CHECK(proto.SerializeToOstream(&output)); } +#ifdef USE_OPENCV cv::Mat ReadImageToCVMat(const string& filename, const int height, const int width, const bool is_color) { cv::Mat cv_img; @@ -98,6 +101,7 @@ cv::Mat ReadImageToCVMat(const string& filename, cv::Mat ReadImageToCVMat(const string& filename) { return ReadImageToCVMat(filename, 0, 0, true); } + // Do the file extension and encoding match? static bool matchExt(const std::string & fn, std::string en) { @@ -111,6 +115,7 @@ static bool matchExt(const std::string & fn, return true; return false; } + bool ReadImageToDatum(const string& filename, const int label, const int height, const int width, const bool is_color, const std::string & encoding, Datum* datum) { @@ -135,6 +140,7 @@ bool ReadImageToDatum(const string& filename, const int label, return false; } } +#endif // USE_OPENCV bool ReadFileToDatum(const string& filename, const int label, Datum* datum) { @@ -156,6 +162,7 @@ bool ReadFileToDatum(const string& filename, const int label, } } +#ifdef USE_OPENCV cv::Mat DecodeDatumToCVMatNative(const Datum& datum) { cv::Mat cv_img; CHECK(datum.encoded()) << "Datum not encoded"; @@ -227,6 +234,5 @@ void CVMatToDatum(const cv::Mat& cv_img, Datum* datum) { } datum->set_data(buffer); } - - +#endif // USE_OPENCV } // namespace caffe diff --git a/tools/compute_image_mean.cpp b/tools/compute_image_mean.cpp index b1fc7cae..2035d515 100644 --- a/tools/compute_image_mean.cpp +++ b/tools/compute_image_mean.cpp @@ -24,6 +24,7 @@ DEFINE_string(backend, "lmdb", int main(int argc, char** argv) { ::google::InitGoogleLogging(argv[0]); +#ifdef USE_OPENCV #ifndef GFLAGS_GFLAGS_H_ namespace gflags = google; #endif @@ -115,5 +116,8 @@ int main(int argc, char** argv) { } LOG(INFO) << "mean_value channel [" << c << "]:" << mean_values[c] / dim; } +#else + LOG(FATAL) << "This tool requires OpenCV; compile with USE_OPENCV."; +#endif // USE_OPENCV return 0; } diff --git a/tools/convert_imageset.cpp b/tools/convert_imageset.cpp index aad1f1fe..e51a2631 100644 --- a/tools/convert_imageset.cpp +++ b/tools/convert_imageset.cpp @@ -43,6 +43,7 @@ DEFINE_string(encode_type, "", "Optional: What type should we encode the image as ('png','jpg',...)."); int main(int argc, char** argv) { +#ifdef USE_OPENCV ::google::InitGoogleLogging(argv[0]); // Print output to stderr (while still logging) FLAGS_alsologtostderr = 1; @@ -150,5 +151,8 @@ int main(int argc, char** argv) { txn->Commit(); LOG(INFO) << "Processed " << count << " files."; } +#else + LOG(FATAL) << "This tool requires OpenCV; compile with USE_OPENCV."; +#endif // USE_OPENCV return 0; }