Make HDF5 optional

CMakeLists.txt

@@ -14,6 +14,19 @@ set(CAFFE_TARGET_VERSION "1.0.0-rc3" CACHE STRING "Caffe logical version")
 set(CAFFE_TARGET_SOVERSION "1.0.0-rc3" CACHE STRING "Caffe soname version")
 add_definitions(-DCAFFE_VERSION=${CAFFE_TARGET_VERSION})
 
+# Search packages for host system instead of packages for target system
+# in case of cross compilation these macro should be defined by toolchain file
+if(NOT COMMAND find_host_package)
+  macro(find_host_package)
+    find_package(${ARGN})
+  endmacro()
+endif()
+if(NOT COMMAND find_host_program)
+  macro(find_host_program)
+    find_program(${ARGN})
+  endmacro()
+endif()
+
 # ---[ Using cmake scripts and modules
 list(APPEND CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}/cmake/Modules)
 
@@ -38,6 +51,7 @@ caffe_option(USE_OPENCV "Build with OpenCV support" ON)
 caffe_option(USE_LEVELDB "Build with levelDB" ON)
 caffe_option(USE_LMDB "Build with lmdb" ON)
 caffe_option(ALLOW_LMDB_NOLOCK "Allow MDB_NOLOCK when reading LMDB files (only if necessary)" OFF)
+caffe_option(USE_HDF5 "Build with hdf5" ON)
 
 # ---[ Dependencies
 include(cmake/Dependencies.cmake)

Makefile

@@ -178,11 +178,12 @@ ifneq ($(CPU_ONLY), 1)
 	LIBRARIES := cudart cublas curand
 endif
 
-LIBRARIES += glog gflags protobuf boost_system boost_filesystem m hdf5_hl hdf5
+LIBRARIES += glog gflags protobuf boost_system boost_filesystem m
 
 # handle IO dependencies
 USE_LEVELDB ?= 1
 USE_LMDB ?= 1
+USE_HDF5 ?= 1
 USE_OPENCV ?= 1
 
 ifeq ($(USE_LEVELDB), 1)
@@ -191,13 +192,16 @@ endif
 ifeq ($(USE_LMDB), 1)
 	LIBRARIES += lmdb
 endif
+ifeq ($(USE_HDF5), 1)
+	LIBRARIES += hdf5_hl hdf5
+endif
 ifeq ($(USE_OPENCV), 1)
-	LIBRARIES += opencv_core opencv_highgui opencv_imgproc 
+	LIBRARIES += opencv_core opencv_highgui opencv_imgproc
 
 	ifeq ($(OPENCV_VERSION), 3)
 		LIBRARIES += opencv_imgcodecs
 	endif
-		
+
 endif
 PYTHON_LIBRARIES ?= boost_python python2.7
 WARNINGS := -Wall -Wno-sign-compare
@@ -341,6 +345,9 @@ ifeq ($(ALLOW_LMDB_NOLOCK), 1)
 	COMMON_FLAGS += -DALLOW_LMDB_NOLOCK
 endif
 endif
+ifeq ($(USE_HDF5), 1)
+	COMMON_FLAGS += -DUSE_HDF5
+endif
 
 # CPU-only configuration
 ifeq ($(CPU_ONLY), 1)

Makefile.config.example

@@ -11,6 +11,7 @@
 # USE_OPENCV := 0
 # USE_LEVELDB := 0
 # USE_LMDB := 0
+# USE_HDF5 := 0
 
 # uncomment to allow MDB_NOLOCK when reading LMDB files (only if necessary)
 #	You should not set this flag if you will be reading LMDBs with any

cmake/ConfigGen.cmake

@@ -71,6 +71,10 @@ function(caffe_generate_export_configs)
     list(APPEND Caffe_DEFINITIONS -DUSE_LEVELDB)
   endif()
 
+  if(USE_HDF5)
+    list(APPEND Caffe_DEFINITIONS -DUSE_HDF5)
+  endif()
+
   if(NOT HAVE_CUDNN)
     set(HAVE_CUDNN FALSE)
   else()

cmake/Dependencies.cmake

@@ -24,9 +24,12 @@ list(APPEND Caffe_LINKER_LIBS ${GFLAGS_LIBRARIES})
 include(cmake/ProtoBuf.cmake)
 
 # ---[ HDF5
-find_package(HDF5 COMPONENTS HL REQUIRED)
-include_directories(SYSTEM ${HDF5_INCLUDE_DIRS} ${HDF5_HL_INCLUDE_DIR})
-list(APPEND Caffe_LINKER_LIBS ${HDF5_LIBRARIES})
+if(USE_HDF5)
+  find_package(HDF5 COMPONENTS HL REQUIRED)
+  include_directories(SYSTEM ${HDF5_INCLUDE_DIRS} ${HDF5_HL_INCLUDE_DIR})
+  list(APPEND Caffe_LINKER_LIBS ${HDF5_LIBRARIES})
+  add_definitions(-DUSE_HDF5)
+endif()
 
 # ---[ LMDB
 if(USE_LMDB)

cmake/Modules/FindProtobuf.cmake

@@ -0,0 +1,231 @@
+# Locate and configure the Google Protocol Buffers library.
+#
+# The following variables can be set and are optional:
+#
+#   PROTOBUF_SRC_ROOT_FOLDER - When compiling with MSVC, if this cache variable is set
+#                              the protobuf-default VS project build locations
+#                              (vsprojects/Debug & vsprojects/Release) will be searched
+#                              for libraries and binaries.
+#
+#   PROTOBUF_IMPORT_DIRS     - List of additional directories to be searched for
+#                              imported .proto files. (New in CMake 2.8.8)
+#
+# Defines the following variables:
+#
+#   PROTOBUF_FOUND - Found the Google Protocol Buffers library (libprotobuf & header files)
+#   PROTOBUF_INCLUDE_DIRS - Include directories for Google Protocol Buffers
+#   PROTOBUF_LIBRARIES - The protobuf libraries
+# [New in CMake 2.8.5]
+#   PROTOBUF_PROTOC_LIBRARIES - The protoc libraries
+#   PROTOBUF_LITE_LIBRARIES - The protobuf-lite libraries
+#
+# The following cache variables are also available to set or use:
+#   PROTOBUF_LIBRARY - The protobuf library
+#   PROTOBUF_PROTOC_LIBRARY   - The protoc library
+#   PROTOBUF_INCLUDE_DIR - The include directory for protocol buffers
+#   PROTOBUF_PROTOC_EXECUTABLE - The protoc compiler
+# [New in CMake 2.8.5]
+#   PROTOBUF_LIBRARY_DEBUG - The protobuf library (debug)
+#   PROTOBUF_PROTOC_LIBRARY_DEBUG   - The protoc library (debug)
+#   PROTOBUF_LITE_LIBRARY - The protobuf lite library
+#   PROTOBUF_LITE_LIBRARY_DEBUG - The protobuf lite library (debug)
+#
+#  ====================================================================
+#  Example:
+#
+#   find_package(Protobuf REQUIRED)
+#   include_directories(${PROTOBUF_INCLUDE_DIRS})
+#
+#   include_directories(${CMAKE_CURRENT_BINARY_DIR})
+#   PROTOBUF_GENERATE_CPP(PROTO_SRCS PROTO_HDRS foo.proto)
+#   add_executable(bar bar.cc ${PROTO_SRCS} ${PROTO_HDRS})
+#   target_link_libraries(bar ${PROTOBUF_LIBRARIES})
+#
+# NOTE: You may need to link against pthreads, depending
+#       on the platform.
+#
+# NOTE: The PROTOBUF_GENERATE_CPP macro & add_executable() or add_library()
+#       calls only work properly within the same directory.
+#
+#  ====================================================================
+#
+# PROTOBUF_GENERATE_CPP (public function)
+#   SRCS = Variable to define with autogenerated
+#          source files
+#   HDRS = Variable to define with autogenerated
+#          header files
+#   ARGN = proto files
+#
+#  ====================================================================
+
+
+#=============================================================================
+# Copyright 2009 Kitware, Inc.
+# Copyright 2009-2011 Philip Lowman <philip@yhbt.com>
+# Copyright 2008 Esben Mose Hansen, Ange Optimization ApS
+#
+# Distributed under the OSI-approved BSD License (the "License");
+# see accompanying file Copyright.txt for details.
+#
+# This software is distributed WITHOUT ANY WARRANTY; without even the
+# implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+# See the License for more information.
+#=============================================================================
+# (To distribute this file outside of CMake, substitute the full
+#  License text for the above reference.)
+
+function(PROTOBUF_GENERATE_CPP SRCS HDRS)
+  if(NOT ARGN)
+    message(SEND_ERROR "Error: PROTOBUF_GENERATE_CPP() called without any proto files")
+    return()
+  endif()
+
+  if(PROTOBUF_GENERATE_CPP_APPEND_PATH)
+    # Create an include path for each file specified
+    foreach(FIL ${ARGN})
+      get_filename_component(ABS_FIL ${FIL} ABSOLUTE)
+      get_filename_component(ABS_PATH ${ABS_FIL} PATH)
+      list(FIND _protobuf_include_path ${ABS_PATH} _contains_already)
+      if(${_contains_already} EQUAL -1)
+          list(APPEND _protobuf_include_path -I ${ABS_PATH})
+      endif()
+    endforeach()
+  else()
+    set(_protobuf_include_path -I ${CMAKE_CURRENT_SOURCE_DIR})
+  endif()
+
+  if(DEFINED PROTOBUF_IMPORT_DIRS)
+    foreach(DIR ${PROTOBUF_IMPORT_DIRS})
+      get_filename_component(ABS_PATH ${DIR} ABSOLUTE)
+      list(FIND _protobuf_include_path ${ABS_PATH} _contains_already)
+      if(${_contains_already} EQUAL -1)
+          list(APPEND _protobuf_include_path -I ${ABS_PATH})
+      endif()
+    endforeach()
+  endif()
+
+  set(${SRCS})
+  set(${HDRS})
+  foreach(FIL ${ARGN})
+    get_filename_component(ABS_FIL ${FIL} ABSOLUTE)
+    get_filename_component(FIL_WE ${FIL} NAME_WE)
+
+    list(APPEND ${SRCS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.cc")
+    list(APPEND ${HDRS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.h")
+
+    add_custom_command(
+      OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.cc"
+             "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.h"
+      COMMAND  ${PROTOBUF_PROTOC_EXECUTABLE}
+      ARGS --cpp_out  ${CMAKE_CURRENT_BINARY_DIR} ${_protobuf_include_path} ${ABS_FIL}
+      DEPENDS ${ABS_FIL}
+      COMMENT "Running C++ protocol buffer compiler on ${FIL}"
+      VERBATIM )
+  endforeach()
+
+  set_source_files_properties(${${SRCS}} ${${HDRS}} PROPERTIES GENERATED TRUE)
+  set(${SRCS} ${${SRCS}} PARENT_SCOPE)
+  set(${HDRS} ${${HDRS}} PARENT_SCOPE)
+endfunction()
+
+# Internal function: search for normal library as well as a debug one
+#    if the debug one is specified also include debug/optimized keywords
+#    in *_LIBRARIES variable
+function(_protobuf_find_libraries name filename)
+   find_library(${name}_LIBRARY
+       NAMES ${filename}
+       PATHS ${PROTOBUF_SRC_ROOT_FOLDER}/vsprojects/Release)
+   mark_as_advanced(${name}_LIBRARY)
+
+   find_library(${name}_LIBRARY_DEBUG
+       NAMES ${filename}
+       PATHS ${PROTOBUF_SRC_ROOT_FOLDER}/vsprojects/Debug)
+   mark_as_advanced(${name}_LIBRARY_DEBUG)
+
+   if(NOT ${name}_LIBRARY_DEBUG)
+      # There is no debug library
+      set(${name}_LIBRARY_DEBUG ${${name}_LIBRARY} PARENT_SCOPE)
+      set(${name}_LIBRARIES     ${${name}_LIBRARY} PARENT_SCOPE)
+   else()
+      # There IS a debug library
+      set(${name}_LIBRARIES
+          optimized ${${name}_LIBRARY}
+          debug     ${${name}_LIBRARY_DEBUG}
+          PARENT_SCOPE
+      )
+   endif()
+endfunction()
+
+# Internal function: find threads library
+function(_protobuf_find_threads)
+    set(CMAKE_THREAD_PREFER_PTHREAD TRUE)
+    find_package(Threads)
+    if(Threads_FOUND)
+        list(APPEND PROTOBUF_LIBRARIES ${CMAKE_THREAD_LIBS_INIT})
+        set(PROTOBUF_LIBRARIES "${PROTOBUF_LIBRARIES}" PARENT_SCOPE)
+    endif()
+endfunction()
+
+#
+# Main.
+#
+
+# By default have PROTOBUF_GENERATE_CPP macro pass -I to protoc
+# for each directory where a proto file is referenced.
+if(NOT DEFINED PROTOBUF_GENERATE_CPP_APPEND_PATH)
+  set(PROTOBUF_GENERATE_CPP_APPEND_PATH TRUE)
+endif()
+
+
+# Google's provided vcproj files generate libraries with a "lib"
+# prefix on Windows
+if(MSVC)
+    set(PROTOBUF_ORIG_FIND_LIBRARY_PREFIXES "${CMAKE_FIND_LIBRARY_PREFIXES}")
+    set(CMAKE_FIND_LIBRARY_PREFIXES "lib" "")
+
+    find_path(PROTOBUF_SRC_ROOT_FOLDER protobuf.pc.in)
+endif()
+
+# The Protobuf library
+_protobuf_find_libraries(PROTOBUF protobuf)
+#DOC "The Google Protocol Buffers RELEASE Library"
+
+_protobuf_find_libraries(PROTOBUF_LITE protobuf-lite)
+
+# The Protobuf Protoc Library
+_protobuf_find_libraries(PROTOBUF_PROTOC protoc)
+
+# Restore original find library prefixes
+if(MSVC)
+    set(CMAKE_FIND_LIBRARY_PREFIXES "${PROTOBUF_ORIG_FIND_LIBRARY_PREFIXES}")
+endif()
+
+if(UNIX)
+    _protobuf_find_threads()
+endif()
+
+# Find the include directory
+find_path(PROTOBUF_INCLUDE_DIR
+    google/protobuf/service.h
+    PATHS ${PROTOBUF_SRC_ROOT_FOLDER}/src
+)
+mark_as_advanced(PROTOBUF_INCLUDE_DIR)
+
+# Find the protoc Executable
+find_host_program(PROTOBUF_PROTOC_EXECUTABLE
+    NAMES protoc
+    DOC "The Google Protocol Buffers Compiler"
+    PATHS
+    ${PROTOBUF_SRC_ROOT_FOLDER}/vsprojects/Release
+    ${PROTOBUF_SRC_ROOT_FOLDER}/vsprojects/Debug
+)
+mark_as_advanced(PROTOBUF_PROTOC_EXECUTABLE)
+
+
+include(FindPackageHandleStandardArgs)
+FIND_PACKAGE_HANDLE_STANDARD_ARGS(PROTOBUF DEFAULT_MSG
+    PROTOBUF_LIBRARY PROTOBUF_INCLUDE_DIR)
+
+if(PROTOBUF_FOUND)
+    set(PROTOBUF_INCLUDE_DIRS ${PROTOBUF_INCLUDE_DIR})
+endif()

cmake/Summary.cmake

@@ -118,6 +118,7 @@ function(caffe_print_configuration_summary)
   caffe_status("  USE_LEVELDB       :   ${USE_LEVELDB}")
   caffe_status("  USE_LMDB          :   ${USE_LMDB}")
   caffe_status("  ALLOW_LMDB_NOLOCK :   ${ALLOW_LMDB_NOLOCK}")
+  caffe_status("  USE_HDF5          :   ${USE_HDF5}")
   caffe_status("")
   caffe_status("Dependencies:")
   caffe_status("  BLAS              : " APPLE THEN "Yes (vecLib)" ELSE "Yes (${BLAS})")

cmake/Templates/caffe_config.h.in

@@ -36,3 +36,4 @@
 #cmakedefine USE_LEVELDB
 #cmakedefine USE_LMDB
 #cmakedefine ALLOW_LMDB_NOLOCK
+#cmakedefine USE_HDF5

docs/installation.md

@@ -34,12 +34,12 @@ Caffe has several dependencies:
     * 5.5, and 5.0 are compatible but considered legacy
 * [BLAS](http://en.wikipedia.org/wiki/Basic_Linear_Algebra_Subprograms) via ATLAS, MKL, or OpenBLAS.
 * [Boost](http://www.boost.org/) >= 1.55
-* `protobuf`, `glog`, `gflags`, `hdf5`
+* `protobuf`, `glog`, `gflags`
 
 Optional dependencies:
 
 * [OpenCV](http://opencv.org/) >= 2.4 including 3.0
-* IO libraries: `lmdb`, `leveldb` (note: leveldb requires `snappy`)
+* IO libraries: `lmdb`, `leveldb` (note: leveldb requires `snappy`), `hdf5`
 * cuDNN for GPU acceleration (v4)
 
 Pycaffe and Matcaffe interfaces have their own natural needs.

include/caffe/util/hdf5.hpp

@@ -1,3 +1,4 @@
+#ifdef USE_HDF5
 #ifndef CAFFE_UTIL_HDF5_H_
 #define CAFFE_UTIL_HDF5_H_
 
@@ -37,3 +38,4 @@ string hdf5_get_name_by_idx(hid_t loc_id, int idx);
 }  // namespace caffe
 
 #endif   // CAFFE_UTIL_HDF5_H_
+#endif   // USE_HDF5

include/caffe/util/math_functions.hpp

@@ -3,6 +3,7 @@
 
 #include <stdint.h>
 #include <cmath>  // for std::fabs and std::signbit
+#include <cstring>  // for memset
 
 #include "glog/logging.h"
 

src/caffe/layers/hdf5_data_layer.cpp

@@ -1,3 +1,4 @@
+#ifdef USE_HDF5
 /*
 TODO:
 - load file in a separate thread ("prefetch")
@@ -164,3 +165,4 @@ INSTANTIATE_CLASS(HDF5DataLayer);
 REGISTER_LAYER_CLASS(HDF5Data);
 
 }  // namespace caffe
+#endif  // USE_HDF5