refactor:Refactor the cmake build (#378)

* Refactor the cmake compilation
OpenAtomFoundation · Jul 20, 2024 · b987ba9 · b987ba9
1 parent 1112122
commit b987ba9
Show file tree

Hide file tree

Showing 28 changed files with 495 additions and 460 deletions.
diff --git a/.github/workflows/import_braft.yml b/.github/workflows/import_braft.yml
@@ -26,7 +26,7 @@ jobs:
     steps:
       - uses: actions/checkout@v4
 
-      - name: Build
+      - name: Build --verbose
         env: 
           CPLUS_INCLUDE_PATH: /opt/homebrew/include
         run: |
@@ -50,7 +50,7 @@ jobs:
 
       - name: Build
         run: |
-          bash build.sh
+          bash build.sh --verbose
 
       - name: Run Go E2E Tests
         working-directory: ${{ github.workspace }}/build-release

diff --git a/.github/workflows/pikiwidb.yml b/.github/workflows/pikiwidb.yml
@@ -30,7 +30,7 @@ jobs:
         run: |
           brew install autoconf
           brew install go
-          sh build.sh
+          sh build.sh --verbose
 
       - name: GTest
         working-directory: ${{ github.workspace }}/build-release
@@ -54,7 +54,7 @@ jobs:
 
       - name: Build
         run: |
-          bash build.sh
+          bash build.sh --verbose
 
       - name: GTest
         working-directory: ${{ github.workspace }}/build-release

diff --git a/.gitignore b/.gitignore
@@ -66,6 +66,8 @@ build-release/
 build-debug/
 buildtrees
 deps
+deps-debug/
+deps-release/
 
 #develop container
 .devcontainer
@@ -83,4 +85,4 @@ build_support/__pycache__
 build_support/clang_format_exclusions.txt
 
 # pkg
-pkg
+pkg
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -6,7 +6,7 @@
 CMAKE_MINIMUM_REQUIRED(VERSION 3.14)
 PROJECT(PikiwiDB)
 
-SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Werror=return-type -D'GIT_COMMIT_ID=\"${GIT_COMMIT_ID}\"'")
+SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Werror=return-type")
 
 # Avoid warning about DOWNLOAD_EXTRACT_TIMESTAMP in CMake 3.24:
 IF (CMAKE_VERSION VERSION_GREATER_EQUAL "3.24.0")
@@ -15,6 +15,7 @@ ENDIF ()
 
 SET(CMAKE_CXX_STANDARD 20)
 SET(CMAKE_EXPORT_COMPILE_COMMANDS ON)
+
 ENABLE_TESTING()
 
 IF (CMAKE_CXX_COMPILER_ID STREQUAL "Clang")
@@ -93,20 +94,23 @@ ELSEIF (HOST_ARCH MATCHES "arm")
 ENDIF ()
 
 IF (CMAKE_BUILD_TYPE STREQUAL "Release")
-    SET(EP_BASE_SUFFIX "build-release/buildtrees")
+    SET(EP_BASE_SUFFIX "build-release")
+    SET(LIB_SUFFIX "deps-release")
 ELSE ()
-    SET(EP_BASE_SUFFIX "build-debug/buildtrees")
+    SET(EP_BASE_SUFFIX "build-debug")
+    SET(LIB_SUFFIX "deps-debug")
 ENDIF ()
 
 SET_PROPERTY(DIRECTORY PROPERTY EP_BASE ${CMAKE_CURRENT_SOURCE_DIR}/${EP_BASE_SUFFIX})
 SET(CMAKE_MODULE_PATH "${CMAKE_CURRENT_LIST_DIR}/cmake/modules/")
-SET(STAGED_INSTALL_PREFIX ${CMAKE_CURRENT_SOURCE_DIR}/deps)
-SET(CMAKE_UTILS_DIR ${CMAKE_CURRENT_SOURCE_DIR}/utils)
-SET(INSTALL_INCLUDEDIR ${STAGED_INSTALL_PREFIX}/include)
-SET(INSTALL_LIBDIR ${STAGED_INSTALL_PREFIX}/lib)
-SET(INSTALL_LIBDIR_64 ${STAGED_INSTALL_PREFIX}/lib64)
-SET(CMAKE_PREFIX_PATH ${CMAKE_PREFIX_PATH} ${STAGED_INSTALL_PREFIX})
 SET(BUILD_SUPPORT_DIR ${PROJECT_SOURCE_DIR}/build_support)
+
+SET(PLIB_INSTALL_DIR ${CMAKE_BINARY_DIR}/lib)
+SET(LIB_INSTALL_PREFIX ${CMAKE_CURRENT_SOURCE_DIR}/${LIB_SUFFIX})
+SET(LIB_INCLUDE_DIR ${LIB_INSTALL_PREFIX}/include)
+SET(LIB_INSTALL_DIR ${LIB_INSTALL_PREFIX}/lib)
+SET(LIB_SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/${EP_BASE_SUFFIX}/Source)
+
 # make sure we use the same compiler for all dependencies
 SET(CMAKE_POSITION_INDEPENDENT_CODE ON)
 
@@ -121,11 +125,13 @@ SET(THIRD_PARTY_PATH ${CMAKE_CURRENT_BINARY_DIR}/third-party)
 SET(THIRD_PARTY_BUILD_TYPE ${CMAKE_BUILD_TYPE})
 SET(EXTERNAL_PROJECT_LOG_ARGS
         LOG_DOWNLOAD 0
-        LOG_UPDATE 1
+        LOG_UPDATE 0
         LOG_CONFIGURE 0
         LOG_BUILD 0
-        LOG_TEST 1
-        LOG_INSTALL 0)
+        LOG_TEST 0
+        LOG_INSTALL 0
+        BUILD_ALWAYS 0
+)
 
 IF(CMAKE_SYSTEM_NAME STREQUAL "Linux")
     SET(LIB rt)
@@ -145,33 +151,36 @@ ELSEIF(CMAKE_SYSTEM_NAME STREQUAL "Darwin")
 ENDIF()
 SET(LIB ${LIB} CACHE INTERNAL "libs which should be linked for executable target")
 
-INCLUDE(FetchContent)
+INCLUDE(ExternalProject)
 
+INCLUDE(cmake/findTools.cmake)
 INCLUDE(cmake/openssl.cmake)
+INCLUDE(cmake/lz4.cmake)
+INCLUDE(cmake/zstd.cmake)
 INCLUDE(cmake/gflags.cmake)
-INCLUDE(cmake/findTools.cmake)
 INCLUDE(cmake/snappy.cmake)
+INCLUDE(cmake/zlib.cmake)
 INCLUDE(cmake/leveldb.cmake)
 INCLUDE(cmake/libevent.cmake)
 INCLUDE(cmake/llhttp.cmake)
 INCLUDE(cmake/fmt.cmake)
 INCLUDE(cmake/spdlog.cmake)
 INCLUDE(cmake/gtest.cmake)
 INCLUDE(cmake/rocksdb.cmake)
-INCLUDE(cmake/zlib.cmake)
 INCLUDE(cmake/protobuf.cmake)
 INCLUDE(cmake/brpc.cmake)
 INCLUDE(cmake/braft.cmake)
 
-ENABLE_TESTING()
-
 SET(PROTO_OUTPUT_DIR "${CMAKE_BINARY_DIR}/generated_pb")
 FILE(MAKE_DIRECTORY "${PROTO_OUTPUT_DIR}")
 
+#INCLUDE(cmake/protogen.cmake)
+
 ADD_SUBDIRECTORY(src/pstd)
 ADD_SUBDIRECTORY(src/net)
 ADD_SUBDIRECTORY(src/praft)
 ADD_SUBDIRECTORY(src/storage)
+
 ADD_SUBDIRECTORY(src)
 
 #############################################################################

diff --git a/build.sh b/build.sh
@@ -16,7 +16,7 @@ BUILD_TYPE=Release
 VERBOSE=0
 CMAKE_FLAGS=""
 MAKE_FLAGS=""
-PREFIX="build"
+PREFIX="cmake-build"
 
 ARGS=`getopt -a -o h -l help,debug,verbose,prefix: -- "$@"`
 function show_help() {

diff --git a/cmake/braft.cmake b/cmake/braft.cmake
@@ -5,12 +5,14 @@
 
 INCLUDE(ExternalProject)
 
-SET(BRAFT_SOURCES_DIR ${THIRD_PARTY_PATH}/braft)
-SET(BRAFT_INSTALL_DIR ${THIRD_PARTY_PATH}/install/braft)
-SET(BRAFT_INCLUDE_DIR "${BRAFT_INSTALL_DIR}/include" CACHE PATH "braft include directory." FORCE)
-SET(BRAFT_LIBRARIES "${BRAFT_INSTALL_DIR}/lib/libbraft.a" CACHE FILEPATH "braft library." FORCE)
+SET(BRAFT_SOURCES_DIR ${LIB_INSTALL_PREFIX})
+SET(BRAFT_INSTALL_DIR ${LIB_INSTALL_PREFIX})
+SET(BRAFT_INCLUDE_DIR "${LIB_INCLUDE_DIR}" CACHE PATH "brpc include directory." FORCE)
+SET(BRAFT_LIBRARIES "${LIB_INSTALL_DIR}/libbraft.a" CACHE FILEPATH "brpc library." FORCE)
 
-SET(prefix_path "${THIRD_PARTY_PATH}/install/brpc|${CMAKE_CURRENT_BINARY_DIR}/_deps/gflags-build|${THIRD_PARTY_PATH}/install/protobuf|${THIRD_PARTY_PATH}/install/zlib|${THIRD_PARTY_PATH}/install/leveldb")
+IF (CMAKE_SYSTEM_NAME STREQUAL "Darwin")
+    SET(BRAFT_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wl,-U,__Z13GetStackTracePPvii")
+ENDIF ()
 
 ExternalProject_Add(
         extern_braft
@@ -19,30 +21,31 @@ ExternalProject_Add(
         GIT_REPOSITORY "https://github.com/pikiwidb/braft.git"
         GIT_TAG master
         GIT_SHALLOW true
-        PREFIX ${BRAFT_SOURCES_DIR}
-        UPDATE_COMMAND ""
-        CMAKE_ARGS -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}
-        -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER}
-        -DCMAKE_CXX_FLAGS=${CMAKE_CXX_FLAGS}
-        -DCMAKE_C_FLAGS=${CMAKE_C_FLAGS}
-        -DCMAKE_INSTALL_PREFIX=${BRAFT_INSTALL_DIR}
-        -DCMAKE_INSTALL_LIBDIR=${BRAFT_INSTALL_DIR}/lib
+        CMAKE_ARGS
+        -DCMAKE_BUILD_TYPE=${LIB_BUILD_TYPE}
+        -DCMAKE_CXX_FLAGS=${BRAFT_CXX_FLAGS}
+        -DCMAKE_INSTALL_PREFIX=${LIB_INSTALL_PREFIX}
+        -DBRPC_LIB=${BRPC_LIBRARIES}
+        -DBRPC_INCLUDE_PATH=${BRPC_INCLUDE_DIR}
         -DCMAKE_POSITION_INDEPENDENT_CODE=ON
-        -DCMAKE_BUILD_TYPE=${THIRD_PARTY_BUILD_TYPE}
-        -DCMAKE_PREFIX_PATH=${prefix_path}
         -DBRPC_WITH_GLOG=OFF
-        -DOPENSSL_ROOT_DIR=${THIRD_PARTY_PATH}/install/openssl
-        ${EXTERNAL_OPTIONAL_ARGS}
-        LIST_SEPARATOR |
-        CMAKE_CACHE_ARGS -DCMAKE_INSTALL_PREFIX:PATH=${BRAFT_INSTALL_DIR}
-        -DCMAKE_INSTALL_LIBDIR:PATH=${BRAFT_INSTALL_DIR}/lib
-        -DCMAKE_POSITION_INDEPENDENT_CODE:BOOL=ON
-        -DCMAKE_BUILD_TYPE:STRING=${THIRD_PARTY_BUILD_TYPE}
-        BUILD_IN_SOURCE 1
-        BUILD_COMMAND $(MAKE) -j ${CPU_CORE} braft-static
-        INSTALL_COMMAND mkdir -p ${BRAFT_INSTALL_DIR}/lib/ COMMAND cp ${BRAFT_SOURCES_DIR}/src/extern_braft/output/lib/libbraft.a ${BRAFT_LIBRARIES} COMMAND rm -rf ${BRAFT_INCLUDE_DIR} COMMAND cp -r ${BRAFT_SOURCES_DIR}/src/extern_braft/output/include ${BRAFT_INCLUDE_DIR}
+        -DCMAKE_FIND_LIBRARY_SUFFIXES=${LIB_INSTALL_PREFIX}
+        -DLEVELDB_LIB=${LEVELDB_LIBRARIES}
+        -DLEVELDB_INCLUDE_PATH=${LEVELDB_INCLUDE_DIR}
+
+        -DGFLAGS_INCLUDE_PATH=${GFLAGS_INCLUDE_DIR}
+        -DGFLAGS_LIB=${GFLAGS_LIBRARIES}
+
+        -DPROTOC_LIB=${PROTOC_LIBRARY}
+        -DPROTOBUF_LIBRARIES=${PROTOBUF_LIBRARY}
+        -DPROTOBUF_INCLUDE_DIRS=${PROTOBUF_INCLUDE_DIR}
+        -DPROTOBUF_PROTOC_EXECUTABLE=${PROTOBUF_PROTOC}
+        -DOPENSSL_INCLUDE_DIR=${OPENSSL_INCLUDE_DIR}
+        -DCMAKE_POSITION_INDEPENDENT_CODE=ON
+        BUILD_COMMAND make -j${CPU_CORE}
 )
-ADD_DEPENDENCIES(extern_braft brpc)
+
+ADD_DEPENDENCIES(extern_braft brpc gflags)
 ADD_LIBRARY(braft STATIC IMPORTED GLOBAL)
 SET_PROPERTY(TARGET braft PROPERTY IMPORTED_LOCATION ${BRAFT_LIBRARIES})
-ADD_DEPENDENCIES(braft extern_braft)
+ADD_DEPENDENCIES(braft extern_braft)
diff --git a/cmake/brpc.cmake b/cmake/brpc.cmake
@@ -3,44 +3,46 @@
 # LICENSE file in the root directory of this source tree. An additional grant
 # of patent rights can be found in the PATENTS file in the same directory.
 
-INCLUDE(ExternalProject)
+SET(BRPC_SOURCES_DIR ${LIB_INSTALL_PREFIX})
+SET(BRPC_INSTALL_DIR ${LIB_INSTALL_PREFIX})
+SET(BRPC_INCLUDE_DIR "${LIB_INCLUDE_DIR}" CACHE PATH "brpc include directory." FORCE)
+SET(BRPC_LIBRARIES "${LIB_INSTALL_DIR}/libbrpc.a" CACHE FILEPATH "brpc library." FORCE)
 
-SET(BRPC_SOURCES_DIR ${THIRD_PARTY_PATH}/brpc)
-SET(BRPC_INSTALL_DIR ${THIRD_PARTY_PATH}/install/brpc)
-SET(BRPC_INCLUDE_DIR "${BRPC_INSTALL_DIR}/include" CACHE PATH "brpc include directory." FORCE)
-SET(BRPC_LIBRARIES "${BRPC_INSTALL_DIR}/lib/libbrpc.a" CACHE FILEPATH "brpc library." FORCE)
+SET(CMAKE_FIND_LIBRARY_SUFFIXES ${LIB_INSTALL_PREFIX})
 
 # Reference https://stackoverflow.com/questions/45414507/pass-a-list-of-prefix-paths-to-externalproject-add-in-cmake-args
-SET(prefix_path "${CMAKE_CURRENT_BINARY_DIR}/_deps/gflags-build|${THIRD_PARTY_PATH}/install/protobuf|${THIRD_PARTY_PATH}/install/zlib|${THIRD_PARTY_PATH}/install/leveldb|${THIRD_PARTY_PATH}/install/openssl")
-SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-deprecated-declarations")
+SET(CMAKE_CPP_FLAGS "${CMAKE_CXX_FLAGS} -Wno-deprecated-declarations")
 # If minimal .a is need, you can set  WITH_DEBUG_SYMBOLS=OFF
-EXTERNALPROJECT_ADD( 
+ExternalProject_Add(
         extern_brpc
         ${EXTERNAL_PROJECT_LOG_ARGS}
         DEPENDS ssl crypto zlib protobuf leveldb gflags
         URL https://github.com/apache/brpc/archive/refs/tags/1.8.0.tar.gz
         URL_HASH SHA256=13ffb2f1f57c679379a20367c744b3e597614a793ec036cd7580aae90798019d
-        PREFIX ${BRPC_SOURCES_DIR}
-        UPDATE_COMMAND ""
-        CMAKE_ARGS -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}
-        -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER}
-        -DCMAKE_CXX_FLAGS=${CMAKE_CXX_FLAGS}
-        -DCMAKE_C_FLAGS=${CMAKE_C_FLAGS}
+        CMAKE_ARGS
+        -DCMAKE_BUILD_TYPE=${LIB_BUILD_TYPE}
+        -DCMAKE_CPP_FLAGS=${CMAKE_CPP_FLAGS}
         -DCMAKE_INSTALL_PREFIX=${BRPC_INSTALL_DIR}
-        -DCMAKE_INSTALL_LIBDIR=${BRPC_INSTALL_DIR}/lib
+        -DCMAKE_FIND_LIBRARY_SUFFIXES=${LIB_INSTALL_PREFIX}
+        -DGFLAGS_INCLUDE_PATH=${GFLAGS_INCLUDE_DIR}
+        -DBUILD_SHARED_LIBS=FALSE
+        -DBUILD_BRPC_TOOLS=OFF
+        -DGFLAGS_LIBRARY=${GFLAGS_LIBRARIES}
+
+        -DLEVELDB_LIB=${LEVELDB_LIBRARIES}
+        -DLEVELDB_INCLUDE_PATH=${LEVELDB_INCLUDE_DIR}
+
+        -DPROTOC_LIB=${PROTOC_LIBRARY}
+        -DPROTOBUF_LIBRARIES=${PROTOBUF_LIBRARY}
+        -DPROTOBUF_INCLUDE_DIRS=${PROTOBUF_INCLUDE_DIR}
+        -DPROTOBUF_PROTOC_EXECUTABLE=${PROTOBUF_PROTOC}
+
+        -DOPENSSL_INCLUDE_DIR=${OPENSSL_INCLUDE_DIR}
         -DCMAKE_POSITION_INDEPENDENT_CODE=ON
-        -DCMAKE_BUILD_TYPE=${THIRD_PARTY_BUILD_TYPE}
-        -DCMAKE_PREFIX_PATH=${prefix_path}
+        -DPROJECT_BINARY_DIR=${LIB_INSTALL_PREFIX}
         -DWITH_GLOG=OFF
         -DDOWNLOAD_GTEST=OFF
-        ${EXTERNAL_OPTIONAL_ARGS}
-        LIST_SEPARATOR |
-        CMAKE_CACHE_ARGS -DCMAKE_INSTALL_PREFIX:PATH=${BRPC_INSTALL_DIR}
-        -DCMAKE_INSTALL_LIBDIR:PATH=${BRPC_INSTALL_DIR}/lib
-        -DCMAKE_POSITION_INDEPENDENT_CODE:BOOL=ON
-        -DCMAKE_BUILD_TYPE:STRING=${THIRD_PARTY_BUILD_TYPE}
-        BUILD_IN_SOURCE 1
-        BUILD_COMMAND $(MAKE) -j ${CPU_CORE} brpc-static
+        BUILD_COMMAND make -j${CPU_CORE}
 )
 ADD_DEPENDENCIES(extern_brpc ssl crypto zlib protobuf leveldb gflags)
 ADD_LIBRARY(brpc STATIC IMPORTED GLOBAL)

diff --git a/cmake/fmt.cmake b/cmake/fmt.cmake
@@ -3,11 +3,30 @@
 # LICENSE file in the root directory of this source tree. An additional grant
 # of patent rights can be found in the PATENTS file in the same directory.
 
-INCLUDE_GUARD()
 
-FetchContent_Declare(fmt
+IF (${LIB_BUILD_TYPE} STREQUAL DEBUG)
+    SET(LIB_FMT libfmtd.a)
+ELSE ()
+    SET(LIB_FMT libfmt.a)
+ENDIF ()
+
+SET(FMT_INCLUDE_DIR "${LIB_INCLUDE_DIR}" CACHE PATH "fmt include directory." FORCE)
+SET(FMT_LIBRARIES "${LIB_INSTALL_DIR}/${LIB_FMT}" CACHE FILEPATH "fmt library directory." FORCE)
+
+ExternalProject_Add(
+        extern_fmt
         URL https://github.com/fmtlib/fmt/archive/10.1.1.zip
         URL_HASH SHA256=3c2e73019178ad72b0614a3124f25de454b9ca3a1afe81d5447b8d3cbdb6d322
-        )
+        CMAKE_ARGS
+        -DCMAKE_INSTALL_PREFIX=${LIB_INSTALL_PREFIX}
+        -DCMAKE_BUILD_TYPE=${LIB_BUILD_TYPE}
+        -DFMT_DOC=FALSE
+        -DFMT_TEST=FALSE
+        -DSHARED_LIBS=FALSE
+        BUILD_COMMAND make -j${CPU_CORE}
+)
+
 
-FetchContent_MakeAvailable(fmt)
+ADD_LIBRARY(fmt STATIC IMPORTED GLOBAL)
+SET_PROPERTY(TARGET fmt PROPERTY IMPORTED_LOCATION ${FMT_LIBRARIES})
+ADD_DEPENDENCIES(fmt extern_fmt)