From a05f58e9183432b7b391b4dc7968f0716557fe03 Mon Sep 17 00:00:00 2001 From: david dali susanibar arce Date: Fri, 21 Apr 2023 15:23:55 -0500 Subject: [PATCH 1/8] feat: enable gcs on macosx --- ci/scripts/java_jni_macos_build.sh | 5 +++++ cpp/build-support/run-test.sh | 3 +++ cpp/thirdparty/versions.txt | 4 ++-- 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/ci/scripts/java_jni_macos_build.sh b/ci/scripts/java_jni_macos_build.sh index 0d4c332a6b11f..2c5f212a10e72 100755 --- a/ci/scripts/java_jni_macos_build.sh +++ b/ci/scripts/java_jni_macos_build.sh @@ -46,6 +46,7 @@ export ARROW_ACERO export ARROW_DATASET : ${ARROW_GANDIVA:=ON} export ARROW_GANDIVA +: ${ARROW_GCS:=ON} : ${ARROW_ORC:=ON} export ARROW_ORC : ${ARROW_PARQUET:=ON} @@ -67,6 +68,7 @@ mkdir -p "${build_dir}/cpp" pushd "${build_dir}/cpp" cmake \ + -Dabsl_SOURCE=BUNDLED \ -DARROW_ACERO=${ARROW_ACERO} \ -DARROW_BUILD_SHARED=OFF \ -DARROW_BUILD_TESTS=${ARROW_BUILD_TESTS} \ @@ -75,15 +77,18 @@ cmake \ -DARROW_DEPENDENCY_USE_SHARED=OFF \ -DARROW_GANDIVA=${ARROW_GANDIVA} \ -DARROW_GANDIVA_STATIC_LIBSTDCPP=ON \ + -DARROW_GCS=${ARROW_GCS} \ -DARROW_ORC=${ARROW_ORC} \ -DARROW_PARQUET=${ARROW_PARQUET} \ -DARROW_S3=${ARROW_S3} \ -DARROW_USE_CCACHE=${ARROW_USE_CCACHE} \ + -DARROW_WITH_SNAPPY=ON \ -DAWSSDK_SOURCE=BUNDLED \ -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} \ -DCMAKE_INSTALL_LIBDIR=lib \ -DCMAKE_INSTALL_PREFIX=${install_dir} \ -DCMAKE_UNITY_BUILD=${CMAKE_UNITY_BUILD} \ + -Dgoogle_cloud_cpp_storage_SOURCE=BUNDLED \ -DGTest_SOURCE=BUNDLED \ -DPARQUET_BUILD_EXAMPLES=OFF \ -DPARQUET_BUILD_EXECUTABLES=OFF \ diff --git a/cpp/build-support/run-test.sh b/cpp/build-support/run-test.sh index 7f68abd31c369..4838c1704965a 100755 --- a/cpp/build-support/run-test.sh +++ b/cpp/build-support/run-test.sh @@ -60,6 +60,9 @@ rm -f $LOGFILE $LOGFILE.gz pipe_cmd=cat +# Install the gcs testbench +sh ../../../../../ci/scripts/install_gcs_testbench.sh default + function setup_sanitizers() { # Sets environment variables for different sanitizers (it configures how) the run_tests. Function works. diff --git a/cpp/thirdparty/versions.txt b/cpp/thirdparty/versions.txt index 8c8bd5d711142..6b99dc73df94d 100644 --- a/cpp/thirdparty/versions.txt +++ b/cpp/thirdparty/versions.txt @@ -23,8 +23,8 @@ # `DEPENDENCIES` array (see the comment on top of the declaration for the # format). -ARROW_ABSL_BUILD_VERSION=20211102.0 -ARROW_ABSL_BUILD_SHA256_CHECKSUM=dcf71b9cba8dc0ca9940c4b316a0c796be8fab42b070bb6b7cab62b48f0e66c4 +ARROW_ABSL_BUILD_VERSION=20230125.2 +ARROW_ABSL_BUILD_SHA256_CHECKSUM=9a2b5752d7bfade0bdeee2701de17c9480620f8b237e1964c1b9967c75374906 ARROW_AWS_C_AUTH_BUILD_VERSION=v0.6.22 ARROW_AWS_C_AUTH_BUILD_SHA256_CHECKSUM=691a6b4418afcd3dc141351b6ad33fccd8e3ff84df0e9e045b42295d284ee14c ARROW_AWS_C_CAL_BUILD_VERSION=v0.5.20 From acf8e93ceef84e838bdf5809dbe5e446bcdd4fcf Mon Sep 17 00:00:00 2001 From: david dali susanibar arce Date: Mon, 24 Apr 2023 09:47:18 -0500 Subject: [PATCH 2/8] Update cpp/build-support/run-test.sh Co-authored-by: Jacob Wujciak-Jens --- cpp/build-support/run-test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/build-support/run-test.sh b/cpp/build-support/run-test.sh index 4838c1704965a..ee0b5b7a30d38 100755 --- a/cpp/build-support/run-test.sh +++ b/cpp/build-support/run-test.sh @@ -61,7 +61,7 @@ rm -f $LOGFILE $LOGFILE.gz pipe_cmd=cat # Install the gcs testbench -sh ../../../../../ci/scripts/install_gcs_testbench.sh default +sh $ROOT/../ci/scripts/install_gcs_testbench.sh default function setup_sanitizers() { # Sets environment variables for different sanitizers (it configures how) the run_tests. Function works. From d6da99d68488565df82cca8a8e0d5e2e301955e2 Mon Sep 17 00:00:00 2001 From: david dali susanibar arce Date: Mon, 24 Apr 2023 10:11:13 -0500 Subject: [PATCH 3/8] fix: Symbol not found: _curl_multi_poll --- java/Brewfile | 1 + 1 file changed, 1 insertion(+) diff --git a/java/Brewfile b/java/Brewfile index af6bd65615d62..cc691044f278b 100644 --- a/java/Brewfile +++ b/java/Brewfile @@ -17,3 +17,4 @@ brew "openjdk@11" brew "sccache" +brew "curl" From 26d808d39314845889b179d53b90367d1ae4d119 Mon Sep 17 00:00:00 2001 From: david dali susanibar arce Date: Fri, 12 May 2023 12:11:06 -0500 Subject: [PATCH 4/8] fix: Symbol not found: _curl_multi_poll --- dev/tasks/java-jars/github.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/dev/tasks/java-jars/github.yml b/dev/tasks/java-jars/github.yml index 4e1f1e4fa161f..e62267db7cf35 100644 --- a/dev/tasks/java-jars/github.yml +++ b/dev/tasks/java-jars/github.yml @@ -108,6 +108,7 @@ jobs: set -e # make brew Java available to CMake if [ "{{ arch }}" = "aarch_64" ]; then + export CURL_ROOT=$(brew --prefix curl) export JAVA_HOME=$(brew --prefix openjdk@11)/libexec/openjdk.jdk/Contents/Home fi arrow/ci/scripts/java_jni_macos_build.sh \ From 2213140d42f0ec688d958ee0c7b114a1f452f3f7 Mon Sep 17 00:00:00 2001 From: david dali susanibar arce Date: Mon, 5 Jun 2023 10:42:27 -0500 Subject: [PATCH 5/8] fix: testbench, clean and rebase branch --- cpp/build-support/run-test.sh | 3 --- dev/tasks/java-jars/github.yml | 1 + java/Brewfile | 1 - 3 files changed, 1 insertion(+), 4 deletions(-) diff --git a/cpp/build-support/run-test.sh b/cpp/build-support/run-test.sh index 77540003da37b..8e42438a23c1c 100755 --- a/cpp/build-support/run-test.sh +++ b/cpp/build-support/run-test.sh @@ -60,9 +60,6 @@ rm -f $LOGFILE $LOGFILE.gz pipe_cmd=cat -# Install the gcs testbench -sh $ROOT/../ci/scripts/install_gcs_testbench.sh default - function setup_sanitizers() { # Sets environment variables for different sanitizers (it configures how) the run_tests. Function works. diff --git a/dev/tasks/java-jars/github.yml b/dev/tasks/java-jars/github.yml index e62267db7cf35..d0dbfb3bd6464 100644 --- a/dev/tasks/java-jars/github.yml +++ b/dev/tasks/java-jars/github.yml @@ -111,6 +111,7 @@ jobs: export CURL_ROOT=$(brew --prefix curl) export JAVA_HOME=$(brew --prefix openjdk@11)/libexec/openjdk.jdk/Contents/Home fi + arrow/ci/scripts/install_gcs_testbench.sh default arrow/ci/scripts/java_jni_macos_build.sh \ $GITHUB_WORKSPACE/arrow \ $GITHUB_WORKSPACE/arrow/cpp-build \ diff --git a/java/Brewfile b/java/Brewfile index cc691044f278b..af6bd65615d62 100644 --- a/java/Brewfile +++ b/java/Brewfile @@ -17,4 +17,3 @@ brew "openjdk@11" brew "sccache" -brew "curl" From e1db91b809315fe93c899435df16c6edd9de468f Mon Sep 17 00:00:00 2001 From: david dali susanibar arce Date: Mon, 5 Jun 2023 20:28:35 -0500 Subject: [PATCH 6/8] fix: define cmake options for GCS dependencies --- ci/scripts/java_jni_macos_build.sh | 1 - cpp/cmake_modules/DefineOptions.cmake | 5 ++++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/ci/scripts/java_jni_macos_build.sh b/ci/scripts/java_jni_macos_build.sh index 918a9510cba33..5aab00d5a5661 100755 --- a/ci/scripts/java_jni_macos_build.sh +++ b/ci/scripts/java_jni_macos_build.sh @@ -83,7 +83,6 @@ cmake \ -DARROW_PARQUET=${ARROW_PARQUET} \ -DARROW_S3=${ARROW_S3} \ -DARROW_USE_CCACHE=${ARROW_USE_CCACHE} \ - -DARROW_WITH_SNAPPY=ON \ -DAWSSDK_SOURCE=BUNDLED \ -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} \ -DCMAKE_INSTALL_LIBDIR=lib \ diff --git a/cpp/cmake_modules/DefineOptions.cmake b/cpp/cmake_modules/DefineOptions.cmake index e2ad9e7a8b932..247a190ae6562 100644 --- a/cpp/cmake_modules/DefineOptions.cmake +++ b/cpp/cmake_modules/DefineOptions.cmake @@ -335,7 +335,10 @@ takes precedence over ccache if a storage backend is configured" ON) ARROW_WITH_UTF8PROC) define_option(ARROW_GCS - "Build Arrow with GCS support (requires the GCloud SDK for C++)" OFF) + "Build Arrow with GCS support (requires the GCloud SDK for C++)" + OFF + DEPENDS + ARROW_WITH_SNAPPY) define_option(ARROW_HDFS "Build the Arrow HDFS bridge" OFF) From c5ad4752304b5e303c7f3727ab4c316cbc90804a Mon Sep 17 00:00:00 2001 From: david dali susanibar arce Date: Wed, 28 Jun 2023 17:11:30 -0500 Subject: [PATCH 7/8] fix: update absl build to 20230125.3 --- cpp/thirdparty/versions.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cpp/thirdparty/versions.txt b/cpp/thirdparty/versions.txt index 2ba2ee5bd8a06..f9da026d075b8 100644 --- a/cpp/thirdparty/versions.txt +++ b/cpp/thirdparty/versions.txt @@ -23,8 +23,8 @@ # `DEPENDENCIES` array (see the comment on top of the declaration for the # format). -ARROW_ABSL_BUILD_VERSION=20230125.2 -ARROW_ABSL_BUILD_SHA256_CHECKSUM=9a2b5752d7bfade0bdeee2701de17c9480620f8b237e1964c1b9967c75374906 +ARROW_ABSL_BUILD_VERSION=20230125.3 +ARROW_ABSL_BUILD_SHA256_CHECKSUM=5366d7e7fa7ba0d915014d387b66d0d002c03236448e1ba9ef98122c13b35c36 ARROW_AWS_C_AUTH_BUILD_VERSION=v0.6.22 ARROW_AWS_C_AUTH_BUILD_SHA256_CHECKSUM=691a6b4418afcd3dc141351b6ad33fccd8e3ff84df0e9e045b42295d284ee14c ARROW_AWS_C_CAL_BUILD_VERSION=v0.5.20 From 828d6892222623bedc0b11f01edf26cfa6206d8e Mon Sep 17 00:00:00 2001 From: david dali susanibar arce Date: Wed, 28 Jun 2023 20:22:52 -0500 Subject: [PATCH 8/8] fix: delete absl build version --- ci/scripts/java_jni_macos_build.sh | 1 - cpp/thirdparty/versions.txt | 4 ++-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/ci/scripts/java_jni_macos_build.sh b/ci/scripts/java_jni_macos_build.sh index 5aab00d5a5661..29522c703be8e 100755 --- a/ci/scripts/java_jni_macos_build.sh +++ b/ci/scripts/java_jni_macos_build.sh @@ -68,7 +68,6 @@ mkdir -p "${build_dir}/cpp" pushd "${build_dir}/cpp" cmake \ - -Dabsl_SOURCE=BUNDLED \ -DARROW_ACERO=${ARROW_ACERO} \ -DARROW_BUILD_SHARED=OFF \ -DARROW_BUILD_TESTS=${ARROW_BUILD_TESTS} \ diff --git a/cpp/thirdparty/versions.txt b/cpp/thirdparty/versions.txt index f9da026d075b8..a229c340414a5 100644 --- a/cpp/thirdparty/versions.txt +++ b/cpp/thirdparty/versions.txt @@ -23,8 +23,8 @@ # `DEPENDENCIES` array (see the comment on top of the declaration for the # format). -ARROW_ABSL_BUILD_VERSION=20230125.3 -ARROW_ABSL_BUILD_SHA256_CHECKSUM=5366d7e7fa7ba0d915014d387b66d0d002c03236448e1ba9ef98122c13b35c36 +ARROW_ABSL_BUILD_VERSION=20211102.0 +ARROW_ABSL_BUILD_SHA256_CHECKSUM=dcf71b9cba8dc0ca9940c4b316a0c796be8fab42b070bb6b7cab62b48f0e66c4 ARROW_AWS_C_AUTH_BUILD_VERSION=v0.6.22 ARROW_AWS_C_AUTH_BUILD_SHA256_CHECKSUM=691a6b4418afcd3dc141351b6ad33fccd8e3ff84df0e9e045b42295d284ee14c ARROW_AWS_C_CAL_BUILD_VERSION=v0.5.20