GH-44667: [Archery] Only suppress Docker progress logs if --quiet i…

…s enabled

.github/workflows/archery.yml

@@ -78,7 +78,7 @@ jobs:
         working-directory: dev/archery
         run: pytest -v archery
       - name: Archery Docker Validation
-        run: archery docker check-config
+        run: archery --quiet docker check-config
       - name: Crossbow Check Config
         working-directory: dev/tasks
         run: archery crossbow check-config

.github/workflows/cpp.yml

@@ -169,7 +169,7 @@ jobs:
           # GH-40558: reduce ASLR to avoid ASAN/LSAN crashes
           sudo sysctl -w vm.mmap_rnd_bits=28
           source ci/scripts/util_enable_core_dumps.sh
-          archery docker run ${{ matrix.image }}
+          archery --quiet docker run ${{ matrix.image }}
       - name: Docker Push
         if: >-
           success() &&
@@ -180,7 +180,7 @@ jobs:
           ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }}
           ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }}
         continue-on-error: true
-        run: archery docker push ${{ matrix.image }}
+        run: archery --quiet docker push ${{ matrix.image }}
 
   build-example:
     name: C++ Minimal Build Example

.github/workflows/dev.yml

@@ -74,7 +74,7 @@ jobs:
           UBUNTU: 22.04
         run: |
           source ci/scripts/util_enable_core_dumps.sh
-          archery docker run -e GITHUB_ACTIONS=true ubuntu-lint
+          archery --quiet docker run -e GITHUB_ACTIONS=true ubuntu-lint
       - name: Docker Push
         if: >-
           success() &&
@@ -85,7 +85,7 @@ jobs:
           ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }}
           ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }}
         continue-on-error: true
-        run: archery docker push ubuntu-lint
+        run: archery --quiet docker push ubuntu-lint
 
   release:
     name: Source Release and Merge Script on ${{ matrix.runs-on }}

.github/workflows/docs.yml

@@ -62,7 +62,7 @@ jobs:
           ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }}
           ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }}
           JDK: 17
-        run: archery docker run debian-docs
+        run: archery --quiet docker run debian-docs
       - name: Docker Push
         if: >-
           success() &&
@@ -73,4 +73,4 @@ jobs:
           ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }}
           ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }}
         continue-on-error: true
-        run: archery docker push debian-docs
+        run: archery --quiet docker push debian-docs

.github/workflows/docs_light.yml

@@ -68,4 +68,4 @@ jobs:
         env:
           ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }}
           ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }}
-        run: archery docker run conda-python-docs
+        run: archery --quiet docker run conda-python-docs

.github/workflows/integration.yml

@@ -112,7 +112,7 @@ jobs:
           ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }}
         run: |
           source ci/scripts/util_enable_core_dumps.sh
-          archery docker run \
+          archery --quiet docker run \
             -e ARCHERY_DEFAULT_BRANCH=${{ github.event.repository.default_branch }} \
             -e ARCHERY_INTEGRATION_WITH_GO=1 \
             -e ARCHERY_INTEGRATION_WITH_NANOARROW=1 \
@@ -128,4 +128,4 @@ jobs:
           ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }}
           ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }}
         continue-on-error: true
-        run: archery docker push conda-integration
+        run: archery --quiet docker push conda-integration

.github/workflows/java.yml

@@ -94,7 +94,7 @@ jobs:
           ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }}
           DEVELOCITY_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
         run: |
-          archery docker run \
+          archery --quiet docker run \
             -e CI=true \
             -e "DEVELOCITY_ACCESS_KEY=$DEVELOCITY_ACCESS_KEY" \
             ${{ matrix.image }}
@@ -108,7 +108,7 @@ jobs:
           ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }}
           ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }}
         continue-on-error: true
-        run: archery docker push ${{ matrix.image }}
+        run: archery --quiet docker push ${{ matrix.image }}
 
   macos:
     name: AMD64 macOS 13 Java JDK ${{ matrix.jdk }}

.github/workflows/java_jni.yml

@@ -88,7 +88,7 @@ jobs:
           ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }}
         run: |
           source ci/scripts/util_enable_core_dumps.sh
-          archery docker run java-jni-manylinux-2014
+          archery --quiet docker run java-jni-manylinux-2014
       - name: Docker Push
         if: >-
           success() &&
@@ -99,7 +99,7 @@ jobs:
           ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }}
           ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }}
         continue-on-error: true
-        run: archery docker push java-jni-manylinux-2014
+        run: archery --quiet docker push java-jni-manylinux-2014
 
   docker_integration_python:
     name: AMD64 Conda Java C Data Interface Integration
@@ -130,7 +130,7 @@ jobs:
           ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }}
           DEVELOCITY_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
         run: |
-          archery docker run \
+          archery --quiet docker run \
             -e CI=true \
             -e "DEVELOCITY_ACCESS_KEY=$DEVELOCITY_ACCESS_KEY" \
             conda-python-java-integration
@@ -144,4 +144,4 @@ jobs:
           ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }}
           ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }}
         continue-on-error: true
-        run: archery docker push conda-python-java-integration
+        run: archery --quiet docker push conda-python-java-integration

.github/workflows/js.yml

@@ -72,7 +72,7 @@ jobs:
           ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }}
         run: |
           source ci/scripts/util_enable_core_dumps.sh
-          archery docker run debian-js
+          archery --quiet docker run debian-js
       - name: Docker Push
         if: >-
           success() &&
@@ -83,7 +83,7 @@ jobs:
           ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }}
           ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }}
         continue-on-error: true
-        run: archery docker push debian-js
+        run: archery --quiet docker push debian-js
 
   macos:
     name: AMD64 macOS 13 NodeJS ${{ matrix.node }}

.github/workflows/python.yml

@@ -125,7 +125,7 @@ jobs:
           ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }}
         run: |
           source ci/scripts/util_enable_core_dumps.sh
-          archery docker run ${{ matrix.image }}
+          archery --quiet docker run ${{ matrix.image }}
       - name: Docker Push
         if: >-
           success() &&
@@ -136,7 +136,7 @@ jobs:
           ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }}
           ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }}
         continue-on-error: true
-        run: archery docker push ${{ matrix.image }}
+        run: archery --quiet docker push ${{ matrix.image }}
 
   macos:
     name: ${{ matrix.architecture }} macOS ${{ matrix.macos-version }} Python 3

.github/workflows/r.yml

@@ -168,7 +168,7 @@ jobs:
           source ci/scripts/util_enable_core_dumps.sh
           # Setting a non-default and non-probable Marquesas French Polynesia time
           # it has both with a .45 offset and very very few people who live there.
-          archery docker run -e TZ=MART -e ARROW_R_FORCE_TESTS=${{ matrix.force-tests }} ubuntu-r
+          archery --quiet docker run -e TZ=MART -e ARROW_R_FORCE_TESTS=${{ matrix.force-tests }} ubuntu-r
       - name: Dump install logs
         run: cat r/check/arrow.Rcheck/00install.out
         if: always()
@@ -191,7 +191,7 @@ jobs:
           ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }}
           ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }}
         continue-on-error: true
-        run: archery docker push ubuntu-r
+        run: archery --quiet docker push ubuntu-r
 
   bundled:
     name: "${{ matrix.config.org }}/${{ matrix.config.image }}:${{ matrix.config.tag }}"
@@ -228,7 +228,7 @@ jobs:
           # Don't set a TZ here to test that case. These builds will have the following warning in them:
           #   System has not been booted with systemd as init system (PID 1). Can't operate.
           #   Failed to connect to bus: Host is down
-          archery docker run -e TZ="" r
+          archery --quiet docker run -e TZ="" r
       - name: Dump install logs
         run: cat r/check/arrow.Rcheck/00install.out
         if: always()
@@ -251,7 +251,7 @@ jobs:
           ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }}
           ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }}
         continue-on-error: true
-        run: archery docker push r
+        run: archery --quiet docker push r
 
   windows-cpp:
     name: AMD64 Windows C++ RTools ${{ matrix.config.rtools }} ${{ matrix.config.arch }}

.github/workflows/ruby.yml

@@ -101,7 +101,7 @@ jobs:
           ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }}
         run: |
           source ci/scripts/util_enable_core_dumps.sh
-          archery docker run \
+          archery --quiet docker run \
             -e ARROW_FLIGHT=ON \
             -e ARROW_FLIGHT_SQL=ON \
             -e ARROW_GCS=ON \
@@ -119,7 +119,7 @@ jobs:
           ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }}
         continue-on-error: true
         shell: bash
-        run: archery docker push ubuntu-ruby
+        run: archery --quiet docker push ubuntu-ruby
 
   macos:
     name: ARM64 macOS 14 GLib & Ruby

.github/workflows/swift.yml

@@ -75,7 +75,7 @@ jobs:
           ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }}
         run: |
           source ci/scripts/util_enable_core_dumps.sh
-          archery docker run ubuntu-swift
+          archery --quiet docker run ubuntu-swift
       - name: Docker Push
         if: >-
           success() &&
@@ -86,4 +86,4 @@ jobs:
           ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }}
           ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }}
         continue-on-error: true
-        run: archery docker push ubuntu-swift
+        run: archery --quiet docker push ubuntu-swift

dev/archery/archery/docker/core.py

@@ -24,6 +24,7 @@
 from ruamel.yaml import YAML
 
 from ..utils.command import Command, default_bin
+from ..utils.logger import ctx as log_ctx
 from ..utils.source import arrow_path
 from ..compat import _ensure_path
 
@@ -43,6 +44,10 @@ def flatten(node, parents=None):
         raise TypeError(node)
 
 
+def verbosity_args():
+    return ['--quiet'] if log_ctx.quiet else []
+
+
 _arch_short_mapping = {
     'arm64v8': 'arm64',
 }
@@ -233,7 +238,7 @@ def _execute_docker(self, *args, **kwargs):
 
     def pull(self, service_name, pull_leaf=True, ignore_pull_failures=True):
         def _pull(service):
-            args = ['pull', '--quiet']
+            args = ['pull'] + verbosity_args()
             if service['image'] in self.pull_memory:
                 return
 
@@ -417,20 +422,23 @@ def run(self, service_name, command=None, *, env=None, volumes=None,
                     args.extend(shlex.split(cmd))
 
             # execute as a plain docker cli command
-            self._execute_docker('run', '--rm', *args)
+            # (we preserve stdout even with `--quiet` enabled, as we usually
+            #  expect to see execution output of the container)
+            self._execute_docker('run', '--rm', *args, stdout=None)
         else:
             # execute as a docker compose command
             args.append(service_name)
             if command is not None:
                 args.append(command)
-            self._execute_compose('run', '--rm', *args)
+            self._execute_compose('run', '--rm', *args, stdout=None)
 
     def push(self, service_name, user=None, password=None):
         def _push(service):
+            args = ['push'] + verbosity_args()
             if self.config.using_docker:
-                return self._execute_docker('push', '--quiet', service['image'])
+                return self._execute_docker(*args, service['image'])
             else:
-                return self._execute_compose('push', '--quiet', service['name'])
+                return self._execute_compose(*args, service['name'])
 
         if user is not None:
             try:

dev/archery/archery/docker/tests/test_docker.py

@@ -232,24 +232,31 @@ def test_config_validation(tmpdir):
     DockerCompose(config_path)  # no issue
 
 
-def assert_docker_calls(compose, expected_args):
-    base_command = ['docker']
+def _assert_docker_or_compose_calls(base_command, expected_args,
+                                    **expected_kwargs):
     expected_commands = []
+    is_run_command = False
     for args in expected_args:
         if isinstance(args, str):
             args = re.split(r"\s", args)
         expected_commands.append(base_command + args)
-    return assert_subprocess_calls(expected_commands, check=True)
+        is_run_command = is_run_command or ('run' in args)
+    if is_run_command:
+        return assert_subprocess_calls(expected_commands, check=True,
+                                       stdout=None, **expected_kwargs)
+    else:
+        return assert_subprocess_calls(expected_commands, check=True,
+                                       **expected_kwargs)
+
+
+def assert_docker_calls(compose, expected_args):
+    base_command = ['docker']
+    return _assert_docker_or_compose_calls(base_command, expected_args)
 
 
 def assert_compose_calls(compose, expected_args, env=mock.ANY):
     base_command = ['docker', 'compose', f'--file={compose.config.path}']
-    expected_commands = []
-    for args in expected_args:
-        if isinstance(args, str):
-            args = re.split(r"\s", args)
-        expected_commands.append(base_command + args)
-    return assert_subprocess_calls(expected_commands, check=True, env=env)
+    return _assert_docker_or_compose_calls(base_command, expected_args, env=env)
 
 
 def test_arrow_example_validation_passes(arrow_compose_path):
@@ -270,7 +277,7 @@ def test_compose_default_params_and_env(arrow_compose_path):
 
 def test_forwarding_env_variables(arrow_compose_path):
     expected_calls = [
-        "pull --quiet --ignore-pull-failures conda-cpp",
+        "pull --ignore-pull-failures conda-cpp",
         "build conda-cpp",
     ]
     expected_env = PartialEnv(
@@ -290,24 +297,24 @@ def test_compose_pull(arrow_compose_path):
     compose = DockerCompose(arrow_compose_path)
 
     expected_calls = [
-        "pull --quiet --ignore-pull-failures conda-cpp",
+        "pull --ignore-pull-failures conda-cpp",
     ]
     with assert_compose_calls(compose, expected_calls):
         compose.clear_pull_memory()
         compose.pull('conda-cpp')
 
     expected_calls = [
-        "pull --quiet --ignore-pull-failures conda-cpp",
-        "pull --quiet --ignore-pull-failures conda-python",
-        "pull --quiet --ignore-pull-failures conda-python-pandas"
+        "pull --ignore-pull-failures conda-cpp",
+        "pull --ignore-pull-failures conda-python",
+        "pull --ignore-pull-failures conda-python-pandas"
     ]
     with assert_compose_calls(compose, expected_calls):
         compose.clear_pull_memory()
         compose.pull('conda-python-pandas')
 
     expected_calls = [
-        "pull --quiet --ignore-pull-failures conda-cpp",
-        "pull --quiet --ignore-pull-failures conda-python",
+        "pull --ignore-pull-failures conda-cpp",
+        "pull --ignore-pull-failures conda-python",
     ]
     with assert_compose_calls(compose, expected_calls):
         compose.clear_pull_memory()
@@ -316,8 +323,8 @@ def test_compose_pull(arrow_compose_path):
 
 def test_compose_pull_params(arrow_compose_path):
     expected_calls = [
-        "pull --quiet --ignore-pull-failures conda-cpp",
-        "pull --quiet --ignore-pull-failures conda-python",
+        "pull --ignore-pull-failures conda-cpp",
+        "pull --ignore-pull-failures conda-python",
     ]
     compose = DockerCompose(arrow_compose_path, params=dict(UBUNTU='18.04'))
     expected_env = PartialEnv(PYTHON='3.8', PANDAS='latest')
@@ -483,7 +490,7 @@ def test_compose_push(arrow_compose_path):
     for image in ["conda-cpp", "conda-python", "conda-python-pandas"]:
         expected_calls.append(
             mock.call(["docker", "compose", f"--file={compose.config.path}",
-                       "push", "--quiet", image], check=True, env=expected_env)
+                       "push", image], check=True, env=expected_env)
         )
     with assert_subprocess_calls(expected_calls):
         compose.push('conda-python-pandas', user='user', password='pass')