diff --git a/src/common/device_helpers.cu b/src/common/device_helpers.cu
index 608a535cd8cb..01e81b16ee0b 100644
--- a/src/common/device_helpers.cu
+++ b/src/common/device_helpers.cu
@@ -7,6 +7,11 @@
 
 namespace dh {
 PinnedMemory::PinnedMemory() {
+  // Use the `GrowOnlyPinnedMemoryImpl` as the only option for now.
+  // See https://github.com/dmlc/xgboost/issues/10933
+  this->impl_.emplace<detail::GrowOnlyPinnedMemoryImpl>();
+  return;
+
 #if defined(xgboost_IS_WIN)
   this->impl_.emplace<detail::GrowOnlyPinnedMemoryImpl>();
 #else
diff --git a/tests/cpp/common/test_device_vector.cu b/tests/cpp/common/test_device_vector.cu
index 6f4c34edfa9f..97ee39b31a1e 100644
--- a/tests/cpp/common/test_device_vector.cu
+++ b/tests/cpp/common/test_device_vector.cu
@@ -32,6 +32,9 @@ class TestVirtualMem : public ::testing::TestWithParam<CUmemLocationType> {
  public:
   void Run() {
     auto type = this->GetParam();
+    if (type == CU_MEM_LOCATION_TYPE_HOST_NUMA) {
+      GTEST_SKIP_("Host numa might require special system capabilities, skipping for now.");
+    }
     detail::GrowOnlyVirtualMemVec vec{type};
     auto prop = xgboost::cudr::MakeAllocProp(type);
     auto gran = xgboost::cudr::GetAllocGranularity(&prop);
@@ -111,15 +114,7 @@ TEST(TestVirtualMem, Version) {
   xgboost::curt::DrVersion(&major, &minor);
   LOG(INFO) << "Latest supported CUDA version by the driver:" << major << "." << minor;
   PinnedMemory pinned;
-#if defined(xgboost_IS_WIN)
   ASSERT_FALSE(pinned.IsVm());
-#else  // defined(xgboost_IS_WIN)
-  if (major >= 12 && minor >= 5) {
-    ASSERT_TRUE(pinned.IsVm());
-  } else {
-    ASSERT_FALSE(pinned.IsVm());
-  }
-#endif  // defined(xgboost_IS_WIN)
 }
 
 TEST(AtomitFetch, Max) {