diff --git a/plugin/federated/CMakeLists.txt b/plugin/federated/CMakeLists.txt
index 4f0d1b05c..de129a12a 100644
--- a/plugin/federated/CMakeLists.txt
+++ b/plugin/federated/CMakeLists.txt
@@ -29,5 +29,5 @@ target_link_libraries(federated_client INTERFACE federated_proto)
 
 # Rabit engine for Federated Learning.
 target_sources(objxgboost PRIVATE federated_server.cc)
-target_link_libraries(objxgboost PRIVATE federated_client)
+target_link_libraries(objxgboost PRIVATE federated_client "-Wl,--exclude-libs,ALL")
 target_compile_definitions(objxgboost PUBLIC -DXGBOOST_USE_FEDERATED=1)
diff --git a/python-package/xgboost/testing.py b/python-package/xgboost/testing.py
index 4a8a9acbf..598ddac5b 100644
--- a/python-package/xgboost/testing.py
+++ b/python-package/xgboost/testing.py
@@ -41,7 +41,7 @@ def skip_ipv6() -> PytestSkip:
     return {"condition": not has_ipv6(), "reason": "IPv6 is required to be enabled."}
 
 
-def timeout(sec: int, *args: Any, enable: bool = False, **kwargs: Any) -> Any:
+def timeout(sec: int, *args: Any, enable: bool = True, **kwargs: Any) -> Any:
     """Make a pytest mark for the `pytest-timeout` package.
 
     Parameters
diff --git a/src/c_api/c_api.cu b/src/c_api/c_api.cu
index 09f133aee..2af36f0ac 100644
--- a/src/c_api/c_api.cu
+++ b/src/c_api/c_api.cu
@@ -42,19 +42,15 @@ void XGBBuildInfoDevice(Json *p_info) {
 }
 
 void XGBoostAPIGuard::SetGPUAttribute() {
-  try {
-    device_id_ = dh::CurrentDevice();
-  } catch (dmlc::Error const&) {
-    // do nothing, running on CPU only machine
-  }
+  // Not calling `safe_cuda` to avoid unnecessary exception handling overhead.
+  // If errors, do nothing, assuming running on CPU only machine.
+  cudaGetDevice(&device_id_);
 }
 
 void XGBoostAPIGuard::RestoreGPUAttribute() {
-  try {
-    dh::safe_cuda(cudaSetDevice(device_id_));
-  } catch (dmlc::Error const&) {
-    // do nothing, running on CPU only machine
-  }
+  // Not calling `safe_cuda` to avoid unnecessary exception handling overhead.
+  // If errors, do nothing, assuming running on CPU only machine.
+  cudaSetDevice(device_id_);
 }
 }                        // namespace xgboost
 
diff --git a/tests/ci_build/Dockerfile.cpu b/tests/ci_build/Dockerfile.cpu
index 0101a2fa7..54c3c3af4 100644
--- a/tests/ci_build/Dockerfile.cpu
+++ b/tests/ci_build/Dockerfile.cpu
@@ -30,7 +30,7 @@ ENV JAVA_HOME /usr/lib/jvm/java-8-openjdk-amd64/
 RUN git clone -b v1.49.1 https://github.com/grpc/grpc.git \
       --recurse-submodules --depth 1 --shallow-submodules && \
     pushd grpc && \
-    cmake -S . -B build -GNinja -DCMAKE_INSTALL_PREFIX=/opt/grpc && \
+    cmake -S . -B build -GNinja -DCMAKE_INSTALL_PREFIX=/opt/grpc -DCMAKE_CXX_VISIBILITY_PRESET=hidden && \
     cmake --build build --target install && \
     popd && \
     rm -rf grpc
diff --git a/tests/ci_build/Dockerfile.gpu_build_centos7 b/tests/ci_build/Dockerfile.gpu_build_centos7
index b6b38575b..4168e430d 100644
--- a/tests/ci_build/Dockerfile.gpu_build_centos7
+++ b/tests/ci_build/Dockerfile.gpu_build_centos7
@@ -39,7 +39,7 @@ ENV GOSU_VERSION 1.10
 RUN git clone -b v1.49.1 https://github.com/grpc/grpc.git \
       --recurse-submodules --depth 1 && \
     pushd grpc && \
-    cmake -S . -B build -GNinja -DCMAKE_INSTALL_PREFIX=/opt/grpc && \
+    cmake -S . -B build -GNinja -DCMAKE_INSTALL_PREFIX=/opt/grpc -DCMAKE_CXX_VISIBILITY_PRESET=hidden && \
     cmake --build build --target install && \
     popd && \
     rm -rf grpc