diff --git a/src/common/cuda_context.cuh b/src/common/cuda_context.cuh
index 17896460f..b7119ef90 100644
--- a/src/common/cuda_context.cuh
+++ b/src/common/cuda_context.cuh
@@ -6,12 +6,6 @@
 #include <thrust/execution_policy.h>
 #include "device_helpers.cuh"
 
-#ifdef XGBOOST_USE_HIP
-namespace thrust {
-    namespace cuda = thrust::hip;
-}
-#endif
-
 namespace xgboost {
 struct CUDAContext {
  private:
diff --git a/src/common/cuda_to_hip.h b/src/common/cuda_to_hip.h
index 6033a80b2..f56cb60a8 100644
--- a/src/common/cuda_to_hip.h
+++ b/src/common/cuda_to_hip.h
@@ -54,4 +54,18 @@
 #define cudaDevAttrMultiProcessorCount                hipDeviceAttributeMultiprocessorCount
 #define cudaOccupancyMaxActiveBlocksPerMultiprocessor hipOccupancyMaxActiveBlocksPerMultiprocessor
 
+namespace thrust {
+    namespace hip {
+    }
+}
+
+namespace thrust {
+    namespace cuda = thrust::hip;
+}
+
+namespace hipcub {
+}
+
+namespace cub = hipcub;
+
 #endif
diff --git a/src/common/hist_util.cu b/src/common/hist_util.cu
index bd0c894f0..1f06c2a6f 100644
--- a/src/common/hist_util.cu
+++ b/src/common/hist_util.cu
@@ -26,12 +26,6 @@
 #include "quantile.h"
 #include "xgboost/host_device_vector.h"
 
-#ifdef XGBOOST_USE_HIP
-namespace thrust {
-    namespace cuda = thrust::hip;
-}
-#endif
-
 namespace xgboost::common {
 constexpr float SketchContainer::kFactor;
 
diff --git a/src/common/hist_util.cuh b/src/common/hist_util.cuh
index aec733ddc..37751b40b 100644
--- a/src/common/hist_util.cuh
+++ b/src/common/hist_util.cuh
@@ -17,14 +17,6 @@
 #include "quantile.cuh"
 #include "xgboost/span.h"  // for IterSpan
 
-#if defined(XGBOOST_USE_HIP)
-namespace cub = hipcub;
-
-namespace thrust {
-    namespace cuda = thrust::hip;
-}
-#endif
-
 namespace xgboost::common {
 namespace cuda {
 /**
diff --git a/src/common/numeric.cu b/src/common/numeric.cu
index c25ee2c6a..01950f8c8 100644
--- a/src/common/numeric.cu
+++ b/src/common/numeric.cu
@@ -8,12 +8,6 @@
 #include "xgboost/context.h"             // Context
 #include "xgboost/host_device_vector.h"  // HostDeviceVector
 
-#ifdef XGBOOST_USE_HIP
-namespace thrust {
-    namespace cuda = thrust::hip;
-}
-#endif
-
 namespace xgboost::common::cuda_impl {
 double Reduce(Context const* ctx, HostDeviceVector<float> const& values) {
   values.SetDevice(ctx->Device());
diff --git a/src/common/quantile.cu b/src/common/quantile.cu
index 849b19480..3db846a56 100644
--- a/src/common/quantile.cu
+++ b/src/common/quantile.cu
@@ -22,12 +22,6 @@
 #include "transform_iterator.h"  // MakeIndexTransformIter
 #include "xgboost/span.h"
 
-#ifdef XGBOOST_USE_HIP
-namespace thrust {
-    namespace cuda = thrust::hip;
-}
-#endif
-
 namespace xgboost {
 namespace common {
 
diff --git a/src/common/quantile.cuh b/src/common/quantile.cuh
index 63d7d1e5a..f5228a855 100644
--- a/src/common/quantile.cuh
+++ b/src/common/quantile.cuh
@@ -10,12 +10,6 @@
 #include "timer.h"
 #include "categorical.h"
 
-#if defined(XGBOOST_USE_HIP)
-namespace thrust {
-    namespace cuda = thrust::hip;
-}
-#endif
-
 namespace xgboost {
 namespace common {
 
diff --git a/src/common/ranking_utils.cu b/src/common/ranking_utils.cu
index e9347aa82..5af963d30 100644
--- a/src/common/ranking_utils.cu
+++ b/src/common/ranking_utils.cu
@@ -25,8 +25,6 @@
 
 #if defined(XGBOOST_USE_HIP)
 #include <hipcub/hipcub.hpp>
-
-namespace cub = hipcub;
 #endif
 
 namespace xgboost::ltr {
diff --git a/src/common/stats.cu b/src/common/stats.cu
index 6cfcd6bae..10c7565bc 100644
--- a/src/common/stats.cu
+++ b/src/common/stats.cu
@@ -15,10 +15,6 @@
 #include "xgboost/host_device_vector.h"  // HostDeviceVector
 #include "xgboost/linalg.h"              // linalg::TensorView, UnravelIndex, Apply
 
-#if defined(XGBOOST_USE_HIP)
-namespace cub = hipcub;
-#endif
-
 namespace xgboost::common::cuda_impl {
 void Median(Context const* ctx, linalg::TensorView<float const, 2> t,
             common::OptionalWeights weights, linalg::Tensor<float, 1>* out) {
diff --git a/src/common/stats.cuh b/src/common/stats.cuh
index 5c909a830..1af89af37 100644
--- a/src/common/stats.cuh
+++ b/src/common/stats.cuh
@@ -23,12 +23,6 @@
 #include "xgboost/context.h"                       // Context
 #include "xgboost/span.h"                          // Span
 
-#ifdef XGBOOST_USE_HIP
-namespace thrust {
-    namespace cuda = thrust::hip;
-}
-#endif
-
 namespace xgboost {
 namespace common {
 namespace detail {
diff --git a/src/data/data.cu b/src/data/data.cu
index 39c44954c..3f9e00292 100644
--- a/src/data/data.cu
+++ b/src/data/data.cu
@@ -15,13 +15,6 @@
 #include "xgboost/json.h"
 #include "xgboost/logging.h"
 
-#if defined(XGBOOST_USE_HIP)
-namespace cub = hipcub;
-namespace thrust {
-    namespace cuda = thrust::hip;
-}
-#endif
-
 namespace xgboost {
 namespace {
 auto SetDeviceToPtr(void const* ptr) {
diff --git a/src/data/device_adapter.cuh b/src/data/device_adapter.cuh
index b1c18ac6a..8c99b13eb 100644
--- a/src/data/device_adapter.cuh
+++ b/src/data/device_adapter.cuh
@@ -17,12 +17,6 @@
 #include "adapter.h"
 #include "array_interface.h"
 
-#if defined(XGBOOST_USE_HIP)
-namespace thrust {
-    namespace cuda = thrust::hip;
-}
-#endif
-
 namespace xgboost {
 namespace data {
 
diff --git a/src/data/iterative_dmatrix.cu b/src/data/iterative_dmatrix.cu
index cc09356c4..828b984ae 100644
--- a/src/data/iterative_dmatrix.cu
+++ b/src/data/iterative_dmatrix.cu
@@ -16,12 +16,6 @@
 #include "simple_batch_iterator.h"
 #include "sparse_page_source.h"
 
-#if defined(XGBOOST_USE_HIP)
-namespace thrust {
-    namespace cuda = thrust::hip;
-}
-#endif
-
 namespace xgboost::data {
 void IterativeDMatrix::InitFromCUDA(Context const* ctx, BatchParam const& p,
                                     DataIterHandle iter_handle, float missing,
diff --git a/src/data/simple_dmatrix.cuh b/src/data/simple_dmatrix.cuh
index a862ed23d..37d474e41 100644
--- a/src/data/simple_dmatrix.cuh
+++ b/src/data/simple_dmatrix.cuh
@@ -13,12 +13,6 @@
 #include "../common/error_msg.h"  // for InfInData
 #include "device_adapter.cuh"     // for HasInfInData
 
-#if defined(XGBOOST_USE_HIP)
-namespace thrust {
-    namespace cuda = thrust::hip;
-}
-#endif
-
 namespace xgboost::data {
 
 #if defined(XGBOOST_USE_CUDA)
diff --git a/src/metric/auc.cu b/src/metric/auc.cu
index d2194034e..6b70cda62 100644
--- a/src/metric/auc.cu
+++ b/src/metric/auc.cu
@@ -25,12 +25,6 @@
 #include "xgboost/data.h"
 #include "xgboost/span.h"
 
-#if defined(XGBOOST_USE_HIP)
-namespace thrust {
-    namespace cuda = thrust::hip;
-}
-#endif
-
 namespace xgboost {
 namespace metric {
 // tag the this file, used by force static link later.
diff --git a/src/metric/elementwise_metric.cu b/src/metric/elementwise_metric.cu
index eb766e964..cab1e9dd6 100644
--- a/src/metric/elementwise_metric.cu
+++ b/src/metric/elementwise_metric.cu
@@ -30,12 +30,6 @@
 #include "../common/device_helpers.cuh"
 #endif  // defined(XGBOOST_USE_CUDA) || defined(XGBOOST_USE_HIP)
 
-#if defined(XGBOOST_USE_HIP)
-namespace thrust {
-    namespace cuda = thrust::hip;
-}
-#endif
-
 namespace xgboost {
 namespace metric {
 // tag the this file, used by force static link later.
diff --git a/src/metric/multiclass_metric.cu b/src/metric/multiclass_metric.cu
index e8f71dfd4..a6d215e6a 100644
--- a/src/metric/multiclass_metric.cu
+++ b/src/metric/multiclass_metric.cu
@@ -24,12 +24,6 @@
 #include "../common/device_helpers.cuh"
 #endif  // XGBOOST_USE_CUDA || XGBOOST_USE_HIP
 
-#if defined(XGBOOST_USE_HIP)
-namespace thrust {
-    namespace cuda = thrust::hip;
-}
-#endif
-
 namespace xgboost {
 namespace metric {
 // tag the this file, used by force static link later.
diff --git a/src/metric/rank_metric.cu b/src/metric/rank_metric.cu
index 30814447a..eb6f1b3a1 100644
--- a/src/metric/rank_metric.cu
+++ b/src/metric/rank_metric.cu
@@ -26,8 +26,6 @@
 
 #if defined(XGBOOST_USE_HIP)
 #include <hipcub/hipcub.hpp>
-
-namespace cub = hipcub;
 #endif
 
 namespace xgboost::metric {
diff --git a/src/metric/survival_metric.cu b/src/metric/survival_metric.cu
index 19c1891e3..dd495f030 100644
--- a/src/metric/survival_metric.cu
+++ b/src/metric/survival_metric.cu
@@ -25,12 +25,6 @@
 #include "../common/device_helpers.cuh"
 #endif  // XGBOOST_USE_CUDA || XGBOOST_USE_HIP
 
-#if defined(XGBOOST_USE_HIP)
-namespace thrust {
-    namespace cuda = thrust::hip;
-}
-#endif
-
 using AFTParam = xgboost::common::AFTParam;
 using ProbabilityDistributionType = xgboost::common::ProbabilityDistributionType;
 template <typename Distribution>
diff --git a/src/objective/adaptive.cu b/src/objective/adaptive.cu
index 4835373ad..c03930b80 100644
--- a/src/objective/adaptive.cu
+++ b/src/objective/adaptive.cu
@@ -19,11 +19,6 @@
 #include "xgboost/context.h"
 
 namespace xgboost {
-
-#if defined(XGBOOST_USE_HIP)
-namespace cub = hipcub;
-#endif
-
 namespace obj {
 namespace detail {
 void EncodeTreeLeafDevice(Context const* ctx, common::Span<bst_node_t const> position,
diff --git a/src/objective/lambdarank_obj.cu b/src/objective/lambdarank_obj.cu
index 9d908c19c..47d7957e8 100644
--- a/src/objective/lambdarank_obj.cu
+++ b/src/objective/lambdarank_obj.cu
@@ -35,8 +35,6 @@
 
 #if defined(XGBOOST_USE_HIP)
 #include <hipcub/hipcub.hpp>
-
-namespace cub = hipcub;
 #endif
 
 namespace xgboost::obj {
diff --git a/src/predictor/gpu_predictor.cu b/src/predictor/gpu_predictor.cu
index 89506a86b..2ae19f0a3 100644
--- a/src/predictor/gpu_predictor.cu
+++ b/src/predictor/gpu_predictor.cu
@@ -29,12 +29,6 @@
 #include "xgboost/tree_model.h"
 #include "xgboost/tree_updater.h"
 
-#if defined(XGBOOST_USE_HIP)
-namespace thrust {
-    namespace cuda = thrust::hip;
-}
-#endif
-
 namespace xgboost::predictor {
 DMLC_REGISTRY_FILE_TAG(gpu_predictor);
 
diff --git a/src/tree/fit_stump.cu b/src/tree/fit_stump.cu
index 2b0a248ce..03055e7c9 100644
--- a/src/tree/fit_stump.cu
+++ b/src/tree/fit_stump.cu
@@ -21,12 +21,6 @@
 #include "xgboost/logging.h"  // CHECK_EQ
 #include "xgboost/span.h"     // span
 
-#if defined(XGBOOST_USE_HIP)
-namespace thrust {
-    namespace cuda = thrust::hip;
-}
-#endif
-
 namespace xgboost::tree::cuda_impl {
 void FitStump(Context const* ctx, MetaInfo const& info,
               linalg::TensorView<GradientPair const, 2> gpair, linalg::VectorView<float> out) {
diff --git a/src/tree/gpu_hist/evaluate_splits.cu b/src/tree/gpu_hist/evaluate_splits.cu
index 70cbca529..de3f0a14d 100644
--- a/src/tree/gpu_hist/evaluate_splits.cu
+++ b/src/tree/gpu_hist/evaluate_splits.cu
@@ -24,10 +24,6 @@
 #define WARP_SIZE WAVEFRONT_SIZE
 #endif
 
-#if defined(XGBOOST_USE_HIP)
-namespace cub = hipcub;
-#endif
-
 namespace xgboost::tree {
 // With constraints
 XGBOOST_DEVICE float LossChangeMissing(const GradientPairInt64 &scan,
diff --git a/src/tree/gpu_hist/evaluator.cu b/src/tree/gpu_hist/evaluator.cu
index 5d00640a4..b416cb442 100644
--- a/src/tree/gpu_hist/evaluator.cu
+++ b/src/tree/gpu_hist/evaluator.cu
@@ -12,12 +12,6 @@
 #include "evaluate_splits.cuh"
 #include "xgboost/data.h"
 
-#if defined(XGBOOST_USE_HIP)
-namespace thrust {
-    namespace cuda = thrust::hip;
-}
-#endif
-
 namespace xgboost::tree {
 void GPUHistEvaluator::Reset(common::HistogramCuts const &cuts, common::Span<FeatureType const> ft,
                              bst_feature_t n_features, TrainParam const &param,
diff --git a/src/tree/gpu_hist/histogram.cu b/src/tree/gpu_hist/histogram.cu
index 64e665afc..b5034cafb 100644
--- a/src/tree/gpu_hist/histogram.cu
+++ b/src/tree/gpu_hist/histogram.cu
@@ -16,12 +16,6 @@
 #include "row_partitioner.cuh"
 #include "xgboost/base.h"
 
-#if defined(XGBOOST_USE_HIP)
-namespace thrust {
-    namespace cuda = thrust::hip;
-}
-#endif
-
 namespace xgboost {
 namespace tree {
 namespace {