diff --git a/src/common/device_helpers.cuh b/src/common/device_helpers.cuh
index 884c83df5..4aadfb0c0 100644
--- a/src/common/device_helpers.cuh
+++ b/src/common/device_helpers.cuh
@@ -2,9 +2,6 @@
  * Copyright 2017-2023 XGBoost contributors
  */
 #pragma once
-
-#if defined(XGBOOST_USE_CUDA)
-
 #include <thrust/binary_search.h>  // thrust::upper_bound
 #include <thrust/device_malloc_allocator.h>
 #include <thrust/device_ptr.h>
@@ -1385,7 +1382,3 @@ class LDGIterator {
   }
 };
 }  // namespace dh
-
-#elif defined(XGBOOST_USE_HIP)
-#include "device_helpers.hip.h"
-#endif
diff --git a/tests/cpp/common/test_bitfield.cu b/tests/cpp/common/test_bitfield.cu
index 49b8cbed5..5b08ec82a 100644
--- a/tests/cpp/common/test_bitfield.cu
+++ b/tests/cpp/common/test_bitfield.cu
@@ -6,7 +6,11 @@
 #include <thrust/device_vector.h>
 #include <vector>
 #include "../../../src/common/bitfield.h"
+#if defined(XGBOOST_USE_CUDA)
 #include "../../../src/common/device_helpers.cuh"
+#elif defined(XGBOOST_USE_HIP)
+#include "../../../src/common/device_helpers.hip.h"
+#endif
 
 namespace xgboost {
 
diff --git a/tests/cpp/common/test_device_helpers.cu b/tests/cpp/common/test_device_helpers.cu
index ae4cffad0..13542cc16 100644
--- a/tests/cpp/common/test_device_helpers.cu
+++ b/tests/cpp/common/test_device_helpers.cu
@@ -6,7 +6,11 @@
 #include <thrust/device_vector.h>
 #include <vector>
 #include <xgboost/base.h>
+#if defined(XGBOOST_USE_CUDA)
 #include "../../../src/common/device_helpers.cuh"
+#elif defined(XGBOOST_USE_HIP)
+#include "../../../src/common/device_helpers.hip.h"
+#endif
 #include "../../../src/common/quantile.h"
 #include "../helpers.h"
 #include "gtest/gtest.h"
diff --git a/tests/cpp/common/test_gpu_compressed_iterator.cu b/tests/cpp/common/test_gpu_compressed_iterator.cu
index 1ffc4494e..94e695940 100644
--- a/tests/cpp/common/test_gpu_compressed_iterator.cu
+++ b/tests/cpp/common/test_gpu_compressed_iterator.cu
@@ -1,5 +1,9 @@
 #include "../../../src/common/compressed_iterator.h"
+#if defined(XGBOOST_USE_CUDA)
 #include "../../../src/common/device_helpers.cuh"
+#elif defined(XGBOOST_USE_HIP)
+#include "../../../src/common/device_helpers.hip.h"
+#endif
 #include "gtest/gtest.h"
 #include <algorithm>
 #include <thrust/device_vector.h>
diff --git a/tests/cpp/common/test_host_device_vector.cu b/tests/cpp/common/test_host_device_vector.cu
index 81b036055..5ac155e09 100644
--- a/tests/cpp/common/test_host_device_vector.cu
+++ b/tests/cpp/common/test_host_device_vector.cu
@@ -6,7 +6,11 @@
 #include <thrust/equal.h>
 #include <thrust/iterator/counting_iterator.h>
 
+#if defined(XGBOOST_USE_CUDA)
 #include "../../../src/common/device_helpers.cuh"
+#elif defined(XGBOOST_USE_HIP)
+#include "../../../src/common/device_helpers.hip.h"
+#endif
 #include <xgboost/host_device_vector.h>
 
 namespace xgboost {
diff --git a/tests/cpp/common/test_span.cu b/tests/cpp/common/test_span.cu
index 79c871b45..afebcf91c 100644
--- a/tests/cpp/common/test_span.cu
+++ b/tests/cpp/common/test_span.cu
@@ -7,7 +7,11 @@
 #include <thrust/device_vector.h>
 #include <thrust/execution_policy.h>
 
+#if defined(XGBOOST_USE_CUDA)
 #include "../../../src/common/device_helpers.cuh"
+#elif defined(XGBOOST_USE_HIP)
+#include "../../../src/common/device_helpers.hip.h"
+#endif
 
 #include <xgboost/span.h>
 #include "test_span.h"
diff --git a/tests/cpp/data/test_array_interface.h b/tests/cpp/data/test_array_interface.h
index 78bce76f5..a4780a5a9 100644
--- a/tests/cpp/data/test_array_interface.h
+++ b/tests/cpp/data/test_array_interface.h
@@ -6,7 +6,11 @@
 
 #include <memory>
 #include "../../../src/common/bitfield.h"
+#if defined(XGBOOST_USE_CUDA)
 #include "../../../src/common/device_helpers.cuh"
+#elif defined(XGBOOST_USE_HIP)
+#include "../../../src/common/device_helpers.hip.h"
+#endif
 
 namespace xgboost {
 
diff --git a/tests/cpp/data/test_ellpack_page.cu b/tests/cpp/data/test_ellpack_page.cu
index 356c84bb0..cf90f4cc2 100644
--- a/tests/cpp/data/test_ellpack_page.cu
+++ b/tests/cpp/data/test_ellpack_page.cu
@@ -7,7 +7,11 @@
 
 #include "../../../src/common/categorical.h"
 #include "../../../src/common/hist_util.h"
+#if defined(XGBOOST_USE_CUDA)
 #include "../../../src/data/ellpack_page.cuh"
+#elif defined(XGBOOST_USE_HIP)
+#include "../../../src/data/ellpack_page.hip.h"
+#endif
 #include "../../../src/tree/param.h"  // TrainParam
 #include "../helpers.h"
 #include "../histogram_helpers.h"
diff --git a/tests/cpp/data/test_ellpack_page_raw_format.cu b/tests/cpp/data/test_ellpack_page_raw_format.cu
index 66d4024ec..bbab2b608 100644
--- a/tests/cpp/data/test_ellpack_page_raw_format.cu
+++ b/tests/cpp/data/test_ellpack_page_raw_format.cu
@@ -4,7 +4,11 @@
 #include <gtest/gtest.h>
 #include <xgboost/data.h>
 
+#if defined(XGBOOST_USE_CUDA)
 #include "../../../src/data/ellpack_page.cuh"
+#elif defined(XGBOOST_USE_HIP)
+#include "../../../src/data/ellpack_page.hip.h"
+#endif
 #include "../../../src/data/sparse_page_source.h"
 #include "../../../src/tree/param.h"  // TrainParam
 #include "../filesystem.h"            // dmlc::TemporaryDirectory
diff --git a/tests/cpp/data/test_metainfo.cu b/tests/cpp/data/test_metainfo.cu
index a86b6b70b..e12248ff8 100644
--- a/tests/cpp/data/test_metainfo.cu
+++ b/tests/cpp/data/test_metainfo.cu
@@ -6,7 +6,11 @@
 #include <xgboost/data.h>
 #include <xgboost/json.h>
 
+#if defined(XGBOOST_USE_CUDA)
 #include "../../../src/common/device_helpers.cuh"
+#elif defined(XGBOOST_USE_HIP)
+#include "../../../src/common/device_helpers.hip.h"
+#endif
 #include "test_array_interface.h"
 #include "test_metainfo.h"
 
diff --git a/tests/cpp/data/test_sparse_page_dmatrix.cu b/tests/cpp/data/test_sparse_page_dmatrix.cu
index 846fe7f63..a61ea3e13 100644
--- a/tests/cpp/data/test_sparse_page_dmatrix.cu
+++ b/tests/cpp/data/test_sparse_page_dmatrix.cu
@@ -4,7 +4,11 @@
 #include <xgboost/data.h>  // for DMatrix
 
 #include "../../../src/common/compressed_iterator.h"
+#if defined(XGBOOST_USE_CUDA)
 #include "../../../src/data/ellpack_page.cuh"
+#elif defined(XGBOOST_USE_HIP)
+#include "../../../src/data/ellpack_page.hip.h"
+#endif
 #include "../../../src/data/sparse_page_dmatrix.h"
 #include "../../../src/tree/param.h"  // TrainParam
 #include "../filesystem.h"            // dmlc::TemporaryDirectory
diff --git a/tests/cpp/tree/gpu_hist/test_gradient_based_sampler.cu b/tests/cpp/tree/gpu_hist/test_gradient_based_sampler.cu
index 95ae02aee..1ecf1d345 100644
--- a/tests/cpp/tree/gpu_hist/test_gradient_based_sampler.cu
+++ b/tests/cpp/tree/gpu_hist/test_gradient_based_sampler.cu
@@ -3,9 +3,13 @@
  */
 #include <gtest/gtest.h>
 
+#if defined(XGBOOST_USE_CUDA)
 #include "../../../../src/data/ellpack_page.cuh"
 #include "../../../../src/tree/gpu_hist/gradient_based_sampler.cuh"
-#include "../../../../src/tree/param.h"
+#elif defined(XGBOOST_USE_HIP)
+#include "../../../../src/data/ellpack_page.hip.h"
+#include "../../../../src/tree/gpu_hist/gradient_based_sampler.hip.h"
+#endif
 #include "../../../../src/tree/param.h"  // TrainParam
 #include "../../filesystem.h"            // dmlc::TemporaryDirectory
 #include "../../helpers.h"
diff --git a/tests/cpp/tree/test_constraints.cu b/tests/cpp/tree/test_constraints.cu
index c9f1639b3..f69d51931 100644
--- a/tests/cpp/tree/test_constraints.cu
+++ b/tests/cpp/tree/test_constraints.cu
@@ -8,9 +8,15 @@
 #include <string>
 #include <bitset>
 #include <set>
+#if defined(XGBOOST_USE_CUDA)
 #include "../../../src/tree/constraints.cuh"
 #include "../../../src/tree/param.h"
 #include "../../../src/common/device_helpers.cuh"
+#elif defined(XGBOOST_USE_HIP)
+#include "../../../src/tree/constraints.hip.h"
+#include "../../../src/tree/param.h"
+#include "../../../src/common/device_helpers.hip.h"
+#endif
 
 namespace xgboost {
 namespace {