diff --git a/tests/cpp/CMakeLists.txt b/tests/cpp/CMakeLists.txt
index 71fedc368..00c099660 100644
--- a/tests/cpp/CMakeLists.txt
+++ b/tests/cpp/CMakeLists.txt
@@ -13,6 +13,11 @@ if (USE_CUDA)
   list(APPEND TEST_SOURCES ${CUDA_TEST_SOURCES})
 endif (USE_CUDA)
 
+if (USE_HIP)
+  file(GLOB_RECURSE HIP_TEST_SOURCES "*.hip")
+  list(APPEND TEST_SOURCES ${HIP_TEST_SOURCES})
+endif (USE_HIP)
+
 if (USE_HIP)
   file(GLOB_RECURSE HIP_TEST_SOURCES "*.cu")
   list(APPEND TEST_SOURCES ${HIP_TEST_SOURCES})
@@ -43,6 +48,11 @@ if (USE_HIP AND PLUGIN_RMM)
   target_include_directories(testxgboost PRIVATE ${HIP_INCLUDE_DIRS})
 endif (USE_HIP AND PLUGIN_RMM)
 
+if (USE_HIP AND PLUGIN_RMM)
+  find_package(HIP)
+  target_include_directories(testxgboost PRIVATE ${HIP_INCLUDE_DIRS})
+endif (USE_HIP AND PLUGIN_RMM)
+
 target_include_directories(testxgboost
   PRIVATE
   ${GTEST_INCLUDE_DIRS}
diff --git a/tests/cpp/helpers.cc b/tests/cpp/helpers.cc
index ebb56d2d3..e2d645f93 100644
--- a/tests/cpp/helpers.cc
+++ b/tests/cpp/helpers.cc
@@ -623,13 +623,27 @@ class RMMAllocator {
   int n_gpu;
   RMMAllocator() : n_gpu(common::AllVisibleGPUs()) {
     int current_device;
+#if defined(XGBOOST_USE_CUDA)
     CHECK_EQ(cudaGetDevice(&current_device), cudaSuccess);
+#elif defined(XGBOOST_USE_HIP)
+    CHECK_EQ(hipGetDevice(&current_device), hipSuccess);
+#endif
     for (int i = 0; i < n_gpu; ++i) {
+#if defined(XGBOOST_USE_CUDA)
       CHECK_EQ(cudaSetDevice(i), cudaSuccess);
+#elif defined(XGBOOST_USE_HIP)
+      CHECK_EQ(hipSetDevice(i), hipSuccess);
+#endif
+
       cuda_mr.push_back(std::make_unique<CUDAMemoryResource>());
       pool_mr.push_back(std::make_unique<PoolMemoryResource>(cuda_mr[i].get()));
     }
+
+#if defined(XGBOOST_USE_CUDA)
     CHECK_EQ(cudaSetDevice(current_device), cudaSuccess);
+#elif defined(XGBOOST_USE_HIP)
+    CHECK_EQ(hipSetDevice(current_device), hipSuccess);
+#endif
   }
   ~RMMAllocator() = default;
 };
diff --git a/tests/cpp/helpers.h b/tests/cpp/helpers.h
index ec0abf32b..1baa096cf 100644
--- a/tests/cpp/helpers.h
+++ b/tests/cpp/helpers.h
@@ -26,13 +26,13 @@
 #include "filesystem.h"  // dmlc::TemporaryDirectory
 #include "xgboost/linalg.h"
 
-#if defined(__CUDACC__)
+#if defined(__CUDACC__) || defined(__HIP_PLATFORM_AMD__)
 #define DeclareUnifiedTest(name) GPU ## name
 #else
 #define DeclareUnifiedTest(name) name
 #endif
 
-#if defined(__CUDACC__)
+#if defined(__CUDACC__) || defined(__HIP_PLATFORM_AMD__)
 #define GPUIDX 0
 #else
 #define GPUIDX -1
@@ -294,7 +294,7 @@ class RandomDataGenerator {
 
   std::shared_ptr<DMatrix> GenerateDMatrix(bool with_label = false, bool float_label = true,
                                            size_t classes = 1) const;
-#if defined(XGBOOST_USE_CUDA)
+#if defined(XGBOOST_USE_CUDA) || defined(XGBOOST_USE_HIP)
   std::shared_ptr<DMatrix> GenerateDeviceDMatrix();
 #endif
   std::shared_ptr<DMatrix> GenerateQuantileDMatrix();
diff --git a/tests/cpp/histogram_helpers.h b/tests/cpp/histogram_helpers.h
index 127f6fe44..9b32c8b83 100644
--- a/tests/cpp/histogram_helpers.h
+++ b/tests/cpp/histogram_helpers.h
@@ -1,9 +1,9 @@
-#if defined(__CUDACC__)
+#if defined(__CUDACC__) || defined(__HIP_PLATFORM_AMD__)
 #include "../../src/data/ellpack_page.cuh"
 #endif
 
 namespace xgboost {
-#if defined(__CUDACC__)
+#if defined(__CUDACC__) || defined(__HIP_PLATFORM_AMD__)
 namespace {
 class HistogramCutsWrapper : public common::HistogramCuts {
  public:
diff --git a/tests/cpp/linear/test_linear.hip b/tests/cpp/linear/test_linear.hip
index e69de29bb..7da4ec908 100644
--- a/tests/cpp/linear/test_linear.hip
+++ b/tests/cpp/linear/test_linear.hip
@@ -0,0 +1,4 @@
+
+#if defined(XGBOOST_USE_HIP)
+#include "test_linear.cu"
+#endif
diff --git a/tests/cpp/metric/test_auc.hip b/tests/cpp/metric/test_auc.hip
index e69de29bb..cbda5bb1d 100644
--- a/tests/cpp/metric/test_auc.hip
+++ b/tests/cpp/metric/test_auc.hip
@@ -0,0 +1,4 @@
+
+#if defined(XGBOOST_USE_HIP)
+#include "test_auc.cu"
+#endif
diff --git a/tests/cpp/metric/test_elementwise_metric.hip b/tests/cpp/metric/test_elementwise_metric.hip
index e69de29bb..299505a76 100644
--- a/tests/cpp/metric/test_elementwise_metric.hip
+++ b/tests/cpp/metric/test_elementwise_metric.hip
@@ -0,0 +1,4 @@
+
+#if defined(XGBOOST_USE_HIP)
+#include "test_elementwise_metric.cu"
+#endif
diff --git a/tests/cpp/metric/test_multiclass_metric.hip b/tests/cpp/metric/test_multiclass_metric.hip
index e69de29bb..9338631b2 100644
--- a/tests/cpp/metric/test_multiclass_metric.hip
+++ b/tests/cpp/metric/test_multiclass_metric.hip
@@ -0,0 +1,4 @@
+
+#if defined(XGBOOST_USE_HIP)
+#include "test_multiclass_metric.cu"
+#endif
diff --git a/tests/cpp/metric/test_rank_metric.cc b/tests/cpp/metric/test_rank_metric.cc
index 1edbd9fc8..faad00455 100644
--- a/tests/cpp/metric/test_rank_metric.cc
+++ b/tests/cpp/metric/test_rank_metric.cc
@@ -3,7 +3,7 @@
 
 #include "../helpers.h"
 
-#if !defined(__CUDACC__)
+#if !defined(__CUDACC__) && !defined(__HIP_PLATFORM_AMD__)
 TEST(Metric, AMS) {
   auto ctx = xgboost::CreateEmptyGenericParam(GPUIDX);
   EXPECT_ANY_THROW(xgboost::Metric::Create("ams", &ctx));
diff --git a/tests/cpp/metric/test_rank_metric.hip b/tests/cpp/metric/test_rank_metric.hip
index e69de29bb..5abf50e12 100644
--- a/tests/cpp/metric/test_rank_metric.hip
+++ b/tests/cpp/metric/test_rank_metric.hip
@@ -0,0 +1,4 @@
+
+#if defined(XGBOOST_USE_HIP)
+#include "test_rank_metric.cu"
+#endif
diff --git a/tests/cpp/metric/test_survival_metric.hip b/tests/cpp/metric/test_survival_metric.hip
index e69de29bb..1dbfe50e2 100644
--- a/tests/cpp/metric/test_survival_metric.hip
+++ b/tests/cpp/metric/test_survival_metric.hip
@@ -0,0 +1,4 @@
+
+#if defined(XGBOOST_USE_HIP)
+#include "test_survival_metric.cu"
+#endif
diff --git a/tests/cpp/objective/test_quantile_obj_gpu.hip b/tests/cpp/objective/test_quantile_obj_gpu.hip
new file mode 100644
index 000000000..e69de29bb
diff --git a/tests/cpp/predictor/test_gpu_predictor.cu b/tests/cpp/predictor/test_gpu_predictor.cu
index 4a3293dbe..1bb954ccd 100644
--- a/tests/cpp/predictor/test_gpu_predictor.cu
+++ b/tests/cpp/predictor/test_gpu_predictor.cu
@@ -146,7 +146,11 @@ TEST(GpuPredictor, LesserFeatures) {
 
 // Very basic test of empty model
 TEST(GPUPredictor, ShapStump) {
+#if defined(XGBOOST_USE_CUDA)
   cudaSetDevice(0);
+#elif defined(XGBOOST_USE_HIP)
+  hipSetDevice(0);
+#endif
 
   Context ctx;
   ctx.gpu_id = 0;
diff --git a/tests/cpp/predictor/test_gpu_predictor.hip b/tests/cpp/predictor/test_gpu_predictor.hip
index e69de29bb..c3310c46c 100644
--- a/tests/cpp/predictor/test_gpu_predictor.hip
+++ b/tests/cpp/predictor/test_gpu_predictor.hip
@@ -0,0 +1,4 @@
+
+#if defined(XGBOOST_USE_HIP)
+#include "test_gpu_predictor.cu"
+#endif
diff --git a/tests/cpp/predictor/test_predictor.cc b/tests/cpp/predictor/test_predictor.cc
index 3e8a94c75..7ab8946f7 100644
--- a/tests/cpp/predictor/test_predictor.cc
+++ b/tests/cpp/predictor/test_predictor.cc
@@ -170,7 +170,7 @@ void TestPredictionWithLesserFeatures(std::string predictor_name) {
   auto m_invalid = RandomDataGenerator(kRows, kTrainCols + 1, 0.5).GenerateDMatrix(false);
   ASSERT_THROW({learner->Predict(m_invalid, false, &prediction, 0, 0);}, dmlc::Error);
 
-#if defined(XGBOOST_USE_CUDA)
+#if defined(XGBOOST_USE_CUDA) || defined(XGBOOST_USE_HIP)
   HostDeviceVector<float> from_cpu;
   learner->SetParam("predictor", "cpu_predictor");
   learner->Predict(m_test, false, &from_cpu, 0, 0);
@@ -184,7 +184,7 @@ void TestPredictionWithLesserFeatures(std::string predictor_name) {
   for (size_t i = 0; i < h_cpu.size(); ++i) {
     ASSERT_NEAR(h_cpu[i], h_gpu[i], kRtEps);
   }
-#endif  // defined(XGBOOST_USE_CUDA)
+#endif  // defined(XGBOOST_USE_CUDA) || defined(XGBOOST_USE_HIP)
 }
 
 void GBTreeModelForTest(gbm::GBTreeModel *model, uint32_t split_ind,
diff --git a/tests/cpp/test_learner.cc b/tests/cpp/test_learner.cc
index 27bfbf21e..79a57b690 100644
--- a/tests/cpp/test_learner.cc
+++ b/tests/cpp/test_learner.cc
@@ -266,7 +266,7 @@ TEST(Learner, BinaryModelIO) {
   ASSERT_EQ(config_str.find("WARNING"), std::string::npos);
 }
 
-#if defined(XGBOOST_USE_CUDA)
+#if defined(XGBOOST_USE_CUDA) || defined(XGBOOST_USE_HIP)
 // Tests for automatic GPU configuration.
 TEST(Learner, GPUConfiguration) {
   using Arg = std::pair<std::string, std::string>;
@@ -325,7 +325,7 @@ TEST(Learner, GPUConfiguration) {
     ASSERT_EQ(learner->Ctx()->gpu_id, 0);
   }
 }
-#endif  // defined(XGBOOST_USE_CUDA)
+#endif  // defined(XGBOOST_USE_CUDA) || defined(XGBOOST_USE_HIP)
 
 TEST(Learner, Seed) {
   auto m = RandomDataGenerator{10, 10, 0}.GenerateDMatrix();
diff --git a/tests/cpp/test_multi_target.cc b/tests/cpp/test_multi_target.cc
index d2e34235c..e96c2eb06 100644
--- a/tests/cpp/test_multi_target.cc
+++ b/tests/cpp/test_multi_target.cc
@@ -116,9 +116,9 @@ TEST_F(TestL1MultiTarget, Exact) { this->RunTest("exact"); }
 
 TEST_F(TestL1MultiTarget, Approx) { this->RunTest("approx"); }
 
-#if defined(XGBOOST_USE_CUDA)
+#if defined(XGBOOST_USE_CUDA) || defined(XGBOOST_USE_HIP)
 TEST_F(TestL1MultiTarget, GpuHist) { this->RunTest("gpu_hist"); }
-#endif  // defined(XGBOOST_USE_CUDA)
+#endif  // defined(XGBOOST_USE_CUDA) || defined(XGBOOST_USE_HIP)
 
 TEST(MultiStrategy, Configure) {
   auto p_fmat = RandomDataGenerator{12ul, 3ul, 0.0}.GenerateDMatrix();
diff --git a/tests/cpp/test_serialization.cc b/tests/cpp/test_serialization.cc
index 15765f09f..b963c8441 100644
--- a/tests/cpp/test_serialization.cc
+++ b/tests/cpp/test_serialization.cc
@@ -338,7 +338,7 @@ TEST_F(SerializationTest, CPUCoordDescent) {
                            fmap_, p_dmat_);
 }
 
-#if defined(XGBOOST_USE_CUDA)
+#if defined(XGBOOST_USE_CUDA) || defined(XGBOOST_USE_HIP)
 TEST_F(SerializationTest, GpuHist) {
   TestLearnerSerialization({{"booster", "gbtree"},
                             {"seed", "0"},
@@ -416,7 +416,7 @@ TEST_F(SerializationTest, GPUCoordDescent) {
                             {"updater", "gpu_coord_descent"}},
                            fmap_, p_dmat_);
 }
-#endif  // defined(XGBOOST_USE_CUDA)
+#endif  // defined(XGBOOST_USE_CUDA) || defined(XGBOOST_USE_HIP)
 
 class L1SerializationTest : public SerializationTest {};
 
@@ -447,7 +447,7 @@ TEST_F(L1SerializationTest, Hist) {
                            fmap_, p_dmat_);
 }
 
-#if defined(XGBOOST_USE_CUDA)
+#if defined(XGBOOST_USE_CUDA) || defined(XGBOOST_USE_HIP)
 TEST_F(L1SerializationTest, GpuHist) {
   TestLearnerSerialization({{"booster", "gbtree"},
                             {"objective", "reg:absoluteerror"},
@@ -456,7 +456,7 @@ TEST_F(L1SerializationTest, GpuHist) {
                             {"tree_method", "gpu_hist"}},
                            fmap_, p_dmat_);
 }
-#endif  //  defined(XGBOOST_USE_CUDA)
+#endif  //  defined(XGBOOST_USE_CUDA) || defined(XGBOOST_USE_HIP)
 
 class LogitSerializationTest : public SerializationTest {
  protected:
@@ -542,7 +542,7 @@ TEST_F(LogitSerializationTest, CPUCoordDescent) {
                            fmap_, p_dmat_);
 }
 
-#if defined(XGBOOST_USE_CUDA)
+#if defined(XGBOOST_USE_CUDA) || defined(XGBOOST_USE_HIP)
 TEST_F(LogitSerializationTest, GpuHist) {
   TestLearnerSerialization({{"booster", "gbtree"},
                             {"objective", "binary:logistic"},
@@ -578,7 +578,7 @@ TEST_F(LogitSerializationTest, GPUCoordDescent) {
                             {"updater", "gpu_coord_descent"}},
                            fmap_, p_dmat_);
 }
-#endif  // defined(XGBOOST_USE_CUDA)
+#endif  // defined(XGBOOST_USE_CUDA) || defined(XGBOOST_USE_HIP)
 
 class MultiClassesSerializationTest : public SerializationTest {
  protected:
@@ -684,7 +684,7 @@ TEST_F(MultiClassesSerializationTest, CPUCoordDescent) {
                            fmap_, p_dmat_);
 }
 
-#if defined(XGBOOST_USE_CUDA)
+#if defined(XGBOOST_USE_CUDA) || defined(XGBOOST_USE_HIP)
 TEST_F(MultiClassesSerializationTest, GpuHist) {
   TestLearnerSerialization({{"booster", "gbtree"},
                             {"num_class", std::to_string(kClasses)},
@@ -731,5 +731,5 @@ TEST_F(MultiClassesSerializationTest, GPUCoordDescent) {
                             {"updater", "gpu_coord_descent"}},
                            fmap_, p_dmat_);
 }
-#endif  // defined(XGBOOST_USE_CUDA)
+#endif  // defined(XGBOOST_USE_CUDA) || defined(XGBOOST_USE_HIP)
 }       // namespace xgboost