diff --git a/CMakeLists.txt b/CMakeLists.txt
index 5844da216..58b9b8fb8 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -256,9 +256,10 @@ if (USE_HIP)
   find_package(rocthrust REQUIRED)
   find_package(hipcub REQUIRED)
 
-  set(CMAKE_HIP_FLAGS "${CMAKE_HIP_FLAGS} -I${HIP_INCLUDE_DIRS} -I${HIP_INCLUDE_DIRS}/hip")
+  set(CMAKE_HIP_FLAGS "${CMAKE_HIP_FLAGS} -I${HIP_INCLUDE_DIRS}")
   set(CMAKE_HIP_FLAGS "${CMAKE_HIP_FLAGS} -Wunused-result -w")
   set(CMAKE_HIP_FLAGS "${CMAKE_HIP_FLAGS} -D__HIP_PLATFORM_AMD__")
+  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -I${HIP_INCLUDE_DIRS}")
   add_subdirectory(${PROJECT_SOURCE_DIR}/rocgputreeshap)
 endif (USE_HIP)
 
diff --git a/src/c_api/c_api.cu b/src/c_api/c_api.cu
index ebcea1c2f..a0bd28b72 100644
--- a/src/c_api/c_api.cu
+++ b/src/c_api/c_api.cu
@@ -55,8 +55,10 @@ void XGBBuildInfoDevice(Json *p_info) {
   info["RCCL_VERSION"] = v;
   info["NCCL_VERSION"] = v;
 #if defined(XGBOOST_USE_DLOPEN_RCCL)
+  info["USE_DLOPEN_NCCL"] = Boolean{true};
   info["USE_DLOPEN_RCCL"] = Boolean{true};
 #else
+  info["USE_DLOPEN_NCCL"] = Boolean{false};
   info["USE_DLOPEN_RCCL"] = Boolean{false};
 #endif  // defined(XGBOOST_USE_DLOPEN_RCCL)
 #else
diff --git a/src/collective/nccl_stub.cc b/src/collective/nccl_stub.cc
index 408432438..44bd3e9a1 100644
--- a/src/collective/nccl_stub.cc
+++ b/src/collective/nccl_stub.cc
@@ -13,7 +13,6 @@
 #include <thrust/system_error.h>       // for system_error
 #elif defined(XGBOOST_USE_RCCL)
 #include "../common/cuda_to_hip.h"
-#include "../common/device_helpers.hip.h"
 #include <hip/hip_runtime_api.h>  // for cudaPeekAtLastError
 #include <dlfcn.h>             // for dlclose, dlsym, dlopen
 #include <rccl.h>
diff --git a/src/collective/nccl_stub.h b/src/collective/nccl_stub.h
index 978f34028..60388ac9e 100644
--- a/src/collective/nccl_stub.h
+++ b/src/collective/nccl_stub.h
@@ -9,7 +9,15 @@
 #include <nccl.h>
 #elif defined(XGBOOST_USE_RCCL)
 #include "../common/cuda_to_hip.h"
-#include "../common/device_helpers.cuh"
+
+#ifndef __HIP_PLATFORM_AMD__
+#define __HIP_PLATFORM_AMD__
+#endif
+
+#ifndef THRUST_DEVICE_SYSTEM
+#define THRUST_DEVICE_SYSTEM THRUST_DEVICE_SYSTEM_HIP
+#endif
+
 #include <hip/hip_runtime_api.h>
 #include <rccl.h>
 #endif
diff --git a/src/common/random.cc b/src/common/random.cc
index e0d1a2255..7d2c34dd8 100644
--- a/src/common/random.cc
+++ b/src/common/random.cc
@@ -19,7 +19,7 @@ std::shared_ptr<HostDeviceVector<bst_feature_t>> ColumnSampler::ColSample(
   auto p_new_features = std::make_shared<HostDeviceVector<bst_feature_t>>();
 
   if (ctx_->IsCUDA()) {
-#if defined(XGBOOST_USE_CUDA)
+#if defined(XGBOOST_USE_CUDA) || defined(XGBOOST_USE_HIP)
     cuda_impl::SampleFeature(ctx_, n, p_features, p_new_features, this->feature_weights_,
                              &this->weight_buffer_, &this->idx_buffer_, &rng_);
     return p_new_features;
diff --git a/src/common/random.h b/src/common/random.h
index 2a94123a3..098e94b74 100644
--- a/src/common/random.h
+++ b/src/common/random.h
@@ -180,7 +180,7 @@ class ColumnSampler {
     if (ctx->IsCPU()) {
       std::iota(feature_set_tree_->HostVector().begin(), feature_set_tree_->HostVector().end(), 0);
     } else {
-#if defined(XGBOOST_USE_CUDA)
+#if defined(XGBOOST_USE_CUDA) || defined(XGBOOST_USE_HIP)
       cuda_impl::InitFeatureSet(ctx, feature_set_tree_);
 #else
       AssertGPUSupport();
diff --git a/src/common/random.hip b/src/common/random.hip
new file mode 100644
index 000000000..8f2a6f7a0
--- /dev/null
+++ b/src/common/random.hip
@@ -0,0 +1,4 @@
+
+#if defined(XGBOOST_USE_HIP)
+#include "random.cu"
+#endif