diff --git a/src/tree/gpu_hist/gradient_based_sampler.cuh b/src/tree/gpu_hist/gradient_based_sampler.cuh
index 5be6c71de..925d4af2a 100644
--- a/src/tree/gpu_hist/gradient_based_sampler.cuh
+++ b/src/tree/gpu_hist/gradient_based_sampler.cuh
@@ -6,7 +6,12 @@
 #include <xgboost/data.h>
 #include <xgboost/span.h>
 
+#if defined(XGBOOST_USE_CUDA)
 #include "../../common/device_helpers.cuh"
+#elif defined(XGBOOST_USE_HIP)
+#include "../../common/device_helpers.hip.h"
+#endif
+
 #include "../../data/ellpack_page.cuh"
 
 namespace xgboost {
diff --git a/src/tree/gpu_hist/gradient_based_sampler.hip b/src/tree/gpu_hist/gradient_based_sampler.hip
index e69de29bb..e7094cd3e 100644
--- a/src/tree/gpu_hist/gradient_based_sampler.hip
+++ b/src/tree/gpu_hist/gradient_based_sampler.hip
@@ -0,0 +1,4 @@
+
+#if defined(XGBOOST_USE_HIP)
+#include "gradient_based_sampler.cu"
+#endif