diff --git a/src/common/cuda_context.cuh b/src/common/cuda_context.cuh
index 9056c1b5e..c8b2e0792 100644
--- a/src/common/cuda_context.cuh
+++ b/src/common/cuda_context.cuh
@@ -1,5 +1,5 @@
 /**
- * Copyright 2022 by XGBoost Contributors
+ * Copyright 2022-2023, XGBoost Contributors
  */
 #ifndef XGBOOST_COMMON_CUDA_CONTEXT_CUH_
 #define XGBOOST_COMMON_CUDA_CONTEXT_CUH_
@@ -17,11 +17,23 @@ struct CUDAContext {
   /**
    * \brief Caching thrust policy.
    */
-  auto CTP() const { return thrust::cuda::par(caching_alloc_).on(dh::DefaultStream()); }
+  auto CTP() const {
+#if THRUST_MAJOR_VERSION >= 2
+    return thrust::cuda::par_nosync(caching_alloc_).on(dh::DefaultStream());
+#else
+    return thrust::cuda::par(caching_alloc_).on(dh::DefaultStream());
+#endif  // THRUST_MAJOR_VERSION >= 2
+  }
   /**
    * \brief Thrust policy without caching allocator.
    */
-  auto TP() const { return thrust::cuda::par(alloc_).on(dh::DefaultStream()); }
+  auto TP() const {
+#if THRUST_MAJOR_VERSION >= 2
+    return thrust::cuda::par_nosync(alloc_).on(dh::DefaultStream());
+#else
+    return thrust::cuda::par(alloc_).on(dh::DefaultStream());
+#endif  // THRUST_MAJOR_VERSION >= 2
+  }
   auto Stream() const { return dh::DefaultStream(); }
 };
 }  // namespace xgboost