diff --git a/CMakeLists.txt b/CMakeLists.txt
index 951400e0c..b6b569894 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -39,6 +39,7 @@ option(ENABLE_ALL_WARNINGS "Enable all compiler warnings. Only effective for GCC
 option(LOG_CAPI_INVOCATION "Log all C API invocations for debugging" OFF)
 option(GOOGLE_TEST "Build google tests" OFF)
 option(USE_DMLC_GTEST "Use google tests bundled with dmlc-core submodule" OFF)
+option(USE_DEVICE_DEBUG "Generate CUDA device debug info." OFF)
 option(USE_NVTX "Build with cuda profiling annotations. Developers only." OFF)
 set(NVTX_HEADER_DIR "" CACHE PATH "Path to the stand-alone nvtx header")
 option(RABIT_MOCK "Build rabit with mock" OFF)
@@ -74,6 +75,9 @@ endif (USE_DEBUG_OUTPUT AND (NOT (CMAKE_BUILD_TYPE MATCHES Debug)))
 if (USE_NCCL AND NOT (USE_CUDA))
   message(SEND_ERROR "`USE_NCCL` must be enabled with `USE_CUDA` flag.")
 endif (USE_NCCL AND NOT (USE_CUDA))
+if (USE_DEVICE_DEBUG AND NOT (USE_CUDA))
+  message(SEND_ERROR "`USE_DEVICE_DEBUG` must be enabled with `USE_CUDA` flag.")
+endif (USE_DEVICE_DEBUG AND NOT (USE_CUDA))
 if (BUILD_WITH_SHARED_NCCL AND (NOT USE_NCCL))
   message(SEND_ERROR "Build XGBoost with -DUSE_NCCL=ON to enable BUILD_WITH_SHARED_NCCL.")
 endif (BUILD_WITH_SHARED_NCCL AND (NOT USE_NCCL))
@@ -123,7 +127,6 @@ if (USE_CUDA)
   endif()
   set(GEN_CODE "")
   format_gencode_flags("${GPU_COMPUTE_VER}" GEN_CODE)
-  message(STATUS "CUDA GEN_CODE: ${GEN_CODE}")
   add_subdirectory(${PROJECT_SOURCE_DIR}/gputreeshap)
 endif (USE_CUDA)
 
@@ -213,9 +216,9 @@ else (BUILD_STATIC_LIB)
 endif (BUILD_STATIC_LIB)
 target_link_libraries(xgboost PRIVATE objxgboost)
 
-if (USE_NVTX)
-  enable_nvtx(xgboost)
-endif (USE_NVTX)
+if (USE_CUDA)
+  xgboost_set_cuda_flags(xgboost)
+endif (USE_CUDA)
 
 #-- Hide all C++ symbols
 if (HIDE_CXX_SYMBOLS)
diff --git a/cmake/Utils.cmake b/cmake/Utils.cmake
index 141be6f57..3e671af1c 100644
--- a/cmake/Utils.cmake
+++ b/cmake/Utils.cmake
@@ -54,23 +54,22 @@ endfunction(msvc_use_static_runtime)
 
 # Set output directory of target, ignoring debug or release
 function(set_output_directory target dir)
-	set_target_properties(${target} PROPERTIES
-		RUNTIME_OUTPUT_DIRECTORY ${dir}
-		RUNTIME_OUTPUT_DIRECTORY_DEBUG ${dir}
-		RUNTIME_OUTPUT_DIRECTORY_RELEASE ${dir}
-		RUNTIME_OUTPUT_DIRECTORY_RELWITHDEBINFO ${dir}
-		RUNTIME_OUTPUT_DIRECTORY_MINSIZEREL ${dir}
-		LIBRARY_OUTPUT_DIRECTORY ${dir}
-		LIBRARY_OUTPUT_DIRECTORY_DEBUG ${dir}
-		LIBRARY_OUTPUT_DIRECTORY_RELEASE ${dir}
-		LIBRARY_OUTPUT_DIRECTORY_RELWITHDEBINFO ${dir}
-		LIBRARY_OUTPUT_DIRECTORY_MINSIZEREL ${dir}
-		ARCHIVE_OUTPUT_DIRECTORY ${dir}
-		ARCHIVE_OUTPUT_DIRECTORY_DEBUG ${dir}
-		ARCHIVE_OUTPUT_DIRECTORY_RELEASE ${dir}
-		ARCHIVE_OUTPUT_DIRECTORY_RELWITHDEBINFO ${dir}
-		ARCHIVE_OUTPUT_DIRECTORY_MINSIZEREL ${dir}
-	)
+  set_target_properties(${target} PROPERTIES
+    RUNTIME_OUTPUT_DIRECTORY ${dir}
+    RUNTIME_OUTPUT_DIRECTORY_DEBUG ${dir}
+    RUNTIME_OUTPUT_DIRECTORY_RELEASE ${dir}
+    RUNTIME_OUTPUT_DIRECTORY_RELWITHDEBINFO ${dir}
+    RUNTIME_OUTPUT_DIRECTORY_MINSIZEREL ${dir}
+    LIBRARY_OUTPUT_DIRECTORY ${dir}
+    LIBRARY_OUTPUT_DIRECTORY_DEBUG ${dir}
+    LIBRARY_OUTPUT_DIRECTORY_RELEASE ${dir}
+    LIBRARY_OUTPUT_DIRECTORY_RELWITHDEBINFO ${dir}
+    LIBRARY_OUTPUT_DIRECTORY_MINSIZEREL ${dir}
+    ARCHIVE_OUTPUT_DIRECTORY ${dir}
+    ARCHIVE_OUTPUT_DIRECTORY_DEBUG ${dir}
+    ARCHIVE_OUTPUT_DIRECTORY_RELEASE ${dir}
+    ARCHIVE_OUTPUT_DIRECTORY_RELWITHDEBINFO ${dir}
+    ARCHIVE_OUTPUT_DIRECTORY_MINSIZEREL ${dir})
 endfunction(set_output_directory)
 
 # Set a default build type to release if none was specified
@@ -91,7 +90,9 @@ function(format_gencode_flags flags out)
   endif()
   # Set up architecture flags
   if(NOT flags)
-    if(CUDA_VERSION VERSION_GREATER_EQUAL "10.0")
+    if (CUDA_VERSION VERSION_GREATER_EQUAL "11.0")
+      set(flags "35;50;52;60;61;70;75;80")
+    elseif(CUDA_VERSION VERSION_GREATER_EQUAL "10.0")
       set(flags "35;50;52;60;61;70;75")
     elseif(CUDA_VERSION VERSION_GREATER_EQUAL "9.0")
       set(flags "35;50;52;60;61;70")
@@ -99,15 +100,25 @@ function(format_gencode_flags flags out)
       set(flags "35;50;52;60;61")
     endif()
   endif()
-  # Generate SASS
-  foreach(ver ${flags})
-    set(${out} "${${out}}--generate-code=arch=compute_${ver},code=sm_${ver};")
-  endforeach()
-  # Generate PTX for last architecture
-  list(GET flags -1 ver)
-  set(${out} "${${out}}--generate-code=arch=compute_${ver},code=compute_${ver};")
 
-  set(${out} "${${out}}" PARENT_SCOPE)
+  if (CMAKE_VERSION VERSION_GREATER_EQUAL "3.18")
+    cmake_policy(SET CMP0104 NEW)
+    foreach(ver ${flags})
+      set(CMAKE_CUDA_ARCHITECTURES "${ver}-real;${ver}-virtual;${CMAKE_CUDA_ARCHITECTURES}")
+    endforeach()
+    set(CMAKE_CUDA_ARCHITECTURES "${CMAKE_CUDA_ARCHITECTURES}" PARENT_SCOPE)
+    message(STATUS "CMAKE_CUDA_ARCHITECTURES: ${CMAKE_CUDA_ARCHITECTURES}")
+  else()
+    # Generate SASS
+    foreach(ver ${flags})
+      set(${out} "${${out}}--generate-code=arch=compute_${ver},code=sm_${ver};")
+    endforeach()
+    # Generate PTX for last architecture
+    list(GET flags -1 ver)
+    set(${out} "${${out}}--generate-code=arch=compute_${ver},code=compute_${ver};")
+    set(${out} "${${out}}" PARENT_SCOPE)
+    message(STATUS "CUDA GEN_CODE: ${GEN_CODE}")
+  endif (CMAKE_VERSION VERSION_GREATER_EQUAL "3.18")
 endfunction(format_gencode_flags flags)
 
 macro(enable_nvtx target)
@@ -116,3 +127,60 @@ macro(enable_nvtx target)
   target_link_libraries(${target} PRIVATE "${NVTX_LIBRARY}")
   target_compile_definitions(${target} PRIVATE -DXGBOOST_USE_NVTX=1)
 endmacro()
+
+# Set CUDA related flags to target.  Must be used after code `format_gencode_flags`.
+function(xgboost_set_cuda_flags target)
+  find_package(OpenMP REQUIRED)
+  target_link_libraries(${target} PUBLIC OpenMP::OpenMP_CXX)
+
+  target_compile_options(${target} PRIVATE
+    $<$<COMPILE_LANGUAGE:CUDA>:--expt-extended-lambda>
+    $<$<COMPILE_LANGUAGE:CUDA>:--expt-relaxed-constexpr>
+    $<$<COMPILE_LANGUAGE:CUDA>:${GEN_CODE}>
+    $<$<COMPILE_LANGUAGE:CUDA>:-Xcompiler=${OpenMP_CXX_FLAGS}>)
+
+  if (CMAKE_VERSION VERSION_GREATER_EQUAL "3.18")
+    set_property(TARGET ${target} PROPERTY CUDA_ARCHITECTURES ${CMAKE_CUDA_ARCHITECTURES})
+  endif (CMAKE_VERSION VERSION_GREATER_EQUAL "3.18")
+
+  if (USE_DEVICE_DEBUG)
+    if (CMAKE_BUILD_TYPE MATCHES "Debug")
+      target_compile_options(${target} PRIVATE
+        $<$<COMPILE_LANGUAGE:CUDA>:-G;-src-in-ptx>)
+    endif(CMAKE_BUILD_TYPE MATCHES "Debug")
+  else (USE_DEVICE_DEBUG)
+    target_compile_options(${target} PRIVATE
+      $<$<COMPILE_LANGUAGE:CUDA>:-lineinfo>)
+  endif (USE_DEVICE_DEBUG)
+
+  if (USE_NVTX)
+    enable_nvtx(${target})
+  endif (USE_NVTX)
+
+  target_compile_definitions(${target} PRIVATE -DXGBOOST_USE_CUDA=1)
+  if (CMAKE_CUDA_COMPILER_VERSION VERSION_LESS 11.0)
+    target_include_directories(${target} PRIVATE ${xgboost_SOURCE_DIR}/cub/)
+  endif (CMAKE_CUDA_COMPILER_VERSION VERSION_LESS 11.0)
+
+  if (MSVC)
+    target_compile_options(${target} PRIVATE
+      $<$<COMPILE_LANGUAGE:CUDA>:-Xcompiler=/utf-8>)
+  endif (MSVC)
+
+  set_target_properties(${target} PROPERTIES
+    CUDA_STANDARD 14
+    CUDA_STANDARD_REQUIRED ON
+    CUDA_SEPARABLE_COMPILATION OFF)
+
+  if (HIDE_CXX_SYMBOLS)
+    target_compile_options(${target} PRIVATE
+      $<$<COMPILE_LANGUAGE:CUDA>:-Xcompiler=-fvisibility=hidden>)
+  endif (HIDE_CXX_SYMBOLS)
+
+  if (USE_NCCL)
+    find_package(Nccl REQUIRED)
+    target_include_directories(${target} PRIVATE ${NCCL_INCLUDE_DIR})
+    target_compile_definitions(${target} PRIVATE -DXGBOOST_USE_NCCL=1)
+    target_link_libraries(${target} PUBLIC ${NCCL_LIBRARY})
+  endif (USE_NCCL)
+endfunction(xgboost_set_cuda_flags)
diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
index 8e62d9ced..edcb64a34 100644
--- a/src/CMakeLists.txt
+++ b/src/CMakeLists.txt
@@ -8,47 +8,7 @@ target_sources(objxgboost PRIVATE ${CPU_SOURCES})
 if (USE_CUDA)
   file(GLOB_RECURSE CUDA_SOURCES *.cu *.cuh)
   target_sources(objxgboost PRIVATE ${CUDA_SOURCES})
-  target_compile_definitions(objxgboost PRIVATE -DXGBOOST_USE_CUDA=1)
-  if (CMAKE_CUDA_COMPILER_VERSION VERSION_LESS 11.0)
-    target_include_directories(objxgboost PRIVATE ${xgboost_SOURCE_DIR}/cub/)
-  endif (CMAKE_CUDA_COMPILER_VERSION VERSION_LESS 11.0)
-  target_compile_options(objxgboost PRIVATE
-    $<$<COMPILE_LANGUAGE:CUDA>:--expt-extended-lambda>
-    $<$<COMPILE_LANGUAGE:CUDA>:--expt-relaxed-constexpr>
-    $<$<COMPILE_LANGUAGE:CUDA>:-lineinfo>
-    $<$<COMPILE_LANGUAGE:CUDA>:${GEN_CODE}>)
-
-  if (USE_NCCL)
-    find_package(Nccl REQUIRED)
-    target_include_directories(objxgboost PRIVATE ${NCCL_INCLUDE_DIR})
-    target_compile_definitions(objxgboost PRIVATE -DXGBOOST_USE_NCCL=1)
-    target_link_libraries(objxgboost PUBLIC ${NCCL_LIBRARY})
-  endif (USE_NCCL)
-
-  if (USE_NVTX)
-    enable_nvtx(objxgboost)
-  endif (USE_NVTX)
-
-  target_compile_options(objxgboost PRIVATE
-    $<$<COMPILE_LANGUAGE:CUDA>:-Xcompiler=${OpenMP_CXX_FLAGS}>
-  )
-
-  if (MSVC)
-    target_compile_options(objxgboost PRIVATE
-      $<$<COMPILE_LANGUAGE:CUDA>:-Xcompiler=/utf-8>
-    )
-  endif (MSVC)
-
-  if (HIDE_CXX_SYMBOLS)
-    target_compile_options(objxgboost PRIVATE
-      $<$<COMPILE_LANGUAGE:CUDA>:-Xcompiler=-fvisibility=hidden>
-    )
-  endif (HIDE_CXX_SYMBOLS)
-
-  set_target_properties(objxgboost PROPERTIES
-    CUDA_STANDARD 14
-    CUDA_STANDARD_REQUIRED ON
-    CUDA_SEPARABLE_COMPILATION OFF)
+  xgboost_set_cuda_flags(objxgboost)
   target_link_libraries(objxgboost PRIVATE GPUTreeShap)
 endif (USE_CUDA)
 
diff --git a/tests/cpp/CMakeLists.txt b/tests/cpp/CMakeLists.txt
index b733eed23..f08f20cc6 100644
--- a/tests/cpp/CMakeLists.txt
+++ b/tests/cpp/CMakeLists.txt
@@ -23,45 +23,10 @@ add_executable(testxgboost ${TEST_SOURCES}
 target_link_libraries(testxgboost PRIVATE objxgboost)
 
 if (USE_CUDA)
-  # OpenMP is mandatory for CUDA
-  find_package(OpenMP REQUIRED)
-  if (CMAKE_CUDA_COMPILER_VERSION VERSION_LESS 11.0)
-    target_include_directories(testxgboost PRIVATE
-      ${xgboost_SOURCE_DIR}/cub/)
-  endif (CMAKE_CUDA_COMPILER_VERSION VERSION_LESS 11.0)
-  target_compile_options(testxgboost PRIVATE
-    $<$<COMPILE_LANGUAGE:CUDA>:--expt-extended-lambda>
-    $<$<COMPILE_LANGUAGE:CUDA>:--expt-relaxed-constexpr>
-    $<$<COMPILE_LANGUAGE:CUDA>:-lineinfo>
-    $<$<COMPILE_LANGUAGE:CUDA>:-Xcompiler=${OpenMP_CXX_FLAGS}>
-    $<$<COMPILE_LANGUAGE:CUDA>:${GEN_CODE}>)
-  target_compile_definitions(testxgboost
-    PRIVATE -DXGBOOST_USE_CUDA=1)
+  xgboost_set_cuda_flags(testxgboost)
+
   find_package(CUDA)
   target_include_directories(testxgboost PRIVATE ${CUDA_INCLUDE_DIRS})
-  set_target_properties(testxgboost PROPERTIES
-    CUDA_SEPARABLE_COMPILATION OFF)
-
-  if (USE_NCCL)
-    find_package(Nccl REQUIRED)
-    target_include_directories(testxgboost PRIVATE ${NCCL_INCLUDE_DIR})
-    target_compile_definitions(testxgboost PRIVATE -DXGBOOST_USE_NCCL=1)
-    target_link_libraries(testxgboost PRIVATE ${NCCL_LIBRARY})
-  endif (USE_NCCL)
-
-  if (USE_NVTX)
-    enable_nvtx(testxgboost)
-  endif (USE_NVTX)
-
-  if (MSVC)
-    target_compile_options(testxgboost PRIVATE
-      $<$<COMPILE_LANGUAGE:CUDA>:-Xcompiler=/utf-8>
-    )
-  endif (MSVC)
-
-  set_target_properties(testxgboost PROPERTIES
-    CUDA_STANDARD 14
-    CUDA_STANDARD_REQUIRED ON)
 endif (USE_CUDA)
 
 if (MSVC)