diff --git a/CMakeLists.txt b/CMakeLists.txt
index d8190c5fb..cc9506ea0 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -32,6 +32,7 @@ option(R_LIB "Build shared library for R package" OFF)
 ## Dev
 option(USE_DEBUG_OUTPUT "Dump internal training results like gradients and predictions to stdout.
 Should only be used for debugging." OFF)
+option(ENABLE_ALL_WARNINGS "Enable all compiler warnings. Only effective for GCC/Clang" OFF)
 option(GOOGLE_TEST "Build google tests" OFF)
 option(USE_DMLC_GTEST "Use google tests bundled with dmlc-core submodule" OFF)
 option(USE_NVTX "Build with cuda profiling annotations. Developers only." OFF)
@@ -79,6 +80,11 @@ endif (R_LIB AND GOOGLE_TEST)
 if (USE_AVX)
   message(SEND_ERROR  "The option 'USE_AVX' is deprecated as experimental AVX features have been removed from XGBoost.")
 endif (USE_AVX)
+if (ENABLE_ALL_WARNINGS)
+  if ((NOT CMAKE_CXX_COMPILER_ID MATCHES "Clang") AND (NOT CMAKE_CXX_COMPILER_ID STREQUAL "GNU"))
+    message(SEND_ERROR "ENABLE_ALL_WARNINGS is only available for Clang and GCC.")
+  endif ((NOT CMAKE_CXX_COMPILER_ID MATCHES "Clang") AND (NOT CMAKE_CXX_COMPILER_ID STREQUAL "GNU"))
+endif (ENABLE_ALL_WARNINGS)
 
 #-- Sanitizer
 if (USE_SANITIZER)
@@ -130,6 +136,9 @@ if (MSVC)
                            -D_CRT_SECURE_NO_WARNINGS -D_CRT_SECURE_NO_DEPRECATE)
   endif (TARGET dmlc_unit_tests)
 endif (MSVC)
+if (ENABLE_ALL_WARNINGS)
+  target_compile_options(dmlc PRIVATE -Wall -Wextra)
+endif (ENABLE_ALL_WARNINGS)
 target_link_libraries(objxgboost PUBLIC dmlc)
 
 # rabit
@@ -159,6 +168,9 @@ foreach(lib rabit rabit_base rabit_empty rabit_mock rabit_mock_static)
     if (HIDE_CXX_SYMBOLS)  # Hide all C++ symbols from Rabit
       set_target_properties(${lib} PROPERTIES CXX_VISIBILITY_PRESET hidden)
     endif (HIDE_CXX_SYMBOLS)
+    if (ENABLE_ALL_WARNINGS)
+      target_compile_options(${lib} PRIVATE -Wall -Wextra)
+    endif (ENABLE_ALL_WARNINGS)
   endif (TARGET ${lib})
 endforeach()
 
diff --git a/R-package/CMakeLists.txt b/R-package/CMakeLists.txt
index d056ffe50..bf72bebde 100644
--- a/R-package/CMakeLists.txt
+++ b/R-package/CMakeLists.txt
@@ -6,6 +6,9 @@ file(GLOB_RECURSE R_SOURCES
   ${CMAKE_CURRENT_LIST_DIR}/src/*.c)
 # Use object library to expose symbols
 add_library(xgboost-r OBJECT ${R_SOURCES})
+if (ENABLE_ALL_WARNINGS)
+  target_compile_options(xgboost-r PRIVATE -Wall -Wextra)
+endif (ENABLE_ALL_WARNINGS)
 target_compile_definitions(xgboost-r
   PUBLIC
   -DXGBOOST_STRICT_R_MODE=1
diff --git a/jvm-packages/CMakeLists.txt b/jvm-packages/CMakeLists.txt
index c549d52ae..c7aa32f97 100644
--- a/jvm-packages/CMakeLists.txt
+++ b/jvm-packages/CMakeLists.txt
@@ -2,6 +2,9 @@ find_package(JNI REQUIRED)
 
 add_library(xgboost4j SHARED
   ${PROJECT_SOURCE_DIR}/jvm-packages/xgboost4j/src/native/xgboost4j.cpp)
+if (ENABLE_ALL_WARNINGS)
+  target_compile_options(xgboost4j PUBLIC -Wall -Wextra)
+endif (ENABLE_ALL_WARNINGS)
 target_link_libraries(xgboost4j PRIVATE objxgboost)
 target_include_directories(xgboost4j
   PRIVATE
diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
index 840a7f713..aae08d5f0 100644
--- a/src/CMakeLists.txt
+++ b/src/CMakeLists.txt
@@ -72,6 +72,11 @@ if (MSVC)
   )
 endif (MSVC)
 
+if (ENABLE_ALL_WARNINGS)
+  target_compile_options(objxgboost PUBLIC
+    $<IF:$<COMPILE_LANGUAGE:CUDA>,-Xcompiler=-Wall -Xcompiler=-Wextra,-Wall -Wextra>)
+endif (ENABLE_ALL_WARNINGS)
+
 set_target_properties(objxgboost PROPERTIES
   POSITION_INDEPENDENT_CODE ON
   CXX_STANDARD 14
diff --git a/tests/ci_build/build_via_cmake.sh b/tests/ci_build/build_via_cmake.sh
index fc7504233..ffb780c8d 100755
--- a/tests/ci_build/build_via_cmake.sh
+++ b/tests/ci_build/build_via_cmake.sh
@@ -4,7 +4,7 @@ set -e
 rm -rf build
 mkdir build
 cd build
-cmake .. "$@" -DGOOGLE_TEST=ON -DUSE_DMLC_GTEST=ON -DCMAKE_VERBOSE_MAKEFILE=ON -GNinja
+cmake .. "$@" -DGOOGLE_TEST=ON -DUSE_DMLC_GTEST=ON -DCMAKE_VERBOSE_MAKEFILE=ON -DENABLE_ALL_WARNINGS=ON -GNinja
 ninja clean
 time ninja -v
 cd ..
diff --git a/tests/cpp/CMakeLists.txt b/tests/cpp/CMakeLists.txt
index f092dc402..6d2097393 100644
--- a/tests/cpp/CMakeLists.txt
+++ b/tests/cpp/CMakeLists.txt
@@ -63,6 +63,10 @@ if (MSVC)
     -D_CRT_SECURE_NO_DEPRECATE
   )
 endif (MSVC)
+if (ENABLE_ALL_WARNINGS)
+  target_compile_options(testxgboost PUBLIC
+    $<IF:$<COMPILE_LANGUAGE:CUDA>,-Xcompiler=-Wall -Xcompiler=-Wextra,-Wall -Wextra>)
+endif (ENABLE_ALL_WARNINGS)
 
 target_include_directories(testxgboost
   PRIVATE