diff --git a/R-package/src/xgboost_R.cc b/R-package/src/xgboost_R.cc
index 0d5311705..cb86ef469 100644
--- a/R-package/src/xgboost_R.cc
+++ b/R-package/src/xgboost_R.cc
@@ -1,17 +1,14 @@
 // Copyright (c) 2014 by Contributors
-#include <cstring>
-#include <cstdio>
-#include <vector>
-#include <string>
-#include <utility>
-#include <sstream>
-
 #include <dmlc/logging.h>
 #include <dmlc/omp.h>
 #include <xgboost/c_api.h>
-
-#include "xgboost_R.h"
-
+#include <vector>
+#include <string>
+#include <utility>
+#include <cstring>
+#include <cstdio>
+#include <sstream>
+#include "./xgboost_R.h"
 
 /*!
  * \brief macro to annotate begin of api
diff --git a/include/xgboost/base.h b/include/xgboost/base.h
index 81b618a92..480242611 100644
--- a/include/xgboost/base.h
+++ b/include/xgboost/base.h
@@ -6,15 +6,14 @@
 #ifndef XGBOOST_BASE_H_
 #define XGBOOST_BASE_H_
 
+#include <dmlc/base.h>
+#include <dmlc/omp.h>
 #include <cmath>
 #include <iostream>
 #include <vector>
 #include <string>
 #include <utility>
 
-#include <dmlc/base.h>
-#include <dmlc/omp.h>
-
 /*!
  * \brief string flag for R library, to leave hooks when needed.
  */
diff --git a/include/xgboost/data.h b/include/xgboost/data.h
index 908b8a15b..d7ab895a8 100644
--- a/include/xgboost/data.h
+++ b/include/xgboost/data.h
@@ -7,13 +7,6 @@
 #ifndef XGBOOST_DATA_H_
 #define XGBOOST_DATA_H_
 
-#include <memory>
-#include <numeric>
-#include <algorithm>
-#include <string>
-#include <utility>
-#include <vector>
-
 #include <dmlc/base.h>
 #include <dmlc/data.h>
 #include <dmlc/serializer.h>
@@ -22,6 +15,13 @@
 #include <xgboost/span.h>
 #include <xgboost/host_device_vector.h>
 
+#include <memory>
+#include <numeric>
+#include <algorithm>
+#include <string>
+#include <utility>
+#include <vector>
+
 namespace xgboost {
 // forward declare dmatrix.
 class DMatrix;
diff --git a/include/xgboost/feature_map.h b/include/xgboost/feature_map.h
index 47c20f8e2..a48e28ba1 100644
--- a/include/xgboost/feature_map.h
+++ b/include/xgboost/feature_map.h
@@ -7,13 +7,13 @@
 #ifndef XGBOOST_FEATURE_MAP_H_
 #define XGBOOST_FEATURE_MAP_H_
 
+#include <xgboost/logging.h>
+
 #include <vector>
 #include <string>
 #include <cstring>
 #include <iostream>
 
-#include <xgboost/logging.h>
-
 namespace xgboost {
 /*!
  * \brief Feature map data structure to help text model dump.
diff --git a/include/xgboost/gbm.h b/include/xgboost/gbm.h
index 95c54b959..d359f5c29 100644
--- a/include/xgboost/gbm.h
+++ b/include/xgboost/gbm.h
@@ -8,13 +8,6 @@
 #ifndef XGBOOST_GBM_H_
 #define XGBOOST_GBM_H_
 
-#include <vector>
-#include <utility>
-#include <string>
-#include <functional>
-#include <unordered_map>
-#include <memory>
-
 #include <dmlc/registry.h>
 #include <dmlc/any.h>
 #include <xgboost/base.h>
@@ -22,6 +15,13 @@
 #include <xgboost/host_device_vector.h>
 #include <xgboost/model.h>
 
+#include <vector>
+#include <utility>
+#include <string>
+#include <functional>
+#include <unordered_map>
+#include <memory>
+
 namespace xgboost {
 
 class Json;
diff --git a/include/xgboost/generic_parameters.h b/include/xgboost/generic_parameters.h
index 19e77b304..752342b9a 100644
--- a/include/xgboost/generic_parameters.h
+++ b/include/xgboost/generic_parameters.h
@@ -5,11 +5,11 @@
 #ifndef XGBOOST_GENERIC_PARAMETERS_H_
 #define XGBOOST_GENERIC_PARAMETERS_H_
 
-#include <string>
-
 #include <xgboost/logging.h>
 #include <xgboost/parameter.h>
 
+#include <string>
+
 namespace xgboost {
 struct GenericParameter : public XGBoostParameter<GenericParameter> {
   // Constant representing the device ID of CPU.
diff --git a/include/xgboost/json.h b/include/xgboost/json.h
index 59861aa44..c190505bf 100644
--- a/include/xgboost/json.h
+++ b/include/xgboost/json.h
@@ -4,15 +4,15 @@
 #ifndef XGBOOST_JSON_H_
 #define XGBOOST_JSON_H_
 
+#include <xgboost/logging.h>
+#include <xgboost/parameter.h>
+#include <string>
+
 #include <map>
 #include <memory>
 #include <vector>
 #include <functional>
 #include <utility>
-#include <string>
-
-#include <xgboost/logging.h>
-#include <xgboost/parameter.h>
 
 namespace xgboost {
 
diff --git a/include/xgboost/json_io.h b/include/xgboost/json_io.h
index 261d71097..a58afe172 100644
--- a/include/xgboost/json_io.h
+++ b/include/xgboost/json_io.h
@@ -3,6 +3,7 @@
  */
 #ifndef XGBOOST_JSON_IO_H_
 #define XGBOOST_JSON_IO_H_
+#include <xgboost/json.h>
 
 #include <memory>
 #include <string>
@@ -13,8 +14,6 @@
 #include <sstream>
 #include <locale>
 
-#include <xgboost/json.h>
-
 namespace xgboost {
 
 template <typename Allocator>
diff --git a/include/xgboost/learner.h b/include/xgboost/learner.h
index cbe064aca..a608bc1b8 100644
--- a/include/xgboost/learner.h
+++ b/include/xgboost/learner.h
@@ -8,12 +8,6 @@
 #ifndef XGBOOST_LEARNER_H_
 #define XGBOOST_LEARNER_H_
 
-#include <utility>
-#include <map>
-#include <memory>
-#include <string>
-#include <vector>
-
 #include <dmlc/any.h>
 #include <rabit/rabit.h>
 #include <xgboost/base.h>
@@ -23,6 +17,12 @@
 #include <xgboost/host_device_vector.h>
 #include <xgboost/model.h>
 
+#include <utility>
+#include <map>
+#include <memory>
+#include <string>
+#include <vector>
+
 namespace xgboost {
 
 class Metric;
diff --git a/include/xgboost/linear_updater.h b/include/xgboost/linear_updater.h
index d46abdcb2..39a0c324a 100644
--- a/include/xgboost/linear_updater.h
+++ b/include/xgboost/linear_updater.h
@@ -3,11 +3,6 @@
  */
 #pragma once
 
-#include <functional>
-#include <string>
-#include <utility>
-#include <vector>
-
 #include <dmlc/registry.h>
 #include <xgboost/base.h>
 #include <xgboost/data.h>
@@ -15,6 +10,12 @@
 #include <xgboost/host_device_vector.h>
 #include <xgboost/model.h>
 
+#include <functional>
+#include <string>
+#include <utility>
+#include <vector>
+
+
 namespace xgboost {
 
 class Json;
diff --git a/include/xgboost/logging.h b/include/xgboost/logging.h
index ff8570db4..07a2e67cf 100644
--- a/include/xgboost/logging.h
+++ b/include/xgboost/logging.h
@@ -8,18 +8,18 @@
 #ifndef XGBOOST_LOGGING_H_
 #define XGBOOST_LOGGING_H_
 
-#include <sstream>
-#include <map>
-#include <string>
-#include <utility>
-#include <vector>
-
 #include <dmlc/logging.h>
 #include <dmlc/thread_local.h>
 
 #include <xgboost/base.h>
 #include <xgboost/parameter.h>
 
+#include <sstream>
+#include <map>
+#include <string>
+#include <utility>
+#include <vector>
+
 namespace xgboost {
 
 class BaseLogger {
diff --git a/include/xgboost/metric.h b/include/xgboost/metric.h
index b7cc77a8c..8e9d2afe6 100644
--- a/include/xgboost/metric.h
+++ b/include/xgboost/metric.h
@@ -7,11 +7,6 @@
 #ifndef XGBOOST_METRIC_H_
 #define XGBOOST_METRIC_H_
 
-#include <vector>
-#include <string>
-#include <functional>
-#include <utility>
-
 #include <dmlc/registry.h>
 #include <xgboost/model.h>
 #include <xgboost/generic_parameters.h>
@@ -19,6 +14,11 @@
 #include <xgboost/base.h>
 #include <xgboost/host_device_vector.h>
 
+#include <vector>
+#include <string>
+#include <functional>
+#include <utility>
+
 namespace xgboost {
 /*!
  * \brief interface of evaluation metric used to evaluate model performance.
diff --git a/include/xgboost/objective.h b/include/xgboost/objective.h
index 957ff9da4..1c0942ed0 100644
--- a/include/xgboost/objective.h
+++ b/include/xgboost/objective.h
@@ -7,11 +7,6 @@
 #ifndef XGBOOST_OBJECTIVE_H_
 #define XGBOOST_OBJECTIVE_H_
 
-#include <vector>
-#include <utility>
-#include <string>
-#include <functional>
-
 #include <dmlc/registry.h>
 #include <xgboost/base.h>
 #include <xgboost/data.h>
@@ -19,6 +14,11 @@
 #include <xgboost/generic_parameters.h>
 #include <xgboost/host_device_vector.h>
 
+#include <vector>
+#include <utility>
+#include <string>
+#include <functional>
+
 namespace xgboost {
 
 /*! \brief interface of objective function */
diff --git a/include/xgboost/parameter.h b/include/xgboost/parameter.h
index 465c101e4..3186651ed 100644
--- a/include/xgboost/parameter.h
+++ b/include/xgboost/parameter.h
@@ -8,11 +8,10 @@
 #ifndef XGBOOST_PARAMETER_H_
 #define XGBOOST_PARAMETER_H_
 
-#include <string>
-#include <type_traits>
-
 #include <dmlc/parameter.h>
 #include <xgboost/base.h>
+#include <string>
+#include <type_traits>
 
 /*!
  * \brief Specialization of FieldEntry for enum class (backed by int)
diff --git a/include/xgboost/predictor.h b/include/xgboost/predictor.h
index b1790e777..0e3f7568c 100644
--- a/include/xgboost/predictor.h
+++ b/include/xgboost/predictor.h
@@ -5,6 +5,10 @@
  *  performs predictions for a gradient booster.
  */
 #pragma once
+#include <xgboost/base.h>
+#include <xgboost/data.h>
+#include <xgboost/generic_parameters.h>
+#include <xgboost/host_device_vector.h>
 
 #include <functional>
 #include <memory>
@@ -14,11 +18,6 @@
 #include <vector>
 #include <mutex>
 
-#include <xgboost/base.h>
-#include <xgboost/data.h>
-#include <xgboost/generic_parameters.h>
-#include <xgboost/host_device_vector.h>
-
 // Forward declarations
 namespace xgboost {
 class TreeUpdater;
diff --git a/include/xgboost/span.h b/include/xgboost/span.h
index 04b737666..29b0b9a79 100644
--- a/include/xgboost/span.h
+++ b/include/xgboost/span.h
@@ -29,15 +29,15 @@
 #ifndef XGBOOST_SPAN_H_
 #define XGBOOST_SPAN_H_
 
+#include <xgboost/base.h>
+#include <xgboost/logging.h>
+
 #include <cinttypes>          // size_t
 #include <limits>             // numeric_limits
 #include <iterator>
 #include <type_traits>
 #include <cstdio>
 
-#include <xgboost/base.h>
-#include <xgboost/logging.h>
-
 /*!
  * The version number 1910 is picked up from GSL.
  *
diff --git a/include/xgboost/tree_model.h b/include/xgboost/tree_model.h
index aa728a918..e7f6dc8ec 100644
--- a/include/xgboost/tree_model.h
+++ b/include/xgboost/tree_model.h
@@ -7,14 +7,6 @@
 #ifndef XGBOOST_TREE_MODEL_H_
 #define XGBOOST_TREE_MODEL_H_
 
-#include <limits>
-#include <vector>
-#include <string>
-#include <cstring>
-#include <algorithm>
-#include <tuple>
-#include <stack>
-
 #include <dmlc/io.h>
 #include <dmlc/parameter.h>
 
@@ -24,6 +16,14 @@
 #include <xgboost/feature_map.h>
 #include <xgboost/model.h>
 
+#include <limits>
+#include <vector>
+#include <string>
+#include <cstring>
+#include <algorithm>
+#include <tuple>
+#include <stack>
+
 namespace xgboost {
 
 struct PathElement;  // forward declaration
diff --git a/include/xgboost/tree_updater.h b/include/xgboost/tree_updater.h
index 805d6161c..a091c81b0 100644
--- a/include/xgboost/tree_updater.h
+++ b/include/xgboost/tree_updater.h
@@ -8,11 +8,6 @@
 #ifndef XGBOOST_TREE_UPDATER_H_
 #define XGBOOST_TREE_UPDATER_H_
 
-#include <functional>
-#include <vector>
-#include <utility>
-#include <string>
-
 #include <dmlc/registry.h>
 #include <xgboost/base.h>
 #include <xgboost/data.h>
@@ -21,6 +16,11 @@
 #include <xgboost/host_device_vector.h>
 #include <xgboost/model.h>
 
+#include <functional>
+#include <vector>
+#include <utility>
+#include <string>
+
 namespace xgboost {
 
 class Json;
diff --git a/src/c_api/c_api.cc b/src/c_api/c_api.cc
index cc50c956f..360baf8af 100644
--- a/src/c_api/c_api.cc
+++ b/src/c_api/c_api.cc
@@ -1,4 +1,7 @@
 // Copyright (c) 2014-2020 by Contributors
+#include <rabit/rabit.h>
+#include <rabit/c_api.h>
+
 #include <cstdio>
 #include <cstring>
 #include <fstream>
@@ -7,9 +10,6 @@
 #include <string>
 #include <memory>
 
-#include <rabit/rabit.h>
-#include <rabit/c_api.h>
-
 #include "xgboost/base.h"
 #include "xgboost/data.h"
 #include "xgboost/host_device_vector.h"
diff --git a/src/cli_main.cc b/src/cli_main.cc
index dc4592099..5c602f374 100644
--- a/src/cli_main.cc
+++ b/src/cli_main.cc
@@ -7,14 +7,6 @@
 #define _CRT_SECURE_NO_WARNINGS
 #define _CRT_SECURE_NO_DEPRECATE
 #define NOMINMAX
-
-#include <iomanip>
-#include <ctime>
-#include <string>
-#include <cstdio>
-#include <cstring>
-#include <vector>
-
 #include <dmlc/timer.h>
 
 #include <xgboost/learner.h>
@@ -23,6 +15,12 @@
 #include <xgboost/logging.h>
 #include <xgboost/parameter.h>
 
+#include <iomanip>
+#include <ctime>
+#include <string>
+#include <cstdio>
+#include <cstring>
+#include <vector>
 #include "common/common.h"
 #include "common/config.h"
 #include "common/io.h"
diff --git a/src/common/base64.h b/src/common/base64.h
index ed8f7d759..4c876b5f8 100644
--- a/src/common/base64.h
+++ b/src/common/base64.h
@@ -8,11 +8,10 @@
 #ifndef XGBOOST_COMMON_BASE64_H_
 #define XGBOOST_COMMON_BASE64_H_
 
+#include <xgboost/logging.h>
 #include <cctype>
 #include <cstdio>
 #include <string>
-
-#include "xgboost/logging.h"
 #include "./io.h"
 
 namespace xgboost {
diff --git a/src/common/common.h b/src/common/common.h
index f4c82e13e..f9300f3d5 100644
--- a/src/common/common.h
+++ b/src/common/common.h
@@ -6,6 +6,9 @@
 #ifndef XGBOOST_COMMON_COMMON_H_
 #define XGBOOST_COMMON_COMMON_H_
 
+#include <xgboost/base.h>
+#include <xgboost/logging.h>
+
 #include <exception>
 #include <limits>
 #include <type_traits>
@@ -13,9 +16,6 @@
 #include <string>
 #include <sstream>
 
-#include <xgboost/base.h>
-#include <xgboost/logging.h>
-
 #if defined(__CUDACC__)
 #include <thrust/system/cuda/error.h>
 #include <thrust/system_error.h>
diff --git a/src/common/compressed_iterator.h b/src/common/compressed_iterator.h
index a042d1ad4..9f60722fb 100644
--- a/src/common/compressed_iterator.h
+++ b/src/common/compressed_iterator.h
@@ -3,11 +3,11 @@
  * \file compressed_iterator.h
  */
 #pragma once
+#include <xgboost/base.h>
 #include <cmath>
 #include <cstddef>
 #include <algorithm>
 
-#include "xgboost/base.h"
 #include "common.h"
 
 #ifdef __CUDACC__
diff --git a/src/common/hist_util.cc b/src/common/hist_util.cc
index a5b4db155..e3ca953d2 100644
--- a/src/common/hist_util.cc
+++ b/src/common/hist_util.cc
@@ -2,21 +2,19 @@
  * Copyright 2017-2020 by Contributors
  * \file hist_util.cc
  */
-#include <numeric>
-#include <vector>
-
 #include <dmlc/timer.h>
 #include <dmlc/omp.h>
 
 #include <rabit/rabit.h>
+#include <numeric>
+#include <vector>
 
 #include "xgboost/base.h"
+#include "../common/common.h"
 #include "hist_util.h"
 #include "random.h"
 #include "column_matrix.h"
 #include "quantile.h"
-
-#include "../common/common.h"
 #include "./../tree/updater_quantile_hist.h"
 
 #if defined(XGBOOST_MM_PREFETCH_PRESENT)
diff --git a/src/common/hist_util.cu b/src/common/hist_util.cu
index cde46a2b4..716d49f9a 100644
--- a/src/common/hist_util.cu
+++ b/src/common/hist_util.cu
@@ -1,10 +1,6 @@
 /*!
  * Copyright 2018 XGBoost contributors
  */
-#include <memory>
-#include <mutex>
-#include <utility>
-#include <vector>
 
 #include <xgboost/logging.h>
 
@@ -17,6 +13,11 @@
 #include <thrust/binary_search.h>
 #include <thrust/execution_policy.h>
 
+#include <memory>
+#include <mutex>
+#include <utility>
+#include <vector>
+
 #include "device_helpers.cuh"
 #include "hist_util.h"
 #include "hist_util.cuh"
diff --git a/src/common/hist_util.h b/src/common/hist_util.h
index aef3f7921..d60960301 100644
--- a/src/common/hist_util.h
+++ b/src/common/hist_util.h
@@ -7,6 +7,8 @@
 #ifndef XGBOOST_COMMON_HIST_UTIL_H_
 #define XGBOOST_COMMON_HIST_UTIL_H_
 
+#include <xgboost/data.h>
+#include <xgboost/generic_parameters.h>
 #include <limits>
 #include <vector>
 #include <algorithm>
@@ -14,9 +16,6 @@
 #include <utility>
 #include <map>
 
-#include "xgboost/data.h"
-#include "xgboost/generic_parameters.h"
-
 #include "row_set.h"
 #include "threading_utils.h"
 #include "../tree/param.h"
diff --git a/src/common/host_device_vector.cc b/src/common/host_device_vector.cc
index e5dc746c0..1dd9997fa 100644
--- a/src/common/host_device_vector.cc
+++ b/src/common/host_device_vector.cc
@@ -4,12 +4,12 @@
 #ifndef XGBOOST_USE_CUDA
 
 // dummy implementation of HostDeviceVector in case CUDA is not used
+
+#include <xgboost/base.h>
+#include <xgboost/data.h>
 #include <cstdint>
 #include <memory>
 #include <utility>
-
-#include "xgboost/base.h"
-#include "xgboost/data.h"
 #include "xgboost/host_device_vector.h"
 
 namespace xgboost {
diff --git a/src/common/host_device_vector.cu b/src/common/host_device_vector.cu
index 87e58cc5b..3ffcf757e 100644
--- a/src/common/host_device_vector.cu
+++ b/src/common/host_device_vector.cu
@@ -1,12 +1,14 @@
 /*!
  * Copyright 2017 XGBoost contributors
  */
-#include <algorithm>
-#include <cstdint>
 
 #include <thrust/fill.h>
 #include <thrust/device_ptr.h>
 
+#include <algorithm>
+#include <cstdint>
+#include <mutex>
+
 #include "xgboost/data.h"
 #include "xgboost/host_device_vector.h"
 #include "device_helpers.cuh"
diff --git a/src/common/io.h b/src/common/io.h
index 6b9cac43f..528296dc7 100644
--- a/src/common/io.h
+++ b/src/common/io.h
@@ -7,11 +7,11 @@
 
 #ifndef XGBOOST_COMMON_IO_H_
 #define XGBOOST_COMMON_IO_H_
-#include <string>
-#include <cstring>
 
 #include <dmlc/io.h>
 #include <rabit/rabit.h>
+#include <string>
+#include <cstring>
 
 #include "common.h"
 
diff --git a/src/common/math.h b/src/common/math.h
index 4e5add27e..41905310e 100644
--- a/src/common/math.h
+++ b/src/common/math.h
@@ -7,14 +7,14 @@
 #ifndef XGBOOST_COMMON_MATH_H_
 #define XGBOOST_COMMON_MATH_H_
 
+#include <xgboost/base.h>
+
 #include <algorithm>
 #include <cmath>
 #include <limits>
 #include <utility>
 #include <vector>
 
-#include <xgboost/base.h>
-
 namespace xgboost {
 namespace common {
 /*!
diff --git a/src/common/probability_distribution.cc b/src/common/probability_distribution.cc
index 593e87fd9..51bcc495b 100644
--- a/src/common/probability_distribution.cc
+++ b/src/common/probability_distribution.cc
@@ -4,9 +4,9 @@
  * \brief Implementation of a few useful probability distributions
  * \author Avinash Barnwal and Hyunsu Cho
  */
-#include <cmath>
 
-#include "xgboost/logging.h"
+#include <xgboost/logging.h>
+#include <cmath>
 #include "probability_distribution.h"
 
 namespace xgboost {
diff --git a/src/common/quantile.h b/src/common/quantile.h
index 569f06aaf..067b041bc 100644
--- a/src/common/quantile.h
+++ b/src/common/quantile.h
@@ -7,15 +7,14 @@
 #ifndef XGBOOST_COMMON_QUANTILE_H_
 #define XGBOOST_COMMON_QUANTILE_H_
 
+#include <dmlc/base.h>
+#include <xgboost/logging.h>
 #include <cmath>
 #include <vector>
 #include <cstring>
 #include <algorithm>
 #include <iostream>
 
-#include <dmlc/base.h>
-#include <xgboost/logging.h>
-
 namespace xgboost {
 namespace common {
 /*!
diff --git a/src/common/random.h b/src/common/random.h
index 4ee61387d..45af80ce0 100644
--- a/src/common/random.h
+++ b/src/common/random.h
@@ -7,6 +7,8 @@
 #ifndef XGBOOST_COMMON_RANDOM_H_
 #define XGBOOST_COMMON_RANDOM_H_
 
+#include <rabit/rabit.h>
+#include <xgboost/logging.h>
 #include <algorithm>
 #include <vector>
 #include <limits>
@@ -15,8 +17,6 @@
 #include <numeric>
 #include <random>
 
-#include "rabit/rabit.h"
-#include "xgboost/logging.h"
 #include "xgboost/host_device_vector.h"
 
 namespace xgboost {
diff --git a/src/common/row_set.h b/src/common/row_set.h
index 2ac1cf467..25f7c739d 100644
--- a/src/common/row_set.h
+++ b/src/common/row_set.h
@@ -7,12 +7,11 @@
 #ifndef XGBOOST_COMMON_ROW_SET_H_
 #define XGBOOST_COMMON_ROW_SET_H_
 
+#include <xgboost/data.h>
 #include <algorithm>
 #include <vector>
 #include <utility>
 
-#include "xgboost/data.h"
-
 namespace xgboost {
 namespace common {
 
diff --git a/src/common/survival_util.cc b/src/common/survival_util.cc
index 4d78a49ca..5630da355 100644
--- a/src/common/survival_util.cc
+++ b/src/common/survival_util.cc
@@ -6,10 +6,9 @@
  * \author Avinash Barnwal, Hyunsu Cho and Toby Hocking
  */
 
+#include <dmlc/registry.h>
 #include <algorithm>
 #include <cmath>
-
-#include <dmlc/registry.h>
 #include "survival_util.h"
 
 /*
diff --git a/src/common/survival_util.h b/src/common/survival_util.h
index fac8b4728..50c6aab51 100644
--- a/src/common/survival_util.h
+++ b/src/common/survival_util.h
@@ -8,9 +8,8 @@
 #ifndef XGBOOST_COMMON_SURVIVAL_UTIL_H_
 #define XGBOOST_COMMON_SURVIVAL_UTIL_H_
 
+#include <xgboost/parameter.h>
 #include <memory>
-
-#include "xgboost/parameter.h"
 #include "probability_distribution.h"
 
 DECLARE_FIELD_ENUM_CLASS(xgboost::common::ProbabilityDistributionType);
diff --git a/src/common/timer.cc b/src/common/timer.cc
index 1cbf869bc..49d08a35c 100644
--- a/src/common/timer.cc
+++ b/src/common/timer.cc
@@ -1,15 +1,14 @@
 /*!
  * Copyright by Contributors 2019
  */
+#include <rabit/rabit.h>
 #include <algorithm>
 #include <type_traits>
 #include <utility>
 #include <vector>
 #include <sstream>
-
-#include <rabit/rabit.h>
-#include "xgboost/json.h"
 #include "timer.h"
+#include "xgboost/json.h"
 
 #if defined(XGBOOST_USE_NVTX)
 #include <nvToolsExt.h>
diff --git a/src/common/timer.h b/src/common/timer.h
index 3fd54e186..b2dc2a428 100644
--- a/src/common/timer.h
+++ b/src/common/timer.h
@@ -2,6 +2,7 @@
  * Copyright by Contributors 2017-2019
  */
 #pragma once
+#include <xgboost/logging.h>
 #include <chrono>
 #include <iostream>
 #include <map>
@@ -9,8 +10,6 @@
 #include <utility>
 #include <vector>
 
-#include <xgboost/logging.h>
-
 namespace xgboost {
 namespace common {
 
diff --git a/src/common/transform.h b/src/common/transform.h
index 3d5970dec..fa2d0d379 100644
--- a/src/common/transform.h
+++ b/src/common/transform.h
@@ -3,14 +3,15 @@
  */
 #ifndef XGBOOST_COMMON_TRANSFORM_H_
 #define XGBOOST_COMMON_TRANSFORM_H_
-#include <utility>
-#include <vector>
-#include <type_traits>  // enable_if
 
 #include <dmlc/omp.h>
 #include <dmlc/common.h>
 
-#include "xgboost/data.h"
+#include <xgboost/data.h>
+#include <utility>
+#include <vector>
+#include <type_traits>  // enable_if
+
 #include "xgboost/host_device_vector.h"
 #include "xgboost/span.h"
 
diff --git a/src/common/version.cc b/src/common/version.cc
index c0cdffb99..3fb2e5c24 100644
--- a/src/common/version.cc
+++ b/src/common/version.cc
@@ -1,12 +1,12 @@
 /*!
  * Copyright 2019 XGBoost contributors
  */
+#include <dmlc/io.h>
+
 #include <string>
 #include <tuple>
 #include <vector>
 
-#include <dmlc/io.h>
-
 #include "xgboost/logging.h"
 #include "xgboost/json.h"
 #include "xgboost/version_config.h"
diff --git a/src/common/version.h b/src/common/version.h
index 3c3b97f16..96885f6a8 100644
--- a/src/common/version.h
+++ b/src/common/version.h
@@ -4,10 +4,10 @@
 #ifndef XGBOOST_COMMON_VERSION_H_
 #define XGBOOST_COMMON_VERSION_H_
 
+#include <dmlc/io.h>
 #include <string>
 #include <tuple>
 
-#include <dmlc/io.h>
 #include "xgboost/base.h"
 
 namespace xgboost {
diff --git a/src/data/adapter.h b/src/data/adapter.h
index cd26d0bf4..e252e1da7 100644
--- a/src/data/adapter.h
+++ b/src/data/adapter.h
@@ -4,6 +4,8 @@
  */
 #ifndef XGBOOST_DATA_ADAPTER_H_
 #define XGBOOST_DATA_ADAPTER_H_
+#include <dmlc/data.h>
+
 #include <cstddef>
 #include <functional>
 #include <limits>
@@ -12,8 +14,6 @@
 #include <utility>
 #include <vector>
 
-#include <dmlc/data.h>
-
 #include "xgboost/logging.h"
 #include "xgboost/base.h"
 #include "xgboost/data.h"
diff --git a/src/data/data.cc b/src/data/data.cc
index ff907d031..2a620b7d4 100644
--- a/src/data/data.cc
+++ b/src/data/data.cc
@@ -2,11 +2,8 @@
  * Copyright 2015-2020 by Contributors
  * \file data.cc
  */
-#include <cstring>
-#include <vector>
-#include <string>
-
 #include <dmlc/registry.h>
+#include <cstring>
 
 #include "dmlc/io.h"
 #include "xgboost/data.h"
diff --git a/src/data/device_dmatrix.cu b/src/data/device_dmatrix.cu
index 88d6a33a0..d11d01b16 100644
--- a/src/data/device_dmatrix.cu
+++ b/src/data/device_dmatrix.cu
@@ -3,16 +3,14 @@
  * \file device_dmatrix.cu
  * \brief Device-memory version of DMatrix.
  */
-#include <memory>
-#include <utility>
 
 #include <thrust/execution_policy.h>
 #include <thrust/iterator/discard_iterator.h>
 #include <thrust/iterator/transform_output_iterator.h>
-
 #include <xgboost/base.h>
 #include <xgboost/data.h>
-
+#include <memory>
+#include <utility>
 #include "../common/hist_util.h"
 #include "adapter.h"
 #include "device_adapter.cuh"
diff --git a/src/data/device_dmatrix.h b/src/data/device_dmatrix.h
index ad7e8aa3b..781461baa 100644
--- a/src/data/device_dmatrix.h
+++ b/src/data/device_dmatrix.h
@@ -6,11 +6,11 @@
 #ifndef XGBOOST_DATA_DEVICE_DMATRIX_H_
 #define XGBOOST_DATA_DEVICE_DMATRIX_H_
 
-#include <memory>
-
 #include <xgboost/base.h>
 #include <xgboost/data.h>
 
+#include <memory>
+
 #include "adapter.h"
 #include "simple_batch_iterator.h"
 #include "simple_dmatrix.h"
diff --git a/src/data/ellpack_page_source.h b/src/data/ellpack_page_source.h
index c5f787f7f..a1ce587c2 100644
--- a/src/data/ellpack_page_source.h
+++ b/src/data/ellpack_page_source.h
@@ -5,11 +5,10 @@
 #ifndef XGBOOST_DATA_ELLPACK_PAGE_SOURCE_H_
 #define XGBOOST_DATA_ELLPACK_PAGE_SOURCE_H_
 
+#include <xgboost/data.h>
 #include <memory>
 #include <string>
 
-#include <xgboost/data.h>
-
 #include "../common/timer.h"
 #include "../common/hist_util.h"
 #include "sparse_page_source.h"
diff --git a/src/data/simple_dmatrix.h b/src/data/simple_dmatrix.h
index 7958785c6..9d2130b41 100644
--- a/src/data/simple_dmatrix.h
+++ b/src/data/simple_dmatrix.h
@@ -7,11 +7,12 @@
 #ifndef XGBOOST_DATA_SIMPLE_DMATRIX_H_
 #define XGBOOST_DATA_SIMPLE_DMATRIX_H_
 
+#include <xgboost/base.h>
+#include <xgboost/data.h>
+
 #include <memory>
 #include <string>
 
-#include <xgboost/base.h>
-#include <xgboost/data.h>
 
 namespace xgboost {
 namespace data {
diff --git a/src/data/sparse_page_dmatrix.h b/src/data/sparse_page_dmatrix.h
index 79c95c581..393172658 100644
--- a/src/data/sparse_page_dmatrix.h
+++ b/src/data/sparse_page_dmatrix.h
@@ -7,14 +7,13 @@
 #ifndef XGBOOST_DATA_SPARSE_PAGE_DMATRIX_H_
 #define XGBOOST_DATA_SPARSE_PAGE_DMATRIX_H_
 
+#include <xgboost/data.h>
 #include <algorithm>
 #include <memory>
 #include <string>
 #include <utility>
 #include <vector>
 
-#include "xgboost/data.h"
-
 #include "ellpack_page_source.h"
 #include "sparse_page_source.h"
 
diff --git a/src/data/sparse_page_source.h b/src/data/sparse_page_source.h
index 214795fad..108af403b 100644
--- a/src/data/sparse_page_source.h
+++ b/src/data/sparse_page_source.h
@@ -7,6 +7,9 @@
 #ifndef XGBOOST_DATA_SPARSE_PAGE_SOURCE_H_
 #define XGBOOST_DATA_SPARSE_PAGE_SOURCE_H_
 
+#include <dmlc/threadediter.h>
+#include <dmlc/timer.h>
+
 #include <algorithm>
 #include <limits>
 #include <locale>
@@ -16,9 +19,6 @@
 #include <vector>
 #include <fstream>
 
-#include <dmlc/threadediter.h>
-#include <dmlc/timer.h>
-
 #include "xgboost/base.h"
 #include "xgboost/data.h"
 
diff --git a/src/data/sparse_page_writer.h b/src/data/sparse_page_writer.h
index f303f8427..f63fcf0f8 100644
--- a/src/data/sparse_page_writer.h
+++ b/src/data/sparse_page_writer.h
@@ -6,6 +6,8 @@
 #ifndef XGBOOST_DATA_SPARSE_PAGE_WRITER_H_
 #define XGBOOST_DATA_SPARSE_PAGE_WRITER_H_
 
+#include <xgboost/data.h>
+#include <dmlc/io.h>
 #include <vector>
 #include <algorithm>
 #include <cstring>
@@ -14,12 +16,9 @@
 #include <memory>
 #include <functional>
 
-#include <xgboost/data.h>
-#include <dmlc/io.h>
-
 #if DMLC_ENABLE_STD_THREAD
 #include <dmlc/concurrency.h>
-#include <thread>  // NOLINT
+#include <thread>
 #endif  // DMLC_ENABLE_STD_THREAD
 
 namespace xgboost {
diff --git a/src/gbm/gblinear.cc b/src/gbm/gblinear.cc
index 4742f4971..e554f8a55 100644
--- a/src/gbm/gblinear.cc
+++ b/src/gbm/gblinear.cc
@@ -5,14 +5,14 @@
  *        the update rule is parallel coordinate descent (shotgun)
  * \author Tianqi Chen
  */
+#include <dmlc/omp.h>
+#include <dmlc/parameter.h>
+
 #include <vector>
 #include <string>
 #include <sstream>
 #include <algorithm>
 
-#include <dmlc/omp.h>
-#include <dmlc/parameter.h>
-
 #include "xgboost/gbm.h"
 #include "xgboost/json.h"
 #include "xgboost/predictor.h"
diff --git a/src/gbm/gblinear_model.h b/src/gbm/gblinear_model.h
index 4aff2f717..f2d0d9a86 100644
--- a/src/gbm/gblinear_model.h
+++ b/src/gbm/gblinear_model.h
@@ -2,14 +2,14 @@
  * Copyright 2018-2019 by Contributors
  */
 #pragma once
+#include <dmlc/io.h>
+#include <dmlc/parameter.h>
+#include <xgboost/learner.h>
+
 #include <vector>
 #include <string>
 #include <cstring>
 
-#include <dmlc/io.h>
-#include <dmlc/parameter.h>
-
-#include <xgboost/learner.h>
 #include "xgboost/base.h"
 #include "xgboost/feature_map.h"
 #include "xgboost/model.h"
diff --git a/src/gbm/gbm.cc b/src/gbm/gbm.cc
index a8218e6a4..87a6ded29 100644
--- a/src/gbm/gbm.cc
+++ b/src/gbm/gbm.cc
@@ -3,12 +3,11 @@
  * \file gbm.cc
  * \brief Registry of gradient boosters.
  */
+#include <dmlc/registry.h>
 #include <string>
 #include <vector>
 #include <memory>
 
-#include <dmlc/registry.h>
-
 #include "xgboost/gbm.h"
 #include "xgboost/learner.h"
 #include "xgboost/generic_parameters.h"
diff --git a/src/gbm/gbtree.cc b/src/gbm/gbtree.cc
index bf5156306..61a3021cb 100644
--- a/src/gbm/gbtree.cc
+++ b/src/gbm/gbtree.cc
@@ -4,6 +4,9 @@
  * \brief gradient boosted tree implementation.
  * \author Tianqi Chen
  */
+#include <dmlc/omp.h>
+#include <dmlc/parameter.h>
+
 #include <vector>
 #include <memory>
 #include <utility>
@@ -11,9 +14,6 @@
 #include <limits>
 #include <algorithm>
 
-#include <dmlc/omp.h>
-#include <dmlc/parameter.h>
-
 #include "xgboost/data.h"
 #include "xgboost/gbm.h"
 #include "xgboost/logging.h"
diff --git a/src/gbm/gbtree.h b/src/gbm/gbtree.h
index ad9c63d14..534c3ad54 100644
--- a/src/gbm/gbtree.h
+++ b/src/gbm/gbtree.h
@@ -7,6 +7,8 @@
 #ifndef XGBOOST_GBM_GBTREE_H_
 #define XGBOOST_GBM_GBTREE_H_
 
+#include <dmlc/omp.h>
+
 #include <vector>
 #include <map>
 #include <memory>
@@ -14,7 +16,6 @@
 #include <string>
 #include <unordered_map>
 
-#include <dmlc/omp.h>
 #include "xgboost/base.h"
 #include "xgboost/data.h"
 #include "xgboost/logging.h"
diff --git a/src/gbm/gbtree_model.h b/src/gbm/gbtree_model.h
index 374f31ef9..7ac7d8f47 100644
--- a/src/gbm/gbtree_model.h
+++ b/src/gbm/gbtree_model.h
@@ -4,10 +4,6 @@
  */
 #ifndef XGBOOST_GBM_GBTREE_MODEL_H_
 #define XGBOOST_GBM_GBTREE_MODEL_H_
-#include <memory>
-#include <utility>
-#include <string>
-#include <vector>
 
 #include <dmlc/parameter.h>
 #include <dmlc/io.h>
@@ -16,6 +12,11 @@
 #include <xgboost/parameter.h>
 #include <xgboost/learner.h>
 
+#include <memory>
+#include <utility>
+#include <string>
+#include <vector>
+
 namespace xgboost {
 
 class Json;
diff --git a/src/learner.cc b/src/learner.cc
index e6c3fee80..34649480c 100644
--- a/src/learner.cc
+++ b/src/learner.cc
@@ -4,6 +4,10 @@
  * \brief Implementation of learning algorithm.
  * \author Tianqi Chen
  */
+#include <dmlc/io.h>
+#include <dmlc/parameter.h>
+#include <dmlc/thread_local.h>
+
 #include <atomic>
 #include <mutex>
 #include <algorithm>
@@ -16,11 +20,7 @@
 #include <utility>
 #include <vector>
 
-#include <dmlc/io.h>
-#include <dmlc/parameter.h>
-#include <dmlc/thread_local.h>
-#include <dmlc/any.h>
-
+#include "dmlc/any.h"
 #include "xgboost/base.h"
 #include "xgboost/data.h"
 #include "xgboost/model.h"
diff --git a/src/logging.cc b/src/logging.cc
index c8aa1d320..f03e1be02 100644
--- a/src/logging.cc
+++ b/src/logging.cc
@@ -4,11 +4,11 @@
  * \brief Implementation of loggers.
  * \author Tianqi Chen
  */
+#include <rabit/rabit.h>
+
 #include <iostream>
 #include <map>
 
-#include <rabit/rabit.h>
-
 #include "xgboost/parameter.h"
 #include "xgboost/logging.h"
 
diff --git a/src/metric/elementwise_metric.cu b/src/metric/elementwise_metric.cu
index 82e8be3a9..85dead9c3 100644
--- a/src/metric/elementwise_metric.cu
+++ b/src/metric/elementwise_metric.cu
@@ -6,11 +6,10 @@
  *
  *  The expressions like wsum == 0 ? esum : esum / wsum is used to handle empty dataset.
  */
-#include <cmath>
-
 #include <rabit/rabit.h>
 #include <xgboost/metric.h>
 #include <dmlc/registry.h>
+#include <cmath>
 
 #include "metric_common.h"
 #include "../common/math.h"
diff --git a/src/metric/multiclass_metric.cu b/src/metric/multiclass_metric.cu
index b019f2e47..377a05010 100644
--- a/src/metric/multiclass_metric.cu
+++ b/src/metric/multiclass_metric.cu
@@ -4,10 +4,9 @@
  * \brief evaluation metrics for multiclass classification.
  * \author Kailong Chen, Tianqi Chen
  */
-#include <cmath>
-
 #include <rabit/rabit.h>
-#include "xgboost/metric.h"
+#include <xgboost/metric.h>
+#include <cmath>
 
 #include "metric_common.h"
 #include "../common/math.h"
diff --git a/src/metric/rank_metric.cc b/src/metric/rank_metric.cc
index 61df43355..b55c764d2 100644
--- a/src/metric/rank_metric.cc
+++ b/src/metric/rank_metric.cc
@@ -19,15 +19,15 @@
 //   this cannot be used when the translation unit is compiled using the 'nvcc' compiler (as the
 //   corresponding headers that brings in those function declaration can't be included with CUDA).
 //   This precludes the CPU and GPU logic to coexist inside a .cu file
-#include <cmath>
-#include <vector>
 
 #include <rabit/rabit.h>
+#include <xgboost/metric.h>
 #include <dmlc/registry.h>
+#include <cmath>
+
+#include <vector>
 
-#include "xgboost/metric.h"
 #include "xgboost/host_device_vector.h"
-
 #include "../common/math.h"
 #include "metric_common.h"
 
diff --git a/src/metric/rank_metric.cu b/src/metric/rank_metric.cu
index 9bcbc6882..290b1fb02 100644
--- a/src/metric/rank_metric.cu
+++ b/src/metric/rank_metric.cu
@@ -4,9 +4,6 @@
  * \brief prediction rank based metrics.
  * \author Kailong Chen, Tianqi Chen
  */
-#include <cmath>
-#include <vector>
-
 #include <rabit/rabit.h>
 #include <dmlc/registry.h>
 
@@ -14,6 +11,9 @@
 #include <xgboost/host_device_vector.h>
 #include <thrust/iterator/discard_iterator.h>
 
+#include <cmath>
+#include <vector>
+
 #include "metric_common.h"
 
 #include "../common/math.h"
diff --git a/src/metric/survival_metric.cc b/src/metric/survival_metric.cc
index 1ea0d6f02..cc0614fe7 100644
--- a/src/metric/survival_metric.cc
+++ b/src/metric/survival_metric.cc
@@ -4,16 +4,16 @@
  * \brief Metrics for survival analysis
  * \author Avinash Barnwal, Hyunsu Cho and Toby Hocking
  */
+
+#include <rabit/rabit.h>
+#include <xgboost/metric.h>
+#include <xgboost/host_device_vector.h>
+#include <dmlc/registry.h>
 #include <cmath>
 #include <memory>
 #include <vector>
 #include <limits>
 
-#include <dmlc/registry.h>
-#include <rabit/rabit.h>
-
-#include "xgboost/metric.h"
-#include "xgboost/host_device_vector.h"
 #include "xgboost/json.h"
 
 #include "../common/math.h"
diff --git a/src/objective/aft_obj.cc b/src/objective/aft_obj.cc
index 40882d257..bf66d9991 100644
--- a/src/objective/aft_obj.cc
+++ b/src/objective/aft_obj.cc
@@ -3,6 +3,10 @@
  * \file rank.cc
  * \brief Definition of aft loss.
  */
+
+#include <dmlc/omp.h>
+#include <xgboost/logging.h>
+#include <xgboost/objective.h>
 #include <vector>
 #include <limits>
 #include <algorithm>
@@ -10,10 +14,6 @@
 #include <utility>
 #include <cmath>
 
-#include <dmlc/omp.h>
-#include <xgboost/logging.h>
-#include <xgboost/objective.h>
-
 #include "xgboost/json.h"
 
 #include "../common/math.h"
diff --git a/src/objective/multiclass_obj.cu b/src/objective/multiclass_obj.cu
index c968e2685..29af5e0d2 100644
--- a/src/objective/multiclass_obj.cu
+++ b/src/objective/multiclass_obj.cu
@@ -4,13 +4,13 @@
  * \brief Definition of multi-class classification objectives.
  * \author Tianqi Chen
  */
+#include <dmlc/omp.h>
+
 #include <vector>
 #include <algorithm>
 #include <limits>
 #include <utility>
 
-#include <dmlc/omp.h>
-
 #include "xgboost/parameter.h"
 #include "xgboost/data.h"
 #include "xgboost/logging.h"
diff --git a/src/objective/objective.cc b/src/objective/objective.cc
index 8641f042f..8f65cec0d 100644
--- a/src/objective/objective.cc
+++ b/src/objective/objective.cc
@@ -3,11 +3,11 @@
  * \file objective.cc
  * \brief Registry of all objective functions.
  */
-#include <sstream>
-
+#include <xgboost/objective.h>
 #include <dmlc/registry.h>
 
-#include "xgboost/objective.h"
+#include <sstream>
+
 #include "xgboost/host_device_vector.h"
 
 namespace dmlc {
diff --git a/src/objective/rank_obj.cu b/src/objective/rank_obj.cu
index 44fd51193..e7a220dbc 100644
--- a/src/objective/rank_obj.cu
+++ b/src/objective/rank_obj.cu
@@ -1,15 +1,14 @@
 /*!
  * Copyright 2015-2019 XGBoost contributors
  */
+#include <dmlc/omp.h>
+#include <dmlc/timer.h>
+#include <xgboost/logging.h>
+#include <xgboost/objective.h>
 #include <vector>
 #include <algorithm>
 #include <utility>
 
-#include <dmlc/omp.h>
-#include <dmlc/timer.h>
-
-#include "xgboost/logging.h"
-#include "xgboost/objective.h"
 #include "xgboost/json.h"
 #include "xgboost/parameter.h"
 
diff --git a/src/objective/regression_loss.h b/src/objective/regression_loss.h
index 2f1cf829a..60ede53dd 100644
--- a/src/objective/regression_loss.h
+++ b/src/objective/regression_loss.h
@@ -3,10 +3,10 @@
  */
 #ifndef XGBOOST_OBJECTIVE_REGRESSION_LOSS_H_
 #define XGBOOST_OBJECTIVE_REGRESSION_LOSS_H_
-#include <algorithm>
 
 #include <dmlc/omp.h>
 #include <xgboost/logging.h>
+#include <algorithm>
 #include "../common/math.h"
 
 namespace xgboost {
diff --git a/src/objective/regression_obj.cu b/src/objective/regression_obj.cu
index 4bc3973d9..810ee9a6b 100644
--- a/src/objective/regression_obj.cu
+++ b/src/objective/regression_obj.cu
@@ -4,13 +4,13 @@
  * \brief Definition of single-value regression and classification objectives.
  * \author Tianqi Chen, Kailong Chen
  */
-#include <cmath>
-#include <memory>
-#include <vector>
 
 #include <dmlc/omp.h>
 #include <xgboost/logging.h>
 #include <xgboost/objective.h>
+#include <cmath>
+#include <memory>
+#include <vector>
 
 #include "xgboost/host_device_vector.h"
 #include "xgboost/json.h"
diff --git a/src/predictor/cpu_predictor.cc b/src/predictor/cpu_predictor.cc
index 2a96d82ad..1c84f4947 100644
--- a/src/predictor/cpu_predictor.cc
+++ b/src/predictor/cpu_predictor.cc
@@ -1,13 +1,13 @@
 /*!
  * Copyright by Contributors 2017-2020
  */
+#include <dmlc/omp.h>
+#include <dmlc/any.h>
+
 #include <cstddef>
 #include <limits>
 #include <mutex>
 
-#include <dmlc/omp.h>
-#include <dmlc/any.h>
-
 #include "xgboost/base.h"
 #include "xgboost/data.h"
 #include "xgboost/predictor.h"
diff --git a/src/predictor/gpu_predictor.cu b/src/predictor/gpu_predictor.cu
index 0e57276d6..9a498136d 100644
--- a/src/predictor/gpu_predictor.cu
+++ b/src/predictor/gpu_predictor.cu
@@ -1,12 +1,11 @@
 /*!
  * Copyright 2017-2020 by Contributors
  */
-#include <memory>
-
 #include <thrust/copy.h>
 #include <thrust/device_ptr.h>
 #include <thrust/device_vector.h>
 #include <thrust/fill.h>
+#include <memory>
 
 #include "xgboost/data.h"
 #include "xgboost/predictor.h"
diff --git a/src/predictor/predictor.cc b/src/predictor/predictor.cc
index 49e1e816d..eb68d382b 100644
--- a/src/predictor/predictor.cc
+++ b/src/predictor/predictor.cc
@@ -1,9 +1,9 @@
 /*!
  * Copyright 2017-2020 by Contributors
  */
+#include <dmlc/registry.h>
 #include <mutex>
 
-#include <dmlc/registry.h>
 #include "xgboost/predictor.h"
 #include "xgboost/data.h"
 #include "xgboost/generic_parameters.h"
diff --git a/src/tree/constraints.cu b/src/tree/constraints.cu
index 8c23064b1..b6db0eda0 100644
--- a/src/tree/constraints.cu
+++ b/src/tree/constraints.cu
@@ -1,15 +1,15 @@
 /*!
  * Copyright 2019 XGBoost contributors
  */
-#include <algorithm>
-#include <string>
-#include <set>
-
 #include <thrust/copy.h>
 #include <thrust/device_vector.h>
 #include <thrust/execution_policy.h>
 #include <thrust/iterator/counting_iterator.h>
 
+#include <algorithm>
+#include <string>
+#include <set>
+
 #include "xgboost/logging.h"
 #include "xgboost/span.h"
 #include "constraints.cuh"
diff --git a/src/tree/gpu_hist/gradient_based_sampler.cu b/src/tree/gpu_hist/gradient_based_sampler.cu
index 558e8a236..eb441f39e 100644
--- a/src/tree/gpu_hist/gradient_based_sampler.cu
+++ b/src/tree/gpu_hist/gradient_based_sampler.cu
@@ -1,16 +1,15 @@
 /*!
  * Copyright 2019 by XGBoost Contributors
  */
-#include <algorithm>
-#include <limits>
-
 #include <thrust/functional.h>
 #include <thrust/random.h>
 #include <thrust/transform.h>
-
 #include <xgboost/host_device_vector.h>
 #include <xgboost/logging.h>
 
+#include <algorithm>
+#include <limits>
+
 #include "../../common/compressed_iterator.h"
 #include "../../common/random.h"
 #include "gradient_based_sampler.cuh"
diff --git a/src/tree/gpu_hist/histogram.cu b/src/tree/gpu_hist/histogram.cu
index bf9094556..edc3046d1 100644
--- a/src/tree/gpu_hist/histogram.cu
+++ b/src/tree/gpu_hist/histogram.cu
@@ -1,13 +1,12 @@
 /*!
  * Copyright 2020 by XGBoost Contributors
  */
+#include <thrust/reduce.h>
+#include <thrust/iterator/transform_iterator.h>
 #include <algorithm>
 #include <ctgmath>
 #include <limits>
 
-#include <thrust/reduce.h>
-#include <thrust/iterator/transform_iterator.h>
-
 #include "xgboost/base.h"
 #include "row_partitioner.cuh"
 
diff --git a/src/tree/gpu_hist/row_partitioner.cu b/src/tree/gpu_hist/row_partitioner.cu
index 742d8e2f7..2b7fbe4af 100644
--- a/src/tree/gpu_hist/row_partitioner.cu
+++ b/src/tree/gpu_hist/row_partitioner.cu
@@ -1,12 +1,10 @@
 /*!
  * Copyright 2017-2019 XGBoost contributors
  */
-#include <vector>
-
 #include <thrust/iterator/discard_iterator.h>
 #include <thrust/iterator/transform_output_iterator.h>
 #include <thrust/sequence.h>
-
+#include <vector>
 #include "../../common/device_helpers.cuh"
 #include "row_partitioner.cuh"
 
diff --git a/src/tree/split_evaluator.cc b/src/tree/split_evaluator.cc
index bbacca279..be166156b 100644
--- a/src/tree/split_evaluator.cc
+++ b/src/tree/split_evaluator.cc
@@ -3,6 +3,9 @@
  * \file split_evaluator.cc
  * \brief Contains implementations of different split evaluators.
  */
+#include <dmlc/json.h>
+#include <dmlc/registry.h>
+
 #include <algorithm>
 #include <unordered_set>
 #include <vector>
@@ -12,9 +15,6 @@
 #include <sstream>
 #include <utility>
 
-#include <dmlc/json.h>
-#include <dmlc/registry.h>
-
 #include "xgboost/logging.h"
 #include "xgboost/parameter.h"
 
diff --git a/src/tree/split_evaluator.h b/src/tree/split_evaluator.h
index 77b31d916..2b3876491 100644
--- a/src/tree/split_evaluator.h
+++ b/src/tree/split_evaluator.h
@@ -7,15 +7,15 @@
 
 #ifndef XGBOOST_TREE_SPLIT_EVALUATOR_H_
 #define XGBOOST_TREE_SPLIT_EVALUATOR_H_
+
+#include <dmlc/registry.h>
+#include <xgboost/base.h>
 #include <functional>
 #include <memory>
 #include <string>
 #include <utility>
 #include <vector>
 
-#include <dmlc/registry.h>
-#include <xgboost/base.h>
-
 #include "param.h"
 
 #define ROOT_PARENT_ID (-1 & ((1U << 31) - 1))
diff --git a/src/tree/tree_model.cc b/src/tree/tree_model.cc
index 8818a07d9..8f45621ca 100644
--- a/src/tree/tree_model.cc
+++ b/src/tree/tree_model.cc
@@ -3,12 +3,6 @@
  * \file tree_model.cc
  * \brief model structure for tree
  */
-#include <sstream>
-#include <limits>
-#include <cmath>
-#include <iomanip>
-#include <stack>
-
 #include <dmlc/registry.h>
 #include <dmlc/json.h>
 
@@ -16,6 +10,12 @@
 #include <xgboost/logging.h>
 #include <xgboost/json.h>
 
+#include <sstream>
+#include <limits>
+#include <cmath>
+#include <iomanip>
+#include <stack>
+
 #include "param.h"
 #include "../common/common.h"
 
diff --git a/src/tree/updater_basemaker-inl.h b/src/tree/updater_basemaker-inl.h
index 42f4901dd..66ab91982 100644
--- a/src/tree/updater_basemaker-inl.h
+++ b/src/tree/updater_basemaker-inl.h
@@ -7,13 +7,15 @@
 #ifndef XGBOOST_TREE_UPDATER_BASEMAKER_INL_H_
 #define XGBOOST_TREE_UPDATER_BASEMAKER_INL_H_
 
+#include <rabit/rabit.h>
+
+
 #include <vector>
 #include <algorithm>
 #include <string>
 #include <limits>
 #include <utility>
 
-#include <rabit/rabit.h>
 #include "xgboost/base.h"
 #include "xgboost/json.h"
 #include "xgboost/tree_updater.h"
diff --git a/src/tree/updater_colmaker.cc b/src/tree/updater_colmaker.cc
index 563160059..951cfdb5e 100644
--- a/src/tree/updater_colmaker.cc
+++ b/src/tree/updater_colmaker.cc
@@ -4,13 +4,12 @@
  * \brief use columnwise update to construct a tree
  * \author Tianqi Chen
  */
+#include <rabit/rabit.h>
 #include <memory>
 #include <vector>
 #include <cmath>
 #include <algorithm>
 
-#include <rabit/rabit.h>
-
 #include "xgboost/parameter.h"
 #include "xgboost/tree_updater.h"
 #include "xgboost/logging.h"
diff --git a/src/tree/updater_gpu_hist.cu b/src/tree/updater_gpu_hist.cu
index 8a52de9e8..25d2645e1 100644
--- a/src/tree/updater_gpu_hist.cu
+++ b/src/tree/updater_gpu_hist.cu
@@ -1,6 +1,9 @@
 /*!
  * Copyright 2017-2020 XGBoost contributors
  */
+#include <thrust/copy.h>
+#include <thrust/reduce.h>
+#include <xgboost/tree_updater.h>
 #include <algorithm>
 #include <cmath>
 #include <memory>
@@ -9,10 +12,6 @@
 #include <utility>
 #include <vector>
 
-#include <thrust/copy.h>
-#include <thrust/reduce.h>
-
-#include "xgboost/tree_updater.h"
 #include "xgboost/host_device_vector.h"
 #include "xgboost/parameter.h"
 #include "xgboost/span.h"
diff --git a/src/tree/updater_histmaker.cc b/src/tree/updater_histmaker.cc
index cc17b5143..c4fdbe3c0 100644
--- a/src/tree/updater_histmaker.cc
+++ b/src/tree/updater_histmaker.cc
@@ -4,10 +4,10 @@
  * \brief use histogram counting to construct a tree
  * \author Tianqi Chen
  */
+#include <rabit/rabit.h>
 #include <vector>
 #include <algorithm>
 
-#include <rabit/rabit.h>
 #include "xgboost/tree_updater.h"
 #include "xgboost/base.h"
 #include "xgboost/logging.h"
diff --git a/src/tree/updater_prune.cc b/src/tree/updater_prune.cc
index 5aa904ed8..76a8916a0 100644
--- a/src/tree/updater_prune.cc
+++ b/src/tree/updater_prune.cc
@@ -4,18 +4,17 @@
  * \brief prune a tree given the statistics
  * \author Tianqi Chen
  */
+#include <rabit/rabit.h>
+#include <xgboost/tree_updater.h>
+
 #include <string>
 #include <memory>
 
-#include <rabit/rabit.h>
-
-#include "xgboost/tree_updater.h"
 #include "xgboost/base.h"
 #include "xgboost/json.h"
 #include "./param.h"
 #include "../common/io.h"
 #include "../common/timer.h"
-
 namespace xgboost {
 namespace tree {
 
diff --git a/src/tree/updater_quantile_hist.cc b/src/tree/updater_quantile_hist.cc
index 51b47a386..30eb01a72 100644
--- a/src/tree/updater_quantile_hist.cc
+++ b/src/tree/updater_quantile_hist.cc
@@ -4,6 +4,9 @@
  * \brief use quantized feature values to construct a tree
  * \author Philip Cho, Tianqi Checn, Egor Smirnov
  */
+#include <dmlc/timer.h>
+#include <rabit/rabit.h>
+
 #include <cmath>
 #include <memory>
 #include <vector>
@@ -14,8 +17,6 @@
 #include <string>
 #include <utility>
 
-#include <dmlc/timer.h>
-#include <rabit/rabit.h>
 #include "xgboost/logging.h"
 #include "xgboost/tree_updater.h"
 
diff --git a/src/tree/updater_quantile_hist.h b/src/tree/updater_quantile_hist.h
index 388584df3..d74c16f72 100644
--- a/src/tree/updater_quantile_hist.h
+++ b/src/tree/updater_quantile_hist.h
@@ -7,6 +7,10 @@
 #ifndef XGBOOST_TREE_UPDATER_QUANTILE_HIST_H_
 #define XGBOOST_TREE_UPDATER_QUANTILE_HIST_H_
 
+#include <dmlc/timer.h>
+#include <rabit/rabit.h>
+#include <xgboost/tree_updater.h>
+
 #include <memory>
 #include <vector>
 #include <string>
@@ -15,13 +19,8 @@
 #include <unordered_map>
 #include <utility>
 
-#include <dmlc/timer.h>
-#include <rabit/rabit.h>
-
-#include "xgboost/tree_updater.h"
 #include "xgboost/data.h"
 #include "xgboost/json.h"
-
 #include "constraints.h"
 #include "./param.h"
 #include "./split_evaluator.h"
diff --git a/src/tree/updater_refresh.cc b/src/tree/updater_refresh.cc
index 5748d9b3a..d63d88c80 100644
--- a/src/tree/updater_refresh.cc
+++ b/src/tree/updater_refresh.cc
@@ -4,12 +4,12 @@
  * \brief refresh the statistics and leaf value on the tree on the dataset
  * \author Tianqi Chen
  */
+#include <rabit/rabit.h>
+#include <xgboost/tree_updater.h>
+
 #include <vector>
 #include <limits>
 
-#include <rabit/rabit.h>
-
-#include "xgboost/tree_updater.h"
 #include "xgboost/json.h"
 #include "./param.h"
 #include "../common/io.h"
diff --git a/src/tree/updater_skmaker.cc b/src/tree/updater_skmaker.cc
index b4468b517..69cb4e58b 100644
--- a/src/tree/updater_skmaker.cc
+++ b/src/tree/updater_skmaker.cc
@@ -5,13 +5,12 @@
           a refresh is needed to make the statistics exactly correct
  * \author Tianqi Chen
  */
+#include <rabit/rabit.h>
+#include <xgboost/base.h>
+#include <xgboost/tree_updater.h>
 #include <vector>
 #include <algorithm>
 
-#include <rabit/rabit.h>
-#include "xgboost/base.h"
-#include "xgboost/tree_updater.h"
-
 #include "../common/quantile.h"
 #include "../common/group_data.h"
 #include "./updater_basemaker-inl.h"
diff --git a/src/tree/updater_sync.cc b/src/tree/updater_sync.cc
index 6ec814d4f..578bfb83c 100644
--- a/src/tree/updater_sync.cc
+++ b/src/tree/updater_sync.cc
@@ -3,11 +3,11 @@
  * \file updater_sync.cc
  * \brief synchronize the tree in all distributed nodes
  */
+#include <xgboost/tree_updater.h>
 #include <vector>
 #include <string>
 #include <limits>
 
-#include "xgboost/tree_updater.h"
 #include "xgboost/json.h"
 #include "../common/io.h"