diff --git a/cmake/Version.cmake b/cmake/Version.cmake
index c8eafc8f1..0159723e1 100644
--- a/cmake/Version.cmake
+++ b/cmake/Version.cmake
@@ -1,8 +1,8 @@
 function (write_version)
   message(STATUS "xgboost VERSION: ${xgboost_VERSION}")
   configure_file(
-    ${xgboost_SOURCE_DIR}/cmake/build_config.h.in
-    ${xgboost_SOURCE_DIR}/include/xgboost/build_config.h @ONLY)
+    ${xgboost_SOURCE_DIR}/cmake/version_config.h.in
+    ${xgboost_SOURCE_DIR}/include/xgboost/version_config.h @ONLY)
   configure_file(
     ${xgboost_SOURCE_DIR}/cmake/Python_version.in
     ${xgboost_SOURCE_DIR}/python-package/xgboost/VERSION
diff --git a/cmake/build_config.h.in b/cmake/build_config.h.in
deleted file mode 100644
index 74d980cdf..000000000
--- a/cmake/build_config.h.in
+++ /dev/null
@@ -1,17 +0,0 @@
-/*!
- * Copyright 2019 by Contributors
- * \file build_config.h
- *
- * Generated from `cmake/build_config.h.in` by cmake.
- */
-#ifndef XGBOOST_BUILD_CONFIG_H_
-#define XGBOOST_BUILD_CONFIG_H_
-
-#cmakedefine XGBOOST_MM_PREFETCH_PRESENT
-#cmakedefine XGBOOST_BUILTIN_PREFETCH_PRESENT
-
-#define XGBOOST_VER_MAJOR @xgboost_VERSION_MAJOR@
-#define XGBOOST_VER_MINOR @xgboost_VERSION_MINOR@
-#define XGBOOST_VER_PATCH @xgboost_VERSION_PATCH@
-
-#endif  // XGBOOST_BUILD_CONFIG_H_
diff --git a/cmake/version_config.h.in b/cmake/version_config.h.in
new file mode 100644
index 000000000..dfde79a5a
--- /dev/null
+++ b/cmake/version_config.h.in
@@ -0,0 +1,11 @@
+/*!
+ * Copyright 2019 XGBoost contributors
+ */
+#ifndef XGBOOST_VERSION_CONFIG_H_
+#define XGBOOST_VERSION_CONFIG_H_
+
+#define XGBOOST_VER_MAJOR @xgboost_VERSION_MAJOR@
+#define XGBOOST_VER_MINOR @xgboost_VERSION_MINOR@
+#define XGBOOST_VER_PATCH @xgboost_VERSION_PATCH@
+
+#endif  // XGBOOST_VERSION_CONFIG_H_
diff --git a/include/xgboost/base.h b/include/xgboost/base.h
index 673bd633f..29d8c95b5 100644
--- a/include/xgboost/base.h
+++ b/include/xgboost/base.h
@@ -84,6 +84,20 @@
 #define XGBOOST_DEVICE
 #endif  // defined (__CUDA__) || defined(__NVCC__)
 
+// These check are for Makefile.
+#if !defined(XGBOOST_MM_PREFETCH_PRESENT) && !defined(XGBOOST_BUILTIN_PREFETCH_PRESENT)
+/* default logic for software pre-fetching */
+#if (defined(_MSC_VER) && (defined(_M_IX86) || defined(_M_AMD64))) || defined(__INTEL_COMPILER)
+// Enable _mm_prefetch for Intel compiler and MSVC+x86
+  #define XGBOOST_MM_PREFETCH_PRESENT
+  #define XGBOOST_BUILTIN_PREFETCH_PRESENT
+#elif defined(__GNUC__)
+// Enable __builtin_prefetch for GCC
+#define XGBOOST_BUILTIN_PREFETCH_PRESENT
+#endif  // GUARDS
+
+#endif  // !defined(XGBOOST_MM_PREFETCH_PRESENT) && !defined()
+
 /*! \brief namespace of xgboost*/
 namespace xgboost {
 /*!
@@ -232,7 +246,4 @@ using XGBoostVersionT = int32_t;
 #endif  // DMLC_USE_CXX11 && defined(__GNUC__) && !defined(__clang_version__)
 }  // namespace xgboost
 
-/* Always keep this #include at the bottom of xgboost/base.h */
-#include <xgboost/build_config.h>
-
 #endif  // XGBOOST_BASE_H_
diff --git a/include/xgboost/build_config.h b/include/xgboost/build_config.h
deleted file mode 100644
index f626f390a..000000000
--- a/include/xgboost/build_config.h
+++ /dev/null
@@ -1,28 +0,0 @@
-/*!
- * Copyright 2019 by Contributors
- * \file build_config.h
- *
- * Generated from `cmake/build_config.h.in` by cmake.
- */
-#ifndef XGBOOST_BUILD_CONFIG_H_
-#define XGBOOST_BUILD_CONFIG_H_
-
-// These check are for Makefile.
-#if !defined(XGBOOST_MM_PREFETCH_PRESENT) && !defined(XGBOOST_BUILTIN_PREFETCH_PRESENT)
-/* default logic for software pre-fetching */
-#if (defined(_MSC_VER) && (defined(_M_IX86) || defined(_M_AMD64))) || defined(__INTEL_COMPILER)
-// Enable _mm_prefetch for Intel compiler and MSVC+x86
-  #define XGBOOST_MM_PREFETCH_PRESENT
-  #define XGBOOST_BUILTIN_PREFETCH_PRESENT
-#elif defined(__GNUC__)
-// Enable __builtin_prefetch for GCC
-#define XGBOOST_BUILTIN_PREFETCH_PRESENT
-#endif  // GUARDS
-
-#endif  // !defined(XGBOOST_MM_PREFETCH_PRESENT) && !defined()
-
-#define XGBOOST_VER_MAJOR 1
-#define XGBOOST_VER_MINOR 0
-#define XGBOOST_VER_PATCH 0
-
-#endif  // XGBOOST_BUILD_CONFIG_H_
diff --git a/include/xgboost/version_config.h b/include/xgboost/version_config.h
new file mode 100644
index 000000000..98b83cc68
--- /dev/null
+++ b/include/xgboost/version_config.h
@@ -0,0 +1,11 @@
+/*!
+ * Copyright 2019 XGBoost contributors
+ */
+#ifndef XGBOOST_VERSION_CONFIG_H_
+#define XGBOOST_VERSION_CONFIG_H_
+
+#define XGBOOST_VER_MAJOR 1
+#define XGBOOST_VER_MINOR 0
+#define XGBOOST_VER_PATCH 0
+
+#endif  // XGBOOST_VERSION_CONFIG_H_
diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
index ad3af9556..9d753aadc 100644
--- a/src/CMakeLists.txt
+++ b/src/CMakeLists.txt
@@ -64,6 +64,17 @@ target_compile_definitions(objxgboost
   $<$<NOT:$<CXX_COMPILER_ID:MSVC>>:_MWAITXINTRIN_H_INCLUDED>
   ${XGBOOST_DEFINITIONS})
 
+if (XGBOOST_MM_PREFETCH_PRESENT)
+  target_compile_definitions(objxgboost
+    PRIVATE
+    -DXGBOOST_MM_PREFETCH_PRESENT=1)
+endif(XGBOOST_MM_PREFETCH_PRESENT)
+if (XGBOOST_BUILTIN_PREFETCH_PRESENT)
+  target_compile_definitions(objxgboost
+    PRIVATE
+    -DXGBOOST_BUILTIN_PREFETCH_PRESENT=1)
+endif (XGBOOST_BUILTIN_PREFETCH_PRESENT)
+
 if (USE_OPENMP)
   find_package(OpenMP REQUIRED)
   if (OpenMP_CXX_FOUND OR OPENMP_FOUND)
diff --git a/src/c_api/c_api.cc b/src/c_api/c_api.cc
index c6369f9f0..081cc5395 100644
--- a/src/c_api/c_api.cc
+++ b/src/c_api/c_api.cc
@@ -1,10 +1,4 @@
 // Copyright (c) 2014-2019 by Contributors
-
-#include <xgboost/data.h>
-#include <xgboost/learner.h>
-#include <xgboost/c_api.h>
-#include <xgboost/logging.h>
-
 #include <dmlc/thread_local.h>
 #include <rabit/rabit.h>
 #include <rabit/c_api.h>
@@ -16,6 +10,12 @@
 #include <string>
 #include <memory>
 
+#include "xgboost/data.h"
+#include "xgboost/learner.h"
+#include "xgboost/c_api.h"
+#include "xgboost/logging.h"
+#include "xgboost/version_config.h"
+
 #include "c_api_error.h"
 #include "../data/simple_csr_source.h"
 #include "../common/math.h"
diff --git a/src/common/hist_util.cc b/src/common/hist_util.cc
index 9db5f33de..83e8c117b 100644
--- a/src/common/hist_util.cc
+++ b/src/common/hist_util.cc
@@ -2,14 +2,16 @@
  * Copyright 2017-2019 by Contributors
   * \file hist_util.cc
  */
-#include "./hist_util.h"
 #include <dmlc/timer.h>
-#include <rabit/rabit.h>
 #include <dmlc/omp.h>
+
+#include <rabit/rabit.h>
 #include <numeric>
 #include <vector>
 
+#include "xgboost/base.h"
 #include "../common/common.h"
+#include "./hist_util.h"
 #include "./random.h"
 #include "./column_matrix.h"
 #include "./quantile.h"
diff --git a/src/common/version.cc b/src/common/version.cc
index 5f25d1057..3fb2e5c24 100644
--- a/src/common/version.cc
+++ b/src/common/version.cc
@@ -9,6 +9,7 @@
 
 #include "xgboost/logging.h"
 #include "xgboost/json.h"
+#include "xgboost/version_config.h"
 #include "version.h"
 
 namespace xgboost {
diff --git a/src/data/data.cc b/src/data/data.cc
index 0ca76c315..b4f42a260 100644
--- a/src/data/data.cc
+++ b/src/data/data.cc
@@ -2,15 +2,16 @@
  * Copyright 2015-2019 by Contributors
  * \file data.cc
  */
-#include <xgboost/data.h>
-#include <xgboost/logging.h>
-#include <xgboost/build_config.h>
 #include <dmlc/registry.h>
 #include <cstring>
 
-#include "./sparse_page_writer.h"
-#include "./simple_dmatrix.h"
-#include "./simple_csr_source.h"
+#include "xgboost/data.h"
+#include "xgboost/logging.h"
+#include "xgboost/version_config.h"
+#include "sparse_page_writer.h"
+#include "simple_dmatrix.h"
+#include "simple_csr_source.h"
+
 #include "../common/io.h"
 #include "../common/version.h"
 #include "../common/group_data.h"
diff --git a/tests/cpp/c_api/test_c_api.cc b/tests/cpp/c_api/test_c_api.cc
index 256068d55..3d096a1ea 100644
--- a/tests/cpp/c_api/test_c_api.cc
+++ b/tests/cpp/c_api/test_c_api.cc
@@ -1,5 +1,6 @@
-// Copyright by Contributors
+// Copyright (c) 2019 by Contributors
 #include <gtest/gtest.h>
+#include <xgboost/version_config.h>
 #include <xgboost/c_api.h>
 #include <xgboost/data.h>
 
diff --git a/tests/cpp/common/test_version.cc b/tests/cpp/common/test_version.cc
index fbbef5992..a2b7ed36c 100644
--- a/tests/cpp/common/test_version.cc
+++ b/tests/cpp/common/test_version.cc
@@ -6,6 +6,7 @@
 #include <dmlc/filesystem.h>
 #include <dmlc/io.h>
 
+#include <xgboost/version_config.h>
 #include <xgboost/json.h>
 #include <xgboost/base.h>
 
@@ -58,4 +59,4 @@ TEST(Version, Basic) {
   str = str.substr(ptr);
   ASSERT_EQ(str.size(), 0);
 }
-}  // namespace xgboost
\ No newline at end of file
+}  // namespace xgboost
diff --git a/tests/cpp/test_learner.cc b/tests/cpp/test_learner.cc
index 0d7f61e7e..2a1088029 100644
--- a/tests/cpp/test_learner.cc
+++ b/tests/cpp/test_learner.cc
@@ -2,9 +2,10 @@
 #include <gtest/gtest.h>
 #include <vector>
 #include "helpers.h"
+#include <dmlc/filesystem.h>
 
-#include "xgboost/learner.h"
-#include "dmlc/filesystem.h"
+#include <xgboost/learner.h>
+#include <xgboost/version_config.h>
 
 namespace xgboost {