diff --git a/CMakeLists.txt b/CMakeLists.txt
index 538ee4790..92c78917c 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -116,11 +116,12 @@ if (R_LIB)
 endif (R_LIB)
 
 # core xgboost
-add_subdirectory(${PROJECT_SOURCE_DIR}/src)
+add_subdirectory(${xgboost_SOURCE_DIR}/plugin)
+add_subdirectory(${xgboost_SOURCE_DIR}/src)
 set(XGBOOST_OBJ_SOURCES "${XGBOOST_OBJ_SOURCES};$<TARGET_OBJECTS:objxgboost>")
 
 #-- Shared library
-add_library(xgboost SHARED ${XGBOOST_OBJ_SOURCES} ${PLUGINS_SOURCES})
+add_library(xgboost SHARED ${XGBOOST_OBJ_SOURCES})
 target_include_directories(xgboost
   INTERFACE
   $<INSTALL_INTERFACE:${CMAKE_INSTALL_PREFIX}/include>
diff --git a/Makefile b/Makefile
index 42d3bfe1a..16d97c0a3 100644
--- a/Makefile
+++ b/Makefile
@@ -42,11 +42,6 @@ ifeq ($(USE_OPENMP), 0)
 endif
 include $(DMLC_CORE)/make/dmlc.mk
 
-# include the plugins
-ifdef XGB_PLUGINS
-include $(XGB_PLUGINS)
-endif
-
 # set compiler defaults for OSX versus *nix
 # let people override either
 OS := $(shell uname)
@@ -67,8 +62,8 @@ export CXX = g++
 endif
 endif
 
-export LDFLAGS= -pthread -lm $(ADD_LDFLAGS) $(DMLC_LDFLAGS) $(PLUGIN_LDFLAGS)
-export CFLAGS= -DDMLC_LOG_CUSTOMIZE=1 -std=c++11 -Wall -Wno-unknown-pragmas -Iinclude $(ADD_CFLAGS) $(PLUGIN_CFLAGS)
+export LDFLAGS= -pthread -lm $(ADD_LDFLAGS) $(DMLC_LDFLAGS)
+export CFLAGS= -DDMLC_LOG_CUSTOMIZE=1 -std=c++11 -Wall -Wno-unknown-pragmas -Iinclude $(ADD_CFLAGS)
 CFLAGS += -I$(DMLC_CORE)/include -I$(RABIT)/include -I$(GTEST_PATH)/include
 #java include path
 export JAVAINCFLAGS = -I${JAVA_HOME}/include -I./java
@@ -130,7 +125,7 @@ $(RABIT)/lib/$(LIB_RABIT): $(wildcard $(RABIT)/src/*.cc)
 jvm: jvm-packages/lib/libxgboost4j.so
 
 SRC = $(wildcard src/*.cc src/*/*.cc)
-ALL_OBJ = $(patsubst src/%.cc, build/%.o, $(SRC)) $(PLUGIN_OBJS)
+ALL_OBJ = $(patsubst src/%.cc, build/%.o, $(SRC))
 AMALGA_OBJ = amalgamation/xgboost-all0.o
 LIB_DEP = $(DMLC_CORE)/libdmlc.a $(RABIT)/lib/$(LIB_RABIT)
 ALL_DEP = $(filter-out build/cli_main.o, $(ALL_OBJ)) $(LIB_DEP)
@@ -142,11 +137,6 @@ build/%.o: src/%.cc
 	$(CXX) $(CFLAGS) -MM -MT build/$*.o $< >build/$*.d
 	$(CXX) -c $(CFLAGS) $< -o $@
 
-build_plugin/%.o: plugin/%.cc
-	@mkdir -p $(@D)
-	$(CXX) $(CFLAGS) -MM -MT build_plugin/$*.o $< >build_plugin/$*.d
-	$(CXX) -c $(CFLAGS) $< -o $@
-
 # The should be equivalent to $(ALL_OBJ)  except for build/cli_main.o
 amalgamation/xgboost-all0.o: amalgamation/xgboost-all0.cc
 	$(CXX) -c $(CFLAGS) $< -o $@
@@ -180,7 +170,7 @@ lint: rcpplint
 	  python-package/xgboost/include python-package/xgboost/lib \
 	  python-package/xgboost/make python-package/xgboost/rabit \
 	  python-package/xgboost/src --pylint-rc ${PWD}/python-package/.pylintrc xgboost \
-	  ${LINT_LANG} include src plugin python-package
+	  ${LINT_LANG} include src python-package
 
 pylint:
 	flake8 --ignore E501 python-package
@@ -200,7 +190,7 @@ cover: check
 endif
 
 clean:
-	$(RM) -rf build build_plugin lib bin *~ */*~ */*/*~ */*/*/*~ */*.o */*/*.o */*/*/*.o #xgboost
+	$(RM) -rf build lib bin *~ */*~ */*/*~ */*/*/*~ */*.o */*/*.o */*/*/*.o #xgboost
 	$(RM) -rf build_tests *.gcov tests/cpp/xgboost_test
 	if [ -d "R-package/src" ]; then \
 		cd R-package/src; \
@@ -278,4 +268,3 @@ Rcheck: Rbuild
 
 -include build/*.d
 -include build/*/*.d
--include build_plugin/*/*.d
diff --git a/doc/build.rst b/doc/build.rst
index 2dadadb53..64f76b73b 100644
--- a/doc/build.rst
+++ b/doc/build.rst
@@ -226,7 +226,7 @@ On Windows, run CMake as follows:
 
   .. code-block:: bash
 
-    make .. -G"Visual Studio 15 2017 Win64" -T v140,cuda=8.0 -DUSE_CUDA=ON
+    cmake .. -G"Visual Studio 15 2017 Win64" -T v140,cuda=8.0 -DUSE_CUDA=ON
 
 To speed up compilation, the compute version specific to your GPU could be passed to cmake as, e.g., ``-DGPU_COMPUTE_VER=50``.
 The above cmake configuration run will create an ``xgboost.sln`` solution file in the build directory. Build this solution in release mode as a x64 build, either from Visual studio or from command line:
diff --git a/doc/parameter.rst b/doc/parameter.rst
index f19ddb462..d0ccacba6 100644
--- a/doc/parameter.rst
+++ b/doc/parameter.rst
@@ -135,7 +135,7 @@ Parameters for Tree Booster
 
 * ``updater`` [default= ``grow_colmaker,prune``]
 
-  - A comma separated string defining the sequence of tree updaters to run, providing a modular way to construct and to modify the trees. This is an advanced parameter that is usually set automatically, depending on some other parameters. However, it could be also set explicitly by a user. The following updater plugins exist:
+  - A comma separated string defining the sequence of tree updaters to run, providing a modular way to construct and to modify the trees. This is an advanced parameter that is usually set automatically, depending on some other parameters. However, it could be also set explicitly by a user. The following updaters exist:
 
     - ``grow_colmaker``: non-distributed column-based construction of trees.
     - ``distcol``: distributed tree construction with column-based data splitting mode.
@@ -150,7 +150,7 @@ Parameters for Tree Booster
 
 * ``refresh_leaf`` [default=1]
 
-  - This is a parameter of the ``refresh`` updater plugin. When this flag is 1, tree leafs as well as tree nodes' stats are updated. When it is 0, only node stats are updated.
+  - This is a parameter of the ``refresh`` updater. When this flag is 1, tree leafs as well as tree nodes' stats are updated. When it is 0, only node stats are updated.
 
 * ``process_type`` [default= ``default``]
 
@@ -158,7 +158,7 @@ Parameters for Tree Booster
   - Choices: ``default``, ``update``
 
     - ``default``: The normal boosting process which creates new trees.
-    - ``update``: Starts from an existing model and only updates its trees. In each boosting iteration, a tree from the initial model is taken, a specified sequence of updater plugins is run for that tree, and a modified tree is added to the new model. The new model would have either the same or smaller number of trees, depending on the number of boosting iteratons performed. Currently, the following built-in updater plugins could be meaningfully used with this process type: ``refresh``, ``prune``. With ``process_type=update``, one cannot use updater plugins that create new trees.
+    - ``update``: Starts from an existing model and only updates its trees. In each boosting iteration, a tree from the initial model is taken, a specified sequence of updaters is run for that tree, and a modified tree is added to the new model. The new model would have either the same or smaller number of trees, depending on the number of boosting iteratons performed. Currently, the following built-in updaters could be meaningfully used with this process type: ``refresh``, ``prune``. With ``process_type=update``, one cannot use updaters that create new trees.
 
 * ``grow_policy`` [default= ``depthwise``]
 
diff --git a/make/config.mk b/make/config.mk
index fc4bb7ae5..bf10e1b82 100644
--- a/make/config.mk
+++ b/make/config.mk
@@ -65,13 +65,3 @@ TEST_COVER = 0
 # path to gtest library (only used when $BUILD_TEST=1)
 # there should be an include path in $GTEST_PATH/include and library in $GTEST_PATH/lib
 GTEST_PATH ?= 
-
-# path to cub library (only used when $CUDA_ENABLED=1)
-# this should point to the cub project root folder
-CUB_PATH ?= cub
-
-# List of additional plugins, checkout plugin folder.
-# uncomment the following lines to include these plugins
-# you can also add your own plugin like this
-#
-# XGB_PLUGINS += plugin/example/plugin.mk
diff --git a/make/travis.mk b/make/travis.mk
index ac2897352..a32da2bd5 100644
--- a/make/travis.mk
+++ b/make/travis.mk
@@ -28,11 +28,3 @@ LIBJVM=$(JAVA_HOME)/jre/lib/amd64/server
 # path to googletest and whether to measure coverage or not
 GTEST_PATH =
 WITH_COVER = 1
-
-# List of additional plugins, checkout plugin folder.
-# uncomment the following lines to include these plugins
-# you can also add your own plugin like this
-#
-XGB_PLUGINS += plugin/example/plugin.mk
-XGB_PLUGINS += plugin/lz4/plugin.mk
-XGB_PLUGINS += plugin/dense_parser/plugin.mk
diff --git a/plugin/CMakeLists.txt b/plugin/CMakeLists.txt
new file mode 100644
index 000000000..0c44784ad
--- /dev/null
+++ b/plugin/CMakeLists.txt
@@ -0,0 +1,9 @@
+if (PLUGIN_LZ4)
+  set(PLUGIN_SOURCES ${PLUGIN_SOURCES}
+    ${xgboost_SOURCE_DIR}/plugin/lz4/sparse_page_lz4_format.cc PARENT_SCOPE)
+endif (PLUGIN_LZ4)
+
+if (PLUGIN_DENSE_PARSER)
+  set(PLUGINS_SOURCES ${PLUGINS_SOURCES}
+    ${xgboost_SOURCE_DIR}/plugin/dense_parser/dense_libsvm.cc PARENT_SCOPE)
+endif (PLUGIN_DENSE_PARSER)
diff --git a/plugin/README.md b/plugin/README.md
index de146a9ec..6e115c465 100644
--- a/plugin/README.md
+++ b/plugin/README.md
@@ -1,30 +1,19 @@
 XGBoost Plugins Modules
 =======================
-This folder contains plugin modules to xgboost that can be optionally installed.
-The plugin system helps us to extend xgboost with additional features,
-and add experimental features that may not yet be ready to be included in the
-main project.
 
-To include a certain plugin, say ```plugin_a```, you only need to add the following line to the config.mk.
+This folder contains plugin modules to xgboost that can be optionally installed.  The
+plugin system helps us to extend xgboost with additional features, and add experimental
+features that may not yet be ready to be included in the main project.
 
-```makefile
-# Add plugin by including the plugin in config.mk
-XGB_PLUGINS += plugin/plugin_a/plugin.mk
+To include a certain plugin, say ```plugin_a```, you only need to add the following line
+to `xgboost/plugin/CMakeLists.txt`
+``` cmake
+set(PLUGIN_SOURCES ${PLUGIN_SOURCES}
+    ${xgboost_SOURCE_DIR}/plugin/plugin_a.cc PARENT_SCOPE)
 ```
+along with specified source file `plugin_a.cc`.
 
-Then rebuild libxgboost by typing ```make```, you can get a new library with the plugin enabled.
-
-Link Static XGBoost Library with Plugins
-----------------------------------------
-This problem only happens when you link ```libxgboost.a```.
-If you only use ```libxgboost.so```(this include python and other bindings),
-you can ignore this section.
-
-When you want to link ```libxgboost.a``` with additional plugins included,
-you will need to enabled whole archive via The following option.
-```bash
---whole-archive libxgboost.a --no-whole-archive
-```
+Then rebuild XGBoost with CMake.
 
 Write Your Own Plugin
 ---------------------
diff --git a/plugin/dense_parser/plugin.mk b/plugin/dense_parser/plugin.mk
deleted file mode 100644
index 027cc42f8..000000000
--- a/plugin/dense_parser/plugin.mk
+++ /dev/null
@@ -1,2 +0,0 @@
-PLUGIN_OBJS += build_plugin/dense_parser/dense_libsvm.o
-PLUGIN_LDFLAGS +=
diff --git a/plugin/example/README.md b/plugin/example/README.md
index 32b709047..0d5ef15ed 100644
--- a/plugin/example/README.md
+++ b/plugin/example/README.md
@@ -1,21 +1,19 @@
 XGBoost Plugin Example
 ======================
-This folder provides an example of xgboost plugin.
+This folder provides an example of implementing xgboost plugin.
 
 There are three steps you need to do to add a plugin to xgboost
 - Create your source .cc file, implement a new extension
   - In this example [custom_obj.cc](custom_obj.cc)
 - Register this extension to xgboost via a registration macro
-  - In this example ```XGBOOST_REGISTER_OBJECTIVE``` in [this line](custom_obj.cc#L75)
-- Create a [plugin.mk](plugin.mk) on this folder
-
-To add this plugin, add the following line to ```config.mk```(template in make/config.mk).
-```makefile
-# Add plugin by include the plugin in config
-XGB_PLUGINS += plugin/plugin_a/plugin.mk
+  - In this example ```XGBOOST_REGISTER_OBJECTIVE``` in [this line](custom_obj.cc#L78)
+- Add a line to `xgboost/plugin/CMakeLists.txt`:
+```
+set(PLUGINS_SOURCES ${PLUGINS_SOURCES}
+    ${xgboost_SOURCE_DIR}/plugin/example/custom_obj.cc PARENT_SCOPE)
 ```
 
 Then you can test this plugin by using ```objective=mylogistic``` parameter.
 
-
-
+<!--  LocalWords:  XGBoost
+ -->
diff --git a/plugin/example/plugin.mk b/plugin/example/plugin.mk
deleted file mode 100644
index 8ebd26d61..000000000
--- a/plugin/example/plugin.mk
+++ /dev/null
@@ -1,4 +0,0 @@
-# Add the object files you like to include in this plugin.
-PLUGIN_OBJS += build_plugin/example/custom_obj.o
-# Add additional dependent libraries this plugin might have
-PLUGIN_LDFLAGS +=
\ No newline at end of file
diff --git a/plugin/lz4/plugin.mk b/plugin/lz4/plugin.mk
deleted file mode 100644
index 7a69027c7..000000000
--- a/plugin/lz4/plugin.mk
+++ /dev/null
@@ -1,2 +0,0 @@
-PLUGIN_OBJS += build_plugin/lz4/sparse_page_lz4_format.o
-PLUGIN_LDFLAGS += -llz4
diff --git a/plugin/updater_gpu/README.md b/plugin/updater_gpu/README.md
deleted file mode 100644
index 614e05c48..000000000
--- a/plugin/updater_gpu/README.md
+++ /dev/null
@@ -1,3 +0,0 @@
-# CUDA Accelerated Tree Construction Algorithms
-
-[The XGBoost GPU documentation has moved here.](https://xgboost.readthedocs.io/en/latest/gpu/index.html)
diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
index 50291355d..f2b29897b 100644
--- a/src/CMakeLists.txt
+++ b/src/CMakeLists.txt
@@ -20,15 +20,6 @@ int main() {
 }
 " XGBOOST_BUILTIN_PREFETCH_PRESENT)
 
-# Add plugins to source files
-if (PLUGIN_LZ4)
-  list(APPEND PLUGINS_SOURCES ${PROJECT_SOURCE_DIR}/plugin/lz4/sparse_page_lz4_format.cc)
-  list(APPEND SRC_LIBS lz4)
-endif (PLUGIN_LZ4)
-if (PLUGIN_DENSE_PARSER)
-  list(APPEND PLUGINS_SOURCES ${PROJECT_SOURCE_DIR}/plugin/dense_parser/dense_libsvm.cc)
-endif (PLUGIN_DENSE_PARSER)
-
 #-- Object library
 # Object library is necessary for jvm-package, which creates its own shared
 # library.
diff --git a/tests/cpp/CMakeLists.txt b/tests/cpp/CMakeLists.txt
index 8afcd6ab9..319abe004 100644
--- a/tests/cpp/CMakeLists.txt
+++ b/tests/cpp/CMakeLists.txt
@@ -12,7 +12,8 @@ if (USE_CUDA)
   file(GLOB_RECURSE CUDA_TEST_SOURCES "*.cu")
   list(APPEND TEST_SOURCES ${CUDA_TEST_SOURCES})
 endif (USE_CUDA)
-add_executable(testxgboost ${TEST_SOURCES} ${XGBOOST_OBJ_SOURCES})
+add_executable(testxgboost ${TEST_SOURCES} ${XGBOOST_OBJ_SOURCES}
+  ${xgboost_SOURCE_DIR}/plugin/example/custom_obj.cc)
 
 if (USE_CUDA)
   # OpenMP is mandatory for CUDA
diff --git a/tests/cpp/plugin/test_example_objective.cc b/tests/cpp/plugin/test_example_objective.cc
new file mode 100644
index 000000000..816f861f4
--- /dev/null
+++ b/tests/cpp/plugin/test_example_objective.cc
@@ -0,0 +1,15 @@
+#include <gtest/gtest.h>
+#include <xgboost/objective.h>
+#include <string>
+#include "../helpers.h"
+
+namespace xgboost {
+
+TEST(Plugin, ExampleObjective) {
+  xgboost::GenericParameter tparam = CreateEmptyGenericParam(0, 0);
+  auto * obj = xgboost::ObjFunction::Create("mylogistic", &tparam);
+  ASSERT_EQ(obj->DefaultEvalMetric(), std::string{"error"});
+  delete obj;
+}
+
+}  // namespace xgboost
diff --git a/tests/travis/run_test.sh b/tests/travis/run_test.sh
index bf413cf8e..1fa0f513a 100755
--- a/tests/travis/run_test.sh
+++ b/tests/travis/run_test.sh
@@ -5,8 +5,6 @@ make -f dmlc-core/scripts/packages.mk lz4
 
 if [ ${TRAVIS_OS_NAME} == "osx" ]; then
     echo 'USE_OPENMP=0' >> config.mk
-    echo 'TMPVAR := $(XGB_PLUGINS)' >> config.mk
-    echo 'XGB_PLUGINS = $(filter-out plugin/lz4/plugin.mk, $(TMPVAR))' >> config.mk
 else
     # use g++-4.8 for linux
     export CXX=g++-4.8