diff --git a/include/xgboost/c_api.h b/include/xgboost/c_api.h
index 18fada464..1f6541255 100644
--- a/include/xgboost/c_api.h
+++ b/include/xgboost/c_api.h
@@ -461,9 +461,69 @@ XGB_DLL int XGBoosterLoadModelFromBuffer(BoosterHandle handle,
  * \param out_dptr the argument to hold the output data pointer
  * \return 0 when success, -1 when failure happens
  */
-XGB_DLL int XGBoosterGetModelRaw(BoosterHandle handle,
-                                 bst_ulong *out_len,
+XGB_DLL int XGBoosterGetModelRaw(BoosterHandle handle, bst_ulong *out_len,
                                  const char **out_dptr);
+
+/*!
+ * \brief Memory snapshot based serialization method.  Saves everything states
+ * into buffer.
+ *
+ * \param handle handle
+ * \param out_len the argument to hold the output length
+ * \param out_dptr the argument to hold the output data pointer
+ * \return 0 when success, -1 when failure happens
+ */
+XGB_DLL int XGBoosterSerializeToBuffer(BoosterHandle handle, bst_ulong *out_len,
+                                       const char **out_dptr);
+/*!
+ * \brief Memory snapshot based serialization method.  Loads the buffer returned
+ *        from `XGBoosterSerializeToBuffer'.
+ *
+ * \param handle handle
+ * \param buf pointer to the buffer
+ * \param len the length of the buffer
+ * \return 0 when success, -1 when failure happens
+ */
+XGB_DLL int XGBoosterUnserializeFromBuffer(BoosterHandle handle,
+                                           const void *buf, bst_ulong len);
+
+/*!
+ * \brief Initialize the booster from rabit checkpoint.
+ *  This is used in distributed training API.
+ * \param handle handle
+ * \param version The output version of the model.
+ * \return 0 when success, -1 when failure happens
+ */
+XGB_DLL int XGBoosterLoadRabitCheckpoint(BoosterHandle handle,
+                                         int* version);
+
+/*!
+ * \brief Save the current checkpoint to rabit.
+ * \param handle handle
+ * \return 0 when success, -1 when failure happens
+ */
+XGB_DLL int XGBoosterSaveRabitCheckpoint(BoosterHandle handle);
+
+
+/*!
+ * \brief Save XGBoost's internal configuration into a JSON document.
+ * \param handle handle to Booster object.
+ * \param out_str A valid pointer to array of characters.  The characters array is
+ *                allocated and managed by XGBoost, while pointer to that array needs to
+ *                be managed by caller.
+ * \return 0 when success, -1 when failure happens
+ */
+XGB_DLL int XGBoosterSaveJsonConfig(BoosterHandle handle, bst_ulong *out_len,
+                                    char const **out_str);
+/*!
+ * \brief Load XGBoost's internal configuration from a JSON document.
+ * \param handle handle to Booster object.
+ * \param json_parameters string representation of a JSON document.
+ * \return 0 when success, -1 when failure happens
+ */
+XGB_DLL int XGBoosterLoadJsonConfig(BoosterHandle handle,
+                                    char const *json_parameters);
+
 /*!
  * \brief dump model, return array of strings representing model dump
  * \param handle handle
@@ -570,25 +630,4 @@ XGB_DLL int XGBoosterSetAttr(BoosterHandle handle,
 XGB_DLL int XGBoosterGetAttrNames(BoosterHandle handle,
                                   bst_ulong* out_len,
                                   const char*** out);
-
-// --- Distributed training API----
-// NOTE: functions in rabit/c_api.h will be also available in libxgboost.so
-/*!
- * \brief Initialize the booster from rabit checkpoint.
- *  This is used in distributed training API.
- * \param handle handle
- * \param version The output version of the model.
- * \return 0 when success, -1 when failure happens
- */
-XGB_DLL int XGBoosterLoadRabitCheckpoint(
-    BoosterHandle handle,
-    int* version);
-
-/*!
- * \brief Save the current checkpoint to rabit.
- * \param handle handle
- * \return 0 when success, -1 when failure happens
- */
-XGB_DLL int XGBoosterSaveRabitCheckpoint(BoosterHandle handle);
-
 #endif  // XGBOOST_C_API_H_
diff --git a/include/xgboost/gbm.h b/include/xgboost/gbm.h
index f93b0dc4f..fde8d2e0d 100644
--- a/include/xgboost/gbm.h
+++ b/include/xgboost/gbm.h
@@ -32,7 +32,7 @@ struct LearnerModelParam;
 /*!
  * \brief interface of gradient boosting model.
  */
-class GradientBooster : public Model {
+class GradientBooster : public Model, public Configurable {
  protected:
   GenericParameter const* generic_param_;
 
diff --git a/include/xgboost/learner.h b/include/xgboost/learner.h
index 8b953af35..853f6bbc7 100644
--- a/include/xgboost/learner.h
+++ b/include/xgboost/learner.h
@@ -45,7 +45,7 @@ class Json;
  *
  *  \endcode
  */
-class Learner : public Model, public rabit::Serializable {
+class Learner : public Model, public Configurable, public rabit::Serializable {
  public:
   /*! \brief virtual destructor */
   ~Learner() override;
@@ -53,16 +53,6 @@ class Learner : public Model, public rabit::Serializable {
    * \brief Configure Learner based on set parameters.
    */
   virtual void Configure() = 0;
-  /*!
-   * \brief load model from stream
-   * \param fi input stream.
-   */
-  void Load(dmlc::Stream* fi) override = 0;
-  /*!
-   * \brief save model to stream.
-   * \param fo output stream
-   */
-  void Save(dmlc::Stream* fo) const override = 0;
   /*!
    * \brief update the model for one iteration
    *  With the specified objective function.
@@ -110,6 +100,13 @@ class Learner : public Model, public rabit::Serializable {
                        bool pred_contribs = false,
                        bool approx_contribs = false,
                        bool pred_interactions = false) = 0;
+
+  void LoadModel(Json const& in) override = 0;
+  void SaveModel(Json* out) const override = 0;
+
+  virtual void LoadModel(dmlc::Stream* fi) = 0;
+  virtual void SaveModel(dmlc::Stream* fo) const = 0;
+
   /*!
    * \brief Set multiple parameters at once.
    *
diff --git a/include/xgboost/parameter.h b/include/xgboost/parameter.h
index f9130b1fa..c3314688a 100644
--- a/include/xgboost/parameter.h
+++ b/include/xgboost/parameter.h
@@ -99,6 +99,7 @@ struct XGBoostParameter : public dmlc::Parameter<Type> {
       return unknown;
     }
   }
+  bool GetInitialised() const { return static_cast<bool>(this->initialised_); }
 };
 }  // namespace xgboost
 
diff --git a/python-package/xgboost/core.py b/python-package/xgboost/core.py
index 444d36ece..4406fef1f 100644
--- a/python-package/xgboost/core.py
+++ b/python-package/xgboost/core.py
@@ -1076,28 +1076,47 @@ class Booster(object):
         self.handle = ctypes.c_void_p()
         _check_call(_LIB.XGBoosterCreate(dmats, c_bst_ulong(len(cache)),
                                          ctypes.byref(self.handle)))
-        self.set_param({'seed': 0})
         self.set_param(params or {})
         if (params is not None) and ('booster' in params):
             self.booster = params['booster']
         else:
             self.booster = 'gbtree'
-        if model_file is not None:
+        if isinstance(model_file, Booster):
+            assert self.handle is not None
+            # We use the pickle interface for getting memory snapshot from
+            # another model, and load the snapshot with this booster.
+            state = model_file.__getstate__()
+            handle = state['handle']
+            del state['handle']
+            ptr = (ctypes.c_char * len(handle)).from_buffer(handle)
+            length = c_bst_ulong(len(handle))
+            _check_call(
+                _LIB.XGBoosterUnserializeFromBuffer(self.handle, ptr, length))
+            self.__dict__.update(state)
+        elif isinstance(model_file, (STRING_TYPES, os_PathLike)):
             self.load_model(model_file)
+        elif model_file is None:
+            pass
+        else:
+            raise TypeError('Unknown type:', model_file)
 
     def __del__(self):
-        if self.handle is not None:
+        if hasattr(self, 'handle') and self.handle is not None:
             _check_call(_LIB.XGBoosterFree(self.handle))
             self.handle = None
 
     def __getstate__(self):
-        # can't pickle ctypes pointers
-        # put model content in bytearray
+        # can't pickle ctypes pointers, put model content in bytearray
         this = self.__dict__.copy()
         handle = this['handle']
         if handle is not None:
-            raw = self.save_raw()
-            this["handle"] = raw
+            length = c_bst_ulong()
+            cptr = ctypes.POINTER(ctypes.c_char)()
+            _check_call(_LIB.XGBoosterSerializeToBuffer(self.handle,
+                                                        ctypes.byref(length),
+                                                        ctypes.byref(cptr)))
+            buf = ctypes2buffer(cptr, length.value)
+            this["handle"] = buf
         return this
 
     def __setstate__(self, state):
@@ -1107,18 +1126,44 @@ class Booster(object):
             buf = handle
             dmats = c_array(ctypes.c_void_p, [])
             handle = ctypes.c_void_p()
-            _check_call(_LIB.XGBoosterCreate(dmats, c_bst_ulong(0), ctypes.byref(handle)))
+            _check_call(_LIB.XGBoosterCreate(
+                dmats, c_bst_ulong(0), ctypes.byref(handle)))
             length = c_bst_ulong(len(buf))
             ptr = (ctypes.c_char * len(buf)).from_buffer(buf)
-            _check_call(_LIB.XGBoosterLoadModelFromBuffer(handle, ptr, length))
+            _check_call(
+                _LIB.XGBoosterUnserializeFromBuffer(handle, ptr, length))
             state['handle'] = handle
         self.__dict__.update(state)
 
+    def save_config(self):
+        '''Output internal parameter configuration of Booster as a JSON
+        string.'''
+        json_string = ctypes.c_char_p()
+        length = c_bst_ulong()
+        _check_call(_LIB.XGBoosterSaveJsonConfig(
+            self.handle,
+            ctypes.byref(length),
+            ctypes.byref(json_string)))
+        json_string = json_string.value.decode()
+        return json_string
+
+    def load_config(self, config):
+        '''Load configuration returned by `save_config`.'''
+        assert isinstance(config, str)
+        _check_call(_LIB.XGBoosterLoadJsonConfig(
+            self.handle,
+            c_str(config)))
+
     def __copy__(self):
         return self.__deepcopy__(None)
 
     def __deepcopy__(self, _):
-        return Booster(model_file=self.save_raw())
+        '''Return a copy of booster.  Caches for DMatrix are not copied so continue
+        training on copied booster will result in lower performance and
+        slightly different result.
+
+        '''
+        return Booster(model_file=self)
 
     def copy(self):
         """Copy the booster object.
@@ -1451,20 +1496,22 @@ class Booster(object):
     def save_model(self, fname):
         """Save the model to a file.
 
-        The model is saved in an XGBoost internal binary format which is
-        universal among the various XGBoost interfaces. Auxiliary attributes of
-        the Python Booster object (such as feature_names) will not be saved.
-        To preserve all attributes, pickle the Booster object.
+        The model is saved in an XGBoost internal format which is universal
+        among the various XGBoost interfaces. Auxiliary attributes of the
+        Python Booster object (such as feature_names) will not be saved.  To
+        preserve all attributes, pickle the Booster object.
 
         Parameters
         ----------
         fname : string or os.PathLike
             Output file name
+
         """
         if isinstance(fname, (STRING_TYPES, os_PathLike)):  # assume file name
-            _check_call(_LIB.XGBoosterSaveModel(self.handle, c_str(os_fspath(fname))))
+            _check_call(_LIB.XGBoosterSaveModel(
+                self.handle, c_str(os_fspath(fname))))
         else:
-            raise TypeError("fname must be a string")
+            raise TypeError("fname must be a string or os_PathLike")
 
     def save_raw(self):
         """Save the model to a in memory buffer representation
@@ -1481,26 +1528,26 @@ class Booster(object):
         return ctypes2buffer(cptr, length.value)
 
     def load_model(self, fname):
-        """Load the model from a file.
+        """Load the model from a file, local or as URI.
 
-        The model is loaded from an XGBoost internal binary format which is
-        universal among the various XGBoost interfaces. Auxiliary attributes of
-        the Python Booster object (such as feature_names) will not be loaded.
-        To preserve all attributes, pickle the Booster object.
+        The model is loaded from an XGBoost format which is universal among the
+        various XGBoost interfaces. Auxiliary attributes of the Python Booster
+        object (such as feature_names) will not be loaded.  To preserve all
+        attributes, pickle the Booster object.
 
         Parameters
         ----------
         fname : string, os.PathLike, or a memory buffer
             Input file name or memory buffer(see also save_raw)
+
         """
         if isinstance(fname, (STRING_TYPES, os_PathLike)):
-            # assume file name, cannot use os.path.exist to check, file can be from URL.
-            _check_call(_LIB.XGBoosterLoadModel(self.handle, c_str(os_fspath(fname))))
+            # assume file name, cannot use os.path.exist to check, file can be
+            # from URL.
+            _check_call(_LIB.XGBoosterLoadModel(
+                self.handle, c_str(os_fspath(fname))))
         else:
-            buf = fname
-            length = c_bst_ulong(len(buf))
-            ptr = (ctypes.c_char * len(buf)).from_buffer(buf)
-            _check_call(_LIB.XGBoosterLoadModelFromBuffer(self.handle, ptr, length))
+            raise TypeError('Unknown file type: ', fname)
 
     def dump_model(self, fout, fmap='', with_stats=False, dump_format="text"):
         """Dump model into a text or JSON file.
diff --git a/python-package/xgboost/training.py b/python-package/xgboost/training.py
index 070bb88cb..f76a607d0 100644
--- a/python-package/xgboost/training.py
+++ b/python-package/xgboost/training.py
@@ -34,9 +34,8 @@ def _train_internal(params, dtrain,
     num_parallel_tree = 1
 
     if xgb_model is not None:
-        if not isinstance(xgb_model, STRING_TYPES):
-            xgb_model = xgb_model.save_raw()
-        bst = Booster(params, [dtrain] + [d[0] for d in evals], model_file=xgb_model)
+        bst = Booster(params, [dtrain] + [d[0] for d in evals],
+                      model_file=xgb_model)
         nboost = len(bst.get_dump())
 
     _params = dict(params) if isinstance(params, list) else params
diff --git a/src/c_api/c_api.cc b/src/c_api/c_api.cc
index 2a8cd282e..baa3dfc54 100644
--- a/src/c_api/c_api.cc
+++ b/src/c_api/c_api.cc
@@ -458,8 +458,8 @@ XGB_DLL int XGDMatrixNumCol(const DMatrixHandle handle,
 
 // xgboost implementation
 XGB_DLL int XGBoosterCreate(const DMatrixHandle dmats[],
-                    xgboost::bst_ulong len,
-                    BoosterHandle *out) {
+                            xgboost::bst_ulong len,
+                            BoosterHandle *out) {
   API_BEGIN();
   std::vector<std::shared_ptr<DMatrix> > mats;
   for (xgboost::bst_ulong i = 0; i < len; ++i) {
@@ -485,6 +485,31 @@ XGB_DLL int XGBoosterSetParam(BoosterHandle handle,
   API_END();
 }
 
+XGB_DLL int XGBoosterLoadJsonConfig(BoosterHandle handle, char const* json_parameters) {
+  API_BEGIN();
+  CHECK_HANDLE();
+  std::string str {json_parameters};
+  Json config { Json::Load(StringView{str.c_str(), str.size()}) };
+  static_cast<Learner*>(handle)->LoadConfig(config);
+  API_END();
+}
+
+XGB_DLL int XGBoosterSaveJsonConfig(BoosterHandle handle,
+                                    xgboost::bst_ulong *out_len,
+                                    char const** out_str) {
+  API_BEGIN();
+  CHECK_HANDLE();
+  Json config { Object() };
+  auto* learner = static_cast<Learner*>(handle);
+  learner->Configure();
+  learner->SaveConfig(&config);
+  std::string& raw_str = XGBAPIThreadLocalStore::Get()->ret_str;
+  Json::Dump(config, &raw_str);
+  *out_str = raw_str.c_str();
+  *out_len = static_cast<xgboost::bst_ulong>(raw_str.length());
+  API_END();
+}
+
 XGB_DLL int XGBoosterUpdateOneIter(BoosterHandle handle,
                                    int iter,
                                    DMatrixHandle dtrain) {
@@ -579,7 +604,7 @@ XGB_DLL int XGBoosterLoadModel(BoosterHandle handle, const char* fname) {
     static_cast<Learner*>(handle)->LoadModel(in);
   } else {
     std::unique_ptr<dmlc::Stream> fi(dmlc::Stream::Create(fname, "r"));
-    static_cast<Learner*>(handle)->Load(fi.get());
+    static_cast<Learner*>(handle)->LoadModel(fi.get());
   }
   API_END();
 }
@@ -598,20 +623,18 @@ XGB_DLL int XGBoosterSaveModel(BoosterHandle handle, const char* c_fname) {
     fo->Write(str.c_str(), str.size());
   } else {
     auto *bst = static_cast<Learner*>(handle);
-    bst->Save(fo.get());
+    bst->SaveModel(fo.get());
   }
   API_END();
 }
 
-// The following two functions are `Load` and `Save` for memory based serialization
-// methods. E.g. Python pickle.
 XGB_DLL int XGBoosterLoadModelFromBuffer(BoosterHandle handle,
                                          const void* buf,
                                          xgboost::bst_ulong len) {
   API_BEGIN();
   CHECK_HANDLE();
   common::MemoryFixSizeBuffer fs((void*)buf, len);  // NOLINT(*)
-  static_cast<Learner*>(handle)->Load(&fs);
+  static_cast<Learner*>(handle)->LoadModel(&fs);
   API_END();
 }
 
@@ -621,6 +644,25 @@ XGB_DLL int XGBoosterGetModelRaw(BoosterHandle handle,
   std::string& raw_str = XGBAPIThreadLocalStore::Get()->ret_str;
   raw_str.resize(0);
 
+  API_BEGIN();
+  CHECK_HANDLE();
+  common::MemoryBufferStream fo(&raw_str);
+  auto *learner = static_cast<Learner*>(handle);
+  learner->Configure();
+  learner->SaveModel(&fo);
+  *out_dptr = dmlc::BeginPtr(raw_str);
+  *out_len = static_cast<xgboost::bst_ulong>(raw_str.length());
+  API_END();
+}
+
+// The following two functions are `Load` and `Save` for memory based
+// serialization methods. E.g. Python pickle.
+XGB_DLL int XGBoosterSerializeToBuffer(BoosterHandle handle,
+                                       xgboost::bst_ulong *out_len,
+                                       const char **out_dptr) {
+  std::string &raw_str = XGBAPIThreadLocalStore::Get()->ret_str;
+  raw_str.resize(0);
+
   API_BEGIN();
   CHECK_HANDLE();
   common::MemoryBufferStream fo(&raw_str);
@@ -632,6 +674,41 @@ XGB_DLL int XGBoosterGetModelRaw(BoosterHandle handle,
   API_END();
 }
 
+XGB_DLL int XGBoosterUnserializeFromBuffer(BoosterHandle handle,
+                                           const void *buf,
+                                           xgboost::bst_ulong len) {
+  API_BEGIN();
+  CHECK_HANDLE();
+  common::MemoryFixSizeBuffer fs((void*)buf, len);  // NOLINT(*)
+  static_cast<Learner*>(handle)->Load(&fs);
+  API_END();
+}
+
+XGB_DLL int XGBoosterLoadRabitCheckpoint(BoosterHandle handle,
+                                         int* version) {
+  API_BEGIN();
+  CHECK_HANDLE();
+  auto* bst = static_cast<Learner*>(handle);
+  *version = rabit::LoadCheckPoint(bst);
+  if (*version != 0) {
+    bst->Configure();
+  }
+  API_END();
+}
+
+XGB_DLL int XGBoosterSaveRabitCheckpoint(BoosterHandle handle) {
+  API_BEGIN();
+  CHECK_HANDLE();
+  auto* learner = static_cast<Learner*>(handle);
+  learner->Configure();
+  if (learner->AllowLazyCheckPoint()) {
+    rabit::LazyCheckPoint(learner);
+  } else {
+    rabit::CheckPoint(learner);
+  }
+  API_END();
+}
+
 inline void XGBoostDumpModelImpl(
     BoosterHandle handle,
     const FeatureMap& fmap,
@@ -758,29 +835,5 @@ XGB_DLL int XGBoosterGetAttrNames(BoosterHandle handle,
   API_END();
 }
 
-XGB_DLL int XGBoosterLoadRabitCheckpoint(BoosterHandle handle,
-                                         int* version) {
-  API_BEGIN();
-  CHECK_HANDLE();
-  auto* bst = static_cast<Learner*>(handle);
-  *version = rabit::LoadCheckPoint(bst);
-  if (*version != 0) {
-    bst->Configure();
-  }
-  API_END();
-}
-
-XGB_DLL int XGBoosterSaveRabitCheckpoint(BoosterHandle handle) {
-  API_BEGIN();
-  CHECK_HANDLE();
-  auto* bst = static_cast<Learner*>(handle);
-  if (bst->AllowLazyCheckPoint()) {
-    rabit::LazyCheckPoint(bst);
-  } else {
-    rabit::CheckPoint(bst);
-  }
-  API_END();
-}
-
 // force link rabit
 static DMLC_ATTRIBUTE_UNUSED int XGBOOST_LINK_RABIT_C_API_ = RabitLinkTag();
diff --git a/src/gbm/gblinear.cc b/src/gbm/gblinear.cc
index d165b16c4..46a1706e4 100644
--- a/src/gbm/gblinear.cc
+++ b/src/gbm/gblinear.cc
@@ -99,6 +99,16 @@ class GBLinear : public GradientBooster {
     model_.LoadModel(model);
   }
 
+  void LoadConfig(Json const& in) override {
+    CHECK_EQ(get<String>(in["name"]), "gblinear");
+    fromJson(in["gblinear_train_param"], &param_);
+  }
+  void SaveConfig(Json* p_out) const override {
+    auto& out = *p_out;
+    out["name"] = String{"gblinear"};
+    out["gblinear_train_param"] = toJson(param_);
+  }
+
   void DoBoost(DMatrix *p_fmat,
                HostDeviceVector<GradientPair> *in_gpair,
                ObjFunction* obj) override {
diff --git a/src/gbm/gblinear_model.h b/src/gbm/gblinear_model.h
index 853f2377c..71b8bcd06 100644
--- a/src/gbm/gblinear_model.h
+++ b/src/gbm/gblinear_model.h
@@ -112,7 +112,8 @@ class GBLinearModel : public Model {
          << "    \"weight\": [" << std::endl;
       for (unsigned i = 0; i < nfeature; ++i) {
         for (int gid = 0; gid < ngroup; ++gid) {
-          if (i != 0 || gid != 0) fo << "," << std::endl;
+          if (i != 0 || gid != 0)
+            fo << "," << std::endl;
           fo << "      " << (*this)[i][gid];
         }
       }
@@ -134,5 +135,6 @@ class GBLinearModel : public Model {
     return v;
   }
 };
+
 }  // namespace gbm
 }  // namespace xgboost
diff --git a/src/gbm/gbtree.cc b/src/gbm/gbtree.cc
index 15f950836..f516f8e2a 100644
--- a/src/gbm/gbtree.cc
+++ b/src/gbm/gbtree.cc
@@ -34,6 +34,7 @@ DMLC_REGISTRY_FILE_TAG(gbtree);
 
 void GBTree::Configure(const Args& cfg) {
   this->cfg_ = cfg;
+  std::string updater_seq = tparam_.updater_seq;
   tparam_.UpdateAllowUnknown(cfg);
 
   model_.Configure(cfg);
@@ -75,24 +76,31 @@ void GBTree::Configure(const Args& cfg) {
         "`tree_method` parameter instead.";
     // Don't drive users to silent XGBOost.
     showed_updater_warning_ = true;
-  } else {
-    this->ConfigureUpdaters();
-    LOG(DEBUG) << "Using updaters: " << tparam_.updater_seq;
   }
 
-  for (auto& up : updaters_) {
-    up->Configure(cfg);
+  this->ConfigureUpdaters();
+  if (updater_seq != tparam_.updater_seq) {
+    updaters_.clear();
+    this->InitUpdater(cfg);
+  } else {
+    for (auto &up : updaters_) {
+      up->Configure(cfg);
+    }
   }
 
   configured_ = true;
 }
 
-// FIXME(trivialfis): This handles updaters and predictor.  Because the choice of updaters
-// depends on whether external memory is used and how large is dataset.  We can remove the
-// dependency on DMatrix once `hist` tree method can handle external memory so that we can
-// make it default.
+// FIXME(trivialfis): This handles updaters.  Because the choice of updaters depends on
+// whether external memory is used and how large is dataset.  We can remove the dependency
+// on DMatrix once `hist` tree method can handle external memory so that we can make it
+// default.
 void GBTree::ConfigureWithKnownData(Args const& cfg, DMatrix* fmat) {
+  CHECK(this->configured_);
   std::string updater_seq = tparam_.updater_seq;
+  CHECK(tparam_.GetInitialised());
+
+  tparam_.UpdateAllowUnknown(cfg);
 
   this->PerformTreeMethodHeuristic(fmat);
   this->ConfigureUpdaters();
@@ -101,9 +109,8 @@ void GBTree::ConfigureWithKnownData(Args const& cfg, DMatrix* fmat) {
   if (updater_seq != tparam_.updater_seq) {
     LOG(DEBUG) << "Using updaters: " << tparam_.updater_seq;
     this->updaters_.clear();
+    this->InitUpdater(cfg);
   }
-
-  this->InitUpdater(cfg);
 }
 
 void GBTree::PerformTreeMethodHeuristic(DMatrix* fmat) {
@@ -141,6 +148,9 @@ void GBTree::PerformTreeMethodHeuristic(DMatrix* fmat) {
 }
 
 void GBTree::ConfigureUpdaters() {
+  if (specified_updater_) {
+    return;
+  }
   // `updater` parameter was manually specified
   /* Choose updaters according to tree_method parameters */
   switch (tparam_.tree_method) {
@@ -289,6 +299,46 @@ void GBTree::CommitModel(std::vector<std::vector<std::unique_ptr<RegTree>>>&& ne
   monitor_.Stop("CommitModel");
 }
 
+void GBTree::LoadConfig(Json const& in) {
+  CHECK_EQ(get<String>(in["name"]), "gbtree");
+  fromJson(in["gbtree_train_param"], &tparam_);
+  int32_t const n_gpus = xgboost::common::AllVisibleGPUs();
+  if (n_gpus == 0 && tparam_.predictor == PredictorType::kGPUPredictor) {
+    tparam_.UpdateAllowUnknown(Args{{"predictor", "auto"}});
+  }
+  if (n_gpus == 0 && tparam_.tree_method == TreeMethod::kGPUHist) {
+    tparam_.UpdateAllowUnknown(Args{{"tree_method", "hist"}});
+    LOG(WARNING)
+        << "Loading from a raw memory buffer on CPU only machine.  "
+           "Change tree_method to hist.";
+  }
+
+  auto const& j_updaters = get<Object const>(in["updater"]);
+  updaters_.clear();
+  for (auto const& kv : j_updaters) {
+    std::unique_ptr<TreeUpdater> up(TreeUpdater::Create(kv.first, generic_param_));
+    up->LoadConfig(kv.second);
+    updaters_.push_back(std::move(up));
+  }
+
+  specified_updater_ = get<Boolean>(in["specified_updater"]);
+}
+
+void GBTree::SaveConfig(Json* p_out) const {
+  auto& out = *p_out;
+  out["name"] = String("gbtree");
+  out["gbtree_train_param"] = toJson(tparam_);
+  out["updater"] = Object();
+
+  auto& j_updaters = out["updater"];
+  for (auto const& up : updaters_) {
+    j_updaters[up->Name()] = Object();
+    auto& j_up = j_updaters[up->Name()];
+    up->SaveConfig(&j_up);
+  }
+  out["specified_updater"] = Boolean{specified_updater_};
+}
+
 void GBTree::LoadModel(Json const& in) {
   CHECK_EQ(get<String>(in["name"]), "gbtree");
   model_.LoadModel(in["model"]);
@@ -324,7 +374,7 @@ class Dart : public GBTree {
     for (size_t i = 0; i < weight_drop_.size(); ++i) {
       j_weight_drop[i] = Number(weight_drop_[i]);
     }
-    out["weight_drop"] = Array(j_weight_drop);
+    out["weight_drop"] = Array(std::move(j_weight_drop));
   }
   void LoadModel(Json const& in) override {
     CHECK_EQ(get<String>(in["name"]), "dart");
@@ -352,6 +402,21 @@ class Dart : public GBTree {
     }
   }
 
+  void LoadConfig(Json const& in) override {
+    CHECK_EQ(get<String>(in["name"]), "dart");
+    auto const& gbtree = in["gbtree"];
+    GBTree::LoadConfig(gbtree);
+    fromJson(in["dart_train_param"], &dparam_);
+  }
+  void SaveConfig(Json* p_out) const override {
+    auto& out = *p_out;
+    out["name"] = String("dart");
+    out["gbtree"] = Object();
+    auto& gbtree = out["gbtree"];
+    GBTree::SaveConfig(&gbtree);
+    out["dart_train_param"] = toJson(dparam_);
+  }
+
   // predict the leaf scores with dropout if ntree_limit = 0
   void PredictBatch(DMatrix* p_fmat,
                     HostDeviceVector<bst_float>* out_preds,
diff --git a/src/gbm/gbtree.h b/src/gbm/gbtree.h
index b2ca62e43..09f1c4f0a 100644
--- a/src/gbm/gbtree.h
+++ b/src/gbm/gbtree.h
@@ -192,6 +192,9 @@ class GBTree : public GradientBooster {
     model_.Save(fo);
   }
 
+  void LoadConfig(Json const& in) override;
+  void SaveConfig(Json* p_out) const override;
+
   void SaveModel(Json* p_out) const override;
   void LoadModel(Json const& in) override;
 
diff --git a/src/gbm/gbtree_model.cc b/src/gbm/gbtree_model.cc
index f4125880e..f2ab0e6fe 100644
--- a/src/gbm/gbtree_model.cc
+++ b/src/gbm/gbtree_model.cc
@@ -46,7 +46,8 @@ void GBTreeModel::SaveModel(Json* p_out) const {
   for (auto const& tree : trees) {
     Json tree_json{Object()};
     tree->SaveModel(&tree_json);
-    tree_json["id"] = std::to_string(t);
+    // The field is not used in XGBoost, but might be useful for external project.
+    tree_json["id"] = Integer(t);
     trees_json.emplace_back(tree_json);
     t++;
   }
diff --git a/src/learner.cc b/src/learner.cc
index 2386914c5..f6f200774 100644
--- a/src/learner.cc
+++ b/src/learner.cc
@@ -30,6 +30,7 @@
 
 #include "common/common.h"
 #include "common/io.h"
+#include "common/observer.h"
 #include "common/random.h"
 #include "common/timer.h"
 #include "common/version.h"
@@ -37,27 +38,6 @@
 namespace {
 
 const char* kMaxDeltaStepDefaultValue = "0.7";
-
-inline bool IsFloat(const std::string& str) {
-  std::stringstream ss(str);
-  float f{};
-  return !((ss >> std::noskipws >> f).rdstate() ^ std::ios_base::eofbit);
-}
-
-inline bool IsInt(const std::string& str) {
-  std::stringstream ss(str);
-  int i{};
-  return !((ss >> std::noskipws >> i).rdstate() ^ std::ios_base::eofbit);
-}
-
-inline std::string RenderParamVal(const std::string& str) {
-  if (IsFloat(str) || IsInt(str)) {
-    return str;
-  } else {
-    return std::string("'") + str + "'";
-  }
-}
-
 }  // anonymous namespace
 
 namespace xgboost {
@@ -323,11 +303,77 @@ class LearnerImpl : public Learner {
     }
   }
 
-  void Load(dmlc::Stream* fi) override {
+  void LoadConfig(Json const& in) override {
+    CHECK(IsA<Object>(in));
+    Version::Load(in, true);
+
+    auto const& learner_parameters = get<Object>(in["learner"]);
+    fromJson(learner_parameters.at("learner_train_param"), &tparam_);
+
+    auto const& gradient_booster = learner_parameters.at("gradient_booster");
+
+    auto const& objective_fn = learner_parameters.at("objective");
+    if (!obj_) {
+      obj_.reset(ObjFunction::Create(tparam_.objective, &generic_parameters_));
+    }
+    obj_->LoadConfig(objective_fn);
+
+    tparam_.booster = get<String>(gradient_booster["name"]);
+    if (!gbm_) {
+      gbm_.reset(GradientBooster::Create(tparam_.booster,
+                                         &generic_parameters_, &learner_model_param_,
+                                         cache_));
+    }
+    gbm_->LoadConfig(gradient_booster);
+
+    auto const& j_metrics = learner_parameters.at("metrics");
+    auto n_metrics = get<Array const>(j_metrics).size();
+    metric_names_.resize(n_metrics);
+    metrics_.resize(n_metrics);
+    for (size_t i = 0; i < n_metrics; ++i) {
+      metric_names_[i]= get<String>(j_metrics[i]);
+      metrics_[i] = std::unique_ptr<Metric>(
+          Metric::Create(metric_names_.back(), &generic_parameters_));
+    }
+
+    fromJson(learner_parameters.at("generic_param"), &generic_parameters_);
+
+    this->need_configuration_ = true;
+  }
+
+  void SaveConfig(Json* p_out) const override {
+    CHECK(!this->need_configuration_) << "Call Configure before saving model.";
+    Version::Save(p_out);
+    Json& out { *p_out };
+    // parameters
+    out["learner"] = Object();
+    auto& learner_parameters = out["learner"];
+
+    learner_parameters["learner_train_param"] = toJson(tparam_);
+    learner_parameters["gradient_booster"] = Object();
+    auto& gradient_booster = learner_parameters["gradient_booster"];
+    gbm_->SaveConfig(&gradient_booster);
+
+    learner_parameters["objective"] = Object();
+    auto& objective_fn = learner_parameters["objective"];
+    obj_->SaveConfig(&objective_fn);
+
+    std::vector<Json> metrics(metrics_.size());
+    for (size_t i = 0; i < metrics_.size(); ++i) {
+      metrics[i] = String(metrics_[i]->Name());
+    }
+    learner_parameters["metrics"] = Array(metrics);
+
+    learner_parameters["generic_param"] = toJson(generic_parameters_);
+  }
+
+  // About to be deprecated by JSON format
+  void LoadModel(dmlc::Stream* fi) override {
     generic_parameters_.UpdateAllowUnknown(Args{});
     tparam_.Init(std::vector<std::pair<std::string, std::string>>{});
     // TODO(tqchen) mark deprecation of old format.
     common::PeekableInStream fp(fi);
+
     // backward compatible header check.
     std::string header;
     header.resize(4);
@@ -338,6 +384,15 @@ class LearnerImpl : public Learner {
         CHECK_EQ(fp.Read(&header[0], 4), 4U);
       }
     }
+
+    if (header[0] == '{') {
+      auto json_stream = common::FixedSizeStream(&fp);
+      std::string buffer;
+      json_stream.Take(&buffer);
+      auto model = Json::Load({buffer.c_str(), buffer.size()});
+      this->LoadModel(model);
+      return;
+    }
     // use the peekable reader.
     fi = &fp;
     // read parameter
@@ -370,43 +425,9 @@ class LearnerImpl : public Learner {
       std::vector<std::pair<std::string, std::string> > attr;
       fi->Read(&attr);
       for (auto& kv : attr) {
-        // Load `predictor`, `gpu_id` parameters from extra attributes
         const std::string prefix = "SAVED_PARAM_";
         if (kv.first.find(prefix) == 0) {
           const std::string saved_param = kv.first.substr(prefix.length());
-          bool is_gpu_predictor = saved_param == "predictor" && kv.second == "gpu_predictor";
-#ifdef XGBOOST_USE_CUDA
-          if (saved_param == "predictor" || saved_param == "gpu_id") {
-            cfg_[saved_param] = kv.second;
-            LOG(INFO)
-              << "Parameter '" << saved_param << "' has been recovered from "
-              << "the saved model. It will be set to "
-              << RenderParamVal(kv.second) << " for prediction. To "
-              << "override the predictor behavior, explicitly set '"
-              << saved_param << "' parameter as follows:\n"
-              << "  * Python package: bst.set_param('"
-              << saved_param << "', [new value])\n"
-              << "  * R package:      xgb.parameters(bst) <- list("
-              << saved_param << " = [new value])\n"
-              << "  * JVM packages:   bst.setParam(\""
-              << saved_param << "\", [new value])";
-          }
-#else
-          if (is_gpu_predictor) {
-            cfg_["predictor"] = "cpu_predictor";
-            kv.second = "cpu_predictor";
-          }
-#endif  // XGBOOST_USE_CUDA
-#if defined(XGBOOST_USE_CUDA)
-          // NO visible GPU in current environment
-          if (is_gpu_predictor && common::AllVisibleGPUs() == 0) {
-            cfg_["predictor"] = "cpu_predictor";
-            kv.second = "cpu_predictor";
-            LOG(INFO) << "Switch gpu_predictor to cpu_predictor.";
-          } else if (is_gpu_predictor) {
-            cfg_["predictor"] = "gpu_predictor";
-          }
-#endif  // defined(XGBOOST_USE_CUDA)
           if (saved_configs_.find(saved_param) != saved_configs_.end()) {
             cfg_[saved_param] = kv.second;
           }
@@ -447,26 +468,12 @@ class LearnerImpl : public Learner {
       tparam_.dsplit = DataSplitMode::kRow;
     }
 
-    // There's no logic for state machine for binary IO, as it has a mix of everything and
-    // half loaded model.
     this->Configure();
   }
 
-  // rabit save model to rabit checkpoint
-  void Save(dmlc::Stream* fo) const override {
-    if (this->need_configuration_) {
-      // Save empty model.  Calling Configure in a dummy LearnerImpl avoids violating
-      // constness.
-      LearnerImpl empty(std::move(this->cache_));
-      empty.SetParams({this->cfg_.cbegin(), this->cfg_.cend()});
-      for (auto const& kv : attributes_) {
-        empty.SetAttr(kv.first, kv.second);
-      }
-      empty.Configure();
-      empty.Save(fo);
-      return;
-    }
-
+  // Save model into binary format.  The code is about to be deprecated by more robust
+  // JSON serialization format.
+  void SaveModel(dmlc::Stream* fo) const override {
     LearnerModelParamLegacy mparam = mparam_;  // make a copy to potentially modify
     std::vector<std::pair<std::string, std::string> > extra_attr;
     // extra attributed to be added just before saving
@@ -479,14 +486,13 @@ class LearnerImpl : public Learner {
       }
     }
     {
-      std::vector<std::string> saved_params{"predictor", "gpu_id"};
+      std::vector<std::string> saved_params;
       // check if rabit_bootstrap_cache were set to non zero before adding to checkpoint
       if (cfg_.find("rabit_bootstrap_cache") != cfg_.end() &&
         (cfg_.find("rabit_bootstrap_cache"))->second != "0") {
         std::copy(saved_configs_.begin(), saved_configs_.end(),
                   std::back_inserter(saved_params));
       }
-      // Write `predictor`, `n_gpus`, `gpu_id` parameters as extra attributes
       for (const auto& key : saved_params) {
         auto it = cfg_.find(key);
         if (it != cfg_.end()) {
@@ -495,19 +501,6 @@ class LearnerImpl : public Learner {
         }
       }
     }
-#if defined(XGBOOST_USE_CUDA)
-    {
-      // Force save gpu_id.
-      if (std::none_of(extra_attr.cbegin(), extra_attr.cend(),
-                   [](std::pair<std::string, std::string> const& it) {
-                     return it.first == "SAVED_PARAM_gpu_id";
-                   })) {
-        mparam.contain_extra_attrs = 1;
-        extra_attr.emplace_back("SAVED_PARAM_gpu_id",
-                                std::to_string(generic_parameters_.gpu_id));
-      }
-    }
-#endif  // defined(XGBOOST_USE_CUDA)
     fo->Write(&mparam, sizeof(LearnerModelParamLegacy));
     fo->Write(tparam_.objective);
     fo->Write(tparam_.booster);
@@ -541,6 +534,69 @@ class LearnerImpl : public Learner {
     }
   }
 
+  void Save(dmlc::Stream* fo) const override {
+    if (generic_parameters_.enable_experimental_json_serialization) {
+      Json memory_snapshot{Object()};
+      memory_snapshot["Model"] = Object();
+      auto &model = memory_snapshot["Model"];
+      this->SaveModel(&model);
+      memory_snapshot["Config"] = Object();
+      auto &config = memory_snapshot["Config"];
+      this->SaveConfig(&config);
+      std::string out_str;
+      Json::Dump(memory_snapshot, &out_str);
+      fo->Write(out_str.c_str(), out_str.size());
+    } else {
+      std::string binary_buf;
+      common::MemoryBufferStream s(&binary_buf);
+      this->SaveModel(&s);
+      Json config{ Object() };
+      // Do not use std::size_t as it's not portable.
+      int64_t const json_offset = binary_buf.size();
+      this->SaveConfig(&config);
+      std::string config_str;
+      Json::Dump(config, &config_str);
+      // concatonate the model and config at final output, it's a temporary solution for
+      // continuing support for binary model format
+      fo->Write(&serialisation_header_[0], serialisation_header_.size());
+      fo->Write(&json_offset, sizeof(json_offset));
+      fo->Write(&binary_buf[0], binary_buf.size());
+      fo->Write(&config_str[0], config_str.size());
+    }
+  }
+
+  void Load(dmlc::Stream* fi) override {
+    common::PeekableInStream fp(fi);
+    char c {0};
+    fp.PeekRead(&c, 1);
+    if (c == '{') {
+      std::string buffer;
+      common::FixedSizeStream{&fp}.Take(&buffer);
+      auto memory_snapshot = Json::Load({buffer.c_str(), buffer.size()});
+      this->LoadModel(memory_snapshot["Model"]);
+      this->LoadConfig(memory_snapshot["Config"]);
+    } else {
+      std::string header;
+      header.resize(serialisation_header_.size());
+      CHECK_EQ(fp.Read(&header[0], header.size()), serialisation_header_.size());
+      CHECK_EQ(header, serialisation_header_);
+
+      int64_t json_offset {-1};
+      CHECK_EQ(fp.Read(&json_offset, sizeof(json_offset)), sizeof(json_offset));
+      CHECK_GT(json_offset, 0);
+      std::string buffer;
+      common::FixedSizeStream{&fp}.Take(&buffer);
+
+      common::MemoryFixSizeBuffer binary_buf(&buffer[0], json_offset);
+      this->LoadModel(&binary_buf);
+
+      common::MemoryFixSizeBuffer json_buf {&buffer[0] + json_offset,
+                                            buffer.size() - json_offset};
+      auto config = Json::Load({buffer.c_str() + json_offset, buffer.size() - json_offset});
+      this->LoadConfig(config);
+    }
+  }
+
   std::vector<std::string> DumpModel(const FeatureMap& fmap,
                                      bool with_stats,
                                      std::string format) const override {
@@ -551,6 +607,7 @@ class LearnerImpl : public Learner {
 
   void UpdateOneIter(int iter, DMatrix* train) override {
     monitor_.Start("UpdateOneIter");
+    TrainingObserver::Instance().Update(iter);
     this->Configure();
     if (generic_parameters_.seed_per_iteration || rabit::IsDistributed()) {
       common::GlobalRandom().seed(generic_parameters_.seed * kRandSeedMagic + iter);
@@ -561,9 +618,13 @@ class LearnerImpl : public Learner {
     monitor_.Start("PredictRaw");
     this->PredictRaw(train, &preds_[train]);
     monitor_.Stop("PredictRaw");
+    TrainingObserver::Instance().Observe(preds_[train], "Predictions");
+
     monitor_.Start("GetGradient");
     obj_->GetGradient(preds_[train], train->Info(), iter, &gpair_);
     monitor_.Stop("GetGradient");
+    TrainingObserver::Instance().Observe(gpair_, "Gradients");
+
     gbm_->DoBoost(train, &gpair_, obj_.get());
     monitor_.Stop("UpdateOneIter");
   }
@@ -792,6 +853,10 @@ class LearnerImpl : public Learner {
   LearnerModelParamLegacy mparam_;
   LearnerModelParam learner_model_param_;
   LearnerTrainParam tparam_;
+  // Used to identify the offset of JSON string when
+  // `enable_experimental_json_serialization' is set to false.  Will be removed once JSON
+  // takes over.
+  std::string const serialisation_header_ { u8"CONFIG-offset:" };
   // configurations
   std::map<std::string, std::string> cfg_;
   std::map<std::string, std::string> attributes_;
@@ -811,9 +876,8 @@ class LearnerImpl : public Learner {
 
   common::Monitor monitor_;
 
-  /*! \brief saved config keys used to restore failed worker */
-  std::set<std::string> saved_configs_ = {"max_depth", "tree_method", "dsplit",
-    "seed", "silent", "num_round", "gamma", "min_child_weight"};
+  /*! \brief (Deprecated) saved config keys used to restore failed worker */
+  std::set<std::string> saved_configs_ = {"num_round"};
 };
 
 std::string const LearnerImpl::kEvalMetric {"eval_metric"};  // NOLINT
diff --git a/src/tree/tree_model.cc b/src/tree/tree_model.cc
index 49b2178c9..c3752f337 100644
--- a/src/tree/tree_model.cc
+++ b/src/tree/tree_model.cc
@@ -682,13 +682,13 @@ void RegTree::LoadModel(Json const& in) {
     s.leaf_child_cnt = get<Integer const>(leaf_child_counts[i]);
 
     auto& n = nodes_[i];
-    auto left = get<Integer const>(lefts[i]);
-    auto right = get<Integer const>(rights[i]);
-    auto parent = get<Integer const>(parents[i]);
-    auto ind = get<Integer const>(indices[i]);
-    auto cond = get<Number const>(conds[i]);
-    auto dft_left = get<Boolean const>(default_left[i]);
-    n = Node(left, right, parent, ind, cond, dft_left);
+    bst_node_t left = get<Integer const>(lefts[i]);
+    bst_node_t right = get<Integer const>(rights[i]);
+    bst_node_t parent = get<Integer const>(parents[i]);
+    bst_feature_t ind = get<Integer const>(indices[i]);
+    float cond { get<Number const>(conds[i]) };
+    bool dft_left { get<Boolean const>(default_left[i]) };
+    n = Node{left, right, parent, ind, cond, dft_left};
   }
 
 
diff --git a/src/tree/updater_gpu_hist.cu b/src/tree/updater_gpu_hist.cu
index 8cc016721..f8b7731c9 100644
--- a/src/tree/updater_gpu_hist.cu
+++ b/src/tree/updater_gpu_hist.cu
@@ -1027,8 +1027,6 @@ class GPUHistMakerSpecialised {
     param_.UpdateAllowUnknown(args);
     generic_param_ = generic_param;
     hist_maker_param_.UpdateAllowUnknown(args);
-    device_ = generic_param_->gpu_id;
-    CHECK_GE(device_, 0) << "Must have at least one device";
     dh::CheckComputeCapability();
 
     monitor_.Init("updater_gpu_hist");
@@ -1041,6 +1039,7 @@ class GPUHistMakerSpecialised {
   void Update(HostDeviceVector<GradientPair>* gpair, DMatrix* dmat,
               const std::vector<RegTree*>& trees) {
     monitor_.StartCuda("Update");
+
     // rescale learning rate according to size of trees
     float lr = param_.learning_rate;
     param_.learning_rate = lr / trees.size();
@@ -1064,6 +1063,8 @@ class GPUHistMakerSpecialised {
   }
 
   void InitDataOnce(DMatrix* dmat) {
+    device_ = generic_param_->gpu_id;
+    CHECK_GE(device_, 0) << "Must have at least one device";
     info_ = &dmat->Info();
     reducer_.Init({device_});
 
@@ -1162,14 +1163,24 @@ class GPUHistMakerSpecialised {
 class GPUHistMaker : public TreeUpdater {
  public:
   void Configure(const Args& args) override {
+    // Used in test to count how many configurations are performed
+    LOG(DEBUG) << "[GPU Hist]: Configure";
     hist_maker_param_.UpdateAllowUnknown(args);
-    float_maker_.reset();
-    double_maker_.reset();
+    // The passed in args can be empty, if we simply purge the old maker without
+    // preserving parameters then we can't do Update on it.
+    TrainParam param;
+    if (float_maker_) {
+      param = float_maker_->param_;
+    } else if (double_maker_) {
+      param = double_maker_->param_;
+    }
     if (hist_maker_param_.single_precision_histogram) {
       float_maker_.reset(new GPUHistMakerSpecialised<GradientPair>());
+      float_maker_->param_ = param;
       float_maker_->Configure(args, tparam_);
     } else {
       double_maker_.reset(new GPUHistMakerSpecialised<GradientPairPrecise>());
+      double_maker_->param_ = param;
       double_maker_->Configure(args, tparam_);
     }
   }
diff --git a/tests/cpp/c_api/test_c_api.cc b/tests/cpp/c_api/test_c_api.cc
index a54cb9bd3..1371d01e7 100644
--- a/tests/cpp/c_api/test_c_api.cc
+++ b/tests/cpp/c_api/test_c_api.cc
@@ -8,6 +8,7 @@
 #include "../helpers.h"
 #include "../../../src/common/io.h"
 
+
 TEST(c_api, XGDMatrixCreateFromMatDT) {
   std::vector<int> col0 = {0, -1, 3};
   std::vector<float> col1 = {-4.0f, 2.0f, 0.0f};
@@ -77,7 +78,41 @@ TEST(c_api, Version) {
   ASSERT_EQ(patch, XGBOOST_VER_PATCH);
 }
 
-TEST(c_api, Json_ModelIO){
+TEST(c_api, ConfigIO) {
+  size_t constexpr kRows = 10;
+  auto pp_dmat = CreateDMatrix(kRows, 10, 0);
+  auto p_dmat = *pp_dmat;
+  std::vector<std::shared_ptr<DMatrix>> mat {p_dmat};
+  std::vector<bst_float> labels(kRows);
+  for (size_t i = 0; i < labels.size(); ++i) {
+    labels[i] = i;
+  }
+  p_dmat->Info().labels_.HostVector() = labels;
+
+  std::shared_ptr<Learner> learner { Learner::Create(mat) };
+
+  BoosterHandle handle = learner.get();
+  learner->UpdateOneIter(0, p_dmat.get());
+
+  char const* out[1];
+  bst_ulong len {0};
+  XGBoosterSaveJsonConfig(handle, &len, out);
+
+  std::string config_str_0 { out[0] };
+  auto config_0 = Json::Load({config_str_0.c_str(), config_str_0.size()});
+  XGBoosterLoadJsonConfig(handle, out[0]);
+
+  bst_ulong len_1 {0};
+  std::string config_str_1 { out[0] };
+  XGBoosterSaveJsonConfig(handle, &len_1, out);
+  auto config_1 = Json::Load({config_str_1.c_str(), config_str_1.size()});
+
+  ASSERT_EQ(config_0, config_1);
+
+  delete pp_dmat;
+}
+
+TEST(c_api, Json_ModelIO) {
   size_t constexpr kRows = 10;
   dmlc::TemporaryDirectory tempdir;
 
diff --git a/tests/cpp/gbm/test_gbtree.cc b/tests/cpp/gbm/test_gbtree.cc
index f0a77b789..58c8c44f2 100644
--- a/tests/cpp/gbm/test_gbtree.cc
+++ b/tests/cpp/gbm/test_gbtree.cc
@@ -117,15 +117,28 @@ TEST(GBTree, Json_IO) {
     CreateTrainedGBM("gbtree", Args{}, kRows, kCols, &mparam, &gparam) };
 
   Json model {Object()};
+  model["model"] = Object();
+  auto& j_model = model["model"];
 
-  gbm->SaveModel(&model);
+  model["config"] = Object();
+  auto& j_param = model["config"];
+
+  gbm->SaveModel(&j_model);
+  gbm->SaveConfig(&j_param);
 
   std::string model_str;
   Json::Dump(model, &model_str);
 
-  auto loaded_model = Json::Load(StringView{model_str.c_str(), model_str.size()});
-  ASSERT_EQ(get<String>(loaded_model["name"]), "gbtree");
-  ASSERT_TRUE(IsA<Object>(loaded_model["model"]["gbtree_model_param"]));
+  model = Json::Load({model_str.c_str(), model_str.size()});
+  ASSERT_EQ(get<String>(model["model"]["name"]), "gbtree");
+
+  auto const& gbtree_model = model["model"]["model"];
+  ASSERT_EQ(get<Array>(gbtree_model["trees"]).size(), 1);
+  ASSERT_EQ(get<Integer>(get<Object>(get<Array>(gbtree_model["trees"]).front()).at("id")), 0);
+  ASSERT_EQ(get<Array>(gbtree_model["tree_info"]).size(), 1);
+
+  auto j_train_param = model["config"]["gbtree_train_param"];
+  ASSERT_EQ(get<String>(j_train_param["num_parallel_tree"]), "1");
 }
 
 TEST(Dart, Json_IO) {
@@ -145,20 +158,21 @@ TEST(Dart, Json_IO) {
   Json model {Object()};
   model["model"] = Object();
   auto& j_model = model["model"];
-  model["parameters"] = Object();
+  model["config"] = Object();
+
+  auto& j_param = model["config"];
 
   gbm->SaveModel(&j_model);
+  gbm->SaveConfig(&j_param);
 
   std::string model_str;
   Json::Dump(model, &model_str);
 
   model = Json::Load({model_str.c_str(), model_str.size()});
 
-  {
-    auto const& gbtree = model["model"]["gbtree"];
-    ASSERT_TRUE(IsA<Object>(gbtree));
-    ASSERT_EQ(get<String>(model["model"]["name"]), "dart");
-    ASSERT_NE(get<Array>(model["model"]["weight_drop"]).size(), 0);
-  }
+  ASSERT_EQ(get<String>(model["model"]["name"]), "dart") << model;
+  ASSERT_EQ(get<String>(model["config"]["name"]), "dart");
+  ASSERT_TRUE(IsA<Object>(model["model"]["gbtree"]));
+  ASSERT_NE(get<Array>(model["model"]["weight_drop"]).size(), 0);
 }
 }  // namespace xgboost
diff --git a/tests/cpp/predictor/test_gpu_predictor.cu b/tests/cpp/predictor/test_gpu_predictor.cu
index 5f2ba1b23..ffc8743b4 100644
--- a/tests/cpp/predictor/test_gpu_predictor.cu
+++ b/tests/cpp/predictor/test_gpu_predictor.cu
@@ -13,23 +13,6 @@
 #include "../helpers.h"
 #include "../../../src/gbm/gbtree_model.h"
 
-namespace {
-
-inline void CheckCAPICall(int ret) {
-  ASSERT_EQ(ret, 0) << XGBGetLastError();
-}
-
-}  // namespace anonymous
-
-const std::map<std::string, std::string>&
-QueryBoosterConfigurationArguments(BoosterHandle handle) {
-  CHECK_NE(handle, static_cast<void*>(nullptr));
-  auto* bst = static_cast<xgboost::Learner*>(handle);
-  bst->Configure();
-  return bst->GetConfigurationArguments();
-}
-
-
 namespace xgboost {
 namespace predictor {
 
@@ -110,77 +93,5 @@ TEST(gpu_predictor, ExternalMemoryTest) {
     }
   }
 }
-
-// Test whether pickling preserves predictor parameters
-TEST(gpu_predictor, PicklingTest) {
-  int const gpuid = 0;
-
-  dmlc::TemporaryDirectory tempdir;
-  const std::string tmp_file = tempdir.path + "/simple.libsvm";
-  CreateBigTestData(tmp_file, 600);
-
-  DMatrixHandle dmat[1];
-  BoosterHandle bst, bst2;
-  std::vector<bst_float> label;
-  for (int i = 0; i < 200; ++i) {
-    label.push_back((i % 2 ? 1 : 0));
-  }
-
-  // Load data matrix
-  ASSERT_EQ(XGDMatrixCreateFromFile(
-      tmp_file.c_str(), 0, &dmat[0]), 0) << XGBGetLastError();
-  ASSERT_EQ(XGDMatrixSetFloatInfo(
-      dmat[0], "label", label.data(), 200), 0) << XGBGetLastError();
-  // Create booster
-  ASSERT_EQ(XGBoosterCreate(dmat, 1, &bst), 0) << XGBGetLastError();
-  // Set parameters
-  ASSERT_EQ(XGBoosterSetParam(bst, "seed", "0"), 0) << XGBGetLastError();
-  ASSERT_EQ(XGBoosterSetParam(bst, "base_score", "0.5"), 0) << XGBGetLastError();
-  ASSERT_EQ(XGBoosterSetParam(bst, "booster", "gbtree"), 0) << XGBGetLastError();
-  ASSERT_EQ(XGBoosterSetParam(bst, "learning_rate", "0.01"), 0) << XGBGetLastError();
-  ASSERT_EQ(XGBoosterSetParam(bst, "max_depth", "8"), 0) << XGBGetLastError();
-  ASSERT_EQ(XGBoosterSetParam(
-      bst, "objective", "binary:logistic"), 0) << XGBGetLastError();
-  ASSERT_EQ(XGBoosterSetParam(bst, "seed", "123"), 0) << XGBGetLastError();
-  ASSERT_EQ(XGBoosterSetParam(
-      bst, "tree_method", "gpu_hist"), 0) << XGBGetLastError();
-  ASSERT_EQ(XGBoosterSetParam(
-      bst, "gpu_id", std::to_string(gpuid).c_str()), 0) << XGBGetLastError();
-  ASSERT_EQ(XGBoosterSetParam(bst, "predictor", "gpu_predictor"), 0) << XGBGetLastError();
-
-  // Run boosting iterations
-  for (int i = 0; i < 10; ++i) {
-    ASSERT_EQ(XGBoosterUpdateOneIter(bst, i, dmat[0]), 0) << XGBGetLastError();
-  }
-
-  // Delete matrix
-  CheckCAPICall(XGDMatrixFree(dmat[0]));
-
-  // Pickle
-  const char* dptr;
-  bst_ulong len;
-  std::string buf;
-  CheckCAPICall(XGBoosterGetModelRaw(bst, &len, &dptr));
-  buf = std::string(dptr, len);
-  CheckCAPICall(XGBoosterFree(bst));
-
-  // Unpickle
-  CheckCAPICall(XGBoosterCreate(nullptr, 0, &bst2));
-  CheckCAPICall(XGBoosterLoadModelFromBuffer(bst2, buf.c_str(), len));
-
-  {  // Query predictor
-    const auto& kwargs = QueryBoosterConfigurationArguments(bst2);
-    ASSERT_EQ(kwargs.at("predictor"), "gpu_predictor");
-    ASSERT_EQ(kwargs.at("gpu_id"), std::to_string(gpuid).c_str());
-  }
-
-  {  // Change predictor and query again
-    CheckCAPICall(XGBoosterSetParam(bst2, "predictor", "cpu_predictor"));
-    const auto& kwargs = QueryBoosterConfigurationArguments(bst2);
-    ASSERT_EQ(kwargs.at("predictor"), "cpu_predictor");
-  }
-
-  CheckCAPICall(XGBoosterFree(bst2));
-}
 }  // namespace predictor
 }  // namespace xgboost
diff --git a/tests/python-gpu/load_pickle.py b/tests/python-gpu/load_pickle.py
index 45d20aae4..3d80a54a2 100644
--- a/tests/python-gpu/load_pickle.py
+++ b/tests/python-gpu/load_pickle.py
@@ -1,20 +1,39 @@
-'''Loading a pickled model generated by test_pickling.py'''
-import pickle
+'''Loading a pickled model generated by test_pickling.py, only used by
+`test_gpu_with_dask.py`'''
 import unittest
 import os
 import xgboost as xgb
-import sys
+import json
 
-sys.path.append("tests/python")
-from test_pickling import build_dataset, model_path
+from test_gpu_pickling import build_dataset, model_path, load_pickle
 
 
 class TestLoadPickle(unittest.TestCase):
     def test_load_pkl(self):
-        assert os.environ['CUDA_VISIBLE_DEVICES'] == ''
-        with open(model_path, 'rb') as fd:
-            bst = pickle.load(fd)
+        '''Test whether prediction is correct.'''
+        assert os.environ['CUDA_VISIBLE_DEVICES'] == '-1'
+        bst = load_pickle(model_path)
         x, y = build_dataset()
         test_x = xgb.DMatrix(x)
         res = bst.predict(test_x)
         assert len(res) == 10
+
+    def test_predictor_type_is_auto(self):
+        '''Under invalid CUDA_VISIBLE_DEVICES, predictor should be set to
+        auto'''
+        assert os.environ['CUDA_VISIBLE_DEVICES'] == '-1'
+        bst = load_pickle(model_path)
+        config = bst.save_config()
+        config = json.loads(config)
+        assert config['learner']['gradient_booster']['gbtree_train_param'][
+            'predictor'] == 'auto'
+
+    def test_predictor_type_is_gpu(self):
+        '''When CUDA_VISIBLE_DEVICES is not specified, keep using
+        `gpu_predictor`'''
+        assert 'CUDA_VISIBLE_DEVICES' not in os.environ.keys()
+        bst = load_pickle(model_path)
+        config = bst.save_config()
+        config = json.loads(config)
+        assert config['learner']['gradient_booster']['gbtree_train_param'][
+            'predictor'] == 'gpu_predictor'
diff --git a/tests/python-gpu/test_pickling.py b/tests/python-gpu/test_gpu_pickling.py
similarity index 56%
rename from tests/python-gpu/test_pickling.py
rename to tests/python-gpu/test_gpu_pickling.py
index 9c077e315..00ffb04b0 100644
--- a/tests/python-gpu/test_pickling.py
+++ b/tests/python-gpu/test_gpu_pickling.py
@@ -4,7 +4,7 @@ import unittest
 import numpy as np
 import subprocess
 import os
-import sys
+import json
 import xgboost as xgb
 from xgboost import XGBClassifier
 
@@ -39,18 +39,17 @@ class TestPickling(unittest.TestCase):
         bst = xgb.train(param, train_x)
 
         save_pickle(bst, model_path)
-        args = ["pytest",
-                "--verbose",
-                "-s",
-                "--fulltrace",
-                "./tests/python-gpu/load_pickle.py"]
+        args = [
+            "pytest", "--verbose", "-s", "--fulltrace",
+            "./tests/python-gpu/load_pickle.py::TestLoadPickle::test_load_pkl"
+        ]
         command = ''
         for arg in args:
             command += arg
             command += ' '
 
-        cuda_environment = {'CUDA_VISIBLE_DEVICES': ''}
-        env = os.environ
+        cuda_environment = {'CUDA_VISIBLE_DEVICES': '-1'}
+        env = os.environ.copy()
         # Passing new_environment directly to `env' argument results
         # in failure on Windows:
         #    Fatal Python error: _Py_HashRandomization_Init: failed to
@@ -62,12 +61,55 @@ class TestPickling(unittest.TestCase):
         assert status == 0
         os.remove(model_path)
 
+    def test_pickled_predictor(self):
+        args_templae = [
+            "pytest",
+            "--verbose",
+            "-s",
+            "--fulltrace"]
+
+        x, y = build_dataset()
+        train_x = xgb.DMatrix(x, label=y)
+
+        param = {'tree_method': 'gpu_hist',
+                 'verbosity': 1, 'predictor': 'gpu_predictor'}
+        bst = xgb.train(param, train_x)
+        config = json.loads(bst.save_config())
+        assert config['learner']['gradient_booster']['gbtree_train_param'][
+            'predictor'] == 'gpu_predictor'
+
+        save_pickle(bst, model_path)
+
+        args = args_templae.copy()
+        args.append(
+            "./tests/python-gpu/"
+            "load_pickle.py::TestLoadPickle::test_predictor_type_is_auto")
+
+        cuda_environment = {'CUDA_VISIBLE_DEVICES': '-1'}
+        env = os.environ.copy()
+        env.update(cuda_environment)
+
+        # Load model in a CPU only environment.
+        status = subprocess.call(args, env=env)
+        assert status == 0
+
+        args = args_templae.copy()
+        args.append(
+            "./tests/python-gpu/"
+            "load_pickle.py::TestLoadPickle::test_predictor_type_is_gpu")
+
+        # Load in environment that has GPU.
+        env = os.environ.copy()
+        assert 'CUDA_VISIBLE_DEVICES' not in env.keys()
+        status = subprocess.call(args, env=env)
+        assert status == 0
+
     def test_predict_sklearn_pickle(self):
         x, y = build_dataset()
 
         kwargs = {'tree_method': 'gpu_hist',
                   'predictor': 'gpu_predictor',
-                  'verbosity': 2,
+                  'verbosity': 1,
                   'objective': 'binary:logistic',
                   'n_estimators': 10}
 
diff --git a/tests/python-gpu/test_gpu_training_continuation.py b/tests/python-gpu/test_gpu_training_continuation.py
index 3d4b053df..ac52fe464 100644
--- a/tests/python-gpu/test_gpu_training_continuation.py
+++ b/tests/python-gpu/test_gpu_training_continuation.py
@@ -7,23 +7,25 @@ rng = np.random.RandomState(1994)
 
 
 class TestGPUTrainingContinuation(unittest.TestCase):
-    def test_training_continuation_binary(self):
-        kRows = 32
-        kCols = 16
+    def run_training_continuation(self, use_json):
+        kRows = 64
+        kCols = 32
         X = np.random.randn(kRows, kCols)
         y = np.random.randn(kRows)
         dtrain = xgb.DMatrix(X, y)
-        params = {'tree_method': 'gpu_hist', 'max_depth': '2'}
-        bst_0 = xgb.train(params, dtrain, num_boost_round=4)
+        params = {'tree_method': 'gpu_hist', 'max_depth': '2',
+                  'gamma': '0.1', 'alpha': '0.01',
+                  'enable_experimental_json_serialization': use_json}
+        bst_0 = xgb.train(params, dtrain, num_boost_round=64)
         dump_0 = bst_0.get_dump(dump_format='json')
 
-        bst_1 = xgb.train(params, dtrain, num_boost_round=2)
-        bst_1 = xgb.train(params, dtrain, num_boost_round=2, xgb_model=bst_1)
+        bst_1 = xgb.train(params, dtrain, num_boost_round=32)
+        bst_1 = xgb.train(params, dtrain, num_boost_round=32, xgb_model=bst_1)
         dump_1 = bst_1.get_dump(dump_format='json')
 
         def recursive_compare(obj_0, obj_1):
             if isinstance(obj_0, float):
-                assert np.isclose(obj_0, obj_1)
+                assert np.isclose(obj_0, obj_1, atol=1e-6)
             elif isinstance(obj_0, str):
                 assert obj_0 == obj_1
             elif isinstance(obj_0, int):
@@ -42,7 +44,14 @@ class TestGPUTrainingContinuation(unittest.TestCase):
                 for i in range(len(obj_0)):
                     recursive_compare(obj_0[i], obj_1[i])
 
+        assert len(dump_0) == len(dump_1)
         for i in range(len(dump_0)):
             obj_0 = json.loads(dump_0[i])
             obj_1 = json.loads(dump_1[i])
             recursive_compare(obj_0, obj_1)
+
+    def test_gpu_training_continuation_binary(self):
+        self.run_training_continuation(False)
+
+    def test_gpu_training_continuation_json(self):
+        self.run_training_continuation(True)
diff --git a/tests/python/test_basic_models.py b/tests/python/test_basic_models.py
index eb6b1a093..eb71fc2fa 100644
--- a/tests/python/test_basic_models.py
+++ b/tests/python/test_basic_models.py
@@ -203,7 +203,7 @@ class TestModels(unittest.TestCase):
         self.assertRaises(ValueError, bst.predict, dm1)
         bst.predict(dm2)  # success
 
-    def test_json_model_io(self):
+    def test_model_json_io(self):
         X = np.random.random((10, 3))
         y = np.random.randint(2, size=(10,))
 
diff --git a/tests/python/test_pickling.py b/tests/python/test_pickling.py
index 1497688d2..be4b9c743 100644
--- a/tests/python/test_pickling.py
+++ b/tests/python/test_pickling.py
@@ -2,6 +2,7 @@ import pickle
 import numpy as np
 import xgboost as xgb
 import os
+import unittest
 
 
 kRows = 100
@@ -14,35 +15,45 @@ def generate_data():
     return X, y
 
 
-def test_model_pickling():
-    xgb_params = {
-        'verbosity': 0,
-        'nthread': 1,
-        'tree_method': 'hist'
-    }
+class TestPickling(unittest.TestCase):
+    def run_model_pickling(self, xgb_params):
+        X, y = generate_data()
+        dtrain = xgb.DMatrix(X, y)
+        bst = xgb.train(xgb_params, dtrain)
 
-    X, y = generate_data()
-    dtrain = xgb.DMatrix(X, y)
-    bst = xgb.train(xgb_params, dtrain)
+        dump_0 = bst.get_dump(dump_format='json')
+        assert dump_0
 
-    dump_0 = bst.get_dump(dump_format='json')
-    assert dump_0
+        filename = 'model.pkl'
 
-    filename = 'model.pkl'
+        with open(filename, 'wb') as fd:
+            pickle.dump(bst, fd)
 
-    with open(filename, 'wb') as fd:
-        pickle.dump(bst, fd)
+        with open(filename, 'rb') as fd:
+            bst = pickle.load(fd)
 
-    with open(filename, 'rb') as fd:
-        bst = pickle.load(fd)
+        with open(filename, 'wb') as fd:
+            pickle.dump(bst, fd)
 
-    with open(filename, 'wb') as fd:
-        pickle.dump(bst, fd)
+        with open(filename, 'rb') as fd:
+            bst = pickle.load(fd)
 
-    with open(filename, 'rb') as fd:
-        bst = pickle.load(fd)
+        assert bst.get_dump(dump_format='json') == dump_0
 
-    assert bst.get_dump(dump_format='json') == dump_0
+        if os.path.exists(filename):
+            os.remove(filename)
 
-    if os.path.exists(filename):
-        os.remove(filename)
+    def test_model_pickling_binary(self):
+        params = {
+            'nthread': 1,
+            'tree_method': 'hist'
+        }
+        self.run_model_pickling(params)
+
+    def test_model_pickling_json(self):
+        params = {
+            'nthread': 1,
+            'tree_method': 'hist',
+            'enable_experimental_json_serialization': True
+        }
+        self.run_model_pickling(params)
diff --git a/tests/python/test_training_continuation.py b/tests/python/test_training_continuation.py
index 98a931bcd..5ebb11445 100644
--- a/tests/python/test_training_continuation.py
+++ b/tests/python/test_training_continuation.py
@@ -10,26 +10,35 @@ rng = np.random.RandomState(1337)
 class TestTrainingContinuation(unittest.TestCase):
     num_parallel_tree = 3
 
-    xgb_params_01 = {
-        'verbosity': 0,
-        'nthread': 1,
-    }
+    def generate_parameters(self, use_json):
+        xgb_params_01_binary = {
+            'nthread': 1,
+        }
 
-    xgb_params_02 = {
-        'verbosity': 0,
-        'nthread': 1,
-        'num_parallel_tree': num_parallel_tree
-    }
+        xgb_params_02_binary = {
+            'nthread': 1,
+            'num_parallel_tree': self.num_parallel_tree
+        }
 
-    xgb_params_03 = {
-        'verbosity': 0,
-        'nthread': 1,
-        'num_class': 5,
-        'num_parallel_tree': num_parallel_tree
-    }
+        xgb_params_03_binary = {
+            'nthread': 1,
+            'num_class': 5,
+            'num_parallel_tree': self.num_parallel_tree
+        }
+        if use_json:
+            xgb_params_01_binary[
+                'enable_experimental_json_serialization'] = True
+            xgb_params_02_binary[
+                'enable_experimental_json_serialization'] = True
+            xgb_params_03_binary[
+                'enable_experimental_json_serialization'] = True
 
-    @pytest.mark.skipif(**tm.no_sklearn())
-    def test_training_continuation(self):
+        return [
+            xgb_params_01_binary, xgb_params_02_binary, xgb_params_03_binary
+        ]
+
+    def run_training_continuation(self, xgb_params_01, xgb_params_02,
+                                  xgb_params_03):
         from sklearn.datasets import load_digits
         from sklearn.metrics import mean_squared_error
 
@@ -45,18 +54,18 @@ class TestTrainingContinuation(unittest.TestCase):
         dtrain_2class = xgb.DMatrix(X_2class, label=y_2class)
         dtrain_5class = xgb.DMatrix(X_5class, label=y_5class)
 
-        gbdt_01 = xgb.train(self.xgb_params_01, dtrain_2class,
+        gbdt_01 = xgb.train(xgb_params_01, dtrain_2class,
                             num_boost_round=10)
         ntrees_01 = len(gbdt_01.get_dump())
         assert ntrees_01 == 10
 
-        gbdt_02 = xgb.train(self.xgb_params_01, dtrain_2class,
+        gbdt_02 = xgb.train(xgb_params_01, dtrain_2class,
                             num_boost_round=0)
         gbdt_02.save_model('xgb_tc.model')
 
-        gbdt_02a = xgb.train(self.xgb_params_01, dtrain_2class,
+        gbdt_02a = xgb.train(xgb_params_01, dtrain_2class,
                              num_boost_round=10, xgb_model=gbdt_02)
-        gbdt_02b = xgb.train(self.xgb_params_01, dtrain_2class,
+        gbdt_02b = xgb.train(xgb_params_01, dtrain_2class,
                              num_boost_round=10, xgb_model="xgb_tc.model")
         ntrees_02a = len(gbdt_02a.get_dump())
         ntrees_02b = len(gbdt_02b.get_dump())
@@ -71,13 +80,13 @@ class TestTrainingContinuation(unittest.TestCase):
         res2 = mean_squared_error(y_2class, gbdt_02b.predict(dtrain_2class))
         assert res1 == res2
 
-        gbdt_03 = xgb.train(self.xgb_params_01, dtrain_2class,
+        gbdt_03 = xgb.train(xgb_params_01, dtrain_2class,
                             num_boost_round=3)
         gbdt_03.save_model('xgb_tc.model')
 
-        gbdt_03a = xgb.train(self.xgb_params_01, dtrain_2class,
+        gbdt_03a = xgb.train(xgb_params_01, dtrain_2class,
                              num_boost_round=7, xgb_model=gbdt_03)
-        gbdt_03b = xgb.train(self.xgb_params_01, dtrain_2class,
+        gbdt_03b = xgb.train(xgb_params_01, dtrain_2class,
                              num_boost_round=7, xgb_model="xgb_tc.model")
         ntrees_03a = len(gbdt_03a.get_dump())
         ntrees_03b = len(gbdt_03b.get_dump())
@@ -88,7 +97,7 @@ class TestTrainingContinuation(unittest.TestCase):
         res2 = mean_squared_error(y_2class, gbdt_03b.predict(dtrain_2class))
         assert res1 == res2
 
-        gbdt_04 = xgb.train(self.xgb_params_02, dtrain_2class,
+        gbdt_04 = xgb.train(xgb_params_02, dtrain_2class,
                             num_boost_round=3)
         assert gbdt_04.best_ntree_limit == (gbdt_04.best_iteration +
                                             1) * self.num_parallel_tree
@@ -100,7 +109,7 @@ class TestTrainingContinuation(unittest.TestCase):
                                       ntree_limit=gbdt_04.best_ntree_limit))
         assert res1 == res2
 
-        gbdt_04 = xgb.train(self.xgb_params_02, dtrain_2class,
+        gbdt_04 = xgb.train(xgb_params_02, dtrain_2class,
                             num_boost_round=7, xgb_model=gbdt_04)
         assert gbdt_04.best_ntree_limit == (
             gbdt_04.best_iteration + 1) * self.num_parallel_tree
@@ -112,11 +121,11 @@ class TestTrainingContinuation(unittest.TestCase):
                                       ntree_limit=gbdt_04.best_ntree_limit))
         assert res1 == res2
 
-        gbdt_05 = xgb.train(self.xgb_params_03, dtrain_5class,
+        gbdt_05 = xgb.train(xgb_params_03, dtrain_5class,
                             num_boost_round=7)
         assert gbdt_05.best_ntree_limit == (
             gbdt_05.best_iteration + 1) * self.num_parallel_tree
-        gbdt_05 = xgb.train(self.xgb_params_03,
+        gbdt_05 = xgb.train(xgb_params_03,
                             dtrain_5class,
                             num_boost_round=3,
                             xgb_model=gbdt_05)
@@ -127,3 +136,32 @@ class TestTrainingContinuation(unittest.TestCase):
         res2 = gbdt_05.predict(dtrain_5class,
                                ntree_limit=gbdt_05.best_ntree_limit)
         np.testing.assert_almost_equal(res1, res2)
+
+    @pytest.mark.skipif(**tm.no_sklearn())
+    def test_training_continuation_binary(self):
+        params = self.generate_parameters(False)
+        self.run_training_continuation(params[0], params[1], params[2])
+
+    @pytest.mark.skipif(**tm.no_sklearn())
+    def test_training_continuation_json(self):
+        params = self.generate_parameters(True)
+        for p in params:
+            p['enable_experimental_json_serialization'] = True
+        self.run_training_continuation(params[0], params[1], params[2])
+
+    @pytest.mark.skipif(**tm.no_sklearn())
+    def test_training_continuation_updaters_binary(self):
+        updaters = 'grow_colmaker,prune,refresh'
+        params = self.generate_parameters(False)
+        for p in params:
+            p['updater'] = updaters
+        self.run_training_continuation(params[0], params[1], params[2])
+
+    @pytest.mark.skipif(**tm.no_sklearn())
+    def test_training_continuation_updaters_json(self):
+        # Picked up from R tests.
+        updaters = 'grow_colmaker,prune,refresh'
+        params = self.generate_parameters(True)
+        for p in params:
+            p['updater'] = updaters
+        self.run_training_continuation(params[0], params[1], params[2])