diff --git a/doc/parameter.rst b/doc/parameter.rst
index aff9a5d18..f6e78b122 100644
--- a/doc/parameter.rst
+++ b/doc/parameter.rst
@@ -420,7 +420,7 @@ Specify the learning task and the corresponding learning objective. The objectiv
 
 * ``seed_per_iteration`` [default=false]
 
-  - Seed PRNG determnisticly via iterator number, this option will be switched on automatically on distributed mode.
+  - Seed PRNG determnisticly via iterator number.
 
 ***********************
 Command Line Parameters
diff --git a/include/xgboost/generic_parameters.h b/include/xgboost/generic_parameters.h
index 583252bcc..9301f10ce 100644
--- a/include/xgboost/generic_parameters.h
+++ b/include/xgboost/generic_parameters.h
@@ -49,10 +49,7 @@ struct GenericParameter : public XGBoostParameter<GenericParameter> {
     DMLC_DECLARE_ALIAS(seed, random_state);
     DMLC_DECLARE_FIELD(seed_per_iteration)
         .set_default(false)
-        .describe(
-            "Seed PRNG determnisticly via iterator number, "
-            "this option will be switched on automatically on distributed "
-            "mode.");
+        .describe("Seed PRNG determnisticly via iterator number.");
     DMLC_DECLARE_FIELD(nthread).set_default(0).describe(
         "Number of threads to use.");
     DMLC_DECLARE_ALIAS(nthread, n_jobs);
diff --git a/src/learner.cc b/src/learner.cc
index 36b13f8b8..c6baf5b5e 100644
--- a/src/learner.cc
+++ b/src/learner.cc
@@ -1062,9 +1062,10 @@ class LearnerImpl : public LearnerIO {
     monitor_.Start("UpdateOneIter");
     TrainingObserver::Instance().Update(iter);
     this->Configure();
-    if (generic_parameters_.seed_per_iteration || rabit::IsDistributed()) {
+    if (generic_parameters_.seed_per_iteration) {
       common::GlobalRandom().seed(generic_parameters_.seed * kRandSeedMagic + iter);
     }
+
     this->CheckDataSplitMode();
     this->ValidateDMatrix(train.get(), true);
 
@@ -1089,9 +1090,10 @@ class LearnerImpl : public LearnerIO {
                     HostDeviceVector<GradientPair>* in_gpair) override {
     monitor_.Start("BoostOneIter");
     this->Configure();
-    if (generic_parameters_.seed_per_iteration || rabit::IsDistributed()) {
+    if (generic_parameters_.seed_per_iteration) {
       common::GlobalRandom().seed(generic_parameters_.seed * kRandSeedMagic + iter);
     }
+
     this->CheckDataSplitMode();
     this->ValidateDMatrix(train.get(), true);
     auto local_cache = this->GetPredictionCache();