diff --git a/include/xgboost/data.h b/include/xgboost/data.h
index 6cae56349..50134c13b 100644
--- a/include/xgboost/data.h
+++ b/include/xgboost/data.h
@@ -285,7 +285,6 @@ template<typename T>
 class BatchIteratorImpl {
  public:
   virtual ~BatchIteratorImpl() {}
-  virtual BatchIteratorImpl* Clone() = 0;
   virtual T& operator*() = 0;
   virtual const T& operator*() const = 0;
   virtual void operator++() = 0;
@@ -298,14 +297,6 @@ class BatchIterator {
   using iterator_category = std::forward_iterator_tag;
   explicit BatchIterator(BatchIteratorImpl<T>* impl) { impl_.reset(impl); }
 
-  BatchIterator(const BatchIterator& other) {
-    if (other.impl_) {
-      impl_.reset(other.impl_->Clone());
-    } else {
-      impl_.reset();
-    }
-  }
-
   void operator++() {
     CHECK(impl_ != nullptr);
     ++(*impl_);
@@ -332,7 +323,7 @@ class BatchIterator {
   }
 
  private:
-  std::unique_ptr<BatchIteratorImpl<T>> impl_;
+  std::shared_ptr<BatchIteratorImpl<T>> impl_;
 };
 
 template<typename T>
diff --git a/src/data/simple_dmatrix.cc b/src/data/simple_dmatrix.cc
index fbe1a1edf..8fb6e2d97 100644
--- a/src/data/simple_dmatrix.cc
+++ b/src/data/simple_dmatrix.cc
@@ -43,9 +43,6 @@ class SimpleBatchIteratorImpl : public BatchIteratorImpl<T> {
   }
   void operator++() override { page_ = nullptr; }
   bool AtEnd() const override { return page_ == nullptr; }
-  SimpleBatchIteratorImpl* Clone() override {
-    return new SimpleBatchIteratorImpl(*this);
-  }
 
  private:
   T* page_{nullptr};
diff --git a/src/data/sparse_page_dmatrix.cc b/src/data/sparse_page_dmatrix.cc
index a755927ba..a1c73f1e6 100644
--- a/src/data/sparse_page_dmatrix.cc
+++ b/src/data/sparse_page_dmatrix.cc
@@ -31,9 +31,6 @@ class SparseBatchIteratorImpl : public BatchIteratorImpl<T> {
   const T& operator*() const override { return source_->Value(); }
   void operator++() override { at_end_ = !source_->Next(); }
   bool AtEnd() const override { return at_end_; }
-  SparseBatchIteratorImpl* Clone() override {
-    return new SparseBatchIteratorImpl(*this);
-  }
 
  private:
   SparsePageSource<T>* source_{nullptr};