diff --git a/src/data/data.cc b/src/data/data.cc
index b4753bcb4..11a36ffdf 100644
--- a/src/data/data.cc
+++ b/src/data/data.cc
@@ -138,12 +138,24 @@ DMatrix* DMatrix::Load(const std::string& uri,
     cache_file = uri.substr(dlm_pos + 1, uri.length());
     fname = uri.substr(0, dlm_pos);
     CHECK_EQ(cache_file.find('#'), std::string::npos)
-        << "Only one `#` is allowed in file path for cache file specification.";
+        << "Only one `#` is allowed in file path for cache file specification.";    
     if (load_row_split) {
       std::ostringstream os;
-      os << "r" << rabit::GetRank()
-	 << "-" <<  rabit::GetWorldSize()
-	 << "." << cache_file;
+      std::vector<std::string> cache_shards = common::Split(cache_file, ':');
+      for (size_t i = 0; i < cache_shards.size(); ++i) {
+	size_t pos = cache_shards[i].rfind('.');
+	if (pos == std::string::npos) {
+	  os << cache_shards[i]
+	     << ".r" << rabit::GetRank()
+	     << "-" <<  rabit::GetWorldSize();
+	} else {
+	  os << cache_shards[i].substr(0, pos)
+	     << ".r" << rabit::GetRank()
+	     << "-" <<  rabit::GetWorldSize()
+	     << cache_shards[i].substr(pos, cache_shards[i].length());
+	}
+	if (i + 1 != cache_shards.size()) os << ':'; 
+      }
       cache_file = os.str();
     }
   } else {