diff --git a/src/cli_main.cc b/src/cli_main.cc
index a08e3fd6d..720a3b185 100644
--- a/src/cli_main.cc
+++ b/src/cli_main.cc
@@ -318,6 +318,7 @@ int CLIRunTask(int argc, char *argv[]) {
     printf("Usage: <config>\n");
     return 0;
   }
+  rabit::Init(argc, argv);
 
   std::vector<std::pair<std::string, std::string> > cfg;
   cfg.push_back(std::make_pair("seed", "0"));
@@ -336,7 +337,6 @@ int CLIRunTask(int argc, char *argv[]) {
   CLIParam param;
   param.Configure(cfg);
 
-  rabit::Init(argc, argv);
   switch (param.task) {
     case kTrain: CLITrain(param); break;
     case kDump2Text: CLIDump2Text(param); break;
diff --git a/src/data/data.cc b/src/data/data.cc
index 02b972d83..b4753bcb4 100644
--- a/src/data/data.cc
+++ b/src/data/data.cc
@@ -141,7 +141,9 @@ DMatrix* DMatrix::Load(const std::string& uri,
         << "Only one `#` is allowed in file path for cache file specification.";
     if (load_row_split) {
       std::ostringstream os;
-      os << cache_file << ".r" << rabit::GetRank();
+      os << "r" << rabit::GetRank()
+	 << "-" <<  rabit::GetWorldSize()
+	 << "." << cache_file;
       cache_file = os.str();
     }
   } else {
@@ -154,9 +156,11 @@ DMatrix* DMatrix::Load(const std::string& uri,
   } else {
     // test option to load in part
     npart = dmlc::GetEnv("XGBOOST_TEST_NPART", 1);
-    if (npart != 1) {
-      LOG(CONSOLE) << "Partial load option on npart=" << npart;
-    }
+  }
+
+  if (npart != 1) {
+    LOG(CONSOLE) << "Load part of data " << partid
+		 << " of " << npart << " parts";
   }
   // legacy handling of binary data loading
   if (file_format == "auto" && !load_row_split) {