Hendrik/xgboost
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

increase shard

Browse Source
This commit is contained in:
Ubuntu 2016-01-28 00:58:48 +00:00 committed by tqchen
parent 724eda2435
commit c36195795a
1 changed files with 16 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
src/data/data.cc
View File

@ -138,12 +138,24 @@ DMatrix* DMatrix::Load(const std::string& uri,
cache_file = uri.substr(dlm_pos + 1, uri.length()); cache_file = uri.substr(dlm_pos + 1, uri.length());
fname = uri.substr(0, dlm_pos); fname = uri.substr(0, dlm_pos);
CHECK_EQ(cache_file.find('#'), std::string::npos) CHECK_EQ(cache_file.find('#'), std::string::npos)
<< "Only one `#` is allowed in file path for cache file specification."; << "Only one `#` is allowed in file path for cache file specification.";
if (load_row_split) { if (load_row_split) {
std::ostringstream os; std::ostringstream os;
os << "r" << rabit::GetRank() std::vector<std::string> cache_shards = common::Split(cache_file, ':');
<< "-" << rabit::GetWorldSize() for (size_t i = 0; i < cache_shards.size(); ++i) {
<< "." << cache_file; size_t pos = cache_shards[i].rfind('.');
if (pos == std::string::npos) {
os << cache_shards[i]
<< ".r" << rabit::GetRank()
<< "-" << rabit::GetWorldSize();
} else {
os << cache_shards[i].substr(0, pos)
<< ".r" << rabit::GetRank()
<< "-" << rabit::GetWorldSize()
<< cache_shards[i].substr(pos, cache_shards[i].length());
}
if (i + 1 != cache_shards.size()) os << ':';
}
cache_file = os.str(); cache_file = os.str();
} }
} else { } else {