add
This commit is contained in:
parent
ddb7e538df
commit
0a7d233c5d
@ -133,7 +133,9 @@ class DMatrixPageBase : public DataMatrix {
|
|||||||
const char* cache_file,
|
const char* cache_file,
|
||||||
bool silent,
|
bool silent,
|
||||||
bool loadsplit) {
|
bool loadsplit) {
|
||||||
|
if (!silent) {
|
||||||
|
utils::Printf("start generate text file from %s\n", uri);
|
||||||
|
}
|
||||||
int rank = 0, npart = 1;
|
int rank = 0, npart = 1;
|
||||||
if (loadsplit) {
|
if (loadsplit) {
|
||||||
rank = rabit::GetRank();
|
rank = rabit::GetRank();
|
||||||
@ -146,6 +148,8 @@ class DMatrixPageBase : public DataMatrix {
|
|||||||
dmlc::InputSplit *in =
|
dmlc::InputSplit *in =
|
||||||
dmlc::InputSplit::Create(uri, rank, npart);
|
dmlc::InputSplit::Create(uri, rank, npart);
|
||||||
std::string line;
|
std::string line;
|
||||||
|
size_t bytes_write = 0;
|
||||||
|
double tstart = rabit::utils::GetTime();
|
||||||
info.Clear();
|
info.Clear();
|
||||||
while (in->ReadRecord(&line)) {
|
while (in->ReadRecord(&line)) {
|
||||||
float label;
|
float label;
|
||||||
@ -162,8 +166,17 @@ class DMatrixPageBase : public DataMatrix {
|
|||||||
RowBatch::Inst row(BeginPtr(feats), feats.size());
|
RowBatch::Inst row(BeginPtr(feats), feats.size());
|
||||||
page.Push(row);
|
page.Push(row);
|
||||||
if (page.MemCostBytes() >= kPageSize) {
|
if (page.MemCostBytes() >= kPageSize) {
|
||||||
page.Save(&fo); page.Clear();
|
bytes_write += page.MemCostBytes();
|
||||||
|
page.Save(&fo);
|
||||||
|
page.Clear();
|
||||||
|
double tdiff = rabit::utils::GetTime() - tstart;
|
||||||
|
if (!silent) {
|
||||||
|
utils::Printf("Writting to %s in %g MB/s, %g MB written\n",
|
||||||
|
cache_file, (bytes_write >> 20UL) / tdiff,
|
||||||
|
(bytes_write >> 20UL));
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
for (size_t i = 0; i < feats.size(); ++i) {
|
for (size_t i = 0; i < feats.size(); ++i) {
|
||||||
info.info.num_col = std::max(info.info.num_col,
|
info.info.num_col = std::max(info.info.num_col,
|
||||||
static_cast<size_t>(feats[i].index+1));
|
static_cast<size_t>(feats[i].index+1));
|
||||||
|
|||||||
@ -7,6 +7,7 @@
|
|||||||
* \author Tianqi Chen
|
* \author Tianqi Chen
|
||||||
*/
|
*/
|
||||||
#include "../../subtree/rabit/include/rabit.h"
|
#include "../../subtree/rabit/include/rabit.h"
|
||||||
|
#include "../../subtree/rabit/include/rabit/timer.h"
|
||||||
#endif // XGBOOST_SYNC_H_
|
#endif // XGBOOST_SYNC_H_
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user