Hendrik/xgboost
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

faster external memory

Browse Source
This commit is contained in:
tqchen 2015-04-25 10:14:56 -07:00
parent b31d1c4ad9
commit 5870b47d76
2 changed files with 9 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/io/page_dmatrix-inl.hpp
View File

@ -20,7 +20,7 @@ namespace io {
class ThreadRowPageIterator: public utils::IIterator<RowBatch> { class ThreadRowPageIterator: public utils::IIterator<RowBatch> {
public: public:
ThreadRowPageIterator(void) { ThreadRowPageIterator(void) {
itr.SetParam("buffer_size", "2"); itr.SetParam("buffer_size", "4");
page_ = NULL; page_ = NULL;
base_rowid_ = 0; base_rowid_ = 0;
} }

13
src/io/page_fmatrix-inl.hpp
View File

@ -112,8 +112,9 @@ struct ColConvertFactory {
#pragma omp parallel #pragma omp parallel
{ {
nthread = omp_get_num_threads(); nthread = omp_get_num_threads();
if (nthread == omp_get_num_procs()) { int max_nthread = std::max(omp_get_num_procs() / 2 - 4, 1);
nthread = std::max(1, nthread / 2); if (nthread > max_nthread) {
nthread = max_nthread;
} }
} }
pcol->Clear(); pcol->Clear();
@ -302,9 +303,11 @@ class FMatrixPage : public IFMatrix {
col_size_[i] += pcol->offset[i + 1] - pcol->offset[i]; col_size_[i] += pcol->offset[i + 1] - pcol->offset[i];
} }
pcol->Save(&fo); pcol->Save(&fo);
bytes_write += pcol->MemCostBytes(); size_t spage = pcol->MemCostBytes();
double tdiff = rabit::utils::GetTime() - tstart; bytes_write += spage;
utils::Printf("Writting to %s in %g MB/s, %lu MB written\n", double tnow = rabit::utils::GetTime();
double tdiff = tnow - tstart;
utils::Printf("Writting to %s in %g MB/s, %lu MB written current speed:%g MB/s\n",
col_data_name_.c_str(), col_data_name_.c_str(),
(bytes_write >> 20UL) / tdiff, (bytes_write >> 20UL) / tdiff,
(bytes_write >> 20UL)); (bytes_write >> 20UL));