Hendrik/xgboost
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

faster external memory

Browse Source
This commit is contained in:
tqchen 2015-04-25 10:14:56 -07:00
parent b31d1c4ad9
commit 5870b47d76
2 changed files with 9 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/io/page_dmatrix-inl.hpp
View File

@ -20,7 +20,7 @@ namespace io {
class ThreadRowPageIterator: public utils::IIterator<RowBatch> {
public:
ThreadRowPageIterator(void) {
itr.SetParam("buffer_size", "2");
itr.SetParam("buffer_size", "4");
page_ = NULL;
base_rowid_ = 0;
}

13
src/io/page_fmatrix-inl.hpp
View File

@ -112,8 +112,9 @@ struct ColConvertFactory {
#pragma omp parallel
{
nthread = omp_get_num_threads();
if (nthread == omp_get_num_procs()) {
nthread = std::max(1, nthread / 2);
int max_nthread = std::max(omp_get_num_procs() / 2 - 4, 1);
if (nthread > max_nthread) {
nthread = max_nthread;
}
}
pcol->Clear();
@ -302,9 +303,11 @@ class FMatrixPage : public IFMatrix {
col_size_[i] += pcol->offset[i + 1] - pcol->offset[i];
}
pcol->Save(&fo);
bytes_write += pcol->MemCostBytes();
double tdiff = rabit::utils::GetTime() - tstart;
utils::Printf("Writting to %s in %g MB/s, %lu MB written\n",
size_t spage = pcol->MemCostBytes();
bytes_write += spage;
double tnow = rabit::utils::GetTime();
double tdiff = tnow - tstart;
utils::Printf("Writting to %s in %g MB/s, %lu MB written current speed:%g MB/s\n",
col_data_name_.c_str(),
(bytes_write >> 20UL) / tdiff,
(bytes_write >> 20UL));