Hendrik/xgboost
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

External data adapters (#5044)

Browse Source 
* Use external data adapters as lightweight intermediate layer between external data and DMatrix
This commit is contained in:
Rory Mitchell
2019-12-04 10:56:17 +13:00
committed by GitHub
parent f2277e7106
commit e3c34c79be
15 changed files with 1058 additions and 593 deletions
Download Patch File Download Diff File Expand all files Collapse all files

54
tests/cpp/common/test_group_data.cc Normal file
View File

@@ -0,0 +1,54 @@
/*!
* Copyright 2019 by Contributors
*/
#include <gtest/gtest.h>
#include <xgboost/data.h>
#include "../../../src/common/group_data.h"
namespace xgboost {
namespace common {
TEST(group_data, ParallelGroupBuilder) {
std::vector<size_t> offsets;
std::vector<Entry> data;
ParallelGroupBuilder<Entry, size_t> builder(&offsets, &data);
builder.InitBudget(0, 1);
// Add two rows with two elements each
builder.AddBudget(0, 0, 2);
builder.AddBudget(1, 0, 2);
builder.InitStorage();
builder.Push(0, Entry(0, 0), 0);
builder.Push(0, Entry(1, 1), 0);
builder.Push(1, Entry(0, 2), 0);
builder.Push(1, Entry(1, 3), 0);
std::vector<Entry> expected_data{
Entry(0, 0),
Entry(1, 1),
Entry(0, 2),
Entry(1, 3),
};
std::vector<size_t> expected_offsets{0, 2, 4};
EXPECT_EQ(data, expected_data);
EXPECT_EQ(offsets, expected_offsets);
// Create new builder, add one more row given already populated offsets/data
ParallelGroupBuilder<Entry, size_t> builder2(&offsets, &data);
builder2.InitBudget(0, 1);
builder2.AddBudget(2, 0, 2);
builder2.InitStorage();
builder2.Push(2, Entry(0, 4), 0);
builder2.Push(2, Entry(1, 5), 0);
expected_data.emplace_back(Entry(0, 4));
expected_data.emplace_back(Entry(1, 5));
expected_offsets.emplace_back(6);
EXPECT_EQ(data, expected_data);
EXPECT_EQ(offsets, expected_offsets);
}
} // namespace common
} // namespace xgboost

104
tests/cpp/data/test_adapter.cc Normal file
View File

@@ -0,0 +1,104 @@
// Copyright (c) 2019 by Contributors
#include <gtest/gtest.h>
#include <xgboost/c_api.h>
#include <xgboost/data.h>
#include <xgboost/version_config.h>
#include "../../../src/data/adapter.h"
#include "../../../src/data/simple_dmatrix.h"
#include "../../../src/common/timer.h"
#include "../helpers.h"
using namespace xgboost; // NOLINT
TEST(c_api, CSRAdapter) {
int m = 3;
int n = 2;
std::vector<float> data = {1, 2, 3, 4, 5};
std::vector<unsigned> feature_idx = {0, 1, 0, 1, 1};
std::vector<size_t> row_ptr = {0, 2, 4, 5};
data::CSRAdapter adapter(row_ptr.data(), feature_idx.data(), data.data(),
row_ptr.size() - 1, data.size(), n);
adapter.Next();
auto & batch = adapter.Value();
auto line0 = batch.GetLine(0);
EXPECT_EQ(line0.GetElement(0).value, 1);
EXPECT_EQ(line0.GetElement(1).value, 2);
auto line1 = batch.GetLine(1);
EXPECT_EQ(line1 .GetElement(0).value, 3);
EXPECT_EQ(line1 .GetElement(1).value, 4);
auto line2 = batch.GetLine(2);
EXPECT_EQ(line2 .GetElement(0).value, 5);
EXPECT_EQ(line2 .GetElement(0).row_idx, 2);
EXPECT_EQ(line2 .GetElement(0).column_idx, 1);
data::SimpleDMatrix dmat(&adapter, -1, std::nan(""));
EXPECT_EQ(dmat.Info().num_col_, 2);
EXPECT_EQ(dmat.Info().num_row_, 3);
EXPECT_EQ(dmat.Info().num_nonzero_, 5);
for (auto &batch : dmat.GetBatches<SparsePage>()) {
for (auto i = 0ull; i < batch.Size(); i++) {
auto inst = batch[i];
for(auto j = 0ull; j < inst.size(); j++)
{
EXPECT_EQ(inst[j].fvalue, data[row_ptr[i] + j]);
EXPECT_EQ(inst[j].index, feature_idx[row_ptr[i] + j]);
}
}
}
}
TEST(c_api, DenseAdapter) {
int m = 3;
int n = 2;
std::vector<float> data = {1, 2, 3, 4, 5, 6};
data::DenseAdapter adapter(data.data(), m, m*n, n);
data::SimpleDMatrix dmat(&adapter,-1,std::numeric_limits<float>::quiet_NaN());
EXPECT_EQ(dmat.Info().num_col_, 2);
EXPECT_EQ(dmat.Info().num_row_, 3);
EXPECT_EQ(dmat.Info().num_nonzero_, 6);
for (auto &batch : dmat.GetBatches<SparsePage>()) {
for (auto i = 0ull; i < batch.Size(); i++) {
auto inst = batch[i];
for(auto j = 0ull; j < inst.size(); j++)
{
EXPECT_EQ(inst[j].fvalue, data[i*n+j]);
EXPECT_EQ(inst[j].index, j);
}
}
}
}
TEST(c_api, CSCAdapter) {
std::vector<float> data = {1, 3, 2, 4, 5};
std::vector<unsigned> row_idx = {0, 1, 0, 1, 2};
std::vector<size_t> col_ptr = {0, 2, 5};
data::CSCAdapter adapter(col_ptr.data(), row_idx.data(), data.data(), 2, 3);
data::SimpleDMatrix dmat(&adapter,-1,std::numeric_limits<float>::quiet_NaN());
EXPECT_EQ(dmat.Info().num_col_, 2);
EXPECT_EQ(dmat.Info().num_row_, 3);
EXPECT_EQ(dmat.Info().num_nonzero_, 5);
auto &batch = *dmat.GetBatches<SparsePage>().begin();
auto inst = batch[0];
EXPECT_EQ(inst[0].fvalue, 1);
EXPECT_EQ(inst[0].index, 0);
EXPECT_EQ(inst[1].fvalue, 2);
EXPECT_EQ(inst[1].index, 1);
inst = batch[1];
EXPECT_EQ(inst[0].fvalue, 3);
EXPECT_EQ(inst[0].index, 0);
EXPECT_EQ(inst[1].fvalue, 4);
EXPECT_EQ(inst[1].index, 1);
inst = batch[2];
EXPECT_EQ(inst[0].fvalue, 5);
EXPECT_EQ(inst[0].index, 1);
}
TEST(c_api, FileAdapter) {
std::string filename = "test.libsvm";
CreateBigTestData(filename, 10);
std::unique_ptr<dmlc::Parser<uint32_t>> parser(dmlc::Parser<uint32_t>::Create(filename.c_str(), 0, 1,"auto"));
data::FileAdapter adapter(parser.get());
}

1
tests/cpp/data/test_data.cc
View File

@@ -101,7 +101,6 @@ TEST(DMatrix, Uri) {
std::string path = tmpdir.path + "/small.csv";
std::ofstream fout(path);
ASSERT_TRUE(fout);
size_t i = 0;
for (size_t r = 0; r < kRows; ++r) {
for (size_t c = 0; c < kCols; ++c) {

63
tests/cpp/data/test_simple_dmatrix.cc
View File

@@ -4,6 +4,9 @@
#include "../../../src/data/simple_dmatrix.h"
#include "../helpers.h"
#include "../../../src/data/adapter.h"
using namespace xgboost; // NOLINT
TEST(SimpleDMatrix, MetaInfo) {
dmlc::TemporaryDirectory tempdir;
@@ -63,3 +66,63 @@ TEST(SimpleDMatrix, ColAccessWithoutBatches) {
EXPECT_EQ(num_col_batch, 1) << "Expected number of batches to be 1";
delete dmat;
}
TEST(SimpleDMatrix, Empty) {
std::vector<float> data{};
std::vector<unsigned> feature_idx = {};
std::vector<size_t> row_ptr = {};
data::CSRAdapter csr_adapter(row_ptr.data(), feature_idx.data(), data.data(), 0, 0, 0);
data::SimpleDMatrix dmat(&csr_adapter,
std::numeric_limits<float>::quiet_NaN(), 1);
CHECK_EQ(dmat.Info().num_nonzero_, 0);
CHECK_EQ(dmat.Info().num_row_, 0);
CHECK_EQ(dmat.Info().num_col_, 0);
for (auto &batch : dmat.GetBatches<SparsePage>()) {
CHECK_EQ(batch.Size(), 0);
}
data::DenseAdapter dense_adapter(nullptr, 0, 0, 0);
dmat = data::SimpleDMatrix(&dense_adapter,
std::numeric_limits<float>::quiet_NaN(), 1);
CHECK_EQ(dmat.Info().num_nonzero_, 0);
CHECK_EQ(dmat.Info().num_row_, 0);
CHECK_EQ(dmat.Info().num_col_, 0);
for (auto &batch : dmat.GetBatches<SparsePage>()) {
CHECK_EQ(batch.Size(), 0);
}
data::CSCAdapter csc_adapter(nullptr, nullptr, nullptr, 0, 0);
dmat = data::SimpleDMatrix(&csc_adapter,
std::numeric_limits<float>::quiet_NaN(), 1);
CHECK_EQ(dmat.Info().num_nonzero_, 0);
CHECK_EQ(dmat.Info().num_row_, 0);
CHECK_EQ(dmat.Info().num_col_, 0);
for (auto &batch : dmat.GetBatches<SparsePage>()) {
CHECK_EQ(batch.Size(), 0);
}
}
TEST(SimpleDMatrix, MissingData) {
std::vector<float> data{0.0, std::nanf(""), 1.0};
std::vector<unsigned> feature_idx = {0, 1, 0};
std::vector<size_t> row_ptr = {0, 2, 3};
data::CSRAdapter adapter(row_ptr.data(), feature_idx.data(), data.data(), 2, 3, 2);
data::SimpleDMatrix dmat(&adapter, std::numeric_limits<float>::quiet_NaN(), 1);
CHECK_EQ(dmat.Info().num_nonzero_, 2);
dmat = data::SimpleDMatrix(&adapter, 1.0, 1);
CHECK_EQ(dmat.Info().num_nonzero_, 1);
}
TEST(SimpleDMatrix, EmptyRow) {
std::vector<float> data{0.0, 1.0};
std::vector<unsigned> feature_idx = {0, 1};
std::vector<size_t> row_ptr = {0, 2, 2};
data::CSRAdapter adapter(row_ptr.data(), feature_idx.data(), data.data(), 2, 2, 2);
data::SimpleDMatrix dmat(&adapter, std::numeric_limits<float>::quiet_NaN(), 1);
CHECK_EQ(dmat.Info().num_nonzero_, 2);
CHECK_EQ(dmat.Info().num_row_, 2);
CHECK_EQ(dmat.Info().num_col_, 2);
}

112
tests/python/test_basic.py
View File

@@ -64,25 +64,6 @@ class TestBasic(unittest.TestCase):
# assert they are the same
assert np.sum(np.abs(preds2 - preds)) == 0
def test_np_view(self):
# Sliced Float32 array
y = np.array([12, 34, 56], np.float32)[::2]
from_view = xgb.DMatrix(np.array([[]]), label=y).get_label()
from_array = xgb.DMatrix(np.array([[]]), label=y + 0).get_label()
assert (from_view.shape == from_array.shape)
assert (from_view == from_array).all()
# Sliced UInt array
z = np.array([12, 34, 56], np.uint32)[::2]
dmat = xgb.DMatrix(np.array([[]]))
dmat.set_uint_info('root_index', z)
from_view = dmat.get_uint_info('root_index')
dmat = xgb.DMatrix(np.array([[]]))
dmat.set_uint_info('root_index', z + 0)
from_array = dmat.get_uint_info('root_index')
assert (from_view.shape == from_array.shape)
assert (from_view == from_array).all()
def test_record_results(self):
dtrain = xgb.DMatrix(dpath + 'agaricus.txt.train')
dtest = xgb.DMatrix(dpath + 'agaricus.txt.test')
@@ -127,72 +108,6 @@ class TestBasic(unittest.TestCase):
# assert they are the same
assert np.sum(np.abs(preds2 - preds)) == 0
def test_dmatrix_init(self):
data = np.random.randn(5, 5)
# different length
self.assertRaises(ValueError, xgb.DMatrix, data,
feature_names=list('abcdef'))
# contains duplicates
self.assertRaises(ValueError, xgb.DMatrix, data,
feature_names=['a', 'b', 'c', 'd', 'd'])
# contains symbol
self.assertRaises(ValueError, xgb.DMatrix, data,
feature_names=['a', 'b', 'c', 'd', 'e<1'])
dm = xgb.DMatrix(data)
dm.feature_names = list('abcde')
assert dm.feature_names == list('abcde')
assert dm.slice([0, 1]).feature_names == dm.feature_names
dm.feature_types = 'q'
assert dm.feature_types == list('qqqqq')
dm.feature_types = list('qiqiq')
assert dm.feature_types == list('qiqiq')
def incorrect_type_set():
dm.feature_types = list('abcde')
self.assertRaises(ValueError, incorrect_type_set)
# reset
dm.feature_names = None
self.assertEqual(dm.feature_names, ['f0', 'f1', 'f2', 'f3', 'f4'])
assert dm.feature_types is None
def test_feature_names(self):
data = np.random.randn(100, 5)
target = np.array([0, 1] * 50)
cases = [['Feature1', 'Feature2', 'Feature3', 'Feature4', 'Feature5'],
[u'要因1', u'要因2', u'要因3', u'要因4', u'要因5']]
for features in cases:
dm = xgb.DMatrix(data, label=target,
feature_names=features)
assert dm.feature_names == features
assert dm.num_row() == 100
assert dm.num_col() == 5
params = {'objective': 'multi:softprob',
'eval_metric': 'mlogloss',
'eta': 0.3,
'num_class': 3}
bst = xgb.train(params, dm, num_boost_round=10)
scores = bst.get_fscore()
assert list(sorted(k for k in scores)) == features
dummy = np.random.randn(5, 5)
dm = xgb.DMatrix(dummy, feature_names=features)
bst.predict(dm)
# different feature name must raises error
dm = xgb.DMatrix(dummy, feature_names=list('abcde'))
self.assertRaises(ValueError, bst.predict, dm)
def test_dump(self):
data = np.random.randn(100, 2)
target = np.array([0, 1] * 50)
@@ -250,27 +165,6 @@ class TestBasic(unittest.TestCase):
assert dm.num_row() == row
assert dm.num_col() == cols
def test_dmatrix_numpy_init(self):
data = np.random.randn(5, 5)
dm = xgb.DMatrix(data)
assert dm.num_row() == 5
assert dm.num_col() == 5
data = np.array([[1, 2], [3, 4]])
dm = xgb.DMatrix(data)
assert dm.num_row() == 2
assert dm.num_col() == 2
# 0d array
self.assertRaises(ValueError, xgb.DMatrix, np.array(1))
# 1d array
self.assertRaises(ValueError, xgb.DMatrix, np.array([1, 2, 3]))
# 3d array
data = np.random.randn(5, 5, 5)
self.assertRaises(ValueError, xgb.DMatrix, data)
# object dtype
data = np.array([['a', 'b'], ['c', 'd']])
self.assertRaises(ValueError, xgb.DMatrix, data)
def test_cv(self):
dm = xgb.DMatrix(dpath + 'agaricus.txt.train')
@@ -336,12 +230,6 @@ class TestBasic(unittest.TestCase):
' dtype=float32)]')
assert output == solution
def test_get_info(self):
dtrain = xgb.DMatrix(dpath + 'agaricus.txt.train')
dtrain.get_float_info('label')
dtrain.get_float_info('weight')
dtrain.get_float_info('base_margin')
dtrain.get_uint_info('root_index')
class TestBasicPathLike(unittest.TestCase):

171
tests/python/test_dmatrix.py Normal file
View File

@@ -0,0 +1,171 @@
# -*- coding: utf-8 -*-
import numpy as np
import xgboost as xgb
import unittest
import scipy.sparse
from scipy.sparse import rand
rng = np.random.RandomState(1)
dpath = 'demo/data/'
rng = np.random.RandomState(1994)
class TestDMatrix(unittest.TestCase):
def test_dmatrix_numpy_init(self):
data = np.random.randn(5, 5)
dm = xgb.DMatrix(data)
assert dm.num_row() == 5
assert dm.num_col() == 5
data = np.array([[1, 2], [3, 4]])
dm = xgb.DMatrix(data)
assert dm.num_row() == 2
assert dm.num_col() == 2
# 0d array
self.assertRaises(ValueError, xgb.DMatrix, np.array(1))
# 1d array
self.assertRaises(ValueError, xgb.DMatrix, np.array([1, 2, 3]))
# 3d array
data = np.random.randn(5, 5, 5)
self.assertRaises(ValueError, xgb.DMatrix, data)
# object dtype
data = np.array([['a', 'b'], ['c', 'd']])
self.assertRaises(ValueError, xgb.DMatrix, data)
def test_csr(self):
indptr = np.array([0, 2, 3, 6])
indices = np.array([0, 2, 2, 0, 1, 2])
data = np.array([1, 2, 3, 4, 5, 6])
X = scipy.sparse.csr_matrix((data, indices, indptr), shape=(3, 3))
dtrain = xgb.DMatrix(X)
assert dtrain.num_row() == 3
assert dtrain.num_col() == 3
def test_csc(self):
row = np.array([0, 2, 2, 0, 1, 2])
col = np.array([0, 0, 1, 2, 2, 2])
data = np.array([1, 2, 3, 4, 5, 6])
X = scipy.sparse.csc_matrix((data, (row, col)), shape=(3, 3))
dtrain = xgb.DMatrix(X)
assert dtrain.num_row() == 3
assert dtrain.num_col() == 3
def test_np_view(self):
# Sliced Float32 array
y = np.array([12, 34, 56], np.float32)[::2]
from_view = xgb.DMatrix(np.array([[]]), label=y).get_label()
from_array = xgb.DMatrix(np.array([[]]), label=y + 0).get_label()
assert (from_view.shape == from_array.shape)
assert (from_view == from_array).all()
# Sliced UInt array
z = np.array([12, 34, 56], np.uint32)[::2]
dmat = xgb.DMatrix(np.array([[]]))
dmat.set_uint_info('root_index', z)
from_view = dmat.get_uint_info('root_index')
dmat = xgb.DMatrix(np.array([[]]))
dmat.set_uint_info('root_index', z + 0)
from_array = dmat.get_uint_info('root_index')
assert (from_view.shape == from_array.shape)
assert (from_view == from_array).all()
def test_feature_names(self):
data = np.random.randn(5, 5)
# different length
self.assertRaises(ValueError, xgb.DMatrix, data,
feature_names=list('abcdef'))
# contains duplicates
self.assertRaises(ValueError, xgb.DMatrix, data,
feature_names=['a', 'b', 'c', 'd', 'd'])
# contains symbol
self.assertRaises(ValueError, xgb.DMatrix, data,
feature_names=['a', 'b', 'c', 'd', 'e<1'])
dm = xgb.DMatrix(data)
dm.feature_names = list('abcde')
assert dm.feature_names == list('abcde')
assert dm.slice([0, 1]).feature_names == dm.feature_names
dm.feature_types = 'q'
assert dm.feature_types == list('qqqqq')
dm.feature_types = list('qiqiq')
assert dm.feature_types == list('qiqiq')
def incorrect_type_set():
dm.feature_types = list('abcde')
self.assertRaises(ValueError, incorrect_type_set)
# reset
dm.feature_names = None
self.assertEqual(dm.feature_names, ['f0', 'f1', 'f2', 'f3', 'f4'])
assert dm.feature_types is None
def test_feature_names(self):
data = np.random.randn(100, 5)
target = np.array([0, 1] * 50)
cases = [['Feature1', 'Feature2', 'Feature3', 'Feature4', 'Feature5'],
[u'要因1', u'要因2', u'要因3', u'要因4', u'要因5']]
for features in cases:
dm = xgb.DMatrix(data, label=target,
feature_names=features)
assert dm.feature_names == features
assert dm.num_row() == 100
assert dm.num_col() == 5
params = {'objective': 'multi:softprob',
'eval_metric': 'mlogloss',
'eta': 0.3,
'num_class': 3}
bst = xgb.train(params, dm, num_boost_round=10)
scores = bst.get_fscore()
assert list(sorted(k for k in scores)) == features
dummy = np.random.randn(5, 5)
dm = xgb.DMatrix(dummy, feature_names=features)
bst.predict(dm)
# different feature name must raises error
dm = xgb.DMatrix(dummy, feature_names=list('abcde'))
self.assertRaises(ValueError, bst.predict, dm)
def test_get_info(self):
dtrain = xgb.DMatrix(dpath + 'agaricus.txt.train')
dtrain.get_float_info('label')
dtrain.get_float_info('weight')
dtrain.get_float_info('base_margin')
dtrain.get_uint_info('root_index')
def test_sparse_dmatrix_csr(self):
nrow = 100
ncol = 1000
x = rand(nrow, ncol, density=0.0005, format='csr', random_state=rng)
assert x.indices.max() < ncol - 1
x.data[:] = 1
dtrain = xgb.DMatrix(x, label=np.random.binomial(1, 0.3, nrow))
assert (dtrain.num_row(), dtrain.num_col()) == (nrow, ncol)
watchlist = [(dtrain, 'train')]
param = {'max_depth': 3, 'objective': 'binary:logistic', 'verbosity': 0}
bst = xgb.train(param, dtrain, 5, watchlist)
bst.predict(dtrain)
def test_sparse_dmatrix_csc(self):
nrow = 1000
ncol = 100
x = rand(nrow, ncol, density=0.0005, format='csc', random_state=rng)
assert x.indices.max() < nrow - 1
x.data[:] = 1
dtrain = xgb.DMatrix(x, label=np.random.binomial(1, 0.3, nrow))
assert (dtrain.num_row(), dtrain.num_col()) == (nrow, ncol)
watchlist = [(dtrain, 'train')]
param = {'max_depth': 3, 'objective': 'binary:logistic', 'verbosity': 0}
bst = xgb.train(param, dtrain, 5, watchlist)
bst.predict(dtrain)

33
tests/python/test_sparse_dmatrix.py
View File

@@ -1,33 +0,0 @@
import numpy as np
import xgboost as xgb
from scipy.sparse import rand
rng = np.random.RandomState(1)
param = {'max_depth': 3, 'objective': 'binary:logistic', 'verbosity': 0}
def test_sparse_dmatrix_csr():
nrow = 100
ncol = 1000
x = rand(nrow, ncol, density=0.0005, format='csr', random_state=rng)
assert x.indices.max() < ncol - 1
x.data[:] = 1
dtrain = xgb.DMatrix(x, label=np.random.binomial(1, 0.3, nrow))
assert (dtrain.num_row(), dtrain.num_col()) == (nrow, ncol)
watchlist = [(dtrain, 'train')]
bst = xgb.train(param, dtrain, 5, watchlist)
bst.predict(dtrain)
def test_sparse_dmatrix_csc():
nrow = 1000
ncol = 100
x = rand(nrow, ncol, density=0.0005, format='csc', random_state=rng)
assert x.indices.max() < nrow - 1
x.data[:] = 1
dtrain = xgb.DMatrix(x, label=np.random.binomial(1, 0.3, nrow))
assert (dtrain.num_row(), dtrain.num_col()) == (nrow, ncol)
watchlist = [(dtrain, 'train')]
bst = xgb.train(param, dtrain, 5, watchlist)
bst.predict(dtrain)