Hendrik/xgboost
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Run training with empty DMatrix. (#4990)

Browse Source 
This makes GPU Hist robust in distributed environment as some workers might not
be associated with any data in either training or evaluation.

* Disable rabit mock test for now: See #5012 .

* Disable dask-cudf test at prediction for now: See #5003

* Launch dask job for all workers despite they might not have any data.
* Check 0 rows in elementwise evaluation metrics.

   Using AUC and AUC-PR still throws an error.  See #4663 for a robust fix.

* Add tests for edge cases.
* Add `LaunchKernel` wrapper handling zero sized grid.
* Move some parts of allreducer into a cu file.
* Don't validate feature names when the booster is empty.

* Sync number of columns in DMatrix.

  As num_feature is required to be the same across all workers in data split
  mode.

* Filtering in dask interface now by default syncs all booster that's not
empty, instead of using rank 0.

* Fix Jenkins' GPU tests.

* Install dask-cuda from source in Jenkins' test.

  Now all tests are actually running.

* Restore GPU Hist tree synchronization test.

* Check UUID of running devices.

  The check is only performed on CUDA version >= 10.x, as 9.x doesn't have UUID field.

* Fix CMake policy and project variables.

  Use xgboost_SOURCE_DIR uniformly, add policy for CMake >= 3.13.

* Fix copying data to CPU

* Fix race condition in cpu predictor.

* Fix duplicated DMatrix construction.

* Don't download extra nccl in CI script.
This commit is contained in:
Jiaming Yuan
2019-11-06 16:13:13 +08:00
committed by GitHub
parent 807a244517
commit 7663de956c
44 changed files with 603 additions and 272 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
tests/ci_build/Dockerfile.cudf
View File

@@ -18,7 +18,7 @@ ENV PATH=/opt/python/bin:$PATH
# Create new Conda environment with cuDF and dask
RUN \
conda create -n cudf_test -c rapidsai -c nvidia -c numba -c conda-forge -c anaconda \
cudf=0.9 python=3.7 anaconda::cudatoolkit=$CUDA_VERSION dask
cudf=0.9 python=3.7 anaconda::cudatoolkit=$CUDA_VERSION dask dask-cuda
# Install other Python packages
RUN \

3
tests/ci_build/Dockerfile.gpu
View File

@@ -17,7 +17,8 @@ ENV PATH=/opt/python/bin:$PATH
# Install Python packages
RUN \
pip install numpy pytest scipy scikit-learn pandas matplotlib wheel kubernetes urllib3 graphviz && \
pip install "dask[complete]"
pip install "dask[complete]" && \
conda install -c rapidsai -c nvidia -c numba -c conda-forge -c anaconda dask-cuda
ENV GOSU_VERSION 1.10

12
tests/ci_build/Dockerfile.gpu_build
View File

@@ -21,18 +21,12 @@ RUN \
# NCCL2 (License: https://docs.nvidia.com/deeplearning/sdk/nccl-sla/index.html)
RUN \
export CUDA_SHORT=`echo $CUDA_VERSION | egrep -o '[0-9]+\.[0-9]'` && \
if [ "${CUDA_SHORT}" != "10.0" ] && [ "${CUDA_SHORT}" != "10.1" ]; then \
wget https://developer.download.nvidia.com/compute/redist/nccl/v2.2/nccl_2.2.13-1%2Bcuda${CUDA_SHORT}_x86_64.txz && \
tar xf "nccl_2.2.13-1+cuda${CUDA_SHORT}_x86_64.txz" && \
cp nccl_2.2.13-1+cuda${CUDA_SHORT}_x86_64/include/nccl.h /usr/include && \
cp nccl_2.2.13-1+cuda${CUDA_SHORT}_x86_64/lib/* /usr/lib && \
rm -f nccl_2.2.13-1+cuda${CUDA_SHORT}_x86_64.txz && \
rm -r nccl_2.2.13-1+cuda${CUDA_SHORT}_x86_64; else \
export NCCL_VERSION=2.4.8-1 && \
wget https://developer.download.nvidia.com/compute/machine-learning/repos/rhel7/x86_64/nvidia-machine-learning-repo-rhel7-1.0.0-1.x86_64.rpm && \
rpm -i nvidia-machine-learning-repo-rhel7-1.0.0-1.x86_64.rpm && \
yum -y update && \
yum install -y libnccl-2.4.2-1+cuda${CUDA_SHORT} libnccl-devel-2.4.2-1+cuda${CUDA_SHORT} libnccl-static-2.4.2-1+cuda${CUDA_SHORT} && \
rm -f nvidia-machine-learning-repo-rhel7-1.0.0-1.x86_64.rpm; fi
yum install -y libnccl-${NCCL_VERSION}+cuda${CUDA_SHORT} libnccl-devel-${NCCL_VERSION}+cuda${CUDA_SHORT} libnccl-static-${NCCL_VERSION}+cuda${CUDA_SHORT} && \
rm -f nvidia-machine-learning-repo-rhel7-1.0.0-1.x86_64.rpm;
ENV PATH=/opt/python/bin:$PATH
ENV CC=/opt/rh/devtoolset-4/root/usr/bin/gcc

4
tests/ci_build/test_python.sh
View File

@@ -33,11 +33,13 @@ case "$suite" in
pytest -v -s --fulltrace -m "(not slow) and mgpu" tests/python-gpu
cd tests/distributed
./runtests-gpu.sh
cd -
pytest -v -s --fulltrace -m "mgpu" tests/python-gpu/test_gpu_with_dask.py
;;
cudf)
source activate cudf_test
python -m pytest -v -s --fulltrace tests/python-gpu/test_from_columnar.py tests/python-gpu/test_gpu_with_dask.py
pytest -v -s --fulltrace -m "not mgpu" tests/python-gpu/test_from_columnar.py
;;
cpu)

10
tests/cpp/CMakeLists.txt
View File

@@ -19,7 +19,7 @@ if (USE_CUDA)
# OpenMP is mandatory for CUDA
find_package(OpenMP REQUIRED)
target_include_directories(testxgboost PRIVATE
${PROJECT_SOURCE_DIR}/cub/)
${xgboost_SOURCE_DIR}/cub/)
target_compile_options(testxgboost PRIVATE
$<$<COMPILE_LANGUAGE:CUDA>:--expt-extended-lambda>
$<$<COMPILE_LANGUAGE:CUDA>:--expt-relaxed-constexpr>
@@ -48,9 +48,9 @@ endif (USE_CUDA)
target_include_directories(testxgboost
PRIVATE
${GTEST_INCLUDE_DIRS}
${PROJECT_SOURCE_DIR}/include
${PROJECT_SOURCE_DIR}/dmlc-core/include
${PROJECT_SOURCE_DIR}/rabit/include)
${xgboost_SOURCE_DIR}/include
${xgboost_SOURCE_DIR}/dmlc-core/include
${xgboost_SOURCE_DIR}/rabit/include)
set_target_properties(
testxgboost PROPERTIES
CXX_STANDARD 11
@@ -67,7 +67,7 @@ target_compile_definitions(testxgboost PRIVATE ${XGBOOST_DEFINITIONS})
if (USE_OPENMP)
target_compile_options(testxgboost PRIVATE $<$<COMPILE_LANGUAGE:CXX>:${OpenMP_CXX_FLAGS}>)
endif (USE_OPENMP)
set_output_directory(testxgboost ${PROJECT_BINARY_DIR})
set_output_directory(testxgboost ${xgboost_BINARY_DIR})
# This grouping organises source files nicely in visual studio
auto_source_group("${TEST_SOURCES}")

6
tests/distributed/distributed_gpu.py
View File

@@ -2,6 +2,7 @@
import sys
import time
import xgboost as xgb
import os
def run_test(name, params_fun):
@@ -48,6 +49,9 @@ def run_test(name, params_fun):
xgb.rabit.finalize()
if os.path.exists(model_name):
os.remove(model_name)
base_params = {
'tree_method': 'gpu_hist',
@@ -81,7 +85,5 @@ def wrap_rf(params_fun):
params_rf_1x4 = wrap_rf(params_basic_1x4)
test_name = sys.argv[1]
run_test(test_name, globals()['params_%s' % test_name])

4
tests/distributed/runtests-gpu.sh
View File

@@ -6,7 +6,7 @@ export DMLC_SUBMIT_CLUSTER=local
submit="timeout 30 python ../../dmlc-core/tracker/dmlc-submit"
echo -e "\n ====== 1. Basic distributed-gpu test with Python: 4 workers; 1 GPU per worker ====== \n"
$submit --num-workers=4 python distributed_gpu.py basic_1x4 || exit 1
$submit --num-workers=$(nvidia-smi -L | wc -l) python distributed_gpu.py basic_1x4 || exit 1
echo -e "\n ====== 2. RF distributed-gpu test with Python: 4 workers; 1 GPU per worker ====== \n"
$submit --num-workers=4 python distributed_gpu.py rf_1x4 || exit 1
$submit --num-workers=$(nvidia-smi -L | wc -l) python distributed_gpu.py rf_1x4 || exit 1

3
tests/pytest.ini Normal file
View File

@@ -0,0 +1,3 @@
[pytest]
markers =
mgpu: Mark a test that requires multiple GPUs to run.

29
tests/python-gpu/test_gpu_updaters.py
View File

@@ -2,6 +2,7 @@ import numpy as np
import sys
import unittest
import pytest
import xgboost
sys.path.append("tests/python")
from regression_test_utilities import run_suite, parameter_combinations, \
@@ -21,7 +22,8 @@ datasets = ["Boston", "Cancer", "Digits", "Sparse regression",
class TestGPU(unittest.TestCase):
def test_gpu_hist(self):
test_param = parameter_combinations({'gpu_id': [0], 'max_depth': [2, 8],
test_param = parameter_combinations({'gpu_id': [0],
'max_depth': [2, 8],
'max_leaves': [255, 4],
'max_bin': [2, 256],
'grow_policy': ['lossguide']})
@@ -36,6 +38,31 @@ class TestGPU(unittest.TestCase):
cpu_results = run_suite(param, select_datasets=datasets)
assert_gpu_results(cpu_results, gpu_results)
def test_with_empty_dmatrix(self):
# FIXME(trivialfis): This should be done with all updaters
kRows = 0
kCols = 100
X = np.empty((kRows, kCols))
y = np.empty((kRows))
dtrain = xgboost.DMatrix(X, y)
bst = xgboost.train({'verbosity': 2,
'tree_method': 'gpu_hist',
'gpu_id': 0},
dtrain,
verbose_eval=True,
num_boost_round=6,
evals=[(dtrain, 'Train')])
kRows = 100
X = np.random.randn(kRows, kCols)
dtest = xgboost.DMatrix(X)
predictions = bst.predict(dtest)
np.testing.assert_allclose(predictions, 0.5, 1e-6)
@pytest.mark.mgpu
def test_specified_gpu_id_gpu_update(self):
variable_param = {'gpu_id': [1],

97
tests/python-gpu/test_gpu_with_dask.py
View File

@@ -1,45 +1,94 @@
import sys
import pytest
import numpy as np
import unittest
if sys.platform.startswith("win"):
pytest.skip("Skipping dask tests on Windows", allow_module_level=True)
try:
from distributed.utils_test import client, loop, cluster_fixture
import dask.dataframe as dd
from xgboost import dask as dxgb
from dask_cuda import LocalCUDACluster
from dask.distributed import Client
import cudf
except ImportError:
client = None
loop = None
cluster_fixture = None
pass
sys.path.append("tests/python")
from test_with_dask import generate_array
import testing as tm
from test_with_dask import generate_array # noqa
import testing as tm # noqa
@pytest.mark.skipif(**tm.no_dask())
@pytest.mark.skipif(**tm.no_cudf())
@pytest.mark.skipif(**tm.no_dask_cudf())
def test_dask_dataframe(client):
X, y = generate_array()
class TestDistributedGPU(unittest.TestCase):
@pytest.mark.skipif(**tm.no_dask())
@pytest.mark.skipif(**tm.no_cudf())
@pytest.mark.skipif(**tm.no_dask_cudf())
@pytest.mark.skipif(**tm.no_dask_cuda())
def test_dask_dataframe(self):
with LocalCUDACluster() as cluster:
with Client(cluster) as client:
X, y = generate_array()
X = dd.from_dask_array(X)
y = dd.from_dask_array(y)
X = dd.from_dask_array(X)
y = dd.from_dask_array(y)
X = X.map_partitions(cudf.from_pandas)
y = y.map_partitions(cudf.from_pandas)
X = X.map_partitions(cudf.from_pandas)
y = y.map_partitions(cudf.from_pandas)
dtrain = dxgb.DaskDMatrix(client, X, y)
out = dxgb.train(client, {'tree_method': 'gpu_hist'},
dtrain=dtrain,
evals=[(dtrain, 'X')],
num_boost_round=2)
dtrain = dxgb.DaskDMatrix(client, X, y)
out = dxgb.train(client, {'tree_method': 'gpu_hist'},
dtrain=dtrain,
evals=[(dtrain, 'X')],
num_boost_round=2)
assert isinstance(out['booster'], dxgb.Booster)
assert len(out['history']['X']['rmse']) == 2
assert isinstance(out['booster'], dxgb.Booster)
assert len(out['history']['X']['rmse']) == 2
predictions = dxgb.predict(out, dtrain)
predictions = predictions.compute()
# FIXME(trivialfis): Re-enable this after #5003 is fixed
# predictions = dxgb.predict(client, out, dtrain).compute()
# assert isinstance(predictions, np.ndarray)
@pytest.mark.skipif(**tm.no_dask())
@pytest.mark.skipif(**tm.no_dask_cuda())
@pytest.mark.mgpu
def test_empty_dmatrix(self):
def _check_outputs(out, predictions):
assert isinstance(out['booster'], dxgb.Booster)
assert len(out['history']['validation']['rmse']) == 2
assert isinstance(predictions, np.ndarray)
assert predictions.shape[0] == 1
parameters = {'tree_method': 'gpu_hist', 'verbosity': 3,
'debug_synchronize': True}
with LocalCUDACluster() as cluster:
with Client(cluster) as client:
kRows, kCols = 1, 97
X = dd.from_array(np.random.randn(kRows, kCols))
y = dd.from_array(np.random.rand(kRows))
dtrain = dxgb.DaskDMatrix(client, X, y)
out = dxgb.train(client, parameters,
dtrain=dtrain,
evals=[(dtrain, 'validation')],
num_boost_round=2)
predictions = dxgb.predict(client=client, model=out,
data=dtrain).compute()
_check_outputs(out, predictions)
# train has more rows than evals
valid = dtrain
kRows += 1
X = dd.from_array(np.random.randn(kRows, kCols))
y = dd.from_array(np.random.rand(kRows))
dtrain = dxgb.DaskDMatrix(client, X, y)
out = dxgb.train(client, parameters,
dtrain=dtrain,
evals=[(valid, 'validation')],
num_boost_round=2)
predictions = dxgb.predict(client=client, model=out,
data=valid).compute()
_check_outputs(out, predictions)

3
tests/python/regression_test_utilities.py
View File

@@ -67,7 +67,8 @@ def get_weights_regression(min_weight, max_weight):
n = 10000
sparsity = 0.25
X, y = datasets.make_regression(n, random_state=rng)
X = np.array([[np.nan if rng.uniform(0, 1) < sparsity else x for x in x_row] for x_row in X])
X = np.array([[np.nan if rng.uniform(0, 1) < sparsity else x
for x in x_row] for x_row in X])
w = np.array([rng.uniform(min_weight, max_weight) for i in range(n)])
return X, y, w

9
tests/python/testing.py
View File

@@ -34,6 +34,15 @@ def no_matplotlib():
'reason': reason}
def no_dask_cuda():
reason = 'dask_cuda is not installed.'
try:
import dask_cuda as _ # noqa
return {'condition': False, 'reason': reason}
except ImportError:
return {'condition': True, 'reason': reason}
def no_cudf():
return {'condition': not CUDF_INSTALLED,
'reason': 'CUDF is not installed'}

7
tests/travis/run_test.sh
View File

@@ -34,6 +34,13 @@ fi
if [ ${TASK} == "cmake_test" ]; then
set -e
if grep -n -R '<<<.*>>>\(.*\)' src include | grep --invert "NOLINT"; then
echo 'Do not use raw CUDA execution configuration syntax with <<<blocks, threads>>>.' \
'try `dh::LaunchKernel`'
exit -1
fi
# Build/test
rm -rf build
mkdir build && cd build