Remove benchmark scripts. (#9992)
This commit is contained in:
Jiaming Yuan
GitHub
parent
d07e8b503e
commit
bde20dd897
@ -10,9 +10,7 @@ facilities.
|
|||||||
dependencies for tests, see conda files in `ci_build`.
|
dependencies for tests, see conda files in `ci_build`.
|
||||||
* python-gpu: Similar to python tests, but for GPU.
|
* python-gpu: Similar to python tests, but for GPU.
|
||||||
* travis: CI facilities for Travis.
|
* travis: CI facilities for Travis.
|
||||||
* distributed: Test for distributed system.
|
* test_distributed: Test for distributed systems including spark and dask.
|
||||||
* benchmark: Legacy benchmark code. There are a number of benchmark projects for
|
|
||||||
XGBoost with much better configurations.
|
|
||||||
|
|
||||||
# Others
|
# Others
|
||||||
* pytest.ini: Describes the `pytest` marker for python tests, some markers are generated
|
* pytest.ini: Describes the `pytest` marker for python tests, some markers are generated
|
||||||
|
|||||||
@ -1,69 +0,0 @@
|
|||||||
#pylint: skip-file
|
|
||||||
import argparse
|
|
||||||
import xgboost as xgb
|
|
||||||
import numpy as np
|
|
||||||
from sklearn.datasets import make_classification
|
|
||||||
from sklearn.model_selection import train_test_split
|
|
||||||
import time
|
|
||||||
import ast
|
|
||||||
|
|
||||||
rng = np.random.RandomState(1994)
|
|
||||||
|
|
||||||
|
|
||||||
def run_benchmark(args):
|
|
||||||
|
|
||||||
try:
|
|
||||||
dtest = xgb.DMatrix('dtest.dm')
|
|
||||||
dtrain = xgb.DMatrix('dtrain.dm')
|
|
||||||
|
|
||||||
if not (dtest.num_col() == args.columns \
|
|
||||||
and dtrain.num_col() == args.columns):
|
|
||||||
raise ValueError("Wrong cols")
|
|
||||||
if not (dtest.num_row() == args.rows * args.test_size \
|
|
||||||
and dtrain.num_row() == args.rows * (1-args.test_size)):
|
|
||||||
raise ValueError("Wrong rows")
|
|
||||||
except:
|
|
||||||
|
|
||||||
print("Generating dataset: {} rows * {} columns".format(args.rows, args.columns))
|
|
||||||
print("{}/{} test/train split".format(args.test_size, 1.0 - args.test_size))
|
|
||||||
tmp = time.time()
|
|
||||||
X, y = make_classification(args.rows, n_features=args.columns, n_redundant=0, n_informative=args.columns, n_repeated=0, random_state=7)
|
|
||||||
if args.sparsity < 1.0:
|
|
||||||
X = np.array([[np.nan if rng.uniform(0, 1) < args.sparsity else x for x in x_row] for x_row in X])
|
|
||||||
|
|
||||||
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=args.test_size, random_state=7)
|
|
||||||
print ("Generate Time: %s seconds" % (str(time.time() - tmp)))
|
|
||||||
tmp = time.time()
|
|
||||||
print ("DMatrix Start")
|
|
||||||
dtrain = xgb.DMatrix(X_train, y_train)
|
|
||||||
dtest = xgb.DMatrix(X_test, y_test, nthread=-1)
|
|
||||||
print ("DMatrix Time: %s seconds" % (str(time.time() - tmp)))
|
|
||||||
|
|
||||||
dtest.save_binary('dtest.dm')
|
|
||||||
dtrain.save_binary('dtrain.dm')
|
|
||||||
|
|
||||||
param = {'objective': 'binary:logistic','booster':'gblinear'}
|
|
||||||
if args.params != '':
|
|
||||||
param.update(ast.literal_eval(args.params))
|
|
||||||
|
|
||||||
param['updater'] = args.updater
|
|
||||||
print("Training with '%s'" % param['updater'])
|
|
||||||
tmp = time.time()
|
|
||||||
xgb.train(param, dtrain, args.iterations, evals=[(dtrain,"train")], early_stopping_rounds = args.columns)
|
|
||||||
print ("Train Time: %s seconds" % (str(time.time() - tmp)))
|
|
||||||
|
|
||||||
parser = argparse.ArgumentParser()
|
|
||||||
parser.add_argument('--updater', default='coord_descent')
|
|
||||||
parser.add_argument('--sparsity', type=float, default=0.0)
|
|
||||||
parser.add_argument('--lambda', type=float, default=1.0)
|
|
||||||
parser.add_argument('--tol', type=float, default=1e-5)
|
|
||||||
parser.add_argument('--alpha', type=float, default=1.0)
|
|
||||||
parser.add_argument('--rows', type=int, default=1000000)
|
|
||||||
parser.add_argument('--iterations', type=int, default=10000)
|
|
||||||
parser.add_argument('--columns', type=int, default=50)
|
|
||||||
parser.add_argument('--test_size', type=float, default=0.25)
|
|
||||||
parser.add_argument('--standardise', type=bool, default=False)
|
|
||||||
parser.add_argument('--params', default='', help='Provide additional parameters as a Python dict string, e.g. --params \"{\'max_depth\':2}\"')
|
|
||||||
args = parser.parse_args()
|
|
||||||
|
|
||||||
run_benchmark(args)
|
|
||||||
@ -1,86 +0,0 @@
|
|||||||
"""Run benchmark on the tree booster."""
|
|
||||||
|
|
||||||
import argparse
|
|
||||||
import ast
|
|
||||||
import time
|
|
||||||
|
|
||||||
import numpy as np
|
|
||||||
import xgboost as xgb
|
|
||||||
|
|
||||||
RNG = np.random.RandomState(1994)
|
|
||||||
|
|
||||||
|
|
||||||
def run_benchmark(args):
|
|
||||||
"""Runs the benchmark."""
|
|
||||||
try:
|
|
||||||
dtest = xgb.DMatrix('dtest.dm')
|
|
||||||
dtrain = xgb.DMatrix('dtrain.dm')
|
|
||||||
|
|
||||||
if not (dtest.num_col() == args.columns
|
|
||||||
and dtrain.num_col() == args.columns):
|
|
||||||
raise ValueError("Wrong cols")
|
|
||||||
if not (dtest.num_row() == args.rows * args.test_size
|
|
||||||
and dtrain.num_row() == args.rows * (1 - args.test_size)):
|
|
||||||
raise ValueError("Wrong rows")
|
|
||||||
except:
|
|
||||||
print("Generating dataset: {} rows * {} columns".format(args.rows, args.columns))
|
|
||||||
print("{}/{} test/train split".format(args.test_size, 1.0 - args.test_size))
|
|
||||||
tmp = time.time()
|
|
||||||
X = RNG.rand(args.rows, args.columns)
|
|
||||||
y = RNG.randint(0, 2, args.rows)
|
|
||||||
if 0.0 < args.sparsity < 1.0:
|
|
||||||
X = np.array([[np.nan if RNG.uniform(0, 1) < args.sparsity else x for x in x_row]
|
|
||||||
for x_row in X])
|
|
||||||
|
|
||||||
train_rows = int(args.rows * (1.0 - args.test_size))
|
|
||||||
test_rows = int(args.rows * args.test_size)
|
|
||||||
X_train = X[:train_rows, :]
|
|
||||||
X_test = X[-test_rows:, :]
|
|
||||||
y_train = y[:train_rows]
|
|
||||||
y_test = y[-test_rows:]
|
|
||||||
print("Generate Time: %s seconds" % (str(time.time() - tmp)))
|
|
||||||
del X, y
|
|
||||||
|
|
||||||
tmp = time.time()
|
|
||||||
print("DMatrix Start")
|
|
||||||
dtrain = xgb.DMatrix(X_train, y_train, nthread=-1)
|
|
||||||
dtest = xgb.DMatrix(X_test, y_test, nthread=-1)
|
|
||||||
print("DMatrix Time: %s seconds" % (str(time.time() - tmp)))
|
|
||||||
del X_train, y_train, X_test, y_test
|
|
||||||
|
|
||||||
dtest.save_binary('dtest.dm')
|
|
||||||
dtrain.save_binary('dtrain.dm')
|
|
||||||
|
|
||||||
param = {'objective': 'binary:logistic'}
|
|
||||||
if args.params != '':
|
|
||||||
param.update(ast.literal_eval(args.params))
|
|
||||||
|
|
||||||
param['tree_method'] = args.tree_method
|
|
||||||
print("Training with '%s'" % param['tree_method'])
|
|
||||||
tmp = time.time()
|
|
||||||
xgb.train(param, dtrain, args.iterations, evals=[(dtest, "test")])
|
|
||||||
print("Train Time: %s seconds" % (str(time.time() - tmp)))
|
|
||||||
|
|
||||||
|
|
||||||
def main():
|
|
||||||
"""The main function.
|
|
||||||
|
|
||||||
Defines and parses command line arguments and calls the benchmark.
|
|
||||||
"""
|
|
||||||
parser = argparse.ArgumentParser()
|
|
||||||
parser.add_argument('--tree_method', default='gpu_hist')
|
|
||||||
parser.add_argument('--sparsity', type=float, default=0.0)
|
|
||||||
parser.add_argument('--rows', type=int, default=1000000)
|
|
||||||
parser.add_argument('--columns', type=int, default=50)
|
|
||||||
parser.add_argument('--iterations', type=int, default=500)
|
|
||||||
parser.add_argument('--test_size', type=float, default=0.25)
|
|
||||||
parser.add_argument('--params', default='',
|
|
||||||
help='Provide additional parameters as a Python dict string, e.g. --params '
|
|
||||||
'\"{\'max_depth\':2}\"')
|
|
||||||
args = parser.parse_args()
|
|
||||||
|
|
||||||
run_benchmark(args)
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == '__main__':
|
|
||||||
main()
|
|
||||||
@ -1,87 +0,0 @@
|
|||||||
"""Generate synthetic data in LIBSVM format."""
|
|
||||||
|
|
||||||
import argparse
|
|
||||||
import io
|
|
||||||
import time
|
|
||||||
|
|
||||||
import numpy as np
|
|
||||||
from sklearn.datasets import make_classification
|
|
||||||
from sklearn.model_selection import train_test_split
|
|
||||||
|
|
||||||
RNG = np.random.RandomState(2019)
|
|
||||||
|
|
||||||
|
|
||||||
def generate_data(args):
|
|
||||||
"""Generates the data."""
|
|
||||||
print("Generating dataset: {} rows * {} columns".format(args.rows, args.columns))
|
|
||||||
print("Sparsity {}".format(args.sparsity))
|
|
||||||
print("{}/{} train/test split".format(1.0 - args.test_size, args.test_size))
|
|
||||||
|
|
||||||
tmp = time.time()
|
|
||||||
n_informative = args.columns * 7 // 10
|
|
||||||
n_redundant = args.columns // 10
|
|
||||||
n_repeated = args.columns // 10
|
|
||||||
print("n_informative: {}, n_redundant: {}, n_repeated: {}".format(n_informative, n_redundant,
|
|
||||||
n_repeated))
|
|
||||||
x, y = make_classification(n_samples=args.rows, n_features=args.columns,
|
|
||||||
n_informative=n_informative, n_redundant=n_redundant,
|
|
||||||
n_repeated=n_repeated, shuffle=False, random_state=RNG)
|
|
||||||
print("Generate Time: {} seconds".format(time.time() - tmp))
|
|
||||||
|
|
||||||
tmp = time.time()
|
|
||||||
x_train, x_test, y_train, y_test = train_test_split(x, y, test_size=args.test_size,
|
|
||||||
random_state=RNG, shuffle=False)
|
|
||||||
print("Train/Test Split Time: {} seconds".format(time.time() - tmp))
|
|
||||||
|
|
||||||
tmp = time.time()
|
|
||||||
write_file('train.libsvm', x_train, y_train, args.sparsity)
|
|
||||||
print("Write Train Time: {} seconds".format(time.time() - tmp))
|
|
||||||
|
|
||||||
tmp = time.time()
|
|
||||||
write_file('test.libsvm', x_test, y_test, args.sparsity)
|
|
||||||
print("Write Test Time: {} seconds".format(time.time() - tmp))
|
|
||||||
|
|
||||||
|
|
||||||
def write_file(filename, x_data, y_data, sparsity):
|
|
||||||
with open(filename, 'w') as f:
|
|
||||||
for x, y in zip(x_data, y_data):
|
|
||||||
write_line(f, x, y, sparsity)
|
|
||||||
|
|
||||||
|
|
||||||
def write_line(f, x, y, sparsity):
|
|
||||||
with io.StringIO() as line:
|
|
||||||
line.write(str(y))
|
|
||||||
for i, col in enumerate(x):
|
|
||||||
if 0.0 < sparsity < 1.0:
|
|
||||||
if RNG.uniform(0, 1) > sparsity:
|
|
||||||
write_feature(line, i, col)
|
|
||||||
else:
|
|
||||||
write_feature(line, i, col)
|
|
||||||
line.write('\n')
|
|
||||||
f.write(line.getvalue())
|
|
||||||
|
|
||||||
|
|
||||||
def write_feature(line, index, feature):
|
|
||||||
line.write(' ')
|
|
||||||
line.write(str(index))
|
|
||||||
line.write(':')
|
|
||||||
line.write(str(feature))
|
|
||||||
|
|
||||||
|
|
||||||
def main():
|
|
||||||
"""The main function.
|
|
||||||
|
|
||||||
Defines and parses command line arguments and calls the generator.
|
|
||||||
"""
|
|
||||||
parser = argparse.ArgumentParser()
|
|
||||||
parser.add_argument('--rows', type=int, default=1000000)
|
|
||||||
parser.add_argument('--columns', type=int, default=50)
|
|
||||||
parser.add_argument('--sparsity', type=float, default=0.0)
|
|
||||||
parser.add_argument('--test_size', type=float, default=0.01)
|
|
||||||
args = parser.parse_args()
|
|
||||||
|
|
||||||
generate_data(args)
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == '__main__':
|
|
||||||
main()
|
|
||||||
Loading…
x
Reference in New Issue
Block a user