Merge model compatibility fixes from 1.0rc branch. (#5305)

* Port test model compatibility.
* Port logit model fix.

https://github.com/dmlc/xgboost/pull/5248
https://github.com/dmlc/xgboost/pull/5281

tests/ci_build/Dockerfile.cpu

@@ -21,8 +21,9 @@ ENV GOSU_VERSION 1.10
 
 # Install Python packages
 RUN \
-    pip install pyyaml cpplint pylint astroid sphinx numpy scipy pandas matplotlib sh recommonmark guzzle_sphinx_theme mock \
-                breathe matplotlib graphviz pytest scikit-learn wheel kubernetes urllib3 jsonschema && \
+    pip install pyyaml cpplint pylint astroid sphinx numpy scipy pandas matplotlib sh \
+    		recommonmark guzzle_sphinx_theme mock breathe matplotlib graphviz \
+		pytest scikit-learn wheel kubernetes urllib3 jsonschema boto3 && \
     pip install https://h2o-release.s3.amazonaws.com/datatable/stable/datatable-0.7.0/datatable-0.7.0-cp37-cp37m-linux_x86_64.whl && \
     pip install "dask[complete]"
 

tests/cpp/test_learner.cc

@@ -183,6 +183,41 @@ TEST(Learner, JsonModelIO) {
   delete pp_dmat;
 }
 
+TEST(Learner, BinaryModelIO) {
+  size_t constexpr kRows = 8;
+  int32_t constexpr kIters = 4;
+  auto pp_dmat = CreateDMatrix(kRows, 10, 0);
+  std::shared_ptr<DMatrix> p_dmat {*pp_dmat};
+  p_dmat->Info().labels_.Resize(kRows);
+
+  std::unique_ptr<Learner> learner{Learner::Create({p_dmat})};
+  learner->SetParam("eval_metric", "rmsle");
+  learner->Configure();
+  for (int32_t iter = 0; iter < kIters; ++iter) {
+    learner->UpdateOneIter(iter, p_dmat);
+  }
+  dmlc::TemporaryDirectory tempdir;
+  std::string const fname = tempdir.path + "binary_model_io.bin";
+  {
+    // Make sure the write is complete before loading.
+    std::unique_ptr<dmlc::Stream> fo(dmlc::Stream::Create(fname.c_str(), "w"));
+    learner->SaveModel(fo.get());
+  }
+
+  learner.reset(Learner::Create({p_dmat}));
+  std::unique_ptr<dmlc::Stream> fi(dmlc::Stream::Create(fname.c_str(), "r"));
+  learner->LoadModel(fi.get());
+  learner->Configure();
+  Json config { Object() };
+  learner->SaveConfig(&config);
+  std::string config_str;
+  Json::Dump(config, &config_str);
+  ASSERT_NE(config_str.find("rmsle"), std::string::npos);
+  ASSERT_EQ(config_str.find("WARNING"), std::string::npos);
+
+  delete pp_dmat;
+}
+
 #if defined(XGBOOST_USE_CUDA)
 // Tests for automatic GPU configuration.
 TEST(Learner, GPUConfiguration) {

tests/python/generate_models.py

@@ -0,0 +1,148 @@
+import xgboost
+import numpy as np
+import os
+
+kRounds = 2
+kRows = 1000
+kCols = 4
+kForests = 2
+kMaxDepth = 2
+kClasses = 3
+
+X = np.random.randn(kRows, kCols)
+w = np.random.uniform(size=kRows)
+
+version = xgboost.__version__
+
+np.random.seed(1994)
+target_dir = 'models'
+
+
+def booster_bin(model):
+    return os.path.join(target_dir,
+                        'xgboost-' + version + '.' + model + '.bin')
+
+
+def booster_json(model):
+    return os.path.join(target_dir,
+                        'xgboost-' + version + '.' + model + '.json')
+
+
+def skl_bin(model):
+    return os.path.join(target_dir,
+                        'xgboost_scikit-' + version + '.' + model + '.bin')
+
+
+def skl_json(model):
+    return os.path.join(target_dir,
+                        'xgboost_scikit-' + version + '.' + model + '.json')
+
+
+def generate_regression_model():
+    print('Regression')
+    y = np.random.randn(kRows)
+
+    data = xgboost.DMatrix(X, label=y, weight=w)
+    booster = xgboost.train({'tree_method': 'hist',
+                             'num_parallel_tree': kForests,
+                             'max_depth': kMaxDepth},
+                            num_boost_round=kRounds, dtrain=data)
+    booster.save_model(booster_bin('reg'))
+    booster.save_model(booster_json('reg'))
+
+    reg = xgboost.XGBRegressor(tree_method='hist',
+                               num_parallel_tree=kForests,
+                               max_depth=kMaxDepth,
+                               n_estimators=kRounds)
+    reg.fit(X, y, w)
+    reg.save_model(skl_bin('reg'))
+    reg.save_model(skl_json('reg'))
+
+
+def generate_logistic_model():
+    print('Logistic')
+    y = np.random.randint(0, 2, size=kRows)
+    assert y.max() == 1 and y.min() == 0
+
+    data = xgboost.DMatrix(X, label=y, weight=w)
+    booster = xgboost.train({'tree_method': 'hist',
+                             'num_parallel_tree': kForests,
+                             'max_depth': kMaxDepth,
+                             'objective': 'binary:logistic'},
+                            num_boost_round=kRounds, dtrain=data)
+    booster.save_model(booster_bin('logit'))
+    booster.save_model(booster_json('logit'))
+
+    reg = xgboost.XGBClassifier(tree_method='hist',
+                                num_parallel_tree=kForests,
+                                max_depth=kMaxDepth,
+                                n_estimators=kRounds)
+    reg.fit(X, y, w)
+    reg.save_model(skl_bin('logit'))
+    reg.save_model(skl_json('logit'))
+
+
+def generate_classification_model():
+    print('Classification')
+    y = np.random.randint(0, kClasses, size=kRows)
+    data = xgboost.DMatrix(X, label=y, weight=w)
+    booster = xgboost.train({'num_class': kClasses,
+                             'tree_method': 'hist',
+                             'num_parallel_tree': kForests,
+                             'max_depth': kMaxDepth},
+                            num_boost_round=kRounds, dtrain=data)
+    booster.save_model(booster_bin('cls'))
+    booster.save_model(booster_json('cls'))
+
+    cls = xgboost.XGBClassifier(tree_method='hist',
+                                num_parallel_tree=kForests,
+                                max_depth=kMaxDepth,
+                                n_estimators=kRounds)
+    cls.fit(X, y, w)
+    cls.save_model(skl_bin('cls'))
+    cls.save_model(skl_json('cls'))
+
+
+def generate_ranking_model():
+    print('Learning to Rank')
+    y = np.random.randint(5, size=kRows)
+    w = np.random.uniform(size=20)
+    g = np.repeat(50, 20)
+
+    data = xgboost.DMatrix(X, y, weight=w)
+    data.set_group(g)
+    booster = xgboost.train({'objective': 'rank:ndcg',
+                             'num_parallel_tree': kForests,
+                             'tree_method': 'hist',
+                             'max_depth': kMaxDepth},
+                            num_boost_round=kRounds,
+                            dtrain=data)
+    booster.save_model(booster_bin('ltr'))
+    booster.save_model(booster_json('ltr'))
+
+    ranker = xgboost.sklearn.XGBRanker(n_estimators=kRounds,
+                                       tree_method='hist',
+                                       objective='rank:ndcg',
+                                       max_depth=kMaxDepth,
+                                       num_parallel_tree=kForests)
+    ranker.fit(X, y, g, sample_weight=w)
+    ranker.save_model(skl_bin('ltr'))
+    ranker.save_model(skl_json('ltr'))
+
+
+def write_versions():
+    versions = {'numpy': np.__version__,
+                'xgboost': version}
+    with open(os.path.join(target_dir, 'version'), 'w') as fd:
+        fd.write(str(versions))
+
+
+if __name__ == '__main__':
+    if not os.path.exists(target_dir):
+        os.mkdir(target_dir)
+
+    generate_regression_model()
+    generate_logistic_model()
+    generate_classification_model()
+    generate_ranking_model()
+    write_versions()

tests/python/test_basic.py

@@ -39,7 +39,7 @@ class TestBasic(unittest.TestCase):
     def test_basic(self):
         dtrain = xgb.DMatrix(dpath + 'agaricus.txt.train')
         dtest = xgb.DMatrix(dpath + 'agaricus.txt.test')
-        param = {'max_depth': 2, 'eta': 1, 'verbosity': 0,
+        param = {'max_depth': 2, 'eta': 1,
                  'objective': 'binary:logistic'}
         # specify validations set to watch performance
         watchlist = [(dtest, 'eval'), (dtrain, 'train')]

tests/python/test_basic_models.py

@@ -284,16 +284,31 @@ class TestModels(unittest.TestCase):
         self.assertRaises(ValueError, bst.predict, dm1)
         bst.predict(dm2)  # success
 
+    def test_model_binary_io(self):
+        model_path = 'test_model_binary_io.bin'
+        parameters = {'tree_method': 'hist', 'booster': 'gbtree',
+                      'scale_pos_weight': '0.5'}
+        X = np.random.random((10, 3))
+        y = np.random.random((10,))
+        dtrain = xgb.DMatrix(X, y)
+        bst = xgb.train(parameters, dtrain, num_boost_round=2)
+        bst.save_model(model_path)
+        bst = xgb.Booster(model_file=model_path)
+        os.remove(model_path)
+        config = json.loads(bst.save_config())
+        assert float(config['learner']['objective'][
+            'reg_loss_param']['scale_pos_weight']) == 0.5
+
     def test_model_json_io(self):
-        model_path = './model.json'
+        model_path = 'test_model_json_io.json'
         parameters = {'tree_method': 'hist', 'booster': 'gbtree'}
         j_model = json_model(model_path, parameters)
         assert isinstance(j_model['learner'], dict)
 
-        bst = xgb.Booster(model_file='./model.json')
+        bst = xgb.Booster(model_file=model_path)
 
         bst.save_model(fname=model_path)
-        with open('./model.json', 'r') as fd:
+        with open(model_path, 'r') as fd:
             j_model = json.load(fd)
         assert isinstance(j_model['learner'], dict)
 
@@ -302,7 +317,7 @@ class TestModels(unittest.TestCase):
     @pytest.mark.skipif(**tm.no_json_schema())
     def test_json_schema(self):
         import jsonschema
-        model_path = './model.json'
+        model_path = 'test_json_schema.json'
         path = os.path.dirname(
             os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
         doc = os.path.join(path, 'doc', 'model.schema')

tests/python/test_model_compatibility.py

@@ -0,0 +1,130 @@
+import xgboost
+import os
+import generate_models as gm
+import json
+import zipfile
+import pytest
+
+
+def run_model_param_check(config):
+    assert config['learner']['learner_model_param']['num_feature'] == str(4)
+    assert config['learner']['learner_train_param']['booster'] == 'gbtree'
+
+
+def run_booster_check(booster, name):
+    config = json.loads(booster.save_config())
+    run_model_param_check(config)
+    if name.find('cls') != -1:
+        assert (len(booster.get_dump()) == gm.kForests * gm.kRounds *
+                gm.kClasses)
+        assert float(
+            config['learner']['learner_model_param']['base_score']) == 0.5
+        assert config['learner']['learner_train_param'][
+            'objective'] == 'multi:softmax'
+    elif name.find('logit') != -1:
+        assert len(booster.get_dump()) == gm.kForests * gm.kRounds
+        assert config['learner']['learner_model_param']['num_class'] == str(0)
+        assert config['learner']['learner_train_param'][
+            'objective'] == 'binary:logistic'
+    elif name.find('ltr') != -1:
+        assert config['learner']['learner_train_param'][
+            'objective'] == 'rank:ndcg'
+    else:
+        assert name.find('reg') != -1
+        assert len(booster.get_dump()) == gm.kForests * gm.kRounds
+        assert float(
+            config['learner']['learner_model_param']['base_score']) == 0.5
+        assert config['learner']['learner_train_param'][
+            'objective'] == 'reg:squarederror'
+
+
+def run_scikit_model_check(name, path):
+    if name.find('reg') != -1:
+        reg = xgboost.XGBRegressor()
+        reg.load_model(path)
+        config = json.loads(reg.get_booster().save_config())
+        if name.find('0.90') != -1:
+            assert config['learner']['learner_train_param'][
+                'objective'] == 'reg:linear'
+        else:
+            assert config['learner']['learner_train_param'][
+                'objective'] == 'reg:squarederror'
+        assert (len(reg.get_booster().get_dump()) ==
+                gm.kRounds * gm.kForests)
+        run_model_param_check(config)
+    elif name.find('cls') != -1:
+        cls = xgboost.XGBClassifier()
+        cls.load_model(path)
+        if name.find('0.90') == -1:
+            assert len(cls.classes_) == gm.kClasses
+            assert len(cls._le.classes_) == gm.kClasses
+            assert cls.n_classes_ == gm.kClasses
+        assert (len(cls.get_booster().get_dump()) ==
+                gm.kRounds * gm.kForests * gm.kClasses), path
+        config = json.loads(cls.get_booster().save_config())
+        assert config['learner']['learner_train_param'][
+            'objective'] == 'multi:softprob', path
+        run_model_param_check(config)
+    elif name.find('ltr') != -1:
+        ltr = xgboost.XGBRanker()
+        ltr.load_model(path)
+        assert (len(ltr.get_booster().get_dump()) ==
+                gm.kRounds * gm.kForests)
+        config = json.loads(ltr.get_booster().save_config())
+        assert config['learner']['learner_train_param'][
+            'objective'] == 'rank:ndcg'
+        run_model_param_check(config)
+    elif name.find('logit') != -1:
+        logit = xgboost.XGBClassifier()
+        logit.load_model(path)
+        assert (len(logit.get_booster().get_dump()) ==
+                gm.kRounds * gm.kForests)
+        config = json.loads(logit.get_booster().save_config())
+        assert config['learner']['learner_train_param'][
+            'objective'] == 'binary:logistic'
+    else:
+        assert False
+
+
+@pytest.mark.ci
+def test_model_compatibility():
+    '''Test model compatibility, can only be run on CI as others don't
+    have the credentials.
+
+    '''
+    path = os.path.dirname(os.path.abspath(__file__))
+    path = os.path.join(path, 'models')
+    try:
+        import boto3
+        import botocore
+    except ImportError:
+        pytest.skip(
+            'Skiping compatibility tests as boto3 is not installed.')
+
+    try:
+        s3_bucket = boto3.resource('s3').Bucket('xgboost-ci-jenkins-artifacts')
+        zip_path = 'xgboost_model_compatibility_test.zip'
+        s3_bucket.download_file(zip_path, zip_path)
+    except botocore.exceptions.NoCredentialsError:
+        pytest.skip(
+            'Skiping compatibility tests as running on non-CI environment.')
+
+    with zipfile.ZipFile(zip_path, 'r') as z:
+        z.extractall(path)
+
+    models = [
+        os.path.join(root, f) for root, subdir, files in os.walk(path)
+        for f in files
+        if f != 'version'
+    ]
+    assert models
+
+    for path in models:
+        name = os.path.basename(path)
+        if name.startswith('xgboost-'):
+            booster = xgboost.Booster(model_file=path)
+            run_booster_check(booster, name)
+        elif name.startswith('xgboost_scikit'):
+            run_scikit_model_check(name, path)
+        else:
+            assert False

tests/python/test_ranking.py

@@ -115,7 +115,6 @@ class TestRanking(unittest.TestCase):
         # model training parameters
         cls.params = {'objective': 'rank:pairwise',
                       'booster': 'gbtree',
-                      'silent': 0,
                       'eval_metric': ['ndcg']
                       }
 
@@ -143,7 +142,7 @@ class TestRanking(unittest.TestCase):
         Test cross-validation with a group specified
         """
         cv = xgboost.cv(self.params, self.dtrain, num_boost_round=2500,
-                    early_stopping_rounds=10, nfold=10, as_pandas=False)
+                        early_stopping_rounds=10, nfold=10, as_pandas=False)
         assert isinstance(cv, dict)
         self.assertSetEqual(set(cv.keys()), {'test-ndcg-mean', 'train-ndcg-mean', 'test-ndcg-std', 'train-ndcg-std'},
                             "CV results dict key mismatch")
@@ -153,7 +152,8 @@ class TestRanking(unittest.TestCase):
         Test cross-validation with a group specified
         """
         cv = xgboost.cv(self.params, self.dtrain, num_boost_round=2500,
-                    early_stopping_rounds=10, shuffle=False, nfold=10, as_pandas=False)
+                        early_stopping_rounds=10, shuffle=False, nfold=10,
+                        as_pandas=False)
         assert isinstance(cv, dict)
         assert len(cv) == 4