GPU implementation of AFT survival objective and metric (#5714)
* Add interval accuracy * De-virtualize AFT functions * Lint * Refactor AFT metric using GPU-CPU reducer * Fix R build * Fix build on Windows * Fix copyright header * Clang-tidy * Fix crashing demo * Fix typos in comment; explain GPU ID * Remove unnecessary #include * Add C++ test for interval accuracy * Fix a bug in accuracy metric: use log pred * Refactor AFT objective using GPU-CPU Transform * Lint * Fix lint * Use Ninja to speed up build * Use time, not /usr/bin/time * Add cpu_build worker class, with concurrency = 1 * Use concurrency = 1 only for CUDA build * concurrency = 1 for clang-tidy * Address reviewer's feedback * Update link to AFT paper
This commit is contained in:
Philip Hyunsu Cho
committed by
GitHub
GitHub
parent
7c2686146e
commit
71b0528a2f
@@ -11,59 +11,56 @@
|
||||
namespace xgboost {
|
||||
namespace common {
|
||||
|
||||
TEST(ProbabilityDistribution, DistributionGeneric) {
|
||||
// Assert d/dx CDF = PDF, d/dx PDF = GradPDF, d/dx GradPDF = HessPDF
|
||||
// Do this for every distribution type
|
||||
for (auto type : {ProbabilityDistributionType::kNormal, ProbabilityDistributionType::kLogistic,
|
||||
ProbabilityDistributionType::kExtreme}) {
|
||||
std::unique_ptr<ProbabilityDistribution> dist{ ProbabilityDistribution::Create(type) };
|
||||
double integral_of_pdf = dist->CDF(-2.0);
|
||||
double integral_of_grad_pdf = dist->PDF(-2.0);
|
||||
double integral_of_hess_pdf = dist->GradPDF(-2.0);
|
||||
// Perform numerical differentiation and integration
|
||||
// Enumerate 4000 grid points in range [-2, 2]
|
||||
for (int i = 0; i <= 4000; ++i) {
|
||||
const double x = static_cast<double>(i) / 1000.0 - 2.0;
|
||||
// Numerical differentiation (p. 246, Numerical Analysis 2nd ed. by Timothy Sauer)
|
||||
EXPECT_NEAR((dist->CDF(x + 1e-5) - dist->CDF(x - 1e-5)) / 2e-5, dist->PDF(x), 6e-11);
|
||||
EXPECT_NEAR((dist->PDF(x + 1e-5) - dist->PDF(x - 1e-5)) / 2e-5, dist->GradPDF(x), 6e-11);
|
||||
EXPECT_NEAR((dist->GradPDF(x + 1e-5) - dist->GradPDF(x - 1e-5)) / 2e-5,
|
||||
dist->HessPDF(x), 6e-11);
|
||||
// Numerical integration using Trapezoid Rule (p. 257, Sauer)
|
||||
integral_of_pdf += 5e-4 * (dist->PDF(x - 1e-3) + dist->PDF(x));
|
||||
integral_of_grad_pdf += 5e-4 * (dist->GradPDF(x - 1e-3) + dist->GradPDF(x));
|
||||
integral_of_hess_pdf += 5e-4 * (dist->HessPDF(x - 1e-3) + dist->HessPDF(x));
|
||||
EXPECT_NEAR(integral_of_pdf, dist->CDF(x), 2e-4);
|
||||
EXPECT_NEAR(integral_of_grad_pdf, dist->PDF(x), 2e-4);
|
||||
EXPECT_NEAR(integral_of_hess_pdf, dist->GradPDF(x), 2e-4);
|
||||
}
|
||||
template <typename Distribution>
|
||||
void RunDistributionGenericTest() {
|
||||
double integral_of_pdf = Distribution::CDF(-2.0);
|
||||
double integral_of_grad_pdf = Distribution::PDF(-2.0);
|
||||
double integral_of_hess_pdf = Distribution::GradPDF(-2.0);
|
||||
// Perform numerical differentiation and integration
|
||||
// Enumerate 4000 grid points in range [-2, 2]
|
||||
for (int i = 0; i <= 4000; ++i) {
|
||||
const double x = static_cast<double>(i) / 1000.0 - 2.0;
|
||||
// Numerical differentiation (p. 246, Numerical Analysis 2nd ed. by Timothy Sauer)
|
||||
EXPECT_NEAR((Distribution::CDF(x + 1e-5) - Distribution::CDF(x - 1e-5)) / 2e-5,
|
||||
Distribution::PDF(x), 6e-11);
|
||||
EXPECT_NEAR((Distribution::PDF(x + 1e-5) - Distribution::PDF(x - 1e-5)) / 2e-5,
|
||||
Distribution::GradPDF(x), 6e-11);
|
||||
EXPECT_NEAR((Distribution::GradPDF(x + 1e-5) - Distribution::GradPDF(x - 1e-5)) / 2e-5,
|
||||
Distribution::HessPDF(x), 6e-11);
|
||||
// Numerical integration using Trapezoid Rule (p. 257, Sauer)
|
||||
integral_of_pdf += 5e-4 * (Distribution::PDF(x - 1e-3) + Distribution::PDF(x));
|
||||
integral_of_grad_pdf += 5e-4 * (Distribution::GradPDF(x - 1e-3) + Distribution::GradPDF(x));
|
||||
integral_of_hess_pdf += 5e-4 * (Distribution::HessPDF(x - 1e-3) + Distribution::HessPDF(x));
|
||||
EXPECT_NEAR(integral_of_pdf, Distribution::CDF(x), 2e-4);
|
||||
EXPECT_NEAR(integral_of_grad_pdf, Distribution::PDF(x), 2e-4);
|
||||
EXPECT_NEAR(integral_of_hess_pdf, Distribution::GradPDF(x), 2e-4);
|
||||
}
|
||||
}
|
||||
|
||||
TEST(ProbabilityDistribution, NormalDist) {
|
||||
std::unique_ptr<ProbabilityDistribution> dist{
|
||||
ProbabilityDistribution::Create(ProbabilityDistributionType::kNormal)
|
||||
};
|
||||
TEST(ProbabilityDistribution, DistributionGeneric) {
|
||||
// Assert d/dx CDF = PDF, d/dx PDF = GradPDF, d/dx GradPDF = HessPDF
|
||||
// Do this for every distribution type
|
||||
RunDistributionGenericTest<NormalDistribution>();
|
||||
RunDistributionGenericTest<LogisticDistribution>();
|
||||
RunDistributionGenericTest<ExtremeDistribution>();
|
||||
}
|
||||
|
||||
TEST(ProbabilityDistribution, NormalDist) {
|
||||
// "Three-sigma rule" (https://en.wikipedia.org/wiki/68–95–99.7_rule)
|
||||
// 68% of values are within 1 standard deviation away from the mean
|
||||
// 95% of values are within 2 standard deviation away from the mean
|
||||
// 99.7% of values are within 3 standard deviation away from the mean
|
||||
EXPECT_NEAR(dist->CDF(0.5) - dist->CDF(-0.5), 0.3829, 0.00005);
|
||||
EXPECT_NEAR(dist->CDF(1.0) - dist->CDF(-1.0), 0.6827, 0.00005);
|
||||
EXPECT_NEAR(dist->CDF(1.5) - dist->CDF(-1.5), 0.8664, 0.00005);
|
||||
EXPECT_NEAR(dist->CDF(2.0) - dist->CDF(-2.0), 0.9545, 0.00005);
|
||||
EXPECT_NEAR(dist->CDF(2.5) - dist->CDF(-2.5), 0.9876, 0.00005);
|
||||
EXPECT_NEAR(dist->CDF(3.0) - dist->CDF(-3.0), 0.9973, 0.00005);
|
||||
EXPECT_NEAR(dist->CDF(3.5) - dist->CDF(-3.5), 0.9995, 0.00005);
|
||||
EXPECT_NEAR(dist->CDF(4.0) - dist->CDF(-4.0), 0.9999, 0.00005);
|
||||
EXPECT_NEAR(NormalDistribution::CDF(0.5) - NormalDistribution::CDF(-0.5), 0.3829, 0.00005);
|
||||
EXPECT_NEAR(NormalDistribution::CDF(1.0) - NormalDistribution::CDF(-1.0), 0.6827, 0.00005);
|
||||
EXPECT_NEAR(NormalDistribution::CDF(1.5) - NormalDistribution::CDF(-1.5), 0.8664, 0.00005);
|
||||
EXPECT_NEAR(NormalDistribution::CDF(2.0) - NormalDistribution::CDF(-2.0), 0.9545, 0.00005);
|
||||
EXPECT_NEAR(NormalDistribution::CDF(2.5) - NormalDistribution::CDF(-2.5), 0.9876, 0.00005);
|
||||
EXPECT_NEAR(NormalDistribution::CDF(3.0) - NormalDistribution::CDF(-3.0), 0.9973, 0.00005);
|
||||
EXPECT_NEAR(NormalDistribution::CDF(3.5) - NormalDistribution::CDF(-3.5), 0.9995, 0.00005);
|
||||
EXPECT_NEAR(NormalDistribution::CDF(4.0) - NormalDistribution::CDF(-4.0), 0.9999, 0.00005);
|
||||
}
|
||||
|
||||
TEST(ProbabilityDistribution, LogisticDist) {
|
||||
std::unique_ptr<ProbabilityDistribution> dist{
|
||||
ProbabilityDistribution::Create(ProbabilityDistributionType::kLogistic)
|
||||
};
|
||||
|
||||
/**
|
||||
* Enforce known properties of the logistic distribution.
|
||||
* (https://en.wikipedia.org/wiki/Logistic_distribution)
|
||||
@@ -74,17 +71,13 @@ TEST(ProbabilityDistribution, LogisticDist) {
|
||||
const double x = static_cast<double>(i) / 1000.0 - 2.0;
|
||||
// PDF = 1/4 * sech(x/2)**2
|
||||
const double sech_x = 1.0 / std::cosh(x * 0.5); // hyperbolic secant at x/2
|
||||
EXPECT_NEAR(0.25 * sech_x * sech_x, dist->PDF(x), 1e-15);
|
||||
EXPECT_NEAR(0.25 * sech_x * sech_x, LogisticDistribution::PDF(x), 1e-15);
|
||||
// CDF = 1/2 + 1/2 * tanh(x/2)
|
||||
EXPECT_NEAR(0.5 + 0.5 * std::tanh(x * 0.5), dist->CDF(x), 1e-15);
|
||||
EXPECT_NEAR(0.5 + 0.5 * std::tanh(x * 0.5), LogisticDistribution::CDF(x), 1e-15);
|
||||
}
|
||||
}
|
||||
|
||||
TEST(ProbabilityDistribution, ExtremeDist) {
|
||||
std::unique_ptr<ProbabilityDistribution> dist{
|
||||
ProbabilityDistribution::Create(ProbabilityDistributionType::kExtreme)
|
||||
};
|
||||
|
||||
/**
|
||||
* Enforce known properties of the extreme distribution (also known as Gumbel distribution).
|
||||
* The mean is the negative of the Euler-Mascheroni constant.
|
||||
@@ -99,9 +92,10 @@ TEST(ProbabilityDistribution, ExtremeDist) {
|
||||
for (int i = 0; i <= 25000; ++i) {
|
||||
const double x = static_cast<double>(i) / 1000.0 - 20.0;
|
||||
// Numerical integration using Trapezoid Rule (p. 257, Sauer)
|
||||
mean += 5e-4 * ((x - 1e-3) * dist->PDF(x - 1e-3) + x * dist->PDF(x));
|
||||
mean +=
|
||||
5e-4 * ((x - 1e-3) * ExtremeDistribution::PDF(x - 1e-3) + x * ExtremeDistribution::PDF(x));
|
||||
}
|
||||
EXPECT_NEAR(mean, -probability_constant::kEulerMascheroni, 1e-7);
|
||||
EXPECT_NEAR(mean, -kEulerMascheroni, 1e-7);
|
||||
|
||||
// Enumerate 25000 grid points in range [-20, 5].
|
||||
// Compute the variance of the distribution using numerical integration.
|
||||
@@ -111,10 +105,10 @@ TEST(ProbabilityDistribution, ExtremeDist) {
|
||||
for (int i = 0; i <= 25000; ++i) {
|
||||
const double x = static_cast<double>(i) / 1000.0 - 20.0;
|
||||
// Numerical integration using Trapezoid Rule (p. 257, Sauer)
|
||||
variance += 5e-4 * ((x - 1e-3 - mean) * (x - 1e-3 - mean) * dist->PDF(x - 1e-3)
|
||||
+ (x - mean) * (x - mean) * dist->PDF(x));
|
||||
variance += 5e-4 * ((x - 1e-3 - mean) * (x - 1e-3 - mean) * ExtremeDistribution::PDF(x - 1e-3)
|
||||
+ (x - mean) * (x - mean) * ExtremeDistribution::PDF(x));
|
||||
}
|
||||
EXPECT_NEAR(variance, probability_constant::kPI * probability_constant::kPI / 6.0, 1e-6);
|
||||
EXPECT_NEAR(variance, kPI * kPI / 6.0, 1e-6);
|
||||
}
|
||||
|
||||
} // namespace common
|
||||
|
||||
@@ -8,36 +8,37 @@
|
||||
namespace xgboost {
|
||||
namespace common {
|
||||
|
||||
inline static void RobustTestSuite(ProbabilityDistributionType dist_type,
|
||||
double y_lower, double y_upper, double sigma) {
|
||||
AFTLoss loss(dist_type);
|
||||
template <typename Distribution>
|
||||
inline static void RobustTestSuite(double y_lower, double y_upper, double sigma) {
|
||||
for (int i = 50; i >= -50; --i) {
|
||||
const double y_pred = std::pow(10.0, static_cast<double>(i));
|
||||
const double z = (std::log(y_lower) - std::log(y_pred)) / sigma;
|
||||
const double gradient = loss.Gradient(y_lower, y_upper, std::log(y_pred), sigma);
|
||||
const double hessian = loss.Hessian(y_lower, y_upper, std::log(y_pred), sigma);
|
||||
const double gradient
|
||||
= AFTLoss<Distribution>::Gradient(y_lower, y_upper, std::log(y_pred), sigma);
|
||||
const double hessian
|
||||
= AFTLoss<Distribution>::Hessian(y_lower, y_upper, std::log(y_pred), sigma);
|
||||
ASSERT_FALSE(std::isnan(gradient)) << "z = " << z << ", y \\in ["
|
||||
<< y_lower << ", " << y_upper << "], y_pred = " << y_pred
|
||||
<< ", dist = " << static_cast<int>(dist_type);
|
||||
<< ", dist = " << static_cast<int>(Distribution::Type());
|
||||
ASSERT_FALSE(std::isinf(gradient)) << "z = " << z << ", y \\in ["
|
||||
<< y_lower << ", " << y_upper << "], y_pred = " << y_pred
|
||||
<< ", dist = " << static_cast<int>(dist_type);
|
||||
<< ", dist = " << static_cast<int>(Distribution::Type());
|
||||
ASSERT_FALSE(std::isnan(hessian)) << "z = " << z << ", y \\in ["
|
||||
<< y_lower << ", " << y_upper << "], y_pred = " << y_pred
|
||||
<< ", dist = " << static_cast<int>(dist_type);
|
||||
<< ", dist = " << static_cast<int>(Distribution::Type());
|
||||
ASSERT_FALSE(std::isinf(hessian)) << "z = " << z << ", y \\in ["
|
||||
<< y_lower << ", " << y_upper << "], y_pred = " << y_pred
|
||||
<< ", dist = " << static_cast<int>(dist_type);
|
||||
<< ", dist = " << static_cast<int>(Distribution::Type());
|
||||
}
|
||||
}
|
||||
|
||||
TEST(AFTLoss, RobustGradientPair) { // Ensure that INF and NAN don't show up in gradient pair
|
||||
RobustTestSuite(ProbabilityDistributionType::kNormal, 16.0, 200.0, 2.0);
|
||||
RobustTestSuite(ProbabilityDistributionType::kLogistic, 16.0, 200.0, 2.0);
|
||||
RobustTestSuite(ProbabilityDistributionType::kExtreme, 16.0, 200.0, 2.0);
|
||||
RobustTestSuite(ProbabilityDistributionType::kNormal, 100.0, 100.0, 2.0);
|
||||
RobustTestSuite(ProbabilityDistributionType::kLogistic, 100.0, 100.0, 2.0);
|
||||
RobustTestSuite(ProbabilityDistributionType::kExtreme, 100.0, 100.0, 2.0);
|
||||
RobustTestSuite<NormalDistribution>(16.0, 200.0, 2.0);
|
||||
RobustTestSuite<LogisticDistribution>(16.0, 200.0, 2.0);
|
||||
RobustTestSuite<ExtremeDistribution>(16.0, 200.0, 2.0);
|
||||
RobustTestSuite<NormalDistribution>(100.0, 100.0, 2.0);
|
||||
RobustTestSuite<LogisticDistribution>(100.0, 100.0, 2.0);
|
||||
RobustTestSuite<ExtremeDistribution>(100.0, 100.0, 2.0);
|
||||
}
|
||||
|
||||
} // namespace common
|
||||
|
||||
Reference in New Issue
Block a user