Hendrik/xgboost
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[EM] Support mmap backed ellpack. (#10602)

Browse Source 
- Support resource view in ellpack.
- Define the CUDA version of MMAP resource.
- Define the CUDA version of malloc resource.
- Refactor cuda runtime API wrappers, and add memory access related wrappers.
- gather windows macros into a single header.
This commit is contained in:
Jiaming Yuan
2024-07-18 08:20:21 +08:00
committed by GitHub
parent e9fbce9791
commit 292bb677e5
59 changed files with 889 additions and 646 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
src/c_api/c_api_error.h
View File

@@ -7,10 +7,9 @@
#define XGBOOST_C_API_C_API_ERROR_H_
#include <dmlc/base.h>
#include <dmlc/logging.h>
#include "c_api_utils.h"
#include "xgboost/collective/result.h"
#include "c_api_utils.h" // for XGBoostAPIGuard
#include "xgboost/logging.h"
/*! \brief macro to guard beginning and end section of all functions */
#ifdef LOG_CAPI_INVOCATION

17
src/cli_main.cc
View File

@@ -4,29 +4,26 @@
* \brief The command line interface program of xgboost.
* This file is not included in dynamic library.
*/
#if !defined(NOMINMAX) && defined(_WIN32)
#define NOMINMAX
#endif // !defined(NOMINMAX)
#include <dmlc/timer.h>
#include <xgboost/learner.h>
#include <xgboost/base.h>
#include <xgboost/data.h>
#include <xgboost/json.h>
#include <xgboost/learner.h>
#include <xgboost/logging.h>
#include <xgboost/parameter.h>
#include <iomanip>
#include <ctime>
#include <string>
#include <cstdio>
#include <cstring>
#include <ctime>
#include <iomanip>
#include <string>
#include <vector>
#include "c_api/c_api_utils.h"
#include "common/common.h"
#include "common/config.h"
#include "common/io.h"
#include "common/version.h"
#include "c_api/c_api_utils.h"
namespace xgboost {
enum CLITask {

7
src/collective/tracker.cc
View File

@@ -7,11 +7,10 @@
#include <sys/socket.h> // socket, AF_INET6, AF_INET, connect, getsockname
#endif // defined(__unix__) || defined(__APPLE__)
#if !defined(NOMINMAX) && defined(_WIN32)
#define NOMINMAX
#endif // !defined(NOMINMAX)
#if defined(_WIN32)
// Guard the include
#include <xgboost/windefs.h>
// Socket API
#include <winsock2.h>
#include <ws2tcpip.h>
#endif // defined(_WIN32)

7
src/common/common.cc
View File

@@ -1,5 +1,5 @@
/**
* Copyright 2015-2023 by Contributors
* Copyright 2015-2024, XGBoost Contributors
*/
#include "common.h"
@@ -54,9 +54,4 @@ void EscapeU8(std::string const &string, std::string *p_buffer) {
}
}
}
#if !defined(XGBOOST_USE_CUDA)
int AllVisibleGPUs() { return 0; }
#endif // !defined(XGBOOST_USE_CUDA)
} // namespace xgboost::common

40
src/common/common.cu
View File

@@ -1,29 +1,21 @@
/*!
* Copyright 2018-2022 XGBoost contributors
/**
* Copyright 2018-2024, XGBoost contributors
*/
#include <thrust/system/cuda/error.h>
#include <thrust/system_error.h>
#include "common.h"
namespace xgboost {
namespace common {
void SetDevice(std::int32_t device) {
if (device >= 0) {
dh::safe_cuda(cudaSetDevice(device));
namespace dh {
void ThrowOnCudaError(cudaError_t code, const char *file, int line) {
if (code != cudaSuccess) {
std::string f;
if (file != nullptr) {
f = file;
}
LOG(FATAL) << thrust::system_error(code, thrust::cuda_category(),
f + ": " + std::to_string(line))
.what();
}
}
int AllVisibleGPUs() {
int n_visgpus = 0;
try {
// When compiled with CUDA but running on CPU only device,
// cudaGetDeviceCount will fail.
dh::safe_cuda(cudaGetDeviceCount(&n_visgpus));
} catch (const dmlc::Error &) {
cudaGetLastError(); // reset error.
return 0;
}
return n_visgpus;
}
} // namespace common
} // namespace xgboost
} // namespace dh

38
src/common/common.h
View File

@@ -1,5 +1,5 @@
/**
* Copyright 2015-2023 by XGBoost Contributors
* Copyright 2015-2024, XGBoost Contributors
* \file common.h
* \brief Common utilities
*/
@@ -19,9 +19,8 @@
#include "xgboost/base.h" // for XGBOOST_DEVICE
#include "xgboost/logging.h" // for LOG, LOG_FATAL, LogMessageFatal
// magic to define functions based on the compiler.
#if defined(__CUDACC__)
#include <thrust/system/cuda/error.h>
#include <thrust/system_error.h>
#define WITH_CUDA() true
@@ -31,23 +30,20 @@
#endif // defined(__CUDACC__)
#if defined(XGBOOST_USE_CUDA)
#include <cuda_runtime_api.h>
#endif
namespace dh {
#if defined(__CUDACC__)
#if defined(XGBOOST_USE_CUDA)
/*
* Error handling functions
* Error handling functions
*/
void ThrowOnCudaError(cudaError_t code, const char *file, int line);
#define safe_cuda(ans) ThrowOnCudaError((ans), __FILE__, __LINE__)
inline cudaError_t ThrowOnCudaError(cudaError_t code, const char *file,
int line) {
if (code != cudaSuccess) {
LOG(FATAL) << thrust::system_error(code, thrust::cuda_category(),
std::string{file} + ": " + // NOLINT
std::to_string(line)).what();
}
return code;
}
#endif // defined(__CUDACC__)
#endif // defined(XGBOOST_USE_CUDA)
} // namespace dh
namespace xgboost::common {
@@ -167,8 +163,6 @@ class Range {
Iterator end_;
};
int AllVisibleGPUs();
inline void AssertGPUSupport() {
#ifndef XGBOOST_USE_CUDA
LOG(FATAL) << "XGBoost version not compiled with GPU support.";
@@ -187,16 +181,6 @@ inline void AssertSYCLSupport() {
#endif // XGBOOST_USE_SYCL
}
void SetDevice(std::int32_t device);
#if !defined(XGBOOST_USE_CUDA)
inline void SetDevice(std::int32_t device) {
if (device >= 0) {
AssertGPUSupport();
}
}
#endif
/**
* @brief Last index of a group in a CSR style of index pointer.
*/

86
src/common/cuda_rt_utils.cc Normal file
View File

@@ -0,0 +1,86 @@
/**
* Copyright 2015-2024, XGBoost Contributors
*/
#include "cuda_rt_utils.h"
#if defined(XGBOOST_USE_CUDA)
#include <cuda_runtime_api.h>
#endif // defined(XGBOOST_USE_CUDA)
#include <cstdint> // for int32_t
#include "common.h" // for safe_cuda
namespace xgboost::common {
#if defined(XGBOOST_USE_CUDA)
std::int32_t AllVisibleGPUs() {
int n_visgpus = 0;
try {
// When compiled with CUDA but running on CPU only device,
// cudaGetDeviceCount will fail.
dh::safe_cuda(cudaGetDeviceCount(&n_visgpus));
} catch (const dmlc::Error &) {
cudaGetLastError(); // reset error.
return 0;
}
return n_visgpus;
}
std::int32_t CurrentDevice() {
std::int32_t device = 0;
dh::safe_cuda(cudaGetDevice(&device));
return device;
}
// alternatively: `nvidia-smi -q | grep Addressing`
bool SupportsPageableMem() {
std::int32_t res{0};
dh::safe_cuda(cudaDeviceGetAttribute(&res, cudaDevAttrPageableMemoryAccess, CurrentDevice()));
return res == 1;
}
bool SupportsAts() {
std::int32_t res{0};
dh::safe_cuda(cudaDeviceGetAttribute(&res, cudaDevAttrPageableMemoryAccessUsesHostPageTables,
CurrentDevice()));
return res == 1;
}
void CheckComputeCapability() {
for (std::int32_t d_idx = 0; d_idx < AllVisibleGPUs(); ++d_idx) {
cudaDeviceProp prop;
dh::safe_cuda(cudaGetDeviceProperties(&prop, d_idx));
std::ostringstream oss;
oss << "CUDA Capability Major/Minor version number: " << prop.major << "." << prop.minor
<< " is insufficient. Need >=3.5";
int failed = prop.major < 3 || (prop.major == 3 && prop.minor < 5);
if (failed) LOG(WARNING) << oss.str() << " for device: " << d_idx;
}
}
void SetDevice(std::int32_t device) {
if (device >= 0) {
dh::safe_cuda(cudaSetDevice(device));
}
}
#else
std::int32_t AllVisibleGPUs() { return 0; }
std::int32_t CurrentDevice() {
AssertGPUSupport();
return -1;
}
bool SupportsPageableMem() { return false; }
bool SupportsAts() { return false; }
void CheckComputeCapability() {}
void SetDevice(std::int32_t device) {
if (device >= 0) {
AssertGPUSupport();
}
}
#endif // !defined(XGBOOST_USE_CUDA)
} // namespace xgboost::common

21
src/common/cuda_rt_utils.h Normal file
View File

@@ -0,0 +1,21 @@
/**
* Copyright 2024, XGBoost contributors
*/
#pragma once
#include <cstdint> // for int32_t
namespace xgboost::common {
std::int32_t AllVisibleGPUs();
std::int32_t CurrentDevice();
// Whether the device supports coherently accessing pageable memory without calling
// `cudaHostRegister` on it
bool SupportsPageableMem();
// Address Translation Service (ATS)
bool SupportsAts();
void CheckComputeCapability();
void SetDevice(std::int32_t device);
} // namespace xgboost::common

26
src/common/device_helpers.cuh
View File

@@ -157,18 +157,6 @@ inline size_t MaxSharedMemoryOptin(int device_idx) {
return static_cast<std::size_t>(max_shared_memory);
}
inline void CheckComputeCapability() {
for (int d_idx = 0; d_idx < xgboost::common::AllVisibleGPUs(); ++d_idx) {
cudaDeviceProp prop;
safe_cuda(cudaGetDeviceProperties(&prop, d_idx));
std::ostringstream oss;
oss << "CUDA Capability Major/Minor version number: " << prop.major << "."
<< prop.minor << " is insufficient. Need >=3.5";
int failed = prop.major < 3 || (prop.major == 3 && prop.minor < 5);
if (failed) LOG(WARNING) << oss.str() << " for device: " << d_idx;
}
}
XGBOOST_DEV_INLINE void AtomicOrByte(unsigned int *__restrict__ buffer,
size_t ibyte, unsigned char b) {
atomicOr(&buffer[ibyte / sizeof(unsigned int)],
@@ -273,13 +261,15 @@ void Iota(Container array, cudaStream_t stream) {
}
// dh::DebugSyncDevice(__FILE__, __LINE__);
inline void DebugSyncDevice(std::string file="", int32_t line = -1) {
if (file != "" && line != -1) {
auto rank = xgboost::collective::GetRank();
LOG(DEBUG) << "R:" << rank << ": " << file << ":" << line;
inline void DebugSyncDevice(char const *file = __builtin_FILE(), int32_t line = __builtin_LINE()) {
{
auto err = cudaDeviceSynchronize();
ThrowOnCudaError(err, file, line);
}
{
auto err = cudaGetLastError();
ThrowOnCudaError(err, file, line);
}
safe_cuda(cudaDeviceSynchronize());
safe_cuda(cudaGetLastError());
}
// Faster to instantiate than caching_device_vector and invokes no synchronisation

134
src/common/io.cc
View File

@@ -1,26 +1,21 @@
/**
* Copyright 2019-2023, by XGBoost Contributors
* Copyright 2019-2024, by XGBoost Contributors
*/
#if !defined(NOMINMAX) && defined(_WIN32)
#define NOMINMAX
#endif // !defined(NOMINMAX)
#if !defined(xgboost_IS_WIN)
#if defined(_MSC_VER) || defined(__MINGW32__)
#define xgboost_IS_WIN 1
#endif // defined(_MSC_VER) || defined(__MINGW32__)
#endif // !defined(xgboost_IS_WIN)
#if defined(__unix__) || defined(__APPLE__)
#include <fcntl.h> // for open, O_RDONLY
#include <sys/mman.h> // for mmap, mmap64, munmap
#include <sys/mman.h> // for mmap, mmap64, munmap, madvise
#include <unistd.h> // for close, getpagesize
#elif defined(xgboost_IS_WIN)
#define WIN32_LEAN_AND_MEAN
#else
#include <xgboost/windefs.h>
#if defined(xgboost_IS_WIN)
#include <windows.h>
#endif // defined(__unix__)
#endif // defined(xgboost_IS_WIN)
#endif // defined(__unix__) || defined(__APPLE__)
#include <algorithm> // for copy, transform
#include <cctype> // for tolower
@@ -31,8 +26,7 @@
#include <filesystem> // for filesystem, weakly_canonical
#include <fstream> // for ifstream
#include <iterator> // for distance
#include <limits> // for numeric_limits
#include <memory> // for unique_ptr
#include <memory> // for unique_ptr, make_unique
#include <string> // for string
#include <system_error> // for error_code, system_category
#include <utility> // for move
@@ -40,7 +34,12 @@
#include "io.h"
#include "xgboost/collective/socket.h" // for LastError
#include "xgboost/logging.h"
#include "xgboost/logging.h" // for CHECK_LE
#include "xgboost/string_view.h" // for StringView
#if !defined(__linux__) && !defined(__GLIBC__) && !defined(xgboost_IS_WIN)
#include <limits> // for numeric_limits
#endif
namespace xgboost::common {
size_t PeekableInStream::Read(void* dptr, size_t size) {
@@ -182,39 +181,9 @@ std::string FileExtension(std::string fname, bool lower) {
// NVCC 11.8 doesn't allow `noexcept(false) = default` altogether.
ResourceHandler::~ResourceHandler() noexcept(false) {} // NOLINT
struct MMAPFile {
#if defined(xgboost_IS_WIN)
HANDLE fd{INVALID_HANDLE_VALUE};
HANDLE file_map{INVALID_HANDLE_VALUE};
#else
std::int32_t fd{0};
#endif
std::byte* base_ptr{nullptr};
std::size_t base_size{0};
std::size_t delta{0};
std::string path;
MMAPFile() = default;
#if defined(xgboost_IS_WIN)
MMAPFile(HANDLE fd, HANDLE fm, std::byte* base_ptr, std::size_t base_size, std::size_t delta,
std::string path)
: fd{fd},
file_map{fm},
base_ptr{base_ptr},
base_size{base_size},
delta{delta},
path{std::move(path)} {}
#else
MMAPFile(std::int32_t fd, std::byte* base_ptr, std::size_t base_size, std::size_t delta,
std::string path)
: fd{fd}, base_ptr{base_ptr}, base_size{base_size}, delta{delta}, path{std::move(path)} {}
#endif
};
std::unique_ptr<MMAPFile> Open(std::string path, std::size_t offset, std::size_t length) {
MMAPFile* detail::OpenMmap(std::string path, std::size_t offset, std::size_t length) {
if (length == 0) {
return std::make_unique<MMAPFile>();
return new MMAPFile{};
}
#if defined(xgboost_IS_WIN)
@@ -234,10 +203,8 @@ std::unique_ptr<MMAPFile> Open(std::string path, std::size_t offset, std::size_t
#if defined(__linux__) || defined(__GLIBC__)
int prot{PROT_READ};
ptr = reinterpret_cast<std::byte*>(mmap64(nullptr, view_size, prot, MAP_PRIVATE, fd, view_start));
madvise(ptr, view_size, MADV_WILLNEED);
CHECK_NE(ptr, MAP_FAILED) << "Failed to map: " << path << ". " << SystemErrorMsg();
auto handle =
std::make_unique<MMAPFile>(fd, ptr, view_size, offset - view_start, std::move(path));
auto handle = new MMAPFile{fd, ptr, view_size, offset - view_start, std::move(path)};
#elif defined(xgboost_IS_WIN)
auto file_size = GetFileSize(fd, nullptr);
DWORD access = PAGE_READONLY;
@@ -248,55 +215,62 @@ std::unique_ptr<MMAPFile> Open(std::string path, std::size_t offset, std::size_t
CHECK(map_file) << "Failed to map: " << path << ". " << SystemErrorMsg();
ptr = reinterpret_cast<std::byte*>(MapViewOfFile(map_file, access, hoff, loff, view_size));
CHECK_NE(ptr, nullptr) << "Failed to map: " << path << ". " << SystemErrorMsg();
auto handle = std::make_unique<MMAPFile>(fd, map_file, ptr, view_size, offset - view_start,
std::move(path));
auto handle = new MMAPFile{fd, map_file, ptr, view_size, offset - view_start, std::move(path)};
#else
CHECK_LE(offset, std::numeric_limits<off_t>::max())
<< "File size has exceeded the limit on the current system.";
int prot{PROT_READ};
ptr = reinterpret_cast<std::byte*>(mmap(nullptr, view_size, prot, MAP_PRIVATE, fd, view_start));
CHECK_NE(ptr, MAP_FAILED) << "Failed to map: " << path << ". " << SystemErrorMsg();
auto handle =
std::make_unique<MMAPFile>(fd, ptr, view_size, offset - view_start, std::move(path));
#endif // defined(__linux__)
auto handle = new MMAPFile{fd, ptr, view_size, offset - view_start, std::move(path)};
#endif // defined(__linux__) || defined(__GLIBC__)
return handle;
}
MmapResource::MmapResource(std::string path, std::size_t offset, std::size_t length)
: ResourceHandler{kMmap}, handle_{Open(std::move(path), offset, length)}, n_{length} {}
MmapResource::~MmapResource() noexcept(false) {
if (!handle_) {
void detail::CloseMmap(MMAPFile* handle) {
if (!handle) {
return;
}
#if defined(xgboost_IS_WIN)
if (handle_->base_ptr) {
CHECK(UnmapViewOfFile(handle_->base_ptr)) "Faled to call munmap: " << SystemErrorMsg();
if (handle->base_ptr) {
CHECK(UnmapViewOfFile(handle->base_ptr)) "Faled to call munmap: " << SystemErrorMsg();
}
if (handle_->fd != INVALID_HANDLE_VALUE) {
CHECK(CloseHandle(handle_->fd)) << "Failed to close handle: " << SystemErrorMsg();
if (handle->fd != INVALID_HANDLE_VALUE) {
CHECK(CloseHandle(handle->fd)) << "Failed to close handle: " << SystemErrorMsg();
}
if (handle_->file_map != INVALID_HANDLE_VALUE) {
CHECK(CloseHandle(handle_->file_map)) << "Failed to close mapping object: " << SystemErrorMsg();
if (handle->file_map != INVALID_HANDLE_VALUE) {
CHECK(CloseHandle(handle->file_map)) << "Failed to close mapping object: " << SystemErrorMsg();
}
#else
if (handle_->base_ptr) {
CHECK_NE(munmap(handle_->base_ptr, handle_->base_size), -1)
<< "Faled to call munmap: " << handle_->path << ". " << SystemErrorMsg();
if (handle->base_ptr) {
CHECK_NE(munmap(handle->base_ptr, handle->base_size), -1)
<< "Faled to call munmap: `" << handle->path << "`. " << SystemErrorMsg();
}
if (handle_->fd != 0) {
CHECK_NE(close(handle_->fd), -1)
<< "Faled to close: " << handle_->path << ". " << SystemErrorMsg();
if (handle->fd != 0) {
CHECK_NE(close(handle->fd), -1)
<< "Faled to close: `" << handle->path << "`. " << SystemErrorMsg();
}
#endif
delete handle;
}
MmapResource::MmapResource(StringView path, std::size_t offset, std::size_t length)
: ResourceHandler{kMmap},
handle_{detail::OpenMmap(std::string{path}, offset, length), detail::CloseMmap},
n_{length} {
#if defined(__unix__) || defined(__APPLE__)
madvise(handle_->base_ptr, handle_->base_size, MADV_WILLNEED);
#endif // defined(__unix__) || defined(__APPLE__)
}
MmapResource::~MmapResource() noexcept(false) = default;
[[nodiscard]] void* MmapResource::Data() {
if (!handle_) {
return nullptr;
}
return handle_->base_ptr + handle_->delta;
return this->handle_->Data();
}
[[nodiscard]] std::size_t MmapResource::Size() const { return n_; }
@@ -329,7 +303,3 @@ AlignedMemWriteStream::~AlignedMemWriteStream() = default;
return this->pimpl_->Tell();
}
} // namespace xgboost::common
#if defined(xgboost_IS_WIN)
#undef xgboost_IS_WIN
#endif // defined(xgboost_IS_WIN)

75
src/common/io.h
View File

@@ -7,7 +7,11 @@
#ifndef XGBOOST_COMMON_IO_H_
#define XGBOOST_COMMON_IO_H_
#include <dmlc/io.h>
#include <xgboost/windefs.h>
#if defined(xgboost_IS_WIN)
#include <windows.h>
#endif // defined(xgboost_IS_WIN)
#include <algorithm> // for min, fill_n, copy_n
#include <array> // for array
@@ -15,6 +19,7 @@
#include <cstdlib> // for malloc, realloc, free
#include <cstring> // for memcpy
#include <fstream> // for ifstream
#include <functional> // for function
#include <limits> // for numeric_limits
#include <memory> // for unique_ptr
#include <string> // for string
@@ -23,6 +28,7 @@
#include <vector> // for vector
#include "common.h" // for DivRoundUp
#include "dmlc/io.h" // for SeekStream
#include "xgboost/string_view.h" // for StringView
namespace xgboost::common {
@@ -224,7 +230,48 @@ inline std::string ReadAll(std::string const &path) {
return content;
}
struct MMAPFile;
/**
* @brief A handle to mmap file.
*/
struct MMAPFile {
#if defined(xgboost_IS_WIN)
HANDLE fd{INVALID_HANDLE_VALUE};
HANDLE file_map{INVALID_HANDLE_VALUE};
#else
std::int32_t fd{0};
#endif // defined(xgboost_IS_WIN)
std::byte* base_ptr{nullptr};
std::size_t base_size{0};
std::size_t delta{0};
std::string path;
MMAPFile() = default;
#if defined(xgboost_IS_WIN)
MMAPFile(HANDLE fd, HANDLE fm, std::byte* base_ptr, std::size_t base_size, std::size_t delta,
std::string path)
: fd{fd},
file_map{fm},
base_ptr{base_ptr},
base_size{base_size},
delta{delta},
path{std::move(path)} {}
#else
MMAPFile(std::int32_t fd, std::byte* base_ptr, std::size_t base_size, std::size_t delta,
std::string path)
: fd{fd}, base_ptr{base_ptr}, base_size{base_size}, delta{delta}, path{std::move(path)} {}
#endif // defined(xgboost_IS_WIN)
void const* Data() const { return this->base_ptr + this->delta; }
void* Data() { return this->base_ptr + this->delta; }
};
namespace detail {
// call mmap
[[nodiscard]] MMAPFile* OpenMmap(std::string path, std::size_t offset, std::size_t length);
// close the mapped file handle.
void CloseMmap(MMAPFile* handle);
} // namespace detail
/**
* @brief Handler for one-shot resource. Unlike `std::pmr::*`, the resource handler is
@@ -237,6 +284,8 @@ class ResourceHandler {
enum Kind : std::uint8_t {
kMalloc = 0,
kMmap = 1,
kCudaMalloc = 2,
kCudaMmap = 3,
};
private:
@@ -251,6 +300,20 @@ class ResourceHandler {
[[nodiscard]] virtual std::size_t Size() const = 0;
[[nodiscard]] auto Type() const { return kind_; }
[[nodiscard]] StringView TypeName() const {
switch (this->Type()) {
case kMalloc:
return "Malloc";
case kMmap:
return "Mmap";
case kCudaMalloc:
return "CudaMalloc";
case kCudaMmap:
return "CudaMmap";
}
LOG(FATAL) << "Unreachable.";
return {};
}
// Allow exceptions for cleaning up resource.
virtual ~ResourceHandler() noexcept(false);
@@ -339,11 +402,11 @@ class MallocResource : public ResourceHandler {
* @brief A class for wrapping mmap as a resource for RAII.
*/
class MmapResource : public ResourceHandler {
std::unique_ptr<MMAPFile> handle_;
std::unique_ptr<MMAPFile, std::function<void(MMAPFile*)>> handle_;
std::size_t n_;
public:
MmapResource(std::string path, std::size_t offset, std::size_t length);
MmapResource(StringView path, std::size_t offset, std::size_t length);
~MmapResource() noexcept(false) override;
[[nodiscard]] void* Data() override;
@@ -471,9 +534,9 @@ class PrivateMmapConstStream : public AlignedResourceReadStream {
* @param offset See the `offset` parameter of `mmap` for details.
* @param length See the `length` parameter of `mmap` for details.
*/
explicit PrivateMmapConstStream(std::string path, std::size_t offset, std::size_t length)
explicit PrivateMmapConstStream(StringView path, std::size_t offset, std::size_t length)
: AlignedResourceReadStream{std::shared_ptr<MmapResource>{ // NOLINT
new MmapResource{std::move(path), offset, length}}} {}
new MmapResource{path, offset, length}}} {}
~PrivateMmapConstStream() noexcept(false) override;
};

26
src/common/ref_resource_view.cuh Normal file
View File

@@ -0,0 +1,26 @@
/**
* Copyright 2024, XGBoost Contributors
*/
#pragma once
#include <cstddef> // for size_t
#include <memory> // for make_shared
#include "cuda_context.cuh" // for CUDAContext
#include "ref_resource_view.h" // for RefResourceView
#include "resource.cuh" // for CudaAllocResource
#include "xgboost/context.h" // for Context
namespace xgboost::common {
/**
* @brief Make a fixed size `RefResourceView` with cudaMalloc resource.
*/
template <typename T>
[[nodiscard]] RefResourceView<T> MakeFixedVecWithCudaMalloc(Context const* ctx,
std::size_t n_elements, T const& init) {
auto resource = std::make_shared<common::CudaMallocResource>(n_elements * sizeof(T));
auto ref = RefResourceView{resource->DataAs<T>(), n_elements, resource};
thrust::fill_n(ctx->CUDACtx()->CTP(), ref.data(), ref.size(), init);
return ref;
}
} // namespace xgboost::common

18
src/common/ref_resource_view.h
View File

@@ -43,24 +43,16 @@ class RefResourceView {
}
public:
RefResourceView(value_type* ptr, size_type n, std::shared_ptr<common::ResourceHandler> mem)
: ptr_{ptr}, size_{n}, mem_{std::move(mem)} {
CHECK_GE(mem_->Size(), n);
}
/**
* @brief Construct a view on ptr with length n. The ptr is held by the mem resource.
*
* @param ptr The pointer to view.
* @param n The length of the view.
* @param mem The owner of the pointer.
* @param init Initialize the view with this value.
*/
RefResourceView(value_type* ptr, size_type n, std::shared_ptr<common::ResourceHandler> mem,
T const& init)
: RefResourceView{ptr, n, mem} {
if (n != 0) {
std::fill_n(ptr_, n, init);
}
RefResourceView(value_type* ptr, size_type n, std::shared_ptr<common::ResourceHandler> mem)
: ptr_{ptr}, size_{n}, mem_{std::move(mem)} {
CHECK_GE(mem_->Size(), n);
}
~RefResourceView() = default;
@@ -159,7 +151,9 @@ template <typename Vec>
template <typename T>
[[nodiscard]] RefResourceView<T> MakeFixedVecWithMalloc(std::size_t n_elements, T const& init) {
auto resource = std::make_shared<common::MallocResource>(n_elements * sizeof(T));
return RefResourceView{resource->DataAs<T>(), n_elements, resource, init};
auto ref = RefResourceView{resource->DataAs<T>(), n_elements, resource};
std::fill_n(ref.data(), ref.size(), init);
return ref;
}
template <typename T>

43
src/common/resource.cu Normal file
View File

@@ -0,0 +1,43 @@
/**
* Copyright 2024, XGBoost Contributors
*/
#include "device_helpers.cuh" // for CurrentDevice
#include "resource.cuh"
#include "xgboost/string_view.h" // for StringView
namespace xgboost::common {
CudaMmapResource::CudaMmapResource(StringView path, std::size_t offset, std::size_t length)
: ResourceHandler{kCudaMmap},
handle_{detail::OpenMmap(std::string{path}, offset, length),
[](MMAPFile* handle) {
// Don't close the mmap while CUDA kernel is running.
if (handle) {
dh::DefaultStream().Sync();
}
detail::CloseMmap(handle);
}},
n_{length} {
auto device = dh::CurrentDevice();
dh::safe_cuda(
cudaMemAdvise(handle_->base_ptr, handle_->base_size, cudaMemAdviseSetReadMostly, device));
dh::safe_cuda(cudaMemAdvise(handle_->base_ptr, handle_->base_size,
cudaMemAdviseSetPreferredLocation, device));
dh::safe_cuda(
cudaMemAdvise(handle_->base_ptr, handle_->base_size, cudaMemAdviseSetAccessedBy, device));
dh::safe_cuda(
cudaMemPrefetchAsync(handle_->base_ptr, handle_->base_size, device, dh::DefaultStream()));
}
[[nodiscard]] void* CudaMmapResource::Data() {
if (!handle_) {
return nullptr;
}
return this->handle_->Data();
}
[[nodiscard]] std::size_t CudaMmapResource::Size() const { return n_; }
CudaMmapResource::~CudaMmapResource() noexcept(false) = default;
PrivateCudaMmapConstStream::~PrivateCudaMmapConstStream() noexcept(false) = default;
} // namespace xgboost::common

54
src/common/resource.cuh Normal file
View File

@@ -0,0 +1,54 @@
/**
* Copyright 2024, XGBoost Contributors
*/
#pragma once
#include <cstddef> // for size_t
#include <functional> // for function
#include "device_vector.cuh" // for DeviceUVector
#include "io.h" // for ResourceHandler, MMAPFile
#include "xgboost/string_view.h" // for StringView
namespace xgboost::common {
/**
* @brief Resource backed by `cudaMalloc`.
*/
class CudaMallocResource : public ResourceHandler {
dh::DeviceUVector<std::byte> storage_;
void Clear() noexcept(true) { this->Resize(0); }
public:
explicit CudaMallocResource(std::size_t n_bytes) : ResourceHandler{kCudaMalloc} {
this->Resize(n_bytes);
}
~CudaMallocResource() noexcept(true) override { this->Clear(); }
void* Data() override { return storage_.data(); }
[[nodiscard]] std::size_t Size() const override { return storage_.size(); }
void Resize(std::size_t n_bytes, std::byte init = std::byte{0}) {
this->storage_.resize(n_bytes, init);
}
};
class CudaMmapResource : public ResourceHandler {
std::unique_ptr<MMAPFile, std::function<void(MMAPFile*)>> handle_;
std::size_t n_;
public:
CudaMmapResource() : ResourceHandler{kCudaMmap} {}
CudaMmapResource(StringView path, std::size_t offset, std::size_t length);
~CudaMmapResource() noexcept(false) override;
[[nodiscard]] void* Data() override;
[[nodiscard]] std::size_t Size() const override;
};
class PrivateCudaMmapConstStream : public AlignedResourceReadStream {
public:
explicit PrivateCudaMmapConstStream(StringView path, std::size_t offset, std::size_t length)
: AlignedResourceReadStream{
std::shared_ptr<CudaMmapResource>{new CudaMmapResource{path, offset, length}}} {}
~PrivateCudaMmapConstStream() noexcept(false) override;
};
} // namespace xgboost::common

7
src/context.cc
View File

@@ -1,5 +1,5 @@
/**
* Copyright 2014-2023 by XGBoost Contributors
* Copyright 2014-2024, XGBoost Contributors
*
* \brief Context object used for controlling runtime parameters.
*/
@@ -11,8 +11,9 @@
#include <optional> // for optional
#include <regex> // for regex_replace, regex_match
#include "common/common.h" // AssertGPUSupport
#include "common/error_msg.h" // WarnDeprecatedGPUId
#include "common/common.h" // AssertGPUSupport
#include "common/cuda_rt_utils.h" // for AllVisibleGPUs
#include "common/error_msg.h" // WarnDeprecatedGPUId
#include "common/threading_utils.h"
#include "xgboost/string_view.h"

150
src/data/ellpack_page.cu
View File

@@ -11,8 +11,9 @@
#include "../common/categorical.h"
#include "../common/cuda_context.cuh"
#include "../common/hist_util.cuh"
#include "../common/transform_iterator.h" // MakeIndexTransformIter
#include "device_adapter.cuh" // for NoInfInData
#include "../common/ref_resource_view.cuh" // for MakeFixedVecWithCudaMalloc
#include "../common/transform_iterator.h" // MakeIndexTransformIter
#include "device_adapter.cuh" // for NoInfInData
#include "ellpack_page.cuh"
#include "ellpack_page.h"
#include "gradient_index.h"
@@ -43,21 +44,19 @@ __global__ void CompressBinEllpackKernel(
common::CompressedBufferWriter wr,
common::CompressedByteT* __restrict__ buffer, // gidx_buffer
const size_t* __restrict__ row_ptrs, // row offset of input data
const Entry* __restrict__ entries, // One batch of input data
const float* __restrict__ cuts, // HistogramCuts::cut_values_
const uint32_t* __restrict__ cut_ptrs, // HistogramCuts::cut_ptrs_
const Entry* __restrict__ entries, // One batch of input data
const float* __restrict__ cuts, // HistogramCuts::cut_values_
const uint32_t* __restrict__ cut_ptrs, // HistogramCuts::cut_ptrs_
common::Span<FeatureType const> feature_types,
size_t base_row, // batch_row_begin
size_t n_rows,
size_t row_stride,
unsigned int null_gidx_value) {
size_t base_row, // batch_row_begin
size_t n_rows, size_t row_stride, std::uint32_t null_gidx_value) {
size_t irow = threadIdx.x + blockIdx.x * blockDim.x;
int ifeature = threadIdx.y + blockIdx.y * blockDim.y;
if (irow >= n_rows || ifeature >= row_stride) {
return;
}
int row_length = static_cast<int>(row_ptrs[irow + 1] - row_ptrs[irow]);
unsigned int bin = null_gidx_value;
std::uint32_t bin = null_gidx_value;
if (ifeature < row_length) {
Entry entry = entries[row_ptrs[irow] - row_ptrs[0] + ifeature];
int feature = entry.index;
@@ -89,25 +88,23 @@ __global__ void CompressBinEllpackKernel(
}
// Construct an ELLPACK matrix with the given number of empty rows.
EllpackPageImpl::EllpackPageImpl(DeviceOrd device,
EllpackPageImpl::EllpackPageImpl(Context const* ctx,
std::shared_ptr<common::HistogramCuts const> cuts, bool is_dense,
bst_idx_t row_stride, bst_idx_t n_rows)
: is_dense(is_dense), cuts_(std::move(cuts)), row_stride{row_stride}, n_rows{n_rows} {
monitor_.Init("ellpack_page");
dh::safe_cuda(cudaSetDevice(device.ordinal));
dh::safe_cuda(cudaSetDevice(ctx->Ordinal()));
monitor_.Start("InitCompressedData");
this->InitCompressedData(device);
monitor_.Stop("InitCompressedData");
this->InitCompressedData(ctx);
}
EllpackPageImpl::EllpackPageImpl(DeviceOrd device,
EllpackPageImpl::EllpackPageImpl(Context const* ctx,
std::shared_ptr<common::HistogramCuts const> cuts,
const SparsePage& page, bool is_dense, size_t row_stride,
common::Span<FeatureType const> feature_types)
: cuts_(std::move(cuts)), is_dense(is_dense), n_rows(page.Size()), row_stride(row_stride) {
this->InitCompressedData(device);
this->CreateHistIndices(device, page, feature_types);
this->InitCompressedData(ctx);
this->CreateHistIndices(ctx->Device(), page, feature_types);
}
// Construct an ELLPACK matrix in memory.
@@ -129,9 +126,7 @@ EllpackPageImpl::EllpackPageImpl(Context const* ctx, DMatrix* dmat, const BatchP
}
monitor_.Stop("Quantiles");
monitor_.Start("InitCompressedData");
this->InitCompressedData(ctx->Device());
monitor_.Stop("InitCompressedData");
this->InitCompressedData(ctx);
dmat->Info().feature_types.SetDevice(ctx->Device());
auto ft = dmat->Info().feature_types.ConstDeviceSpan();
@@ -234,7 +229,7 @@ void CopyDataToEllpack(const AdapterBatchT& batch, common::Span<FeatureType cons
auto device_accessor = dst->GetDeviceAccessor(device);
common::CompressedBufferWriter writer(device_accessor.NumSymbols());
auto d_compressed_buffer = dst->gidx_buffer.DevicePointer();
auto d_compressed_buffer = dst->gidx_buffer.data();
// We redirect the scan output into this functor to do the actual writing
WriteCompressedEllpackFunctor<AdapterBatchT> functor(
@@ -275,7 +270,7 @@ void WriteNullValues(EllpackPageImpl* dst, DeviceOrd device, common::Span<size_t
// Write the null values
auto device_accessor = dst->GetDeviceAccessor(device);
common::CompressedBufferWriter writer(device_accessor.NumSymbols());
auto d_compressed_buffer = dst->gidx_buffer.DevicePointer();
auto d_compressed_buffer = dst->gidx_buffer.data();
auto row_stride = dst->row_stride;
dh::LaunchN(row_stride * dst->n_rows, [=] __device__(size_t idx) {
// For some reason this variable got captured as const
@@ -290,20 +285,20 @@ void WriteNullValues(EllpackPageImpl* dst, DeviceOrd device, common::Span<size_t
}
template <typename AdapterBatch>
EllpackPageImpl::EllpackPageImpl(AdapterBatch batch, float missing, DeviceOrd device, bool is_dense,
common::Span<size_t> row_counts_span,
EllpackPageImpl::EllpackPageImpl(Context const* ctx, AdapterBatch batch, float missing,
bool is_dense, common::Span<size_t> row_counts_span,
common::Span<FeatureType const> feature_types, size_t row_stride,
size_t n_rows, std::shared_ptr<common::HistogramCuts const> cuts) {
dh::safe_cuda(cudaSetDevice(device.ordinal));
dh::safe_cuda(cudaSetDevice(ctx->Ordinal()));
*this = EllpackPageImpl(device, cuts, is_dense, row_stride, n_rows);
CopyDataToEllpack(batch, feature_types, this, device, missing);
WriteNullValues(this, device, row_counts_span);
*this = EllpackPageImpl(ctx, cuts, is_dense, row_stride, n_rows);
CopyDataToEllpack(batch, feature_types, this, ctx->Device(), missing);
WriteNullValues(this, ctx->Device(), row_counts_span);
}
#define ELLPACK_BATCH_SPECIALIZE(__BATCH_T) \
template EllpackPageImpl::EllpackPageImpl( \
__BATCH_T batch, float missing, DeviceOrd device, bool is_dense, \
Context const* ctx, __BATCH_T batch, float missing, bool is_dense, \
common::Span<size_t> row_counts_span, common::Span<FeatureType const> feature_types, \
size_t row_stride, size_t n_rows, std::shared_ptr<common::HistogramCuts const> cuts);
@@ -365,12 +360,10 @@ EllpackPageImpl::EllpackPageImpl(Context const* ctx, GHistIndexMatrix const& pag
row_stride = *std::max_element(it, it + page.Size());
CHECK(ctx->IsCUDA());
monitor_.Start("InitCompressedData");
InitCompressedData(ctx->Device());
monitor_.Stop("InitCompressedData");
InitCompressedData(ctx);
// copy gidx
common::CompressedByteT* d_compressed_buffer = gidx_buffer.DevicePointer();
common::CompressedByteT* d_compressed_buffer = gidx_buffer.data();
dh::device_vector<size_t> row_ptr(page.row_ptr.size());
auto d_row_ptr = dh::ToSpan(row_ptr);
dh::safe_cuda(cudaMemcpyAsync(d_row_ptr.data(), page.row_ptr.data(), d_row_ptr.size_bytes(),
@@ -389,20 +382,20 @@ struct CopyPage {
// The number of elements to skip.
size_t offset;
CopyPage(EllpackPageImpl *dst, EllpackPageImpl const *src, size_t offset)
: cbw{dst->NumSymbols()}, dst_data_d{dst->gidx_buffer.DevicePointer()},
src_iterator_d{src->gidx_buffer.DevicePointer(), src->NumSymbols()},
CopyPage(EllpackPageImpl* dst, EllpackPageImpl const* src, size_t offset)
: cbw{dst->NumSymbols()},
dst_data_d{dst->gidx_buffer.data()},
src_iterator_d{src->gidx_buffer.data(), src->NumSymbols()},
offset(offset) {}
__device__ void operator()(size_t element_id) {
cbw.AtomicWriteSymbol(dst_data_d, src_iterator_d[element_id],
element_id + offset);
cbw.AtomicWriteSymbol(dst_data_d, src_iterator_d[element_id], element_id + offset);
}
};
// Copy the data from the given EllpackPage to the current page.
size_t EllpackPageImpl::Copy(DeviceOrd device, EllpackPageImpl const* page, size_t offset) {
monitor_.Start("Copy");
size_t EllpackPageImpl::Copy(Context const* ctx, EllpackPageImpl const* page, bst_idx_t offset) {
monitor_.Start(__func__);
bst_idx_t num_elements = page->n_rows * page->row_stride;
CHECK_EQ(row_stride, page->row_stride);
CHECK_EQ(NumSymbols(), page->NumSymbols());
@@ -411,10 +404,8 @@ size_t EllpackPageImpl::Copy(DeviceOrd device, EllpackPageImpl const* page, size
LOG(FATAL) << "Concatenating the same Ellpack.";
return this->n_rows * this->row_stride;
}
gidx_buffer.SetDevice(device);
page->gidx_buffer.SetDevice(device);
dh::LaunchN(num_elements, CopyPage(this, page, offset));
monitor_.Stop("Copy");
dh::LaunchN(num_elements, CopyPage{this, page, offset});
monitor_.Stop(__func__);
return num_elements;
}
@@ -423,8 +414,8 @@ struct CompactPage {
common::CompressedBufferWriter cbw;
common::CompressedByteT* dst_data_d;
common::CompressedIterator<uint32_t> src_iterator_d;
/*! \brief An array that maps the rows from the full DMatrix to the compacted
* page.
/**
* @brief An array that maps the rows from the full DMatrix to the compacted page.
*
* The total size is the number of rows in the original, uncompacted DMatrix.
* Elements are the row ids in the compacted page. Rows not needed are set to
@@ -438,24 +429,24 @@ struct CompactPage {
size_t base_rowid;
size_t row_stride;
CompactPage(EllpackPageImpl* dst, EllpackPageImpl const* src,
common::Span<size_t> row_indexes)
CompactPage(EllpackPageImpl* dst, EllpackPageImpl const* src, common::Span<size_t> row_indexes)
: cbw{dst->NumSymbols()},
dst_data_d{dst->gidx_buffer.DevicePointer()},
src_iterator_d{src->gidx_buffer.DevicePointer(), src->NumSymbols()},
dst_data_d{dst->gidx_buffer.data()},
src_iterator_d{src->gidx_buffer.data(), src->NumSymbols()},
row_indexes(row_indexes),
base_rowid{src->base_rowid},
row_stride{src->row_stride} {}
__device__ void operator()(size_t row_id) {
__device__ void operator()(bst_idx_t row_id) {
size_t src_row = base_rowid + row_id;
size_t dst_row = row_indexes[src_row];
if (dst_row == SIZE_MAX) return;
if (dst_row == SIZE_MAX) {
return;
}
size_t dst_offset = dst_row * row_stride;
size_t src_offset = row_id * row_stride;
for (size_t j = 0; j < row_stride; j++) {
cbw.AtomicWriteSymbol(dst_data_d, src_iterator_d[src_offset + j],
dst_offset + j);
cbw.AtomicWriteSymbol(dst_data_d, src_iterator_d[src_offset + j], dst_offset + j);
}
}
};
@@ -467,28 +458,22 @@ void EllpackPageImpl::Compact(Context const* ctx, EllpackPageImpl const* page,
CHECK_EQ(row_stride, page->row_stride);
CHECK_EQ(NumSymbols(), page->NumSymbols());
CHECK_LE(page->base_rowid + page->n_rows, row_indexes.size());
gidx_buffer.SetDevice(ctx->Device());
page->gidx_buffer.SetDevice(ctx->Device());
auto cuctx = ctx->CUDACtx();
dh::LaunchN(page->n_rows, cuctx->Stream(), CompactPage(this, page, row_indexes));
dh::LaunchN(page->n_rows, cuctx->Stream(), CompactPage{this, page, row_indexes});
monitor_.Stop(__func__);
}
// Initialize the buffer to stored compressed features.
void EllpackPageImpl::InitCompressedData(DeviceOrd device) {
size_t num_symbols = NumSymbols();
void EllpackPageImpl::InitCompressedData(Context const* ctx) {
monitor_.Start(__func__);
auto num_symbols = NumSymbols();
// Required buffer size for storing data matrix in ELLPack format.
size_t compressed_size_bytes =
std::size_t compressed_size_bytes =
common::CompressedBufferWriter::CalculateBufferSize(row_stride * n_rows, num_symbols);
gidx_buffer.SetDevice(device);
// Don't call fill unnecessarily
if (gidx_buffer.Size() == 0) {
gidx_buffer.Resize(compressed_size_bytes, 0);
} else {
gidx_buffer.Resize(compressed_size_bytes, 0);
thrust::fill(dh::tbegin(gidx_buffer), dh::tend(gidx_buffer), 0);
}
auto init = static_cast<common::CompressedByteT>(0);
gidx_buffer = common::MakeFixedVecWithCudaMalloc(ctx, compressed_size_bytes, init);
monitor_.Stop(__func__);
}
// Compress a CSR page into ELLPACK.
@@ -496,7 +481,7 @@ void EllpackPageImpl::CreateHistIndices(DeviceOrd device,
const SparsePage& row_batch,
common::Span<FeatureType const> feature_types) {
if (row_batch.Size() == 0) return;
unsigned int null_gidx_value = NumSymbols() - 1;
std::uint32_t null_gidx_value = NumSymbols() - 1;
const auto& offset_vec = row_batch.offset.ConstHostVector();
@@ -541,13 +526,11 @@ void EllpackPageImpl::CreateHistIndices(DeviceOrd device,
const dim3 grid3(common::DivRoundUp(batch_nrows, block3.x),
common::DivRoundUp(row_stride, block3.y), 1);
auto device_accessor = GetDeviceAccessor(device);
dh::LaunchKernel {grid3, block3}(
CompressBinEllpackKernel, common::CompressedBufferWriter(NumSymbols()),
gidx_buffer.DevicePointer(), row_ptrs.data().get(),
entries_d.data().get(), device_accessor.gidx_fvalue_map.data(),
device_accessor.feature_segments.data(), feature_types,
batch_row_begin, batch_nrows, row_stride,
null_gidx_value);
dh::LaunchKernel{grid3, block3}( // NOLINT
CompressBinEllpackKernel, common::CompressedBufferWriter(NumSymbols()), gidx_buffer.data(),
row_ptrs.data().get(), entries_d.data().get(), device_accessor.gidx_fvalue_map.data(),
device_accessor.feature_segments.data(), feature_types, batch_row_begin, batch_nrows,
row_stride, null_gidx_value);
}
}
@@ -566,26 +549,31 @@ size_t EllpackPageImpl::MemCostBytes(size_t num_rows, size_t row_stride,
EllpackDeviceAccessor EllpackPageImpl::GetDeviceAccessor(
DeviceOrd device, common::Span<FeatureType const> feature_types) const {
gidx_buffer.SetDevice(device);
return {device,
cuts_,
is_dense,
row_stride,
base_rowid,
n_rows,
common::CompressedIterator<uint32_t>(gidx_buffer.ConstDevicePointer(),
NumSymbols()),
common::CompressedIterator<uint32_t>(gidx_buffer.data(), NumSymbols()),
feature_types};
}
EllpackDeviceAccessor EllpackPageImpl::GetHostAccessor(
Context const* ctx, std::vector<common::CompressedByteT>* h_gidx_buffer,
common::Span<FeatureType const> feature_types) const {
h_gidx_buffer->resize(gidx_buffer.size());
CHECK_EQ(h_gidx_buffer->size(), gidx_buffer.size());
CHECK_NE(gidx_buffer.size(), 0);
dh::safe_cuda(cudaMemcpyAsync(h_gidx_buffer->data(), gidx_buffer.data(), gidx_buffer.size_bytes(),
cudaMemcpyDefault, dh::DefaultStream()));
return {DeviceOrd::CPU(),
cuts_,
is_dense,
row_stride,
base_rowid,
n_rows,
common::CompressedIterator<uint32_t>(gidx_buffer.ConstHostPointer(), NumSymbols()),
common::CompressedIterator<uint32_t>(h_gidx_buffer->data(), NumSymbols()),
feature_types};
}
} // namespace xgboost

101
src/data/ellpack_page.cuh
View File

@@ -1,23 +1,25 @@
/**
* Copyright 2019-2023, XGBoost Contributors
* Copyright 2019-2024, XGBoost Contributors
*/
#ifndef XGBOOST_DATA_ELLPACK_PAGE_CUH_
#define XGBOOST_DATA_ELLPACK_PAGE_CUH_
#include <thrust/binary_search.h>
#include <xgboost/data.h>
#include "../common/categorical.h"
#include "../common/compressed_iterator.h"
#include "../common/device_helpers.cuh"
#include "../common/hist_util.h"
#include "../common/ref_resource_view.h" // for RefResourceView
#include "ellpack_page.h"
#include "xgboost/data.h"
namespace xgboost {
/** \brief Struct for accessing and manipulating an ELLPACK matrix on the
* device. Does not own underlying memory and may be trivially copied into
* kernels.*/
/**
* @brief Struct for accessing and manipulating an ELLPACK matrix on the device.
*
* Does not own underlying memory and may be trivially copied into kernels.
*/
struct EllpackDeviceAccessor {
/*! \brief Whether or not if the matrix is dense. */
bool is_dense;
@@ -128,31 +130,31 @@ class GHistIndexMatrix;
class EllpackPageImpl {
public:
/*!
* \brief Default constructor.
/**
* @brief Default constructor.
*
* This is used in the external memory case. An empty ELLPACK page is constructed with its content
* set later by the reader.
*/
EllpackPageImpl() = default;
/*!
* \brief Constructor from an existing EllpackInfo.
/**
* @brief Constructor from an existing EllpackInfo.
*
* This is used in the sampling case. The ELLPACK page is constructed from an existing EllpackInfo
* and the given number of rows.
* This is used in the sampling case. The ELLPACK page is constructed from an existing
* Ellpack page and the given number of rows.
*/
EllpackPageImpl(DeviceOrd device, std::shared_ptr<common::HistogramCuts const> cuts,
EllpackPageImpl(Context const* ctx, std::shared_ptr<common::HistogramCuts const> cuts,
bool is_dense, bst_idx_t row_stride, bst_idx_t n_rows);
/*!
* \brief Constructor used for external memory.
/**
* @brief Constructor used for external memory.
*/
EllpackPageImpl(DeviceOrd device, std::shared_ptr<common::HistogramCuts const> cuts,
EllpackPageImpl(Context const* ctx, std::shared_ptr<common::HistogramCuts const> cuts,
const SparsePage& page, bool is_dense, size_t row_stride,
common::Span<FeatureType const> feature_types);
/*!
* \brief Constructor from an existing DMatrix.
/**
* @brief Constructor from an existing DMatrix.
*
* This is used in the in-memory case. The ELLPACK page is constructed from an existing DMatrix
* in CSR format.
@@ -160,37 +162,39 @@ class EllpackPageImpl {
explicit EllpackPageImpl(Context const* ctx, DMatrix* dmat, const BatchParam& parm);
template <typename AdapterBatch>
explicit EllpackPageImpl(AdapterBatch batch, float missing, DeviceOrd device, bool is_dense,
explicit EllpackPageImpl(Context const* ctx, AdapterBatch batch, float missing, bool is_dense,
common::Span<size_t> row_counts_span,
common::Span<FeatureType const> feature_types, size_t row_stride,
size_t n_rows, std::shared_ptr<common::HistogramCuts const> cuts);
/**
* \brief Constructor from an existing CPU gradient index.
* @brief Constructor from an existing CPU gradient index.
*/
explicit EllpackPageImpl(Context const* ctx, GHistIndexMatrix const& page,
common::Span<FeatureType const> ft);
/*! \brief Copy the elements of the given ELLPACK page into this page.
/**
* @brief Copy the elements of the given ELLPACK page into this page.
*
* @param device The GPU device to use.
* @param ctx The GPU context.
* @param page The ELLPACK page to copy from.
* @param offset The number of elements to skip before copying.
* @returns The number of elements copied.
*/
size_t Copy(DeviceOrd device, EllpackPageImpl const *page, size_t offset);
bst_idx_t Copy(Context const* ctx, EllpackPageImpl const* page, bst_idx_t offset);
/*! \brief Compact the given ELLPACK page into the current page.
/**
* @brief Compact the given ELLPACK page into the current page.
*
* @param context The GPU context.
* @param ctx The GPU context.
* @param page The ELLPACK page to compact from.
* @param row_indexes Row indexes for the compacted page.
*/
void Compact(Context const* ctx, EllpackPageImpl const* page, common::Span<size_t> row_indexes);
/*! \return Number of instances in the page. */
/** @return Number of instances in the page. */
[[nodiscard]] bst_idx_t Size() const;
/*! \brief Set the base row id for this page. */
/** @brief Set the base row id for this page. */
void SetBaseRowId(std::size_t row_id) {
base_rowid = row_id;
}
@@ -199,43 +203,54 @@ class EllpackPageImpl {
[[nodiscard]] std::shared_ptr<common::HistogramCuts const> CutsShared() const { return cuts_; }
void SetCuts(std::shared_ptr<common::HistogramCuts const> cuts) { cuts_ = cuts; }
/*! \return Estimation of memory cost of this page. */
/** @return Estimation of memory cost of this page. */
static size_t MemCostBytes(size_t num_rows, size_t row_stride, const common::HistogramCuts&cuts) ;
/*! \brief Return the total number of symbols (total number of bins plus 1 for
* not found). */
/**
* @brief Return the total number of symbols (total number of bins plus 1 for not
* found).
*/
[[nodiscard]] std::size_t NumSymbols() const { return cuts_->TotalBins() + 1; }
/**
* @brief Get an accessor that can be passed into CUDA kernels.
*/
[[nodiscard]] EllpackDeviceAccessor GetDeviceAccessor(
DeviceOrd device, common::Span<FeatureType const> feature_types = {}) const;
/**
* @brief Get an accessor for host code.
*/
[[nodiscard]] EllpackDeviceAccessor GetHostAccessor(
Context const* ctx, std::vector<common::CompressedByteT>* h_gidx_buffer,
common::Span<FeatureType const> feature_types = {}) const;
private:
/*!
* \brief Compress a single page of CSR data into ELLPACK.
/**
* @brief Compress a single page of CSR data into ELLPACK.
*
* @param device The GPU device to use.
* @param row_batch The CSR page.
*/
void CreateHistIndices(DeviceOrd device,
const SparsePage& row_batch,
void CreateHistIndices(DeviceOrd device, const SparsePage& row_batch,
common::Span<FeatureType const> feature_types);
/*!
* \brief Initialize the buffer to store compressed features.
/**
* @brief Initialize the buffer to store compressed features.
*/
void InitCompressedData(DeviceOrd device);
void InitCompressedData(Context const* ctx);
public:
/*! \brief Whether or not if the matrix is dense. */
/** @brief Whether or not if the matrix is dense. */
bool is_dense;
/*! \brief Row length for ELLPACK. */
/** @brief Row length for ELLPACK. */
bst_idx_t row_stride;
bst_idx_t base_rowid{0};
bst_idx_t n_rows{};
/*! \brief global index of histogram, which is stored in ELLPACK format. */
HostDeviceVector<common::CompressedByteT> gidx_buffer;
bst_idx_t n_rows{0};
/**
* @brief Index of the gradient histogram, which is stored in ELLPACK format.
*
* This can be backed by various storage types.
*/
common::RefResourceView<common::CompressedByteT> gidx_buffer;
private:
std::shared_ptr<common::HistogramCuts const> cuts_;

97
src/data/ellpack_page_raw_format.cu
View File

@@ -4,11 +4,12 @@
#include <dmlc/registry.h>
#include <cstddef> // for size_t
#include <cstdint> // for uint64_t
#include <vector> // for vector
#include "../common/io.h" // for AlignedResourceReadStream, AlignedFileWriteStream
#include "../common/ref_resource_view.h" // for ReadVec, WriteVec
#include "ellpack_page.cuh" // for EllpackPage
#include "../common/io.h" // for AlignedResourceReadStream, AlignedFileWriteStream
#include "../common/ref_resource_view.cuh" // for MakeFixedVecWithCudaMalloc
#include "../common/ref_resource_view.h" // for ReadVec, WriteVec
#include "ellpack_page.cuh" // for EllpackPage
#include "ellpack_page_raw_format.h"
#include "ellpack_page_source.h"
@@ -16,8 +17,10 @@ namespace xgboost::data {
DMLC_REGISTRY_FILE_TAG(ellpack_page_raw_format);
namespace {
// Function to support system without HMM or ATS
template <typename T>
[[nodiscard]] bool ReadDeviceVec(common::AlignedResourceReadStream* fi, HostDeviceVector<T>* vec) {
[[nodiscard]] bool ReadDeviceVec(common::AlignedResourceReadStream* fi,
common::RefResourceView<T>* vec) {
std::uint64_t n{0};
if (!fi->Read(&n)) {
return false;
@@ -33,34 +36,34 @@ template <typename T>
return false;
}
vec->Resize(n);
auto d_vec = vec->DeviceSpan();
dh::safe_cuda(
cudaMemcpyAsync(d_vec.data(), ptr, n_bytes, cudaMemcpyDefault, dh::DefaultStream()));
auto ctx = Context{}.MakeCUDA(common::CurrentDevice());
*vec = common::MakeFixedVecWithCudaMalloc(&ctx, n, static_cast<T>(0));
dh::safe_cuda(cudaMemcpyAsync(vec->data(), ptr, n_bytes, cudaMemcpyDefault, dh::DefaultStream()));
return true;
}
} // namespace
#define RET_IF_NOT(expr) \
if (!(expr)) { \
return false; \
}
[[nodiscard]] bool EllpackPageRawFormat::Read(EllpackPage* page,
common::AlignedResourceReadStream* fi) {
auto* impl = page->Impl();
impl->SetCuts(this->cuts_);
if (!fi->Read(&impl->n_rows)) {
return false;
}
if (!fi->Read(&impl->is_dense)) {
return false;
}
if (!fi->Read(&impl->row_stride)) {
return false;
}
impl->gidx_buffer.SetDevice(device_);
if (!ReadDeviceVec(fi, &impl->gidx_buffer)) {
return false;
}
if (!fi->Read(&impl->base_rowid)) {
return false;
RET_IF_NOT(fi->Read(&impl->n_rows));
RET_IF_NOT(fi->Read(&impl->is_dense));
RET_IF_NOT(fi->Read(&impl->row_stride));
if (has_hmm_ats_) {
RET_IF_NOT(common::ReadVec(fi, &impl->gidx_buffer));
} else {
RET_IF_NOT(ReadDeviceVec(fi, &impl->gidx_buffer));
}
RET_IF_NOT(fi->Read(&impl->base_rowid));
dh::DefaultStream().Sync();
return true;
}
@@ -71,8 +74,10 @@ template <typename T>
bytes += fo->Write(impl->n_rows);
bytes += fo->Write(impl->is_dense);
bytes += fo->Write(impl->row_stride);
CHECK(!impl->gidx_buffer.ConstHostVector().empty());
bytes += common::WriteVec(fo, impl->gidx_buffer.HostVector());
std::vector<common::CompressedByteT> h_gidx_buffer;
Context ctx = Context{}.MakeCUDA(common::CurrentDevice());
[[maybe_unused]] auto h_accessor = impl->GetHostAccessor(&ctx, &h_gidx_buffer);
bytes += common::WriteVec(fo, h_gidx_buffer);
bytes += fo->Write(impl->base_rowid);
dh::DefaultStream().Sync();
return bytes;
@@ -82,33 +87,20 @@ template <typename T>
auto* impl = page->Impl();
CHECK(this->cuts_->cut_values_.DeviceCanRead());
impl->SetCuts(this->cuts_);
if (!fi->Read(&impl->n_rows)) {
return false;
}
if (!fi->Read(&impl->is_dense)) {
return false;
}
if (!fi->Read(&impl->row_stride)) {
return false;
}
RET_IF_NOT(fi->Read(&impl->n_rows));
RET_IF_NOT(fi->Read(&impl->is_dense));
RET_IF_NOT(fi->Read(&impl->row_stride));
// Read vec
Context ctx = Context{}.MakeCUDA(common::CurrentDevice());
bst_idx_t n{0};
if (!fi->Read(&n)) {
return false;
}
RET_IF_NOT(fi->Read(&n));
if (n != 0) {
impl->gidx_buffer.SetDevice(device_);
impl->gidx_buffer.Resize(n);
auto span = impl->gidx_buffer.DeviceSpan();
if (!fi->Read(span.data(), span.size_bytes())) {
return false;
}
}
if (!fi->Read(&impl->base_rowid)) {
return false;
impl->gidx_buffer =
common::MakeFixedVecWithCudaMalloc(&ctx, n, static_cast<common::CompressedByteT>(0));
RET_IF_NOT(fi->Read(impl->gidx_buffer.data(), impl->gidx_buffer.size_bytes()));
}
RET_IF_NOT(fi->Read(&impl->base_rowid));
dh::DefaultStream().Sync();
return true;
@@ -123,16 +115,17 @@ template <typename T>
bytes += fo->Write(impl->row_stride);
// Write vector
bst_idx_t n = impl->gidx_buffer.Size();
bst_idx_t n = impl->gidx_buffer.size();
bytes += fo->Write(n);
if (!impl->gidx_buffer.Empty()) {
auto span = impl->gidx_buffer.ConstDeviceSpan();
bytes += fo->Write(span.data(), span.size_bytes());
if (!impl->gidx_buffer.empty()) {
bytes += fo->Write(impl->gidx_buffer.data(), impl->gidx_buffer.size_bytes());
}
bytes += fo->Write(impl->base_rowid);
dh::DefaultStream().Sync();
return bytes;
}
#undef RET_IF_NOT
} // namespace xgboost::data

7
src/data/ellpack_page_raw_format.h
View File

@@ -26,10 +26,13 @@ class EllpackHostCacheStream;
class EllpackPageRawFormat : public SparsePageFormat<EllpackPage> {
std::shared_ptr<common::HistogramCuts const> cuts_;
DeviceOrd device_;
// Supports CUDA HMM or ATS
bool has_hmm_ats_{false};
public:
explicit EllpackPageRawFormat(std::shared_ptr<common::HistogramCuts const> cuts, DeviceOrd device)
: cuts_{std::move(cuts)}, device_{device} {}
explicit EllpackPageRawFormat(std::shared_ptr<common::HistogramCuts const> cuts, DeviceOrd device,
bool has_hmm_ats)
: cuts_{std::move(cuts)}, device_{device}, has_hmm_ats_{has_hmm_ats} {}
[[nodiscard]] bool Read(EllpackPage* page, common::AlignedResourceReadStream* fi) override;
[[nodiscard]] std::size_t Write(const EllpackPage& page,
common::AlignedFileWriteStream* fo) override;

56
src/data/ellpack_page_source.cu
View File

@@ -11,6 +11,7 @@
#include "../common/common.h" // for safe_cuda
#include "../common/cuda_pinned_allocator.h" // for pinned_allocator
#include "../common/device_helpers.cuh" // for CUDAStreamView, DefaultStream
#include "../common/resource.cuh" // for PrivateCudaMmapConstStream
#include "ellpack_page.cuh" // for EllpackPageImpl
#include "ellpack_page.h" // for EllpackPage
#include "ellpack_page_source.h"
@@ -86,16 +87,16 @@ void EllpackHostCacheStream::Seek(bst_idx_t offset_bytes) { this->p_impl_->Seek(
void EllpackHostCacheStream::Bound(bst_idx_t offset_bytes) { this->p_impl_->Bound(offset_bytes); }
/**
* EllpackFormatType
* EllpackCacheStreamPolicy
*/
template <typename S, template <typename> typename F>
EllpackFormatStreamPolicy<S, F>::EllpackFormatStreamPolicy()
EllpackCacheStreamPolicy<S, F>::EllpackCacheStreamPolicy()
: p_cache_{std::make_shared<EllpackHostCache>()} {}
template <typename S, template <typename> typename F>
[[nodiscard]] std::unique_ptr<typename EllpackFormatStreamPolicy<S, F>::WriterT>
EllpackFormatStreamPolicy<S, F>::CreateWriter(StringView, std::uint32_t iter) {
[[nodiscard]] std::unique_ptr<typename EllpackCacheStreamPolicy<S, F>::WriterT>
EllpackCacheStreamPolicy<S, F>::CreateWriter(StringView, std::uint32_t iter) {
auto fo = std::make_unique<EllpackHostCacheStream>(this->p_cache_);
if (iter == 0) {
CHECK(this->p_cache_->cache.empty());
@@ -106,9 +107,8 @@ EllpackFormatStreamPolicy<S, F>::CreateWriter(StringView, std::uint32_t iter) {
}
template <typename S, template <typename> typename F>
[[nodiscard]] std::unique_ptr<typename EllpackFormatStreamPolicy<S, F>::ReaderT>
EllpackFormatStreamPolicy<S, F>::CreateReader(StringView, bst_idx_t offset,
bst_idx_t length) const {
[[nodiscard]] std::unique_ptr<typename EllpackCacheStreamPolicy<S, F>::ReaderT>
EllpackCacheStreamPolicy<S, F>::CreateReader(StringView, bst_idx_t offset, bst_idx_t length) const {
auto fi = std::make_unique<ReaderT>(this->p_cache_);
fi->Seek(offset);
fi->Bound(offset + length);
@@ -117,18 +117,40 @@ EllpackFormatStreamPolicy<S, F>::CreateReader(StringView, bst_idx_t offset,
}
// Instantiation
template EllpackFormatStreamPolicy<EllpackPage, EllpackFormatPolicy>::EllpackFormatStreamPolicy();
template EllpackCacheStreamPolicy<EllpackPage, EllpackFormatPolicy>::EllpackCacheStreamPolicy();
template std::unique_ptr<
typename EllpackFormatStreamPolicy<EllpackPage, EllpackFormatPolicy>::WriterT>
EllpackFormatStreamPolicy<EllpackPage, EllpackFormatPolicy>::CreateWriter(StringView name,
std::uint32_t iter);
typename EllpackCacheStreamPolicy<EllpackPage, EllpackFormatPolicy>::WriterT>
EllpackCacheStreamPolicy<EllpackPage, EllpackFormatPolicy>::CreateWriter(StringView name,
std::uint32_t iter);
template std::unique_ptr<
typename EllpackFormatStreamPolicy<EllpackPage, EllpackFormatPolicy>::ReaderT>
EllpackFormatStreamPolicy<EllpackPage, EllpackFormatPolicy>::CreateReader(
typename EllpackCacheStreamPolicy<EllpackPage, EllpackFormatPolicy>::ReaderT>
EllpackCacheStreamPolicy<EllpackPage, EllpackFormatPolicy>::CreateReader(
StringView name, std::uint64_t offset, std::uint64_t length) const;
/**
* EllpackMmapStreamPolicy
*/
template <typename S, template <typename> typename F>
[[nodiscard]] std::unique_ptr<typename EllpackMmapStreamPolicy<S, F>::ReaderT>
EllpackMmapStreamPolicy<S, F>::CreateReader(StringView name, bst_idx_t offset,
bst_idx_t length) const {
if (has_hmm_) {
return std::make_unique<common::PrivateCudaMmapConstStream>(name, offset, length);
} else {
return std::make_unique<common::PrivateMmapConstStream>(name, offset, length);
}
}
// Instantiation
template std::unique_ptr<
typename EllpackMmapStreamPolicy<EllpackPage, EllpackFormatPolicy>::ReaderT>
EllpackMmapStreamPolicy<EllpackPage, EllpackFormatPolicy>::CreateReader(StringView name,
bst_idx_t offset,
bst_idx_t length) const;
/**
* EllpackPageSourceImpl
*/
@@ -146,8 +168,8 @@ void EllpackPageSourceImpl<F>::Fetch() {
auto const& csr = this->source_->Page();
this->page_.reset(new EllpackPage{});
auto* impl = this->page_->Impl();
*impl = EllpackPageImpl{this->Device(), this->GetCuts(), *csr,
is_dense_, row_stride_, feature_types_};
Context ctx = Context{}.MakeCUDA(this->Device().ordinal);
*impl = EllpackPageImpl{&ctx, this->GetCuts(), *csr, is_dense_, row_stride_, feature_types_};
this->page_->SetBaseRowId(csr->base_rowid);
this->WriteCache();
}
@@ -157,5 +179,7 @@ void EllpackPageSourceImpl<F>::Fetch() {
template void
EllpackPageSourceImpl<DefaultFormatStreamPolicy<EllpackPage, EllpackFormatPolicy>>::Fetch();
template void
EllpackPageSourceImpl<EllpackFormatStreamPolicy<EllpackPage, EllpackFormatPolicy>>::Fetch();
EllpackPageSourceImpl<EllpackCacheStreamPolicy<EllpackPage, EllpackFormatPolicy>>::Fetch();
template void
EllpackPageSourceImpl<EllpackMmapStreamPolicy<EllpackPage, EllpackFormatPolicy>>::Fetch();
} // namespace xgboost::data

45
src/data/ellpack_page_source.h
View File

@@ -9,6 +9,7 @@
#include <memory> // for shared_ptr
#include <utility> // for move
#include "../common/cuda_rt_utils.h" // for SupportsPageableMem
#include "../common/hist_util.h" // for HistogramCuts
#include "ellpack_page.h" // for EllpackPage
#include "ellpack_page_raw_format.h" // for EllpackPageRawFormat
@@ -59,14 +60,19 @@ template <typename S>
class EllpackFormatPolicy {
std::shared_ptr<common::HistogramCuts const> cuts_{nullptr};
DeviceOrd device_;
bool has_hmm_{common::SupportsPageableMem()};
public:
using FormatT = EllpackPageRawFormat;
public:
EllpackFormatPolicy() = default;
// For testing with the HMM flag.
explicit EllpackFormatPolicy(bool has_hmm) : has_hmm_{has_hmm} {}
[[nodiscard]] auto CreatePageFormat() const {
CHECK_EQ(cuts_->cut_values_.Device(), device_);
std::unique_ptr<FormatT> fmt{new EllpackPageRawFormat{cuts_, device_}};
std::unique_ptr<FormatT> fmt{new EllpackPageRawFormat{cuts_, device_, has_hmm_}};
return fmt;
}
@@ -83,7 +89,7 @@ class EllpackFormatPolicy {
};
template <typename S, template <typename> typename F>
class EllpackFormatStreamPolicy : public F<S> {
class EllpackCacheStreamPolicy : public F<S> {
std::shared_ptr<EllpackHostCache> p_cache_;
public:
@@ -91,13 +97,42 @@ class EllpackFormatStreamPolicy : public F<S> {
using ReaderT = EllpackHostCacheStream;
public:
EllpackFormatStreamPolicy();
EllpackCacheStreamPolicy();
[[nodiscard]] std::unique_ptr<WriterT> CreateWriter(StringView name, std::uint32_t iter);
[[nodiscard]] std::unique_ptr<ReaderT> CreateReader(StringView name, bst_idx_t offset,
bst_idx_t length) const;
};
template <typename S, template <typename> typename F>
class EllpackMmapStreamPolicy : public F<S> {
bool has_hmm_{common::SupportsPageableMem()};
public:
using WriterT = common::AlignedFileWriteStream;
using ReaderT = common::AlignedResourceReadStream;
public:
EllpackMmapStreamPolicy() = default;
// For testing with the HMM flag.
template <
typename std::enable_if_t<std::is_same_v<F<S>, EllpackFormatPolicy<EllpackPage>>>* = nullptr>
explicit EllpackMmapStreamPolicy(bool has_hmm) : F<S>{has_hmm}, has_hmm_{has_hmm} {}
[[nodiscard]] std::unique_ptr<WriterT> CreateWriter(StringView name, std::uint32_t iter) {
std::unique_ptr<common::AlignedFileWriteStream> fo;
if (iter == 0) {
fo = std::make_unique<common::AlignedFileWriteStream>(name, "wb");
} else {
fo = std::make_unique<common::AlignedFileWriteStream>(name, "ab");
}
return fo;
}
[[nodiscard]] std::unique_ptr<ReaderT> CreateReader(StringView name, bst_idx_t offset,
bst_idx_t length) const;
};
template <typename F>
class EllpackPageSourceImpl : public PageSourceIncMixIn<EllpackPage, F> {
using Super = PageSourceIncMixIn<EllpackPage, F>;
@@ -128,11 +163,11 @@ class EllpackPageSourceImpl : public PageSourceIncMixIn<EllpackPage, F> {
// Cache to host
using EllpackPageHostSource =
EllpackPageSourceImpl<EllpackFormatStreamPolicy<EllpackPage, EllpackFormatPolicy>>;
EllpackPageSourceImpl<EllpackCacheStreamPolicy<EllpackPage, EllpackFormatPolicy>>;
// Cache to disk
using EllpackPageSource =
EllpackPageSourceImpl<DefaultFormatStreamPolicy<EllpackPage, EllpackFormatPolicy>>;
EllpackPageSourceImpl<EllpackMmapStreamPolicy<EllpackPage, EllpackFormatPolicy>>;
#if !defined(XGBOOST_USE_CUDA)
template <typename F>

6
src/data/gradient_index.cu
View File

@@ -16,7 +16,8 @@ template <typename BinT, typename CompressOffset>
void SetIndexData(Context const* ctx, EllpackPageImpl const* page,
std::vector<size_t>* p_hit_count_tloc, CompressOffset&& get_offset,
GHistIndexMatrix* out) {
auto accessor = page->GetHostAccessor();
std::vector<common::CompressedByteT> h_gidx_buffer;
auto accessor = page->GetHostAccessor(ctx, &h_gidx_buffer);
auto const kNull = static_cast<bst_bin_t>(accessor.NullValue());
common::Span<BinT> index_data_span = {out->index.data<BinT>(), out->index.Size()};
@@ -47,7 +48,8 @@ void GetRowPtrFromEllpack(Context const* ctx, EllpackPageImpl const* page,
if (page->is_dense) {
std::fill(row_ptr.begin() + 1, row_ptr.end(), page->row_stride);
} else {
auto accessor = page->GetHostAccessor();
std::vector<common::CompressedByteT> h_gidx_buffer;
auto accessor = page->GetHostAccessor(ctx, &h_gidx_buffer);
auto const kNull = static_cast<bst_bin_t>(accessor.NullValue());
common::ParallelFor(page->Size(), ctx->Threads(), [&](auto i) {

49
src/data/histogram_cut_format.h
View File

@@ -1,49 +0,0 @@
/**
* Copyright 2021-2024, XGBoost contributors
*/
#ifndef XGBOOST_DATA_HISTOGRAM_CUT_FORMAT_H_
#define XGBOOST_DATA_HISTOGRAM_CUT_FORMAT_H_
#include <dmlc/io.h> // for Stream
#include <cstddef> // for size_t
#include "../common/hist_util.h" // for HistogramCuts
#include "../common/io.h" // for AlignedResourceReadStream, AlignedFileWriteStream
#include "../common/ref_resource_view.h" // for WriteVec, ReadVec
namespace xgboost::data {
inline bool ReadHistogramCuts(common::HistogramCuts *cuts, common::AlignedResourceReadStream *fi) {
if (!common::ReadVec(fi, &cuts->cut_values_.HostVector())) {
return false;
}
if (!common::ReadVec(fi, &cuts->cut_ptrs_.HostVector())) {
return false;
}
if (!common::ReadVec(fi, &cuts->min_vals_.HostVector())) {
return false;
}
bool has_cat{false};
if (!fi->Read(&has_cat)) {
return false;
}
decltype(cuts->MaxCategory()) max_cat{0};
if (!fi->Read(&max_cat)) {
return false;
}
cuts->SetCategorical(has_cat, max_cat);
return true;
}
inline std::size_t WriteHistogramCuts(common::HistogramCuts const &cuts,
common::AlignedFileWriteStream *fo) {
std::size_t bytes = 0;
bytes += common::WriteVec(fo, cuts.Values());
bytes += common::WriteVec(fo, cuts.Ptrs());
bytes += common::WriteVec(fo, cuts.MinValues());
bytes += fo->Write(cuts.HasCategorical());
bytes += fo->Write(cuts.MaxCategory());
return bytes;
}
} // namespace xgboost::data
#endif // XGBOOST_DATA_HISTOGRAM_CUT_FORMAT_H_

15
src/data/iterative_dmatrix.cu
View File

@@ -5,6 +5,7 @@
#include <memory>
#include "../collective/allreduce.h"
#include "../common/cuda_rt_utils.h" // for AllVisibleGPUs
#include "../common/hist_util.cuh"
#include "batch_utils.h" // for RegenGHist
#include "device_adapter.cuh"
@@ -45,11 +46,17 @@ void IterativeDMatrix::InitFromCUDA(Context const* ctx, BatchParam const& p,
int32_t current_device;
dh::safe_cuda(cudaGetDevice(&current_device));
auto get_ctx = [&]() {
Context d_ctx = (ctx->IsCUDA()) ? *ctx : Context{}.MakeCUDA(current_device);
CHECK(!d_ctx.IsCPU());
return d_ctx;
};
auto get_device = [&]() {
auto d = (ctx->IsCUDA()) ? ctx->Device() : DeviceOrd::CUDA(current_device);
CHECK(!d.IsCPU());
return d;
};
fmat_ctx_ = get_ctx();
/**
* Generate quantiles
@@ -118,7 +125,7 @@ void IterativeDMatrix::InitFromCUDA(Context const* ctx, BatchParam const& p,
// that case device id is invalid.
ellpack_.reset(new EllpackPage);
*(ellpack_->Impl()) =
EllpackPageImpl(get_device(), cuts, this->IsDense(), row_stride, accumulated_rows);
EllpackPageImpl(&fmat_ctx_, cuts, this->IsDense(), row_stride, accumulated_rows);
}
};
@@ -142,10 +149,10 @@ void IterativeDMatrix::InitFromCUDA(Context const* ctx, BatchParam const& p,
proxy->Info().feature_types.SetDevice(get_device());
auto d_feature_types = proxy->Info().feature_types.ConstDeviceSpan();
auto new_impl = cuda_impl::Dispatch(proxy, [&](auto const& value) {
return EllpackPageImpl(value, missing, get_device(), is_dense, row_counts_span,
d_feature_types, row_stride, rows, cuts);
return EllpackPageImpl(&fmat_ctx_, value, missing, is_dense, row_counts_span, d_feature_types,
row_stride, rows, cuts);
});
size_t num_elements = ellpack_->Impl()->Copy(get_device(), &new_impl, offset);
std::size_t num_elements = ellpack_->Impl()->Copy(&fmat_ctx_, &new_impl, offset);
offset += num_elements;
proxy->Info().num_row_ = num_rows();

2
src/data/sparse_page_source.h
View File

@@ -226,7 +226,7 @@ class SparsePageSourceImpl : public BatchIteratorImpl<S>, public FormatStreamPol
}
// An heuristic for number of pre-fetched batches. We can make it part of BatchParam
// to let user adjust number of pre-fetched batches when needed.
std::int32_t kPrefetches = 3;
std::int32_t constexpr kPrefetches = 3;
std::int32_t n_prefetches = std::min(nthreads_, kPrefetches);
n_prefetches = std::max(n_prefetches, 1);
std::int32_t n_prefetch_batches = std::min(static_cast<bst_idx_t>(n_prefetches), n_batches_);

4
src/gbm/gblinear.cc
View File

@@ -10,12 +10,12 @@
#include <algorithm>
#include <numeric>
#include <sstream>
#include <string>
#include <vector>
#include "../common/common.h"
#include "../common/error_msg.h" // NoCategorical, DeprecatedFunc
#include "../common/cuda_rt_utils.h" // for AllVisibleGPUs
#include "../common/error_msg.h" // NoCategorical, DeprecatedFunc
#include "../common/threading_utils.h"
#include "../common/timer.h"
#include "gblinear_model.h"

8
src/gbm/gbtree.cc
View File

@@ -1,5 +1,5 @@
/**
* Copyright 2014-2023 by Contributors
* Copyright 2014-2024, XGBoost Contributors
* \file gbtree.cc
* \brief gradient boosted tree implementation.
* \author Tianqi Chen
@@ -10,14 +10,14 @@
#include <dmlc/parameter.h>
#include <algorithm> // for equal
#include <cinttypes> // for uint32_t
#include <limits>
#include <cstdint> // for uint32_t
#include <memory>
#include <string>
#include <utility>
#include <vector>
#include "../common/common.h"
#include "../common/cuda_rt_utils.h" // for AllVisibleGPUs
#include "../common/error_msg.h" // for UnknownDevice, WarnOldSerialization, InplacePredictProxy
#include "../common/random.h"
#include "../common/threading_utils.h"
@@ -347,7 +347,7 @@ void GBTree::LoadConfig(Json const& in) {
// This would cause all trees to be pushed to trees_to_update
// e.g. updating a model, then saving and loading it would result in an empty model
tparam_.process_type = TreeProcessType::kDefault;
std::int32_t const n_gpus = xgboost::common::AllVisibleGPUs();
std::int32_t const n_gpus = common::AllVisibleGPUs();
auto msg = StringView{
R"(

2
src/learner.cc
View File

@@ -1321,7 +1321,7 @@ class LearnerImpl : public LearnerIO {
std::ostringstream os;
os.precision(std::numeric_limits<double>::max_digits10);
os << '[' << iter << ']' << std::setiosflags(std::ios::fixed);
if (metrics_.empty() && tparam_.disable_default_eval_metric <= 0) {
if (metrics_.empty() && !tparam_.disable_default_eval_metric) {
metrics_.emplace_back(Metric::Create(obj_->DefaultEvalMetric(), &ctx_));
auto config = obj_->DefaultMetricConfig();
if (!IsA<Null>(config)) {

1
src/predictor/gpu_predictor.cu
View File

@@ -16,6 +16,7 @@
#include "../common/categorical.h"
#include "../common/common.h"
#include "../common/cuda_context.cuh" // for CUDAContext
#include "../common/cuda_rt_utils.h" // for AllVisibleGPUs
#include "../common/device_helpers.cuh"
#include "../common/error_msg.h" // for InplacePredictProxy
#include "../data/device_adapter.cuh"

3
src/tree/fit_stump.cu
View File

@@ -3,9 +3,6 @@
*
* @brief Utilities for estimating initial score.
*/
#if !defined(NOMINMAX) && defined(_WIN32)
#define NOMINMAX
#endif // !defined(NOMINMAX)
#include <thrust/execution_policy.h> // cuda::par
#include <thrust/iterator/counting_iterator.h> // thrust::make_counting_iterator

16
src/tree/fit_stump.h
View File

@@ -1,5 +1,5 @@
/**
* Copyright 2022 by XGBoost Contributors
* Copyright 2022-2024, XGBoost Contributors
*
* \brief Utilities for estimating initial score.
*/
@@ -7,18 +7,12 @@
#ifndef XGBOOST_TREE_FIT_STUMP_H_
#define XGBOOST_TREE_FIT_STUMP_H_
#if !defined(NOMINMAX) && defined(_WIN32)
#define NOMINMAX
#endif // !defined(NOMINMAX)
#include <algorithm> // std::max
#include "../common/common.h" // AssertGPUSupport
#include "xgboost/base.h" // GradientPair
#include "xgboost/context.h" // Context
#include "xgboost/data.h" // MetaInfo
#include "xgboost/host_device_vector.h" // HostDeviceVector
#include "xgboost/linalg.h" // TensorView
#include "xgboost/base.h" // GradientPair
#include "xgboost/context.h" // Context
#include "xgboost/data.h" // MetaInfo
#include "xgboost/linalg.h" // TensorView
namespace xgboost {
namespace tree {

22
src/tree/gpu_hist/gradient_based_sampler.cu
View File

@@ -163,14 +163,14 @@ GradientBasedSample ExternalMemoryNoSampling::Sample(Context const* ctx,
if (!page_concatenated_) {
// Concatenate all the external memory ELLPACK pages into a single in-memory page.
page_.reset(nullptr);
size_t offset = 0;
bst_idx_t offset = 0;
for (auto& batch : dmat->GetBatches<EllpackPage>(ctx, batch_param_)) {
auto page = batch.Impl();
if (!page_) {
page_ = std::make_unique<EllpackPageImpl>(ctx->Device(), page->CutsShared(), page->is_dense,
page_ = std::make_unique<EllpackPageImpl>(ctx, page->CutsShared(), page->is_dense,
page->row_stride, dmat->Info().num_row_);
}
size_t num_elements = page_->Copy(ctx->Device(), page, offset);
bst_idx_t num_elements = page_->Copy(ctx, page, offset);
offset += num_elements;
}
page_concatenated_ = true;
@@ -228,11 +228,11 @@ GradientBasedSample ExternalMemoryUniformSampling::Sample(Context const* ctx,
auto first_page = (*batch_iterator.begin()).Impl();
// Create a new ELLPACK page with empty rows.
page_.reset(); // Release the device memory first before reallocating
page_.reset(new EllpackPageImpl(ctx->Device(), first_page->CutsShared(), first_page->is_dense,
page_.reset(new EllpackPageImpl(ctx, first_page->CutsShared(), first_page->is_dense,
first_page->row_stride, sample_rows));
// Compact the ELLPACK pages into the single sample page.
thrust::fill(cuctx->CTP(), dh::tbegin(page_->gidx_buffer), dh::tend(page_->gidx_buffer), 0);
thrust::fill(cuctx->CTP(), page_->gidx_buffer.begin(), page_->gidx_buffer.end(), 0);
for (auto& batch : batch_iterator) {
page_->Compact(ctx, batch.Impl(), dh::ToSpan(sample_row_index_));
}
@@ -283,10 +283,10 @@ GradientBasedSample ExternalMemoryGradientBasedSampling::Sample(Context const* c
// Perform Poisson sampling in place.
thrust::transform(cuctx->CTP(), dh::tbegin(gpair), dh::tend(gpair),
thrust::counting_iterator<size_t>(0), dh::tbegin(gpair),
PoissonSampling(dh::ToSpan(threshold_), threshold_index,
RandomWeight(common::GlobalRandom()())));
PoissonSampling{dh::ToSpan(threshold_), threshold_index,
RandomWeight(common::GlobalRandom()())});
// Count the sampled rows.
size_t sample_rows =
bst_idx_t sample_rows =
thrust::count_if(cuctx->CTP(), dh::tbegin(gpair), dh::tend(gpair), IsNonZero());
// Compact gradient pairs.
gpair_.resize(sample_rows);
@@ -302,10 +302,10 @@ GradientBasedSample ExternalMemoryGradientBasedSampling::Sample(Context const* c
auto first_page = (*batch_iterator.begin()).Impl();
// Create a new ELLPACK page with empty rows.
page_.reset(); // Release the device memory first before reallocating
page_.reset(new EllpackPageImpl(ctx->Device(), first_page->CutsShared(), dmat->IsDense(),
first_page->row_stride, sample_rows));
page_.reset(new EllpackPageImpl{ctx, first_page->CutsShared(), dmat->IsDense(),
first_page->row_stride, sample_rows});
// Compact the ELLPACK pages into the single sample page.
thrust::fill(cuctx->CTP(), dh::tbegin(page_->gidx_buffer), dh::tend(page_->gidx_buffer), 0);
thrust::fill(cuctx->CTP(), page_->gidx_buffer.begin(), page_->gidx_buffer.end(), 0);
for (auto& batch : batch_iterator) {
page_->Compact(ctx, batch.Impl(), dh::ToSpan(sample_row_index_));
}

22
src/tree/gpu_hist/gradient_based_sampler.cuh
View File

@@ -1,20 +1,19 @@
/**
* Copyright 2019-2023, XGBoost Contributors
* Copyright 2019-2024, XGBoost Contributors
*/
#pragma once
#include <xgboost/base.h>
#include <xgboost/data.h>
#include <xgboost/span.h>
#include <cstddef> // for size_t
#include "../../common/device_helpers.cuh"
#include "../../data/ellpack_page.cuh"
namespace xgboost {
namespace tree {
#include "../../common/device_vector.cuh" // for device_vector, caching_device_vector
#include "../../data/ellpack_page.cuh" // for EllpackPageImpl
#include "xgboost/base.h" // for GradientPair
#include "xgboost/data.h" // for BatchParam
#include "xgboost/span.h" // for Span
namespace xgboost::tree {
struct GradientBasedSample {
/*!\brief Number of sampled rows. */
size_t sample_rows;
std::size_t sample_rows;
/*!\brief Sampled rows in ELLPACK format. */
EllpackPageImpl const* page;
/*!\brief Gradient pairs for the sampled rows. */
@@ -137,5 +136,4 @@ class GradientBasedSampler {
common::Monitor monitor_;
std::unique_ptr<SamplingStrategy> strategy_;
};
}; // namespace tree
}; // namespace xgboost
}; // namespace xgboost::tree

23
src/tree/updater_gpu_hist.cu
View File

@@ -16,7 +16,8 @@
#include "../collective/broadcast.h"
#include "../common/bitfield.h"
#include "../common/categorical.h"
#include "../common/cuda_context.cuh" // CUDAContext
#include "../common/cuda_context.cuh" // for CUDAContext
#include "../common/cuda_rt_utils.h" // for CheckComputeCapability
#include "../common/device_helpers.cuh"
#include "../common/hist_util.h"
#include "../common/random.h" // for ColumnSampler, GlobalRandom
@@ -826,7 +827,7 @@ class GPUHistMaker : public TreeUpdater {
// Used in test to count how many configurations are performed
LOG(DEBUG) << "[GPU Hist]: Configure";
hist_maker_param_.UpdateAllowUnknown(args);
dh::CheckComputeCapability();
common::CheckComputeCapability();
initialised_ = false;
monitor_.Init("updater_gpu_hist");
@@ -852,17 +853,13 @@ class GPUHistMaker : public TreeUpdater {
CHECK_EQ(gpair->Shape(1), 1) << MTNotImplemented();
auto gpair_hdv = gpair->Data();
// build tree
try {
std::size_t t_idx{0};
for (xgboost::RegTree* tree : trees) {
this->UpdateTree(param, gpair_hdv, dmat, tree, &out_position[t_idx]);
this->hist_maker_param_.CheckTreesSynchronized(ctx_, tree);
++t_idx;
}
dh::safe_cuda(cudaGetLastError());
} catch (const std::exception& e) {
LOG(FATAL) << "Exception in gpu_hist: " << e.what() << std::endl;
std::size_t t_idx{0};
for (xgboost::RegTree* tree : trees) {
this->UpdateTree(param, gpair_hdv, dmat, tree, &out_position[t_idx]);
this->hist_maker_param_.CheckTreesSynchronized(ctx_, tree);
++t_idx;
}
dh::safe_cuda(cudaGetLastError());
monitor_.Stop("Update");
}
@@ -958,7 +955,7 @@ class GPUGlobalApproxMaker : public TreeUpdater {
if (hist_maker_param_.max_cached_hist_node != HistMakerTrainParam::DefaultNodes()) {
LOG(WARNING) << "The `max_cached_hist_node` is ignored in GPU.";
}
dh::CheckComputeCapability();
common::CheckComputeCapability();
initialised_ = false;
monitor_.Init(this->Name());