Hendrik/xgboost
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Better error message for ncclUnhandledCudaError. (#7190)

Browse Source
This commit is contained in:
Jiaming Yuan 2021-08-27 10:29:22 +08:00 committed by GitHub
parent b70e07da1f
commit e7d7ab6bc3
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 9 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
src/common/device_helpers.cuh
View File

@ -113,9 +113,14 @@ inline ncclResult_t ThrowOnNcclError(ncclResult_t code, const char *file,
int line) {
if (code != ncclSuccess) {
std::stringstream ss;
ss << "NCCL failure :" << ncclGetErrorString(code) << " ";
ss << file << "(" << line << ")";
throw std::runtime_error(ss.str());
ss << "NCCL failure :" << ncclGetErrorString(code);
if (code == ncclUnhandledCudaError) {
// nccl usually preserves the last error so we can get more details.
auto err = cudaPeekAtLastError();
ss << " " << thrust::system_error(err, thrust::cuda_category()).what();
}
ss << " " << file << "(" << line << ")";
LOG(FATAL) << ss.str();
}
return code;