Hendrik/xgboost
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Handle special characters in JSON model dump. (#9474)

Browse Source
This commit is contained in:
Jiaming Yuan
2023-08-14 15:49:00 +08:00
committed by GitHub
parent f03463c45b
commit 05d7000096
7 changed files with 127 additions and 103 deletions
Download Patch File Download Diff File Expand all files Collapse all files

63
src/common/common.cc
View File

@@ -1,16 +1,17 @@
/*!
* Copyright 2015-2019 by Contributors
* \file common.cc
* \brief Enable all kinds of global variables in common.
/**
* Copyright 2015-2023 by Contributors
*/
#include <dmlc/thread_local.h>
#include <xgboost/logging.h>
#include "common.h"
#include "./random.h"
namespace xgboost {
namespace common {
#include <dmlc/thread_local.h> // for ThreadLocalStore
#include <cstdint> // for uint8_t
#include <cstdio> // for snprintf, size_t
#include <string> // for string
#include "./random.h" // for GlobalRandomEngine, GlobalRandom
namespace xgboost::common {
/*! \brief thread local entry for random. */
struct RandomThreadLocalEntry {
/*! \brief the random engine instance. */
@@ -19,15 +20,43 @@ struct RandomThreadLocalEntry {
using RandomThreadLocalStore = dmlc::ThreadLocalStore<RandomThreadLocalEntry>;
GlobalRandomEngine& GlobalRandom() {
return RandomThreadLocalStore::Get()->engine;
GlobalRandomEngine &GlobalRandom() { return RandomThreadLocalStore::Get()->engine; }
void EscapeU8(std::string const &string, std::string *p_buffer) {
auto &buffer = *p_buffer;
for (size_t i = 0; i < string.length(); i++) {
const auto ch = string[i];
if (ch == '\\') {
if (i < string.size() && string[i + 1] == 'u') {
buffer += "\\";
} else {
buffer += "\\\\";
}
} else if (ch == '"') {
buffer += "\\\"";
} else if (ch == '\b') {
buffer += "\\b";
} else if (ch == '\f') {
buffer += "\\f";
} else if (ch == '\n') {
buffer += "\\n";
} else if (ch == '\r') {
buffer += "\\r";
} else if (ch == '\t') {
buffer += "\\t";
} else if (static_cast<uint8_t>(ch) <= 0x1f) {
// Unit separator
char buf[8];
snprintf(buf, sizeof buf, "\\u%04x", ch);
buffer += buf;
} else {
buffer += ch;
}
}
}
#if !defined(XGBOOST_USE_CUDA)
int AllVisibleGPUs() {
return 0;
}
int AllVisibleGPUs() { return 0; }
#endif // !defined(XGBOOST_USE_CUDA)
} // namespace common
} // namespace xgboost
} // namespace xgboost::common

41
src/common/common.h
View File

@@ -6,20 +6,19 @@
#ifndef XGBOOST_COMMON_COMMON_H_
#define XGBOOST_COMMON_COMMON_H_
#include <xgboost/base.h>
#include <xgboost/logging.h>
#include <xgboost/span.h>
#include <algorithm> // for max
#include <array> // for array
#include <cmath> // for ceil
#include <cstddef> // for size_t
#include <cstdint> // for int32_t, int64_t
#include <sstream> // for basic_istream, operator<<, istringstream
#include <string> // for string, basic_string, getline, char_traits
#include <tuple> // for make_tuple
#include <utility> // for forward, index_sequence, make_index_sequence
#include <vector> // for vector
#include <algorithm>
#include <exception>
#include <functional>
#include <limits>
#include <numeric>
#include <sstream>
#include <string>
#include <type_traits>
#include <utility>
#include <vector>
#include "xgboost/base.h" // for XGBOOST_DEVICE
#include "xgboost/logging.h" // for LOG, LOG_FATAL, LogMessageFatal
#if defined(__CUDACC__)
#include <thrust/system/cuda/error.h>
@@ -52,8 +51,7 @@ inline cudaError_t ThrowOnCudaError(cudaError_t code, const char *file,
#endif // defined(__CUDACC__)
} // namespace dh
namespace xgboost {
namespace common {
namespace xgboost::common {
/*!
* \brief Split a string by delimiter
* \param s String to be split.
@@ -69,19 +67,13 @@ inline std::vector<std::string> Split(const std::string& s, char delim) {
return ret;
}
void EscapeU8(std::string const &string, std::string *p_buffer);
template <typename T>
XGBOOST_DEVICE T Max(T a, T b) {
return a < b ? b : a;
}
// simple routine to convert any data to string
template<typename T>
inline std::string ToString(const T& data) {
std::ostringstream os;
os << data;
return os.str();
}
template <typename T1, typename T2>
XGBOOST_DEVICE T1 DivRoundUp(const T1 a, const T2 b) {
return static_cast<T1>(std::ceil(static_cast<double>(a) / b));
@@ -195,6 +187,5 @@ template <typename Indexable>
XGBOOST_DEVICE size_t LastOf(size_t group, Indexable const &indptr) {
return indptr[group + 1] - 1;
}
} // namespace common
} // namespace xgboost
} // namespace xgboost::common
#endif // XGBOOST_COMMON_COMMON_H_

90
src/common/json.cc
View File

@@ -1,23 +1,29 @@
/*!
* Copyright (c) by Contributors 2019-2022
/**
* Copyright 2019-2023, XGBoost Contributors
*/
#include "xgboost/json.h"
#include <dmlc/endian.h>
#include <array> // for array
#include <cctype> // for isdigit
#include <cmath> // for isinf, isnan
#include <cstdio> // for EOF
#include <cstdlib> // for size_t, strtof
#include <cstring> // for memcpy
#include <initializer_list> // for initializer_list
#include <iterator> // for distance
#include <limits> // for numeric_limits
#include <memory> // for allocator
#include <sstream> // for operator<<, basic_ostream, operator&, ios, stringstream
#include <system_error> // for errc
#include <cctype>
#include <cmath>
#include <cstddef>
#include <iterator>
#include <limits>
#include <sstream>
#include "./math.h"
#include "charconv.h"
#include "xgboost/base.h"
#include "xgboost/json_io.h"
#include "xgboost/logging.h"
#include "xgboost/string_view.h"
#include "./math.h" // for CheckNAN
#include "charconv.h" // for to_chars, NumericLimits, from_chars, to_chars_result
#include "common.h" // for EscapeU8
#include "xgboost/base.h" // for XGBOOST_EXPECT
#include "xgboost/intrusive_ptr.h" // for IntrusivePtr
#include "xgboost/json_io.h" // for JsonReader, UBJReader, UBJWriter, JsonWriter, ToBigEn...
#include "xgboost/logging.h" // for LOG, LOG_FATAL, LogMessageFatal, LogCheck_NE, CHECK
#include "xgboost/string_view.h" // for StringView, operator<<
namespace xgboost {
@@ -57,12 +63,12 @@ void JsonWriter::Visit(JsonObject const* obj) {
}
void JsonWriter::Visit(JsonNumber const* num) {
char number[NumericLimits<float>::kToCharsSize];
auto res = to_chars(number, number + sizeof(number), num->GetNumber());
std::array<char, NumericLimits<float>::kToCharsSize> number;
auto res = to_chars(number.data(), number.data() + number.size(), num->GetNumber());
auto end = res.ptr;
auto ori_size = stream_->size();
stream_->resize(stream_->size() + end - number);
std::memcpy(stream_->data() + ori_size, number, end - number);
stream_->resize(stream_->size() + end - number.data());
std::memcpy(stream_->data() + ori_size, number.data(), end - number.data());
}
void JsonWriter::Visit(JsonInteger const* num) {
@@ -88,43 +94,15 @@ void JsonWriter::Visit(JsonNull const* ) {
}
void JsonWriter::Visit(JsonString const* str) {
std::string buffer;
buffer += '"';
auto const& string = str->GetString();
for (size_t i = 0; i < string.length(); i++) {
const char ch = string[i];
if (ch == '\\') {
if (i < string.size() && string[i+1] == 'u') {
buffer += "\\";
} else {
buffer += "\\\\";
}
} else if (ch == '"') {
buffer += "\\\"";
} else if (ch == '\b') {
buffer += "\\b";
} else if (ch == '\f') {
buffer += "\\f";
} else if (ch == '\n') {
buffer += "\\n";
} else if (ch == '\r') {
buffer += "\\r";
} else if (ch == '\t') {
buffer += "\\t";
} else if (static_cast<uint8_t>(ch) <= 0x1f) {
// Unit separator
char buf[8];
snprintf(buf, sizeof buf, "\\u%04x", ch);
buffer += buf;
} else {
buffer += ch;
}
}
buffer += '"';
std::string buffer;
buffer += '"';
auto const& string = str->GetString();
common::EscapeU8(string, &buffer);
buffer += '"';
auto s = stream_->size();
stream_->resize(s + buffer.size());
std::memcpy(stream_->data() + s, buffer.data(), buffer.size());
auto s = stream_->size();
stream_->resize(s + buffer.size());
std::memcpy(stream_->data() + s, buffer.data(), buffer.size());
}
void JsonWriter::Visit(JsonBoolean const* boolean) {

1
src/common/numeric.h
View File

@@ -10,6 +10,7 @@
#include <cstddef> // for size_t
#include <cstdint> // for int32_t
#include <iterator> // for iterator_traits
#include <numeric> // for accumulate
#include <vector>
#include "common.h" // AssertGPUSupport