/*! * Copyright 2019 by Contributors * \file bitfield.h */ #ifndef XGBOOST_COMMON_BITFIELD_H_ #define XGBOOST_COMMON_BITFIELD_H_ #include #include #include #include #include #include #include #include "span.h" namespace xgboost { #if defined(__CUDACC__) using BitFieldAtomicType = unsigned long long; // NOLINT __forceinline__ __device__ BitFieldAtomicType AtomicOr(BitFieldAtomicType* address, BitFieldAtomicType val) { BitFieldAtomicType old = *address, assumed; // NOLINT do { assumed = old; old = atomicCAS(address, assumed, val | assumed); } while (assumed != old); return old; } __forceinline__ __device__ BitFieldAtomicType AtomicAnd(BitFieldAtomicType* address, BitFieldAtomicType val) { BitFieldAtomicType old = *address, assumed; // NOLINT do { assumed = old; old = atomicCAS(address, assumed, val & assumed); } while (assumed != old); return old; } #endif // defined(__CUDACC__) /*! * \brief A non-owning type with auxiliary methods defined for manipulating bits. * * \tparam Direction Whether the bits start from left or from right. */ template struct BitFieldContainer { using value_type = VT; using pointer = value_type*; static value_type constexpr kValueSize = sizeof(value_type) * 8; static value_type constexpr kOne = 1; // force correct type. struct Pos { value_type int_pos {0}; value_type bit_pos {0}; }; common::Span bits_; static_assert(!std::is_signed::value, "Must use unsiged type as underlying storage."); XGBOOST_DEVICE static Pos ToBitPos(value_type pos) { Pos pos_v; if (pos == 0) { return pos_v; } pos_v.int_pos = pos / kValueSize; pos_v.bit_pos = pos % kValueSize; return pos_v; } public: BitFieldContainer() = default; XGBOOST_DEVICE BitFieldContainer(common::Span bits) : bits_{bits} {} XGBOOST_DEVICE BitFieldContainer(BitFieldContainer const& other) : bits_{other.bits_} {} static size_t ComputeStorageSize(size_t size) { auto pos = ToBitPos(size); if (size < kValueSize) { return 1; } if (pos.bit_pos != 0) { return pos.int_pos + 2; } else { return pos.int_pos + 1; } } #if defined(__CUDA_ARCH__) __device__ BitFieldContainer& operator|=(BitFieldContainer const& rhs) { auto tid = blockIdx.x * blockDim.x + threadIdx.x; size_t min_size = min(bits_.size(), rhs.bits_.size()); if (tid < min_size) { bits_[tid] |= rhs.bits_[tid]; } return *this; } #else BitFieldContainer& operator|=(BitFieldContainer const& rhs) { size_t min_size = std::min(bits_.size(), rhs.bits_.size()); for (size_t i = 0; i < min_size; ++i) { bits_[i] |= rhs.bits_[i]; } return *this; } #endif // #if defined(__CUDA_ARCH__) #if defined(__CUDA_ARCH__) __device__ BitFieldContainer& operator&=(BitFieldContainer const& rhs) { size_t min_size = min(bits_.size(), rhs.bits_.size()); auto tid = blockIdx.x * blockDim.x + threadIdx.x; if (tid < min_size) { bits_[tid] &= rhs.bits_[tid]; } return *this; } #else BitFieldContainer& operator&=(BitFieldContainer const& rhs) { size_t min_size = std::min(bits_.size(), rhs.bits_.size()); for (size_t i = 0; i < min_size; ++i) { bits_[i] &= rhs.bits_[i]; } return *this; } #endif // defined(__CUDA_ARCH__) #if defined(__CUDA_ARCH__) __device__ void Set(value_type pos) { Pos pos_v = Direction::Shift(ToBitPos(pos)); value_type& value = bits_[pos_v.int_pos]; value_type set_bit = kOne << pos_v.bit_pos; static_assert(sizeof(BitFieldAtomicType) == sizeof(value_type), ""); AtomicOr(reinterpret_cast(&value), set_bit); } __device__ void Clear(value_type pos) { Pos pos_v = Direction::Shift(ToBitPos(pos)); value_type& value = bits_[pos_v.int_pos]; value_type clear_bit = ~(kOne << pos_v.bit_pos); static_assert(sizeof(BitFieldAtomicType) == sizeof(value_type), ""); AtomicAnd(reinterpret_cast(&value), clear_bit); } #else void Set(value_type pos) { Pos pos_v = Direction::Shift(ToBitPos(pos)); value_type& value = bits_[pos_v.int_pos]; value_type set_bit = kOne << pos_v.bit_pos; value |= set_bit; } void Clear(value_type pos) { Pos pos_v = Direction::Shift(ToBitPos(pos)); value_type& value = bits_[pos_v.int_pos]; value_type clear_bit = ~(kOne << pos_v.bit_pos); value &= clear_bit; } #endif // defined(__CUDA_ARCH__) XGBOOST_DEVICE bool Check(Pos pos_v) const { pos_v = Direction::Shift(pos_v); value_type const value = bits_[pos_v.int_pos]; value_type const test_bit = kOne << pos_v.bit_pos; value_type result = test_bit & value; return static_cast(result); } XGBOOST_DEVICE bool Check(value_type pos) const { Pos pos_v = ToBitPos(pos); return Check(pos_v); } XGBOOST_DEVICE size_t Size() const { return kValueSize * bits_.size(); } XGBOOST_DEVICE pointer Data() const { return bits_.data(); } friend std::ostream& operator<<(std::ostream& os, BitFieldContainer field) { os << "Bits " << "storage size: " << field.bits_.size() << "\n"; for (typename common::Span::index_type i = 0; i < field.bits_.size(); ++i) { std::bitset::kValueSize> bset(field.bits_[i]); os << bset << "\n"; } return os; } }; // Bits start from left most bits (most significant bit). template struct LBitsPolicy : public BitFieldContainer> { using Container = BitFieldContainer>; using Pos = typename Container::Pos; using value_type = typename Container::value_type; XGBOOST_DEVICE static Pos Shift(Pos pos) { pos.bit_pos = Container::kValueSize - pos.bit_pos - Container::kOne; return pos; } }; // Bits start from right most bit (least significant bit) of each entry, but integer index // is from left to right. template struct RBitsPolicy : public BitFieldContainer> { using Container = BitFieldContainer>; using Pos = typename Container::Pos; using value_type = typename Container::value_type; XGBOOST_DEVICE static Pos Shift(Pos pos) { return pos; } }; // Format: BitField, underlying type must be unsigned. using LBitField64 = BitFieldContainer>; using RBitField8 = BitFieldContainer>; #if defined(__CUDACC__) template inline void PrintDeviceBits(std::string name, BitFieldContainer field) { std::cout << "Bits: " << name << std::endl; std::vector::value_type> h_field_bits(field.bits_.size()); thrust::copy(thrust::device_ptr::value_type>(field.bits_.data()), thrust::device_ptr::value_type>( field.bits_.data() + field.bits_.size()), h_field_bits.data()); BitFieldContainer h_field; h_field.bits_ = {h_field_bits.data(), h_field_bits.data() + h_field_bits.size()}; std::cout << h_field; } inline void PrintDeviceStorage(std::string name, common::Span list) { std::cout << name << std::endl; std::vector h_list(list.size()); thrust::copy(thrust::device_ptr(list.data()), thrust::device_ptr(list.data() + list.size()), h_list.data()); for (auto v : h_list) { std::cout << v << ", "; } std::cout << std::endl; } #endif // defined(__CUDACC__) } // namespace xgboost #endif // XGBOOST_COMMON_BITFIELD_H_