libartbase/base/bit_memory_region.h - SHIFTPHONES/android_art - Gitiles

 /*
  * Copyright (C) 2017 The Android Open Source Project
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
  *
  *      http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */

 #ifndef ART_LIBARTBASE_BASE_BIT_MEMORY_REGION_H_
 #define ART_LIBARTBASE_BASE_BIT_MEMORY_REGION_H_

 #include "memory_region.h"

 #include "bit_utils.h"
 #include "memory_tool.h"

 #include <array>

 namespace art {

 // Bit memory region is a bit offset subregion of a normal memoryregion. This is useful for
 // abstracting away the bit start offset to avoid needing passing as an argument everywhere.
 class BitMemoryRegion final : public ValueObject {
  public:
   BitMemoryRegion() = default;
   ALWAYS_INLINE BitMemoryRegion(uint8_t* data, ssize_t bit_start, size_t bit_size) {
     // Normalize the data pointer. Note that bit_start may be negative.
     data_ = AlignDown(data + (bit_start >> kBitsPerByteLog2), kPageSize);
     bit_start_ = bit_start + kBitsPerByte * (data - data_);
     bit_size_ = bit_size;
   }
   ALWAYS_INLINE explicit BitMemoryRegion(MemoryRegion region)
     : BitMemoryRegion(region.begin(), /* bit_start */ 0, region.size_in_bits()) {
   }
   ALWAYS_INLINE BitMemoryRegion(MemoryRegion region, size_t bit_offset, size_t bit_length)
     : BitMemoryRegion(region) {
     *this = Subregion(bit_offset, bit_length);
   }

   ALWAYS_INLINE bool IsValid() const { return data_ != nullptr; }

   const uint8_t* data() const {
     DCHECK_ALIGNED(bit_start_, kBitsPerByte);
     return data_ + bit_start_ / kBitsPerByte;
   }

   size_t size_in_bits() const {
     return bit_size_;
   }

   void Resize(size_t bit_size) {
     bit_size_ = bit_size;
   }

   ALWAYS_INLINE BitMemoryRegion Subregion(size_t bit_offset, size_t bit_length) const {
     DCHECK_LE(bit_offset, bit_size_);
     DCHECK_LE(bit_length, bit_size_ - bit_offset);
     BitMemoryRegion result = *this;
     result.bit_start_ += bit_offset;
     result.bit_size_ = bit_length;
     return result;
   }

   ALWAYS_INLINE BitMemoryRegion Subregion(size_t bit_offset) const {
     DCHECK_LE(bit_offset, bit_size_);
     BitMemoryRegion result = *this;
     result.bit_start_ += bit_offset;
     result.bit_size_ -= bit_offset;
     return result;
   }

   // Load a single bit in the region. The bit at offset 0 is the least
   // significant bit in the first byte.
   ALWAYS_INLINE bool LoadBit(size_t bit_offset) const {
     DCHECK_LT(bit_offset, bit_size_);
     size_t index = (bit_start_ + bit_offset) / kBitsPerByte;
     size_t shift = (bit_start_ + bit_offset) % kBitsPerByte;
     return ((data_[index] >> shift) & 1) != 0;
   }

   ALWAYS_INLINE void StoreBit(size_t bit_offset, bool value) {
     DCHECK_LT(bit_offset, bit_size_);
     size_t index = (bit_start_ + bit_offset) / kBitsPerByte;
     size_t shift = (bit_start_ + bit_offset) % kBitsPerByte;
     data_[index] &= ~(1 << shift);  // Clear bit.
     data_[index] |= (value ? 1 : 0) << shift;  // Set bit.
     DCHECK_EQ(value, LoadBit(bit_offset));
   }

   // Load `bit_length` bits from `data` starting at given `bit_offset`.
   // The least significant bit is stored in the smallest memory offset.
   template<typename Result = size_t>
   ATTRIBUTE_NO_SANITIZE_ADDRESS  // We might touch extra bytes due to the alignment.
   ATTRIBUTE_NO_SANITIZE_HWADDRESS  // The hwasan uses different attribute.
   ALWAYS_INLINE Result LoadBits(size_t bit_offset, size_t bit_length) const {
     static_assert(std::is_integral_v<Result>, "Result must be integral");
     static_assert(std::is_unsigned_v<Result>, "Result must be unsigned");
     DCHECK(IsAligned<sizeof(Result)>(data_));
     DCHECK_LE(bit_offset, bit_size_);
     DCHECK_LE(bit_length, bit_size_ - bit_offset);
     DCHECK_LE(bit_length, BitSizeOf<Result>());
     if (bit_length == 0) {
       return 0;
     }
     // Load naturally-aligned value which contains the least significant bit.
     Result* data = reinterpret_cast<Result*>(data_);
     size_t width = BitSizeOf<Result>();
     size_t index = (bit_start_ + bit_offset) / width;
     size_t shift = (bit_start_ + bit_offset) % width;
     Result value = data[index] >> shift;
     // Load extra value containing the most significant bit (it might be the same one).
     // We can not just load the following value as that could potentially cause SIGSEGV.
     Result extra = data[index + (shift + (bit_length - 1)) / width];
     // Mask to clear unwanted bits (the 1s are needed to avoid avoid undefined shift).
     Result clear = (std::numeric_limits<Result>::max() << 1) << (bit_length - 1);
     // Prepend the extra value.  We add explicit '& (width - 1)' so that the shift is defined.
     // It is a no-op for `shift != 0` and if `shift == 0` then `value == extra` because of
     // bit_length <= width causing the `value` and `extra` to be read from the same location.
     // The '& (width - 1)' is implied by the shift instruction on ARM and removed by compiler.
     return (value | (extra << ((width - shift) & (width - 1)))) & ~clear;
   }

   // Store `bit_length` bits in `data` starting at given `bit_offset`.
   // The least significant bit is stored in the smallest memory offset.
   ALWAYS_INLINE void StoreBits(size_t bit_offset, size_t value, size_t bit_length) {
     DCHECK_LE(bit_offset, bit_size_);
     DCHECK_LE(bit_length, bit_size_ - bit_offset);
     DCHECK_LE(bit_length, BitSizeOf<size_t>());
     DCHECK_LE(value, MaxInt<size_t>(bit_length));
     if (bit_length == 0) {
       return;
     }
     // Write data byte by byte to avoid races with other threads
     // on bytes that do not overlap with this region.
     size_t mask = std::numeric_limits<size_t>::max() >> (BitSizeOf<size_t>() - bit_length);
     size_t index = (bit_start_ + bit_offset) / kBitsPerByte;
     size_t shift = (bit_start_ + bit_offset) % kBitsPerByte;
     data_[index] &= ~(mask << shift);  // Clear bits.
     data_[index] |= (value << shift);  // Set bits.
     size_t finished_bits = kBitsPerByte - shift;
     for (int i = 1; finished_bits < bit_length; i++, finished_bits += kBitsPerByte) {
       data_[index + i] &= ~(mask >> finished_bits);  // Clear bits.
       data_[index + i] |= (value >> finished_bits);  // Set bits.
     }
     DCHECK_EQ(value, LoadBits(bit_offset, bit_length));
   }

   // Copy bits from other bit region.
   ALWAYS_INLINE void CopyBits(const BitMemoryRegion& src) {
     DCHECK_EQ(size_in_bits(), src.size_in_bits());
     // Hopefully, the loads of the unused `value` shall be optimized away.
     VisitChunks(
         [this, &src](size_t offset, size_t num_bits, size_t value ATTRIBUTE_UNUSED) ALWAYS_INLINE {
           StoreChunk(offset, src.LoadBits(offset, num_bits), num_bits);
           return true;
         });
   }

   // And bits from other bit region.
   ALWAYS_INLINE void AndBits(const BitMemoryRegion& src) {
     DCHECK_EQ(size_in_bits(), src.size_in_bits());
     VisitChunks([this, &src](size_t offset, size_t num_bits, size_t value) ALWAYS_INLINE {
       StoreChunk(offset, value & src.LoadBits(offset, num_bits), num_bits);
       return true;
     });
   }

   // Or bits from other bit region.
   ALWAYS_INLINE void OrBits(const BitMemoryRegion& src) {
     DCHECK_EQ(size_in_bits(), src.size_in_bits());
     VisitChunks([this, &src](size_t offset, size_t num_bits, size_t value) ALWAYS_INLINE {
       StoreChunk(offset, value | src.LoadBits(offset, num_bits), num_bits);
       return true;
     });
   }

   // Xor bits from other bit region.
   ALWAYS_INLINE void XorBits(const BitMemoryRegion& src) {
     DCHECK_EQ(size_in_bits(), src.size_in_bits());
     VisitChunks([this, &src](size_t offset, size_t num_bits, size_t value) ALWAYS_INLINE {
       StoreChunk(offset, value ^ src.LoadBits(offset, num_bits), num_bits);
       return true;
     });
   }

   // Count the number of set bits within this region.
   ALWAYS_INLINE size_t PopCount() const {
     size_t result = 0u;
     VisitChunks([&](size_t offset ATTRIBUTE_UNUSED,
                     size_t num_bits ATTRIBUTE_UNUSED,
                     size_t value) ALWAYS_INLINE {
                       result += POPCOUNT(value);
                       return true;
                     });
     return result;
   }

   // Count the number of set bits within the given bit range.
   ALWAYS_INLINE size_t PopCount(size_t bit_offset, size_t bit_length) const {
     return Subregion(bit_offset, bit_length).PopCount();
   }

   // Check if this region has all bits clear.
   ALWAYS_INLINE bool HasAllBitsClear() const {
     return VisitChunks([](size_t offset ATTRIBUTE_UNUSED,
                           size_t num_bits ATTRIBUTE_UNUSED,
                           size_t value) ALWAYS_INLINE {
                             return value == 0u;
                           });
   }

   // Check if this region has any bit set.
   ALWAYS_INLINE bool HasSomeBitSet() const {
     return !HasAllBitsClear();
   }

   // Check if there is any bit set within the given bit range.
   ALWAYS_INLINE bool HasSomeBitSet(size_t bit_offset, size_t bit_length) const {
     return Subregion(bit_offset, bit_length).HasSomeBitSet();
   }

   static int Compare(const BitMemoryRegion& lhs, const BitMemoryRegion& rhs) {
     if (lhs.size_in_bits() != rhs.size_in_bits()) {
       return (lhs.size_in_bits() < rhs.size_in_bits()) ? -1 : 1;
     }
     int result = 0;
     bool equals = lhs.VisitChunks(
         [&](size_t offset, size_t num_bits, size_t lhs_value) ALWAYS_INLINE {
           size_t rhs_value = rhs.LoadBits(offset, num_bits);
           if (lhs_value == rhs_value) {
             return true;
           }
           // We have found a difference. To avoid the comparison being dependent on how the region
           // is split into chunks, check the lowest bit that differs. (Android is little-endian.)
           int bit = CTZ(lhs_value ^ rhs_value);
           result = ((rhs_value >> bit) & 1u) != 0u ? 1 : -1;
           return false;  // Stop iterating.
         });
     DCHECK_EQ(equals, result == 0);
     return result;
   }

   static bool Equals(const BitMemoryRegion& lhs, const BitMemoryRegion& rhs) {
     if (lhs.size_in_bits() != rhs.size_in_bits()) {
       return false;
     }
     return lhs.VisitChunks([&rhs](size_t offset, size_t num_bits, size_t lhs_value) ALWAYS_INLINE {
       return lhs_value == rhs.LoadBits(offset, num_bits);
     });
   }

  private:
   // Visit the region in aligned `size_t` chunks. The first and last chunk may have fewer bits.
   //
   // Returns `true` if the iteration visited all chunks successfully, i.e. none of the
   // calls to `visitor(offset, num_bits, value)` returned `false`; otherwise `false`.
   template <typename VisitorType>
   ATTRIBUTE_NO_SANITIZE_ADDRESS  // We might touch extra bytes due to the alignment.
   ATTRIBUTE_NO_SANITIZE_HWADDRESS  // The hwasan uses different attribute.
   ALWAYS_INLINE bool VisitChunks(VisitorType&& visitor) const {
     constexpr size_t kChunkSize = BitSizeOf<size_t>();
     size_t remaining_bits = bit_size_;
     if (remaining_bits == 0) {
       return true;
     }
     DCHECK(IsAligned<sizeof(size_t)>(data_));
     const size_t* data = reinterpret_cast<const size_t*>(data_);
     size_t offset = 0u;
     size_t bit_start = bit_start_;
     data += bit_start / kChunkSize;
     if ((bit_start % kChunkSize) != 0u) {
       size_t leading_bits = kChunkSize - (bit_start % kChunkSize);
       size_t value = (*data) >> (bit_start % kChunkSize);
       if (leading_bits > remaining_bits) {
         leading_bits = remaining_bits;
         value = value & ~(std::numeric_limits<size_t>::max() << remaining_bits);
       }
       if (!visitor(offset, leading_bits, value)) {
         return false;
       }
       offset += leading_bits;
       remaining_bits -= leading_bits;
       ++data;
     }
     while (remaining_bits >= kChunkSize) {
       size_t value = *data;
       if (!visitor(offset, kChunkSize, value)) {
         return false;
       }
       offset += kChunkSize;
       remaining_bits -= kChunkSize;
       ++data;
     }
     if (remaining_bits != 0u) {
       size_t value = (*data) & ~(std::numeric_limits<size_t>::max() << remaining_bits);
       if (!visitor(offset, remaining_bits, value)) {
         return false;
       }
     }
     return true;
   }

   ALWAYS_INLINE void StoreChunk(size_t bit_offset, size_t value, size_t bit_length) {
     if (bit_length == BitSizeOf<size_t>()) {
       DCHECK_ALIGNED(bit_start_ + bit_offset, BitSizeOf<size_t>());
       uint8_t* data = data_ + (bit_start_ + bit_offset) / kBitsPerByte;
       DCHECK_ALIGNED(data, sizeof(size_t));
       reinterpret_cast<size_t*>(data)[0] = value;
     } else {
       StoreBits(bit_offset, value, bit_length);
     }
   }

   uint8_t* data_ = nullptr;  // The pointer is page aligned.
   size_t bit_start_ = 0;
   size_t bit_size_ = 0;
 };

 constexpr uint32_t kVarintBits = 4;  // Minimum number of bits used for varint.
 constexpr uint32_t kVarintMax = 11;  // Maximum value which is stored "inline".

 class BitMemoryReader {
  public:
   BitMemoryReader(BitMemoryReader&&) = default;
   explicit BitMemoryReader(BitMemoryRegion data)
       : finished_region_(data.Subregion(0, 0) /* set the length to zero */ ) {
   }
   explicit BitMemoryReader(const uint8_t* data, ssize_t bit_offset = 0)
       : finished_region_(const_cast<uint8_t*>(data), bit_offset, /* bit_length */ 0) {
   }

   const uint8_t* data() const { return finished_region_.data(); }

   BitMemoryRegion GetReadRegion() const { return finished_region_; }

   size_t NumberOfReadBits() const { return finished_region_.size_in_bits(); }

   ALWAYS_INLINE BitMemoryRegion ReadRegion(size_t bit_length) {
     size_t bit_offset = finished_region_.size_in_bits();
     finished_region_.Resize(bit_offset + bit_length);
     return finished_region_.Subregion(bit_offset, bit_length);
   }

   template<typename Result = size_t>
   ALWAYS_INLINE Result ReadBits(size_t bit_length) {
     return ReadRegion(bit_length).LoadBits<Result>(/* bit_offset */ 0, bit_length);
   }

   ALWAYS_INLINE bool ReadBit() {
     return ReadRegion(/* bit_length */ 1).LoadBit(/* bit_offset */ 0);
   }

   // Read variable-length bit-packed integer.
   // The first four bits determine the variable length of the encoded integer:
   //   Values 0..11 represent the result as-is, with no further following bits.
   //   Values 12..15 mean the result is in the next 8/16/24/32-bits respectively.
   ALWAYS_INLINE uint32_t ReadVarint() {
     uint32_t x = ReadBits(kVarintBits);
     return (x <= kVarintMax) ? x : ReadBits((x - kVarintMax) * kBitsPerByte);
   }

   // Read N 'interleaved' varints (different to just reading consecutive varints).
   // All small values are stored first and the large values are stored after them.
   // This requires fewer bit-reads compared to indidually storing the varints.
   template<size_t N>
   ALWAYS_INLINE std::array<uint32_t, N> ReadInterleavedVarints() {
     static_assert(N * kVarintBits <= sizeof(uint64_t) * kBitsPerByte, "N too big");
     std::array<uint32_t, N> values;
     // StackMap BitTable uses over 8 varints in the header, so we need uint64_t.
     uint64_t data = ReadBits<uint64_t>(N * kVarintBits);
     for (size_t i = 0; i < N; i++) {
       values[i] = BitFieldExtract(data, i * kVarintBits, kVarintBits);
     }
     // Do the second part in its own loop as that seems to produce better code in clang.
     for (size_t i = 0; i < N; i++) {
       if (UNLIKELY(values[i] > kVarintMax)) {
         values[i] = ReadBits((values[i] - kVarintMax) * kBitsPerByte);
       }
     }
     return values;
   }

  private:
   // Represents all of the bits which were read so far. There is no upper bound.
   // Therefore, by definition, the "cursor" is always at the end of the region.
   BitMemoryRegion finished_region_;

   DISALLOW_COPY_AND_ASSIGN(BitMemoryReader);
 };

 template<typename Vector>
 class BitMemoryWriter {
  public:
   explicit BitMemoryWriter(Vector* out, size_t bit_offset = 0)
       : out_(out), bit_start_(bit_offset), bit_offset_(bit_offset) {
     DCHECK_EQ(NumberOfWrittenBits(), 0u);
   }

   void Truncate(size_t bit_offset) {
     DCHECK_GE(bit_offset, bit_start_);
     DCHECK_LE(bit_offset, bit_offset_);
     bit_offset_ = bit_offset;
     DCHECK_LE(BitsToBytesRoundUp(bit_offset), out_->size());
     out_->resize(BitsToBytesRoundUp(bit_offset));  // Shrink.
   }

   BitMemoryRegion GetWrittenRegion() const {
     return BitMemoryRegion(out_->data(), bit_start_, bit_offset_ - bit_start_);
   }

   const uint8_t* data() const { return out_->data(); }

   size_t NumberOfWrittenBits() const { return bit_offset_ - bit_start_; }

   ALWAYS_INLINE BitMemoryRegion Allocate(size_t bit_length) {
     out_->resize(BitsToBytesRoundUp(bit_offset_ + bit_length));
     BitMemoryRegion region(out_->data(), bit_offset_, bit_length);
     DCHECK_LE(bit_length, std::numeric_limits<size_t>::max() - bit_offset_) << "Overflow";
     bit_offset_ += bit_length;
     return region;
   }

   ALWAYS_INLINE void WriteRegion(const BitMemoryRegion& region) {
     Allocate(region.size_in_bits()).CopyBits(region);
   }

   ALWAYS_INLINE void WriteBits(uint32_t value, size_t bit_length) {
     Allocate(bit_length).StoreBits(/* bit_offset */ 0, value, bit_length);
   }

   ALWAYS_INLINE void WriteBit(bool value) {
     Allocate(1).StoreBit(/* bit_offset */ 0, value);
   }

   template<size_t N>
   ALWAYS_INLINE void WriteInterleavedVarints(std::array<uint32_t, N> values) {
     // Write small values (or the number of bytes needed for the large values).
     for (uint32_t value : values) {
       if (value > kVarintMax) {
         WriteBits(kVarintMax + BitsToBytesRoundUp(MinimumBitsToStore(value)), kVarintBits);
       } else {
         WriteBits(value, kVarintBits);
       }
     }
     // Write large values.
     for (uint32_t value : values) {
       if (value > kVarintMax) {
         WriteBits(value, BitsToBytesRoundUp(MinimumBitsToStore(value)) * kBitsPerByte);
       }
     }
   }

   ALWAYS_INLINE void WriteVarint(uint32_t value) {
     WriteInterleavedVarints<1>({value});
   }

   void WriteBytesAligned(const uint8_t* bytes, size_t length) {
     DCHECK_ALIGNED(bit_start_, kBitsPerByte);
     DCHECK_ALIGNED(bit_offset_, kBitsPerByte);
     DCHECK_EQ(BitsToBytesRoundUp(bit_offset_), out_->size());
     out_->insert(out_->end(), bytes, bytes + length);
     bit_offset_ += length * kBitsPerByte;
   }

   ALWAYS_INLINE void ByteAlign() {
     DCHECK_ALIGNED(bit_start_, kBitsPerByte);
     bit_offset_ = RoundUp(bit_offset_, kBitsPerByte);
   }

  private:
   Vector* out_;
   size_t bit_start_;
   size_t bit_offset_;

   DISALLOW_COPY_AND_ASSIGN(BitMemoryWriter);
 };

 }  // namespace art

 #endif  // ART_LIBARTBASE_BASE_BIT_MEMORY_REGION_H_
	/*
	* Copyright (C) 2017 The Android Open Source Project
	*
	* Licensed under the Apache License, Version 2.0 (the "License");
	* you may not use this file except in compliance with the License.
	* You may obtain a copy of the License at
	*
	* http://www.apache.org/licenses/LICENSE-2.0
	*
	* Unless required by applicable law or agreed to in writing, software
	* distributed under the License is distributed on an "AS IS" BASIS,
	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	* See the License for the specific language governing permissions and
	* limitations under the License.
	*/

	#ifndef ART_LIBARTBASE_BASE_BIT_MEMORY_REGION_H_
	#define ART_LIBARTBASE_BASE_BIT_MEMORY_REGION_H_

	#include "memory_region.h"

	#include "bit_utils.h"
	#include "memory_tool.h"

	#include <array>

	namespace art {

	// Bit memory region is a bit offset subregion of a normal memoryregion. This is useful for
	// abstracting away the bit start offset to avoid needing passing as an argument everywhere.
	class BitMemoryRegion final : public ValueObject {
	public:
	BitMemoryRegion() = default;
	ALWAYS_INLINE BitMemoryRegion(uint8_t* data, ssize_t bit_start, size_t bit_size) {
	// Normalize the data pointer. Note that bit_start may be negative.
	data_ = AlignDown(data + (bit_start >> kBitsPerByteLog2), kPageSize);
	bit_start_ = bit_start + kBitsPerByte * (data - data_);
	bit_size_ = bit_size;
	}
	ALWAYS_INLINE explicit BitMemoryRegion(MemoryRegion region)
	: BitMemoryRegion(region.begin(), /* bit_start */ 0, region.size_in_bits()) {
	}
	ALWAYS_INLINE BitMemoryRegion(MemoryRegion region, size_t bit_offset, size_t bit_length)
	: BitMemoryRegion(region) {
	*this = Subregion(bit_offset, bit_length);
	}

	ALWAYS_INLINE bool IsValid() const { return data_ != nullptr; }

	const uint8_t* data() const {
	DCHECK_ALIGNED(bit_start_, kBitsPerByte);
	return data_ + bit_start_ / kBitsPerByte;
	}

	size_t size_in_bits() const {
	return bit_size_;
	}

	void Resize(size_t bit_size) {
	bit_size_ = bit_size;
	}

	ALWAYS_INLINE BitMemoryRegion Subregion(size_t bit_offset, size_t bit_length) const {
	DCHECK_LE(bit_offset, bit_size_);
	DCHECK_LE(bit_length, bit_size_ - bit_offset);
	BitMemoryRegion result = *this;
	result.bit_start_ += bit_offset;
	result.bit_size_ = bit_length;
	return result;
	}

	ALWAYS_INLINE BitMemoryRegion Subregion(size_t bit_offset) const {
	DCHECK_LE(bit_offset, bit_size_);
	BitMemoryRegion result = *this;
	result.bit_start_ += bit_offset;
	result.bit_size_ -= bit_offset;
	return result;
	}

	// Load a single bit in the region. The bit at offset 0 is the least
	// significant bit in the first byte.
	ALWAYS_INLINE bool LoadBit(size_t bit_offset) const {
	DCHECK_LT(bit_offset, bit_size_);
	size_t index = (bit_start_ + bit_offset) / kBitsPerByte;
	size_t shift = (bit_start_ + bit_offset) % kBitsPerByte;
	return ((data_[index] >> shift) & 1) != 0;
	}

	ALWAYS_INLINE void StoreBit(size_t bit_offset, bool value) {
	DCHECK_LT(bit_offset, bit_size_);
	size_t index = (bit_start_ + bit_offset) / kBitsPerByte;
	size_t shift = (bit_start_ + bit_offset) % kBitsPerByte;
	data_[index] &= ~(1 << shift); // Clear bit.
	data_[index] \|= (value ? 1 : 0) << shift; // Set bit.
	DCHECK_EQ(value, LoadBit(bit_offset));
	}

	// Load `bit_length` bits from `data` starting at given `bit_offset`.
	// The least significant bit is stored in the smallest memory offset.
	template<typename Result = size_t>
	ATTRIBUTE_NO_SANITIZE_ADDRESS // We might touch extra bytes due to the alignment.
	ATTRIBUTE_NO_SANITIZE_HWADDRESS // The hwasan uses different attribute.
	ALWAYS_INLINE Result LoadBits(size_t bit_offset, size_t bit_length) const {
	static_assert(std::is_integral_v<Result>, "Result must be integral");
	static_assert(std::is_unsigned_v<Result>, "Result must be unsigned");
	DCHECK(IsAligned<sizeof(Result)>(data_));
	DCHECK_LE(bit_offset, bit_size_);
	DCHECK_LE(bit_length, bit_size_ - bit_offset);
	DCHECK_LE(bit_length, BitSizeOf<Result>());
	if (bit_length == 0) {
	return 0;
	}
	// Load naturally-aligned value which contains the least significant bit.
	Result* data = reinterpret_cast<Result*>(data_);
	size_t width = BitSizeOf<Result>();
	size_t index = (bit_start_ + bit_offset) / width;
	size_t shift = (bit_start_ + bit_offset) % width;
	Result value = data[index] >> shift;
	// Load extra value containing the most significant bit (it might be the same one).
	// We can not just load the following value as that could potentially cause SIGSEGV.
	Result extra = data[index + (shift + (bit_length - 1)) / width];
	// Mask to clear unwanted bits (the 1s are needed to avoid avoid undefined shift).
	Result clear = (std::numeric_limits<Result>::max() << 1) << (bit_length - 1);
	// Prepend the extra value. We add explicit '& (width - 1)' so that the shift is defined.
	// It is a no-op for `shift != 0` and if `shift == 0` then `value == extra` because of
	// bit_length <= width causing the `value` and `extra` to be read from the same location.
	// The '& (width - 1)' is implied by the shift instruction on ARM and removed by compiler.
	return (value \| (extra << ((width - shift) & (width - 1)))) & ~clear;
	}

	// Store `bit_length` bits in `data` starting at given `bit_offset`.
	// The least significant bit is stored in the smallest memory offset.
	ALWAYS_INLINE void StoreBits(size_t bit_offset, size_t value, size_t bit_length) {
	DCHECK_LE(bit_offset, bit_size_);
	DCHECK_LE(bit_length, bit_size_ - bit_offset);
	DCHECK_LE(bit_length, BitSizeOf<size_t>());
	DCHECK_LE(value, MaxInt<size_t>(bit_length));
	if (bit_length == 0) {
	return;
	}
	// Write data byte by byte to avoid races with other threads
	// on bytes that do not overlap with this region.
	size_t mask = std::numeric_limits<size_t>::max() >> (BitSizeOf<size_t>() - bit_length);
	size_t index = (bit_start_ + bit_offset) / kBitsPerByte;
	size_t shift = (bit_start_ + bit_offset) % kBitsPerByte;
	data_[index] &= ~(mask << shift); // Clear bits.
	data_[index] \|= (value << shift); // Set bits.
	size_t finished_bits = kBitsPerByte - shift;
	for (int i = 1; finished_bits < bit_length; i++, finished_bits += kBitsPerByte) {
	data_[index + i] &= ~(mask >> finished_bits); // Clear bits.
	data_[index + i] \|= (value >> finished_bits); // Set bits.
	}
	DCHECK_EQ(value, LoadBits(bit_offset, bit_length));
	}

	// Copy bits from other bit region.
	ALWAYS_INLINE void CopyBits(const BitMemoryRegion& src) {
	DCHECK_EQ(size_in_bits(), src.size_in_bits());
	// Hopefully, the loads of the unused `value` shall be optimized away.
	VisitChunks(
	[this, &src](size_t offset, size_t num_bits, size_t value ATTRIBUTE_UNUSED) ALWAYS_INLINE {
	StoreChunk(offset, src.LoadBits(offset, num_bits), num_bits);
	return true;
	});
	}

	// And bits from other bit region.
	ALWAYS_INLINE void AndBits(const BitMemoryRegion& src) {
	DCHECK_EQ(size_in_bits(), src.size_in_bits());
	VisitChunks([this, &src](size_t offset, size_t num_bits, size_t value) ALWAYS_INLINE {
	StoreChunk(offset, value & src.LoadBits(offset, num_bits), num_bits);
	return true;
	});
	}

	// Or bits from other bit region.
	ALWAYS_INLINE void OrBits(const BitMemoryRegion& src) {
	DCHECK_EQ(size_in_bits(), src.size_in_bits());
	VisitChunks([this, &src](size_t offset, size_t num_bits, size_t value) ALWAYS_INLINE {
	StoreChunk(offset, value \| src.LoadBits(offset, num_bits), num_bits);
	return true;
	});
	}

	// Xor bits from other bit region.
	ALWAYS_INLINE void XorBits(const BitMemoryRegion& src) {
	DCHECK_EQ(size_in_bits(), src.size_in_bits());
	VisitChunks([this, &src](size_t offset, size_t num_bits, size_t value) ALWAYS_INLINE {
	StoreChunk(offset, value ^ src.LoadBits(offset, num_bits), num_bits);
	return true;
	});
	}

	// Count the number of set bits within this region.
	ALWAYS_INLINE size_t PopCount() const {
	size_t result = 0u;
	VisitChunks([&](size_t offset ATTRIBUTE_UNUSED,
	size_t num_bits ATTRIBUTE_UNUSED,
	size_t value) ALWAYS_INLINE {
	result += POPCOUNT(value);
	return true;
	});
	return result;
	}

	// Count the number of set bits within the given bit range.
	ALWAYS_INLINE size_t PopCount(size_t bit_offset, size_t bit_length) const {
	return Subregion(bit_offset, bit_length).PopCount();
	}

	// Check if this region has all bits clear.
	ALWAYS_INLINE bool HasAllBitsClear() const {
	return VisitChunks([](size_t offset ATTRIBUTE_UNUSED,
	size_t num_bits ATTRIBUTE_UNUSED,
	size_t value) ALWAYS_INLINE {
	return value == 0u;
	});
	}

	// Check if this region has any bit set.
	ALWAYS_INLINE bool HasSomeBitSet() const {
	return !HasAllBitsClear();
	}

	// Check if there is any bit set within the given bit range.
	ALWAYS_INLINE bool HasSomeBitSet(size_t bit_offset, size_t bit_length) const {
	return Subregion(bit_offset, bit_length).HasSomeBitSet();
	}

	static int Compare(const BitMemoryRegion& lhs, const BitMemoryRegion& rhs) {
	if (lhs.size_in_bits() != rhs.size_in_bits()) {
	return (lhs.size_in_bits() < rhs.size_in_bits()) ? -1 : 1;
	}
	int result = 0;
	bool equals = lhs.VisitChunks(
	[&](size_t offset, size_t num_bits, size_t lhs_value) ALWAYS_INLINE {
	size_t rhs_value = rhs.LoadBits(offset, num_bits);
	if (lhs_value == rhs_value) {
	return true;
	}
	// We have found a difference. To avoid the comparison being dependent on how the region
	// is split into chunks, check the lowest bit that differs. (Android is little-endian.)
	int bit = CTZ(lhs_value ^ rhs_value);
	result = ((rhs_value >> bit) & 1u) != 0u ? 1 : -1;
	return false; // Stop iterating.
	});
	DCHECK_EQ(equals, result == 0);
	return result;
	}

	static bool Equals(const BitMemoryRegion& lhs, const BitMemoryRegion& rhs) {
	if (lhs.size_in_bits() != rhs.size_in_bits()) {
	return false;
	}
	return lhs.VisitChunks([&rhs](size_t offset, size_t num_bits, size_t lhs_value) ALWAYS_INLINE {
	return lhs_value == rhs.LoadBits(offset, num_bits);
	});
	}

	private:
	// Visit the region in aligned `size_t` chunks. The first and last chunk may have fewer bits.
	//
	// Returns `true` if the iteration visited all chunks successfully, i.e. none of the
	// calls to `visitor(offset, num_bits, value)` returned `false`; otherwise `false`.
	template <typename VisitorType>
	ATTRIBUTE_NO_SANITIZE_ADDRESS // We might touch extra bytes due to the alignment.
	ATTRIBUTE_NO_SANITIZE_HWADDRESS // The hwasan uses different attribute.
	ALWAYS_INLINE bool VisitChunks(VisitorType&& visitor) const {
	constexpr size_t kChunkSize = BitSizeOf<size_t>();
	size_t remaining_bits = bit_size_;
	if (remaining_bits == 0) {
	return true;
	}
	DCHECK(IsAligned<sizeof(size_t)>(data_));
	const size_t* data = reinterpret_cast<const size_t*>(data_);
	size_t offset = 0u;
	size_t bit_start = bit_start_;
	data += bit_start / kChunkSize;
	if ((bit_start % kChunkSize) != 0u) {
	size_t leading_bits = kChunkSize - (bit_start % kChunkSize);
	size_t value = (*data) >> (bit_start % kChunkSize);
	if (leading_bits > remaining_bits) {
	leading_bits = remaining_bits;
	value = value & ~(std::numeric_limits<size_t>::max() << remaining_bits);
	}
	if (!visitor(offset, leading_bits, value)) {
	return false;
	}
	offset += leading_bits;
	remaining_bits -= leading_bits;
	++data;
	}
	while (remaining_bits >= kChunkSize) {
	size_t value = *data;
	if (!visitor(offset, kChunkSize, value)) {
	return false;
	}
	offset += kChunkSize;
	remaining_bits -= kChunkSize;
	++data;
	}
	if (remaining_bits != 0u) {
	size_t value = (*data) & ~(std::numeric_limits<size_t>::max() << remaining_bits);
	if (!visitor(offset, remaining_bits, value)) {
	return false;
	}
	}
	return true;
	}

	ALWAYS_INLINE void StoreChunk(size_t bit_offset, size_t value, size_t bit_length) {
	if (bit_length == BitSizeOf<size_t>()) {
	DCHECK_ALIGNED(bit_start_ + bit_offset, BitSizeOf<size_t>());
	uint8_t* data = data_ + (bit_start_ + bit_offset) / kBitsPerByte;
	DCHECK_ALIGNED(data, sizeof(size_t));
	reinterpret_cast<size_t*>(data)[0] = value;
	} else {
	StoreBits(bit_offset, value, bit_length);
	}
	}

	uint8_t* data_ = nullptr; // The pointer is page aligned.
	size_t bit_start_ = 0;
	size_t bit_size_ = 0;
	};

	constexpr uint32_t kVarintBits = 4; // Minimum number of bits used for varint.
	constexpr uint32_t kVarintMax = 11; // Maximum value which is stored "inline".

	class BitMemoryReader {
	public:
	BitMemoryReader(BitMemoryReader&&) = default;
	explicit BitMemoryReader(BitMemoryRegion data)
	: finished_region_(data.Subregion(0, 0) /* set the length to zero */ ) {
	}
	explicit BitMemoryReader(const uint8_t* data, ssize_t bit_offset = 0)
	: finished_region_(const_cast<uint8_t>(data), bit_offset, / bit_length */ 0) {
	}

	const uint8_t* data() const { return finished_region_.data(); }

	BitMemoryRegion GetReadRegion() const { return finished_region_; }

	size_t NumberOfReadBits() const { return finished_region_.size_in_bits(); }

	ALWAYS_INLINE BitMemoryRegion ReadRegion(size_t bit_length) {
	size_t bit_offset = finished_region_.size_in_bits();
	finished_region_.Resize(bit_offset + bit_length);
	return finished_region_.Subregion(bit_offset, bit_length);
	}

	template<typename Result = size_t>
	ALWAYS_INLINE Result ReadBits(size_t bit_length) {
	return ReadRegion(bit_length).LoadBits<Result>(/* bit_offset */ 0, bit_length);
	}

	ALWAYS_INLINE bool ReadBit() {
	return ReadRegion(/* bit_length / 1).LoadBit(/ bit_offset */ 0);
	}

	// Read variable-length bit-packed integer.
	// The first four bits determine the variable length of the encoded integer:
	// Values 0..11 represent the result as-is, with no further following bits.
	// Values 12..15 mean the result is in the next 8/16/24/32-bits respectively.
	ALWAYS_INLINE uint32_t ReadVarint() {
	uint32_t x = ReadBits(kVarintBits);
	return (x <= kVarintMax) ? x : ReadBits((x - kVarintMax) * kBitsPerByte);
	}

	// Read N 'interleaved' varints (different to just reading consecutive varints).
	// All small values are stored first and the large values are stored after them.
	// This requires fewer bit-reads compared to indidually storing the varints.
	template<size_t N>
	ALWAYS_INLINE std::array<uint32_t, N> ReadInterleavedVarints() {
	static_assert(N * kVarintBits <= sizeof(uint64_t) * kBitsPerByte, "N too big");
	std::array<uint32_t, N> values;
	// StackMap BitTable uses over 8 varints in the header, so we need uint64_t.
	uint64_t data = ReadBits<uint64_t>(N * kVarintBits);
	for (size_t i = 0; i < N; i++) {
	values[i] = BitFieldExtract(data, i * kVarintBits, kVarintBits);
	}
	// Do the second part in its own loop as that seems to produce better code in clang.
	for (size_t i = 0; i < N; i++) {
	if (UNLIKELY(values[i] > kVarintMax)) {
	values[i] = ReadBits((values[i] - kVarintMax) * kBitsPerByte);
	}
	}
	return values;
	}

	private:
	// Represents all of the bits which were read so far. There is no upper bound.
	// Therefore, by definition, the "cursor" is always at the end of the region.
	BitMemoryRegion finished_region_;

	DISALLOW_COPY_AND_ASSIGN(BitMemoryReader);
	};

	template<typename Vector>
	class BitMemoryWriter {
	public:
	explicit BitMemoryWriter(Vector* out, size_t bit_offset = 0)
	: out_(out), bit_start_(bit_offset), bit_offset_(bit_offset) {
	DCHECK_EQ(NumberOfWrittenBits(), 0u);
	}

	void Truncate(size_t bit_offset) {
	DCHECK_GE(bit_offset, bit_start_);
	DCHECK_LE(bit_offset, bit_offset_);
	bit_offset_ = bit_offset;
	DCHECK_LE(BitsToBytesRoundUp(bit_offset), out_->size());
	out_->resize(BitsToBytesRoundUp(bit_offset)); // Shrink.
	}

	BitMemoryRegion GetWrittenRegion() const {
	return BitMemoryRegion(out_->data(), bit_start_, bit_offset_ - bit_start_);
	}

	const uint8_t* data() const { return out_->data(); }

	size_t NumberOfWrittenBits() const { return bit_offset_ - bit_start_; }

	ALWAYS_INLINE BitMemoryRegion Allocate(size_t bit_length) {
	out_->resize(BitsToBytesRoundUp(bit_offset_ + bit_length));
	BitMemoryRegion region(out_->data(), bit_offset_, bit_length);
	DCHECK_LE(bit_length, std::numeric_limits<size_t>::max() - bit_offset_) << "Overflow";
	bit_offset_ += bit_length;
	return region;
	}

	ALWAYS_INLINE void WriteRegion(const BitMemoryRegion& region) {
	Allocate(region.size_in_bits()).CopyBits(region);
	}

	ALWAYS_INLINE void WriteBits(uint32_t value, size_t bit_length) {
	Allocate(bit_length).StoreBits(/* bit_offset */ 0, value, bit_length);
	}

	ALWAYS_INLINE void WriteBit(bool value) {
	Allocate(1).StoreBit(/* bit_offset */ 0, value);
	}

	template<size_t N>
	ALWAYS_INLINE void WriteInterleavedVarints(std::array<uint32_t, N> values) {
	// Write small values (or the number of bytes needed for the large values).
	for (uint32_t value : values) {
	if (value > kVarintMax) {
	WriteBits(kVarintMax + BitsToBytesRoundUp(MinimumBitsToStore(value)), kVarintBits);
	} else {
	WriteBits(value, kVarintBits);
	}
	}
	// Write large values.
	for (uint32_t value : values) {
	if (value > kVarintMax) {
	WriteBits(value, BitsToBytesRoundUp(MinimumBitsToStore(value)) * kBitsPerByte);
	}
	}
	}

	ALWAYS_INLINE void WriteVarint(uint32_t value) {
	WriteInterleavedVarints<1>({value});
	}

	void WriteBytesAligned(const uint8_t* bytes, size_t length) {
	DCHECK_ALIGNED(bit_start_, kBitsPerByte);
	DCHECK_ALIGNED(bit_offset_, kBitsPerByte);
	DCHECK_EQ(BitsToBytesRoundUp(bit_offset_), out_->size());
	out_->insert(out_->end(), bytes, bytes + length);
	bit_offset_ += length * kBitsPerByte;
	}

	ALWAYS_INLINE void ByteAlign() {
	DCHECK_ALIGNED(bit_start_, kBitsPerByte);
	bit_offset_ = RoundUp(bit_offset_, kBitsPerByte);
	}

	private:
	Vector* out_;
	size_t bit_start_;
	size_t bit_offset_;

	DISALLOW_COPY_AND_ASSIGN(BitMemoryWriter);
	};

	} // namespace art

	#endif // ART_LIBARTBASE_BASE_BIT_MEMORY_REGION_H_