Split inter-region ref bitmap for every space
Currently, one bitmap is created which covers both region-space and
non-moving-space. Also, this bitmap is created and destroyed in each GC
cycle. This becomes a problem on 32-bit processes (especially during
gcstress tests) due to limited availability of address space.
In this change, two separate bitmaps are created for both the spaces.
This is done once during startup, and then the bitmaps are reused for
the lifetime of process.
Test: art/test/testrunner/testrunner.py --host --gcstress --32
Bug: 112720851
Change-Id: I6fc1dbd5dab10b39b2fd2d436a678e319feb78e7
diff --git a/runtime/gc/collector/concurrent_copying.cc b/runtime/gc/collector/concurrent_copying.cc
index f7f3a8d..e95b135 100644
--- a/runtime/gc/collector/concurrent_copying.cc
+++ b/runtime/gc/collector/concurrent_copying.cc
@@ -95,7 +95,8 @@
weak_ref_access_enabled_(true),
copied_live_bytes_ratio_sum_(0.f),
gc_count_(0),
- inter_region_bitmap_(nullptr),
+ region_space_inter_region_bitmap_(nullptr),
+ non_moving_space_inter_region_bitmap_(nullptr),
reclaimed_bytes_ratio_sum_(0.f),
young_gen_(young_gen),
skipped_blocks_lock_("concurrent copying bytes blocks lock", kMarkSweepMarkStackLock),
@@ -286,12 +287,32 @@
gc_barrier_->Increment(self, barrier_count);
}
+void ConcurrentCopying::CreateInterRegionRefBitmaps() {
+ DCHECK(kEnableGenerationalConcurrentCopyingCollection);
+ DCHECK(region_space_inter_region_bitmap_ == nullptr);
+ DCHECK(non_moving_space_inter_region_bitmap_ == nullptr);
+ DCHECK(region_space_ != nullptr);
+ DCHECK(heap_->non_moving_space_ != nullptr);
+ // Region-space
+ region_space_inter_region_bitmap_ = accounting::ContinuousSpaceBitmap::Create(
+ "region-space inter region ref bitmap",
+ reinterpret_cast<uint8_t*>(region_space_->Begin()),
+ region_space_->Limit() - region_space_->Begin());
+ CHECK(region_space_inter_region_bitmap_ != nullptr)
+ << "Couldn't allocate region-space inter region ref bitmap";
+
+ // non-moving-space
+ non_moving_space_inter_region_bitmap_ = accounting::ContinuousSpaceBitmap::Create(
+ "non-moving-space inter region ref bitmap",
+ reinterpret_cast<uint8_t*>(heap_->non_moving_space_->Begin()),
+ heap_->non_moving_space_->Limit() - heap_->non_moving_space_->Begin());
+ CHECK(non_moving_space_inter_region_bitmap_ != nullptr)
+ << "Couldn't allocate non-moving-space inter region ref bitmap";
+}
+
void ConcurrentCopying::BindBitmaps() {
Thread* self = Thread::Current();
WriterMutexLock mu(self, *Locks::heap_bitmap_lock_);
- uintptr_t continuous_spaces_begin = UINTPTR_MAX;
- uintptr_t continuous_spaces_limit = 0;
- DCHECK(inter_region_bitmap_ == nullptr);
// Mark all of the spaces we never collect as immune.
for (const auto& space : heap_->GetContinuousSpaces()) {
if (space->GetGcRetentionPolicy() == space::kGcRetentionPolicyNeverCollect ||
@@ -301,6 +322,7 @@
} else {
CHECK(!space->IsZygoteSpace());
CHECK(!space->IsImageSpace());
+ CHECK(space == region_space_ || space == heap_->non_moving_space_);
if (kEnableGenerationalConcurrentCopyingCollection) {
if (space == region_space_) {
region_space_bitmap_ = region_space_->GetMarkBitmap();
@@ -323,11 +345,6 @@
// be captured after the thread-flip of this GC cycle, as that is when
// the young-gen for the next GC cycle starts getting populated.
heap_->GetCardTable()->ClearCardRange(space->Begin(), space->Limit());
-
- continuous_spaces_begin =
- std::min(continuous_spaces_begin, reinterpret_cast<uintptr_t>(space->Begin()));
- continuous_spaces_limit =
- std::max(continuous_spaces_limit, reinterpret_cast<uintptr_t>(space->Limit()));
}
} else {
if (space == region_space_) {
@@ -339,18 +356,10 @@
}
}
}
- if (kEnableGenerationalConcurrentCopyingCollection) {
- if (young_gen_) {
- for (const auto& space : GetHeap()->GetDiscontinuousSpaces()) {
- CHECK(space->IsLargeObjectSpace());
- space->AsLargeObjectSpace()->CopyLiveToMarked();
- }
- } else {
- inter_region_bitmap_.reset(accounting::ContinuousSpaceBitmap::Create(
- "inter region ref bitmap",
- reinterpret_cast<uint8_t*>(continuous_spaces_begin),
- continuous_spaces_limit - continuous_spaces_begin));
- CHECK(inter_region_bitmap_ != nullptr) << "Couldn't allocate inter region ref bitmap";
+ if (kEnableGenerationalConcurrentCopyingCollection && young_gen_) {
+ for (const auto& space : GetHeap()->GetDiscontinuousSpaces()) {
+ CHECK(space->IsLargeObjectSpace());
+ space->AsLargeObjectSpace()->CopyLiveToMarked();
}
}
}
@@ -1112,12 +1121,22 @@
}
ComputeLiveBytesAndMarkRefFieldsVisitor</*kHandleInterRegionRefs*/ true>
visitor(this, obj_region_idx);
- ref->VisitReferences</*kVisitNativeRoots=*/true, kDefaultVerifyFlags, kWithoutReadBarrier>(
+ ref->VisitReferences</*kVisitNativeRoots=*/ true, kDefaultVerifyFlags, kWithoutReadBarrier>(
visitor, visitor);
// Mark the corresponding card dirty if the object contains any
// inter-region reference.
if (visitor.ContainsInterRegionRefs()) {
- inter_region_bitmap_->Set(ref);
+ if (obj_region_idx == static_cast<size_t>(-1)) {
+ // If an inter-region ref has been found in a non-region-space, then it
+ // must be non-moving-space. This is because this function cannot be
+ // called on a immune-space object, and a large-object-space object has
+ // only class object reference, which is either in some immune-space, or
+ // in non-moving-space.
+ DCHECK(heap_->non_moving_space_->HasAddress(ref));
+ non_moving_space_inter_region_bitmap_->Set(ref);
+ } else {
+ region_space_inter_region_bitmap_->Set(ref);
+ }
}
}
@@ -1427,11 +1446,15 @@
}
}
ScanDirtyObject</*kNoUnEvac*/ true>(obj);
- } else if (space != region_space_ || region_space_->IsInUnevacFromSpace(obj)) {
+ } else if (space != region_space_) {
+ DCHECK(space == heap_->non_moving_space_);
// We need to process un-evac references as they may be unprocessed,
// if they skipped the marking phase due to heap mutation.
ScanDirtyObject</*kNoUnEvac*/ false>(obj);
- inter_region_bitmap_->Clear(obj);
+ non_moving_space_inter_region_bitmap_->Clear(obj);
+ } else if (region_space_->IsInUnevacFromSpace(obj)) {
+ ScanDirtyObject</*kNoUnEvac*/ false>(obj);
+ region_space_inter_region_bitmap_->Clear(obj);
}
},
accounting::CardTable::kCardAged);
@@ -1443,16 +1466,20 @@
ScanDirtyObject</*kNoUnEvac*/ true>(obj);
};
if (space == region_space_) {
- region_space_->ScanUnevacFromSpace(inter_region_bitmap_.get(), visitor);
+ region_space_->ScanUnevacFromSpace(region_space_inter_region_bitmap_, visitor);
} else {
- inter_region_bitmap_->VisitMarkedRange(reinterpret_cast<uintptr_t>(space->Begin()),
- reinterpret_cast<uintptr_t>(space->End()),
- visitor);
+ DCHECK(space == heap_->non_moving_space_);
+ non_moving_space_inter_region_bitmap_->VisitMarkedRange(
+ reinterpret_cast<uintptr_t>(space->Begin()),
+ reinterpret_cast<uintptr_t>(space->End()),
+ visitor);
}
}
}
// Done scanning unevac space.
done_scanning_.store(true, std::memory_order_release);
+ // NOTE: inter-region-ref bitmaps can be cleared here to release memory, if needed.
+ // Currently we do it in ReclaimPhase().
if (kVerboseMode) {
LOG(INFO) << "GC end of ScanCardsForSpace";
}
@@ -3527,7 +3554,8 @@
// We do not currently use the region space cards at all, madvise them away to save ram.
heap_->GetCardTable()->ClearCardRange(region_space_->Begin(), region_space_->Limit());
} else if (kEnableGenerationalConcurrentCopyingCollection && !young_gen_) {
- inter_region_bitmap_.reset();
+ region_space_inter_region_bitmap_->Clear();
+ non_moving_space_inter_region_bitmap_->Clear();
}
{
MutexLock mu(self, skipped_blocks_lock_);
diff --git a/runtime/gc/collector/concurrent_copying.h b/runtime/gc/collector/concurrent_copying.h
index aabfc8e..70d2624 100644
--- a/runtime/gc/collector/concurrent_copying.h
+++ b/runtime/gc/collector/concurrent_copying.h
@@ -98,6 +98,9 @@
return kCollectorTypeCC;
}
void RevokeAllThreadLocalBuffers() override;
+ // Creates inter-region ref bitmaps for region-space and non-moving-space.
+ // Gets called in Heap construction after the two spaces are created.
+ void CreateInterRegionRefBitmaps();
void SetRegionSpace(space::RegionSpace* region_space) {
DCHECK(region_space != nullptr);
region_space_ = region_space;
@@ -391,7 +394,8 @@
size_t gc_count_;
// Bit is set if the corresponding object has inter-region references that
// were found during the marking phase of two-phase full-heap GC cycle.
- std::unique_ptr<accounting::ContinuousSpaceBitmap> inter_region_bitmap_;
+ accounting::ContinuousSpaceBitmap* region_space_inter_region_bitmap_;
+ accounting::ContinuousSpaceBitmap* non_moving_space_inter_region_bitmap_;
// reclaimed_bytes_ratio = reclaimed_bytes/num_allocated_bytes per GC cycle
float reclaimed_bytes_ratio_sum_;
diff --git a/runtime/gc/heap.cc b/runtime/gc/heap.cc
index d868aba..bf8aaae 100644
--- a/runtime/gc/heap.cc
+++ b/runtime/gc/heap.cc
@@ -666,6 +666,9 @@
concurrent_copying_collector_->SetRegionSpace(region_space_);
if (kEnableGenerationalConcurrentCopyingCollection) {
young_concurrent_copying_collector_->SetRegionSpace(region_space_);
+ // At this point, non-moving space should be created.
+ DCHECK(non_moving_space_ != nullptr);
+ concurrent_copying_collector_->CreateInterRegionRefBitmaps();
}
garbage_collectors_.push_back(concurrent_copying_collector_);
if (kEnableGenerationalConcurrentCopyingCollection) {
@@ -2736,7 +2739,7 @@
// active_concurrent_copying_collector_. So we should not concurrency here.
active_concurrent_copying_collector_ = (gc_type == collector::kGcTypeSticky) ?
young_concurrent_copying_collector_ : concurrent_copying_collector_;
- active_concurrent_copying_collector_->SetRegionSpace(region_space_);
+ DCHECK(active_concurrent_copying_collector_->RegionSpace() == region_space_);
}
collector = active_concurrent_copying_collector_;
break;