blob: 6f4767e39189e874765e67fdc26d17746865a4fb [file] [log] [blame]
Elliott Hughes2faa5f12012-01-30 14:42:07 -08001/*
2 * Copyright (C) 2011 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
Carl Shapiro69759ea2011-07-21 18:13:35 -070016
Brian Carlstrom578bbdc2011-07-21 14:07:47 -070017#include "heap.h"
Carl Shapiro58551df2011-07-24 03:09:51 -070018
Brian Carlstrom58ae9412011-10-04 00:56:06 -070019#include <limits>
Ian Rogers700a4022014-05-19 16:49:03 -070020#include <memory>
Mathieu Chartier31000802015-06-14 14:14:37 -070021#include <unwind.h> // For GC verification.
Carl Shapiro58551df2011-07-24 03:09:51 -070022#include <vector>
23
Mathieu Chartierc7853442015-03-27 14:35:38 -070024#include "art_field-inl.h"
Mathieu Chartierbad02672014-08-25 13:08:22 -070025#include "base/allocator.h"
Mathieu Chartier8d447252015-10-26 10:21:14 -070026#include "base/arena_allocator.h"
Ian Rogersc7dd2952014-10-21 23:31:19 -070027#include "base/dumpable.h"
Mathieu Chartierb2f99362013-11-20 17:26:00 -080028#include "base/histogram-inl.h"
Elliott Hughes1aa246d2012-12-13 09:29:36 -080029#include "base/stl_util.h"
Mathieu Chartier32ce2ad2016-03-04 14:58:03 -080030#include "base/systrace.h"
Vladimir Marko80afd022015-05-19 18:08:00 +010031#include "base/time_utils.h"
Mathieu Chartier987ccff2013-07-08 11:05:21 -070032#include "common_throws.h"
Ian Rogers48931882013-01-22 14:35:16 -080033#include "cutils/sched_policy.h"
Elliott Hughes767a1472011-10-26 18:49:02 -070034#include "debugger.h"
Elliott Hughes956af0f2014-12-11 14:34:28 -080035#include "dex_file-inl.h"
Ian Rogers1d54e732013-05-02 21:10:01 -070036#include "gc/accounting/atomic_stack.h"
37#include "gc/accounting/card_table-inl.h"
38#include "gc/accounting/heap_bitmap-inl.h"
39#include "gc/accounting/mod_union_table-inl.h"
Hiroshi Yamauchi38e68e92014-03-07 13:59:08 -080040#include "gc/accounting/remembered_set.h"
Ian Rogers1d54e732013-05-02 21:10:01 -070041#include "gc/accounting/space_bitmap-inl.h"
Hiroshi Yamauchid5307ec2014-03-27 21:07:51 -070042#include "gc/collector/concurrent_copying.h"
Mathieu Chartier52e4b432014-06-10 11:22:31 -070043#include "gc/collector/mark_compact.h"
Mathieu Chartier3cf22532015-07-09 15:15:09 -070044#include "gc/collector/mark_sweep.h"
Ian Rogers1d54e732013-05-02 21:10:01 -070045#include "gc/collector/partial_mark_sweep.h"
Mathieu Chartier590fee92013-09-13 13:46:47 -070046#include "gc/collector/semi_space.h"
Ian Rogers1d54e732013-05-02 21:10:01 -070047#include "gc/collector/sticky_mark_sweep.h"
Mathieu Chartier78f7b4c2014-05-06 10:57:27 -070048#include "gc/reference_processor.h"
Mathieu Chartier590fee92013-09-13 13:46:47 -070049#include "gc/space/bump_pointer_space.h"
Hiroshi Yamauchi50b29282013-07-30 13:58:37 -070050#include "gc/space/dlmalloc_space-inl.h"
Ian Rogers1d54e732013-05-02 21:10:01 -070051#include "gc/space/image_space.h"
52#include "gc/space/large_object_space.h"
Hiroshi Yamauchi2cd334a2015-01-09 14:03:35 -080053#include "gc/space/region_space.h"
Hiroshi Yamauchicf58d4a2013-09-26 14:21:22 -070054#include "gc/space/rosalloc_space-inl.h"
Ian Rogers1d54e732013-05-02 21:10:01 -070055#include "gc/space/space-inl.h"
Mathieu Chartiera1602f22014-01-13 17:19:19 -080056#include "gc/space/zygote_space.h"
Mathieu Chartiera5eae692014-12-17 17:56:03 -080057#include "gc/task_processor.h"
Mathieu Chartierd8891782014-03-02 13:28:37 -080058#include "entrypoints/quick/quick_alloc_entrypoints.h"
Hiroshi Yamauchi3b4c1892013-09-12 21:33:12 -070059#include "heap-inl.h"
Brian Carlstrom9cff8e12011-08-18 16:47:29 -070060#include "image.h"
Mathieu Chartiereb175f72014-10-31 11:49:27 -070061#include "intern_table.h"
Nicolas Geoffrayb6e20ae2016-03-07 14:29:04 +000062#include "jit/jit.h"
63#include "jit/jit_code_cache.h"
Ian Rogers2dd0e2c2013-01-24 12:42:14 -080064#include "mirror/class-inl.h"
Ian Rogers2dd0e2c2013-01-24 12:42:14 -080065#include "mirror/object-inl.h"
66#include "mirror/object_array-inl.h"
Mathieu Chartier8fa2dad2014-03-13 12:22:56 -070067#include "mirror/reference-inl.h"
Brian Carlstrom5643b782012-02-05 12:32:53 -080068#include "os.h"
Ian Rogers53b8b092014-03-13 23:45:53 -070069#include "reflection.h"
Mathieu Chartier0de9f732013-11-22 17:58:48 -080070#include "runtime.h"
Mathieu Chartier7664f5c2012-06-08 18:15:32 -070071#include "ScopedLocalRef.h"
Ian Rogers00f7d0e2012-07-19 15:28:27 -070072#include "scoped_thread_state_change.h"
Mathieu Chartiereb8167a2014-05-07 15:43:14 -070073#include "handle_scope-inl.h"
Elliott Hughes8d768a92011-09-14 16:35:25 -070074#include "thread_list.h"
Elliott Hugheseac76672012-05-24 21:56:51 -070075#include "well_known_classes.h"
Carl Shapiro69759ea2011-07-21 18:13:35 -070076
77namespace art {
Mathieu Chartier50482232013-11-21 11:48:14 -080078
Ian Rogers1d54e732013-05-02 21:10:01 -070079namespace gc {
Carl Shapiro69759ea2011-07-21 18:13:35 -070080
Mathieu Chartier91e30632014-03-25 15:58:50 -070081static constexpr size_t kCollectorTransitionStressIterations = 0;
82static constexpr size_t kCollectorTransitionStressWait = 10 * 1000; // Microseconds
Ian Rogers1d54e732013-05-02 21:10:01 -070083// Minimum amount of remaining bytes before a concurrent GC is triggered.
Mathieu Chartier720ef762013-08-17 14:46:54 -070084static constexpr size_t kMinConcurrentRemainingBytes = 128 * KB;
Mathieu Chartier74762802014-01-24 10:21:35 -080085static constexpr size_t kMaxConcurrentRemainingBytes = 512 * KB;
Mathieu Chartierdf86d1f2014-04-08 13:44:04 -070086// Sticky GC throughput adjustment, divided by 4. Increasing this causes sticky GC to occur more
Mathieu Chartier73d1e172014-04-11 17:53:48 -070087// relative to partial/full GC. This may be desirable since sticky GCs interfere less with mutator
Mathieu Chartierdf86d1f2014-04-08 13:44:04 -070088// threads (lower pauses, use less memory bandwidth).
Mathieu Chartier73d1e172014-04-11 17:53:48 -070089static constexpr double kStickyGcThroughputAdjustment = 1.0;
Mathieu Chartierc1790162014-05-23 10:54:50 -070090// Whether or not we compact the zygote in PreZygoteFork.
Mathieu Chartier31f44142014-04-08 14:40:03 -070091static constexpr bool kCompactZygote = kMovingCollector;
Mathieu Chartierc1790162014-05-23 10:54:50 -070092// How many reserve entries are at the end of the allocation stack, these are only needed if the
93// allocation stack overflows.
94static constexpr size_t kAllocationStackReserveSize = 1024;
95// Default mark stack size in bytes.
96static const size_t kDefaultMarkStackSize = 64 * KB;
Zuo Wangf37a88b2014-07-10 04:26:41 -070097// Define space name.
98static const char* kDlMallocSpaceName[2] = {"main dlmalloc space", "main dlmalloc space 1"};
99static const char* kRosAllocSpaceName[2] = {"main rosalloc space", "main rosalloc space 1"};
100static const char* kMemMapSpaceName[2] = {"main space", "main space 1"};
Mathieu Chartier7247af52014-11-19 10:51:42 -0800101static const char* kNonMovingSpaceName = "non moving space";
102static const char* kZygoteSpaceName = "zygote space";
Mathieu Chartierb363f662014-07-16 13:28:58 -0700103static constexpr size_t kGSSBumpPointerSpaceCapacity = 32 * MB;
Mathieu Chartier95a505c2014-12-10 18:45:30 -0800104static constexpr bool kGCALotMode = false;
105// GC alot mode uses a small allocation stack to stress test a lot of GC.
106static constexpr size_t kGcAlotAllocationStackSize = 4 * KB /
107 sizeof(mirror::HeapReference<mirror::Object>);
108// Verify objet has a small allocation stack size since searching the allocation stack is slow.
109static constexpr size_t kVerifyObjectAllocationStackSize = 16 * KB /
110 sizeof(mirror::HeapReference<mirror::Object>);
111static constexpr size_t kDefaultAllocationStackSize = 8 * MB /
112 sizeof(mirror::HeapReference<mirror::Object>);
Mathieu Chartierb5de3bb2015-06-05 13:21:05 -0700113// System.runFinalization can deadlock with native allocations, to deal with this, we have a
114// timeout on how long we wait for finalizers to run. b/21544853
115static constexpr uint64_t kNativeAllocationFinalizeTimeout = MsToNs(250u);
Mathieu Chartier0051be62012-10-12 17:47:11 -0700116
Andreas Gampeace0dc12016-01-20 13:33:13 -0800117// For deterministic compilation, we need the heap to be at a well-known address.
118static constexpr uint32_t kAllocSpaceBeginForDeterministicAoT = 0x40000000;
Hiroshi Yamauchib62f2e62016-03-23 15:51:24 -0700119// Dump the rosalloc stats on SIGQUIT.
120static constexpr bool kDumpRosAllocStatsOnSigQuit = false;
Andreas Gampeace0dc12016-01-20 13:33:13 -0800121
Mathieu Chartier5d2a3f72016-05-11 11:35:39 -0700122static constexpr size_t kNativeAllocationHistogramBuckets = 16;
123
Hiroshi Yamauchib6bab0f2016-07-18 17:07:26 -0700124// Extra added to the heap growth multiplier. Used to adjust the GC ergonomics for the read barrier
125// config.
126static constexpr double kExtraHeapGrowthMultiplier = kUseReadBarrier ? 1.0 : 0.0;
127
Mathieu Chartierf8cb1782016-03-18 18:45:41 -0700128static inline bool CareAboutPauseTimes() {
129 return Runtime::Current()->InJankPerceptibleProcessState();
130}
131
Mathieu Chartiera4f6af92015-08-11 17:35:25 -0700132Heap::Heap(size_t initial_size,
133 size_t growth_limit,
134 size_t min_free,
135 size_t max_free,
136 double target_utilization,
137 double foreground_heap_growth_multiplier,
138 size_t capacity,
139 size_t non_moving_space_capacity,
140 const std::string& image_file_name,
141 const InstructionSet image_instruction_set,
142 CollectorType foreground_collector_type,
Mathieu Chartier2dbe6272014-09-16 10:43:23 -0700143 CollectorType background_collector_type,
Mathieu Chartiera4f6af92015-08-11 17:35:25 -0700144 space::LargeObjectSpaceType large_object_space_type,
145 size_t large_object_threshold,
146 size_t parallel_gc_threads,
147 size_t conc_gc_threads,
148 bool low_memory_mode,
149 size_t long_pause_log_threshold,
150 size_t long_gc_log_threshold,
151 bool ignore_max_footprint,
152 bool use_tlab,
153 bool verify_pre_gc_heap,
154 bool verify_pre_sweeping_heap,
155 bool verify_post_gc_heap,
156 bool verify_pre_gc_rosalloc,
157 bool verify_pre_sweeping_rosalloc,
158 bool verify_post_gc_rosalloc,
159 bool gc_stress_mode,
Mathieu Chartier56fe2582016-07-14 13:30:03 -0700160 bool measure_gc_performance,
Mathieu Chartier31000802015-06-14 14:14:37 -0700161 bool use_homogeneous_space_compaction_for_oom,
Zuo Wangf37a88b2014-07-10 04:26:41 -0700162 uint64_t min_interval_homogeneous_space_compaction_by_oom)
Mathieu Chartiercbb2d202013-11-14 17:45:16 -0800163 : non_moving_space_(nullptr),
Mathieu Chartiere6da9af2013-12-16 11:54:42 -0800164 rosalloc_space_(nullptr),
165 dlmalloc_space_(nullptr),
Mathieu Chartierfc5b5282014-01-09 16:15:36 -0800166 main_space_(nullptr),
Mathieu Chartier7bf82af2013-12-06 16:51:45 -0800167 collector_type_(kCollectorTypeNone),
Mathieu Chartier31f44142014-04-08 14:40:03 -0700168 foreground_collector_type_(foreground_collector_type),
Mathieu Chartiere6da9af2013-12-16 11:54:42 -0800169 background_collector_type_(background_collector_type),
Mathieu Chartier31f44142014-04-08 14:40:03 -0700170 desired_collector_type_(foreground_collector_type_),
Mathieu Chartiera5eae692014-12-17 17:56:03 -0800171 pending_task_lock_(nullptr),
Mathieu Chartier2775ee42013-08-20 17:43:47 -0700172 parallel_gc_threads_(parallel_gc_threads),
173 conc_gc_threads_(conc_gc_threads),
Mathieu Chartiere0a53e92013-08-05 10:17:40 -0700174 low_memory_mode_(low_memory_mode),
Mathieu Chartier2775ee42013-08-20 17:43:47 -0700175 long_pause_log_threshold_(long_pause_log_threshold),
176 long_gc_log_threshold_(long_gc_log_threshold),
177 ignore_max_footprint_(ignore_max_footprint),
Mathieu Chartier8e4a96d2014-05-21 10:44:32 -0700178 zygote_creation_lock_("zygote creation lock", kZygoteCreationLock),
Mathieu Chartiere4cab172014-08-19 18:24:04 -0700179 zygote_space_(nullptr),
Mathieu Chartier2dbe6272014-09-16 10:43:23 -0700180 large_object_threshold_(large_object_threshold),
Hiroshi Yamauchi76f55b02015-08-21 16:10:39 -0700181 disable_thread_flip_count_(0),
182 thread_flip_running_(false),
Mathieu Chartierd5a89ee2014-01-31 09:55:13 -0800183 collector_type_running_(kCollectorTypeNone),
Ian Rogers1d54e732013-05-02 21:10:01 -0700184 last_gc_type_(collector::kGcTypeNone),
Mathieu Chartierbdd0fb92013-07-02 10:16:15 -0700185 next_gc_type_(collector::kGcTypePartial),
Mathieu Chartier80de7a62012-11-27 17:21:50 -0800186 capacity_(capacity),
Mathieu Chartier2fde5332012-09-14 14:51:54 -0700187 growth_limit_(growth_limit),
Mathieu Chartier0051be62012-10-12 17:47:11 -0700188 max_allowed_footprint_(initial_size),
Mathieu Chartier987ccff2013-07-08 11:05:21 -0700189 native_footprint_gc_watermark_(initial_size),
Mathieu Chartier590fee92013-09-13 13:46:47 -0700190 native_need_to_run_finalization_(false),
Mathieu Chartier7bf82af2013-12-06 16:51:45 -0800191 concurrent_start_bytes_(std::numeric_limits<size_t>::max()),
Ian Rogers1d54e732013-05-02 21:10:01 -0700192 total_bytes_freed_ever_(0),
193 total_objects_freed_ever_(0),
Elliott Hughesb3bd5f02012-03-08 21:05:27 -0800194 num_bytes_allocated_(0),
Mathieu Chartier987ccff2013-07-08 11:05:21 -0700195 native_bytes_allocated_(0),
Mathieu Chartier5d2a3f72016-05-11 11:35:39 -0700196 native_histogram_lock_("Native allocation lock"),
197 native_allocation_histogram_("Native allocation sizes",
198 1U,
199 kNativeAllocationHistogramBuckets),
200 native_free_histogram_("Native free sizes", 1U, kNativeAllocationHistogramBuckets),
Hiroshi Yamauchi4460a842015-03-09 11:57:48 -0700201 num_bytes_freed_revoke_(0),
Mathieu Chartierc7b83a02012-09-11 18:07:39 -0700202 verify_missing_card_marks_(false),
203 verify_system_weaks_(false),
Mathieu Chartier938a03b2014-01-16 15:10:31 -0800204 verify_pre_gc_heap_(verify_pre_gc_heap),
Mathieu Chartier6f365cc2014-04-23 12:42:27 -0700205 verify_pre_sweeping_heap_(verify_pre_sweeping_heap),
Mathieu Chartier938a03b2014-01-16 15:10:31 -0800206 verify_post_gc_heap_(verify_post_gc_heap),
Mathieu Chartierfd678be2012-08-30 14:50:54 -0700207 verify_mod_union_table_(false),
Hiroshi Yamauchia4adbfd2014-02-04 18:12:17 -0800208 verify_pre_gc_rosalloc_(verify_pre_gc_rosalloc),
Mathieu Chartier6f365cc2014-04-23 12:42:27 -0700209 verify_pre_sweeping_rosalloc_(verify_pre_sweeping_rosalloc),
Hiroshi Yamauchia4adbfd2014-02-04 18:12:17 -0800210 verify_post_gc_rosalloc_(verify_post_gc_rosalloc),
Mathieu Chartier31000802015-06-14 14:14:37 -0700211 gc_stress_mode_(gc_stress_mode),
Mathieu Chartier0418ae22013-07-31 13:35:46 -0700212 /* For GC a lot mode, we limit the allocations stacks to be kGcAlotInterval allocations. This
213 * causes a lot of GC since we do a GC for alloc whenever the stack is full. When heap
214 * verification is enabled, we limit the size of allocation stacks to speed up their
215 * searching.
216 */
Mathieu Chartier95a505c2014-12-10 18:45:30 -0800217 max_allocation_stack_size_(kGCALotMode ? kGcAlotAllocationStackSize
218 : (kVerifyObjectSupport > kVerifyObjectModeFast) ? kVerifyObjectAllocationStackSize :
219 kDefaultAllocationStackSize),
Mathieu Chartiere6da9af2013-12-16 11:54:42 -0800220 current_allocator_(kAllocatorTypeDlMalloc),
221 current_non_moving_allocator_(kAllocatorTypeNonMoving),
Mathieu Chartier590fee92013-09-13 13:46:47 -0700222 bump_pointer_space_(nullptr),
223 temp_space_(nullptr),
Hiroshi Yamauchi2cd334a2015-01-09 14:03:35 -0800224 region_space_(nullptr),
Mathieu Chartier0051be62012-10-12 17:47:11 -0700225 min_free_(min_free),
226 max_free_(max_free),
227 target_utilization_(target_utilization),
Hiroshi Yamauchib6bab0f2016-07-18 17:07:26 -0700228 foreground_heap_growth_multiplier_(
229 foreground_heap_growth_multiplier + kExtraHeapGrowthMultiplier),
Mathieu Chartier155dfe92012-10-09 14:24:49 -0700230 total_wait_time_(0),
Mathieu Chartier4e305412014-02-19 10:54:44 -0800231 verify_object_mode_(kVerifyObjectModeDisabled),
Mathieu Chartier1d27b342014-01-28 12:51:09 -0800232 disable_moving_gc_count_(0),
Evgenii Stepanov1e133742015-05-20 12:30:59 -0700233 is_running_on_memory_tool_(Runtime::Current()->IsRunningOnMemoryTool()),
Zuo Wangf37a88b2014-07-10 04:26:41 -0700234 use_tlab_(use_tlab),
235 main_space_backup_(nullptr),
Mathieu Chartierb363f662014-07-16 13:28:58 -0700236 min_interval_homogeneous_space_compaction_by_oom_(
237 min_interval_homogeneous_space_compaction_by_oom),
Zuo Wangf37a88b2014-07-10 04:26:41 -0700238 last_time_homogeneous_space_compaction_by_oom_(NanoTime()),
Mathieu Chartiera5eae692014-12-17 17:56:03 -0800239 pending_collector_transition_(nullptr),
240 pending_heap_trim_(nullptr),
Hiroshi Yamauchia1c9f012015-04-02 10:18:12 -0700241 use_homogeneous_space_compaction_for_oom_(use_homogeneous_space_compaction_for_oom),
242 running_collection_is_blocking_(false),
243 blocking_gc_count_(0U),
244 blocking_gc_time_(0U),
245 last_update_time_gc_count_rate_histograms_( // Round down by the window duration.
246 (NanoTime() / kGcCountRateHistogramWindowDuration) * kGcCountRateHistogramWindowDuration),
247 gc_count_last_window_(0U),
248 blocking_gc_count_last_window_(0U),
249 gc_count_rate_histogram_("gc count rate histogram", 1U, kGcCountRateMaxBucketCount),
250 blocking_gc_count_rate_histogram_("blocking gc count rate histogram", 1U,
Man Cao8c2ff642015-05-27 17:25:30 -0700251 kGcCountRateMaxBucketCount),
Mathieu Chartier31000802015-06-14 14:14:37 -0700252 alloc_tracking_enabled_(false),
253 backtrace_lock_(nullptr),
254 seen_backtrace_count_(0u),
Mathieu Chartier51168372015-08-12 16:40:32 -0700255 unique_backtrace_count_(0u),
Jeff Haodcdc85b2015-12-04 14:06:18 -0800256 gc_disabled_for_shutdown_(false) {
Elliott Hughes4dd9b4d2011-12-12 18:29:24 -0800257 if (VLOG_IS_ON(heap) || VLOG_IS_ON(startup)) {
Elliott Hughesb3bd5f02012-03-08 21:05:27 -0800258 LOG(INFO) << "Heap() entering";
Brian Carlstrom0a5b14d2011-09-27 13:29:15 -0700259 }
Mathieu Chartier32ce2ad2016-03-04 14:58:03 -0800260 ScopedTrace trace(__FUNCTION__);
Mathieu Chartier31000802015-06-14 14:14:37 -0700261 Runtime* const runtime = Runtime::Current();
Mathieu Chartier50482232013-11-21 11:48:14 -0800262 // If we aren't the zygote, switch to the default non zygote allocator. This may update the
263 // entrypoints.
Mathieu Chartier31000802015-06-14 14:14:37 -0700264 const bool is_zygote = runtime->IsZygote();
Mathieu Chartier8e219ae2014-08-19 14:29:46 -0700265 if (!is_zygote) {
Mathieu Chartier31f44142014-04-08 14:40:03 -0700266 // Background compaction is currently not supported for command line runs.
267 if (background_collector_type_ != foreground_collector_type_) {
Mathieu Chartier52ba1992014-05-07 14:39:21 -0700268 VLOG(heap) << "Disabling background compaction for non zygote";
Mathieu Chartier31f44142014-04-08 14:40:03 -0700269 background_collector_type_ = foreground_collector_type_;
Mathieu Chartierbd0a6532014-02-27 11:14:21 -0800270 }
Mathieu Chartier50482232013-11-21 11:48:14 -0800271 }
Mathieu Chartiera5f9de02014-02-28 16:48:42 -0800272 ChangeCollector(desired_collector_type_);
Ian Rogers1d54e732013-05-02 21:10:01 -0700273 live_bitmap_.reset(new accounting::HeapBitmap(this));
274 mark_bitmap_.reset(new accounting::HeapBitmap(this));
Ian Rogers30fab402012-01-23 15:43:46 -0800275 // Requested begin for the alloc space, to follow the mapped image and oat files
Ian Rogers13735952014-10-08 12:43:28 -0700276 uint8_t* requested_alloc_space_begin = nullptr;
Hiroshi Yamauchi2cd334a2015-01-09 14:03:35 -0800277 if (foreground_collector_type_ == kCollectorTypeCC) {
278 // Need to use a low address so that we can allocate a contiguous
279 // 2 * Xmx space when there's no image (dex2oat for target).
280 CHECK_GE(300 * MB, non_moving_space_capacity);
281 requested_alloc_space_begin = reinterpret_cast<uint8_t*>(300 * MB) - non_moving_space_capacity;
282 }
Jeff Haodcdc85b2015-12-04 14:06:18 -0800283
284 // Load image space(s).
Brian Carlstrom5643b782012-02-05 12:32:53 -0800285 if (!image_file_name.empty()) {
Jeff Haodcdc85b2015-12-04 14:06:18 -0800286 // For code reuse, handle this like a work queue.
287 std::vector<std::string> image_file_names;
288 image_file_names.push_back(image_file_name);
Andreas Gampe8994a042015-12-30 19:03:17 +0000289 // The loaded spaces. Secondary images may fail to load, in which case we need to remove
290 // already added spaces.
291 std::vector<space::Space*> added_image_spaces;
Mathieu Chartier582b68f2016-02-04 09:50:22 -0800292 uint8_t* const original_requested_alloc_space_begin = requested_alloc_space_begin;
Jeff Haodcdc85b2015-12-04 14:06:18 -0800293 for (size_t index = 0; index < image_file_names.size(); ++index) {
294 std::string& image_name = image_file_names[index];
Jeff Haodcdc85b2015-12-04 14:06:18 -0800295 std::string error_msg;
Mathieu Chartierfbc31082016-01-24 11:59:56 -0800296 space::ImageSpace* boot_image_space = space::ImageSpace::CreateBootImage(
297 image_name.c_str(),
298 image_instruction_set,
299 index > 0,
300 &error_msg);
Jeff Haodcdc85b2015-12-04 14:06:18 -0800301 if (boot_image_space != nullptr) {
302 AddSpace(boot_image_space);
Andreas Gampe8994a042015-12-30 19:03:17 +0000303 added_image_spaces.push_back(boot_image_space);
Jeff Haodcdc85b2015-12-04 14:06:18 -0800304 // Oat files referenced by image files immediately follow them in memory, ensure alloc space
305 // isn't going to get in the middle
306 uint8_t* oat_file_end_addr = boot_image_space->GetImageHeader().GetOatFileEnd();
307 CHECK_GT(oat_file_end_addr, boot_image_space->End());
308 requested_alloc_space_begin = AlignUp(oat_file_end_addr, kPageSize);
309 boot_image_spaces_.push_back(boot_image_space);
310
311 if (index == 0) {
312 // If this was the first space, check whether there are more images to load.
313 const OatFile* boot_oat_file = boot_image_space->GetOatFile();
314 if (boot_oat_file == nullptr) {
315 continue;
316 }
317
318 const OatHeader& boot_oat_header = boot_oat_file->GetOatHeader();
319 const char* boot_classpath =
Jeff Haof0192c82016-03-28 20:39:50 -0700320 boot_oat_header.GetStoreValueByKey(OatHeader::kBootClassPathKey);
Jeff Haodcdc85b2015-12-04 14:06:18 -0800321 if (boot_classpath == nullptr) {
322 continue;
323 }
324
Andreas Gampe8994a042015-12-30 19:03:17 +0000325 space::ImageSpace::CreateMultiImageLocations(image_file_name,
326 boot_classpath,
327 &image_file_names);
Jeff Haodcdc85b2015-12-04 14:06:18 -0800328 }
329 } else {
330 LOG(ERROR) << "Could not create image space with image file '" << image_file_name << "'. "
331 << "Attempting to fall back to imageless running. Error was: " << error_msg
332 << "\nAttempted image: " << image_name;
Andreas Gampe8994a042015-12-30 19:03:17 +0000333 // Remove already loaded spaces.
334 for (space::Space* loaded_space : added_image_spaces) {
335 RemoveSpace(loaded_space);
Mathieu Chartierb08f3052016-02-02 17:24:39 -0800336 delete loaded_space;
Andreas Gampe8994a042015-12-30 19:03:17 +0000337 }
Mathieu Chartierb08f3052016-02-02 17:24:39 -0800338 boot_image_spaces_.clear();
Mathieu Chartier582b68f2016-02-04 09:50:22 -0800339 requested_alloc_space_begin = original_requested_alloc_space_begin;
Jeff Haodcdc85b2015-12-04 14:06:18 -0800340 break;
341 }
Alex Light64ad14d2014-08-19 14:23:13 -0700342 }
Brian Carlstrom69b15fb2011-09-03 12:25:21 -0700343 }
Zuo Wangf37a88b2014-07-10 04:26:41 -0700344 /*
345 requested_alloc_space_begin -> +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-
Mathieu Chartier6a7824d2014-08-22 14:53:04 -0700346 +- nonmoving space (non_moving_space_capacity)+-
Zuo Wangf37a88b2014-07-10 04:26:41 -0700347 +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-
Mathieu Chartier8e219ae2014-08-19 14:29:46 -0700348 +-????????????????????????????????????????????+-
349 +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-
Mathieu Chartierb363f662014-07-16 13:28:58 -0700350 +-main alloc space / bump space 1 (capacity_) +-
Zuo Wangf37a88b2014-07-10 04:26:41 -0700351 +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-
Mathieu Chartierb363f662014-07-16 13:28:58 -0700352 +-????????????????????????????????????????????+-
353 +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-
354 +-main alloc space2 / bump space 2 (capacity_)+-
Zuo Wangf37a88b2014-07-10 04:26:41 -0700355 +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-
356 */
Hiroshi Yamauchi2cd334a2015-01-09 14:03:35 -0800357 // We don't have hspace compaction enabled with GSS or CC.
358 if (foreground_collector_type_ == kCollectorTypeGSS ||
359 foreground_collector_type_ == kCollectorTypeCC) {
Hiroshi Yamauchi20ed5af2014-11-17 18:05:44 -0800360 use_homogeneous_space_compaction_for_oom_ = false;
361 }
Mathieu Chartierb363f662014-07-16 13:28:58 -0700362 bool support_homogeneous_space_compaction =
Mathieu Chartier0deeb812014-08-21 18:28:20 -0700363 background_collector_type_ == gc::kCollectorTypeHomogeneousSpaceCompact ||
Hiroshi Yamauchi20ed5af2014-11-17 18:05:44 -0800364 use_homogeneous_space_compaction_for_oom_;
Mathieu Chartierb363f662014-07-16 13:28:58 -0700365 // We may use the same space the main space for the non moving space if we don't need to compact
366 // from the main space.
367 // This is not the case if we support homogeneous compaction or have a moving background
368 // collector type.
Mathieu Chartierb363f662014-07-16 13:28:58 -0700369 bool separate_non_moving_space = is_zygote ||
370 support_homogeneous_space_compaction || IsMovingGc(foreground_collector_type_) ||
371 IsMovingGc(background_collector_type_);
Mathieu Chartier76ce9172016-01-27 10:44:20 -0800372 if (foreground_collector_type_ == kCollectorTypeGSS) {
Mathieu Chartierb363f662014-07-16 13:28:58 -0700373 separate_non_moving_space = false;
374 }
375 std::unique_ptr<MemMap> main_mem_map_1;
376 std::unique_ptr<MemMap> main_mem_map_2;
Andreas Gampeace0dc12016-01-20 13:33:13 -0800377
378 // Gross hack to make dex2oat deterministic.
Mathieu Chartierc68e77b2016-01-28 09:49:55 -0800379 if (foreground_collector_type_ == kCollectorTypeMS &&
380 requested_alloc_space_begin == nullptr &&
381 Runtime::Current()->IsAotCompiler()) {
382 // Currently only enabled for MS collector since that is what the deterministic dex2oat uses.
383 // b/26849108
Andreas Gampeace0dc12016-01-20 13:33:13 -0800384 requested_alloc_space_begin = reinterpret_cast<uint8_t*>(kAllocSpaceBeginForDeterministicAoT);
385 }
Ian Rogers13735952014-10-08 12:43:28 -0700386 uint8_t* request_begin = requested_alloc_space_begin;
Mathieu Chartierb363f662014-07-16 13:28:58 -0700387 if (request_begin != nullptr && separate_non_moving_space) {
Mathieu Chartier6a7824d2014-08-22 14:53:04 -0700388 request_begin += non_moving_space_capacity;
Mathieu Chartierb363f662014-07-16 13:28:58 -0700389 }
390 std::string error_str;
391 std::unique_ptr<MemMap> non_moving_space_mem_map;
392 if (separate_non_moving_space) {
Mathieu Chartier32ce2ad2016-03-04 14:58:03 -0800393 ScopedTrace trace2("Create separate non moving space");
Mathieu Chartier7247af52014-11-19 10:51:42 -0800394 // If we are the zygote, the non moving space becomes the zygote space when we run
395 // PreZygoteFork the first time. In this case, call the map "zygote space" since we can't
396 // rename the mem map later.
397 const char* space_name = is_zygote ? kZygoteSpaceName: kNonMovingSpaceName;
Mathieu Chartierb363f662014-07-16 13:28:58 -0700398 // Reserve the non moving mem map before the other two since it needs to be at a specific
399 // address.
400 non_moving_space_mem_map.reset(
Mathieu Chartier7247af52014-11-19 10:51:42 -0800401 MemMap::MapAnonymous(space_name, requested_alloc_space_begin,
Vladimir Marko5c42c292015-02-25 12:02:49 +0000402 non_moving_space_capacity, PROT_READ | PROT_WRITE, true, false,
403 &error_str));
Mathieu Chartierb363f662014-07-16 13:28:58 -0700404 CHECK(non_moving_space_mem_map != nullptr) << error_str;
Mathieu Chartierc44ce2e2014-08-25 16:32:41 -0700405 // Try to reserve virtual memory at a lower address if we have a separate non moving space.
Ian Rogers13735952014-10-08 12:43:28 -0700406 request_begin = reinterpret_cast<uint8_t*>(300 * MB);
Mathieu Chartierb363f662014-07-16 13:28:58 -0700407 }
Hiroshi Yamauchi3dbf2342015-03-17 16:01:11 -0700408 // Attempt to create 2 mem maps at or after the requested begin.
Hiroshi Yamauchi2cd334a2015-01-09 14:03:35 -0800409 if (foreground_collector_type_ != kCollectorTypeCC) {
Mathieu Chartier32ce2ad2016-03-04 14:58:03 -0800410 ScopedTrace trace2("Create main mem map");
Mathieu Chartier966f5332016-01-25 12:53:03 -0800411 if (separate_non_moving_space || !is_zygote) {
412 main_mem_map_1.reset(MapAnonymousPreferredAddress(kMemMapSpaceName[0],
413 request_begin,
414 capacity_,
415 &error_str));
Hiroshi Yamauchi3dbf2342015-03-17 16:01:11 -0700416 } else {
Mathieu Chartier966f5332016-01-25 12:53:03 -0800417 // If no separate non-moving space and we are the zygote, the main space must come right
418 // after the image space to avoid a gap. This is required since we want the zygote space to
419 // be adjacent to the image space.
Hiroshi Yamauchi3dbf2342015-03-17 16:01:11 -0700420 main_mem_map_1.reset(MemMap::MapAnonymous(kMemMapSpaceName[0], request_begin, capacity_,
421 PROT_READ | PROT_WRITE, true, false,
422 &error_str));
423 }
Hiroshi Yamauchi2cd334a2015-01-09 14:03:35 -0800424 CHECK(main_mem_map_1.get() != nullptr) << error_str;
425 }
Mathieu Chartierb363f662014-07-16 13:28:58 -0700426 if (support_homogeneous_space_compaction ||
427 background_collector_type_ == kCollectorTypeSS ||
428 foreground_collector_type_ == kCollectorTypeSS) {
Mathieu Chartier32ce2ad2016-03-04 14:58:03 -0800429 ScopedTrace trace2("Create main mem map 2");
Mathieu Chartierb363f662014-07-16 13:28:58 -0700430 main_mem_map_2.reset(MapAnonymousPreferredAddress(kMemMapSpaceName[1], main_mem_map_1->End(),
Ian Rogers6a3c1fc2014-10-31 00:33:20 -0700431 capacity_, &error_str));
Mathieu Chartierb363f662014-07-16 13:28:58 -0700432 CHECK(main_mem_map_2.get() != nullptr) << error_str;
433 }
Mathieu Chartier32ce2ad2016-03-04 14:58:03 -0800434
Mathieu Chartierb363f662014-07-16 13:28:58 -0700435 // Create the non moving space first so that bitmaps don't take up the address range.
436 if (separate_non_moving_space) {
Mathieu Chartier32ce2ad2016-03-04 14:58:03 -0800437 ScopedTrace trace2("Add non moving space");
Mathieu Chartier31f44142014-04-08 14:40:03 -0700438 // Non moving space is always dlmalloc since we currently don't have support for multiple
Zuo Wangf37a88b2014-07-10 04:26:41 -0700439 // active rosalloc spaces.
Mathieu Chartierb363f662014-07-16 13:28:58 -0700440 const size_t size = non_moving_space_mem_map->Size();
441 non_moving_space_ = space::DlMallocSpace::CreateFromMemMap(
Mathieu Chartier6a7824d2014-08-22 14:53:04 -0700442 non_moving_space_mem_map.release(), "zygote / non moving space", kDefaultStartingSize,
Mathieu Chartierb363f662014-07-16 13:28:58 -0700443 initial_size, size, size, false);
Mathieu Chartier78408882014-04-11 18:06:01 -0700444 non_moving_space_->SetFootprintLimit(non_moving_space_->Capacity());
Mathieu Chartierb363f662014-07-16 13:28:58 -0700445 CHECK(non_moving_space_ != nullptr) << "Failed creating non moving space "
446 << requested_alloc_space_begin;
447 AddSpace(non_moving_space_);
448 }
449 // Create other spaces based on whether or not we have a moving GC.
Hiroshi Yamauchi2cd334a2015-01-09 14:03:35 -0800450 if (foreground_collector_type_ == kCollectorTypeCC) {
Hiroshi Yamauchi47995302016-06-10 14:59:43 -0700451 region_space_ = space::RegionSpace::Create("main space (region space)", capacity_ * 2, request_begin);
Hiroshi Yamauchi2cd334a2015-01-09 14:03:35 -0800452 AddSpace(region_space_);
Richard Uhler054a0782015-04-07 10:56:50 -0700453 } else if (IsMovingGc(foreground_collector_type_) &&
454 foreground_collector_type_ != kCollectorTypeGSS) {
Mathieu Chartierb363f662014-07-16 13:28:58 -0700455 // Create bump pointer spaces.
456 // We only to create the bump pointer if the foreground collector is a compacting GC.
457 // TODO: Place bump-pointer spaces somewhere to minimize size of card table.
458 bump_pointer_space_ = space::BumpPointerSpace::CreateFromMemMap("Bump pointer space 1",
459 main_mem_map_1.release());
460 CHECK(bump_pointer_space_ != nullptr) << "Failed to create bump pointer space";
461 AddSpace(bump_pointer_space_);
462 temp_space_ = space::BumpPointerSpace::CreateFromMemMap("Bump pointer space 2",
463 main_mem_map_2.release());
464 CHECK(temp_space_ != nullptr) << "Failed to create bump pointer space";
465 AddSpace(temp_space_);
466 CHECK(separate_non_moving_space);
Hiroshi Yamauchi5ccd4982014-03-11 12:19:04 -0700467 } else {
Mathieu Chartierb363f662014-07-16 13:28:58 -0700468 CreateMainMallocSpace(main_mem_map_1.release(), initial_size, growth_limit_, capacity_);
469 CHECK(main_space_ != nullptr);
470 AddSpace(main_space_);
471 if (!separate_non_moving_space) {
Zuo Wangf37a88b2014-07-10 04:26:41 -0700472 non_moving_space_ = main_space_;
Mathieu Chartierb363f662014-07-16 13:28:58 -0700473 CHECK(!non_moving_space_->CanMoveObjects());
474 }
475 if (foreground_collector_type_ == kCollectorTypeGSS) {
476 CHECK_EQ(foreground_collector_type_, background_collector_type_);
477 // Create bump pointer spaces instead of a backup space.
478 main_mem_map_2.release();
479 bump_pointer_space_ = space::BumpPointerSpace::Create("Bump pointer space 1",
480 kGSSBumpPointerSpaceCapacity, nullptr);
481 CHECK(bump_pointer_space_ != nullptr);
482 AddSpace(bump_pointer_space_);
483 temp_space_ = space::BumpPointerSpace::Create("Bump pointer space 2",
484 kGSSBumpPointerSpaceCapacity, nullptr);
485 CHECK(temp_space_ != nullptr);
486 AddSpace(temp_space_);
487 } else if (main_mem_map_2.get() != nullptr) {
488 const char* name = kUseRosAlloc ? kRosAllocSpaceName[1] : kDlMallocSpaceName[1];
489 main_space_backup_.reset(CreateMallocSpaceFromMemMap(main_mem_map_2.release(), initial_size,
490 growth_limit_, capacity_, name, true));
491 CHECK(main_space_backup_.get() != nullptr);
492 // Add the space so its accounted for in the heap_begin and heap_end.
493 AddSpace(main_space_backup_.get());
Zuo Wangf37a88b2014-07-10 04:26:41 -0700494 }
Hiroshi Yamauchi5ccd4982014-03-11 12:19:04 -0700495 }
Mathieu Chartier31f44142014-04-08 14:40:03 -0700496 CHECK(non_moving_space_ != nullptr);
Mathieu Chartierb363f662014-07-16 13:28:58 -0700497 CHECK(!non_moving_space_->CanMoveObjects());
Mathieu Chartiereb5710e2013-07-25 15:19:42 -0700498 // Allocate the large object space.
Igor Murashkinaaebaa02015-01-26 10:55:53 -0800499 if (large_object_space_type == space::LargeObjectSpaceType::kFreeList) {
Mathieu Chartier2dbe6272014-09-16 10:43:23 -0700500 large_object_space_ = space::FreeListSpace::Create("free list large object space", nullptr,
501 capacity_);
502 CHECK(large_object_space_ != nullptr) << "Failed to create large object space";
Igor Murashkinaaebaa02015-01-26 10:55:53 -0800503 } else if (large_object_space_type == space::LargeObjectSpaceType::kMap) {
Mathieu Chartier2dbe6272014-09-16 10:43:23 -0700504 large_object_space_ = space::LargeObjectMapSpace::Create("mem map large object space");
505 CHECK(large_object_space_ != nullptr) << "Failed to create large object space";
Mathieu Chartiereb5710e2013-07-25 15:19:42 -0700506 } else {
Mathieu Chartier2dbe6272014-09-16 10:43:23 -0700507 // Disable the large object space by making the cutoff excessively large.
508 large_object_threshold_ = std::numeric_limits<size_t>::max();
509 large_object_space_ = nullptr;
Mathieu Chartiereb5710e2013-07-25 15:19:42 -0700510 }
Mathieu Chartier2dbe6272014-09-16 10:43:23 -0700511 if (large_object_space_ != nullptr) {
512 AddSpace(large_object_space_);
513 }
Ian Rogers1d54e732013-05-02 21:10:01 -0700514 // Compute heap capacity. Continuous spaces are sorted in order of Begin().
Mathieu Chartier590fee92013-09-13 13:46:47 -0700515 CHECK(!continuous_spaces_.empty());
516 // Relies on the spaces being sorted.
Ian Rogers13735952014-10-08 12:43:28 -0700517 uint8_t* heap_begin = continuous_spaces_.front()->Begin();
518 uint8_t* heap_end = continuous_spaces_.back()->Limit();
Mathieu Chartier590fee92013-09-13 13:46:47 -0700519 size_t heap_capacity = heap_end - heap_begin;
Mathieu Chartierb363f662014-07-16 13:28:58 -0700520 // Remove the main backup space since it slows down the GC to have unused extra spaces.
Mathieu Chartier0310da52014-12-01 13:40:48 -0800521 // TODO: Avoid needing to do this.
Mathieu Chartierb363f662014-07-16 13:28:58 -0700522 if (main_space_backup_.get() != nullptr) {
523 RemoveSpace(main_space_backup_.get());
524 }
Elliott Hughes6c9c06d2011-11-07 16:43:47 -0800525 // Allocate the card table.
Mathieu Chartierfbc31082016-01-24 11:59:56 -0800526 // We currently don't support dynamically resizing the card table.
527 // Since we don't know where in the low_4gb the app image will be located, make the card table
528 // cover the whole low_4gb. TODO: Extend the card table in AddSpace.
529 UNUSED(heap_capacity);
530 // Start at 64 KB, we can be sure there are no spaces mapped this low since the address range is
531 // reserved by the kernel.
532 static constexpr size_t kMinHeapAddress = 4 * KB;
533 card_table_.reset(accounting::CardTable::Create(reinterpret_cast<uint8_t*>(kMinHeapAddress),
534 4 * GB - kMinHeapAddress));
Mathieu Chartier2cebb242015-04-21 16:50:40 -0700535 CHECK(card_table_.get() != nullptr) << "Failed to create card table";
Hiroshi Yamauchi2cd334a2015-01-09 14:03:35 -0800536 if (foreground_collector_type_ == kCollectorTypeCC && kUseTableLookupReadBarrier) {
537 rb_table_.reset(new accounting::ReadBarrierTable());
538 DCHECK(rb_table_->IsAllCleared());
539 }
Jeff Haodcdc85b2015-12-04 14:06:18 -0800540 if (HasBootImageSpace()) {
Mathieu Chartier4858a932015-01-23 13:18:53 -0800541 // Don't add the image mod union table if we are running without an image, this can crash if
542 // we use the CardCache implementation.
Jeff Haodcdc85b2015-12-04 14:06:18 -0800543 for (space::ImageSpace* image_space : GetBootImageSpaces()) {
544 accounting::ModUnionTable* mod_union_table = new accounting::ModUnionTableToZygoteAllocspace(
545 "Image mod-union table", this, image_space);
546 CHECK(mod_union_table != nullptr) << "Failed to create image mod-union table";
547 AddModUnionTable(mod_union_table);
548 }
Mathieu Chartier4858a932015-01-23 13:18:53 -0800549 }
Mathieu Chartier96bcd452014-06-17 09:50:02 -0700550 if (collector::SemiSpace::kUseRememberedSet && non_moving_space_ != main_space_) {
Hiroshi Yamauchi38e68e92014-03-07 13:59:08 -0800551 accounting::RememberedSet* non_moving_space_rem_set =
552 new accounting::RememberedSet("Non-moving space remembered set", this, non_moving_space_);
553 CHECK(non_moving_space_rem_set != nullptr) << "Failed to create non-moving space remembered set";
554 AddRememberedSet(non_moving_space_rem_set);
555 }
Mathieu Chartierb363f662014-07-16 13:28:58 -0700556 // TODO: Count objects in the image space here?
Ian Rogers3e5cf302014-05-20 16:40:37 -0700557 num_bytes_allocated_.StoreRelaxed(0);
Mathieu Chartierc1790162014-05-23 10:54:50 -0700558 mark_stack_.reset(accounting::ObjectStack::Create("mark stack", kDefaultMarkStackSize,
559 kDefaultMarkStackSize));
560 const size_t alloc_stack_capacity = max_allocation_stack_size_ + kAllocationStackReserveSize;
561 allocation_stack_.reset(accounting::ObjectStack::Create(
562 "allocation stack", max_allocation_stack_size_, alloc_stack_capacity));
563 live_stack_.reset(accounting::ObjectStack::Create(
564 "live stack", max_allocation_stack_size_, alloc_stack_capacity));
Mathieu Chartier65db8802012-11-20 12:36:46 -0800565 // It's still too early to take a lock because there are no threads yet, but we can create locks
566 // now. We don't create it earlier to make it clear that you can't use locks during heap
567 // initialization.
Mathieu Chartierfd678be2012-08-30 14:50:54 -0700568 gc_complete_lock_ = new Mutex("GC complete lock");
Ian Rogersc604d732012-10-14 16:09:54 -0700569 gc_complete_cond_.reset(new ConditionVariable("GC complete condition variable",
570 *gc_complete_lock_));
Hiroshi Yamauchi76f55b02015-08-21 16:10:39 -0700571 thread_flip_lock_ = new Mutex("GC thread flip lock");
572 thread_flip_cond_.reset(new ConditionVariable("GC thread flip condition variable",
573 *thread_flip_lock_));
Mathieu Chartiera5eae692014-12-17 17:56:03 -0800574 task_processor_.reset(new TaskProcessor());
Mathieu Chartier3cf22532015-07-09 15:15:09 -0700575 reference_processor_.reset(new ReferenceProcessor());
Mathieu Chartiera5eae692014-12-17 17:56:03 -0800576 pending_task_lock_ = new Mutex("Pending task lock");
Mathieu Chartier2775ee42013-08-20 17:43:47 -0700577 if (ignore_max_footprint_) {
578 SetIdealFootprint(std::numeric_limits<size_t>::max());
Mathieu Chartier590fee92013-09-13 13:46:47 -0700579 concurrent_start_bytes_ = std::numeric_limits<size_t>::max();
Mathieu Chartier2775ee42013-08-20 17:43:47 -0700580 }
Mathieu Chartier590fee92013-09-13 13:46:47 -0700581 CHECK_NE(max_allowed_footprint_, 0U);
Mathieu Chartier2b82db42012-11-14 17:29:05 -0800582 // Create our garbage collectors.
Mathieu Chartier50482232013-11-21 11:48:14 -0800583 for (size_t i = 0; i < 2; ++i) {
584 const bool concurrent = i != 0;
Mathieu Chartierdfe30832015-03-06 15:28:34 -0800585 if ((MayUseCollector(kCollectorTypeCMS) && concurrent) ||
586 (MayUseCollector(kCollectorTypeMS) && !concurrent)) {
587 garbage_collectors_.push_back(new collector::MarkSweep(this, concurrent));
588 garbage_collectors_.push_back(new collector::PartialMarkSweep(this, concurrent));
589 garbage_collectors_.push_back(new collector::StickyMarkSweep(this, concurrent));
590 }
Mathieu Chartier50482232013-11-21 11:48:14 -0800591 }
Mathieu Chartier50482232013-11-21 11:48:14 -0800592 if (kMovingCollector) {
Mathieu Chartierdfe30832015-03-06 15:28:34 -0800593 if (MayUseCollector(kCollectorTypeSS) || MayUseCollector(kCollectorTypeGSS) ||
594 MayUseCollector(kCollectorTypeHomogeneousSpaceCompact) ||
595 use_homogeneous_space_compaction_for_oom_) {
596 // TODO: Clean this up.
597 const bool generational = foreground_collector_type_ == kCollectorTypeGSS;
598 semi_space_collector_ = new collector::SemiSpace(this, generational,
599 generational ? "generational" : "");
600 garbage_collectors_.push_back(semi_space_collector_);
601 }
602 if (MayUseCollector(kCollectorTypeCC)) {
Mathieu Chartier56fe2582016-07-14 13:30:03 -0700603 concurrent_copying_collector_ = new collector::ConcurrentCopying(this,
604 "",
605 measure_gc_performance);
Mathieu Chartierdfe30832015-03-06 15:28:34 -0800606 garbage_collectors_.push_back(concurrent_copying_collector_);
607 }
608 if (MayUseCollector(kCollectorTypeMC)) {
609 mark_compact_collector_ = new collector::MarkCompact(this);
610 garbage_collectors_.push_back(mark_compact_collector_);
611 }
Mathieu Chartier0325e622012-09-05 14:22:51 -0700612 }
Jeff Haodcdc85b2015-12-04 14:06:18 -0800613 if (!GetBootImageSpaces().empty() && non_moving_space_ != nullptr &&
Andreas Gampee1cb2982014-08-27 11:01:09 -0700614 (is_zygote || separate_non_moving_space || foreground_collector_type_ == kCollectorTypeGSS)) {
Mathieu Chartierb363f662014-07-16 13:28:58 -0700615 // Check that there's no gap between the image space and the non moving space so that the
Andreas Gampee1cb2982014-08-27 11:01:09 -0700616 // immune region won't break (eg. due to a large object allocated in the gap). This is only
617 // required when we're the zygote or using GSS.
Mathieu Chartiera06ba052016-01-06 13:51:52 -0800618 // Space with smallest Begin().
619 space::ImageSpace* first_space = nullptr;
620 for (space::ImageSpace* space : boot_image_spaces_) {
621 if (first_space == nullptr || space->Begin() < first_space->Begin()) {
622 first_space = space;
623 }
624 }
625 bool no_gap = MemMap::CheckNoGaps(first_space->GetMemMap(), non_moving_space_->GetMemMap());
Hiroshi Yamauchi3eed93d2014-06-04 11:43:59 -0700626 if (!no_gap) {
David Srbecky5dedb802015-06-17 00:08:02 +0100627 PrintFileToLog("/proc/self/maps", LogSeverity::ERROR);
Vladimir Marko17a924a2015-05-08 15:17:32 +0100628 MemMap::DumpMaps(LOG(ERROR), true);
Mathieu Chartierc7853442015-03-27 14:35:38 -0700629 LOG(FATAL) << "There's a gap between the image space and the non-moving space";
Hiroshi Yamauchi3eed93d2014-06-04 11:43:59 -0700630 }
631 }
Mathieu Chartier31000802015-06-14 14:14:37 -0700632 instrumentation::Instrumentation* const instrumentation = runtime->GetInstrumentation();
633 if (gc_stress_mode_) {
634 backtrace_lock_ = new Mutex("GC complete lock");
635 }
Evgenii Stepanov1e133742015-05-20 12:30:59 -0700636 if (is_running_on_memory_tool_ || gc_stress_mode_) {
Mathieu Chartier31000802015-06-14 14:14:37 -0700637 instrumentation->InstrumentQuickAllocEntryPoints();
Hiroshi Yamauchi3b4c1892013-09-12 21:33:12 -0700638 }
Elliott Hughes4dd9b4d2011-12-12 18:29:24 -0800639 if (VLOG_IS_ON(heap) || VLOG_IS_ON(startup)) {
Elliott Hughesb3bd5f02012-03-08 21:05:27 -0800640 LOG(INFO) << "Heap() exiting";
Brian Carlstrom0a5b14d2011-09-27 13:29:15 -0700641 }
Carl Shapiro69759ea2011-07-21 18:13:35 -0700642}
643
Mathieu Chartiera4f6af92015-08-11 17:35:25 -0700644MemMap* Heap::MapAnonymousPreferredAddress(const char* name,
645 uint8_t* request_begin,
646 size_t capacity,
647 std::string* out_error_str) {
Mathieu Chartierb363f662014-07-16 13:28:58 -0700648 while (true) {
Kyungmin Leeef32b8f2014-10-23 09:32:05 +0900649 MemMap* map = MemMap::MapAnonymous(name, request_begin, capacity,
Vladimir Marko5c42c292015-02-25 12:02:49 +0000650 PROT_READ | PROT_WRITE, true, false, out_error_str);
Mathieu Chartierb363f662014-07-16 13:28:58 -0700651 if (map != nullptr || request_begin == nullptr) {
652 return map;
653 }
654 // Retry a second time with no specified request begin.
655 request_begin = nullptr;
656 }
Mathieu Chartierb363f662014-07-16 13:28:58 -0700657}
658
Mathieu Chartierdfe30832015-03-06 15:28:34 -0800659bool Heap::MayUseCollector(CollectorType type) const {
660 return foreground_collector_type_ == type || background_collector_type_ == type;
661}
662
Mathieu Chartiera4f6af92015-08-11 17:35:25 -0700663space::MallocSpace* Heap::CreateMallocSpaceFromMemMap(MemMap* mem_map,
664 size_t initial_size,
665 size_t growth_limit,
666 size_t capacity,
667 const char* name,
668 bool can_move_objects) {
Zuo Wangf37a88b2014-07-10 04:26:41 -0700669 space::MallocSpace* malloc_space = nullptr;
670 if (kUseRosAlloc) {
671 // Create rosalloc space.
672 malloc_space = space::RosAllocSpace::CreateFromMemMap(mem_map, name, kDefaultStartingSize,
673 initial_size, growth_limit, capacity,
674 low_memory_mode_, can_move_objects);
675 } else {
676 malloc_space = space::DlMallocSpace::CreateFromMemMap(mem_map, name, kDefaultStartingSize,
677 initial_size, growth_limit, capacity,
678 can_move_objects);
679 }
680 if (collector::SemiSpace::kUseRememberedSet) {
681 accounting::RememberedSet* rem_set =
682 new accounting::RememberedSet(std::string(name) + " remembered set", this, malloc_space);
683 CHECK(rem_set != nullptr) << "Failed to create main space remembered set";
684 AddRememberedSet(rem_set);
685 }
686 CHECK(malloc_space != nullptr) << "Failed to create " << name;
687 malloc_space->SetFootprintLimit(malloc_space->Capacity());
688 return malloc_space;
689}
690
Mathieu Chartier31f44142014-04-08 14:40:03 -0700691void Heap::CreateMainMallocSpace(MemMap* mem_map, size_t initial_size, size_t growth_limit,
692 size_t capacity) {
693 // Is background compaction is enabled?
694 bool can_move_objects = IsMovingGc(background_collector_type_) !=
Zuo Wangf37a88b2014-07-10 04:26:41 -0700695 IsMovingGc(foreground_collector_type_) || use_homogeneous_space_compaction_for_oom_;
Mathieu Chartier31f44142014-04-08 14:40:03 -0700696 // If we are the zygote and don't yet have a zygote space, it means that the zygote fork will
697 // happen in the future. If this happens and we have kCompactZygote enabled we wish to compact
698 // from the main space to the zygote space. If background compaction is enabled, always pass in
699 // that we can move objets.
700 if (kCompactZygote && Runtime::Current()->IsZygote() && !can_move_objects) {
701 // After the zygote we want this to be false if we don't have background compaction enabled so
702 // that getting primitive array elements is faster.
Mathieu Chartierb363f662014-07-16 13:28:58 -0700703 // We never have homogeneous compaction with GSS and don't need a space with movable objects.
Mathieu Chartiere4cab172014-08-19 18:24:04 -0700704 can_move_objects = !HasZygoteSpace() && foreground_collector_type_ != kCollectorTypeGSS;
Mathieu Chartier31f44142014-04-08 14:40:03 -0700705 }
Mathieu Chartier96bcd452014-06-17 09:50:02 -0700706 if (collector::SemiSpace::kUseRememberedSet && main_space_ != nullptr) {
707 RemoveRememberedSet(main_space_);
708 }
Zuo Wangf37a88b2014-07-10 04:26:41 -0700709 const char* name = kUseRosAlloc ? kRosAllocSpaceName[0] : kDlMallocSpaceName[0];
710 main_space_ = CreateMallocSpaceFromMemMap(mem_map, initial_size, growth_limit, capacity, name,
711 can_move_objects);
712 SetSpaceAsDefault(main_space_);
Mathieu Chartier31f44142014-04-08 14:40:03 -0700713 VLOG(heap) << "Created main space " << main_space_;
714}
715
Mathieu Chartier50482232013-11-21 11:48:14 -0800716void Heap::ChangeAllocator(AllocatorType allocator) {
Mathieu Chartier50482232013-11-21 11:48:14 -0800717 if (current_allocator_ != allocator) {
Mathieu Chartierd8891782014-03-02 13:28:37 -0800718 // These two allocators are only used internally and don't have any entrypoints.
719 CHECK_NE(allocator, kAllocatorTypeLOS);
720 CHECK_NE(allocator, kAllocatorTypeNonMoving);
Mathieu Chartier50482232013-11-21 11:48:14 -0800721 current_allocator_ = allocator;
Mathieu Chartierd8891782014-03-02 13:28:37 -0800722 MutexLock mu(nullptr, *Locks::runtime_shutdown_lock_);
Mathieu Chartier50482232013-11-21 11:48:14 -0800723 SetQuickAllocEntryPointsAllocator(current_allocator_);
724 Runtime::Current()->GetInstrumentation()->ResetQuickAllocEntryPoints();
725 }
726}
727
Mathieu Chartier6a7824d2014-08-22 14:53:04 -0700728void Heap::DisableMovingGc() {
Mathieu Chartier31f44142014-04-08 14:40:03 -0700729 if (IsMovingGc(foreground_collector_type_)) {
Mathieu Chartier6a7824d2014-08-22 14:53:04 -0700730 foreground_collector_type_ = kCollectorTypeCMS;
Mathieu Chartier6dda8982014-03-06 11:11:48 -0800731 }
Mathieu Chartier31f44142014-04-08 14:40:03 -0700732 if (IsMovingGc(background_collector_type_)) {
733 background_collector_type_ = foreground_collector_type_;
Mathieu Chartier6dda8982014-03-06 11:11:48 -0800734 }
Mathieu Chartier31f44142014-04-08 14:40:03 -0700735 TransitionCollector(foreground_collector_type_);
Mathieu Chartier4f55e222015-09-04 13:26:21 -0700736 Thread* const self = Thread::Current();
Mathieu Chartier6a7824d2014-08-22 14:53:04 -0700737 ScopedThreadStateChange tsc(self, kSuspended);
Mathieu Chartier4f55e222015-09-04 13:26:21 -0700738 ScopedSuspendAll ssa(__FUNCTION__);
Mathieu Chartier6a7824d2014-08-22 14:53:04 -0700739 // Something may have caused the transition to fail.
Mathieu Chartiere4927f62014-08-23 13:56:03 -0700740 if (!IsMovingGc(collector_type_) && non_moving_space_ != main_space_) {
Mathieu Chartier6a7824d2014-08-22 14:53:04 -0700741 CHECK(main_space_ != nullptr);
742 // The allocation stack may have non movable objects in it. We need to flush it since the GC
743 // can't only handle marking allocation stack objects of one non moving space and one main
744 // space.
745 {
746 WriterMutexLock mu(self, *Locks::heap_bitmap_lock_);
747 FlushAllocStack();
748 }
749 main_space_->DisableMovingObjects();
750 non_moving_space_ = main_space_;
751 CHECK(!non_moving_space_->CanMoveObjects());
752 }
Mathieu Chartier6dda8982014-03-06 11:11:48 -0800753}
754
Mathieu Chartier15d34022014-02-26 17:16:38 -0800755std::string Heap::SafeGetClassDescriptor(mirror::Class* klass) {
756 if (!IsValidContinuousSpaceObjectAddress(klass)) {
757 return StringPrintf("<non heap address klass %p>", klass);
758 }
759 mirror::Class* component_type = klass->GetComponentType<kVerifyNone>();
760 if (IsValidContinuousSpaceObjectAddress(component_type) && klass->IsArrayClass<kVerifyNone>()) {
761 std::string result("[");
762 result += SafeGetClassDescriptor(component_type);
763 return result;
764 } else if (UNLIKELY(klass->IsPrimitive<kVerifyNone>())) {
765 return Primitive::Descriptor(klass->GetPrimitiveType<kVerifyNone>());
Nicolas Geoffray3a090922015-11-24 09:17:30 +0000766 } else if (UNLIKELY(klass->IsProxyClass<kVerifyNone>())) {
767 return Runtime::Current()->GetClassLinker()->GetDescriptorForProxy(klass);
Mathieu Chartier15d34022014-02-26 17:16:38 -0800768 } else {
Mathieu Chartierc2f4d022014-03-03 16:11:42 -0800769 mirror::DexCache* dex_cache = klass->GetDexCache<kVerifyNone>();
Mathieu Chartier15d34022014-02-26 17:16:38 -0800770 if (!IsValidContinuousSpaceObjectAddress(dex_cache)) {
771 return StringPrintf("<non heap address dex_cache %p>", dex_cache);
772 }
773 const DexFile* dex_file = dex_cache->GetDexFile();
774 uint16_t class_def_idx = klass->GetDexClassDefIndex();
775 if (class_def_idx == DexFile::kDexNoIndex16) {
776 return "<class def not found>";
777 }
778 const DexFile::ClassDef& class_def = dex_file->GetClassDef(class_def_idx);
779 const DexFile::TypeId& type_id = dex_file->GetTypeId(class_def.class_idx_);
780 return dex_file->GetTypeDescriptor(type_id);
781 }
782}
783
784std::string Heap::SafePrettyTypeOf(mirror::Object* obj) {
785 if (obj == nullptr) {
786 return "null";
787 }
788 mirror::Class* klass = obj->GetClass<kVerifyNone>();
789 if (klass == nullptr) {
790 return "(class=null)";
791 }
792 std::string result(SafeGetClassDescriptor(klass));
793 if (obj->IsClass()) {
Mathieu Chartierc2f4d022014-03-03 16:11:42 -0800794 result += "<" + SafeGetClassDescriptor(obj->AsClass<kVerifyNone>()) + ">";
Mathieu Chartier15d34022014-02-26 17:16:38 -0800795 }
796 return result;
797}
798
799void Heap::DumpObject(std::ostream& stream, mirror::Object* obj) {
800 if (obj == nullptr) {
801 stream << "(obj=null)";
802 return;
803 }
804 if (IsAligned<kObjectAlignment>(obj)) {
805 space::Space* space = nullptr;
806 // Don't use find space since it only finds spaces which actually contain objects instead of
807 // spaces which may contain objects (e.g. cleared bump pointer spaces).
808 for (const auto& cur_space : continuous_spaces_) {
809 if (cur_space->HasAddress(obj)) {
810 space = cur_space;
811 break;
812 }
813 }
Mathieu Chartier15d34022014-02-26 17:16:38 -0800814 // Unprotect all the spaces.
Andreas Gampe277ccbd2014-11-03 21:36:10 -0800815 for (const auto& con_space : continuous_spaces_) {
816 mprotect(con_space->Begin(), con_space->Capacity(), PROT_READ | PROT_WRITE);
Mathieu Chartier15d34022014-02-26 17:16:38 -0800817 }
818 stream << "Object " << obj;
819 if (space != nullptr) {
820 stream << " in space " << *space;
821 }
Mathieu Chartierc2f4d022014-03-03 16:11:42 -0800822 mirror::Class* klass = obj->GetClass<kVerifyNone>();
Mathieu Chartier15d34022014-02-26 17:16:38 -0800823 stream << "\nclass=" << klass;
824 if (klass != nullptr) {
825 stream << " type= " << SafePrettyTypeOf(obj);
826 }
827 // Re-protect the address we faulted on.
828 mprotect(AlignDown(obj, kPageSize), kPageSize, PROT_NONE);
829 }
830}
831
Mathieu Chartier590fee92013-09-13 13:46:47 -0700832bool Heap::IsCompilingBoot() const {
Mathieu Chartiere5f13e52015-02-24 09:37:21 -0800833 if (!Runtime::Current()->IsAotCompiler()) {
Alex Light64ad14d2014-08-19 14:23:13 -0700834 return false;
835 }
Mathieu Chartiera9d82fe2016-01-25 20:06:11 -0800836 ScopedObjectAccess soa(Thread::Current());
Mathieu Chartier590fee92013-09-13 13:46:47 -0700837 for (const auto& space : continuous_spaces_) {
Mathieu Chartier4e305412014-02-19 10:54:44 -0800838 if (space->IsImageSpace() || space->IsZygoteSpace()) {
Mathieu Chartier590fee92013-09-13 13:46:47 -0700839 return false;
840 }
841 }
842 return true;
843}
844
Mathieu Chartier1d27b342014-01-28 12:51:09 -0800845void Heap::IncrementDisableMovingGC(Thread* self) {
Mathieu Chartier590fee92013-09-13 13:46:47 -0700846 // Need to do this holding the lock to prevent races where the GC is about to run / running when
847 // we attempt to disable it.
Mathieu Chartiercaa82d62014-02-02 16:51:17 -0800848 ScopedThreadStateChange tsc(self, kWaitingForGcToComplete);
Mathieu Chartier590fee92013-09-13 13:46:47 -0700849 MutexLock mu(self, *gc_complete_lock_);
Mathieu Chartier1d27b342014-01-28 12:51:09 -0800850 ++disable_moving_gc_count_;
Mathieu Chartier31f44142014-04-08 14:40:03 -0700851 if (IsMovingGc(collector_type_running_)) {
Mathieu Chartier89a201e2014-05-02 10:27:26 -0700852 WaitForGcToCompleteLocked(kGcCauseDisableMovingGc, self);
Mathieu Chartierd5a89ee2014-01-31 09:55:13 -0800853 }
Mathieu Chartier590fee92013-09-13 13:46:47 -0700854}
855
Mathieu Chartier1d27b342014-01-28 12:51:09 -0800856void Heap::DecrementDisableMovingGC(Thread* self) {
Mathieu Chartier590fee92013-09-13 13:46:47 -0700857 MutexLock mu(self, *gc_complete_lock_);
Mathieu Chartierb735bd92015-06-24 17:04:17 -0700858 CHECK_GT(disable_moving_gc_count_, 0U);
Mathieu Chartier1d27b342014-01-28 12:51:09 -0800859 --disable_moving_gc_count_;
Mathieu Chartier590fee92013-09-13 13:46:47 -0700860}
861
Hiroshi Yamauchi76f55b02015-08-21 16:10:39 -0700862void Heap::IncrementDisableThreadFlip(Thread* self) {
863 // Supposed to be called by mutators. If thread_flip_running_ is true, block. Otherwise, go ahead.
864 CHECK(kUseReadBarrier);
Hiroshi Yamauchi20a0be02016-02-19 15:44:06 -0800865 bool is_nested = self->GetDisableThreadFlipCount() > 0;
866 self->IncrementDisableThreadFlipCount();
867 if (is_nested) {
868 // If this is a nested JNI critical section enter, we don't need to wait or increment the global
869 // counter. The global counter is incremented only once for a thread for the outermost enter.
870 return;
871 }
Hiroshi Yamauchi76f55b02015-08-21 16:10:39 -0700872 ScopedThreadStateChange tsc(self, kWaitingForGcThreadFlip);
873 MutexLock mu(self, *thread_flip_lock_);
874 bool has_waited = false;
875 uint64_t wait_start = NanoTime();
876 while (thread_flip_running_) {
877 has_waited = true;
878 thread_flip_cond_->Wait(self);
879 }
880 ++disable_thread_flip_count_;
881 if (has_waited) {
882 uint64_t wait_time = NanoTime() - wait_start;
883 total_wait_time_ += wait_time;
884 if (wait_time > long_pause_log_threshold_) {
885 LOG(INFO) << __FUNCTION__ << " blocked for " << PrettyDuration(wait_time);
886 }
887 }
888}
889
890void Heap::DecrementDisableThreadFlip(Thread* self) {
891 // Supposed to be called by mutators. Decrement disable_thread_flip_count_ and potentially wake up
892 // the GC waiting before doing a thread flip.
893 CHECK(kUseReadBarrier);
Hiroshi Yamauchi20a0be02016-02-19 15:44:06 -0800894 self->DecrementDisableThreadFlipCount();
895 bool is_outermost = self->GetDisableThreadFlipCount() == 0;
896 if (!is_outermost) {
897 // If this is not an outermost JNI critical exit, we don't need to decrement the global counter.
898 // The global counter is decremented only once for a thread for the outermost exit.
899 return;
900 }
Hiroshi Yamauchi76f55b02015-08-21 16:10:39 -0700901 MutexLock mu(self, *thread_flip_lock_);
902 CHECK_GT(disable_thread_flip_count_, 0U);
903 --disable_thread_flip_count_;
Hiroshi Yamauchi20a0be02016-02-19 15:44:06 -0800904 if (disable_thread_flip_count_ == 0) {
905 // Potentially notify the GC thread blocking to begin a thread flip.
906 thread_flip_cond_->Broadcast(self);
907 }
Hiroshi Yamauchi76f55b02015-08-21 16:10:39 -0700908}
909
910void Heap::ThreadFlipBegin(Thread* self) {
911 // Supposed to be called by GC. Set thread_flip_running_ to be true. If disable_thread_flip_count_
912 // > 0, block. Otherwise, go ahead.
913 CHECK(kUseReadBarrier);
914 ScopedThreadStateChange tsc(self, kWaitingForGcThreadFlip);
915 MutexLock mu(self, *thread_flip_lock_);
916 bool has_waited = false;
917 uint64_t wait_start = NanoTime();
918 CHECK(!thread_flip_running_);
Hiroshi Yamauchi20a0be02016-02-19 15:44:06 -0800919 // Set this to true before waiting so that frequent JNI critical enter/exits won't starve
920 // GC. This like a writer preference of a reader-writer lock.
Hiroshi Yamauchi76f55b02015-08-21 16:10:39 -0700921 thread_flip_running_ = true;
922 while (disable_thread_flip_count_ > 0) {
923 has_waited = true;
924 thread_flip_cond_->Wait(self);
925 }
926 if (has_waited) {
927 uint64_t wait_time = NanoTime() - wait_start;
928 total_wait_time_ += wait_time;
929 if (wait_time > long_pause_log_threshold_) {
930 LOG(INFO) << __FUNCTION__ << " blocked for " << PrettyDuration(wait_time);
931 }
932 }
933}
934
935void Heap::ThreadFlipEnd(Thread* self) {
936 // Supposed to be called by GC. Set thread_flip_running_ to false and potentially wake up mutators
937 // waiting before doing a JNI critical.
938 CHECK(kUseReadBarrier);
939 MutexLock mu(self, *thread_flip_lock_);
940 CHECK(thread_flip_running_);
941 thread_flip_running_ = false;
Hiroshi Yamauchi20a0be02016-02-19 15:44:06 -0800942 // Potentially notify mutator threads blocking to enter a JNI critical section.
Hiroshi Yamauchi76f55b02015-08-21 16:10:39 -0700943 thread_flip_cond_->Broadcast(self);
944}
945
Mathieu Chartierf8cb1782016-03-18 18:45:41 -0700946void Heap::UpdateProcessState(ProcessState old_process_state, ProcessState new_process_state) {
947 if (old_process_state != new_process_state) {
948 const bool jank_perceptible = new_process_state == kProcessStateJankPerceptible;
Mathieu Chartier91e30632014-03-25 15:58:50 -0700949 for (size_t i = 1; i <= kCollectorTransitionStressIterations; ++i) {
950 // Start at index 1 to avoid "is always false" warning.
951 // Have iteration 1 always transition the collector.
Mathieu Chartierf8cb1782016-03-18 18:45:41 -0700952 TransitionCollector((((i & 1) == 1) == jank_perceptible)
953 ? foreground_collector_type_
954 : background_collector_type_);
Mathieu Chartier91e30632014-03-25 15:58:50 -0700955 usleep(kCollectorTransitionStressWait);
956 }
Mathieu Chartierf8cb1782016-03-18 18:45:41 -0700957 if (jank_perceptible) {
Mathieu Chartiera5f9de02014-02-28 16:48:42 -0800958 // Transition back to foreground right away to prevent jank.
Mathieu Chartier31f44142014-04-08 14:40:03 -0700959 RequestCollectorTransition(foreground_collector_type_, 0);
Mathieu Chartiere6da9af2013-12-16 11:54:42 -0800960 } else {
Mathieu Chartiera5f9de02014-02-28 16:48:42 -0800961 // Don't delay for debug builds since we may want to stress test the GC.
Zuo Wangf37a88b2014-07-10 04:26:41 -0700962 // If background_collector_type_ is kCollectorTypeHomogeneousSpaceCompact then we have
963 // special handling which does a homogenous space compaction once but then doesn't transition
964 // the collector.
965 RequestCollectorTransition(background_collector_type_,
966 kIsDebugBuild ? 0 : kCollectorTransitionWait);
Mathieu Chartiere6da9af2013-12-16 11:54:42 -0800967 }
Mathieu Chartiere6da9af2013-12-16 11:54:42 -0800968 }
Mathieu Chartierca2a24d2013-11-25 15:12:12 -0800969}
970
Mathieu Chartier02b6a782012-10-26 13:51:26 -0700971void Heap::CreateThreadPool() {
Mathieu Chartier2775ee42013-08-20 17:43:47 -0700972 const size_t num_threads = std::max(parallel_gc_threads_, conc_gc_threads_);
973 if (num_threads != 0) {
Mathieu Chartierbcd5e9d2013-11-13 14:33:28 -0800974 thread_pool_.reset(new ThreadPool("Heap thread pool", num_threads));
Mathieu Chartier94c32c52013-08-09 11:14:04 -0700975 }
Mathieu Chartier02b6a782012-10-26 13:51:26 -0700976}
977
Hiroshi Yamauchi2cd334a2015-01-09 14:03:35 -0800978// Visit objects when threads aren't suspended. If concurrent moving
979// GC, disable moving GC and suspend threads and then visit objects.
Mathieu Chartier83c8ee02014-01-28 14:50:23 -0800980void Heap::VisitObjects(ObjectCallback callback, void* arg) {
Hiroshi Yamauchi0c8c3032015-01-16 16:54:35 -0800981 Thread* self = Thread::Current();
Hiroshi Yamauchi2cd334a2015-01-09 14:03:35 -0800982 Locks::mutator_lock_->AssertSharedHeld(self);
983 DCHECK(!Locks::mutator_lock_->IsExclusiveHeld(self)) << "Call VisitObjectsPaused() instead";
984 if (IsGcConcurrentAndMoving()) {
985 // Concurrent moving GC. Just suspending threads isn't sufficient
986 // because a collection isn't one big pause and we could suspend
987 // threads in the middle (between phases) of a concurrent moving
988 // collection where it's not easily known which objects are alive
989 // (both the region space and the non-moving space) or which
990 // copies of objects to visit, and the to-space invariant could be
991 // easily broken. Visit objects while GC isn't running by using
992 // IncrementDisableMovingGC() and threads are suspended.
993 IncrementDisableMovingGC(self);
Mathieu Chartierf1d666e2015-09-03 16:13:34 -0700994 {
995 ScopedThreadSuspension sts(self, kWaitingForVisitObjects);
Mathieu Chartier4f55e222015-09-04 13:26:21 -0700996 ScopedSuspendAll ssa(__FUNCTION__);
Mathieu Chartierf1d666e2015-09-03 16:13:34 -0700997 VisitObjectsInternalRegionSpace(callback, arg);
998 VisitObjectsInternal(callback, arg);
Mathieu Chartierf1d666e2015-09-03 16:13:34 -0700999 }
Hiroshi Yamauchi2cd334a2015-01-09 14:03:35 -08001000 DecrementDisableMovingGC(self);
Hiroshi Yamauchi0c8c3032015-01-16 16:54:35 -08001001 } else {
1002 // GCs can move objects, so don't allow this.
1003 ScopedAssertNoThreadSuspension ants(self, "Visiting objects");
Hiroshi Yamauchi2cd334a2015-01-09 14:03:35 -08001004 DCHECK(region_space_ == nullptr);
Hiroshi Yamauchi0c8c3032015-01-16 16:54:35 -08001005 VisitObjectsInternal(callback, arg);
1006 }
1007}
1008
Hiroshi Yamauchi2cd334a2015-01-09 14:03:35 -08001009// Visit objects when threads are already suspended.
1010void Heap::VisitObjectsPaused(ObjectCallback callback, void* arg) {
1011 Thread* self = Thread::Current();
1012 Locks::mutator_lock_->AssertExclusiveHeld(self);
1013 VisitObjectsInternalRegionSpace(callback, arg);
1014 VisitObjectsInternal(callback, arg);
1015}
1016
1017// Visit objects in the region spaces.
1018void Heap::VisitObjectsInternalRegionSpace(ObjectCallback callback, void* arg) {
1019 Thread* self = Thread::Current();
1020 Locks::mutator_lock_->AssertExclusiveHeld(self);
1021 if (region_space_ != nullptr) {
1022 DCHECK(IsGcConcurrentAndMoving());
1023 if (!zygote_creation_lock_.IsExclusiveHeld(self)) {
1024 // Exclude the pre-zygote fork time where the semi-space collector
1025 // calls VerifyHeapReferences() as part of the zygote compaction
1026 // which then would call here without the moving GC disabled,
1027 // which is fine.
1028 DCHECK(IsMovingGCDisabled(self));
1029 }
1030 region_space_->Walk(callback, arg);
1031 }
1032}
1033
1034// Visit objects in the other spaces.
Hiroshi Yamauchi0c8c3032015-01-16 16:54:35 -08001035void Heap::VisitObjectsInternal(ObjectCallback callback, void* arg) {
Mathieu Chartier590fee92013-09-13 13:46:47 -07001036 if (bump_pointer_space_ != nullptr) {
Mathieu Chartier692fafd2013-11-29 17:24:40 -08001037 // Visit objects in bump pointer space.
1038 bump_pointer_space_->Walk(callback, arg);
Mathieu Chartier590fee92013-09-13 13:46:47 -07001039 }
1040 // TODO: Switch to standard begin and end to use ranged a based loop.
Mathieu Chartiercb535da2015-01-23 13:50:03 -08001041 for (auto* it = allocation_stack_->Begin(), *end = allocation_stack_->End(); it < end; ++it) {
1042 mirror::Object* const obj = it->AsMirrorPtr();
Mathieu Chartierebdf3f32014-02-13 10:23:27 -08001043 if (obj != nullptr && obj->GetClass() != nullptr) {
1044 // Avoid the race condition caused by the object not yet being written into the allocation
Mathieu Chartiera5eae692014-12-17 17:56:03 -08001045 // stack or the class not yet being written in the object. Or, if
1046 // kUseThreadLocalAllocationStack, there can be nulls on the allocation stack.
Mathieu Chartierebdf3f32014-02-13 10:23:27 -08001047 callback(obj, arg);
1048 }
Mathieu Chartier590fee92013-09-13 13:46:47 -07001049 }
Hiroshi Yamauchi0c8c3032015-01-16 16:54:35 -08001050 {
1051 ReaderMutexLock mu(Thread::Current(), *Locks::heap_bitmap_lock_);
1052 GetLiveBitmap()->Walk(callback, arg);
1053 }
Mathieu Chartier590fee92013-09-13 13:46:47 -07001054}
1055
1056void Heap::MarkAllocStackAsLive(accounting::ObjectStack* stack) {
Mathieu Chartier00b59152014-07-25 10:13:51 -07001057 space::ContinuousSpace* space1 = main_space_ != nullptr ? main_space_ : non_moving_space_;
1058 space::ContinuousSpace* space2 = non_moving_space_;
Mathieu Chartiere6da9af2013-12-16 11:54:42 -08001059 // TODO: Generalize this to n bitmaps?
Mathieu Chartier00b59152014-07-25 10:13:51 -07001060 CHECK(space1 != nullptr);
1061 CHECK(space2 != nullptr);
Mathieu Chartiere6da9af2013-12-16 11:54:42 -08001062 MarkAllocStack(space1->GetLiveBitmap(), space2->GetLiveBitmap(),
Mathieu Chartier2dbe6272014-09-16 10:43:23 -07001063 (large_object_space_ != nullptr ? large_object_space_->GetLiveBitmap() : nullptr),
1064 stack);
Mathieu Chartier590fee92013-09-13 13:46:47 -07001065}
1066
Mathieu Chartier02b6a782012-10-26 13:51:26 -07001067void Heap::DeleteThreadPool() {
Mathieu Chartier2775ee42013-08-20 17:43:47 -07001068 thread_pool_.reset(nullptr);
Mathieu Chartier02b6a782012-10-26 13:51:26 -07001069}
1070
Mathieu Chartier1b54f9c2014-04-30 16:45:02 -07001071void Heap::AddSpace(space::Space* space) {
Zuo Wangf37a88b2014-07-10 04:26:41 -07001072 CHECK(space != nullptr);
Mathieu Chartier590fee92013-09-13 13:46:47 -07001073 WriterMutexLock mu(Thread::Current(), *Locks::heap_bitmap_lock_);
1074 if (space->IsContinuousSpace()) {
1075 DCHECK(!space->IsDiscontinuousSpace());
1076 space::ContinuousSpace* continuous_space = space->AsContinuousSpace();
1077 // Continuous spaces don't necessarily have bitmaps.
Mathieu Chartiera8e8f9c2014-04-09 14:51:05 -07001078 accounting::ContinuousSpaceBitmap* live_bitmap = continuous_space->GetLiveBitmap();
1079 accounting::ContinuousSpaceBitmap* mark_bitmap = continuous_space->GetMarkBitmap();
Mathieu Chartier590fee92013-09-13 13:46:47 -07001080 if (live_bitmap != nullptr) {
Mathieu Chartier2796a162014-07-25 11:50:47 -07001081 CHECK(mark_bitmap != nullptr);
Mathieu Chartier590fee92013-09-13 13:46:47 -07001082 live_bitmap_->AddContinuousSpaceBitmap(live_bitmap);
1083 mark_bitmap_->AddContinuousSpaceBitmap(mark_bitmap);
Mathieu Chartier357e9be2012-08-01 11:00:14 -07001084 }
Mathieu Chartier590fee92013-09-13 13:46:47 -07001085 continuous_spaces_.push_back(continuous_space);
Mathieu Chartier590fee92013-09-13 13:46:47 -07001086 // Ensure that spaces remain sorted in increasing order of start address.
1087 std::sort(continuous_spaces_.begin(), continuous_spaces_.end(),
1088 [](const space::ContinuousSpace* a, const space::ContinuousSpace* b) {
1089 return a->Begin() < b->Begin();
1090 });
Mathieu Chartier590fee92013-09-13 13:46:47 -07001091 } else {
Mathieu Chartier2796a162014-07-25 11:50:47 -07001092 CHECK(space->IsDiscontinuousSpace());
Mathieu Chartier590fee92013-09-13 13:46:47 -07001093 space::DiscontinuousSpace* discontinuous_space = space->AsDiscontinuousSpace();
Mathieu Chartierbbd695c2014-04-16 09:48:48 -07001094 live_bitmap_->AddLargeObjectBitmap(discontinuous_space->GetLiveBitmap());
1095 mark_bitmap_->AddLargeObjectBitmap(discontinuous_space->GetMarkBitmap());
Mathieu Chartier590fee92013-09-13 13:46:47 -07001096 discontinuous_spaces_.push_back(discontinuous_space);
1097 }
1098 if (space->IsAllocSpace()) {
1099 alloc_spaces_.push_back(space->AsAllocSpace());
Mathieu Chartier357e9be2012-08-01 11:00:14 -07001100 }
Elliott Hughesb3bd5f02012-03-08 21:05:27 -08001101}
1102
Mathieu Chartier1b54f9c2014-04-30 16:45:02 -07001103void Heap::SetSpaceAsDefault(space::ContinuousSpace* continuous_space) {
1104 WriterMutexLock mu(Thread::Current(), *Locks::heap_bitmap_lock_);
1105 if (continuous_space->IsDlMallocSpace()) {
1106 dlmalloc_space_ = continuous_space->AsDlMallocSpace();
1107 } else if (continuous_space->IsRosAllocSpace()) {
1108 rosalloc_space_ = continuous_space->AsRosAllocSpace();
1109 }
1110}
1111
1112void Heap::RemoveSpace(space::Space* space) {
Mathieu Chartiere6da9af2013-12-16 11:54:42 -08001113 DCHECK(space != nullptr);
1114 WriterMutexLock mu(Thread::Current(), *Locks::heap_bitmap_lock_);
1115 if (space->IsContinuousSpace()) {
1116 DCHECK(!space->IsDiscontinuousSpace());
1117 space::ContinuousSpace* continuous_space = space->AsContinuousSpace();
1118 // Continuous spaces don't necessarily have bitmaps.
Mathieu Chartiera8e8f9c2014-04-09 14:51:05 -07001119 accounting::ContinuousSpaceBitmap* live_bitmap = continuous_space->GetLiveBitmap();
1120 accounting::ContinuousSpaceBitmap* mark_bitmap = continuous_space->GetMarkBitmap();
Mathieu Chartiere6da9af2013-12-16 11:54:42 -08001121 if (live_bitmap != nullptr) {
1122 DCHECK(mark_bitmap != nullptr);
1123 live_bitmap_->RemoveContinuousSpaceBitmap(live_bitmap);
1124 mark_bitmap_->RemoveContinuousSpaceBitmap(mark_bitmap);
1125 }
1126 auto it = std::find(continuous_spaces_.begin(), continuous_spaces_.end(), continuous_space);
1127 DCHECK(it != continuous_spaces_.end());
1128 continuous_spaces_.erase(it);
Mathieu Chartiere6da9af2013-12-16 11:54:42 -08001129 } else {
1130 DCHECK(space->IsDiscontinuousSpace());
1131 space::DiscontinuousSpace* discontinuous_space = space->AsDiscontinuousSpace();
Mathieu Chartierbbd695c2014-04-16 09:48:48 -07001132 live_bitmap_->RemoveLargeObjectBitmap(discontinuous_space->GetLiveBitmap());
1133 mark_bitmap_->RemoveLargeObjectBitmap(discontinuous_space->GetMarkBitmap());
Mathieu Chartiere6da9af2013-12-16 11:54:42 -08001134 auto it = std::find(discontinuous_spaces_.begin(), discontinuous_spaces_.end(),
1135 discontinuous_space);
1136 DCHECK(it != discontinuous_spaces_.end());
1137 discontinuous_spaces_.erase(it);
1138 }
1139 if (space->IsAllocSpace()) {
1140 auto it = std::find(alloc_spaces_.begin(), alloc_spaces_.end(), space->AsAllocSpace());
1141 DCHECK(it != alloc_spaces_.end());
1142 alloc_spaces_.erase(it);
1143 }
1144}
1145
Elliott Hughes8b788fe2013-04-17 15:57:01 -07001146void Heap::DumpGcPerformanceInfo(std::ostream& os) {
Mathieu Chartier155dfe92012-10-09 14:24:49 -07001147 // Dump cumulative timings.
Elliott Hughes8b788fe2013-04-17 15:57:01 -07001148 os << "Dumping cumulative Gc timings\n";
Mathieu Chartier155dfe92012-10-09 14:24:49 -07001149 uint64_t total_duration = 0;
Mathieu Chartier2b82db42012-11-14 17:29:05 -08001150 // Dump cumulative loggers for each GC type.
Mathieu Chartier2b82db42012-11-14 17:29:05 -08001151 uint64_t total_paused_time = 0;
Mathieu Chartier5a487192014-04-08 11:14:54 -07001152 for (auto& collector : garbage_collectors_) {
Mathieu Chartier104fa0c2014-08-07 14:26:27 -07001153 total_duration += collector->GetCumulativeTimings().GetTotalNs();
1154 total_paused_time += collector->GetTotalPausedTimeNs();
1155 collector->DumpPerformanceInfo(os);
Mathieu Chartier155dfe92012-10-09 14:24:49 -07001156 }
Mathieu Chartier155dfe92012-10-09 14:24:49 -07001157 if (total_duration != 0) {
Brian Carlstrom2d888622013-07-18 17:02:00 -07001158 const double total_seconds = static_cast<double>(total_duration / 1000) / 1000000.0;
Elliott Hughes8b788fe2013-04-17 15:57:01 -07001159 os << "Total time spent in GC: " << PrettyDuration(total_duration) << "\n";
1160 os << "Mean GC size throughput: "
Ian Rogers1d54e732013-05-02 21:10:01 -07001161 << PrettySize(GetBytesFreedEver() / total_seconds) << "/s\n";
Elliott Hughes8b788fe2013-04-17 15:57:01 -07001162 os << "Mean GC object throughput: "
Ian Rogers1d54e732013-05-02 21:10:01 -07001163 << (GetObjectsFreedEver() / total_seconds) << " objects/s\n";
Mathieu Chartier155dfe92012-10-09 14:24:49 -07001164 }
Mathieu Chartierdd162fb2014-08-06 17:06:33 -07001165 uint64_t total_objects_allocated = GetObjectsAllocatedEver();
Mathieu Chartierc30a7252014-08-12 10:13:48 -07001166 os << "Total number of allocations " << total_objects_allocated << "\n";
Hiroshi Yamauchia1c9f012015-04-02 10:18:12 -07001167 os << "Total bytes allocated " << PrettySize(GetBytesAllocatedEver()) << "\n";
1168 os << "Total bytes freed " << PrettySize(GetBytesFreedEver()) << "\n";
Mathieu Chartierc30a7252014-08-12 10:13:48 -07001169 os << "Free memory " << PrettySize(GetFreeMemory()) << "\n";
Mathieu Chartierdd162fb2014-08-06 17:06:33 -07001170 os << "Free memory until GC " << PrettySize(GetFreeMemoryUntilGC()) << "\n";
1171 os << "Free memory until OOME " << PrettySize(GetFreeMemoryUntilOOME()) << "\n";
Mathieu Chartierc30a7252014-08-12 10:13:48 -07001172 os << "Total memory " << PrettySize(GetTotalMemory()) << "\n";
1173 os << "Max memory " << PrettySize(GetMaxMemory()) << "\n";
Mathieu Chartiere4cab172014-08-19 18:24:04 -07001174 if (HasZygoteSpace()) {
1175 os << "Zygote space size " << PrettySize(zygote_space_->Size()) << "\n";
1176 }
Elliott Hughes8b788fe2013-04-17 15:57:01 -07001177 os << "Total mutator paused time: " << PrettyDuration(total_paused_time) << "\n";
Hiroshi Yamauchia1c9f012015-04-02 10:18:12 -07001178 os << "Total time waiting for GC to complete: " << PrettyDuration(total_wait_time_) << "\n";
1179 os << "Total GC count: " << GetGcCount() << "\n";
1180 os << "Total GC time: " << PrettyDuration(GetGcTime()) << "\n";
1181 os << "Total blocking GC count: " << GetBlockingGcCount() << "\n";
1182 os << "Total blocking GC time: " << PrettyDuration(GetBlockingGcTime()) << "\n";
1183
1184 {
1185 MutexLock mu(Thread::Current(), *gc_complete_lock_);
1186 if (gc_count_rate_histogram_.SampleSize() > 0U) {
1187 os << "Histogram of GC count per " << NsToMs(kGcCountRateHistogramWindowDuration) << " ms: ";
1188 gc_count_rate_histogram_.DumpBins(os);
1189 os << "\n";
1190 }
1191 if (blocking_gc_count_rate_histogram_.SampleSize() > 0U) {
1192 os << "Histogram of blocking GC count per "
1193 << NsToMs(kGcCountRateHistogramWindowDuration) << " ms: ";
1194 blocking_gc_count_rate_histogram_.DumpBins(os);
1195 os << "\n";
1196 }
1197 }
1198
Hiroshi Yamauchib62f2e62016-03-23 15:51:24 -07001199 if (kDumpRosAllocStatsOnSigQuit && rosalloc_space_ != nullptr) {
1200 rosalloc_space_->DumpStats(os);
1201 }
1202
Mathieu Chartier5d2a3f72016-05-11 11:35:39 -07001203 {
1204 MutexLock mu(Thread::Current(), native_histogram_lock_);
1205 if (native_allocation_histogram_.SampleSize() > 0u) {
1206 os << "Histogram of native allocation ";
1207 native_allocation_histogram_.DumpBins(os);
1208 os << " bucket size " << native_allocation_histogram_.BucketWidth() << "\n";
1209 }
1210 if (native_free_histogram_.SampleSize() > 0u) {
1211 os << "Histogram of native free ";
1212 native_free_histogram_.DumpBins(os);
1213 os << " bucket size " << native_free_histogram_.BucketWidth() << "\n";
1214 }
1215 }
1216
Mathieu Chartier73d1e172014-04-11 17:53:48 -07001217 BaseMutex::DumpAll(os);
Mathieu Chartier155dfe92012-10-09 14:24:49 -07001218}
1219
Hiroshi Yamauchi37670172015-06-10 17:20:54 -07001220void Heap::ResetGcPerformanceInfo() {
1221 for (auto& collector : garbage_collectors_) {
1222 collector->ResetMeasurements();
1223 }
Hiroshi Yamauchi37670172015-06-10 17:20:54 -07001224 total_bytes_freed_ever_ = 0;
1225 total_objects_freed_ever_ = 0;
1226 total_wait_time_ = 0;
1227 blocking_gc_count_ = 0;
1228 blocking_gc_time_ = 0;
1229 gc_count_last_window_ = 0;
1230 blocking_gc_count_last_window_ = 0;
1231 last_update_time_gc_count_rate_histograms_ = // Round down by the window duration.
1232 (NanoTime() / kGcCountRateHistogramWindowDuration) * kGcCountRateHistogramWindowDuration;
1233 {
1234 MutexLock mu(Thread::Current(), *gc_complete_lock_);
1235 gc_count_rate_histogram_.Reset();
1236 blocking_gc_count_rate_histogram_.Reset();
1237 }
1238}
1239
Hiroshi Yamauchia1c9f012015-04-02 10:18:12 -07001240uint64_t Heap::GetGcCount() const {
1241 uint64_t gc_count = 0U;
1242 for (auto& collector : garbage_collectors_) {
1243 gc_count += collector->GetCumulativeTimings().GetIterations();
1244 }
1245 return gc_count;
1246}
1247
1248uint64_t Heap::GetGcTime() const {
1249 uint64_t gc_time = 0U;
1250 for (auto& collector : garbage_collectors_) {
1251 gc_time += collector->GetCumulativeTimings().GetTotalNs();
1252 }
1253 return gc_time;
1254}
1255
1256uint64_t Heap::GetBlockingGcCount() const {
1257 return blocking_gc_count_;
1258}
1259
1260uint64_t Heap::GetBlockingGcTime() const {
1261 return blocking_gc_time_;
1262}
1263
1264void Heap::DumpGcCountRateHistogram(std::ostream& os) const {
1265 MutexLock mu(Thread::Current(), *gc_complete_lock_);
1266 if (gc_count_rate_histogram_.SampleSize() > 0U) {
1267 gc_count_rate_histogram_.DumpBins(os);
1268 }
1269}
1270
1271void Heap::DumpBlockingGcCountRateHistogram(std::ostream& os) const {
1272 MutexLock mu(Thread::Current(), *gc_complete_lock_);
1273 if (blocking_gc_count_rate_histogram_.SampleSize() > 0U) {
1274 blocking_gc_count_rate_histogram_.DumpBins(os);
1275 }
1276}
1277
Elliott Hughesb3bd5f02012-03-08 21:05:27 -08001278Heap::~Heap() {
Mathieu Chartier590fee92013-09-13 13:46:47 -07001279 VLOG(heap) << "Starting ~Heap()";
Mathieu Chartier590fee92013-09-13 13:46:47 -07001280 STLDeleteElements(&garbage_collectors_);
1281 // If we don't reset then the mark stack complains in its destructor.
Mathieu Chartier357e9be2012-08-01 11:00:14 -07001282 allocation_stack_->Reset();
Man Cao8c2ff642015-05-27 17:25:30 -07001283 allocation_records_.reset();
Mathieu Chartier357e9be2012-08-01 11:00:14 -07001284 live_stack_->Reset();
Mathieu Chartier11409ae2013-09-23 11:49:36 -07001285 STLDeleteValues(&mod_union_tables_);
Mathieu Chartier0767c9a2014-03-26 12:53:19 -07001286 STLDeleteValues(&remembered_sets_);
Ian Rogers1d54e732013-05-02 21:10:01 -07001287 STLDeleteElements(&continuous_spaces_);
1288 STLDeleteElements(&discontinuous_spaces_);
Ian Rogers00f7d0e2012-07-19 15:28:27 -07001289 delete gc_complete_lock_;
Andreas Gampe6be4f2a2015-11-10 13:34:17 -08001290 delete thread_flip_lock_;
Mathieu Chartiera5eae692014-12-17 17:56:03 -08001291 delete pending_task_lock_;
Mathieu Chartier31000802015-06-14 14:14:37 -07001292 delete backtrace_lock_;
1293 if (unique_backtrace_count_.LoadRelaxed() != 0 || seen_backtrace_count_.LoadRelaxed() != 0) {
1294 LOG(INFO) << "gc stress unique=" << unique_backtrace_count_.LoadRelaxed()
1295 << " total=" << seen_backtrace_count_.LoadRelaxed() +
1296 unique_backtrace_count_.LoadRelaxed();
1297 }
Mathieu Chartier590fee92013-09-13 13:46:47 -07001298 VLOG(heap) << "Finished ~Heap()";
Carl Shapiro69759ea2011-07-21 18:13:35 -07001299}
1300
Ian Rogers1d54e732013-05-02 21:10:01 -07001301space::ContinuousSpace* Heap::FindContinuousSpaceFromObject(const mirror::Object* obj,
1302 bool fail_ok) const {
Mathieu Chartier02e25112013-08-14 16:14:24 -07001303 for (const auto& space : continuous_spaces_) {
1304 if (space->Contains(obj)) {
1305 return space;
Mathieu Chartierb062fdd2012-07-03 09:51:48 -07001306 }
1307 }
Ian Rogers1d54e732013-05-02 21:10:01 -07001308 if (!fail_ok) {
1309 LOG(FATAL) << "object " << reinterpret_cast<const void*>(obj) << " not inside any spaces!";
1310 }
Mathieu Chartier2cebb242015-04-21 16:50:40 -07001311 return nullptr;
Mathieu Chartierb062fdd2012-07-03 09:51:48 -07001312}
1313
Ian Rogers1d54e732013-05-02 21:10:01 -07001314space::DiscontinuousSpace* Heap::FindDiscontinuousSpaceFromObject(const mirror::Object* obj,
1315 bool fail_ok) const {
Mathieu Chartier02e25112013-08-14 16:14:24 -07001316 for (const auto& space : discontinuous_spaces_) {
1317 if (space->Contains(obj)) {
1318 return space;
Ian Rogers1d54e732013-05-02 21:10:01 -07001319 }
1320 }
1321 if (!fail_ok) {
1322 LOG(FATAL) << "object " << reinterpret_cast<const void*>(obj) << " not inside any spaces!";
1323 }
Mathieu Chartier2cebb242015-04-21 16:50:40 -07001324 return nullptr;
Ian Rogers1d54e732013-05-02 21:10:01 -07001325}
1326
1327space::Space* Heap::FindSpaceFromObject(const mirror::Object* obj, bool fail_ok) const {
1328 space::Space* result = FindContinuousSpaceFromObject(obj, true);
Mathieu Chartier2cebb242015-04-21 16:50:40 -07001329 if (result != nullptr) {
Ian Rogers1d54e732013-05-02 21:10:01 -07001330 return result;
1331 }
Ian Rogers6a3c1fc2014-10-31 00:33:20 -07001332 return FindDiscontinuousSpaceFromObject(obj, fail_ok);
Ian Rogers1d54e732013-05-02 21:10:01 -07001333}
1334
Hiroshi Yamauchi654dd482014-07-09 12:54:32 -07001335void Heap::ThrowOutOfMemoryError(Thread* self, size_t byte_count, AllocatorType allocator_type) {
Mathieu Chartiere8f3f032016-04-04 16:49:44 -07001336 // If we're in a stack overflow, do not create a new exception. It would require running the
1337 // constructor, which will of course still be in a stack overflow.
1338 if (self->IsHandlingStackOverflow()) {
1339 self->SetException(Runtime::Current()->GetPreAllocatedOutOfMemoryError());
1340 return;
1341 }
1342
Hiroshi Yamauchi3b4c1892013-09-12 21:33:12 -07001343 std::ostringstream oss;
Ian Rogersef7d42f2014-01-06 12:55:46 -08001344 size_t total_bytes_free = GetFreeMemory();
Hiroshi Yamauchi3b4c1892013-09-12 21:33:12 -07001345 oss << "Failed to allocate a " << byte_count << " byte allocation with " << total_bytes_free
Mathieu Chartierdd162fb2014-08-06 17:06:33 -07001346 << " free bytes and " << PrettySize(GetFreeMemoryUntilOOME()) << " until OOM";
Hiroshi Yamauchi3b4c1892013-09-12 21:33:12 -07001347 // If the allocation failed due to fragmentation, print out the largest continuous allocation.
Zuo Wangf37a88b2014-07-10 04:26:41 -07001348 if (total_bytes_free >= byte_count) {
Mathieu Chartierb363f662014-07-16 13:28:58 -07001349 space::AllocSpace* space = nullptr;
Hiroshi Yamauchi654dd482014-07-09 12:54:32 -07001350 if (allocator_type == kAllocatorTypeNonMoving) {
1351 space = non_moving_space_;
1352 } else if (allocator_type == kAllocatorTypeRosAlloc ||
1353 allocator_type == kAllocatorTypeDlMalloc) {
1354 space = main_space_;
Mathieu Chartierb363f662014-07-16 13:28:58 -07001355 } else if (allocator_type == kAllocatorTypeBumpPointer ||
1356 allocator_type == kAllocatorTypeTLAB) {
1357 space = bump_pointer_space_;
Hiroshi Yamauchi2cd334a2015-01-09 14:03:35 -08001358 } else if (allocator_type == kAllocatorTypeRegion ||
1359 allocator_type == kAllocatorTypeRegionTLAB) {
1360 space = region_space_;
Hiroshi Yamauchi3b4c1892013-09-12 21:33:12 -07001361 }
Hiroshi Yamauchi654dd482014-07-09 12:54:32 -07001362 if (space != nullptr) {
1363 space->LogFragmentationAllocFailure(oss, byte_count);
1364 }
Mathieu Chartier155dfe92012-10-09 14:24:49 -07001365 }
Hiroshi Yamauchi3b4c1892013-09-12 21:33:12 -07001366 self->ThrowOutOfMemoryError(oss.str().c_str());
1367}
Mathieu Chartiere0f0cb32012-08-28 11:26:00 -07001368
Mathieu Chartiera5eae692014-12-17 17:56:03 -08001369void Heap::DoPendingCollectorTransition() {
1370 CollectorType desired_collector_type = desired_collector_type_;
Mathieu Chartierb2728552014-09-08 20:08:41 +00001371 // Launch homogeneous space compaction if it is desired.
1372 if (desired_collector_type == kCollectorTypeHomogeneousSpaceCompact) {
1373 if (!CareAboutPauseTimes()) {
1374 PerformHomogeneousSpaceCompact();
Mathieu Chartiera5eae692014-12-17 17:56:03 -08001375 } else {
1376 VLOG(gc) << "Homogeneous compaction ignored due to jank perceptible process state";
Mathieu Chartierb2728552014-09-08 20:08:41 +00001377 }
Mathieu Chartiera5eae692014-12-17 17:56:03 -08001378 } else {
1379 TransitionCollector(desired_collector_type);
Mathieu Chartierb2728552014-09-08 20:08:41 +00001380 }
Mathieu Chartiera5eae692014-12-17 17:56:03 -08001381}
1382
1383void Heap::Trim(Thread* self) {
Mathieu Chartier8d447252015-10-26 10:21:14 -07001384 Runtime* const runtime = Runtime::Current();
Mathieu Chartier440e4ce2014-03-31 16:36:35 -07001385 if (!CareAboutPauseTimes()) {
1386 // Deflate the monitors, this can cause a pause but shouldn't matter since we don't care
1387 // about pauses.
Mathieu Chartier32ce2ad2016-03-04 14:58:03 -08001388 ScopedTrace trace("Deflating monitors");
1389 ScopedSuspendAll ssa(__FUNCTION__);
1390 uint64_t start_time = NanoTime();
1391 size_t count = runtime->GetMonitorList()->DeflateMonitors();
1392 VLOG(heap) << "Deflating " << count << " monitors took "
1393 << PrettyDuration(NanoTime() - start_time);
Mathieu Chartier440e4ce2014-03-31 16:36:35 -07001394 }
Mathieu Chartiera5eae692014-12-17 17:56:03 -08001395 TrimIndirectReferenceTables(self);
1396 TrimSpaces(self);
Mathieu Chartier8d447252015-10-26 10:21:14 -07001397 // Trim arenas that may have been used by JIT or verifier.
Mathieu Chartier8d447252015-10-26 10:21:14 -07001398 runtime->GetArenaPool()->TrimMaps();
Mathieu Chartiera5f9de02014-02-28 16:48:42 -08001399}
1400
Mathieu Chartier91c2f0c2014-11-26 11:21:15 -08001401class TrimIndirectReferenceTableClosure : public Closure {
1402 public:
1403 explicit TrimIndirectReferenceTableClosure(Barrier* barrier) : barrier_(barrier) {
1404 }
1405 virtual void Run(Thread* thread) OVERRIDE NO_THREAD_SAFETY_ANALYSIS {
Mathieu Chartier91c2f0c2014-11-26 11:21:15 -08001406 thread->GetJniEnv()->locals.Trim();
Lei Lidd9943d2015-02-02 14:24:44 +08001407 // If thread is a running mutator, then act on behalf of the trim thread.
1408 // See the code in ThreadList::RunCheckpoint.
Mathieu Chartier10d25082015-10-28 18:36:09 -07001409 barrier_->Pass(Thread::Current());
Mathieu Chartier91c2f0c2014-11-26 11:21:15 -08001410 }
1411
1412 private:
1413 Barrier* const barrier_;
1414};
1415
Mathieu Chartiera5eae692014-12-17 17:56:03 -08001416void Heap::TrimIndirectReferenceTables(Thread* self) {
1417 ScopedObjectAccess soa(self);
Mathieu Chartier32ce2ad2016-03-04 14:58:03 -08001418 ScopedTrace trace(__PRETTY_FUNCTION__);
Mathieu Chartiera5eae692014-12-17 17:56:03 -08001419 JavaVMExt* vm = soa.Vm();
1420 // Trim globals indirect reference table.
1421 vm->TrimGlobals();
1422 // Trim locals indirect reference tables.
1423 Barrier barrier(0);
1424 TrimIndirectReferenceTableClosure closure(&barrier);
1425 ScopedThreadStateChange tsc(self, kWaitingForCheckPointsToRun);
1426 size_t barrier_count = Runtime::Current()->GetThreadList()->RunCheckpoint(&closure);
Lei Lidd9943d2015-02-02 14:24:44 +08001427 if (barrier_count != 0) {
1428 barrier.Increment(self, barrier_count);
1429 }
Mathieu Chartiera5eae692014-12-17 17:56:03 -08001430}
Mathieu Chartier91c2f0c2014-11-26 11:21:15 -08001431
Mathieu Chartieraa516822015-10-02 15:53:37 -07001432void Heap::StartGC(Thread* self, GcCause cause, CollectorType collector_type) {
1433 MutexLock mu(self, *gc_complete_lock_);
1434 // Ensure there is only one GC at a time.
1435 WaitForGcToCompleteLocked(cause, self);
1436 collector_type_running_ = collector_type;
1437}
1438
Mathieu Chartiera5eae692014-12-17 17:56:03 -08001439void Heap::TrimSpaces(Thread* self) {
Mathieu Chartiera5f9de02014-02-28 16:48:42 -08001440 {
Mathieu Chartiercaa82d62014-02-02 16:51:17 -08001441 // Need to do this before acquiring the locks since we don't want to get suspended while
1442 // holding any locks.
1443 ScopedThreadStateChange tsc(self, kWaitingForGcToComplete);
Mathieu Chartierd5a89ee2014-01-31 09:55:13 -08001444 // Pretend we are doing a GC to prevent background compaction from deleting the space we are
1445 // trimming.
Mathieu Chartieraa516822015-10-02 15:53:37 -07001446 StartGC(self, kGcCauseTrim, kCollectorTypeHeapTrim);
Mathieu Chartierd5a89ee2014-01-31 09:55:13 -08001447 }
Mathieu Chartier32ce2ad2016-03-04 14:58:03 -08001448 ScopedTrace trace(__PRETTY_FUNCTION__);
Mathieu Chartiera5eae692014-12-17 17:56:03 -08001449 const uint64_t start_ns = NanoTime();
Mathieu Chartier590fee92013-09-13 13:46:47 -07001450 // Trim the managed spaces.
1451 uint64_t total_alloc_space_allocated = 0;
1452 uint64_t total_alloc_space_size = 0;
1453 uint64_t managed_reclaimed = 0;
Mathieu Chartiera9d82fe2016-01-25 20:06:11 -08001454 {
1455 ScopedObjectAccess soa(self);
1456 for (const auto& space : continuous_spaces_) {
1457 if (space->IsMallocSpace()) {
1458 gc::space::MallocSpace* malloc_space = space->AsMallocSpace();
1459 if (malloc_space->IsRosAllocSpace() || !CareAboutPauseTimes()) {
1460 // Don't trim dlmalloc spaces if we care about pauses since this can hold the space lock
1461 // for a long period of time.
1462 managed_reclaimed += malloc_space->Trim();
1463 }
1464 total_alloc_space_size += malloc_space->Size();
Mathieu Chartiera5b5c552014-06-24 14:48:59 -07001465 }
Mathieu Chartier590fee92013-09-13 13:46:47 -07001466 }
1467 }
Mathieu Chartier2dbe6272014-09-16 10:43:23 -07001468 total_alloc_space_allocated = GetBytesAllocated();
1469 if (large_object_space_ != nullptr) {
1470 total_alloc_space_allocated -= large_object_space_->GetBytesAllocated();
1471 }
Mathieu Chartier31f44142014-04-08 14:40:03 -07001472 if (bump_pointer_space_ != nullptr) {
1473 total_alloc_space_allocated -= bump_pointer_space_->Size();
1474 }
Hiroshi Yamauchi2cd334a2015-01-09 14:03:35 -08001475 if (region_space_ != nullptr) {
1476 total_alloc_space_allocated -= region_space_->GetBytesAllocated();
1477 }
Mathieu Chartier590fee92013-09-13 13:46:47 -07001478 const float managed_utilization = static_cast<float>(total_alloc_space_allocated) /
1479 static_cast<float>(total_alloc_space_size);
1480 uint64_t gc_heap_end_ns = NanoTime();
Mathieu Chartierd5a89ee2014-01-31 09:55:13 -08001481 // We never move things in the native heap, so we can finish the GC at this point.
1482 FinishGC(self, collector::kGcTypeNone);
Ian Rogers872dd822014-10-30 11:19:14 -07001483
Mathieu Chartier590fee92013-09-13 13:46:47 -07001484 VLOG(heap) << "Heap trim of managed (duration=" << PrettyDuration(gc_heap_end_ns - start_ns)
Dimitry Ivanove6465bc2015-12-14 18:55:02 -08001485 << ", advised=" << PrettySize(managed_reclaimed) << ") heap. Managed heap utilization of "
1486 << static_cast<int>(100 * managed_utilization) << "%.";
Mathieu Chartier590fee92013-09-13 13:46:47 -07001487}
1488
1489bool Heap::IsValidObjectAddress(const mirror::Object* obj) const {
1490 // Note: we deliberately don't take the lock here, and mustn't test anything that would require
1491 // taking the lock.
1492 if (obj == nullptr) {
Elliott Hughes88c5c352012-03-15 18:49:48 -07001493 return true;
1494 }
Mathieu Chartier15d34022014-02-26 17:16:38 -08001495 return IsAligned<kObjectAlignment>(obj) && FindSpaceFromObject(obj, true) != nullptr;
Mathieu Chartier590fee92013-09-13 13:46:47 -07001496}
1497
Mathieu Chartierd68ac702014-02-11 14:50:51 -08001498bool Heap::IsNonDiscontinuousSpaceHeapAddress(const mirror::Object* obj) const {
1499 return FindContinuousSpaceFromObject(obj, true) != nullptr;
1500}
1501
Mathieu Chartier15d34022014-02-26 17:16:38 -08001502bool Heap::IsValidContinuousSpaceObjectAddress(const mirror::Object* obj) const {
1503 if (obj == nullptr || !IsAligned<kObjectAlignment>(obj)) {
1504 return false;
1505 }
1506 for (const auto& space : continuous_spaces_) {
1507 if (space->HasAddress(obj)) {
1508 return true;
1509 }
1510 }
1511 return false;
Elliott Hughesa2501992011-08-26 19:39:54 -07001512}
1513
Ian Rogersef7d42f2014-01-06 12:55:46 -08001514bool Heap::IsLiveObjectLocked(mirror::Object* obj, bool search_allocation_stack,
Mathieu Chartier0f72e412013-09-06 16:40:01 -07001515 bool search_live_stack, bool sorted) {
Mathieu Chartier938a03b2014-01-16 15:10:31 -08001516 if (UNLIKELY(!IsAligned<kObjectAlignment>(obj))) {
1517 return false;
1518 }
1519 if (bump_pointer_space_ != nullptr && bump_pointer_space_->HasAddress(obj)) {
Mathieu Chartier4e305412014-02-19 10:54:44 -08001520 mirror::Class* klass = obj->GetClass<kVerifyNone>();
Mathieu Chartier938a03b2014-01-16 15:10:31 -08001521 if (obj == klass) {
Mathieu Chartier9be9a7a2014-01-24 14:07:33 -08001522 // This case happens for java.lang.Class.
Mathieu Chartier938a03b2014-01-16 15:10:31 -08001523 return true;
1524 }
1525 return VerifyClassClass(klass) && IsLiveObjectLocked(klass);
1526 } else if (temp_space_ != nullptr && temp_space_->HasAddress(obj)) {
Mathieu Chartier4e305412014-02-19 10:54:44 -08001527 // If we are in the allocated region of the temp space, then we are probably live (e.g. during
1528 // a GC). When a GC isn't running End() - Begin() is 0 which means no objects are contained.
1529 return temp_space_->Contains(obj);
Ian Rogers1d54e732013-05-02 21:10:01 -07001530 }
Hiroshi Yamauchi2cd334a2015-01-09 14:03:35 -08001531 if (region_space_ != nullptr && region_space_->HasAddress(obj)) {
1532 return true;
1533 }
Mathieu Chartierf082d3c2013-07-29 17:04:07 -07001534 space::ContinuousSpace* c_space = FindContinuousSpaceFromObject(obj, true);
Mathieu Chartierbbd695c2014-04-16 09:48:48 -07001535 space::DiscontinuousSpace* d_space = nullptr;
Mathieu Chartier938a03b2014-01-16 15:10:31 -08001536 if (c_space != nullptr) {
Mathieu Chartierf082d3c2013-07-29 17:04:07 -07001537 if (c_space->GetLiveBitmap()->Test(obj)) {
Ian Rogers1d54e732013-05-02 21:10:01 -07001538 return true;
1539 }
1540 } else {
Mathieu Chartierf082d3c2013-07-29 17:04:07 -07001541 d_space = FindDiscontinuousSpaceFromObject(obj, true);
Mathieu Chartier938a03b2014-01-16 15:10:31 -08001542 if (d_space != nullptr) {
Mathieu Chartierbbd695c2014-04-16 09:48:48 -07001543 if (d_space->GetLiveBitmap()->Test(obj)) {
Ian Rogers1d54e732013-05-02 21:10:01 -07001544 return true;
1545 }
1546 }
1547 }
Mathieu Chartierf082d3c2013-07-29 17:04:07 -07001548 // This is covering the allocation/live stack swapping that is done without mutators suspended.
Mathieu Chartier0f72e412013-09-06 16:40:01 -07001549 for (size_t i = 0; i < (sorted ? 1 : 5); ++i) {
1550 if (i > 0) {
1551 NanoSleep(MsToNs(10));
Ian Rogers1d54e732013-05-02 21:10:01 -07001552 }
Mathieu Chartier0f72e412013-09-06 16:40:01 -07001553 if (search_allocation_stack) {
1554 if (sorted) {
Mathieu Chartier407f7022014-02-18 14:37:05 -08001555 if (allocation_stack_->ContainsSorted(obj)) {
Mathieu Chartier0f72e412013-09-06 16:40:01 -07001556 return true;
1557 }
Mathieu Chartier407f7022014-02-18 14:37:05 -08001558 } else if (allocation_stack_->Contains(obj)) {
Mathieu Chartier0f72e412013-09-06 16:40:01 -07001559 return true;
1560 }
1561 }
1562
1563 if (search_live_stack) {
1564 if (sorted) {
Mathieu Chartier407f7022014-02-18 14:37:05 -08001565 if (live_stack_->ContainsSorted(obj)) {
Mathieu Chartier0f72e412013-09-06 16:40:01 -07001566 return true;
1567 }
Mathieu Chartier407f7022014-02-18 14:37:05 -08001568 } else if (live_stack_->Contains(obj)) {
Mathieu Chartier0f72e412013-09-06 16:40:01 -07001569 return true;
1570 }
1571 }
Ian Rogers1d54e732013-05-02 21:10:01 -07001572 }
Mathieu Chartierf082d3c2013-07-29 17:04:07 -07001573 // We need to check the bitmaps again since there is a race where we mark something as live and
1574 // then clear the stack containing it.
Mathieu Chartier938a03b2014-01-16 15:10:31 -08001575 if (c_space != nullptr) {
Mathieu Chartierf082d3c2013-07-29 17:04:07 -07001576 if (c_space->GetLiveBitmap()->Test(obj)) {
1577 return true;
1578 }
1579 } else {
1580 d_space = FindDiscontinuousSpaceFromObject(obj, true);
Mathieu Chartierbbd695c2014-04-16 09:48:48 -07001581 if (d_space != nullptr && d_space->GetLiveBitmap()->Test(obj)) {
Mathieu Chartierf082d3c2013-07-29 17:04:07 -07001582 return true;
1583 }
1584 }
Ian Rogers1d54e732013-05-02 21:10:01 -07001585 return false;
Elliott Hughes6a5bd492011-10-28 14:33:57 -07001586}
1587
Mathieu Chartier4c13a3f2014-07-14 14:57:16 -07001588std::string Heap::DumpSpaces() const {
1589 std::ostringstream oss;
1590 DumpSpaces(oss);
1591 return oss.str();
1592}
1593
1594void Heap::DumpSpaces(std::ostream& stream) const {
Mathieu Chartier02e25112013-08-14 16:14:24 -07001595 for (const auto& space : continuous_spaces_) {
Mathieu Chartiera8e8f9c2014-04-09 14:51:05 -07001596 accounting::ContinuousSpaceBitmap* live_bitmap = space->GetLiveBitmap();
1597 accounting::ContinuousSpaceBitmap* mark_bitmap = space->GetMarkBitmap();
Mathieu Chartier590fee92013-09-13 13:46:47 -07001598 stream << space << " " << *space << "\n";
1599 if (live_bitmap != nullptr) {
1600 stream << live_bitmap << " " << *live_bitmap << "\n";
1601 }
1602 if (mark_bitmap != nullptr) {
1603 stream << mark_bitmap << " " << *mark_bitmap << "\n";
1604 }
Mathieu Chartiercc236d72012-07-20 10:29:05 -07001605 }
Mathieu Chartier02e25112013-08-14 16:14:24 -07001606 for (const auto& space : discontinuous_spaces_) {
Mathieu Chartier590fee92013-09-13 13:46:47 -07001607 stream << space << " " << *space << "\n";
Mathieu Chartier128c52c2012-10-16 14:12:41 -07001608 }
Mathieu Chartiercc236d72012-07-20 10:29:05 -07001609}
1610
Ian Rogersef7d42f2014-01-06 12:55:46 -08001611void Heap::VerifyObjectBody(mirror::Object* obj) {
Stephen Hines22c6a812014-07-16 11:03:43 -07001612 if (verify_object_mode_ == kVerifyObjectModeDisabled) {
1613 return;
1614 }
1615
Mathieu Chartier0f72e412013-09-06 16:40:01 -07001616 // Ignore early dawn of the universe verifications.
Ian Rogers3e5cf302014-05-20 16:40:37 -07001617 if (UNLIKELY(static_cast<size_t>(num_bytes_allocated_.LoadRelaxed()) < 10 * KB)) {
Ian Rogers62d6c772013-02-27 08:32:07 -08001618 return;
1619 }
Roland Levillain14d90572015-07-16 10:52:26 +01001620 CHECK_ALIGNED(obj, kObjectAlignment) << "Object isn't aligned";
Ian Rogersb0fa5dc2014-04-28 16:47:08 -07001621 mirror::Class* c = obj->GetFieldObject<mirror::Class, kVerifyNone>(mirror::Object::ClassOffset());
Mathieu Chartier4e305412014-02-19 10:54:44 -08001622 CHECK(c != nullptr) << "Null class in object " << obj;
Roland Levillain14d90572015-07-16 10:52:26 +01001623 CHECK_ALIGNED(c, kObjectAlignment) << "Class " << c << " not aligned in object " << obj;
Mathieu Chartier938a03b2014-01-16 15:10:31 -08001624 CHECK(VerifyClassClass(c));
Mathieu Chartier0325e622012-09-05 14:22:51 -07001625
Mathieu Chartier4e305412014-02-19 10:54:44 -08001626 if (verify_object_mode_ > kVerifyObjectModeFast) {
1627 // Note: the bitmap tests below are racy since we don't hold the heap bitmap lock.
Mathieu Chartier4c13a3f2014-07-14 14:57:16 -07001628 CHECK(IsLiveObjectLocked(obj)) << "Object is dead " << obj << "\n" << DumpSpaces();
Mathieu Chartierdcf8d722012-08-02 14:55:54 -07001629 }
Ian Rogers0cfe1fb2011-08-26 03:29:44 -07001630}
1631
Ian Rogers2dd0e2c2013-01-24 12:42:14 -08001632void Heap::VerificationCallback(mirror::Object* obj, void* arg) {
Mathieu Chartier357e9be2012-08-01 11:00:14 -07001633 reinterpret_cast<Heap*>(arg)->VerifyObjectBody(obj);
Ian Rogers0cfe1fb2011-08-26 03:29:44 -07001634}
1635
1636void Heap::VerifyHeap() {
Ian Rogers50b35e22012-10-04 10:09:15 -07001637 ReaderMutexLock mu(Thread::Current(), *Locks::heap_bitmap_lock_);
Mathieu Chartierb062fdd2012-07-03 09:51:48 -07001638 GetLiveBitmap()->Walk(Heap::VerificationCallback, this);
Ian Rogers0cfe1fb2011-08-26 03:29:44 -07001639}
1640
Mathieu Chartiere76e70f2014-05-02 16:35:37 -07001641void Heap::RecordFree(uint64_t freed_objects, int64_t freed_bytes) {
Mathieu Chartier601276a2014-03-20 15:12:30 -07001642 // Use signed comparison since freed bytes can be negative when background compaction foreground
1643 // transitions occurs. This is caused by the moving objects from a bump pointer space to a
1644 // free list backed space typically increasing memory footprint due to padding and binning.
Ian Rogers3e5cf302014-05-20 16:40:37 -07001645 DCHECK_LE(freed_bytes, static_cast<int64_t>(num_bytes_allocated_.LoadRelaxed()));
Mathieu Chartiere76e70f2014-05-02 16:35:37 -07001646 // Note: This relies on 2s complement for handling negative freed_bytes.
Ian Rogers3e5cf302014-05-20 16:40:37 -07001647 num_bytes_allocated_.FetchAndSubSequentiallyConsistent(static_cast<ssize_t>(freed_bytes));
Elliott Hughes9d5ccec2011-09-19 13:19:50 -07001648 if (Runtime::Current()->HasStatsEnabled()) {
Elliott Hughes9d5ccec2011-09-19 13:19:50 -07001649 RuntimeStats* thread_stats = Thread::Current()->GetStats();
Mathieu Chartier357e9be2012-08-01 11:00:14 -07001650 thread_stats->freed_objects += freed_objects;
Elliott Hughes307f75d2011-10-12 18:04:40 -07001651 thread_stats->freed_bytes += freed_bytes;
Mathieu Chartier2fde5332012-09-14 14:51:54 -07001652 // TODO: Do this concurrently.
1653 RuntimeStats* global_stats = Runtime::Current()->GetStats();
1654 global_stats->freed_objects += freed_objects;
1655 global_stats->freed_bytes += freed_bytes;
Elliott Hughes9d5ccec2011-09-19 13:19:50 -07001656 }
Carl Shapiro58551df2011-07-24 03:09:51 -07001657}
1658
Hiroshi Yamauchi4460a842015-03-09 11:57:48 -07001659void Heap::RecordFreeRevoke() {
1660 // Subtract num_bytes_freed_revoke_ from num_bytes_allocated_ to cancel out the
1661 // the ahead-of-time, bulk counting of bytes allocated in rosalloc thread-local buffers.
1662 // If there's a concurrent revoke, ok to not necessarily reset num_bytes_freed_revoke_
1663 // all the way to zero exactly as the remainder will be subtracted at the next GC.
1664 size_t bytes_freed = num_bytes_freed_revoke_.LoadSequentiallyConsistent();
1665 CHECK_GE(num_bytes_freed_revoke_.FetchAndSubSequentiallyConsistent(bytes_freed),
1666 bytes_freed) << "num_bytes_freed_revoke_ underflow";
1667 CHECK_GE(num_bytes_allocated_.FetchAndSubSequentiallyConsistent(bytes_freed),
1668 bytes_freed) << "num_bytes_allocated_ underflow";
1669 GetCurrentGcIteration()->SetFreedRevoke(bytes_freed);
1670}
1671
Zuo Wangf37a88b2014-07-10 04:26:41 -07001672space::RosAllocSpace* Heap::GetRosAllocSpace(gc::allocator::RosAlloc* rosalloc) const {
Mathieu Chartiera9d82fe2016-01-25 20:06:11 -08001673 if (rosalloc_space_ != nullptr && rosalloc_space_->GetRosAlloc() == rosalloc) {
1674 return rosalloc_space_;
1675 }
Zuo Wangf37a88b2014-07-10 04:26:41 -07001676 for (const auto& space : continuous_spaces_) {
1677 if (space->AsContinuousSpace()->IsRosAllocSpace()) {
1678 if (space->AsContinuousSpace()->AsRosAllocSpace()->GetRosAlloc() == rosalloc) {
1679 return space->AsContinuousSpace()->AsRosAllocSpace();
1680 }
1681 }
1682 }
1683 return nullptr;
1684}
1685
Mathieu Chartiereebc3af2016-02-29 18:13:38 -08001686static inline bool EntrypointsInstrumented() SHARED_REQUIRES(Locks::mutator_lock_) {
1687 instrumentation::Instrumentation* const instrumentation =
1688 Runtime::Current()->GetInstrumentation();
1689 return instrumentation != nullptr && instrumentation->AllocEntrypointsInstrumented();
1690}
1691
Mathieu Chartiera4f6af92015-08-11 17:35:25 -07001692mirror::Object* Heap::AllocateInternalWithGc(Thread* self,
1693 AllocatorType allocator,
Mathieu Chartiereebc3af2016-02-29 18:13:38 -08001694 bool instrumented,
Mathieu Chartiera4f6af92015-08-11 17:35:25 -07001695 size_t alloc_size,
1696 size_t* bytes_allocated,
Ian Rogers6fac4472014-02-25 17:01:10 -08001697 size_t* usable_size,
Hiroshi Yamauchi4460a842015-03-09 11:57:48 -07001698 size_t* bytes_tl_bulk_allocated,
Mathieu Chartierc528dba2013-11-26 12:00:11 -08001699 mirror::Class** klass) {
Mathieu Chartiere6da9af2013-12-16 11:54:42 -08001700 bool was_default_allocator = allocator == GetCurrentAllocator();
Mathieu Chartierf4f38432014-09-03 11:21:08 -07001701 // Make sure there is no pending exception since we may need to throw an OOME.
1702 self->AssertNoPendingException();
Mathieu Chartierc528dba2013-11-26 12:00:11 -08001703 DCHECK(klass != nullptr);
Mathieu Chartiereb8167a2014-05-07 15:43:14 -07001704 StackHandleScope<1> hs(self);
1705 HandleWrapper<mirror::Class> h(hs.NewHandleWrapper(klass));
1706 klass = nullptr; // Invalidate for safety.
Mathieu Chartier866fb2a2012-09-10 10:47:49 -07001707 // The allocation failed. If the GC is running, block until it completes, and then retry the
1708 // allocation.
Mathieu Chartier89a201e2014-05-02 10:27:26 -07001709 collector::GcType last_gc = WaitForGcToComplete(kGcCauseForAlloc, self);
Mathieu Chartiereebc3af2016-02-29 18:13:38 -08001710 // If we were the default allocator but the allocator changed while we were suspended,
1711 // abort the allocation.
1712 if ((was_default_allocator && allocator != GetCurrentAllocator()) ||
1713 (!instrumented && EntrypointsInstrumented())) {
1714 return nullptr;
1715 }
Ian Rogers1d54e732013-05-02 21:10:01 -07001716 if (last_gc != collector::kGcTypeNone) {
Mathieu Chartier866fb2a2012-09-10 10:47:49 -07001717 // A GC was in progress and we blocked, retry allocation now that memory has been freed.
Mathieu Chartiereb8167a2014-05-07 15:43:14 -07001718 mirror::Object* ptr = TryToAllocate<true, false>(self, allocator, alloc_size, bytes_allocated,
Hiroshi Yamauchi4460a842015-03-09 11:57:48 -07001719 usable_size, bytes_tl_bulk_allocated);
Mathieu Chartiereb8167a2014-05-07 15:43:14 -07001720 if (ptr != nullptr) {
1721 return ptr;
1722 }
Carl Shapiro69759ea2011-07-21 18:13:35 -07001723 }
1724
Mathieu Chartier5ae2c932014-03-28 16:22:20 -07001725 collector::GcType tried_type = next_gc_type_;
Mathieu Chartiereb8167a2014-05-07 15:43:14 -07001726 const bool gc_ran =
1727 CollectGarbageInternal(tried_type, kGcCauseForAlloc, false) != collector::kGcTypeNone;
Mathieu Chartiereebc3af2016-02-29 18:13:38 -08001728 if ((was_default_allocator && allocator != GetCurrentAllocator()) ||
1729 (!instrumented && EntrypointsInstrumented())) {
Mathieu Chartiereb8167a2014-05-07 15:43:14 -07001730 return nullptr;
1731 }
1732 if (gc_ran) {
1733 mirror::Object* ptr = TryToAllocate<true, false>(self, allocator, alloc_size, bytes_allocated,
Hiroshi Yamauchi4460a842015-03-09 11:57:48 -07001734 usable_size, bytes_tl_bulk_allocated);
Mathieu Chartiereb8167a2014-05-07 15:43:14 -07001735 if (ptr != nullptr) {
1736 return ptr;
Mathieu Chartier5ae2c932014-03-28 16:22:20 -07001737 }
1738 }
1739
Mathieu Chartier866fb2a2012-09-10 10:47:49 -07001740 // Loop through our different Gc types and try to Gc until we get enough free memory.
Mathieu Chartiercbb2d202013-11-14 17:45:16 -08001741 for (collector::GcType gc_type : gc_plan_) {
Mathieu Chartier5ae2c932014-03-28 16:22:20 -07001742 if (gc_type == tried_type) {
1743 continue;
1744 }
Mathieu Chartiercbb2d202013-11-14 17:45:16 -08001745 // Attempt to run the collector, if we succeed, re-try the allocation.
Andreas Gampe277ccbd2014-11-03 21:36:10 -08001746 const bool plan_gc_ran =
Mathieu Chartiere6da9af2013-12-16 11:54:42 -08001747 CollectGarbageInternal(gc_type, kGcCauseForAlloc, false) != collector::kGcTypeNone;
Mathieu Chartiereebc3af2016-02-29 18:13:38 -08001748 if ((was_default_allocator && allocator != GetCurrentAllocator()) ||
1749 (!instrumented && EntrypointsInstrumented())) {
Mathieu Chartiere6da9af2013-12-16 11:54:42 -08001750 return nullptr;
1751 }
Andreas Gampe277ccbd2014-11-03 21:36:10 -08001752 if (plan_gc_ran) {
Mathieu Chartier866fb2a2012-09-10 10:47:49 -07001753 // Did we free sufficient memory for the allocation to succeed?
Mathieu Chartiereb8167a2014-05-07 15:43:14 -07001754 mirror::Object* ptr = TryToAllocate<true, false>(self, allocator, alloc_size, bytes_allocated,
Hiroshi Yamauchi4460a842015-03-09 11:57:48 -07001755 usable_size, bytes_tl_bulk_allocated);
Mathieu Chartiereb8167a2014-05-07 15:43:14 -07001756 if (ptr != nullptr) {
1757 return ptr;
1758 }
Mathieu Chartier866fb2a2012-09-10 10:47:49 -07001759 }
Ian Rogers00f7d0e2012-07-19 15:28:27 -07001760 }
Ian Rogers00f7d0e2012-07-19 15:28:27 -07001761 // Allocations have failed after GCs; this is an exceptional state.
Mathieu Chartiereb8167a2014-05-07 15:43:14 -07001762 // Try harder, growing the heap if necessary.
1763 mirror::Object* ptr = TryToAllocate<true, true>(self, allocator, alloc_size, bytes_allocated,
Hiroshi Yamauchi4460a842015-03-09 11:57:48 -07001764 usable_size, bytes_tl_bulk_allocated);
Mathieu Chartiereb8167a2014-05-07 15:43:14 -07001765 if (ptr != nullptr) {
1766 return ptr;
Carl Shapiro69759ea2011-07-21 18:13:35 -07001767 }
Mathieu Chartiereb8167a2014-05-07 15:43:14 -07001768 // Most allocations should have succeeded by now, so the heap is really full, really fragmented,
1769 // or the requested size is really big. Do another GC, collecting SoftReferences this time. The
1770 // VM spec requires that all SoftReferences have been collected and cleared before throwing
1771 // OOME.
1772 VLOG(gc) << "Forcing collection of SoftReferences for " << PrettySize(alloc_size)
1773 << " allocation";
1774 // TODO: Run finalization, but this may cause more allocations to occur.
1775 // We don't need a WaitForGcToComplete here either.
1776 DCHECK(!gc_plan_.empty());
1777 CollectGarbageInternal(gc_plan_.back(), kGcCauseForAlloc, true);
Mathieu Chartiereebc3af2016-02-29 18:13:38 -08001778 if ((was_default_allocator && allocator != GetCurrentAllocator()) ||
1779 (!instrumented && EntrypointsInstrumented())) {
Mathieu Chartiereb8167a2014-05-07 15:43:14 -07001780 return nullptr;
Mathieu Chartiercbb2d202013-11-14 17:45:16 -08001781 }
Hiroshi Yamauchi4460a842015-03-09 11:57:48 -07001782 ptr = TryToAllocate<true, true>(self, allocator, alloc_size, bytes_allocated, usable_size,
1783 bytes_tl_bulk_allocated);
Mathieu Chartier6a7824d2014-08-22 14:53:04 -07001784 if (ptr == nullptr) {
Zuo Wangf37a88b2014-07-10 04:26:41 -07001785 const uint64_t current_time = NanoTime();
Mathieu Chartier6a7824d2014-08-22 14:53:04 -07001786 switch (allocator) {
1787 case kAllocatorTypeRosAlloc:
1788 // Fall-through.
1789 case kAllocatorTypeDlMalloc: {
1790 if (use_homogeneous_space_compaction_for_oom_ &&
1791 current_time - last_time_homogeneous_space_compaction_by_oom_ >
1792 min_interval_homogeneous_space_compaction_by_oom_) {
1793 last_time_homogeneous_space_compaction_by_oom_ = current_time;
1794 HomogeneousSpaceCompactResult result = PerformHomogeneousSpaceCompact();
Mathieu Chartiereebc3af2016-02-29 18:13:38 -08001795 // Thread suspension could have occurred.
1796 if ((was_default_allocator && allocator != GetCurrentAllocator()) ||
1797 (!instrumented && EntrypointsInstrumented())) {
1798 return nullptr;
1799 }
Mathieu Chartier6a7824d2014-08-22 14:53:04 -07001800 switch (result) {
1801 case HomogeneousSpaceCompactResult::kSuccess:
1802 // If the allocation succeeded, we delayed an oom.
1803 ptr = TryToAllocate<true, true>(self, allocator, alloc_size, bytes_allocated,
Hiroshi Yamauchi4460a842015-03-09 11:57:48 -07001804 usable_size, bytes_tl_bulk_allocated);
Mathieu Chartier6a7824d2014-08-22 14:53:04 -07001805 if (ptr != nullptr) {
1806 count_delayed_oom_++;
1807 }
1808 break;
1809 case HomogeneousSpaceCompactResult::kErrorReject:
1810 // Reject due to disabled moving GC.
1811 break;
1812 case HomogeneousSpaceCompactResult::kErrorVMShuttingDown:
1813 // Throw OOM by default.
1814 break;
1815 default: {
Ian Rogers2c4257b2014-10-24 14:20:06 -07001816 UNIMPLEMENTED(FATAL) << "homogeneous space compaction result: "
1817 << static_cast<size_t>(result);
1818 UNREACHABLE();
Mathieu Chartier6a7824d2014-08-22 14:53:04 -07001819 }
Zuo Wangf37a88b2014-07-10 04:26:41 -07001820 }
Mathieu Chartier6a7824d2014-08-22 14:53:04 -07001821 // Always print that we ran homogeneous space compation since this can cause jank.
1822 VLOG(heap) << "Ran heap homogeneous space compaction, "
1823 << " requested defragmentation "
1824 << count_requested_homogeneous_space_compaction_.LoadSequentiallyConsistent()
1825 << " performed defragmentation "
1826 << count_performed_homogeneous_space_compaction_.LoadSequentiallyConsistent()
1827 << " ignored homogeneous space compaction "
1828 << count_ignored_homogeneous_space_compaction_.LoadSequentiallyConsistent()
1829 << " delayed count = "
1830 << count_delayed_oom_.LoadSequentiallyConsistent();
Zuo Wangf37a88b2014-07-10 04:26:41 -07001831 }
Mathieu Chartier6a7824d2014-08-22 14:53:04 -07001832 break;
Zuo Wangf37a88b2014-07-10 04:26:41 -07001833 }
Mathieu Chartier6a7824d2014-08-22 14:53:04 -07001834 case kAllocatorTypeNonMoving: {
1835 // Try to transition the heap if the allocation failure was due to the space being full.
1836 if (!IsOutOfMemoryOnAllocation<false>(allocator, alloc_size)) {
1837 // If we aren't out of memory then the OOM was probably from the non moving space being
1838 // full. Attempt to disable compaction and turn the main space into a non moving space.
1839 DisableMovingGc();
Mathieu Chartiereebc3af2016-02-29 18:13:38 -08001840 // Thread suspension could have occurred.
1841 if ((was_default_allocator && allocator != GetCurrentAllocator()) ||
1842 (!instrumented && EntrypointsInstrumented())) {
1843 return nullptr;
1844 }
Mathieu Chartier6a7824d2014-08-22 14:53:04 -07001845 // If we are still a moving GC then something must have caused the transition to fail.
1846 if (IsMovingGc(collector_type_)) {
1847 MutexLock mu(self, *gc_complete_lock_);
1848 // If we couldn't disable moving GC, just throw OOME and return null.
1849 LOG(WARNING) << "Couldn't disable moving GC with disable GC count "
1850 << disable_moving_gc_count_;
1851 } else {
1852 LOG(WARNING) << "Disabled moving GC due to the non moving space being full";
1853 ptr = TryToAllocate<true, true>(self, allocator, alloc_size, bytes_allocated,
Hiroshi Yamauchi4460a842015-03-09 11:57:48 -07001854 usable_size, bytes_tl_bulk_allocated);
Mathieu Chartier6a7824d2014-08-22 14:53:04 -07001855 }
1856 }
1857 break;
1858 }
1859 default: {
1860 // Do nothing for others allocators.
1861 }
Zuo Wangf37a88b2014-07-10 04:26:41 -07001862 }
1863 }
1864 // If the allocation hasn't succeeded by this point, throw an OOM error.
Mathieu Chartiereb8167a2014-05-07 15:43:14 -07001865 if (ptr == nullptr) {
Hiroshi Yamauchi654dd482014-07-09 12:54:32 -07001866 ThrowOutOfMemoryError(self, alloc_size, allocator);
Mathieu Chartiereb8167a2014-05-07 15:43:14 -07001867 }
Mathieu Chartiercbb2d202013-11-14 17:45:16 -08001868 return ptr;
Carl Shapiro69759ea2011-07-21 18:13:35 -07001869}
1870
Mathieu Chartier155dfe92012-10-09 14:24:49 -07001871void Heap::SetTargetHeapUtilization(float target) {
1872 DCHECK_GT(target, 0.0f); // asserted in Java code
1873 DCHECK_LT(target, 1.0f);
1874 target_utilization_ = target;
1875}
1876
Ian Rogers1d54e732013-05-02 21:10:01 -07001877size_t Heap::GetObjectsAllocated() const {
Mathieu Chartier4f55e222015-09-04 13:26:21 -07001878 Thread* const self = Thread::Current();
Mathieu Chartierb43390c2015-05-12 10:47:11 -07001879 ScopedThreadStateChange tsc(self, kWaitingForGetObjectsAllocated);
Mathieu Chartierb43390c2015-05-12 10:47:11 -07001880 // Need SuspendAll here to prevent lock violation if RosAlloc does it during InspectAll.
Mathieu Chartier4f55e222015-09-04 13:26:21 -07001881 ScopedSuspendAll ssa(__FUNCTION__);
1882 ReaderMutexLock mu(self, *Locks::heap_bitmap_lock_);
Ian Rogers1d54e732013-05-02 21:10:01 -07001883 size_t total = 0;
Mathieu Chartier4f55e222015-09-04 13:26:21 -07001884 for (space::AllocSpace* space : alloc_spaces_) {
1885 total += space->GetObjectsAllocated();
Ian Rogers1d54e732013-05-02 21:10:01 -07001886 }
Mathieu Chartier155dfe92012-10-09 14:24:49 -07001887 return total;
1888}
1889
Mathieu Chartierdd162fb2014-08-06 17:06:33 -07001890uint64_t Heap::GetObjectsAllocatedEver() const {
Mathieu Chartier4edd8472015-06-01 10:47:36 -07001891 uint64_t total = GetObjectsFreedEver();
1892 // If we are detached, we can't use GetObjectsAllocated since we can't change thread states.
1893 if (Thread::Current() != nullptr) {
1894 total += GetObjectsAllocated();
1895 }
1896 return total;
Ian Rogers1d54e732013-05-02 21:10:01 -07001897}
1898
Mathieu Chartierdd162fb2014-08-06 17:06:33 -07001899uint64_t Heap::GetBytesAllocatedEver() const {
Mathieu Chartier692fafd2013-11-29 17:24:40 -08001900 return GetBytesFreedEver() + GetBytesAllocated();
Mathieu Chartier155dfe92012-10-09 14:24:49 -07001901}
1902
Elliott Hughes9d5ccec2011-09-19 13:19:50 -07001903class InstanceCounter {
1904 public:
Mathieu Chartiera4f6af92015-08-11 17:35:25 -07001905 InstanceCounter(const std::vector<mirror::Class*>& classes,
1906 bool use_is_assignable_from,
1907 uint64_t* counts)
Mathieu Chartier90443472015-07-16 20:32:27 -07001908 SHARED_REQUIRES(Locks::mutator_lock_)
Mathieu Chartiera4f6af92015-08-11 17:35:25 -07001909 : classes_(classes), use_is_assignable_from_(use_is_assignable_from), counts_(counts) {}
1910
Mathieu Chartier412c7fc2014-02-07 12:18:39 -08001911 static void Callback(mirror::Object* obj, void* arg)
Mathieu Chartier90443472015-07-16 20:32:27 -07001912 SHARED_REQUIRES(Locks::mutator_lock_, Locks::heap_bitmap_lock_) {
Mathieu Chartier412c7fc2014-02-07 12:18:39 -08001913 InstanceCounter* instance_counter = reinterpret_cast<InstanceCounter*>(arg);
1914 mirror::Class* instance_class = obj->GetClass();
1915 CHECK(instance_class != nullptr);
1916 for (size_t i = 0; i < instance_counter->classes_.size(); ++i) {
Mathieu Chartierf1820852015-07-10 13:19:51 -07001917 mirror::Class* klass = instance_counter->classes_[i];
Mathieu Chartier412c7fc2014-02-07 12:18:39 -08001918 if (instance_counter->use_is_assignable_from_) {
Mathieu Chartierf1820852015-07-10 13:19:51 -07001919 if (klass != nullptr && klass->IsAssignableFrom(instance_class)) {
Mathieu Chartier412c7fc2014-02-07 12:18:39 -08001920 ++instance_counter->counts_[i];
Elliott Hughesec0f83d2013-01-15 16:54:08 -08001921 }
Mathieu Chartierf1820852015-07-10 13:19:51 -07001922 } else if (instance_class == klass) {
Mathieu Chartier412c7fc2014-02-07 12:18:39 -08001923 ++instance_counter->counts_[i];
Elliott Hughes9d5ccec2011-09-19 13:19:50 -07001924 }
1925 }
1926 }
1927
Mathieu Chartier7469ebf2012-09-24 16:28:36 -07001928 private:
Ian Rogers2dd0e2c2013-01-24 12:42:14 -08001929 const std::vector<mirror::Class*>& classes_;
Elliott Hughesec0f83d2013-01-15 16:54:08 -08001930 bool use_is_assignable_from_;
1931 uint64_t* const counts_;
Elliott Hughesec0f83d2013-01-15 16:54:08 -08001932 DISALLOW_COPY_AND_ASSIGN(InstanceCounter);
Elliott Hughes9d5ccec2011-09-19 13:19:50 -07001933};
1934
Ian Rogers2dd0e2c2013-01-24 12:42:14 -08001935void Heap::CountInstances(const std::vector<mirror::Class*>& classes, bool use_is_assignable_from,
Elliott Hughesec0f83d2013-01-15 16:54:08 -08001936 uint64_t* counts) {
Elliott Hughesec0f83d2013-01-15 16:54:08 -08001937 InstanceCounter counter(classes, use_is_assignable_from, counts);
Mathieu Chartier412c7fc2014-02-07 12:18:39 -08001938 VisitObjects(InstanceCounter::Callback, &counter);
Elliott Hughes9d5ccec2011-09-19 13:19:50 -07001939}
1940
Elliott Hughes3b78c942013-01-15 17:35:41 -08001941class InstanceCollector {
1942 public:
Ian Rogers2dd0e2c2013-01-24 12:42:14 -08001943 InstanceCollector(mirror::Class* c, int32_t max_count, std::vector<mirror::Object*>& instances)
Mathieu Chartier90443472015-07-16 20:32:27 -07001944 SHARED_REQUIRES(Locks::mutator_lock_)
Elliott Hughes3b78c942013-01-15 17:35:41 -08001945 : class_(c), max_count_(max_count), instances_(instances) {
1946 }
Mathieu Chartier412c7fc2014-02-07 12:18:39 -08001947 static void Callback(mirror::Object* obj, void* arg)
Mathieu Chartier90443472015-07-16 20:32:27 -07001948 SHARED_REQUIRES(Locks::mutator_lock_, Locks::heap_bitmap_lock_) {
Mathieu Chartier412c7fc2014-02-07 12:18:39 -08001949 DCHECK(arg != nullptr);
1950 InstanceCollector* instance_collector = reinterpret_cast<InstanceCollector*>(arg);
Mathieu Chartier2d5f39e2014-09-19 17:52:37 -07001951 if (obj->GetClass() == instance_collector->class_) {
Mathieu Chartier412c7fc2014-02-07 12:18:39 -08001952 if (instance_collector->max_count_ == 0 ||
1953 instance_collector->instances_.size() < instance_collector->max_count_) {
1954 instance_collector->instances_.push_back(obj);
Elliott Hughes3b78c942013-01-15 17:35:41 -08001955 }
1956 }
1957 }
1958
1959 private:
Mathieu Chartier2d5f39e2014-09-19 17:52:37 -07001960 const mirror::Class* const class_;
1961 const uint32_t max_count_;
Ian Rogers2dd0e2c2013-01-24 12:42:14 -08001962 std::vector<mirror::Object*>& instances_;
Elliott Hughes3b78c942013-01-15 17:35:41 -08001963 DISALLOW_COPY_AND_ASSIGN(InstanceCollector);
1964};
1965
Mathieu Chartiera4f6af92015-08-11 17:35:25 -07001966void Heap::GetInstances(mirror::Class* c,
1967 int32_t max_count,
Ian Rogers2dd0e2c2013-01-24 12:42:14 -08001968 std::vector<mirror::Object*>& instances) {
Elliott Hughes3b78c942013-01-15 17:35:41 -08001969 InstanceCollector collector(c, max_count, instances);
Mathieu Chartier412c7fc2014-02-07 12:18:39 -08001970 VisitObjects(&InstanceCollector::Callback, &collector);
Elliott Hughes3b78c942013-01-15 17:35:41 -08001971}
1972
Elliott Hughes0cbaff52013-01-16 15:28:01 -08001973class ReferringObjectsFinder {
1974 public:
Mathieu Chartiera4f6af92015-08-11 17:35:25 -07001975 ReferringObjectsFinder(mirror::Object* object,
1976 int32_t max_count,
Ian Rogers2dd0e2c2013-01-24 12:42:14 -08001977 std::vector<mirror::Object*>& referring_objects)
Mathieu Chartier90443472015-07-16 20:32:27 -07001978 SHARED_REQUIRES(Locks::mutator_lock_)
Elliott Hughes0cbaff52013-01-16 15:28:01 -08001979 : object_(object), max_count_(max_count), referring_objects_(referring_objects) {
1980 }
1981
Mathieu Chartier412c7fc2014-02-07 12:18:39 -08001982 static void Callback(mirror::Object* obj, void* arg)
Mathieu Chartier90443472015-07-16 20:32:27 -07001983 SHARED_REQUIRES(Locks::mutator_lock_, Locks::heap_bitmap_lock_) {
Mathieu Chartier412c7fc2014-02-07 12:18:39 -08001984 reinterpret_cast<ReferringObjectsFinder*>(arg)->operator()(obj);
1985 }
1986
Elliott Hughes0cbaff52013-01-16 15:28:01 -08001987 // For bitmap Visit.
1988 // TODO: Fix lock analysis to not use NO_THREAD_SAFETY_ANALYSIS, requires support for
1989 // annotalysis on visitors.
Mathieu Chartier0e54cd02014-03-20 12:41:23 -07001990 void operator()(mirror::Object* o) const NO_THREAD_SAFETY_ANALYSIS {
Mathieu Chartier059ef3d2015-08-18 13:54:21 -07001991 o->VisitReferences(*this, VoidFunctor());
Elliott Hughes0cbaff52013-01-16 15:28:01 -08001992 }
1993
Mathieu Chartier3b05e9b2014-03-25 09:29:43 -07001994 // For Object::VisitReferences.
Mathieu Chartierda7c6502015-07-23 16:01:26 -07001995 void operator()(mirror::Object* obj, MemberOffset offset, bool is_static ATTRIBUTE_UNUSED) const
Mathieu Chartier90443472015-07-16 20:32:27 -07001996 SHARED_REQUIRES(Locks::mutator_lock_) {
Ian Rogersb0fa5dc2014-04-28 16:47:08 -07001997 mirror::Object* ref = obj->GetFieldObject<mirror::Object>(offset);
Mathieu Chartier407f7022014-02-18 14:37:05 -08001998 if (ref == object_ && (max_count_ == 0 || referring_objects_.size() < max_count_)) {
1999 referring_objects_.push_back(obj);
Elliott Hughes0cbaff52013-01-16 15:28:01 -08002000 }
2001 }
2002
Mathieu Chartierda7c6502015-07-23 16:01:26 -07002003 void VisitRootIfNonNull(mirror::CompressedReference<mirror::Object>* root ATTRIBUTE_UNUSED)
2004 const {}
2005 void VisitRoot(mirror::CompressedReference<mirror::Object>* root ATTRIBUTE_UNUSED) const {}
2006
Elliott Hughes0cbaff52013-01-16 15:28:01 -08002007 private:
Mathieu Chartier2d5f39e2014-09-19 17:52:37 -07002008 const mirror::Object* const object_;
2009 const uint32_t max_count_;
Ian Rogers2dd0e2c2013-01-24 12:42:14 -08002010 std::vector<mirror::Object*>& referring_objects_;
Elliott Hughes0cbaff52013-01-16 15:28:01 -08002011 DISALLOW_COPY_AND_ASSIGN(ReferringObjectsFinder);
2012};
2013
Ian Rogers2dd0e2c2013-01-24 12:42:14 -08002014void Heap::GetReferringObjects(mirror::Object* o, int32_t max_count,
2015 std::vector<mirror::Object*>& referring_objects) {
Elliott Hughes0cbaff52013-01-16 15:28:01 -08002016 ReferringObjectsFinder finder(o, max_count, referring_objects);
Mathieu Chartier412c7fc2014-02-07 12:18:39 -08002017 VisitObjects(&ReferringObjectsFinder::Callback, &finder);
Elliott Hughes0cbaff52013-01-16 15:28:01 -08002018}
2019
Ian Rogers30fab402012-01-23 15:43:46 -08002020void Heap::CollectGarbage(bool clear_soft_references) {
Mathieu Chartier866fb2a2012-09-10 10:47:49 -07002021 // Even if we waited for a GC we still need to do another GC since weaks allocated during the
2022 // last GC will not have necessarily been cleared.
Mathieu Chartier7bf82af2013-12-06 16:51:45 -08002023 CollectGarbageInternal(gc_plan_.back(), kGcCauseExplicit, clear_soft_references);
Carl Shapiro69759ea2011-07-21 18:13:35 -07002024}
2025
Mathieu Chartierdb00eaf2015-08-31 17:10:05 -07002026bool Heap::SupportHomogeneousSpaceCompactAndCollectorTransitions() const {
2027 return main_space_backup_.get() != nullptr && main_space_ != nullptr &&
2028 foreground_collector_type_ == kCollectorTypeCMS;
2029}
2030
Zuo Wangf37a88b2014-07-10 04:26:41 -07002031HomogeneousSpaceCompactResult Heap::PerformHomogeneousSpaceCompact() {
2032 Thread* self = Thread::Current();
2033 // Inc requested homogeneous space compaction.
2034 count_requested_homogeneous_space_compaction_++;
2035 // Store performed homogeneous space compaction at a new request arrival.
Zuo Wangf37a88b2014-07-10 04:26:41 -07002036 ScopedThreadStateChange tsc(self, kWaitingPerformingGc);
2037 Locks::mutator_lock_->AssertNotHeld(self);
2038 {
Andreas Gampe277ccbd2014-11-03 21:36:10 -08002039 ScopedThreadStateChange tsc2(self, kWaitingForGcToComplete);
Zuo Wangf37a88b2014-07-10 04:26:41 -07002040 MutexLock mu(self, *gc_complete_lock_);
2041 // Ensure there is only one GC at a time.
2042 WaitForGcToCompleteLocked(kGcCauseHomogeneousSpaceCompact, self);
2043 // Homogeneous space compaction is a copying transition, can't run it if the moving GC disable count
2044 // is non zero.
Mathieu Chartier6a7824d2014-08-22 14:53:04 -07002045 // If the collector type changed to something which doesn't benefit from homogeneous space compaction,
Zuo Wangf37a88b2014-07-10 04:26:41 -07002046 // exit.
Mathieu Chartier6a7824d2014-08-22 14:53:04 -07002047 if (disable_moving_gc_count_ != 0 || IsMovingGc(collector_type_) ||
2048 !main_space_->CanMoveObjects()) {
Mathieu Chartierdb00eaf2015-08-31 17:10:05 -07002049 return kErrorReject;
2050 }
2051 if (!SupportHomogeneousSpaceCompactAndCollectorTransitions()) {
2052 return kErrorUnsupported;
Zuo Wangf37a88b2014-07-10 04:26:41 -07002053 }
2054 collector_type_running_ = kCollectorTypeHomogeneousSpaceCompact;
2055 }
2056 if (Runtime::Current()->IsShuttingDown(self)) {
2057 // Don't allow heap transitions to happen if the runtime is shutting down since these can
2058 // cause objects to get finalized.
2059 FinishGC(self, collector::kGcTypeNone);
2060 return HomogeneousSpaceCompactResult::kErrorVMShuttingDown;
2061 }
Mathieu Chartier4f55e222015-09-04 13:26:21 -07002062 collector::GarbageCollector* collector;
2063 {
2064 ScopedSuspendAll ssa(__FUNCTION__);
2065 uint64_t start_time = NanoTime();
2066 // Launch compaction.
2067 space::MallocSpace* to_space = main_space_backup_.release();
2068 space::MallocSpace* from_space = main_space_;
2069 to_space->GetMemMap()->Protect(PROT_READ | PROT_WRITE);
2070 const uint64_t space_size_before_compaction = from_space->Size();
2071 AddSpace(to_space);
2072 // Make sure that we will have enough room to copy.
2073 CHECK_GE(to_space->GetFootprintLimit(), from_space->GetFootprintLimit());
2074 collector = Compact(to_space, from_space, kGcCauseHomogeneousSpaceCompact);
2075 const uint64_t space_size_after_compaction = to_space->Size();
2076 main_space_ = to_space;
2077 main_space_backup_.reset(from_space);
2078 RemoveSpace(from_space);
2079 SetSpaceAsDefault(main_space_); // Set as default to reset the proper dlmalloc space.
2080 // Update performed homogeneous space compaction count.
2081 count_performed_homogeneous_space_compaction_++;
2082 // Print statics log and resume all threads.
2083 uint64_t duration = NanoTime() - start_time;
2084 VLOG(heap) << "Heap homogeneous space compaction took " << PrettyDuration(duration) << " size: "
2085 << PrettySize(space_size_before_compaction) << " -> "
2086 << PrettySize(space_size_after_compaction) << " compact-ratio: "
2087 << std::fixed << static_cast<double>(space_size_after_compaction) /
2088 static_cast<double>(space_size_before_compaction);
2089 }
Zuo Wangf37a88b2014-07-10 04:26:41 -07002090 // Finish GC.
Mathieu Chartier3cf22532015-07-09 15:15:09 -07002091 reference_processor_->EnqueueClearedReferences(self);
Zuo Wangf37a88b2014-07-10 04:26:41 -07002092 GrowForUtilization(semi_space_collector_);
Hiroshi Yamauchie4d99872015-02-26 12:53:45 -08002093 LogGC(kGcCauseHomogeneousSpaceCompact, collector);
Zuo Wangf37a88b2014-07-10 04:26:41 -07002094 FinishGC(self, collector::kGcTypeFull);
Mathieu Chartier598302a2015-09-23 14:52:39 -07002095 {
2096 ScopedObjectAccess soa(self);
2097 soa.Vm()->UnloadNativeLibraries();
2098 }
Zuo Wangf37a88b2014-07-10 04:26:41 -07002099 return HomogeneousSpaceCompactResult::kSuccess;
2100}
2101
Mathieu Chartiere6da9af2013-12-16 11:54:42 -08002102void Heap::TransitionCollector(CollectorType collector_type) {
2103 if (collector_type == collector_type_) {
2104 return;
2105 }
Hiroshi Yamauchia4adbfd2014-02-04 18:12:17 -08002106 VLOG(heap) << "TransitionCollector: " << static_cast<int>(collector_type_)
2107 << " -> " << static_cast<int>(collector_type);
Mathieu Chartiere6da9af2013-12-16 11:54:42 -08002108 uint64_t start_time = NanoTime();
Ian Rogers3e5cf302014-05-20 16:40:37 -07002109 uint32_t before_allocated = num_bytes_allocated_.LoadSequentiallyConsistent();
Mathieu Chartier52e4b432014-06-10 11:22:31 -07002110 Runtime* const runtime = Runtime::Current();
Mathieu Chartier52e4b432014-06-10 11:22:31 -07002111 Thread* const self = Thread::Current();
Mathieu Chartiere6da9af2013-12-16 11:54:42 -08002112 ScopedThreadStateChange tsc(self, kWaitingPerformingGc);
2113 Locks::mutator_lock_->AssertNotHeld(self);
Mathieu Chartier1d27b342014-01-28 12:51:09 -08002114 // Busy wait until we can GC (StartGC can fail if we have a non-zero
2115 // compacting_gc_disable_count_, this should rarely occurs).
Mathieu Chartierd5a89ee2014-01-31 09:55:13 -08002116 for (;;) {
Mathieu Chartiercaa82d62014-02-02 16:51:17 -08002117 {
Andreas Gampe277ccbd2014-11-03 21:36:10 -08002118 ScopedThreadStateChange tsc2(self, kWaitingForGcToComplete);
Mathieu Chartiercaa82d62014-02-02 16:51:17 -08002119 MutexLock mu(self, *gc_complete_lock_);
2120 // Ensure there is only one GC at a time.
Mathieu Chartier89a201e2014-05-02 10:27:26 -07002121 WaitForGcToCompleteLocked(kGcCauseCollectorTransition, self);
Mathieu Chartiere4927f62014-08-23 13:56:03 -07002122 // Currently we only need a heap transition if we switch from a moving collector to a
2123 // non-moving one, or visa versa.
2124 const bool copying_transition = IsMovingGc(collector_type_) != IsMovingGc(collector_type);
Mathieu Chartierb38d4832014-04-10 10:56:55 -07002125 // If someone else beat us to it and changed the collector before we could, exit.
2126 // This is safe to do before the suspend all since we set the collector_type_running_ before
2127 // we exit the loop. If another thread attempts to do the heap transition before we exit,
2128 // then it would get blocked on WaitForGcToCompleteLocked.
2129 if (collector_type == collector_type_) {
2130 return;
2131 }
Mathieu Chartiercaa82d62014-02-02 16:51:17 -08002132 // GC can be disabled if someone has a used GetPrimitiveArrayCritical but not yet released.
2133 if (!copying_transition || disable_moving_gc_count_ == 0) {
2134 // TODO: Not hard code in semi-space collector?
2135 collector_type_running_ = copying_transition ? kCollectorTypeSS : collector_type;
2136 break;
2137 }
Mathieu Chartierd5a89ee2014-01-31 09:55:13 -08002138 }
Mathieu Chartiercaa82d62014-02-02 16:51:17 -08002139 usleep(1000);
Mathieu Chartiere6da9af2013-12-16 11:54:42 -08002140 }
Mathieu Chartier52e4b432014-06-10 11:22:31 -07002141 if (runtime->IsShuttingDown(self)) {
Hiroshi Yamauchia6a8d142014-05-12 16:57:33 -07002142 // Don't allow heap transitions to happen if the runtime is shutting down since these can
2143 // cause objects to get finalized.
2144 FinishGC(self, collector::kGcTypeNone);
2145 return;
2146 }
Hiroshi Yamauchie4d99872015-02-26 12:53:45 -08002147 collector::GarbageCollector* collector = nullptr;
Mathieu Chartier4f55e222015-09-04 13:26:21 -07002148 {
2149 ScopedSuspendAll ssa(__FUNCTION__);
2150 switch (collector_type) {
2151 case kCollectorTypeSS: {
2152 if (!IsMovingGc(collector_type_)) {
2153 // Create the bump pointer space from the backup space.
2154 CHECK(main_space_backup_ != nullptr);
2155 std::unique_ptr<MemMap> mem_map(main_space_backup_->ReleaseMemMap());
2156 // We are transitioning from non moving GC -> moving GC, since we copied from the bump
2157 // pointer space last transition it will be protected.
2158 CHECK(mem_map != nullptr);
Hiroshi Yamauchic1276c82014-08-07 10:27:17 -07002159 mem_map->Protect(PROT_READ | PROT_WRITE);
Mathieu Chartier4f55e222015-09-04 13:26:21 -07002160 bump_pointer_space_ = space::BumpPointerSpace::CreateFromMemMap("Bump pointer space",
2161 mem_map.release());
2162 AddSpace(bump_pointer_space_);
2163 collector = Compact(bump_pointer_space_, main_space_, kGcCauseCollectorTransition);
2164 // Use the now empty main space mem map for the bump pointer temp space.
2165 mem_map.reset(main_space_->ReleaseMemMap());
2166 // Unset the pointers just in case.
2167 if (dlmalloc_space_ == main_space_) {
2168 dlmalloc_space_ = nullptr;
2169 } else if (rosalloc_space_ == main_space_) {
2170 rosalloc_space_ = nullptr;
2171 }
2172 // Remove the main space so that we don't try to trim it, this doens't work for debug
2173 // builds since RosAlloc attempts to read the magic number from a protected page.
2174 RemoveSpace(main_space_);
2175 RemoveRememberedSet(main_space_);
2176 delete main_space_; // Delete the space since it has been removed.
2177 main_space_ = nullptr;
2178 RemoveRememberedSet(main_space_backup_.get());
2179 main_space_backup_.reset(nullptr); // Deletes the space.
2180 temp_space_ = space::BumpPointerSpace::CreateFromMemMap("Bump pointer space 2",
2181 mem_map.release());
2182 AddSpace(temp_space_);
Hiroshi Yamauchic1276c82014-08-07 10:27:17 -07002183 }
Mathieu Chartier4f55e222015-09-04 13:26:21 -07002184 break;
Mathieu Chartiere6da9af2013-12-16 11:54:42 -08002185 }
Mathieu Chartier4f55e222015-09-04 13:26:21 -07002186 case kCollectorTypeMS:
2187 // Fall through.
2188 case kCollectorTypeCMS: {
2189 if (IsMovingGc(collector_type_)) {
2190 CHECK(temp_space_ != nullptr);
2191 std::unique_ptr<MemMap> mem_map(temp_space_->ReleaseMemMap());
2192 RemoveSpace(temp_space_);
2193 temp_space_ = nullptr;
2194 mem_map->Protect(PROT_READ | PROT_WRITE);
2195 CreateMainMallocSpace(mem_map.get(),
2196 kDefaultInitialSize,
2197 std::min(mem_map->Size(), growth_limit_),
2198 mem_map->Size());
2199 mem_map.release();
2200 // Compact to the main space from the bump pointer space, don't need to swap semispaces.
2201 AddSpace(main_space_);
2202 collector = Compact(main_space_, bump_pointer_space_, kGcCauseCollectorTransition);
2203 mem_map.reset(bump_pointer_space_->ReleaseMemMap());
2204 RemoveSpace(bump_pointer_space_);
2205 bump_pointer_space_ = nullptr;
2206 const char* name = kUseRosAlloc ? kRosAllocSpaceName[1] : kDlMallocSpaceName[1];
2207 // Temporarily unprotect the backup mem map so rosalloc can write the debug magic number.
2208 if (kIsDebugBuild && kUseRosAlloc) {
2209 mem_map->Protect(PROT_READ | PROT_WRITE);
2210 }
2211 main_space_backup_.reset(CreateMallocSpaceFromMemMap(
2212 mem_map.get(),
2213 kDefaultInitialSize,
2214 std::min(mem_map->Size(), growth_limit_),
2215 mem_map->Size(),
2216 name,
2217 true));
2218 if (kIsDebugBuild && kUseRosAlloc) {
2219 mem_map->Protect(PROT_NONE);
2220 }
2221 mem_map.release();
2222 }
2223 break;
2224 }
2225 default: {
2226 LOG(FATAL) << "Attempted to transition to invalid collector type "
2227 << static_cast<size_t>(collector_type);
2228 break;
2229 }
Mathieu Chartiere6da9af2013-12-16 11:54:42 -08002230 }
Mathieu Chartier4f55e222015-09-04 13:26:21 -07002231 ChangeCollector(collector_type);
Mathieu Chartiere6da9af2013-12-16 11:54:42 -08002232 }
Mathieu Chartiere6da9af2013-12-16 11:54:42 -08002233 // Can't call into java code with all threads suspended.
Mathieu Chartier3cf22532015-07-09 15:15:09 -07002234 reference_processor_->EnqueueClearedReferences(self);
Mathieu Chartiere6da9af2013-12-16 11:54:42 -08002235 uint64_t duration = NanoTime() - start_time;
Mathieu Chartierafe49982014-03-27 10:55:04 -07002236 GrowForUtilization(semi_space_collector_);
Hiroshi Yamauchie4d99872015-02-26 12:53:45 -08002237 DCHECK(collector != nullptr);
2238 LogGC(kGcCauseCollectorTransition, collector);
Mathieu Chartiere6da9af2013-12-16 11:54:42 -08002239 FinishGC(self, collector::kGcTypeFull);
Mathieu Chartier598302a2015-09-23 14:52:39 -07002240 {
2241 ScopedObjectAccess soa(self);
2242 soa.Vm()->UnloadNativeLibraries();
2243 }
Ian Rogers3e5cf302014-05-20 16:40:37 -07002244 int32_t after_allocated = num_bytes_allocated_.LoadSequentiallyConsistent();
Mathieu Chartiere6da9af2013-12-16 11:54:42 -08002245 int32_t delta_allocated = before_allocated - after_allocated;
Mathieu Chartier19d46b42014-06-17 15:04:40 -07002246 std::string saved_str;
2247 if (delta_allocated >= 0) {
2248 saved_str = " saved at least " + PrettySize(delta_allocated);
2249 } else {
2250 saved_str = " expanded " + PrettySize(-delta_allocated);
2251 }
Mathieu Chartierf8cb1782016-03-18 18:45:41 -07002252 VLOG(heap) << "Collector transition to " << collector_type << " took "
2253 << PrettyDuration(duration) << saved_str;
Mathieu Chartiere6da9af2013-12-16 11:54:42 -08002254}
2255
Mathieu Chartier0de9f732013-11-22 17:58:48 -08002256void Heap::ChangeCollector(CollectorType collector_type) {
Mathieu Chartier7bf82af2013-12-06 16:51:45 -08002257 // TODO: Only do this with all mutators suspended to avoid races.
2258 if (collector_type != collector_type_) {
Mathieu Chartier52e4b432014-06-10 11:22:31 -07002259 if (collector_type == kCollectorTypeMC) {
2260 // Don't allow mark compact unless support is compiled in.
2261 CHECK(kMarkCompactSupport);
2262 }
Mathieu Chartier7bf82af2013-12-06 16:51:45 -08002263 collector_type_ = collector_type;
2264 gc_plan_.clear();
2265 switch (collector_type_) {
Hiroshi Yamauchi2cd334a2015-01-09 14:03:35 -08002266 case kCollectorTypeCC: {
2267 gc_plan_.push_back(collector::kGcTypeFull);
2268 if (use_tlab_) {
2269 ChangeAllocator(kAllocatorTypeRegionTLAB);
2270 } else {
2271 ChangeAllocator(kAllocatorTypeRegion);
2272 }
2273 break;
2274 }
Mathieu Chartier52e4b432014-06-10 11:22:31 -07002275 case kCollectorTypeMC: // Fall-through.
Hiroshi Yamauchi3e417802014-03-20 12:03:02 -07002276 case kCollectorTypeSS: // Fall-through.
Hiroshi Yamauchi6f4ffe42014-01-13 12:30:44 -08002277 case kCollectorTypeGSS: {
Mathieu Chartier7bf82af2013-12-06 16:51:45 -08002278 gc_plan_.push_back(collector::kGcTypeFull);
Mathieu Chartier692fafd2013-11-29 17:24:40 -08002279 if (use_tlab_) {
2280 ChangeAllocator(kAllocatorTypeTLAB);
2281 } else {
2282 ChangeAllocator(kAllocatorTypeBumpPointer);
2283 }
Mathieu Chartier7bf82af2013-12-06 16:51:45 -08002284 break;
2285 }
2286 case kCollectorTypeMS: {
Mathieu Chartier7bf82af2013-12-06 16:51:45 -08002287 gc_plan_.push_back(collector::kGcTypeSticky);
2288 gc_plan_.push_back(collector::kGcTypePartial);
2289 gc_plan_.push_back(collector::kGcTypeFull);
Mathieu Chartiere6da9af2013-12-16 11:54:42 -08002290 ChangeAllocator(kUseRosAlloc ? kAllocatorTypeRosAlloc : kAllocatorTypeDlMalloc);
Mathieu Chartier7bf82af2013-12-06 16:51:45 -08002291 break;
2292 }
2293 case kCollectorTypeCMS: {
Mathieu Chartier7bf82af2013-12-06 16:51:45 -08002294 gc_plan_.push_back(collector::kGcTypeSticky);
2295 gc_plan_.push_back(collector::kGcTypePartial);
2296 gc_plan_.push_back(collector::kGcTypeFull);
Mathieu Chartiere6da9af2013-12-16 11:54:42 -08002297 ChangeAllocator(kUseRosAlloc ? kAllocatorTypeRosAlloc : kAllocatorTypeDlMalloc);
Mathieu Chartier7bf82af2013-12-06 16:51:45 -08002298 break;
2299 }
2300 default: {
Ian Rogers2c4257b2014-10-24 14:20:06 -07002301 UNIMPLEMENTED(FATAL);
2302 UNREACHABLE();
Mathieu Chartier7bf82af2013-12-06 16:51:45 -08002303 }
Mathieu Chartier0de9f732013-11-22 17:58:48 -08002304 }
Hiroshi Yamauchi3e417802014-03-20 12:03:02 -07002305 if (IsGcConcurrent()) {
Mathieu Chartier7bf82af2013-12-06 16:51:45 -08002306 concurrent_start_bytes_ =
2307 std::max(max_allowed_footprint_, kMinConcurrentRemainingBytes) - kMinConcurrentRemainingBytes;
2308 } else {
2309 concurrent_start_bytes_ = std::numeric_limits<size_t>::max();
Mathieu Chartier0de9f732013-11-22 17:58:48 -08002310 }
2311 }
2312}
2313
Mathieu Chartier85a43c02014-01-07 17:59:00 -08002314// Special compacting collector which uses sub-optimal bin packing to reduce zygote space size.
Ian Rogers6fac4472014-02-25 17:01:10 -08002315class ZygoteCompactingCollector FINAL : public collector::SemiSpace {
Mathieu Chartier85a43c02014-01-07 17:59:00 -08002316 public:
Roland Levillain3887c462015-08-12 18:15:42 +01002317 ZygoteCompactingCollector(gc::Heap* heap, bool is_running_on_memory_tool)
Evgenii Stepanov1e133742015-05-20 12:30:59 -07002318 : SemiSpace(heap, false, "zygote collector"),
2319 bin_live_bitmap_(nullptr),
2320 bin_mark_bitmap_(nullptr),
2321 is_running_on_memory_tool_(is_running_on_memory_tool) {}
Mathieu Chartier85a43c02014-01-07 17:59:00 -08002322
2323 void BuildBins(space::ContinuousSpace* space) {
2324 bin_live_bitmap_ = space->GetLiveBitmap();
2325 bin_mark_bitmap_ = space->GetMarkBitmap();
2326 BinContext context;
2327 context.prev_ = reinterpret_cast<uintptr_t>(space->Begin());
2328 context.collector_ = this;
2329 WriterMutexLock mu(Thread::Current(), *Locks::heap_bitmap_lock_);
2330 // Note: This requires traversing the space in increasing order of object addresses.
2331 bin_live_bitmap_->Walk(Callback, reinterpret_cast<void*>(&context));
2332 // Add the last bin which spans after the last object to the end of the space.
2333 AddBin(reinterpret_cast<uintptr_t>(space->End()) - context.prev_, context.prev_);
2334 }
2335
2336 private:
2337 struct BinContext {
2338 uintptr_t prev_; // The end of the previous object.
2339 ZygoteCompactingCollector* collector_;
2340 };
2341 // Maps from bin sizes to locations.
2342 std::multimap<size_t, uintptr_t> bins_;
2343 // Live bitmap of the space which contains the bins.
Mathieu Chartiera8e8f9c2014-04-09 14:51:05 -07002344 accounting::ContinuousSpaceBitmap* bin_live_bitmap_;
Mathieu Chartier85a43c02014-01-07 17:59:00 -08002345 // Mark bitmap of the space which contains the bins.
Mathieu Chartiera8e8f9c2014-04-09 14:51:05 -07002346 accounting::ContinuousSpaceBitmap* bin_mark_bitmap_;
Evgenii Stepanov1e133742015-05-20 12:30:59 -07002347 const bool is_running_on_memory_tool_;
Mathieu Chartier85a43c02014-01-07 17:59:00 -08002348
2349 static void Callback(mirror::Object* obj, void* arg)
Mathieu Chartier90443472015-07-16 20:32:27 -07002350 SHARED_REQUIRES(Locks::mutator_lock_) {
Mathieu Chartier85a43c02014-01-07 17:59:00 -08002351 DCHECK(arg != nullptr);
2352 BinContext* context = reinterpret_cast<BinContext*>(arg);
2353 ZygoteCompactingCollector* collector = context->collector_;
2354 uintptr_t object_addr = reinterpret_cast<uintptr_t>(obj);
2355 size_t bin_size = object_addr - context->prev_;
2356 // Add the bin consisting of the end of the previous object to the start of the current object.
2357 collector->AddBin(bin_size, context->prev_);
2358 context->prev_ = object_addr + RoundUp(obj->SizeOf(), kObjectAlignment);
2359 }
2360
2361 void AddBin(size_t size, uintptr_t position) {
Evgenii Stepanov1e133742015-05-20 12:30:59 -07002362 if (is_running_on_memory_tool_) {
2363 MEMORY_TOOL_MAKE_DEFINED(reinterpret_cast<void*>(position), size);
2364 }
Mathieu Chartier85a43c02014-01-07 17:59:00 -08002365 if (size != 0) {
2366 bins_.insert(std::make_pair(size, position));
2367 }
2368 }
2369
Mathieu Chartiera4f6af92015-08-11 17:35:25 -07002370 virtual bool ShouldSweepSpace(space::ContinuousSpace* space ATTRIBUTE_UNUSED) const {
Mathieu Chartier85a43c02014-01-07 17:59:00 -08002371 // Don't sweep any spaces since we probably blasted the internal accounting of the free list
2372 // allocator.
2373 return false;
2374 }
2375
2376 virtual mirror::Object* MarkNonForwardedObject(mirror::Object* obj)
Mathieu Chartier90443472015-07-16 20:32:27 -07002377 REQUIRES(Locks::heap_bitmap_lock_, Locks::mutator_lock_) {
Hiroshi Yamauchi8711d1f2015-03-13 16:48:55 -07002378 size_t obj_size = obj->SizeOf();
2379 size_t alloc_size = RoundUp(obj_size, kObjectAlignment);
Mathieu Chartier5dc08a62014-01-10 10:10:23 -08002380 mirror::Object* forward_address;
Mathieu Chartier85a43c02014-01-07 17:59:00 -08002381 // Find the smallest bin which we can move obj in.
Hiroshi Yamauchi8711d1f2015-03-13 16:48:55 -07002382 auto it = bins_.lower_bound(alloc_size);
Mathieu Chartier85a43c02014-01-07 17:59:00 -08002383 if (it == bins_.end()) {
2384 // No available space in the bins, place it in the target space instead (grows the zygote
2385 // space).
Hiroshi Yamauchi4460a842015-03-09 11:57:48 -07002386 size_t bytes_allocated, dummy;
Hiroshi Yamauchi8711d1f2015-03-13 16:48:55 -07002387 forward_address = to_space_->Alloc(self_, alloc_size, &bytes_allocated, nullptr, &dummy);
Mathieu Chartier85a43c02014-01-07 17:59:00 -08002388 if (to_space_live_bitmap_ != nullptr) {
2389 to_space_live_bitmap_->Set(forward_address);
Mathieu Chartier938a03b2014-01-16 15:10:31 -08002390 } else {
2391 GetHeap()->GetNonMovingSpace()->GetLiveBitmap()->Set(forward_address);
2392 GetHeap()->GetNonMovingSpace()->GetMarkBitmap()->Set(forward_address);
Mathieu Chartier85a43c02014-01-07 17:59:00 -08002393 }
2394 } else {
2395 size_t size = it->first;
2396 uintptr_t pos = it->second;
2397 bins_.erase(it); // Erase the old bin which we replace with the new smaller bin.
2398 forward_address = reinterpret_cast<mirror::Object*>(pos);
2399 // Set the live and mark bits so that sweeping system weaks works properly.
2400 bin_live_bitmap_->Set(forward_address);
2401 bin_mark_bitmap_->Set(forward_address);
Hiroshi Yamauchi8711d1f2015-03-13 16:48:55 -07002402 DCHECK_GE(size, alloc_size);
2403 // Add a new bin with the remaining space.
2404 AddBin(size - alloc_size, pos + alloc_size);
Mathieu Chartier85a43c02014-01-07 17:59:00 -08002405 }
Hiroshi Yamauchi8711d1f2015-03-13 16:48:55 -07002406 // Copy the object over to its new location. Don't use alloc_size to avoid valgrind error.
2407 memcpy(reinterpret_cast<void*>(forward_address), obj, obj_size);
Hiroshi Yamauchi624468c2014-03-31 15:14:47 -07002408 if (kUseBakerOrBrooksReadBarrier) {
2409 obj->AssertReadBarrierPointer();
2410 if (kUseBrooksReadBarrier) {
2411 DCHECK_EQ(forward_address->GetReadBarrierPointer(), obj);
2412 forward_address->SetReadBarrierPointer(forward_address);
2413 }
2414 forward_address->AssertReadBarrierPointer();
Hiroshi Yamauchi9d04a202014-01-31 13:35:49 -08002415 }
Mathieu Chartier85a43c02014-01-07 17:59:00 -08002416 return forward_address;
2417 }
2418};
2419
Mathieu Chartiera1602f22014-01-13 17:19:19 -08002420void Heap::UnBindBitmaps() {
Mathieu Chartierf5997b42014-06-20 10:37:54 -07002421 TimingLogger::ScopedTiming t("UnBindBitmaps", GetCurrentGcIteration()->GetTimings());
Mathieu Chartiera1602f22014-01-13 17:19:19 -08002422 for (const auto& space : GetContinuousSpaces()) {
2423 if (space->IsContinuousMemMapAllocSpace()) {
2424 space::ContinuousMemMapAllocSpace* alloc_space = space->AsContinuousMemMapAllocSpace();
2425 if (alloc_space->HasBoundBitmaps()) {
2426 alloc_space->UnBindBitmaps();
2427 }
2428 }
2429 }
2430}
2431
Mathieu Chartiercc236d72012-07-20 10:29:05 -07002432void Heap::PreZygoteFork() {
Mathieu Chartierfaed9952015-03-31 16:28:53 -07002433 if (!HasZygoteSpace()) {
2434 // We still want to GC in case there is some unreachable non moving objects that could cause a
2435 // suboptimal bin packing when we compact the zygote space.
2436 CollectGarbageInternal(collector::kGcTypeFull, kGcCauseBackground, false);
Mathieu Chartier76ce9172016-01-27 10:44:20 -08002437 // Trim the pages at the end of the non moving space. Trim while not holding zygote lock since
2438 // the trim process may require locking the mutator lock.
2439 non_moving_space_->Trim();
Mathieu Chartierfaed9952015-03-31 16:28:53 -07002440 }
Ian Rogers81d425b2012-09-27 16:03:43 -07002441 Thread* self = Thread::Current();
2442 MutexLock mu(self, zygote_creation_lock_);
Mathieu Chartiercc236d72012-07-20 10:29:05 -07002443 // Try to see if we have any Zygote spaces.
Mathieu Chartiere4cab172014-08-19 18:24:04 -07002444 if (HasZygoteSpace()) {
Mathieu Chartiercc236d72012-07-20 10:29:05 -07002445 return;
2446 }
Mathieu Chartierea0831f2015-12-29 13:17:37 -08002447 Runtime::Current()->GetInternTable()->AddNewTable();
Mathieu Chartierc2e20622014-11-03 11:41:47 -08002448 Runtime::Current()->GetClassLinker()->MoveClassTableToPreZygote();
Mathieu Chartier590fee92013-09-13 13:46:47 -07002449 VLOG(heap) << "Starting PreZygoteFork";
Mathieu Chartier31f44142014-04-08 14:40:03 -07002450 // The end of the non-moving space may be protected, unprotect it so that we can copy the zygote
2451 // there.
Mathieu Chartier590fee92013-09-13 13:46:47 -07002452 non_moving_space_->GetMemMap()->Protect(PROT_READ | PROT_WRITE);
Mathieu Chartier6a7824d2014-08-22 14:53:04 -07002453 const bool same_space = non_moving_space_ == main_space_;
Mathieu Chartier31f44142014-04-08 14:40:03 -07002454 if (kCompactZygote) {
Hiroshi Yamauchia4adbfd2014-02-04 18:12:17 -08002455 // Temporarily disable rosalloc verification because the zygote
2456 // compaction will mess up the rosalloc internal metadata.
2457 ScopedDisableRosAllocVerification disable_rosalloc_verif(this);
Evgenii Stepanov1e133742015-05-20 12:30:59 -07002458 ZygoteCompactingCollector zygote_collector(this, is_running_on_memory_tool_);
Mathieu Chartier85a43c02014-01-07 17:59:00 -08002459 zygote_collector.BuildBins(non_moving_space_);
Mathieu Chartier50482232013-11-21 11:48:14 -08002460 // Create a new bump pointer space which we will compact into.
Mathieu Chartier590fee92013-09-13 13:46:47 -07002461 space::BumpPointerSpace target_space("zygote bump space", non_moving_space_->End(),
2462 non_moving_space_->Limit());
2463 // Compact the bump pointer space to a new zygote bump pointer space.
Mathieu Chartier31f44142014-04-08 14:40:03 -07002464 bool reset_main_space = false;
2465 if (IsMovingGc(collector_type_)) {
Hiroshi Yamauchi2cd334a2015-01-09 14:03:35 -08002466 if (collector_type_ == kCollectorTypeCC) {
2467 zygote_collector.SetFromSpace(region_space_);
2468 } else {
2469 zygote_collector.SetFromSpace(bump_pointer_space_);
2470 }
Mathieu Chartier31f44142014-04-08 14:40:03 -07002471 } else {
2472 CHECK(main_space_ != nullptr);
Hiroshi Yamauchid04495e2015-03-11 19:09:07 -07002473 CHECK_NE(main_space_, non_moving_space_)
2474 << "Does not make sense to compact within the same space";
Mathieu Chartier31f44142014-04-08 14:40:03 -07002475 // Copy from the main space.
2476 zygote_collector.SetFromSpace(main_space_);
2477 reset_main_space = true;
2478 }
Mathieu Chartier85a43c02014-01-07 17:59:00 -08002479 zygote_collector.SetToSpace(&target_space);
Mathieu Chartier1b54f9c2014-04-30 16:45:02 -07002480 zygote_collector.SetSwapSemiSpaces(false);
Hiroshi Yamauchi6f4ffe42014-01-13 12:30:44 -08002481 zygote_collector.Run(kGcCauseCollectorTransition, false);
Mathieu Chartier31f44142014-04-08 14:40:03 -07002482 if (reset_main_space) {
2483 main_space_->GetMemMap()->Protect(PROT_READ | PROT_WRITE);
2484 madvise(main_space_->Begin(), main_space_->Capacity(), MADV_DONTNEED);
2485 MemMap* mem_map = main_space_->ReleaseMemMap();
2486 RemoveSpace(main_space_);
Mathieu Chartier96bcd452014-06-17 09:50:02 -07002487 space::Space* old_main_space = main_space_;
Mathieu Chartier0310da52014-12-01 13:40:48 -08002488 CreateMainMallocSpace(mem_map, kDefaultInitialSize, std::min(mem_map->Size(), growth_limit_),
2489 mem_map->Size());
Mathieu Chartier96bcd452014-06-17 09:50:02 -07002490 delete old_main_space;
Mathieu Chartier31f44142014-04-08 14:40:03 -07002491 AddSpace(main_space_);
2492 } else {
Hiroshi Yamauchi2cd334a2015-01-09 14:03:35 -08002493 if (collector_type_ == kCollectorTypeCC) {
2494 region_space_->GetMemMap()->Protect(PROT_READ | PROT_WRITE);
2495 } else {
2496 bump_pointer_space_->GetMemMap()->Protect(PROT_READ | PROT_WRITE);
2497 }
Mathieu Chartier31f44142014-04-08 14:40:03 -07002498 }
2499 if (temp_space_ != nullptr) {
2500 CHECK(temp_space_->IsEmpty());
2501 }
Mathieu Chartier10fb83a2014-06-15 15:15:43 -07002502 total_objects_freed_ever_ += GetCurrentGcIteration()->GetFreedObjects();
2503 total_bytes_freed_ever_ += GetCurrentGcIteration()->GetFreedBytes();
Mathieu Chartier590fee92013-09-13 13:46:47 -07002504 // Update the end and write out image.
2505 non_moving_space_->SetEnd(target_space.End());
2506 non_moving_space_->SetLimit(target_space.Limit());
Mathieu Chartierfaed9952015-03-31 16:28:53 -07002507 VLOG(heap) << "Create zygote space with size=" << non_moving_space_->Size() << " bytes";
Mathieu Chartier357e9be2012-08-01 11:00:14 -07002508 }
Mathieu Chartier6a7824d2014-08-22 14:53:04 -07002509 // Change the collector to the post zygote one.
Mathieu Chartier31f44142014-04-08 14:40:03 -07002510 ChangeCollector(foreground_collector_type_);
Mathieu Chartiera1602f22014-01-13 17:19:19 -08002511 // Save the old space so that we can remove it after we complete creating the zygote space.
2512 space::MallocSpace* old_alloc_space = non_moving_space_;
Mathieu Chartier590fee92013-09-13 13:46:47 -07002513 // Turn the current alloc space into a zygote space and obtain the new alloc space composed of
Mathieu Chartiera1602f22014-01-13 17:19:19 -08002514 // the remaining available space.
2515 // Remove the old space before creating the zygote space since creating the zygote space sets
Mathieu Chartier2cebb242015-04-21 16:50:40 -07002516 // the old alloc space's bitmaps to null.
Mathieu Chartiera1602f22014-01-13 17:19:19 -08002517 RemoveSpace(old_alloc_space);
Hiroshi Yamauchi38e68e92014-03-07 13:59:08 -08002518 if (collector::SemiSpace::kUseRememberedSet) {
2519 // Sanity bound check.
2520 FindRememberedSetFromSpace(old_alloc_space)->AssertAllDirtyCardsAreWithinSpace();
2521 // Remove the remembered set for the now zygote space (the old
2522 // non-moving space). Note now that we have compacted objects into
2523 // the zygote space, the data in the remembered set is no longer
2524 // needed. The zygote space will instead have a mod-union table
2525 // from this point on.
2526 RemoveRememberedSet(old_alloc_space);
2527 }
Mathieu Chartier7247af52014-11-19 10:51:42 -08002528 // Remaining space becomes the new non moving space.
2529 zygote_space_ = old_alloc_space->CreateZygoteSpace(kNonMovingSpaceName, low_memory_mode_,
Mathieu Chartiere4cab172014-08-19 18:24:04 -07002530 &non_moving_space_);
Mathieu Chartierb363f662014-07-16 13:28:58 -07002531 CHECK(!non_moving_space_->CanMoveObjects());
2532 if (same_space) {
2533 main_space_ = non_moving_space_;
2534 SetSpaceAsDefault(main_space_);
2535 }
Mathieu Chartiera1602f22014-01-13 17:19:19 -08002536 delete old_alloc_space;
Mathieu Chartiere4cab172014-08-19 18:24:04 -07002537 CHECK(HasZygoteSpace()) << "Failed creating zygote space";
2538 AddSpace(zygote_space_);
Mathieu Chartier31f44142014-04-08 14:40:03 -07002539 non_moving_space_->SetFootprintLimit(non_moving_space_->Capacity());
2540 AddSpace(non_moving_space_);
Mathieu Chartier11409ae2013-09-23 11:49:36 -07002541 // Create the zygote space mod union table.
2542 accounting::ModUnionTable* mod_union_table =
Mathieu Chartiere4cab172014-08-19 18:24:04 -07002543 new accounting::ModUnionTableCardCache("zygote space mod-union table", this,
2544 zygote_space_);
Mathieu Chartier11409ae2013-09-23 11:49:36 -07002545 CHECK(mod_union_table != nullptr) << "Failed to create zygote space mod-union table";
Mathieu Chartiere4cab172014-08-19 18:24:04 -07002546 // Set all the cards in the mod-union table since we don't know which objects contain references
2547 // to large objects.
2548 mod_union_table->SetCards();
Mathieu Chartier11409ae2013-09-23 11:49:36 -07002549 AddModUnionTable(mod_union_table);
Mathieu Chartierf6c2a272015-06-03 17:32:42 -07002550 large_object_space_->SetAllLargeObjectsAsZygoteObjects(self);
Hiroshi Yamauchi38e68e92014-03-07 13:59:08 -08002551 if (collector::SemiSpace::kUseRememberedSet) {
Hiroshi Yamauchi38e68e92014-03-07 13:59:08 -08002552 // Add a new remembered set for the post-zygote non-moving space.
2553 accounting::RememberedSet* post_zygote_non_moving_space_rem_set =
2554 new accounting::RememberedSet("Post-zygote non-moving space remembered set", this,
2555 non_moving_space_);
2556 CHECK(post_zygote_non_moving_space_rem_set != nullptr)
2557 << "Failed to create post-zygote non-moving space remembered set";
2558 AddRememberedSet(post_zygote_non_moving_space_rem_set);
2559 }
Mathieu Chartiercc236d72012-07-20 10:29:05 -07002560}
2561
Mathieu Chartier357e9be2012-08-01 11:00:14 -07002562void Heap::FlushAllocStack() {
Mathieu Chartiere6da9af2013-12-16 11:54:42 -08002563 MarkAllocStackAsLive(allocation_stack_.get());
Mathieu Chartier357e9be2012-08-01 11:00:14 -07002564 allocation_stack_->Reset();
2565}
2566
Mathieu Chartiera8e8f9c2014-04-09 14:51:05 -07002567void Heap::MarkAllocStack(accounting::ContinuousSpaceBitmap* bitmap1,
2568 accounting::ContinuousSpaceBitmap* bitmap2,
Mathieu Chartierbbd695c2014-04-16 09:48:48 -07002569 accounting::LargeObjectBitmap* large_objects,
Ian Rogers1d54e732013-05-02 21:10:01 -07002570 accounting::ObjectStack* stack) {
Mathieu Chartiere6da9af2013-12-16 11:54:42 -08002571 DCHECK(bitmap1 != nullptr);
2572 DCHECK(bitmap2 != nullptr);
Mathieu Chartiercb535da2015-01-23 13:50:03 -08002573 const auto* limit = stack->End();
2574 for (auto* it = stack->Begin(); it != limit; ++it) {
2575 const mirror::Object* obj = it->AsMirrorPtr();
Hiroshi Yamauchif5b0e202014-02-11 17:02:22 -08002576 if (!kUseThreadLocalAllocationStack || obj != nullptr) {
2577 if (bitmap1->HasAddress(obj)) {
2578 bitmap1->Set(obj);
2579 } else if (bitmap2->HasAddress(obj)) {
2580 bitmap2->Set(obj);
2581 } else {
Mathieu Chartier2dbe6272014-09-16 10:43:23 -07002582 DCHECK(large_objects != nullptr);
Hiroshi Yamauchif5b0e202014-02-11 17:02:22 -08002583 large_objects->Set(obj);
2584 }
Mathieu Chartiere0f0cb32012-08-28 11:26:00 -07002585 }
Mathieu Chartier357e9be2012-08-01 11:00:14 -07002586 }
2587}
2588
Mathieu Chartier590fee92013-09-13 13:46:47 -07002589void Heap::SwapSemiSpaces() {
Mathieu Chartier31f44142014-04-08 14:40:03 -07002590 CHECK(bump_pointer_space_ != nullptr);
2591 CHECK(temp_space_ != nullptr);
Mathieu Chartier590fee92013-09-13 13:46:47 -07002592 std::swap(bump_pointer_space_, temp_space_);
2593}
2594
Hiroshi Yamauchie4d99872015-02-26 12:53:45 -08002595collector::GarbageCollector* Heap::Compact(space::ContinuousMemMapAllocSpace* target_space,
2596 space::ContinuousMemMapAllocSpace* source_space,
2597 GcCause gc_cause) {
Mathieu Chartier590fee92013-09-13 13:46:47 -07002598 CHECK(kMovingCollector);
Mathieu Chartier590fee92013-09-13 13:46:47 -07002599 if (target_space != source_space) {
Mathieu Chartier6f365cc2014-04-23 12:42:27 -07002600 // Don't swap spaces since this isn't a typical semi space collection.
2601 semi_space_collector_->SetSwapSemiSpaces(false);
Mathieu Chartier590fee92013-09-13 13:46:47 -07002602 semi_space_collector_->SetFromSpace(source_space);
2603 semi_space_collector_->SetToSpace(target_space);
Zuo Wangf37a88b2014-07-10 04:26:41 -07002604 semi_space_collector_->Run(gc_cause, false);
Hiroshi Yamauchie4d99872015-02-26 12:53:45 -08002605 return semi_space_collector_;
Mathieu Chartier52e4b432014-06-10 11:22:31 -07002606 } else {
2607 CHECK(target_space->IsBumpPointerSpace())
2608 << "In-place compaction is only supported for bump pointer spaces";
2609 mark_compact_collector_->SetSpace(target_space->AsBumpPointerSpace());
2610 mark_compact_collector_->Run(kGcCauseCollectorTransition, false);
Hiroshi Yamauchie4d99872015-02-26 12:53:45 -08002611 return mark_compact_collector_;
Mathieu Chartier590fee92013-09-13 13:46:47 -07002612 }
2613}
Anwar Ghuloum67f99412013-08-12 14:19:48 -07002614
Mathieu Chartiera4f6af92015-08-11 17:35:25 -07002615collector::GcType Heap::CollectGarbageInternal(collector::GcType gc_type,
2616 GcCause gc_cause,
Ian Rogers1d54e732013-05-02 21:10:01 -07002617 bool clear_soft_references) {
Ian Rogers81d425b2012-09-27 16:03:43 -07002618 Thread* self = Thread::Current();
Mathieu Chartier590fee92013-09-13 13:46:47 -07002619 Runtime* runtime = Runtime::Current();
Mathieu Chartiercbb2d202013-11-14 17:45:16 -08002620 // If the heap can't run the GC, silently fail and return that no GC was run.
2621 switch (gc_type) {
Mathieu Chartiercbb2d202013-11-14 17:45:16 -08002622 case collector::kGcTypePartial: {
Mathieu Chartiere4cab172014-08-19 18:24:04 -07002623 if (!HasZygoteSpace()) {
Mathieu Chartiercbb2d202013-11-14 17:45:16 -08002624 return collector::kGcTypeNone;
2625 }
2626 break;
2627 }
2628 default: {
2629 // Other GC types don't have any special cases which makes them not runnable. The main case
2630 // here is full GC.
2631 }
2632 }
Mathieu Chartier65db8802012-11-20 12:36:46 -08002633 ScopedThreadStateChange tsc(self, kWaitingPerformingGc);
Ian Rogers81d425b2012-09-27 16:03:43 -07002634 Locks::mutator_lock_->AssertNotHeld(self);
Ian Rogers120f1c72012-09-28 17:17:10 -07002635 if (self->IsHandlingStackOverflow()) {
Mathieu Chartier50c138f2015-01-07 16:00:03 -08002636 // If we are throwing a stack overflow error we probably don't have enough remaining stack
2637 // space to run the GC.
2638 return collector::kGcTypeNone;
Ian Rogers120f1c72012-09-28 17:17:10 -07002639 }
Mathieu Chartierd5a89ee2014-01-31 09:55:13 -08002640 bool compacting_gc;
2641 {
2642 gc_complete_lock_->AssertNotHeld(self);
Andreas Gampe277ccbd2014-11-03 21:36:10 -08002643 ScopedThreadStateChange tsc2(self, kWaitingForGcToComplete);
Mathieu Chartierd5a89ee2014-01-31 09:55:13 -08002644 MutexLock mu(self, *gc_complete_lock_);
2645 // Ensure there is only one GC at a time.
Mathieu Chartier89a201e2014-05-02 10:27:26 -07002646 WaitForGcToCompleteLocked(gc_cause, self);
Mathieu Chartier31f44142014-04-08 14:40:03 -07002647 compacting_gc = IsMovingGc(collector_type_);
Mathieu Chartierd5a89ee2014-01-31 09:55:13 -08002648 // GC can be disabled if someone has a used GetPrimitiveArrayCritical.
2649 if (compacting_gc && disable_moving_gc_count_ != 0) {
2650 LOG(WARNING) << "Skipping GC due to disable moving GC count " << disable_moving_gc_count_;
2651 return collector::kGcTypeNone;
2652 }
Mathieu Chartier51168372015-08-12 16:40:32 -07002653 if (gc_disabled_for_shutdown_) {
2654 return collector::kGcTypeNone;
2655 }
Mathieu Chartierd5a89ee2014-01-31 09:55:13 -08002656 collector_type_running_ = collector_type_;
Ian Rogers00f7d0e2012-07-19 15:28:27 -07002657 }
Mathieu Chartier590fee92013-09-13 13:46:47 -07002658 if (gc_cause == kGcCauseForAlloc && runtime->HasStatsEnabled()) {
2659 ++runtime->GetStats()->gc_for_alloc_count;
2660 ++self->GetStats()->gc_for_alloc_count;
Mathieu Chartier2fde5332012-09-14 14:51:54 -07002661 }
Mathieu Chartiere2c2f6e2014-12-16 18:49:31 -08002662 const uint64_t bytes_allocated_before_gc = GetBytesAllocated();
2663 // Approximate heap size.
2664 ATRACE_INT("Heap size (KB)", bytes_allocated_before_gc / KB);
Mathieu Chartier65db8802012-11-20 12:36:46 -08002665
Ian Rogers1d54e732013-05-02 21:10:01 -07002666 DCHECK_LT(gc_type, collector::kGcTypeMax);
2667 DCHECK_NE(gc_type, collector::kGcTypeNone);
Anwar Ghuloum67f99412013-08-12 14:19:48 -07002668
Mathieu Chartier590fee92013-09-13 13:46:47 -07002669 collector::GarbageCollector* collector = nullptr;
Mathieu Chartier50482232013-11-21 11:48:14 -08002670 // TODO: Clean this up.
Mathieu Chartier1d27b342014-01-28 12:51:09 -08002671 if (compacting_gc) {
Mathieu Chartier692fafd2013-11-29 17:24:40 -08002672 DCHECK(current_allocator_ == kAllocatorTypeBumpPointer ||
Hiroshi Yamauchi2cd334a2015-01-09 14:03:35 -08002673 current_allocator_ == kAllocatorTypeTLAB ||
2674 current_allocator_ == kAllocatorTypeRegion ||
2675 current_allocator_ == kAllocatorTypeRegionTLAB);
Mathieu Chartier52e4b432014-06-10 11:22:31 -07002676 switch (collector_type_) {
2677 case kCollectorTypeSS:
2678 // Fall-through.
2679 case kCollectorTypeGSS:
2680 semi_space_collector_->SetFromSpace(bump_pointer_space_);
2681 semi_space_collector_->SetToSpace(temp_space_);
2682 semi_space_collector_->SetSwapSemiSpaces(true);
2683 collector = semi_space_collector_;
2684 break;
2685 case kCollectorTypeCC:
Hiroshi Yamauchi2cd334a2015-01-09 14:03:35 -08002686 concurrent_copying_collector_->SetRegionSpace(region_space_);
Mathieu Chartier52e4b432014-06-10 11:22:31 -07002687 collector = concurrent_copying_collector_;
2688 break;
2689 case kCollectorTypeMC:
2690 mark_compact_collector_->SetSpace(bump_pointer_space_);
2691 collector = mark_compact_collector_;
2692 break;
2693 default:
2694 LOG(FATAL) << "Invalid collector type " << static_cast<size_t>(collector_type_);
Hiroshi Yamauchid5307ec2014-03-27 21:07:51 -07002695 }
Hiroshi Yamauchi2cd334a2015-01-09 14:03:35 -08002696 if (collector != mark_compact_collector_ && collector != concurrent_copying_collector_) {
Mathieu Chartier52e4b432014-06-10 11:22:31 -07002697 temp_space_->GetMemMap()->Protect(PROT_READ | PROT_WRITE);
Hiroshi Yamauchi6edb9ae2016-02-08 14:18:21 -08002698 if (kIsDebugBuild) {
2699 // Try to read each page of the memory map in case mprotect didn't work properly b/19894268.
2700 temp_space_->GetMemMap()->TryReadable();
2701 }
Mathieu Chartier52e4b432014-06-10 11:22:31 -07002702 CHECK(temp_space_->IsEmpty());
2703 }
2704 gc_type = collector::kGcTypeFull; // TODO: Not hard code this in.
Mathieu Chartiere6da9af2013-12-16 11:54:42 -08002705 } else if (current_allocator_ == kAllocatorTypeRosAlloc ||
2706 current_allocator_ == kAllocatorTypeDlMalloc) {
Mathieu Chartierafe49982014-03-27 10:55:04 -07002707 collector = FindCollectorByGcType(gc_type);
Mathieu Chartier50482232013-11-21 11:48:14 -08002708 } else {
2709 LOG(FATAL) << "Invalid current allocator " << current_allocator_;
Mathieu Chartier590fee92013-09-13 13:46:47 -07002710 }
Mathieu Chartier08cef222014-10-22 17:18:34 -07002711 if (IsGcConcurrent()) {
2712 // Disable concurrent GC check so that we don't have spammy JNI requests.
2713 // This gets recalculated in GrowForUtilization. It is important that it is disabled /
2714 // calculated in the same thread so that there aren't any races that can cause it to become
2715 // permanantly disabled. b/17942071
2716 concurrent_start_bytes_ = std::numeric_limits<size_t>::max();
2717 }
Nicolas Geoffrayb6e20ae2016-03-07 14:29:04 +00002718
Calin Juravleffc87072016-04-20 14:22:09 +01002719 // It's time to clear all inline caches, in case some classes can be unloaded.
2720 if ((gc_type == collector::kGcTypeFull) && (runtime->GetJit() != nullptr)) {
Nicolas Geoffrayb6e20ae2016-03-07 14:29:04 +00002721 runtime->GetJit()->GetCodeCache()->ClearGcRootsInInlineCaches(self);
2722 }
2723
Mathieu Chartier7bf82af2013-12-06 16:51:45 -08002724 CHECK(collector != nullptr)
Hiroshi Yamauchi3e417802014-03-20 12:03:02 -07002725 << "Could not find garbage collector with collector_type="
2726 << static_cast<size_t>(collector_type_) << " and gc_type=" << gc_type;
Mathieu Chartier6f365cc2014-04-23 12:42:27 -07002727 collector->Run(gc_cause, clear_soft_references || runtime->IsZygote());
Mathieu Chartier10fb83a2014-06-15 15:15:43 -07002728 total_objects_freed_ever_ += GetCurrentGcIteration()->GetFreedObjects();
2729 total_bytes_freed_ever_ += GetCurrentGcIteration()->GetFreedBytes();
Mathieu Chartiera5eae692014-12-17 17:56:03 -08002730 RequestTrim(self);
Mathieu Chartier39e32612013-11-12 16:28:05 -08002731 // Enqueue cleared references.
Mathieu Chartier3cf22532015-07-09 15:15:09 -07002732 reference_processor_->EnqueueClearedReferences(self);
Mathieu Chartier590fee92013-09-13 13:46:47 -07002733 // Grow the heap so that we know when to perform the next GC.
Mathieu Chartiere2c2f6e2014-12-16 18:49:31 -08002734 GrowForUtilization(collector, bytes_allocated_before_gc);
Hiroshi Yamauchie4d99872015-02-26 12:53:45 -08002735 LogGC(gc_cause, collector);
2736 FinishGC(self, gc_type);
2737 // Inform DDMS that a GC completed.
2738 Dbg::GcDidFinish();
Mathieu Chartier598302a2015-09-23 14:52:39 -07002739 // Unload native libraries for class unloading. We do this after calling FinishGC to prevent
2740 // deadlocks in case the JNI_OnUnload function does allocations.
2741 {
2742 ScopedObjectAccess soa(self);
2743 soa.Vm()->UnloadNativeLibraries();
2744 }
Hiroshi Yamauchie4d99872015-02-26 12:53:45 -08002745 return gc_type;
2746}
2747
2748void Heap::LogGC(GcCause gc_cause, collector::GarbageCollector* collector) {
Mathieu Chartier10fb83a2014-06-15 15:15:43 -07002749 const size_t duration = GetCurrentGcIteration()->GetDurationNs();
2750 const std::vector<uint64_t>& pause_times = GetCurrentGcIteration()->GetPauseTimes();
Mathieu Chartier62ab87b2014-04-28 12:22:07 -07002751 // Print the GC if it is an explicit GC (e.g. Runtime.gc()) or a slow GC
Mathieu Chartierf5997b42014-06-20 10:37:54 -07002752 // (mutator time blocked >= long_pause_log_threshold_).
Mathieu Chartier62ab87b2014-04-28 12:22:07 -07002753 bool log_gc = gc_cause == kGcCauseExplicit;
2754 if (!log_gc && CareAboutPauseTimes()) {
Mathieu Chartiere53225c2013-08-19 10:59:11 -07002755 // GC for alloc pauses the allocating thread, so consider it as a pause.
Mathieu Chartier62ab87b2014-04-28 12:22:07 -07002756 log_gc = duration > long_gc_log_threshold_ ||
Mathieu Chartiere6da9af2013-12-16 11:54:42 -08002757 (gc_cause == kGcCauseForAlloc && duration > long_pause_log_threshold_);
Mathieu Chartier62ab87b2014-04-28 12:22:07 -07002758 for (uint64_t pause : pause_times) {
2759 log_gc = log_gc || pause >= long_pause_log_threshold_;
Mathieu Chartiere53225c2013-08-19 10:59:11 -07002760 }
Mathieu Chartier62ab87b2014-04-28 12:22:07 -07002761 }
2762 if (log_gc) {
2763 const size_t percent_free = GetPercentFree();
2764 const size_t current_heap_size = GetBytesAllocated();
2765 const size_t total_memory = GetTotalMemory();
2766 std::ostringstream pause_string;
2767 for (size_t i = 0; i < pause_times.size(); ++i) {
Hiroshi Yamauchie4d99872015-02-26 12:53:45 -08002768 pause_string << PrettyDuration((pause_times[i] / 1000) * 1000)
2769 << ((i != pause_times.size() - 1) ? "," : "");
Mathieu Chartiere53225c2013-08-19 10:59:11 -07002770 }
Mathieu Chartier62ab87b2014-04-28 12:22:07 -07002771 LOG(INFO) << gc_cause << " " << collector->GetName()
Mathieu Chartier10fb83a2014-06-15 15:15:43 -07002772 << " GC freed " << current_gc_iteration_.GetFreedObjects() << "("
2773 << PrettySize(current_gc_iteration_.GetFreedBytes()) << ") AllocSpace objects, "
2774 << current_gc_iteration_.GetFreedLargeObjects() << "("
2775 << PrettySize(current_gc_iteration_.GetFreedLargeObjectBytes()) << ") LOS objects, "
Mathieu Chartier62ab87b2014-04-28 12:22:07 -07002776 << percent_free << "% free, " << PrettySize(current_heap_size) << "/"
2777 << PrettySize(total_memory) << ", " << "paused " << pause_string.str()
2778 << " total " << PrettyDuration((duration / 1000) * 1000);
Ian Rogersc7dd2952014-10-21 23:31:19 -07002779 VLOG(heap) << Dumpable<TimingLogger>(*current_gc_iteration_.GetTimings());
Mathieu Chartier2b82db42012-11-14 17:29:05 -08002780 }
Ian Rogers00f7d0e2012-07-19 15:28:27 -07002781}
Mathieu Chartiera6399032012-06-11 18:49:50 -07002782
Mathieu Chartiere6da9af2013-12-16 11:54:42 -08002783void Heap::FinishGC(Thread* self, collector::GcType gc_type) {
2784 MutexLock mu(self, *gc_complete_lock_);
Mathieu Chartierd5a89ee2014-01-31 09:55:13 -08002785 collector_type_running_ = kCollectorTypeNone;
2786 if (gc_type != collector::kGcTypeNone) {
2787 last_gc_type_ = gc_type;
Hiroshi Yamauchia1c9f012015-04-02 10:18:12 -07002788
2789 // Update stats.
2790 ++gc_count_last_window_;
2791 if (running_collection_is_blocking_) {
2792 // If the currently running collection was a blocking one,
2793 // increment the counters and reset the flag.
2794 ++blocking_gc_count_;
2795 blocking_gc_time_ += GetCurrentGcIteration()->GetDurationNs();
2796 ++blocking_gc_count_last_window_;
2797 }
2798 // Update the gc count rate histograms if due.
2799 UpdateGcCountRateHistograms();
Mathieu Chartierd5a89ee2014-01-31 09:55:13 -08002800 }
Hiroshi Yamauchia1c9f012015-04-02 10:18:12 -07002801 // Reset.
2802 running_collection_is_blocking_ = false;
Mathieu Chartiere6da9af2013-12-16 11:54:42 -08002803 // Wake anyone who may have been waiting for the GC to complete.
2804 gc_complete_cond_->Broadcast(self);
2805}
2806
Hiroshi Yamauchia1c9f012015-04-02 10:18:12 -07002807void Heap::UpdateGcCountRateHistograms() {
2808 // Invariant: if the time since the last update includes more than
2809 // one windows, all the GC runs (if > 0) must have happened in first
2810 // window because otherwise the update must have already taken place
2811 // at an earlier GC run. So, we report the non-first windows with
2812 // zero counts to the histograms.
2813 DCHECK_EQ(last_update_time_gc_count_rate_histograms_ % kGcCountRateHistogramWindowDuration, 0U);
2814 uint64_t now = NanoTime();
2815 DCHECK_GE(now, last_update_time_gc_count_rate_histograms_);
2816 uint64_t time_since_last_update = now - last_update_time_gc_count_rate_histograms_;
2817 uint64_t num_of_windows = time_since_last_update / kGcCountRateHistogramWindowDuration;
2818 if (time_since_last_update >= kGcCountRateHistogramWindowDuration) {
2819 // Record the first window.
2820 gc_count_rate_histogram_.AddValue(gc_count_last_window_ - 1); // Exclude the current run.
2821 blocking_gc_count_rate_histogram_.AddValue(running_collection_is_blocking_ ?
2822 blocking_gc_count_last_window_ - 1 : blocking_gc_count_last_window_);
2823 // Record the other windows (with zero counts).
2824 for (uint64_t i = 0; i < num_of_windows - 1; ++i) {
2825 gc_count_rate_histogram_.AddValue(0);
2826 blocking_gc_count_rate_histogram_.AddValue(0);
2827 }
2828 // Update the last update time and reset the counters.
2829 last_update_time_gc_count_rate_histograms_ =
2830 (now / kGcCountRateHistogramWindowDuration) * kGcCountRateHistogramWindowDuration;
2831 gc_count_last_window_ = 1; // Include the current run.
2832 blocking_gc_count_last_window_ = running_collection_is_blocking_ ? 1 : 0;
2833 }
2834 DCHECK_EQ(last_update_time_gc_count_rate_histograms_ % kGcCountRateHistogramWindowDuration, 0U);
2835}
2836
Mathieu Chartierbb87e0f2015-04-03 11:21:55 -07002837class RootMatchesObjectVisitor : public SingleRootVisitor {
2838 public:
2839 explicit RootMatchesObjectVisitor(const mirror::Object* obj) : obj_(obj) { }
2840
2841 void VisitRoot(mirror::Object* root, const RootInfo& info)
Mathieu Chartier90443472015-07-16 20:32:27 -07002842 OVERRIDE SHARED_REQUIRES(Locks::mutator_lock_) {
Mathieu Chartierbb87e0f2015-04-03 11:21:55 -07002843 if (root == obj_) {
2844 LOG(INFO) << "Object " << obj_ << " is a root " << info.ToString();
2845 }
Mathieu Chartierfd678be2012-08-30 14:50:54 -07002846 }
Mathieu Chartierbb87e0f2015-04-03 11:21:55 -07002847
2848 private:
2849 const mirror::Object* const obj_;
2850};
2851
Mathieu Chartierfd678be2012-08-30 14:50:54 -07002852
2853class ScanVisitor {
2854 public:
Brian Carlstromdf629502013-07-17 22:39:56 -07002855 void operator()(const mirror::Object* obj) const {
Mathieu Chartier0f72e412013-09-06 16:40:01 -07002856 LOG(ERROR) << "Would have rescanned object " << obj;
Mathieu Chartierfd678be2012-08-30 14:50:54 -07002857 }
2858};
2859
Ian Rogers1d54e732013-05-02 21:10:01 -07002860// Verify a reference from an object.
Mathieu Chartierbb87e0f2015-04-03 11:21:55 -07002861class VerifyReferenceVisitor : public SingleRootVisitor {
Mathieu Chartierfd678be2012-08-30 14:50:54 -07002862 public:
Roland Levillain3887c462015-08-12 18:15:42 +01002863 VerifyReferenceVisitor(Heap* heap, Atomic<size_t>* fail_count, bool verify_referent)
Mathieu Chartier90443472015-07-16 20:32:27 -07002864 SHARED_REQUIRES(Locks::mutator_lock_, Locks::heap_bitmap_lock_)
Mathieu Chartier8ab7e782014-05-19 16:55:27 -07002865 : heap_(heap), fail_count_(fail_count), verify_referent_(verify_referent) {}
Ian Rogers1d54e732013-05-02 21:10:01 -07002866
Mathieu Chartier8ab7e782014-05-19 16:55:27 -07002867 size_t GetFailureCount() const {
Mathieu Chartiere9e55ac2014-05-21 17:48:25 -07002868 return fail_count_->LoadSequentiallyConsistent();
Mathieu Chartierfd678be2012-08-30 14:50:54 -07002869 }
2870
Mathieu Chartierda7c6502015-07-23 16:01:26 -07002871 void operator()(mirror::Class* klass ATTRIBUTE_UNUSED, mirror::Reference* ref) const
Mathieu Chartier90443472015-07-16 20:32:27 -07002872 SHARED_REQUIRES(Locks::mutator_lock_) {
Mathieu Chartier78f7b4c2014-05-06 10:57:27 -07002873 if (verify_referent_) {
Mathieu Chartier8ab7e782014-05-19 16:55:27 -07002874 VerifyReference(ref, ref->GetReferent(), mirror::Reference::ReferentOffset());
Mathieu Chartier78f7b4c2014-05-06 10:57:27 -07002875 }
Mathieu Chartier407f7022014-02-18 14:37:05 -08002876 }
2877
Mathieu Chartierda7c6502015-07-23 16:01:26 -07002878 void operator()(mirror::Object* obj, MemberOffset offset, bool is_static ATTRIBUTE_UNUSED) const
Mathieu Chartier90443472015-07-16 20:32:27 -07002879 SHARED_REQUIRES(Locks::mutator_lock_) {
Mathieu Chartier8ab7e782014-05-19 16:55:27 -07002880 VerifyReference(obj, obj->GetFieldObject<mirror::Object>(offset), offset);
Mathieu Chartier407f7022014-02-18 14:37:05 -08002881 }
2882
Mathieu Chartier8ab7e782014-05-19 16:55:27 -07002883 bool IsLive(mirror::Object* obj) const NO_THREAD_SAFETY_ANALYSIS {
2884 return heap_->IsLiveObjectLocked(obj, true, false, true);
2885 }
2886
Mathieu Chartierda7c6502015-07-23 16:01:26 -07002887 void VisitRootIfNonNull(mirror::CompressedReference<mirror::Object>* root) const
2888 SHARED_REQUIRES(Locks::mutator_lock_) {
2889 if (!root->IsNull()) {
2890 VisitRoot(root);
2891 }
2892 }
2893 void VisitRoot(mirror::CompressedReference<mirror::Object>* root) const
2894 SHARED_REQUIRES(Locks::mutator_lock_) {
2895 const_cast<VerifyReferenceVisitor*>(this)->VisitRoot(
2896 root->AsMirrorPtr(), RootInfo(kRootVMInternal));
2897 }
2898
2899 virtual void VisitRoot(mirror::Object* root, const RootInfo& root_info) OVERRIDE
Mathieu Chartier90443472015-07-16 20:32:27 -07002900 SHARED_REQUIRES(Locks::mutator_lock_) {
Mathieu Chartierbb87e0f2015-04-03 11:21:55 -07002901 if (root == nullptr) {
2902 LOG(ERROR) << "Root is null with info " << root_info.GetType();
2903 } else if (!VerifyReference(nullptr, root, MemberOffset(0))) {
2904 LOG(ERROR) << "Root " << root << " is dead with type " << PrettyTypeOf(root)
Mathieu Chartiere34fa1d2015-01-14 14:55:47 -08002905 << " thread_id= " << root_info.GetThreadId() << " root_type= " << root_info.GetType();
Mathieu Chartier8ab7e782014-05-19 16:55:27 -07002906 }
2907 }
2908
2909 private:
Mathieu Chartier407f7022014-02-18 14:37:05 -08002910 // TODO: Fix the no thread safety analysis.
Mathieu Chartier8ab7e782014-05-19 16:55:27 -07002911 // Returns false on failure.
2912 bool VerifyReference(mirror::Object* obj, mirror::Object* ref, MemberOffset offset) const
Ian Rogers2dd0e2c2013-01-24 12:42:14 -08002913 NO_THREAD_SAFETY_ANALYSIS {
Mathieu Chartier938a03b2014-01-16 15:10:31 -08002914 if (ref == nullptr || IsLive(ref)) {
2915 // Verify that the reference is live.
Mathieu Chartier8ab7e782014-05-19 16:55:27 -07002916 return true;
Mathieu Chartier938a03b2014-01-16 15:10:31 -08002917 }
Mathieu Chartiere9e55ac2014-05-21 17:48:25 -07002918 if (fail_count_->FetchAndAddSequentiallyConsistent(1) == 0) {
Mathieu Chartier938a03b2014-01-16 15:10:31 -08002919 // Print message on only on first failure to prevent spam.
2920 LOG(ERROR) << "!!!!!!!!!!!!!!Heap corruption detected!!!!!!!!!!!!!!!!!!!";
Mathieu Chartier938a03b2014-01-16 15:10:31 -08002921 }
2922 if (obj != nullptr) {
Mathieu Chartier8ab7e782014-05-19 16:55:27 -07002923 // Only do this part for non roots.
Ian Rogers1d54e732013-05-02 21:10:01 -07002924 accounting::CardTable* card_table = heap_->GetCardTable();
2925 accounting::ObjectStack* alloc_stack = heap_->allocation_stack_.get();
2926 accounting::ObjectStack* live_stack = heap_->live_stack_.get();
Ian Rogers13735952014-10-08 12:43:28 -07002927 uint8_t* card_addr = card_table->CardFromAddr(obj);
Mathieu Chartier938a03b2014-01-16 15:10:31 -08002928 LOG(ERROR) << "Object " << obj << " references dead object " << ref << " at offset "
2929 << offset << "\n card value = " << static_cast<int>(*card_addr);
2930 if (heap_->IsValidObjectAddress(obj->GetClass())) {
2931 LOG(ERROR) << "Obj type " << PrettyTypeOf(obj);
2932 } else {
2933 LOG(ERROR) << "Object " << obj << " class(" << obj->GetClass() << ") not a heap address";
Mathieu Chartier0f72e412013-09-06 16:40:01 -07002934 }
Mathieu Chartier938a03b2014-01-16 15:10:31 -08002935
Mathieu Chartierb363f662014-07-16 13:28:58 -07002936 // Attempt to find the class inside of the recently freed objects.
Mathieu Chartier938a03b2014-01-16 15:10:31 -08002937 space::ContinuousSpace* ref_space = heap_->FindContinuousSpaceFromObject(ref, true);
2938 if (ref_space != nullptr && ref_space->IsMallocSpace()) {
2939 space::MallocSpace* space = ref_space->AsMallocSpace();
2940 mirror::Class* ref_class = space->FindRecentFreedObject(ref);
2941 if (ref_class != nullptr) {
2942 LOG(ERROR) << "Reference " << ref << " found as a recently freed object with class "
2943 << PrettyClass(ref_class);
Mathieu Chartier0f72e412013-09-06 16:40:01 -07002944 } else {
Mathieu Chartier938a03b2014-01-16 15:10:31 -08002945 LOG(ERROR) << "Reference " << ref << " not found as a recently freed object";
Mathieu Chartier0f72e412013-09-06 16:40:01 -07002946 }
Mathieu Chartier938a03b2014-01-16 15:10:31 -08002947 }
Mathieu Chartier0f72e412013-09-06 16:40:01 -07002948
Mathieu Chartier938a03b2014-01-16 15:10:31 -08002949 if (ref->GetClass() != nullptr && heap_->IsValidObjectAddress(ref->GetClass()) &&
2950 ref->GetClass()->IsClass()) {
2951 LOG(ERROR) << "Ref type " << PrettyTypeOf(ref);
2952 } else {
2953 LOG(ERROR) << "Ref " << ref << " class(" << ref->GetClass()
2954 << ") is not a valid heap address";
2955 }
2956
Ian Rogers13735952014-10-08 12:43:28 -07002957 card_table->CheckAddrIsInCardTable(reinterpret_cast<const uint8_t*>(obj));
Mathieu Chartier938a03b2014-01-16 15:10:31 -08002958 void* cover_begin = card_table->AddrFromCard(card_addr);
2959 void* cover_end = reinterpret_cast<void*>(reinterpret_cast<size_t>(cover_begin) +
2960 accounting::CardTable::kCardSize);
2961 LOG(ERROR) << "Card " << reinterpret_cast<void*>(card_addr) << " covers " << cover_begin
2962 << "-" << cover_end;
Mathieu Chartiera8e8f9c2014-04-09 14:51:05 -07002963 accounting::ContinuousSpaceBitmap* bitmap =
2964 heap_->GetLiveBitmap()->GetContinuousSpaceBitmap(obj);
Mathieu Chartier938a03b2014-01-16 15:10:31 -08002965
2966 if (bitmap == nullptr) {
2967 LOG(ERROR) << "Object " << obj << " has no bitmap";
Mathieu Chartier4e305412014-02-19 10:54:44 -08002968 if (!VerifyClassClass(obj->GetClass())) {
Mathieu Chartier938a03b2014-01-16 15:10:31 -08002969 LOG(ERROR) << "Object " << obj << " failed class verification!";
Mathieu Chartier0f72e412013-09-06 16:40:01 -07002970 }
Mathieu Chartier938a03b2014-01-16 15:10:31 -08002971 } else {
Ian Rogers1d54e732013-05-02 21:10:01 -07002972 // Print out how the object is live.
Mathieu Chartier938a03b2014-01-16 15:10:31 -08002973 if (bitmap->Test(obj)) {
Ian Rogers1d54e732013-05-02 21:10:01 -07002974 LOG(ERROR) << "Object " << obj << " found in live bitmap";
2975 }
Mathieu Chartier0f72e412013-09-06 16:40:01 -07002976 if (alloc_stack->Contains(const_cast<mirror::Object*>(obj))) {
Ian Rogers1d54e732013-05-02 21:10:01 -07002977 LOG(ERROR) << "Object " << obj << " found in allocation stack";
2978 }
Mathieu Chartier0f72e412013-09-06 16:40:01 -07002979 if (live_stack->Contains(const_cast<mirror::Object*>(obj))) {
Ian Rogers1d54e732013-05-02 21:10:01 -07002980 LOG(ERROR) << "Object " << obj << " found in live stack";
2981 }
Mathieu Chartier0f72e412013-09-06 16:40:01 -07002982 if (alloc_stack->Contains(const_cast<mirror::Object*>(ref))) {
2983 LOG(ERROR) << "Ref " << ref << " found in allocation stack";
2984 }
2985 if (live_stack->Contains(const_cast<mirror::Object*>(ref))) {
2986 LOG(ERROR) << "Ref " << ref << " found in live stack";
2987 }
Ian Rogers1d54e732013-05-02 21:10:01 -07002988 // Attempt to see if the card table missed the reference.
2989 ScanVisitor scan_visitor;
Ian Rogers13735952014-10-08 12:43:28 -07002990 uint8_t* byte_cover_begin = reinterpret_cast<uint8_t*>(card_table->AddrFromCard(card_addr));
Lei Li727b2942015-01-15 11:26:34 +08002991 card_table->Scan<false>(bitmap, byte_cover_begin,
2992 byte_cover_begin + accounting::CardTable::kCardSize, scan_visitor);
Mathieu Chartierc7b83a02012-09-11 18:07:39 -07002993 }
Mathieu Chartier938a03b2014-01-16 15:10:31 -08002994
2995 // Search to see if any of the roots reference our object.
Mathieu Chartierbb87e0f2015-04-03 11:21:55 -07002996 RootMatchesObjectVisitor visitor1(obj);
2997 Runtime::Current()->VisitRoots(&visitor1);
Mathieu Chartier938a03b2014-01-16 15:10:31 -08002998 // Search to see if any of the roots reference our reference.
Mathieu Chartierbb87e0f2015-04-03 11:21:55 -07002999 RootMatchesObjectVisitor visitor2(ref);
3000 Runtime::Current()->VisitRoots(&visitor2);
Mathieu Chartierfd678be2012-08-30 14:50:54 -07003001 }
Mathieu Chartier8ab7e782014-05-19 16:55:27 -07003002 return false;
Mathieu Chartierfd678be2012-08-30 14:50:54 -07003003 }
3004
Ian Rogers1d54e732013-05-02 21:10:01 -07003005 Heap* const heap_;
Mathieu Chartier8ab7e782014-05-19 16:55:27 -07003006 Atomic<size_t>* const fail_count_;
3007 const bool verify_referent_;
Mathieu Chartierfd678be2012-08-30 14:50:54 -07003008};
3009
Ian Rogers1d54e732013-05-02 21:10:01 -07003010// Verify all references within an object, for use with HeapBitmap::Visit.
Mathieu Chartierfd678be2012-08-30 14:50:54 -07003011class VerifyObjectVisitor {
3012 public:
Roland Levillain3887c462015-08-12 18:15:42 +01003013 VerifyObjectVisitor(Heap* heap, Atomic<size_t>* fail_count, bool verify_referent)
Mathieu Chartiera4f6af92015-08-11 17:35:25 -07003014 : heap_(heap), fail_count_(fail_count), verify_referent_(verify_referent) {}
Mathieu Chartierfd678be2012-08-30 14:50:54 -07003015
Mathieu Chartierda7c6502015-07-23 16:01:26 -07003016 void operator()(mirror::Object* obj)
Mathieu Chartier90443472015-07-16 20:32:27 -07003017 SHARED_REQUIRES(Locks::mutator_lock_, Locks::heap_bitmap_lock_) {
Ian Rogers1d54e732013-05-02 21:10:01 -07003018 // Note: we are verifying the references in obj but not obj itself, this is because obj must
3019 // be live or else how did we find it in the live bitmap?
Mathieu Chartier8ab7e782014-05-19 16:55:27 -07003020 VerifyReferenceVisitor visitor(heap_, fail_count_, verify_referent_);
Mathieu Chartier0f72e412013-09-06 16:40:01 -07003021 // The class doesn't count as a reference but we should verify it anyways.
Mathieu Chartier059ef3d2015-08-18 13:54:21 -07003022 obj->VisitReferences(visitor, visitor);
Mathieu Chartierfd678be2012-08-30 14:50:54 -07003023 }
3024
Mathieu Chartier590fee92013-09-13 13:46:47 -07003025 static void VisitCallback(mirror::Object* obj, void* arg)
Mathieu Chartier90443472015-07-16 20:32:27 -07003026 SHARED_REQUIRES(Locks::mutator_lock_, Locks::heap_bitmap_lock_) {
Mathieu Chartier590fee92013-09-13 13:46:47 -07003027 VerifyObjectVisitor* visitor = reinterpret_cast<VerifyObjectVisitor*>(arg);
3028 visitor->operator()(obj);
3029 }
3030
Mathieu Chartierda7c6502015-07-23 16:01:26 -07003031 void VerifyRoots() SHARED_REQUIRES(Locks::mutator_lock_) REQUIRES(!Locks::heap_bitmap_lock_) {
Mathieu Chartierbb87e0f2015-04-03 11:21:55 -07003032 ReaderMutexLock mu(Thread::Current(), *Locks::heap_bitmap_lock_);
3033 VerifyReferenceVisitor visitor(heap_, fail_count_, verify_referent_);
3034 Runtime::Current()->VisitRoots(&visitor);
3035 }
3036
Mathieu Chartier8ab7e782014-05-19 16:55:27 -07003037 size_t GetFailureCount() const {
Mathieu Chartiere9e55ac2014-05-21 17:48:25 -07003038 return fail_count_->LoadSequentiallyConsistent();
Mathieu Chartierfd678be2012-08-30 14:50:54 -07003039 }
3040
3041 private:
Ian Rogers1d54e732013-05-02 21:10:01 -07003042 Heap* const heap_;
Mathieu Chartier8ab7e782014-05-19 16:55:27 -07003043 Atomic<size_t>* const fail_count_;
Mathieu Chartier78f7b4c2014-05-06 10:57:27 -07003044 const bool verify_referent_;
Mathieu Chartierfd678be2012-08-30 14:50:54 -07003045};
3046
Mathieu Chartierc1790162014-05-23 10:54:50 -07003047void Heap::PushOnAllocationStackWithInternalGC(Thread* self, mirror::Object** obj) {
3048 // Slow path, the allocation stack push back must have already failed.
3049 DCHECK(!allocation_stack_->AtomicPushBack(*obj));
3050 do {
3051 // TODO: Add handle VerifyObject.
3052 StackHandleScope<1> hs(self);
3053 HandleWrapper<mirror::Object> wrapper(hs.NewHandleWrapper(obj));
3054 // Push our object into the reserve region of the allocaiton stack. This is only required due
3055 // to heap verification requiring that roots are live (either in the live bitmap or in the
3056 // allocation stack).
3057 CHECK(allocation_stack_->AtomicPushBackIgnoreGrowthLimit(*obj));
3058 CollectGarbageInternal(collector::kGcTypeSticky, kGcCauseForAlloc, false);
3059 } while (!allocation_stack_->AtomicPushBack(*obj));
3060}
3061
3062void Heap::PushOnThreadLocalAllocationStackWithInternalGC(Thread* self, mirror::Object** obj) {
3063 // Slow path, the allocation stack push back must have already failed.
3064 DCHECK(!self->PushOnThreadLocalAllocationStack(*obj));
Mathieu Chartiercb535da2015-01-23 13:50:03 -08003065 StackReference<mirror::Object>* start_address;
3066 StackReference<mirror::Object>* end_address;
Mathieu Chartierc1790162014-05-23 10:54:50 -07003067 while (!allocation_stack_->AtomicBumpBack(kThreadLocalAllocationStackSize, &start_address,
3068 &end_address)) {
3069 // TODO: Add handle VerifyObject.
3070 StackHandleScope<1> hs(self);
3071 HandleWrapper<mirror::Object> wrapper(hs.NewHandleWrapper(obj));
3072 // Push our object into the reserve region of the allocaiton stack. This is only required due
3073 // to heap verification requiring that roots are live (either in the live bitmap or in the
3074 // allocation stack).
3075 CHECK(allocation_stack_->AtomicPushBackIgnoreGrowthLimit(*obj));
3076 // Push into the reserve allocation stack.
3077 CollectGarbageInternal(collector::kGcTypeSticky, kGcCauseForAlloc, false);
3078 }
3079 self->SetThreadLocalAllocationStack(start_address, end_address);
3080 // Retry on the new thread-local allocation stack.
3081 CHECK(self->PushOnThreadLocalAllocationStack(*obj)); // Must succeed.
3082}
3083
Mathieu Chartierfd678be2012-08-30 14:50:54 -07003084// Must do this with mutators suspended since we are directly accessing the allocation stacks.
Mathieu Chartier8ab7e782014-05-19 16:55:27 -07003085size_t Heap::VerifyHeapReferences(bool verify_referents) {
Hiroshi Yamauchi1ed90612014-02-14 15:00:51 -08003086 Thread* self = Thread::Current();
3087 Locks::mutator_lock_->AssertExclusiveHeld(self);
Mathieu Chartierfd678be2012-08-30 14:50:54 -07003088 // Lets sort our allocation stacks so that we can efficiently binary search them.
Ian Rogers1d54e732013-05-02 21:10:01 -07003089 allocation_stack_->Sort();
3090 live_stack_->Sort();
Hiroshi Yamauchi1ed90612014-02-14 15:00:51 -08003091 // Since we sorted the allocation stack content, need to revoke all
3092 // thread-local allocation stacks.
3093 RevokeAllThreadLocalAllocationStacks(self);
Mathieu Chartier8ab7e782014-05-19 16:55:27 -07003094 Atomic<size_t> fail_count_(0);
3095 VerifyObjectVisitor visitor(this, &fail_count_, verify_referents);
Mathieu Chartier0f72e412013-09-06 16:40:01 -07003096 // Verify objects in the allocation stack since these will be objects which were:
3097 // 1. Allocated prior to the GC (pre GC verification).
3098 // 2. Allocated during the GC (pre sweep GC verification).
Mathieu Chartier0f72e412013-09-06 16:40:01 -07003099 // We don't want to verify the objects in the live stack since they themselves may be
Mathieu Chartierfd678be2012-08-30 14:50:54 -07003100 // pointing to dead objects if they are not reachable.
Hiroshi Yamauchi2cd334a2015-01-09 14:03:35 -08003101 VisitObjectsPaused(VerifyObjectVisitor::VisitCallback, &visitor);
Mathieu Chartier590fee92013-09-13 13:46:47 -07003102 // Verify the roots:
Mathieu Chartierbb87e0f2015-04-03 11:21:55 -07003103 visitor.VerifyRoots();
Mathieu Chartier8ab7e782014-05-19 16:55:27 -07003104 if (visitor.GetFailureCount() > 0) {
Mathieu Chartier0f72e412013-09-06 16:40:01 -07003105 // Dump mod-union tables.
Mathieu Chartier11409ae2013-09-23 11:49:36 -07003106 for (const auto& table_pair : mod_union_tables_) {
3107 accounting::ModUnionTable* mod_union_table = table_pair.second;
3108 mod_union_table->Dump(LOG(ERROR) << mod_union_table->GetName() << ": ");
3109 }
Hiroshi Yamauchi38e68e92014-03-07 13:59:08 -08003110 // Dump remembered sets.
3111 for (const auto& table_pair : remembered_sets_) {
3112 accounting::RememberedSet* remembered_set = table_pair.second;
3113 remembered_set->Dump(LOG(ERROR) << remembered_set->GetName() << ": ");
3114 }
Mathieu Chartier4c13a3f2014-07-14 14:57:16 -07003115 DumpSpaces(LOG(ERROR));
Mathieu Chartierfd678be2012-08-30 14:50:54 -07003116 }
Mathieu Chartier8ab7e782014-05-19 16:55:27 -07003117 return visitor.GetFailureCount();
Mathieu Chartierc7b83a02012-09-11 18:07:39 -07003118}
3119
3120class VerifyReferenceCardVisitor {
3121 public:
3122 VerifyReferenceCardVisitor(Heap* heap, bool* failed)
Mathieu Chartier90443472015-07-16 20:32:27 -07003123 SHARED_REQUIRES(Locks::mutator_lock_,
Mathieu Chartierc7b83a02012-09-11 18:07:39 -07003124 Locks::heap_bitmap_lock_)
Ian Rogers1d54e732013-05-02 21:10:01 -07003125 : heap_(heap), failed_(failed) {
Mathieu Chartierc7b83a02012-09-11 18:07:39 -07003126 }
3127
Mathieu Chartierda7c6502015-07-23 16:01:26 -07003128 // There is no card marks for native roots on a class.
3129 void VisitRootIfNonNull(mirror::CompressedReference<mirror::Object>* root ATTRIBUTE_UNUSED)
3130 const {}
3131 void VisitRoot(mirror::CompressedReference<mirror::Object>* root ATTRIBUTE_UNUSED) const {}
3132
Mathieu Chartier4da7f2f2012-11-13 12:51:01 -08003133 // TODO: Fix lock analysis to not use NO_THREAD_SAFETY_ANALYSIS, requires support for
3134 // annotalysis on visitors.
Mathieu Chartier407f7022014-02-18 14:37:05 -08003135 void operator()(mirror::Object* obj, MemberOffset offset, bool is_static) const
3136 NO_THREAD_SAFETY_ANALYSIS {
Ian Rogersb0fa5dc2014-04-28 16:47:08 -07003137 mirror::Object* ref = obj->GetFieldObject<mirror::Object>(offset);
Mathieu Chartier4da7f2f2012-11-13 12:51:01 -08003138 // Filter out class references since changing an object's class does not mark the card as dirty.
3139 // Also handles large objects, since the only reference they hold is a class reference.
Mathieu Chartier407f7022014-02-18 14:37:05 -08003140 if (ref != nullptr && !ref->IsClass()) {
Ian Rogers1d54e732013-05-02 21:10:01 -07003141 accounting::CardTable* card_table = heap_->GetCardTable();
Mathieu Chartierc7b83a02012-09-11 18:07:39 -07003142 // If the object is not dirty and it is referencing something in the live stack other than
3143 // class, then it must be on a dirty card.
Mathieu Chartier7469ebf2012-09-24 16:28:36 -07003144 if (!card_table->AddrIsInCardTable(obj)) {
3145 LOG(ERROR) << "Object " << obj << " is not in the address range of the card table";
3146 *failed_ = true;
Mathieu Chartier2b82db42012-11-14 17:29:05 -08003147 } else if (!card_table->IsDirty(obj)) {
Mathieu Chartier938a03b2014-01-16 15:10:31 -08003148 // TODO: Check mod-union tables.
Mathieu Chartier4da7f2f2012-11-13 12:51:01 -08003149 // Card should be either kCardDirty if it got re-dirtied after we aged it, or
3150 // kCardDirty - 1 if it didnt get touched since we aged it.
Ian Rogers1d54e732013-05-02 21:10:01 -07003151 accounting::ObjectStack* live_stack = heap_->live_stack_.get();
Mathieu Chartier407f7022014-02-18 14:37:05 -08003152 if (live_stack->ContainsSorted(ref)) {
3153 if (live_stack->ContainsSorted(obj)) {
Mathieu Chartierc7b83a02012-09-11 18:07:39 -07003154 LOG(ERROR) << "Object " << obj << " found in live stack";
3155 }
3156 if (heap_->GetLiveBitmap()->Test(obj)) {
3157 LOG(ERROR) << "Object " << obj << " found in live bitmap";
3158 }
3159 LOG(ERROR) << "Object " << obj << " " << PrettyTypeOf(obj)
3160 << " references " << ref << " " << PrettyTypeOf(ref) << " in live stack";
3161
3162 // Print which field of the object is dead.
3163 if (!obj->IsObjectArray()) {
Ian Rogersef7d42f2014-01-06 12:55:46 -08003164 mirror::Class* klass = is_static ? obj->AsClass() : obj->GetClass();
Mathieu Chartierc7853442015-03-27 14:35:38 -07003165 CHECK(klass != nullptr);
Mathieu Chartierc0fe56a2015-08-11 13:01:23 -07003166 for (ArtField& field : (is_static ? klass->GetSFields() : klass->GetIFields())) {
Mathieu Chartier54d220e2015-07-30 16:20:06 -07003167 if (field.GetOffset().Int32Value() == offset.Int32Value()) {
Mathieu Chartierc7b83a02012-09-11 18:07:39 -07003168 LOG(ERROR) << (is_static ? "Static " : "") << "field in the live stack is "
Mathieu Chartier54d220e2015-07-30 16:20:06 -07003169 << PrettyField(&field);
Mathieu Chartierc7b83a02012-09-11 18:07:39 -07003170 break;
3171 }
3172 }
3173 } else {
Ian Rogersef7d42f2014-01-06 12:55:46 -08003174 mirror::ObjectArray<mirror::Object>* object_array =
Ian Rogers2dd0e2c2013-01-24 12:42:14 -08003175 obj->AsObjectArray<mirror::Object>();
Mathieu Chartierc7b83a02012-09-11 18:07:39 -07003176 for (int32_t i = 0; i < object_array->GetLength(); ++i) {
3177 if (object_array->Get(i) == ref) {
3178 LOG(ERROR) << (is_static ? "Static " : "") << "obj[" << i << "] = ref";
3179 }
3180 }
3181 }
3182
3183 *failed_ = true;
3184 }
3185 }
3186 }
3187 }
3188
3189 private:
Ian Rogers1d54e732013-05-02 21:10:01 -07003190 Heap* const heap_;
3191 bool* const failed_;
Mathieu Chartierc7b83a02012-09-11 18:07:39 -07003192};
3193
3194class VerifyLiveStackReferences {
3195 public:
Brian Carlstrom93ba8932013-07-17 21:31:49 -07003196 explicit VerifyLiveStackReferences(Heap* heap)
Mathieu Chartierc7b83a02012-09-11 18:07:39 -07003197 : heap_(heap),
Brian Carlstrom93ba8932013-07-17 21:31:49 -07003198 failed_(false) {}
Mathieu Chartierc7b83a02012-09-11 18:07:39 -07003199
Mathieu Chartier11409ae2013-09-23 11:49:36 -07003200 void operator()(mirror::Object* obj) const
Mathieu Chartier90443472015-07-16 20:32:27 -07003201 SHARED_REQUIRES(Locks::mutator_lock_, Locks::heap_bitmap_lock_) {
Mathieu Chartierc7b83a02012-09-11 18:07:39 -07003202 VerifyReferenceCardVisitor visitor(heap_, const_cast<bool*>(&failed_));
Mathieu Chartier059ef3d2015-08-18 13:54:21 -07003203 obj->VisitReferences(visitor, VoidFunctor());
Mathieu Chartierc7b83a02012-09-11 18:07:39 -07003204 }
3205
3206 bool Failed() const {
3207 return failed_;
3208 }
3209
3210 private:
Ian Rogers1d54e732013-05-02 21:10:01 -07003211 Heap* const heap_;
Mathieu Chartierc7b83a02012-09-11 18:07:39 -07003212 bool failed_;
3213};
3214
3215bool Heap::VerifyMissingCardMarks() {
Hiroshi Yamauchi1ed90612014-02-14 15:00:51 -08003216 Thread* self = Thread::Current();
3217 Locks::mutator_lock_->AssertExclusiveHeld(self);
Mathieu Chartier2b82db42012-11-14 17:29:05 -08003218 // We need to sort the live stack since we binary search it.
Ian Rogers1d54e732013-05-02 21:10:01 -07003219 live_stack_->Sort();
Hiroshi Yamauchi1ed90612014-02-14 15:00:51 -08003220 // Since we sorted the allocation stack content, need to revoke all
3221 // thread-local allocation stacks.
3222 RevokeAllThreadLocalAllocationStacks(self);
Mathieu Chartierc7b83a02012-09-11 18:07:39 -07003223 VerifyLiveStackReferences visitor(this);
3224 GetLiveBitmap()->Visit(visitor);
Mathieu Chartierc7b83a02012-09-11 18:07:39 -07003225 // We can verify objects in the live stack since none of these should reference dead objects.
Mathieu Chartiercb535da2015-01-23 13:50:03 -08003226 for (auto* it = live_stack_->Begin(); it != live_stack_->End(); ++it) {
3227 if (!kUseThreadLocalAllocationStack || it->AsMirrorPtr() != nullptr) {
3228 visitor(it->AsMirrorPtr());
Hiroshi Yamauchif5b0e202014-02-11 17:02:22 -08003229 }
Mathieu Chartierc7b83a02012-09-11 18:07:39 -07003230 }
Mathieu Chartier4c13a3f2014-07-14 14:57:16 -07003231 return !visitor.Failed();
Mathieu Chartierfd678be2012-08-30 14:50:54 -07003232}
3233
Mathieu Chartiera4f6af92015-08-11 17:35:25 -07003234void Heap::SwapStacks() {
Hiroshi Yamauchif5b0e202014-02-11 17:02:22 -08003235 if (kUseThreadLocalAllocationStack) {
3236 live_stack_->AssertAllZero();
3237 }
Mathieu Chartierd22d5482012-11-06 17:14:12 -08003238 allocation_stack_.swap(live_stack_);
Mathieu Chartierc7b83a02012-09-11 18:07:39 -07003239}
3240
Hiroshi Yamauchif5b0e202014-02-11 17:02:22 -08003241void Heap::RevokeAllThreadLocalAllocationStacks(Thread* self) {
Hiroshi Yamauchif5b0e202014-02-11 17:02:22 -08003242 // This must be called only during the pause.
Hiroshi Yamauchi2cd334a2015-01-09 14:03:35 -08003243 DCHECK(Locks::mutator_lock_->IsExclusiveHeld(self));
Hiroshi Yamauchif5b0e202014-02-11 17:02:22 -08003244 MutexLock mu(self, *Locks::runtime_shutdown_lock_);
3245 MutexLock mu2(self, *Locks::thread_list_lock_);
3246 std::list<Thread*> thread_list = Runtime::Current()->GetThreadList()->GetList();
3247 for (Thread* t : thread_list) {
3248 t->RevokeThreadLocalAllocationStack();
3249 }
3250}
3251
Ian Rogers68d8b422014-07-17 11:09:10 -07003252void Heap::AssertThreadLocalBuffersAreRevoked(Thread* thread) {
3253 if (kIsDebugBuild) {
3254 if (rosalloc_space_ != nullptr) {
3255 rosalloc_space_->AssertThreadLocalBuffersAreRevoked(thread);
3256 }
3257 if (bump_pointer_space_ != nullptr) {
3258 bump_pointer_space_->AssertThreadLocalBuffersAreRevoked(thread);
3259 }
3260 }
3261}
3262
Hiroshi Yamauchic93c5302014-03-20 16:15:37 -07003263void Heap::AssertAllBumpPointerSpaceThreadLocalBuffersAreRevoked() {
3264 if (kIsDebugBuild) {
3265 if (bump_pointer_space_ != nullptr) {
3266 bump_pointer_space_->AssertAllThreadLocalBuffersAreRevoked();
3267 }
3268 }
3269}
3270
Mathieu Chartier11409ae2013-09-23 11:49:36 -07003271accounting::ModUnionTable* Heap::FindModUnionTableFromSpace(space::Space* space) {
3272 auto it = mod_union_tables_.find(space);
3273 if (it == mod_union_tables_.end()) {
3274 return nullptr;
3275 }
3276 return it->second;
3277}
3278
Hiroshi Yamauchi38e68e92014-03-07 13:59:08 -08003279accounting::RememberedSet* Heap::FindRememberedSetFromSpace(space::Space* space) {
3280 auto it = remembered_sets_.find(space);
3281 if (it == remembered_sets_.end()) {
3282 return nullptr;
3283 }
3284 return it->second;
3285}
3286
Mathieu Chartiera4f6af92015-08-11 17:35:25 -07003287void Heap::ProcessCards(TimingLogger* timings,
3288 bool use_rem_sets,
3289 bool process_alloc_space_cards,
Lei Li4add3b42015-01-15 11:55:26 +08003290 bool clear_alloc_space_cards) {
Mathieu Chartierf5997b42014-06-20 10:37:54 -07003291 TimingLogger::ScopedTiming t(__FUNCTION__, timings);
Ian Rogers1d54e732013-05-02 21:10:01 -07003292 // Clear cards and keep track of cards cleared in the mod-union table.
Mathieu Chartier02e25112013-08-14 16:14:24 -07003293 for (const auto& space : continuous_spaces_) {
Mathieu Chartier11409ae2013-09-23 11:49:36 -07003294 accounting::ModUnionTable* table = FindModUnionTableFromSpace(space);
Hiroshi Yamauchi38e68e92014-03-07 13:59:08 -08003295 accounting::RememberedSet* rem_set = FindRememberedSetFromSpace(space);
Mathieu Chartier11409ae2013-09-23 11:49:36 -07003296 if (table != nullptr) {
3297 const char* name = space->IsZygoteSpace() ? "ZygoteModUnionClearCards" :
3298 "ImageModUnionClearCards";
Andreas Gampe277ccbd2014-11-03 21:36:10 -08003299 TimingLogger::ScopedTiming t2(name, timings);
Mathieu Chartier11409ae2013-09-23 11:49:36 -07003300 table->ClearCards();
Hiroshi Yamauchi38e68e92014-03-07 13:59:08 -08003301 } else if (use_rem_sets && rem_set != nullptr) {
3302 DCHECK(collector::SemiSpace::kUseRememberedSet && collector_type_ == kCollectorTypeGSS)
3303 << static_cast<int>(collector_type_);
Andreas Gampe277ccbd2014-11-03 21:36:10 -08003304 TimingLogger::ScopedTiming t2("AllocSpaceRemSetClearCards", timings);
Hiroshi Yamauchi38e68e92014-03-07 13:59:08 -08003305 rem_set->ClearCards();
Lei Li4add3b42015-01-15 11:55:26 +08003306 } else if (process_alloc_space_cards) {
Andreas Gampe277ccbd2014-11-03 21:36:10 -08003307 TimingLogger::ScopedTiming t2("AllocSpaceClearCards", timings);
Lei Li4add3b42015-01-15 11:55:26 +08003308 if (clear_alloc_space_cards) {
Mathieu Chartierfbc31082016-01-24 11:59:56 -08003309 uint8_t* end = space->End();
3310 if (space->IsImageSpace()) {
3311 // Image space end is the end of the mirror objects, it is not necessarily page or card
3312 // aligned. Align up so that the check in ClearCardRange does not fail.
3313 end = AlignUp(end, accounting::CardTable::kCardSize);
3314 }
3315 card_table_->ClearCardRange(space->Begin(), end);
Lei Li4add3b42015-01-15 11:55:26 +08003316 } else {
3317 // No mod union table for the AllocSpace. Age the cards so that the GC knows that these
3318 // cards were dirty before the GC started.
3319 // TODO: Need to use atomic for the case where aged(cleaning thread) -> dirty(other thread)
3320 // -> clean(cleaning thread).
3321 // The races are we either end up with: Aged card, unaged card. Since we have the
3322 // checkpoint roots and then we scan / update mod union tables after. We will always
3323 // scan either card. If we end up with the non aged card, we scan it it in the pause.
3324 card_table_->ModifyCardsAtomic(space->Begin(), space->End(), AgeCardVisitor(),
3325 VoidFunctor());
3326 }
Mathieu Chartier7469ebf2012-09-24 16:28:36 -07003327 }
3328 }
3329}
3330
Mathieu Chartier97509952015-07-13 14:35:43 -07003331struct IdentityMarkHeapReferenceVisitor : public MarkObjectVisitor {
3332 virtual mirror::Object* MarkObject(mirror::Object* obj) OVERRIDE {
3333 return obj;
3334 }
3335 virtual void MarkHeapReference(mirror::HeapReference<mirror::Object>*) OVERRIDE {
3336 }
3337};
Mathieu Chartier11409ae2013-09-23 11:49:36 -07003338
Mathieu Chartier6f365cc2014-04-23 12:42:27 -07003339void Heap::PreGcVerificationPaused(collector::GarbageCollector* gc) {
3340 Thread* const self = Thread::Current();
Mathieu Chartier10fb83a2014-06-15 15:15:43 -07003341 TimingLogger* const timings = current_gc_iteration_.GetTimings();
Mathieu Chartierf5997b42014-06-20 10:37:54 -07003342 TimingLogger::ScopedTiming t(__FUNCTION__, timings);
Mathieu Chartier2b82db42012-11-14 17:29:05 -08003343 if (verify_pre_gc_heap_) {
Andreas Gampe277ccbd2014-11-03 21:36:10 -08003344 TimingLogger::ScopedTiming t2("(Paused)PreGcVerifyHeapReferences", timings);
Mathieu Chartier8ab7e782014-05-19 16:55:27 -07003345 size_t failures = VerifyHeapReferences();
3346 if (failures > 0) {
3347 LOG(FATAL) << "Pre " << gc->GetName() << " heap verification failed with " << failures
3348 << " failures";
Mathieu Chartier2b82db42012-11-14 17:29:05 -08003349 }
Mathieu Chartier4da7f2f2012-11-13 12:51:01 -08003350 }
Mathieu Chartier2b82db42012-11-14 17:29:05 -08003351 // Check that all objects which reference things in the live stack are on dirty cards.
3352 if (verify_missing_card_marks_) {
Andreas Gampe277ccbd2014-11-03 21:36:10 -08003353 TimingLogger::ScopedTiming t2("(Paused)PreGcVerifyMissingCardMarks", timings);
Mathieu Chartier6f365cc2014-04-23 12:42:27 -07003354 ReaderMutexLock mu(self, *Locks::heap_bitmap_lock_);
Mathieu Chartiera4f6af92015-08-11 17:35:25 -07003355 SwapStacks();
Mathieu Chartier6f365cc2014-04-23 12:42:27 -07003356 // Sort the live stack so that we can quickly binary search it later.
Mathieu Chartier4c13a3f2014-07-14 14:57:16 -07003357 CHECK(VerifyMissingCardMarks()) << "Pre " << gc->GetName()
3358 << " missing card mark verification failed\n" << DumpSpaces();
Mathieu Chartiera4f6af92015-08-11 17:35:25 -07003359 SwapStacks();
Mathieu Chartier2b82db42012-11-14 17:29:05 -08003360 }
Mathieu Chartier2b82db42012-11-14 17:29:05 -08003361 if (verify_mod_union_table_) {
Andreas Gampe277ccbd2014-11-03 21:36:10 -08003362 TimingLogger::ScopedTiming t2("(Paused)PreGcVerifyModUnionTables", timings);
Mathieu Chartier2b82db42012-11-14 17:29:05 -08003363 ReaderMutexLock reader_lock(self, *Locks::heap_bitmap_lock_);
Mathieu Chartier11409ae2013-09-23 11:49:36 -07003364 for (const auto& table_pair : mod_union_tables_) {
3365 accounting::ModUnionTable* mod_union_table = table_pair.second;
Mathieu Chartier97509952015-07-13 14:35:43 -07003366 IdentityMarkHeapReferenceVisitor visitor;
3367 mod_union_table->UpdateAndMarkReferences(&visitor);
Mathieu Chartier11409ae2013-09-23 11:49:36 -07003368 mod_union_table->Verify();
3369 }
Mathieu Chartier6f365cc2014-04-23 12:42:27 -07003370 }
3371}
3372
3373void Heap::PreGcVerification(collector::GarbageCollector* gc) {
Mathieu Chartier0651d412014-04-29 14:37:57 -07003374 if (verify_pre_gc_heap_ || verify_missing_card_marks_ || verify_mod_union_table_) {
Mathieu Chartier6f365cc2014-04-23 12:42:27 -07003375 collector::GarbageCollector::ScopedPause pause(gc);
3376 PreGcVerificationPaused(gc);
3377 }
3378}
3379
Mathieu Chartiera4f6af92015-08-11 17:35:25 -07003380void Heap::PrePauseRosAllocVerification(collector::GarbageCollector* gc ATTRIBUTE_UNUSED) {
Mathieu Chartier6f365cc2014-04-23 12:42:27 -07003381 // TODO: Add a new runtime option for this?
3382 if (verify_pre_gc_rosalloc_) {
Mathieu Chartier10fb83a2014-06-15 15:15:43 -07003383 RosAllocVerification(current_gc_iteration_.GetTimings(), "PreGcRosAllocVerification");
Mathieu Chartier2b82db42012-11-14 17:29:05 -08003384 }
Mathieu Chartier4da7f2f2012-11-13 12:51:01 -08003385}
3386
Ian Rogers1d54e732013-05-02 21:10:01 -07003387void Heap::PreSweepingGcVerification(collector::GarbageCollector* gc) {
Mathieu Chartier6f365cc2014-04-23 12:42:27 -07003388 Thread* const self = Thread::Current();
Mathieu Chartier10fb83a2014-06-15 15:15:43 -07003389 TimingLogger* const timings = current_gc_iteration_.GetTimings();
Mathieu Chartierf5997b42014-06-20 10:37:54 -07003390 TimingLogger::ScopedTiming t(__FUNCTION__, timings);
Mathieu Chartier2b82db42012-11-14 17:29:05 -08003391 // Called before sweeping occurs since we want to make sure we are not going so reclaim any
3392 // reachable objects.
Mathieu Chartier6f365cc2014-04-23 12:42:27 -07003393 if (verify_pre_sweeping_heap_) {
Andreas Gampe277ccbd2014-11-03 21:36:10 -08003394 TimingLogger::ScopedTiming t2("(Paused)PostSweepingVerifyHeapReferences", timings);
Ian Rogers1d54e732013-05-02 21:10:01 -07003395 CHECK_NE(self->GetState(), kRunnable);
Hiroshi Yamauchi0c8c3032015-01-16 16:54:35 -08003396 {
3397 WriterMutexLock mu(self, *Locks::heap_bitmap_lock_);
3398 // Swapping bound bitmaps does nothing.
3399 gc->SwapBitmaps();
3400 }
Mathieu Chartier78f7b4c2014-05-06 10:57:27 -07003401 // Pass in false since concurrent reference processing can mean that the reference referents
3402 // may point to dead objects at the point which PreSweepingGcVerification is called.
Mathieu Chartier8ab7e782014-05-19 16:55:27 -07003403 size_t failures = VerifyHeapReferences(false);
3404 if (failures > 0) {
3405 LOG(FATAL) << "Pre sweeping " << gc->GetName() << " GC verification failed with " << failures
3406 << " failures";
Mathieu Chartier6f365cc2014-04-23 12:42:27 -07003407 }
Hiroshi Yamauchi0c8c3032015-01-16 16:54:35 -08003408 {
3409 WriterMutexLock mu(self, *Locks::heap_bitmap_lock_);
3410 gc->SwapBitmaps();
3411 }
Mathieu Chartier6f365cc2014-04-23 12:42:27 -07003412 }
3413 if (verify_pre_sweeping_rosalloc_) {
3414 RosAllocVerification(timings, "PreSweepingRosAllocVerification");
3415 }
3416}
3417
3418void Heap::PostGcVerificationPaused(collector::GarbageCollector* gc) {
3419 // Only pause if we have to do some verification.
3420 Thread* const self = Thread::Current();
Mathieu Chartier10fb83a2014-06-15 15:15:43 -07003421 TimingLogger* const timings = GetCurrentGcIteration()->GetTimings();
Mathieu Chartierf5997b42014-06-20 10:37:54 -07003422 TimingLogger::ScopedTiming t(__FUNCTION__, timings);
Mathieu Chartier6f365cc2014-04-23 12:42:27 -07003423 if (verify_system_weaks_) {
3424 ReaderMutexLock mu2(self, *Locks::heap_bitmap_lock_);
3425 collector::MarkSweep* mark_sweep = down_cast<collector::MarkSweep*>(gc);
3426 mark_sweep->VerifySystemWeaks();
3427 }
3428 if (verify_post_gc_rosalloc_) {
Mathieu Chartierf5997b42014-06-20 10:37:54 -07003429 RosAllocVerification(timings, "(Paused)PostGcRosAllocVerification");
Mathieu Chartier6f365cc2014-04-23 12:42:27 -07003430 }
3431 if (verify_post_gc_heap_) {
Andreas Gampe277ccbd2014-11-03 21:36:10 -08003432 TimingLogger::ScopedTiming t2("(Paused)PostGcVerifyHeapReferences", timings);
Mathieu Chartier8ab7e782014-05-19 16:55:27 -07003433 size_t failures = VerifyHeapReferences();
3434 if (failures > 0) {
3435 LOG(FATAL) << "Pre " << gc->GetName() << " heap verification failed with " << failures
3436 << " failures";
Ian Rogers00f7d0e2012-07-19 15:28:27 -07003437 }
Ian Rogers00f7d0e2012-07-19 15:28:27 -07003438 }
Mathieu Chartier2b82db42012-11-14 17:29:05 -08003439}
Ian Rogers00f7d0e2012-07-19 15:28:27 -07003440
Ian Rogers1d54e732013-05-02 21:10:01 -07003441void Heap::PostGcVerification(collector::GarbageCollector* gc) {
Mathieu Chartier6f365cc2014-04-23 12:42:27 -07003442 if (verify_system_weaks_ || verify_post_gc_rosalloc_ || verify_post_gc_heap_) {
3443 collector::GarbageCollector::ScopedPause pause(gc);
Mathieu Chartierd35326f2014-08-18 15:02:59 -07003444 PostGcVerificationPaused(gc);
Ian Rogers00f7d0e2012-07-19 15:28:27 -07003445 }
Carl Shapiro69759ea2011-07-21 18:13:35 -07003446}
3447
Mathieu Chartier6f365cc2014-04-23 12:42:27 -07003448void Heap::RosAllocVerification(TimingLogger* timings, const char* name) {
Mathieu Chartierf5997b42014-06-20 10:37:54 -07003449 TimingLogger::ScopedTiming t(name, timings);
Mathieu Chartier6f365cc2014-04-23 12:42:27 -07003450 for (const auto& space : continuous_spaces_) {
3451 if (space->IsRosAllocSpace()) {
3452 VLOG(heap) << name << " : " << space->GetName();
3453 space->AsRosAllocSpace()->Verify();
Hiroshi Yamauchia4adbfd2014-02-04 18:12:17 -08003454 }
3455 }
3456}
3457
Mathieu Chartier89a201e2014-05-02 10:27:26 -07003458collector::GcType Heap::WaitForGcToComplete(GcCause cause, Thread* self) {
Mathieu Chartiercaa82d62014-02-02 16:51:17 -08003459 ScopedThreadStateChange tsc(self, kWaitingForGcToComplete);
Mathieu Chartier590fee92013-09-13 13:46:47 -07003460 MutexLock mu(self, *gc_complete_lock_);
Mathieu Chartier89a201e2014-05-02 10:27:26 -07003461 return WaitForGcToCompleteLocked(cause, self);
Mathieu Chartier590fee92013-09-13 13:46:47 -07003462}
3463
Mathieu Chartier89a201e2014-05-02 10:27:26 -07003464collector::GcType Heap::WaitForGcToCompleteLocked(GcCause cause, Thread* self) {
Ian Rogers1d54e732013-05-02 21:10:01 -07003465 collector::GcType last_gc_type = collector::kGcTypeNone;
Mathieu Chartier590fee92013-09-13 13:46:47 -07003466 uint64_t wait_start = NanoTime();
Mathieu Chartierd5a89ee2014-01-31 09:55:13 -08003467 while (collector_type_running_ != kCollectorTypeNone) {
Hiroshi Yamauchia1c9f012015-04-02 10:18:12 -07003468 if (self != task_processor_->GetRunningThread()) {
3469 // The current thread is about to wait for a currently running
3470 // collection to finish. If the waiting thread is not the heap
3471 // task daemon thread, the currently running collection is
3472 // considered as a blocking GC.
3473 running_collection_is_blocking_ = true;
3474 VLOG(gc) << "Waiting for a blocking GC " << cause;
3475 }
Mathieu Chartier32ce2ad2016-03-04 14:58:03 -08003476 ScopedTrace trace("GC: Wait For Completion");
Mathieu Chartier590fee92013-09-13 13:46:47 -07003477 // We must wait, change thread state then sleep on gc_complete_cond_;
3478 gc_complete_cond_->Wait(self);
3479 last_gc_type = last_gc_type_;
Mathieu Chartier7664f5c2012-06-08 18:15:32 -07003480 }
Mathieu Chartier590fee92013-09-13 13:46:47 -07003481 uint64_t wait_time = NanoTime() - wait_start;
3482 total_wait_time_ += wait_time;
3483 if (wait_time > long_pause_log_threshold_) {
Mathieu Chartier89a201e2014-05-02 10:27:26 -07003484 LOG(INFO) << "WaitForGcToComplete blocked for " << PrettyDuration(wait_time)
3485 << " for cause " << cause;
Mathieu Chartier590fee92013-09-13 13:46:47 -07003486 }
Hiroshi Yamauchia1c9f012015-04-02 10:18:12 -07003487 if (self != task_processor_->GetRunningThread()) {
3488 // The current thread is about to run a collection. If the thread
3489 // is not the heap task daemon thread, it's considered as a
3490 // blocking GC (i.e., blocking itself).
3491 running_collection_is_blocking_ = true;
3492 VLOG(gc) << "Starting a blocking GC " << cause;
3493 }
Mathieu Chartier866fb2a2012-09-10 10:47:49 -07003494 return last_gc_type;
Carl Shapiro69759ea2011-07-21 18:13:35 -07003495}
3496
Elliott Hughesc967f782012-04-16 10:23:15 -07003497void Heap::DumpForSigQuit(std::ostream& os) {
Ian Rogers1d54e732013-05-02 21:10:01 -07003498 os << "Heap: " << GetPercentFree() << "% free, " << PrettySize(GetBytesAllocated()) << "/"
Mathieu Chartier2fde5332012-09-14 14:51:54 -07003499 << PrettySize(GetTotalMemory()) << "; " << GetObjectsAllocated() << " objects\n";
Elliott Hughes8b788fe2013-04-17 15:57:01 -07003500 DumpGcPerformanceInfo(os);
Elliott Hughesc967f782012-04-16 10:23:15 -07003501}
3502
3503size_t Heap::GetPercentFree() {
Mathieu Chartierd30e1d62014-06-09 13:25:22 -07003504 return static_cast<size_t>(100.0f * static_cast<float>(GetFreeMemory()) / max_allowed_footprint_);
Elliott Hughesc967f782012-04-16 10:23:15 -07003505}
3506
Elliott Hughes4dd9b4d2011-12-12 18:29:24 -08003507void Heap::SetIdealFootprint(size_t max_allowed_footprint) {
Mathieu Chartier2fde5332012-09-14 14:51:54 -07003508 if (max_allowed_footprint > GetMaxMemory()) {
Mathieu Chartierfd678be2012-08-30 14:50:54 -07003509 VLOG(gc) << "Clamp target GC heap from " << PrettySize(max_allowed_footprint) << " to "
Mathieu Chartier2fde5332012-09-14 14:51:54 -07003510 << PrettySize(GetMaxMemory());
3511 max_allowed_footprint = GetMaxMemory();
3512 }
Mathieu Chartier1c23e1e2012-10-12 14:14:11 -07003513 max_allowed_footprint_ = max_allowed_footprint;
Shih-wei Liao8c2f6412011-10-03 22:58:14 -07003514}
3515
Mathieu Chartier590fee92013-09-13 13:46:47 -07003516bool Heap::IsMovableObject(const mirror::Object* obj) const {
3517 if (kMovingCollector) {
Mathieu Chartier31f44142014-04-08 14:40:03 -07003518 space::Space* space = FindContinuousSpaceFromObject(obj, true);
3519 if (space != nullptr) {
3520 // TODO: Check large object?
3521 return space->CanMoveObjects();
Mathieu Chartier590fee92013-09-13 13:46:47 -07003522 }
Mathieu Chartier590fee92013-09-13 13:46:47 -07003523 }
3524 return false;
3525}
3526
Mathieu Chartier987ccff2013-07-08 11:05:21 -07003527void Heap::UpdateMaxNativeFootprint() {
Ian Rogers3e5cf302014-05-20 16:40:37 -07003528 size_t native_size = native_bytes_allocated_.LoadRelaxed();
Mathieu Chartier987ccff2013-07-08 11:05:21 -07003529 // TODO: Tune the native heap utilization to be a value other than the java heap utilization.
3530 size_t target_size = native_size / GetTargetHeapUtilization();
3531 if (target_size > native_size + max_free_) {
3532 target_size = native_size + max_free_;
3533 } else if (target_size < native_size + min_free_) {
3534 target_size = native_size + min_free_;
3535 }
Mathieu Chartier8ec31f92014-09-03 10:30:11 -07003536 native_footprint_gc_watermark_ = std::min(growth_limit_, target_size);
Mathieu Chartier987ccff2013-07-08 11:05:21 -07003537}
3538
Mathieu Chartierafe49982014-03-27 10:55:04 -07003539collector::GarbageCollector* Heap::FindCollectorByGcType(collector::GcType gc_type) {
3540 for (const auto& collector : garbage_collectors_) {
3541 if (collector->GetCollectorType() == collector_type_ &&
3542 collector->GetGcType() == gc_type) {
3543 return collector;
3544 }
3545 }
3546 return nullptr;
3547}
3548
Mathieu Chartier2f8da3e2014-04-15 15:37:02 -07003549double Heap::HeapGrowthMultiplier() const {
3550 // If we don't care about pause times we are background, so return 1.0.
3551 if (!CareAboutPauseTimes() || IsLowMemoryMode()) {
3552 return 1.0;
3553 }
3554 return foreground_heap_growth_multiplier_;
3555}
3556
Mathieu Chartiere2c2f6e2014-12-16 18:49:31 -08003557void Heap::GrowForUtilization(collector::GarbageCollector* collector_ran,
3558 uint64_t bytes_allocated_before_gc) {
Mathieu Chartier2fde5332012-09-14 14:51:54 -07003559 // We know what our utilization is at this moment.
3560 // This doesn't actually resize any memory. It just lets the heap grow more when necessary.
Mathieu Chartier2f8da3e2014-04-15 15:37:02 -07003561 const uint64_t bytes_allocated = GetBytesAllocated();
Mathieu Chartier2f8da3e2014-04-15 15:37:02 -07003562 uint64_t target_size;
Mathieu Chartierafe49982014-03-27 10:55:04 -07003563 collector::GcType gc_type = collector_ran->GetGcType();
Mathieu Chartiere2c2f6e2014-12-16 18:49:31 -08003564 const double multiplier = HeapGrowthMultiplier(); // Use the multiplier to grow more for
3565 // foreground.
3566 const uint64_t adjusted_min_free = static_cast<uint64_t>(min_free_ * multiplier);
3567 const uint64_t adjusted_max_free = static_cast<uint64_t>(max_free_ * multiplier);
Mathieu Chartierbdd0fb92013-07-02 10:16:15 -07003568 if (gc_type != collector::kGcTypeSticky) {
3569 // Grow the heap for non sticky GC.
Mathieu Chartiere2c2f6e2014-12-16 18:49:31 -08003570 ssize_t delta = bytes_allocated / GetTargetHeapUtilization() - bytes_allocated;
Mathieu Chartier2f8da3e2014-04-15 15:37:02 -07003571 CHECK_GE(delta, 0);
3572 target_size = bytes_allocated + delta * multiplier;
Mathieu Chartiere2c2f6e2014-12-16 18:49:31 -08003573 target_size = std::min(target_size, bytes_allocated + adjusted_max_free);
3574 target_size = std::max(target_size, bytes_allocated + adjusted_min_free);
Mathieu Chartier590fee92013-09-13 13:46:47 -07003575 native_need_to_run_finalization_ = true;
Mathieu Chartierbdd0fb92013-07-02 10:16:15 -07003576 next_gc_type_ = collector::kGcTypeSticky;
3577 } else {
Mathieu Chartierafe49982014-03-27 10:55:04 -07003578 collector::GcType non_sticky_gc_type =
Mathieu Chartiere4cab172014-08-19 18:24:04 -07003579 HasZygoteSpace() ? collector::kGcTypePartial : collector::kGcTypeFull;
Mathieu Chartierafe49982014-03-27 10:55:04 -07003580 // Find what the next non sticky collector will be.
3581 collector::GarbageCollector* non_sticky_collector = FindCollectorByGcType(non_sticky_gc_type);
3582 // If the throughput of the current sticky GC >= throughput of the non sticky collector, then
3583 // do another sticky collection next.
3584 // We also check that the bytes allocated aren't over the footprint limit in order to prevent a
3585 // pathological case where dead objects which aren't reclaimed by sticky could get accumulated
3586 // if the sticky GC throughput always remained >= the full/partial throughput.
Mathieu Chartier10fb83a2014-06-15 15:15:43 -07003587 if (current_gc_iteration_.GetEstimatedThroughput() * kStickyGcThroughputAdjustment >=
Mathieu Chartierafe49982014-03-27 10:55:04 -07003588 non_sticky_collector->GetEstimatedMeanThroughput() &&
Mathieu Chartier10fb83a2014-06-15 15:15:43 -07003589 non_sticky_collector->NumberOfIterations() > 0 &&
Mathieu Chartierafe49982014-03-27 10:55:04 -07003590 bytes_allocated <= max_allowed_footprint_) {
Mathieu Chartierbdd0fb92013-07-02 10:16:15 -07003591 next_gc_type_ = collector::kGcTypeSticky;
3592 } else {
Mathieu Chartierafe49982014-03-27 10:55:04 -07003593 next_gc_type_ = non_sticky_gc_type;
Mathieu Chartierbdd0fb92013-07-02 10:16:15 -07003594 }
Mathieu Chartierbdd0fb92013-07-02 10:16:15 -07003595 // If we have freed enough memory, shrink the heap back down.
Mathieu Chartiere2c2f6e2014-12-16 18:49:31 -08003596 if (bytes_allocated + adjusted_max_free < max_allowed_footprint_) {
3597 target_size = bytes_allocated + adjusted_max_free;
Mathieu Chartierbdd0fb92013-07-02 10:16:15 -07003598 } else {
Mathieu Chartier2f8da3e2014-04-15 15:37:02 -07003599 target_size = std::max(bytes_allocated, static_cast<uint64_t>(max_allowed_footprint_));
Mathieu Chartierbdd0fb92013-07-02 10:16:15 -07003600 }
3601 }
Mathieu Chartier2775ee42013-08-20 17:43:47 -07003602 if (!ignore_max_footprint_) {
3603 SetIdealFootprint(target_size);
Hiroshi Yamauchi3e417802014-03-20 12:03:02 -07003604 if (IsGcConcurrent()) {
Mathieu Chartiere2c2f6e2014-12-16 18:49:31 -08003605 const uint64_t freed_bytes = current_gc_iteration_.GetFreedBytes() +
Hiroshi Yamauchi4460a842015-03-09 11:57:48 -07003606 current_gc_iteration_.GetFreedLargeObjectBytes() +
3607 current_gc_iteration_.GetFreedRevokeBytes();
Mathieu Chartiere2c2f6e2014-12-16 18:49:31 -08003608 // Bytes allocated will shrink by freed_bytes after the GC runs, so if we want to figure out
3609 // how many bytes were allocated during the GC we need to add freed_bytes back on.
3610 CHECK_GE(bytes_allocated + freed_bytes, bytes_allocated_before_gc);
3611 const uint64_t bytes_allocated_during_gc = bytes_allocated + freed_bytes -
3612 bytes_allocated_before_gc;
Mathieu Chartier2775ee42013-08-20 17:43:47 -07003613 // Calculate when to perform the next ConcurrentGC.
Mathieu Chartier2775ee42013-08-20 17:43:47 -07003614 // Calculate the estimated GC duration.
Mathieu Chartier10fb83a2014-06-15 15:15:43 -07003615 const double gc_duration_seconds = NsToMs(current_gc_iteration_.GetDurationNs()) / 1000.0;
Mathieu Chartier2775ee42013-08-20 17:43:47 -07003616 // Estimate how many remaining bytes we will have when we need to start the next GC.
Mathieu Chartiere2c2f6e2014-12-16 18:49:31 -08003617 size_t remaining_bytes = bytes_allocated_during_gc * gc_duration_seconds;
Mathieu Chartier74762802014-01-24 10:21:35 -08003618 remaining_bytes = std::min(remaining_bytes, kMaxConcurrentRemainingBytes);
Mathieu Chartier2775ee42013-08-20 17:43:47 -07003619 remaining_bytes = std::max(remaining_bytes, kMinConcurrentRemainingBytes);
3620 if (UNLIKELY(remaining_bytes > max_allowed_footprint_)) {
3621 // A never going to happen situation that from the estimated allocation rate we will exceed
3622 // the applications entire footprint with the given estimated allocation rate. Schedule
Mathieu Chartier74762802014-01-24 10:21:35 -08003623 // another GC nearly straight away.
3624 remaining_bytes = kMinConcurrentRemainingBytes;
Mathieu Chartier2775ee42013-08-20 17:43:47 -07003625 }
Mathieu Chartier74762802014-01-24 10:21:35 -08003626 DCHECK_LE(remaining_bytes, max_allowed_footprint_);
Mathieu Chartierdd162fb2014-08-06 17:06:33 -07003627 DCHECK_LE(max_allowed_footprint_, GetMaxMemory());
Mathieu Chartier74762802014-01-24 10:21:35 -08003628 // Start a concurrent GC when we get close to the estimated remaining bytes. When the
3629 // allocation rate is very high, remaining_bytes could tell us that we should start a GC
3630 // right away.
Mathieu Chartier2f8da3e2014-04-15 15:37:02 -07003631 concurrent_start_bytes_ = std::max(max_allowed_footprint_ - remaining_bytes,
3632 static_cast<size_t>(bytes_allocated));
Mathieu Chartier65db8802012-11-20 12:36:46 -08003633 }
Mathieu Chartier65db8802012-11-20 12:36:46 -08003634 }
Carl Shapiro69759ea2011-07-21 18:13:35 -07003635}
3636
Mathieu Chartier379d09f2015-01-08 11:28:13 -08003637void Heap::ClampGrowthLimit() {
Mathieu Chartierddac4232015-04-02 10:08:03 -07003638 // Use heap bitmap lock to guard against races with BindLiveToMarkBitmap.
Mathieu Chartiera9d82fe2016-01-25 20:06:11 -08003639 ScopedObjectAccess soa(Thread::Current());
3640 WriterMutexLock mu(soa.Self(), *Locks::heap_bitmap_lock_);
Mathieu Chartier379d09f2015-01-08 11:28:13 -08003641 capacity_ = growth_limit_;
3642 for (const auto& space : continuous_spaces_) {
3643 if (space->IsMallocSpace()) {
3644 gc::space::MallocSpace* malloc_space = space->AsMallocSpace();
3645 malloc_space->ClampGrowthLimit();
3646 }
3647 }
3648 // This space isn't added for performance reasons.
3649 if (main_space_backup_.get() != nullptr) {
3650 main_space_backup_->ClampGrowthLimit();
3651 }
3652}
3653
jeffhaoc1160702011-10-27 15:48:45 -07003654void Heap::ClearGrowthLimit() {
Mathieu Chartier80de7a62012-11-27 17:21:50 -08003655 growth_limit_ = capacity_;
Mathieu Chartiera9d82fe2016-01-25 20:06:11 -08003656 ScopedObjectAccess soa(Thread::Current());
Mathieu Chartier0310da52014-12-01 13:40:48 -08003657 for (const auto& space : continuous_spaces_) {
3658 if (space->IsMallocSpace()) {
3659 gc::space::MallocSpace* malloc_space = space->AsMallocSpace();
3660 malloc_space->ClearGrowthLimit();
3661 malloc_space->SetFootprintLimit(malloc_space->Capacity());
3662 }
3663 }
3664 // This space isn't added for performance reasons.
3665 if (main_space_backup_.get() != nullptr) {
3666 main_space_backup_->ClearGrowthLimit();
3667 main_space_backup_->SetFootprintLimit(main_space_backup_->Capacity());
3668 }
jeffhaoc1160702011-10-27 15:48:45 -07003669}
3670
Mathieu Chartier8668c3c2014-04-24 16:48:11 -07003671void Heap::AddFinalizerReference(Thread* self, mirror::Object** object) {
Ian Rogers00f7d0e2012-07-19 15:28:27 -07003672 ScopedObjectAccess soa(self);
Mathieu Chartier8668c3c2014-04-24 16:48:11 -07003673 ScopedLocalRef<jobject> arg(self->GetJniEnv(), soa.AddLocalReference<jobject>(*object));
Ian Rogers53b8b092014-03-13 23:45:53 -07003674 jvalue args[1];
3675 args[0].l = arg.get();
3676 InvokeWithJValues(soa, nullptr, WellKnownClasses::java_lang_ref_FinalizerReference_add, args);
Mathieu Chartier8668c3c2014-04-24 16:48:11 -07003677 // Restore object in case it gets moved.
3678 *object = soa.Decode<mirror::Object*>(arg.get());
Ian Rogers00f7d0e2012-07-19 15:28:27 -07003679}
3680
Hiroshi Yamauchi0ae98992015-05-01 14:33:19 -07003681void Heap::RequestConcurrentGCAndSaveObject(Thread* self, bool force_full, mirror::Object** obj) {
Mathieu Chartiereb8167a2014-05-07 15:43:14 -07003682 StackHandleScope<1> hs(self);
3683 HandleWrapper<mirror::Object> wrapper(hs.NewHandleWrapper(obj));
Hiroshi Yamauchi0ae98992015-05-01 14:33:19 -07003684 RequestConcurrentGC(self, force_full);
Mathieu Chartiereb8167a2014-05-07 15:43:14 -07003685}
3686
Mathieu Chartiera5eae692014-12-17 17:56:03 -08003687class Heap::ConcurrentGCTask : public HeapTask {
3688 public:
Roland Levillain3887c462015-08-12 18:15:42 +01003689 ConcurrentGCTask(uint64_t target_time, bool force_full)
3690 : HeapTask(target_time), force_full_(force_full) { }
Mathieu Chartiera5eae692014-12-17 17:56:03 -08003691 virtual void Run(Thread* self) OVERRIDE {
3692 gc::Heap* heap = Runtime::Current()->GetHeap();
Hiroshi Yamauchi0ae98992015-05-01 14:33:19 -07003693 heap->ConcurrentGC(self, force_full_);
Mathieu Chartiera5eae692014-12-17 17:56:03 -08003694 heap->ClearConcurrentGCRequest();
Ian Rogers120f1c72012-09-28 17:17:10 -07003695 }
Hiroshi Yamauchi0ae98992015-05-01 14:33:19 -07003696
3697 private:
3698 const bool force_full_; // If true, force full (or partial) collection.
Mathieu Chartiera5eae692014-12-17 17:56:03 -08003699};
3700
Mathieu Chartier90443472015-07-16 20:32:27 -07003701static bool CanAddHeapTask(Thread* self) REQUIRES(!Locks::runtime_shutdown_lock_) {
Mathieu Chartiera5eae692014-12-17 17:56:03 -08003702 Runtime* runtime = Runtime::Current();
3703 return runtime != nullptr && runtime->IsFinishedStarting() && !runtime->IsShuttingDown(self) &&
3704 !self->IsHandlingStackOverflow();
3705}
3706
3707void Heap::ClearConcurrentGCRequest() {
3708 concurrent_gc_pending_.StoreRelaxed(false);
3709}
3710
Hiroshi Yamauchi0ae98992015-05-01 14:33:19 -07003711void Heap::RequestConcurrentGC(Thread* self, bool force_full) {
Mathieu Chartierac195162015-02-20 18:44:28 +00003712 if (CanAddHeapTask(self) &&
3713 concurrent_gc_pending_.CompareExchangeStrongSequentiallyConsistent(false, true)) {
Hiroshi Yamauchi0ae98992015-05-01 14:33:19 -07003714 task_processor_->AddTask(self, new ConcurrentGCTask(NanoTime(), // Start straight away.
3715 force_full));
Mathieu Chartiera5eae692014-12-17 17:56:03 -08003716 }
Mathieu Chartier7664f5c2012-06-08 18:15:32 -07003717}
3718
Hiroshi Yamauchi0ae98992015-05-01 14:33:19 -07003719void Heap::ConcurrentGC(Thread* self, bool force_full) {
Mathieu Chartiera5eae692014-12-17 17:56:03 -08003720 if (!Runtime::Current()->IsShuttingDown(self)) {
3721 // Wait for any GCs currently running to finish.
3722 if (WaitForGcToComplete(kGcCauseBackground, self) == collector::kGcTypeNone) {
3723 // If the we can't run the GC type we wanted to run, find the next appropriate one and try that
3724 // instead. E.g. can't do partial, so do full instead.
Hiroshi Yamauchi0ae98992015-05-01 14:33:19 -07003725 collector::GcType next_gc_type = next_gc_type_;
3726 // If forcing full and next gc type is sticky, override with a non-sticky type.
3727 if (force_full && next_gc_type == collector::kGcTypeSticky) {
3728 next_gc_type = HasZygoteSpace() ? collector::kGcTypePartial : collector::kGcTypeFull;
3729 }
3730 if (CollectGarbageInternal(next_gc_type, kGcCauseBackground, false) ==
Mathieu Chartiera5eae692014-12-17 17:56:03 -08003731 collector::kGcTypeNone) {
3732 for (collector::GcType gc_type : gc_plan_) {
3733 // Attempt to run the collector, if we succeed, we are done.
Hiroshi Yamauchi0ae98992015-05-01 14:33:19 -07003734 if (gc_type > next_gc_type &&
Mathieu Chartiera5eae692014-12-17 17:56:03 -08003735 CollectGarbageInternal(gc_type, kGcCauseBackground, false) !=
3736 collector::kGcTypeNone) {
3737 break;
3738 }
Mathieu Chartierf9ed0d32013-11-21 16:42:47 -08003739 }
3740 }
3741 }
Mathieu Chartiercc236d72012-07-20 10:29:05 -07003742 }
Mathieu Chartier7664f5c2012-06-08 18:15:32 -07003743}
3744
Mathieu Chartiera5eae692014-12-17 17:56:03 -08003745class Heap::CollectorTransitionTask : public HeapTask {
3746 public:
Mathieu Chartiera4f6af92015-08-11 17:35:25 -07003747 explicit CollectorTransitionTask(uint64_t target_time) : HeapTask(target_time) {}
3748
Mathieu Chartiera5eae692014-12-17 17:56:03 -08003749 virtual void Run(Thread* self) OVERRIDE {
3750 gc::Heap* heap = Runtime::Current()->GetHeap();
3751 heap->DoPendingCollectorTransition();
3752 heap->ClearPendingCollectorTransition(self);
Mathieu Chartiera5f9de02014-02-28 16:48:42 -08003753 }
Mathieu Chartiera5eae692014-12-17 17:56:03 -08003754};
3755
3756void Heap::ClearPendingCollectorTransition(Thread* self) {
3757 MutexLock mu(self, *pending_task_lock_);
3758 pending_collector_transition_ = nullptr;
Mathieu Chartiera5f9de02014-02-28 16:48:42 -08003759}
3760
Mathieu Chartiera5eae692014-12-17 17:56:03 -08003761void Heap::RequestCollectorTransition(CollectorType desired_collector_type, uint64_t delta_time) {
3762 Thread* self = Thread::Current();
3763 desired_collector_type_ = desired_collector_type;
3764 if (desired_collector_type_ == collector_type_ || !CanAddHeapTask(self)) {
3765 return;
3766 }
3767 CollectorTransitionTask* added_task = nullptr;
3768 const uint64_t target_time = NanoTime() + delta_time;
3769 {
3770 MutexLock mu(self, *pending_task_lock_);
3771 // If we have an existing collector transition, update the targe time to be the new target.
3772 if (pending_collector_transition_ != nullptr) {
3773 task_processor_->UpdateTargetRunTime(self, pending_collector_transition_, target_time);
3774 return;
3775 }
3776 added_task = new CollectorTransitionTask(target_time);
3777 pending_collector_transition_ = added_task;
3778 }
3779 task_processor_->AddTask(self, added_task);
3780}
3781
3782class Heap::HeapTrimTask : public HeapTask {
3783 public:
3784 explicit HeapTrimTask(uint64_t delta_time) : HeapTask(NanoTime() + delta_time) { }
3785 virtual void Run(Thread* self) OVERRIDE {
3786 gc::Heap* heap = Runtime::Current()->GetHeap();
3787 heap->Trim(self);
3788 heap->ClearPendingTrim(self);
3789 }
3790};
3791
3792void Heap::ClearPendingTrim(Thread* self) {
3793 MutexLock mu(self, *pending_task_lock_);
3794 pending_heap_trim_ = nullptr;
3795}
3796
3797void Heap::RequestTrim(Thread* self) {
3798 if (!CanAddHeapTask(self)) {
3799 return;
3800 }
Ian Rogers48931882013-01-22 14:35:16 -08003801 // GC completed and now we must decide whether to request a heap trim (advising pages back to the
3802 // kernel) or not. Issuing a request will also cause trimming of the libc heap. As a trim scans
3803 // a space it will hold its lock and can become a cause of jank.
3804 // Note, the large object space self trims and the Zygote space was trimmed and unchanging since
3805 // forking.
3806
Elliott Hughes8cf5bc02012-02-02 16:32:16 -08003807 // We don't have a good measure of how worthwhile a trim might be. We can't use the live bitmap
3808 // because that only marks object heads, so a large array looks like lots of empty space. We
3809 // don't just call dlmalloc all the time, because the cost of an _attempted_ trim is proportional
3810 // to utilization (which is probably inversely proportional to how much benefit we can expect).
3811 // We could try mincore(2) but that's only a measure of how many pages we haven't given away,
3812 // not how much use we're making of those pages.
Mathieu Chartiera5eae692014-12-17 17:56:03 -08003813 HeapTrimTask* added_task = nullptr;
Mathieu Chartier440e4ce2014-03-31 16:36:35 -07003814 {
Mathieu Chartiera5eae692014-12-17 17:56:03 -08003815 MutexLock mu(self, *pending_task_lock_);
3816 if (pending_heap_trim_ != nullptr) {
3817 // Already have a heap trim request in task processor, ignore this request.
Mathieu Chartier440e4ce2014-03-31 16:36:35 -07003818 return;
Mathieu Chartiera5f9de02014-02-28 16:48:42 -08003819 }
Mathieu Chartiera5eae692014-12-17 17:56:03 -08003820 added_task = new HeapTrimTask(kHeapTrimWait);
3821 pending_heap_trim_ = added_task;
Mathieu Chartierc39e3422013-08-07 16:41:36 -07003822 }
Mathieu Chartiera5eae692014-12-17 17:56:03 -08003823 task_processor_->AddTask(self, added_task);
Mathieu Chartiera5f9de02014-02-28 16:48:42 -08003824}
3825
Hiroshi Yamauchicf58d4a2013-09-26 14:21:22 -07003826void Heap::RevokeThreadLocalBuffers(Thread* thread) {
Mathieu Chartiere6da9af2013-12-16 11:54:42 -08003827 if (rosalloc_space_ != nullptr) {
Hiroshi Yamauchi4460a842015-03-09 11:57:48 -07003828 size_t freed_bytes_revoke = rosalloc_space_->RevokeThreadLocalBuffers(thread);
3829 if (freed_bytes_revoke > 0U) {
3830 num_bytes_freed_revoke_.FetchAndAddSequentiallyConsistent(freed_bytes_revoke);
3831 CHECK_GE(num_bytes_allocated_.LoadRelaxed(), num_bytes_freed_revoke_.LoadRelaxed());
3832 }
Mathieu Chartiere6da9af2013-12-16 11:54:42 -08003833 }
Mathieu Chartier692fafd2013-11-29 17:24:40 -08003834 if (bump_pointer_space_ != nullptr) {
Hiroshi Yamauchi4460a842015-03-09 11:57:48 -07003835 CHECK_EQ(bump_pointer_space_->RevokeThreadLocalBuffers(thread), 0U);
Mathieu Chartier692fafd2013-11-29 17:24:40 -08003836 }
Hiroshi Yamauchi2cd334a2015-01-09 14:03:35 -08003837 if (region_space_ != nullptr) {
Hiroshi Yamauchi4460a842015-03-09 11:57:48 -07003838 CHECK_EQ(region_space_->RevokeThreadLocalBuffers(thread), 0U);
Hiroshi Yamauchi2cd334a2015-01-09 14:03:35 -08003839 }
Hiroshi Yamauchicf58d4a2013-09-26 14:21:22 -07003840}
3841
Hiroshi Yamauchic93c5302014-03-20 16:15:37 -07003842void Heap::RevokeRosAllocThreadLocalBuffers(Thread* thread) {
3843 if (rosalloc_space_ != nullptr) {
Hiroshi Yamauchi4460a842015-03-09 11:57:48 -07003844 size_t freed_bytes_revoke = rosalloc_space_->RevokeThreadLocalBuffers(thread);
3845 if (freed_bytes_revoke > 0U) {
3846 num_bytes_freed_revoke_.FetchAndAddSequentiallyConsistent(freed_bytes_revoke);
3847 CHECK_GE(num_bytes_allocated_.LoadRelaxed(), num_bytes_freed_revoke_.LoadRelaxed());
3848 }
Hiroshi Yamauchic93c5302014-03-20 16:15:37 -07003849 }
3850}
3851
Hiroshi Yamauchicf58d4a2013-09-26 14:21:22 -07003852void Heap::RevokeAllThreadLocalBuffers() {
Mathieu Chartiere6da9af2013-12-16 11:54:42 -08003853 if (rosalloc_space_ != nullptr) {
Hiroshi Yamauchi4460a842015-03-09 11:57:48 -07003854 size_t freed_bytes_revoke = rosalloc_space_->RevokeAllThreadLocalBuffers();
3855 if (freed_bytes_revoke > 0U) {
3856 num_bytes_freed_revoke_.FetchAndAddSequentiallyConsistent(freed_bytes_revoke);
3857 CHECK_GE(num_bytes_allocated_.LoadRelaxed(), num_bytes_freed_revoke_.LoadRelaxed());
3858 }
Mathieu Chartiere6da9af2013-12-16 11:54:42 -08003859 }
Mathieu Chartier692fafd2013-11-29 17:24:40 -08003860 if (bump_pointer_space_ != nullptr) {
Hiroshi Yamauchi4460a842015-03-09 11:57:48 -07003861 CHECK_EQ(bump_pointer_space_->RevokeAllThreadLocalBuffers(), 0U);
Mathieu Chartier692fafd2013-11-29 17:24:40 -08003862 }
Hiroshi Yamauchi2cd334a2015-01-09 14:03:35 -08003863 if (region_space_ != nullptr) {
Hiroshi Yamauchi4460a842015-03-09 11:57:48 -07003864 CHECK_EQ(region_space_->RevokeAllThreadLocalBuffers(), 0U);
Hiroshi Yamauchi2cd334a2015-01-09 14:03:35 -08003865 }
Hiroshi Yamauchicf58d4a2013-09-26 14:21:22 -07003866}
3867
Mathieu Chartier987ccff2013-07-08 11:05:21 -07003868bool Heap::IsGCRequestPending() const {
Mathieu Chartiera5eae692014-12-17 17:56:03 -08003869 return concurrent_gc_pending_.LoadRelaxed();
Mathieu Chartier987ccff2013-07-08 11:05:21 -07003870}
3871
Mathieu Chartierb5de3bb2015-06-05 13:21:05 -07003872void Heap::RunFinalization(JNIEnv* env, uint64_t timeout) {
3873 env->CallStaticVoidMethod(WellKnownClasses::dalvik_system_VMRuntime,
3874 WellKnownClasses::dalvik_system_VMRuntime_runFinalization,
3875 static_cast<jlong>(timeout));
Mathieu Chartier590fee92013-09-13 13:46:47 -07003876}
3877
Mathieu Chartier8ec31f92014-09-03 10:30:11 -07003878void Heap::RegisterNativeAllocation(JNIEnv* env, size_t bytes) {
Mathieu Chartier590fee92013-09-13 13:46:47 -07003879 Thread* self = ThreadForEnv(env);
Mathieu Chartier5d2a3f72016-05-11 11:35:39 -07003880 {
3881 MutexLock mu(self, native_histogram_lock_);
3882 native_allocation_histogram_.AddValue(bytes);
3883 }
Mathieu Chartier590fee92013-09-13 13:46:47 -07003884 if (native_need_to_run_finalization_) {
Mathieu Chartierb5de3bb2015-06-05 13:21:05 -07003885 RunFinalization(env, kNativeAllocationFinalizeTimeout);
Mathieu Chartier590fee92013-09-13 13:46:47 -07003886 UpdateMaxNativeFootprint();
3887 native_need_to_run_finalization_ = false;
3888 }
Mathieu Chartier987ccff2013-07-08 11:05:21 -07003889 // Total number of native bytes allocated.
Ian Rogers3e5cf302014-05-20 16:40:37 -07003890 size_t new_native_bytes_allocated = native_bytes_allocated_.FetchAndAddSequentiallyConsistent(bytes);
3891 new_native_bytes_allocated += bytes;
3892 if (new_native_bytes_allocated > native_footprint_gc_watermark_) {
Mathieu Chartiere4cab172014-08-19 18:24:04 -07003893 collector::GcType gc_type = HasZygoteSpace() ? collector::kGcTypePartial :
Mathieu Chartiercbb2d202013-11-14 17:45:16 -08003894 collector::kGcTypeFull;
3895
Mathieu Chartier987ccff2013-07-08 11:05:21 -07003896 // The second watermark is higher than the gc watermark. If you hit this it means you are
3897 // allocating native objects faster than the GC can keep up with.
Mathieu Chartier08487452014-09-02 16:21:01 -07003898 if (new_native_bytes_allocated > growth_limit_) {
Mathieu Chartier89a201e2014-05-02 10:27:26 -07003899 if (WaitForGcToComplete(kGcCauseForNativeAlloc, self) != collector::kGcTypeNone) {
Mathieu Chartier590fee92013-09-13 13:46:47 -07003900 // Just finished a GC, attempt to run finalizers.
Mathieu Chartierb5de3bb2015-06-05 13:21:05 -07003901 RunFinalization(env, kNativeAllocationFinalizeTimeout);
Mathieu Chartier590fee92013-09-13 13:46:47 -07003902 CHECK(!env->ExceptionCheck());
Lin Zang60e27162015-03-10 18:53:21 +08003903 // Native bytes allocated may be updated by finalization, refresh it.
3904 new_native_bytes_allocated = native_bytes_allocated_.LoadRelaxed();
Mathieu Chartier590fee92013-09-13 13:46:47 -07003905 }
3906 // If we still are over the watermark, attempt a GC for alloc and run finalizers.
Mathieu Chartier08487452014-09-02 16:21:01 -07003907 if (new_native_bytes_allocated > growth_limit_) {
Hiroshi Yamauchi6f4ffe42014-01-13 12:30:44 -08003908 CollectGarbageInternal(gc_type, kGcCauseForNativeAlloc, false);
Mathieu Chartierb5de3bb2015-06-05 13:21:05 -07003909 RunFinalization(env, kNativeAllocationFinalizeTimeout);
Mathieu Chartier590fee92013-09-13 13:46:47 -07003910 native_need_to_run_finalization_ = false;
3911 CHECK(!env->ExceptionCheck());
3912 }
3913 // We have just run finalizers, update the native watermark since it is very likely that
3914 // finalizers released native managed allocations.
3915 UpdateMaxNativeFootprint();
3916 } else if (!IsGCRequestPending()) {
Hiroshi Yamauchi3e417802014-03-20 12:03:02 -07003917 if (IsGcConcurrent()) {
Hiroshi Yamauchi0ae98992015-05-01 14:33:19 -07003918 RequestConcurrentGC(self, true); // Request non-sticky type.
Mathieu Chartier590fee92013-09-13 13:46:47 -07003919 } else {
Hiroshi Yamauchid20aba12014-04-11 15:31:09 -07003920 CollectGarbageInternal(gc_type, kGcCauseForNativeAlloc, false);
Mathieu Chartier987ccff2013-07-08 11:05:21 -07003921 }
3922 }
3923 }
3924}
3925
Mathieu Chartier8ec31f92014-09-03 10:30:11 -07003926void Heap::RegisterNativeFree(JNIEnv* env, size_t bytes) {
3927 size_t expected_size;
Mathieu Chartier5d2a3f72016-05-11 11:35:39 -07003928 {
3929 MutexLock mu(Thread::Current(), native_histogram_lock_);
3930 native_free_histogram_.AddValue(bytes);
3931 }
Mathieu Chartier987ccff2013-07-08 11:05:21 -07003932 do {
Ian Rogers3e5cf302014-05-20 16:40:37 -07003933 expected_size = native_bytes_allocated_.LoadRelaxed();
Mathieu Chartier8ec31f92014-09-03 10:30:11 -07003934 if (UNLIKELY(bytes > expected_size)) {
Mathieu Chartier590fee92013-09-13 13:46:47 -07003935 ScopedObjectAccess soa(env);
3936 env->ThrowNew(WellKnownClasses::java_lang_RuntimeException,
Mathieu Chartier8ec31f92014-09-03 10:30:11 -07003937 StringPrintf("Attempted to free %zd native bytes with only %zd native bytes "
Mathieu Chartier590fee92013-09-13 13:46:47 -07003938 "registered as allocated", bytes, expected_size).c_str());
3939 break;
3940 }
Mathieu Chartier8ec31f92014-09-03 10:30:11 -07003941 } while (!native_bytes_allocated_.CompareExchangeWeakRelaxed(expected_size,
3942 expected_size - bytes));
Mathieu Chartier987ccff2013-07-08 11:05:21 -07003943}
3944
Ian Rogersef7d42f2014-01-06 12:55:46 -08003945size_t Heap::GetTotalMemory() const {
Mathieu Chartierdd162fb2014-08-06 17:06:33 -07003946 return std::max(max_allowed_footprint_, GetBytesAllocated());
Hiroshi Yamauchi09b07a92013-07-15 13:17:06 -07003947}
3948
Mathieu Chartier11409ae2013-09-23 11:49:36 -07003949void Heap::AddModUnionTable(accounting::ModUnionTable* mod_union_table) {
3950 DCHECK(mod_union_table != nullptr);
3951 mod_union_tables_.Put(mod_union_table->GetSpace(), mod_union_table);
3952}
3953
Mathieu Chartierc645f1d2014-03-06 18:11:53 -08003954void Heap::CheckPreconditionsForAllocObject(mirror::Class* c, size_t byte_count) {
Mathieu Chartiera5eae692014-12-17 17:56:03 -08003955 CHECK(c == nullptr || (c->IsClassClass() && byte_count >= sizeof(mirror::Class)) ||
Mathieu Chartier52a7f5c2015-08-18 18:35:52 -07003956 (c->IsVariableSize() || c->GetObjectSize() == byte_count)) << c->GetClassFlags();
Mathieu Chartierc645f1d2014-03-06 18:11:53 -08003957 CHECK_GE(byte_count, sizeof(mirror::Object));
3958}
3959
Hiroshi Yamauchi38e68e92014-03-07 13:59:08 -08003960void Heap::AddRememberedSet(accounting::RememberedSet* remembered_set) {
3961 CHECK(remembered_set != nullptr);
3962 space::Space* space = remembered_set->GetSpace();
3963 CHECK(space != nullptr);
Mathieu Chartier8e4a96d2014-05-21 10:44:32 -07003964 CHECK(remembered_sets_.find(space) == remembered_sets_.end()) << space;
Hiroshi Yamauchi38e68e92014-03-07 13:59:08 -08003965 remembered_sets_.Put(space, remembered_set);
Mathieu Chartier8e4a96d2014-05-21 10:44:32 -07003966 CHECK(remembered_sets_.find(space) != remembered_sets_.end()) << space;
Hiroshi Yamauchi38e68e92014-03-07 13:59:08 -08003967}
3968
3969void Heap::RemoveRememberedSet(space::Space* space) {
3970 CHECK(space != nullptr);
3971 auto it = remembered_sets_.find(space);
3972 CHECK(it != remembered_sets_.end());
Mathieu Chartier5189e242014-07-24 11:11:05 -07003973 delete it->second;
Hiroshi Yamauchi38e68e92014-03-07 13:59:08 -08003974 remembered_sets_.erase(it);
3975 CHECK(remembered_sets_.find(space) == remembered_sets_.end());
3976}
3977
Mathieu Chartier4aeec172014-03-27 16:09:46 -07003978void Heap::ClearMarkedObjects() {
3979 // Clear all of the spaces' mark bitmaps.
3980 for (const auto& space : GetContinuousSpaces()) {
Mathieu Chartiera8e8f9c2014-04-09 14:51:05 -07003981 accounting::ContinuousSpaceBitmap* mark_bitmap = space->GetMarkBitmap();
Mathieu Chartier4aeec172014-03-27 16:09:46 -07003982 if (space->GetLiveBitmap() != mark_bitmap) {
3983 mark_bitmap->Clear();
3984 }
3985 }
3986 // Clear the marked objects in the discontinous space object sets.
3987 for (const auto& space : GetDiscontinuousSpaces()) {
Mathieu Chartierbbd695c2014-04-16 09:48:48 -07003988 space->GetMarkBitmap()->Clear();
Mathieu Chartier4aeec172014-03-27 16:09:46 -07003989 }
3990}
3991
Man Cao8c2ff642015-05-27 17:25:30 -07003992void Heap::SetAllocationRecords(AllocRecordObjectMap* records) {
3993 allocation_records_.reset(records);
3994}
3995
Man Cao1ed11b92015-06-11 22:47:35 -07003996void Heap::VisitAllocationRecords(RootVisitor* visitor) const {
3997 if (IsAllocTrackingEnabled()) {
3998 MutexLock mu(Thread::Current(), *Locks::alloc_tracker_lock_);
3999 if (IsAllocTrackingEnabled()) {
4000 GetAllocationRecords()->VisitRoots(visitor);
4001 }
4002 }
4003}
4004
Mathieu Chartier97509952015-07-13 14:35:43 -07004005void Heap::SweepAllocationRecords(IsMarkedVisitor* visitor) const {
Man Cao8c2ff642015-05-27 17:25:30 -07004006 if (IsAllocTrackingEnabled()) {
4007 MutexLock mu(Thread::Current(), *Locks::alloc_tracker_lock_);
4008 if (IsAllocTrackingEnabled()) {
Mathieu Chartier97509952015-07-13 14:35:43 -07004009 GetAllocationRecords()->SweepAllocationRecords(visitor);
Man Cao8c2ff642015-05-27 17:25:30 -07004010 }
4011 }
4012}
4013
Man Cao42c3c332015-06-23 16:38:25 -07004014void Heap::AllowNewAllocationRecords() const {
Hiroshi Yamauchifdbd13c2015-09-02 16:16:58 -07004015 CHECK(!kUseReadBarrier);
Hiroshi Yamauchi6f0c6cd2016-03-18 17:17:52 -07004016 MutexLock mu(Thread::Current(), *Locks::alloc_tracker_lock_);
4017 AllocRecordObjectMap* allocation_records = GetAllocationRecords();
4018 if (allocation_records != nullptr) {
4019 allocation_records->AllowNewAllocationRecords();
Man Cao42c3c332015-06-23 16:38:25 -07004020 }
4021}
4022
4023void Heap::DisallowNewAllocationRecords() const {
Hiroshi Yamauchifdbd13c2015-09-02 16:16:58 -07004024 CHECK(!kUseReadBarrier);
Hiroshi Yamauchi6f0c6cd2016-03-18 17:17:52 -07004025 MutexLock mu(Thread::Current(), *Locks::alloc_tracker_lock_);
4026 AllocRecordObjectMap* allocation_records = GetAllocationRecords();
4027 if (allocation_records != nullptr) {
4028 allocation_records->DisallowNewAllocationRecords();
Man Cao42c3c332015-06-23 16:38:25 -07004029 }
4030}
4031
Hiroshi Yamauchifdbd13c2015-09-02 16:16:58 -07004032void Heap::BroadcastForNewAllocationRecords() const {
4033 CHECK(kUseReadBarrier);
Hiroshi Yamauchi6f0c6cd2016-03-18 17:17:52 -07004034 // Always broadcast without checking IsAllocTrackingEnabled() because IsAllocTrackingEnabled() may
4035 // be set to false while some threads are waiting for system weak access in
4036 // AllocRecordObjectMap::RecordAllocation() and we may fail to wake them up. b/27467554.
4037 MutexLock mu(Thread::Current(), *Locks::alloc_tracker_lock_);
4038 AllocRecordObjectMap* allocation_records = GetAllocationRecords();
4039 if (allocation_records != nullptr) {
4040 allocation_records->BroadcastForNewAllocationRecords();
Hiroshi Yamauchifdbd13c2015-09-02 16:16:58 -07004041 }
4042}
4043
Mathieu Chartier31000802015-06-14 14:14:37 -07004044// Based on debug malloc logic from libc/bionic/debug_stacktrace.cpp.
4045class StackCrawlState {
4046 public:
4047 StackCrawlState(uintptr_t* frames, size_t max_depth, size_t skip_count)
4048 : frames_(frames), frame_count_(0), max_depth_(max_depth), skip_count_(skip_count) {
4049 }
4050 size_t GetFrameCount() const {
4051 return frame_count_;
4052 }
4053 static _Unwind_Reason_Code Callback(_Unwind_Context* context, void* arg) {
4054 auto* const state = reinterpret_cast<StackCrawlState*>(arg);
4055 const uintptr_t ip = _Unwind_GetIP(context);
4056 // The first stack frame is get_backtrace itself. Skip it.
4057 if (ip != 0 && state->skip_count_ > 0) {
4058 --state->skip_count_;
4059 return _URC_NO_REASON;
4060 }
4061 // ip may be off for ARM but it shouldn't matter since we only use it for hashing.
4062 state->frames_[state->frame_count_] = ip;
4063 state->frame_count_++;
4064 return state->frame_count_ >= state->max_depth_ ? _URC_END_OF_STACK : _URC_NO_REASON;
4065 }
4066
4067 private:
4068 uintptr_t* const frames_;
4069 size_t frame_count_;
4070 const size_t max_depth_;
4071 size_t skip_count_;
4072};
4073
4074static size_t get_backtrace(uintptr_t* frames, size_t max_depth) {
4075 StackCrawlState state(frames, max_depth, 0u);
4076 _Unwind_Backtrace(&StackCrawlState::Callback, &state);
4077 return state.GetFrameCount();
4078}
4079
4080void Heap::CheckGcStressMode(Thread* self, mirror::Object** obj) {
4081 auto* const runtime = Runtime::Current();
4082 if (gc_stress_mode_ && runtime->GetClassLinker()->IsInitialized() &&
4083 !runtime->IsActiveTransaction() && mirror::Class::HasJavaLangClass()) {
4084 // Check if we should GC.
4085 bool new_backtrace = false;
4086 {
4087 static constexpr size_t kMaxFrames = 16u;
4088 uintptr_t backtrace[kMaxFrames];
4089 const size_t frames = get_backtrace(backtrace, kMaxFrames);
4090 uint64_t hash = 0;
4091 for (size_t i = 0; i < frames; ++i) {
4092 hash = hash * 2654435761 + backtrace[i];
4093 hash += (hash >> 13) ^ (hash << 6);
4094 }
4095 MutexLock mu(self, *backtrace_lock_);
4096 new_backtrace = seen_backtraces_.find(hash) == seen_backtraces_.end();
4097 if (new_backtrace) {
4098 seen_backtraces_.insert(hash);
4099 }
4100 }
4101 if (new_backtrace) {
4102 StackHandleScope<1> hs(self);
4103 auto h = hs.NewHandleWrapper(obj);
4104 CollectGarbage(false);
4105 unique_backtrace_count_.FetchAndAddSequentiallyConsistent(1);
4106 } else {
4107 seen_backtrace_count_.FetchAndAddSequentiallyConsistent(1);
4108 }
4109 }
4110}
4111
Mathieu Chartier51168372015-08-12 16:40:32 -07004112void Heap::DisableGCForShutdown() {
4113 Thread* const self = Thread::Current();
4114 CHECK(Runtime::Current()->IsShuttingDown(self));
4115 MutexLock mu(self, *gc_complete_lock_);
4116 gc_disabled_for_shutdown_ = true;
4117}
4118
Mathieu Chartierfbc31082016-01-24 11:59:56 -08004119bool Heap::ObjectIsInBootImageSpace(mirror::Object* obj) const {
4120 for (gc::space::ImageSpace* space : boot_image_spaces_) {
4121 if (space->HasAddress(obj)) {
4122 return true;
4123 }
4124 }
4125 return false;
4126}
4127
Mingyao Yang6ea1a0e2016-01-29 12:12:49 -08004128bool Heap::IsInBootImageOatFile(const void* p) const {
4129 for (gc::space::ImageSpace* space : boot_image_spaces_) {
4130 if (space->GetOatFile()->Contains(p)) {
4131 return true;
4132 }
4133 }
4134 return false;
4135}
4136
Mathieu Chartierfbc31082016-01-24 11:59:56 -08004137void Heap::GetBootImagesSize(uint32_t* boot_image_begin,
4138 uint32_t* boot_image_end,
4139 uint32_t* boot_oat_begin,
4140 uint32_t* boot_oat_end) {
4141 DCHECK(boot_image_begin != nullptr);
4142 DCHECK(boot_image_end != nullptr);
4143 DCHECK(boot_oat_begin != nullptr);
4144 DCHECK(boot_oat_end != nullptr);
4145 *boot_image_begin = 0u;
4146 *boot_image_end = 0u;
4147 *boot_oat_begin = 0u;
4148 *boot_oat_end = 0u;
4149 for (gc::space::ImageSpace* space_ : GetBootImageSpaces()) {
4150 const uint32_t image_begin = PointerToLowMemUInt32(space_->Begin());
4151 const uint32_t image_size = space_->GetImageHeader().GetImageSize();
4152 if (*boot_image_begin == 0 || image_begin < *boot_image_begin) {
4153 *boot_image_begin = image_begin;
4154 }
4155 *boot_image_end = std::max(*boot_image_end, image_begin + image_size);
4156 const OatFile* boot_oat_file = space_->GetOatFile();
4157 const uint32_t oat_begin = PointerToLowMemUInt32(boot_oat_file->Begin());
4158 const uint32_t oat_size = boot_oat_file->Size();
4159 if (*boot_oat_begin == 0 || oat_begin < *boot_oat_begin) {
4160 *boot_oat_begin = oat_begin;
4161 }
4162 *boot_oat_end = std::max(*boot_oat_end, oat_begin + oat_size);
4163 }
4164}
4165
Ian Rogers1d54e732013-05-02 21:10:01 -07004166} // namespace gc
Carl Shapiro69759ea2011-07-21 18:13:35 -07004167} // namespace art