1 /*
2 * Copyright (c) 2001, 2017, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "precompiled.hpp"
26 #include "classfile/metadataOnStackMark.hpp"
27 #include "classfile/symbolTable.hpp"
28 #include "code/codeCache.hpp"
29 #include "gc/g1/concurrentMarkThread.inline.hpp"
30 #include "gc/g1/g1CollectedHeap.inline.hpp"
31 #include "gc/g1/g1CollectorState.hpp"
32 #include "gc/g1/g1ConcurrentMark.inline.hpp"
33 #include "gc/g1/g1HeapVerifier.hpp"
34 #include "gc/g1/g1OopClosures.inline.hpp"
35 #include "gc/g1/g1CardLiveData.inline.hpp"
36 #include "gc/g1/g1Policy.hpp"
37 #include "gc/g1/g1StringDedup.hpp"
38 #include "gc/g1/heapRegion.inline.hpp"
39 #include "gc/g1/heapRegionRemSet.hpp"
40 #include "gc/g1/heapRegionSet.inline.hpp"
41 #include "gc/shared/gcId.hpp"
42 #include "gc/shared/gcTimer.hpp"
43 #include "gc/shared/gcTrace.hpp"
44 #include "gc/shared/gcTraceTime.inline.hpp"
45 #include "gc/shared/genOopClosures.inline.hpp"
46 #include "gc/shared/referencePolicy.hpp"
47 #include "gc/shared/strongRootsScope.hpp"
48 #include "gc/shared/suspendibleThreadSet.hpp"
49 #include "gc/shared/taskqueue.inline.hpp"
50 #include "gc/shared/vmGCOperations.hpp"
51 #include "gc/shared/weakProcessor.hpp"
52 #include "logging/log.hpp"
53 #include "memory/allocation.hpp"
54 #include "memory/resourceArea.hpp"
55 #include "oops/oop.inline.hpp"
56 #include "runtime/atomic.hpp"
57 #include "runtime/handles.inline.hpp"
58 #include "runtime/java.hpp"
59 #include "runtime/prefetch.inline.hpp"
60 #include "services/memTracker.hpp"
61 #include "utilities/align.hpp"
62 #include "utilities/growableArray.hpp"
63
64 bool G1CMBitMapClosure::do_addr(HeapWord* const addr) {
65 assert(addr < _cm->finger(), "invariant");
66 assert(addr >= _task->finger(), "invariant");
67
68 // We move that task's local finger along.
69 _task->move_finger_to(addr);
70
71 _task->scan_task_entry(G1TaskQueueEntry::from_oop(oop(addr)));
72 // we only partially drain the local queue and global stack
73 _task->drain_local_queue(true);
74 _task->drain_global_stack(true);
75
76 // if the has_aborted flag has been raised, we need to bail out of
77 // the iteration
78 return !_task->has_aborted();
79 }
80
81 G1CMMarkStack::G1CMMarkStack() :
82 _max_chunk_capacity(0),
83 _base(NULL),
84 _chunk_capacity(0) {
85 set_empty();
86 }
87
88 bool G1CMMarkStack::resize(size_t new_capacity) {
89 assert(is_empty(), "Only resize when stack is empty.");
90 assert(new_capacity <= _max_chunk_capacity,
91 "Trying to resize stack to " SIZE_FORMAT " chunks when the maximum is " SIZE_FORMAT, new_capacity, _max_chunk_capacity);
92
93 TaskQueueEntryChunk* new_base = MmapArrayAllocator<TaskQueueEntryChunk>::allocate_or_null(new_capacity, mtGC);
94
95 if (new_base == NULL) {
96 log_warning(gc)("Failed to reserve memory for new overflow mark stack with " SIZE_FORMAT " chunks and size " SIZE_FORMAT "B.", new_capacity, new_capacity * sizeof(TaskQueueEntryChunk));
97 return false;
98 }
99 // Release old mapping.
100 if (_base != NULL) {
101 MmapArrayAllocator<TaskQueueEntryChunk>::free(_base, _chunk_capacity);
102 }
103
104 _base = new_base;
105 _chunk_capacity = new_capacity;
106 set_empty();
107
108 return true;
109 }
110
111 size_t G1CMMarkStack::capacity_alignment() {
112 return (size_t)lcm(os::vm_allocation_granularity(), sizeof(TaskQueueEntryChunk)) / sizeof(G1TaskQueueEntry);
113 }
114
115 bool G1CMMarkStack::initialize(size_t initial_capacity, size_t max_capacity) {
116 guarantee(_max_chunk_capacity == 0, "G1CMMarkStack already initialized.");
117
118 size_t const TaskEntryChunkSizeInVoidStar = sizeof(TaskQueueEntryChunk) / sizeof(G1TaskQueueEntry);
119
120 _max_chunk_capacity = align_up(max_capacity, capacity_alignment()) / TaskEntryChunkSizeInVoidStar;
121 size_t initial_chunk_capacity = align_up(initial_capacity, capacity_alignment()) / TaskEntryChunkSizeInVoidStar;
122
123 guarantee(initial_chunk_capacity <= _max_chunk_capacity,
124 "Maximum chunk capacity " SIZE_FORMAT " smaller than initial capacity " SIZE_FORMAT,
125 _max_chunk_capacity,
126 initial_chunk_capacity);
127
128 log_debug(gc)("Initialize mark stack with " SIZE_FORMAT " chunks, maximum " SIZE_FORMAT,
129 initial_chunk_capacity, _max_chunk_capacity);
130
131 return resize(initial_chunk_capacity);
132 }
133
134 void G1CMMarkStack::expand() {
135 if (_chunk_capacity == _max_chunk_capacity) {
136 log_debug(gc)("Can not expand overflow mark stack further, already at maximum capacity of " SIZE_FORMAT " chunks.", _chunk_capacity);
137 return;
138 }
139 size_t old_capacity = _chunk_capacity;
140 // Double capacity if possible
141 size_t new_capacity = MIN2(old_capacity * 2, _max_chunk_capacity);
142
143 if (resize(new_capacity)) {
144 log_debug(gc)("Expanded mark stack capacity from " SIZE_FORMAT " to " SIZE_FORMAT " chunks",
145 old_capacity, new_capacity);
146 } else {
147 log_warning(gc)("Failed to expand mark stack capacity from " SIZE_FORMAT " to " SIZE_FORMAT " chunks",
148 old_capacity, new_capacity);
149 }
150 }
151
152 G1CMMarkStack::~G1CMMarkStack() {
153 if (_base != NULL) {
154 MmapArrayAllocator<TaskQueueEntryChunk>::free(_base, _chunk_capacity);
155 }
156 }
157
158 void G1CMMarkStack::add_chunk_to_list(TaskQueueEntryChunk* volatile* list, TaskQueueEntryChunk* elem) {
159 elem->next = *list;
160 *list = elem;
161 }
162
163 void G1CMMarkStack::add_chunk_to_chunk_list(TaskQueueEntryChunk* elem) {
164 MutexLockerEx x(MarkStackChunkList_lock, Mutex::_no_safepoint_check_flag);
165 add_chunk_to_list(&_chunk_list, elem);
166 _chunks_in_chunk_list++;
167 }
168
169 void G1CMMarkStack::add_chunk_to_free_list(TaskQueueEntryChunk* elem) {
170 MutexLockerEx x(MarkStackFreeList_lock, Mutex::_no_safepoint_check_flag);
171 add_chunk_to_list(&_free_list, elem);
172 }
173
174 G1CMMarkStack::TaskQueueEntryChunk* G1CMMarkStack::remove_chunk_from_list(TaskQueueEntryChunk* volatile* list) {
175 TaskQueueEntryChunk* result = *list;
176 if (result != NULL) {
177 *list = (*list)->next;
178 }
179 return result;
180 }
181
182 G1CMMarkStack::TaskQueueEntryChunk* G1CMMarkStack::remove_chunk_from_chunk_list() {
183 MutexLockerEx x(MarkStackChunkList_lock, Mutex::_no_safepoint_check_flag);
184 TaskQueueEntryChunk* result = remove_chunk_from_list(&_chunk_list);
185 if (result != NULL) {
186 _chunks_in_chunk_list--;
187 }
188 return result;
189 }
190
191 G1CMMarkStack::TaskQueueEntryChunk* G1CMMarkStack::remove_chunk_from_free_list() {
192 MutexLockerEx x(MarkStackFreeList_lock, Mutex::_no_safepoint_check_flag);
193 return remove_chunk_from_list(&_free_list);
194 }
195
196 G1CMMarkStack::TaskQueueEntryChunk* G1CMMarkStack::allocate_new_chunk() {
197 // This dirty read of _hwm is okay because we only ever increase the _hwm in parallel code.
198 // Further this limits _hwm to a value of _chunk_capacity + #threads, avoiding
199 // wraparound of _hwm.
200 if (_hwm >= _chunk_capacity) {
201 return NULL;
202 }
203
204 size_t cur_idx = Atomic::add(1u, &_hwm) - 1;
205 if (cur_idx >= _chunk_capacity) {
206 return NULL;
207 }
208
209 TaskQueueEntryChunk* result = ::new (&_base[cur_idx]) TaskQueueEntryChunk;
210 result->next = NULL;
211 return result;
212 }
213
214 bool G1CMMarkStack::par_push_chunk(G1TaskQueueEntry* ptr_arr) {
215 // Get a new chunk.
216 TaskQueueEntryChunk* new_chunk = remove_chunk_from_free_list();
217
218 if (new_chunk == NULL) {
219 // Did not get a chunk from the free list. Allocate from backing memory.
220 new_chunk = allocate_new_chunk();
221
222 if (new_chunk == NULL) {
223 return false;
224 }
225 }
226
227 Copy::conjoint_memory_atomic(ptr_arr, new_chunk->data, EntriesPerChunk * sizeof(G1TaskQueueEntry));
228
229 add_chunk_to_chunk_list(new_chunk);
230
231 return true;
232 }
233
234 bool G1CMMarkStack::par_pop_chunk(G1TaskQueueEntry* ptr_arr) {
235 TaskQueueEntryChunk* cur = remove_chunk_from_chunk_list();
236
237 if (cur == NULL) {
238 return false;
239 }
240
241 Copy::conjoint_memory_atomic(cur->data, ptr_arr, EntriesPerChunk * sizeof(G1TaskQueueEntry));
242
243 add_chunk_to_free_list(cur);
244 return true;
245 }
246
247 void G1CMMarkStack::set_empty() {
248 _chunks_in_chunk_list = 0;
249 _hwm = 0;
250 _chunk_list = NULL;
251 _free_list = NULL;
252 }
253
254 G1CMRootRegions::G1CMRootRegions() :
255 _cm(NULL), _scan_in_progress(false),
256 _should_abort(false), _claimed_survivor_index(0) { }
257
258 void G1CMRootRegions::init(const G1SurvivorRegions* survivors, G1ConcurrentMark* cm) {
259 _survivors = survivors;
260 _cm = cm;
261 }
262
263 void G1CMRootRegions::prepare_for_scan() {
264 assert(!scan_in_progress(), "pre-condition");
265
266 // Currently, only survivors can be root regions.
267 _claimed_survivor_index = 0;
268 _scan_in_progress = _survivors->regions()->is_nonempty();
269 _should_abort = false;
270 }
271
272 HeapRegion* G1CMRootRegions::claim_next() {
273 if (_should_abort) {
274 // If someone has set the should_abort flag, we return NULL to
275 // force the caller to bail out of their loop.
276 return NULL;
277 }
278
279 // Currently, only survivors can be root regions.
280 const GrowableArray<HeapRegion*>* survivor_regions = _survivors->regions();
281
282 int claimed_index = Atomic::add(1, &_claimed_survivor_index) - 1;
283 if (claimed_index < survivor_regions->length()) {
284 return survivor_regions->at(claimed_index);
285 }
286 return NULL;
287 }
288
289 uint G1CMRootRegions::num_root_regions() const {
290 return (uint)_survivors->regions()->length();
291 }
292
293 void G1CMRootRegions::notify_scan_done() {
294 MutexLockerEx x(RootRegionScan_lock, Mutex::_no_safepoint_check_flag);
295 _scan_in_progress = false;
296 RootRegionScan_lock->notify_all();
297 }
298
299 void G1CMRootRegions::cancel_scan() {
300 notify_scan_done();
301 }
302
303 void G1CMRootRegions::scan_finished() {
304 assert(scan_in_progress(), "pre-condition");
305
306 // Currently, only survivors can be root regions.
307 if (!_should_abort) {
308 assert(_claimed_survivor_index >= 0, "otherwise comparison is invalid: %d", _claimed_survivor_index);
309 assert((uint)_claimed_survivor_index >= _survivors->length(),
310 "we should have claimed all survivors, claimed index = %u, length = %u",
311 (uint)_claimed_survivor_index, _survivors->length());
312 }
313
314 notify_scan_done();
315 }
316
317 bool G1CMRootRegions::wait_until_scan_finished() {
318 if (!scan_in_progress()) return false;
319
320 {
321 MutexLockerEx x(RootRegionScan_lock, Mutex::_no_safepoint_check_flag);
322 while (scan_in_progress()) {
323 RootRegionScan_lock->wait(Mutex::_no_safepoint_check_flag);
324 }
325 }
326 return true;
327 }
328
329 // Returns the maximum number of workers to be used in a concurrent
330 // phase based on the number of GC workers being used in a STW
331 // phase.
332 static uint scale_concurrent_worker_threads(uint num_gc_workers) {
333 return MAX2((num_gc_workers + 2) / 4, 1U);
334 }
335
336 G1ConcurrentMark::G1ConcurrentMark(G1CollectedHeap* g1h,
337 G1RegionToSpaceMapper* prev_bitmap_storage,
338 G1RegionToSpaceMapper* next_bitmap_storage) :
339 // _cm_thread set inside the constructor
340 _g1h(g1h),
341 _completed_initialization(false),
342
343 _cleanup_list("Concurrent Mark Cleanup List"),
344 _mark_bitmap_1(),
345 _mark_bitmap_2(),
346 _prev_mark_bitmap(&_mark_bitmap_1),
347 _next_mark_bitmap(&_mark_bitmap_2),
348
349 _heap_start(_g1h->reserved_region().start()),
350 _heap_end(_g1h->reserved_region().end()),
351
352 _root_regions(),
353
354 _global_mark_stack(),
355
356 // _finger set in set_non_marking_state
357
358 _max_num_tasks(ParallelGCThreads),
359 // _num_active_tasks set in set_non_marking_state()
360 // _tasks set inside the constructor
361
362 _task_queues(new G1CMTaskQueueSet((int) _max_num_tasks)),
363 _terminator(ParallelTaskTerminator((int) _max_num_tasks, _task_queues)),
364
365 _first_overflow_barrier_sync(),
366 _second_overflow_barrier_sync(),
367
368 _has_overflown(false),
369 _concurrent(false),
370 _has_aborted(false),
371 _restart_for_overflow(false),
372 _concurrent_marking_in_progress(false),
373 _gc_timer_cm(new (ResourceObj::C_HEAP, mtGC) ConcurrentGCTimer()),
374 _gc_tracer_cm(new (ResourceObj::C_HEAP, mtGC) G1OldTracer()),
375
376 // _verbose_level set below
377
378 _init_times(),
379 _remark_times(),
380 _remark_mark_times(),
381 _remark_weak_ref_times(),
382 _cleanup_times(),
383 _total_counting_time(0.0),
384 _total_rs_scrub_time(0.0),
385
386 _accum_task_vtime(NULL),
387
388 _concurrent_workers(NULL),
389 _num_concurrent_workers(0),
390 _max_concurrent_workers(0)
391 {
392 _mark_bitmap_1.initialize(g1h->reserved_region(), prev_bitmap_storage);
393 _mark_bitmap_2.initialize(g1h->reserved_region(), next_bitmap_storage);
394
395 // Create & start ConcurrentMark thread.
396 _cm_thread = new ConcurrentMarkThread(this);
397 if (_cm_thread->osthread() == NULL) {
398 vm_shutdown_during_initialization("Could not create ConcurrentMarkThread");
399 }
400
401 assert(CGC_lock != NULL, "CGC_lock must be initialized");
402
403 SATBMarkQueueSet& satb_qs = JavaThread::satb_mark_queue_set();
404 satb_qs.set_buffer_size(G1SATBBufferSize);
405
406 _root_regions.init(_g1h->survivor(), this);
407
408 if (FLAG_IS_DEFAULT(ConcGCThreads) || ConcGCThreads == 0) {
409 // Calculate the number of concurrent worker threads by scaling
410 // the number of parallel GC threads.
411 uint marking_thread_num = scale_concurrent_worker_threads(ParallelGCThreads);
412 FLAG_SET_ERGO(uint, ConcGCThreads, marking_thread_num);
413 }
414
415 assert(ConcGCThreads > 0, "ConcGCThreads have been set.");
416 if (ConcGCThreads > ParallelGCThreads) {
417 log_warning(gc)("More ConcGCThreads (%u) than ParallelGCThreads (%u).",
418 ConcGCThreads, ParallelGCThreads);
419 return;
420 }
421
422 log_debug(gc)("ConcGCThreads: %u", ConcGCThreads);
423 log_debug(gc)("ParallelGCThreads: %u", ParallelGCThreads);
424
425 _num_concurrent_workers = ConcGCThreads;
426 _max_concurrent_workers = _num_concurrent_workers;
427
428 _concurrent_workers = new WorkGang("G1 Conc", _max_concurrent_workers, false, true);
429 _concurrent_workers->initialize_workers();
430
431 if (FLAG_IS_DEFAULT(MarkStackSize)) {
432 size_t mark_stack_size =
433 MIN2(MarkStackSizeMax,
434 MAX2(MarkStackSize, (size_t) (_max_concurrent_workers * TASKQUEUE_SIZE)));
435 // Verify that the calculated value for MarkStackSize is in range.
436 // It would be nice to use the private utility routine from Arguments.
437 if (!(mark_stack_size >= 1 && mark_stack_size <= MarkStackSizeMax)) {
438 log_warning(gc)("Invalid value calculated for MarkStackSize (" SIZE_FORMAT "): "
439 "must be between 1 and " SIZE_FORMAT,
440 mark_stack_size, MarkStackSizeMax);
441 return;
442 }
443 FLAG_SET_ERGO(size_t, MarkStackSize, mark_stack_size);
444 } else {
445 // Verify MarkStackSize is in range.
446 if (FLAG_IS_CMDLINE(MarkStackSize)) {
447 if (FLAG_IS_DEFAULT(MarkStackSizeMax)) {
448 if (!(MarkStackSize >= 1 && MarkStackSize <= MarkStackSizeMax)) {
449 log_warning(gc)("Invalid value specified for MarkStackSize (" SIZE_FORMAT "): "
450 "must be between 1 and " SIZE_FORMAT,
451 MarkStackSize, MarkStackSizeMax);
452 return;
453 }
454 } else if (FLAG_IS_CMDLINE(MarkStackSizeMax)) {
455 if (!(MarkStackSize >= 1 && MarkStackSize <= MarkStackSizeMax)) {
456 log_warning(gc)("Invalid value specified for MarkStackSize (" SIZE_FORMAT ")"
457 " or for MarkStackSizeMax (" SIZE_FORMAT ")",
458 MarkStackSize, MarkStackSizeMax);
459 return;
460 }
461 }
462 }
463 }
464
465 if (!_global_mark_stack.initialize(MarkStackSize, MarkStackSizeMax)) {
466 vm_exit_during_initialization("Failed to allocate initial concurrent mark overflow mark stack.");
467 }
468
469 _tasks = NEW_C_HEAP_ARRAY(G1CMTask*, _max_num_tasks, mtGC);
470 _accum_task_vtime = NEW_C_HEAP_ARRAY(double, _max_num_tasks, mtGC);
471
472 // so that the assertion in MarkingTaskQueue::task_queue doesn't fail
473 _num_active_tasks = _max_num_tasks;
474
475 for (uint i = 0; i < _max_num_tasks; ++i) {
476 G1CMTaskQueue* task_queue = new G1CMTaskQueue();
477 task_queue->initialize();
478 _task_queues->register_queue(i, task_queue);
479
480 _tasks[i] = new G1CMTask(i, this, task_queue);
481
482 _accum_task_vtime[i] = 0.0;
483 }
484
485 set_non_marking_state();
486 _completed_initialization = true;
487 }
488
489 void G1ConcurrentMark::reset() {
490 // Starting values for these two. This should be called in a STW
491 // phase.
492 MemRegion reserved = _g1h->g1_reserved();
493 _heap_start = reserved.start();
494 _heap_end = reserved.end();
495
496 // Separated the asserts so that we know which one fires.
497 assert(_heap_start != NULL, "heap bounds should look ok");
498 assert(_heap_end != NULL, "heap bounds should look ok");
499 assert(_heap_start < _heap_end, "heap bounds should look ok");
500
501 // Reset all the marking data structures and any necessary flags
502 reset_marking_state();
503
504 // We reset all of them, since different phases will use
505 // different number of active threads. So, it's easiest to have all
506 // of them ready.
507 for (uint i = 0; i < _max_num_tasks; ++i) {
508 _tasks[i]->reset(_next_mark_bitmap);
509 }
510
511 // we need this to make sure that the flag is on during the evac
512 // pause with initial mark piggy-backed
513 set_concurrent_marking_in_progress();
514 }
515
516
517 void G1ConcurrentMark::reset_marking_state() {
518 _global_mark_stack.set_empty();
519
520 // Expand the marking stack, if we have to and if we can.
521 if (has_overflown()) {
522 _global_mark_stack.expand();
523 }
524
525 clear_has_overflown();
526 _finger = _heap_start;
527
528 for (uint i = 0; i < _max_num_tasks; ++i) {
529 G1CMTaskQueue* queue = _task_queues->queue(i);
530 queue->set_empty();
531 }
532 }
533
534 void G1ConcurrentMark::set_concurrency(uint active_tasks) {
535 assert(active_tasks <= _max_num_tasks, "we should not have more");
536
537 _num_active_tasks = active_tasks;
538 // Need to update the three data structures below according to the
539 // number of active threads for this phase.
540 _terminator = ParallelTaskTerminator((int) active_tasks, _task_queues);
541 _first_overflow_barrier_sync.set_n_workers((int) active_tasks);
542 _second_overflow_barrier_sync.set_n_workers((int) active_tasks);
543 }
544
545 void G1ConcurrentMark::set_concurrency_and_phase(uint active_tasks, bool concurrent) {
546 set_concurrency(active_tasks);
547
548 _concurrent = concurrent;
549 // We propagate this to all tasks, not just the active ones.
550 for (uint i = 0; i < _max_num_tasks; ++i) {
551 _tasks[i]->set_concurrent(concurrent);
552 }
553
554 if (concurrent) {
555 set_concurrent_marking_in_progress();
556 } else {
557 // We currently assume that the concurrent flag has been set to
558 // false before we start remark. At this point we should also be
559 // in a STW phase.
560 assert(!concurrent_marking_in_progress(), "invariant");
561 assert(out_of_regions(),
562 "only way to get here: _finger: " PTR_FORMAT ", _heap_end: " PTR_FORMAT,
563 p2i(_finger), p2i(_heap_end));
564 }
565 }
566
567 void G1ConcurrentMark::set_non_marking_state() {
568 // We set the global marking state to some default values when we're
569 // not doing marking.
570 reset_marking_state();
571 _num_active_tasks = 0;
572 clear_concurrent_marking_in_progress();
573 }
574
575 G1ConcurrentMark::~G1ConcurrentMark() {
576 // The G1ConcurrentMark instance is never freed.
577 ShouldNotReachHere();
578 }
579
580 class G1ClearBitMapTask : public AbstractGangTask {
581 public:
582 static size_t chunk_size() { return M; }
583
584 private:
585 // Heap region closure used for clearing the given mark bitmap.
586 class G1ClearBitmapHRClosure : public HeapRegionClosure {
587 private:
588 G1CMBitMap* _bitmap;
589 G1ConcurrentMark* _cm;
590 public:
591 G1ClearBitmapHRClosure(G1CMBitMap* bitmap, G1ConcurrentMark* cm) : HeapRegionClosure(), _cm(cm), _bitmap(bitmap) {
592 }
593
594 virtual bool do_heap_region(HeapRegion* r) {
595 size_t const chunk_size_in_words = G1ClearBitMapTask::chunk_size() / HeapWordSize;
596
597 HeapWord* cur = r->bottom();
598 HeapWord* const end = r->end();
599
600 while (cur < end) {
601 MemRegion mr(cur, MIN2(cur + chunk_size_in_words, end));
602 _bitmap->clear_range(mr);
603
604 cur += chunk_size_in_words;
605
606 // Abort iteration if after yielding the marking has been aborted.
607 if (_cm != NULL && _cm->do_yield_check() && _cm->has_aborted()) {
608 return true;
609 }
610 // Repeat the asserts from before the start of the closure. We will do them
611 // as asserts here to minimize their overhead on the product. However, we
612 // will have them as guarantees at the beginning / end of the bitmap
613 // clearing to get some checking in the product.
614 assert(_cm == NULL || _cm->cm_thread()->during_cycle(), "invariant");
615 assert(_cm == NULL || !G1CollectedHeap::heap()->collector_state()->mark_in_progress(), "invariant");
616 }
617 assert(cur == end, "Must have completed iteration over the bitmap for region %u.", r->hrm_index());
618
619 return false;
620 }
621 };
622
623 G1ClearBitmapHRClosure _cl;
624 HeapRegionClaimer _hr_claimer;
625 bool _suspendible; // If the task is suspendible, workers must join the STS.
626
627 public:
628 G1ClearBitMapTask(G1CMBitMap* bitmap, G1ConcurrentMark* cm, uint n_workers, bool suspendible) :
629 AbstractGangTask("G1 Clear Bitmap"),
630 _cl(bitmap, suspendible ? cm : NULL),
631 _hr_claimer(n_workers),
632 _suspendible(suspendible)
633 { }
634
635 void work(uint worker_id) {
636 SuspendibleThreadSetJoiner sts_join(_suspendible);
637 G1CollectedHeap::heap()->heap_region_par_iterate_from_worker_offset(&_cl, &_hr_claimer, worker_id);
638 }
639
640 bool is_complete() {
641 return _cl.is_complete();
642 }
643 };
644
645 void G1ConcurrentMark::clear_bitmap(G1CMBitMap* bitmap, WorkGang* workers, bool may_yield) {
646 assert(may_yield || SafepointSynchronize::is_at_safepoint(), "Non-yielding bitmap clear only allowed at safepoint.");
647
648 size_t const num_bytes_to_clear = (HeapRegion::GrainBytes * _g1h->num_regions()) / G1CMBitMap::heap_map_factor();
649 size_t const num_chunks = align_up(num_bytes_to_clear, G1ClearBitMapTask::chunk_size()) / G1ClearBitMapTask::chunk_size();
650
651 uint const num_workers = (uint)MIN2(num_chunks, (size_t)workers->active_workers());
652
653 G1ClearBitMapTask cl(bitmap, this, num_workers, may_yield);
654
655 log_debug(gc, ergo)("Running %s with %u workers for " SIZE_FORMAT " work units.", cl.name(), num_workers, num_chunks);
656 workers->run_task(&cl, num_workers);
657 guarantee(!may_yield || cl.is_complete(), "Must have completed iteration when not yielding.");
658 }
659
660 void G1ConcurrentMark::cleanup_for_next_mark() {
661 // Make sure that the concurrent mark thread looks to still be in
662 // the current cycle.
663 guarantee(cm_thread()->during_cycle(), "invariant");
664
665 // We are finishing up the current cycle by clearing the next
666 // marking bitmap and getting it ready for the next cycle. During
667 // this time no other cycle can start. So, let's make sure that this
668 // is the case.
669 guarantee(!_g1h->collector_state()->mark_in_progress(), "invariant");
670
671 clear_bitmap(_next_mark_bitmap, _concurrent_workers, true);
672
673 // Clear the live count data. If the marking has been aborted, the abort()
674 // call already did that.
675 if (!has_aborted()) {
676 clear_live_data(_concurrent_workers);
677 DEBUG_ONLY(verify_live_data_clear());
678 }
679
680 // Repeat the asserts from above.
681 guarantee(cm_thread()->during_cycle(), "invariant");
682 guarantee(!_g1h->collector_state()->mark_in_progress(), "invariant");
683 }
684
685 void G1ConcurrentMark::clear_prev_bitmap(WorkGang* workers) {
686 assert(SafepointSynchronize::is_at_safepoint(), "Should only clear the entire prev bitmap at a safepoint.");
687 clear_bitmap(_prev_mark_bitmap, workers, false);
688 }
689
690 class CheckBitmapClearHRClosure : public HeapRegionClosure {
691 G1CMBitMap* _bitmap;
692 bool _error;
693 public:
694 CheckBitmapClearHRClosure(G1CMBitMap* bitmap) : _bitmap(bitmap) {
695 }
696
697 virtual bool do_heap_region(HeapRegion* r) {
698 // This closure can be called concurrently to the mutator, so we must make sure
699 // that the result of the getNextMarkedWordAddress() call is compared to the
700 // value passed to it as limit to detect any found bits.
701 // end never changes in G1.
702 HeapWord* end = r->end();
703 return _bitmap->get_next_marked_addr(r->bottom(), end) != end;
704 }
705 };
706
707 bool G1ConcurrentMark::next_mark_bitmap_is_clear() {
708 CheckBitmapClearHRClosure cl(_next_mark_bitmap);
709 _g1h->heap_region_iterate(&cl);
710 return cl.is_complete();
711 }
712
713 class NoteStartOfMarkHRClosure: public HeapRegionClosure {
714 public:
715 bool do_heap_region(HeapRegion* r) {
716 r->note_start_of_marking();
717 return false;
718 }
719 };
720
721 void G1ConcurrentMark::checkpoint_roots_initial_pre() {
722 G1CollectedHeap* g1h = G1CollectedHeap::heap();
723
724 _has_aborted = false;
725
726 // Initialize marking structures. This has to be done in a STW phase.
727 reset();
728
729 // For each region note start of marking.
730 NoteStartOfMarkHRClosure startcl;
731 g1h->heap_region_iterate(&startcl);
732 }
733
734
735 void G1ConcurrentMark::checkpoint_roots_initial_post() {
736 G1CollectedHeap* g1h = G1CollectedHeap::heap();
737
738 // Start Concurrent Marking weak-reference discovery.
739 ReferenceProcessor* rp = g1h->ref_processor_cm();
740 // enable ("weak") refs discovery
741 rp->enable_discovery();
742 rp->setup_policy(false); // snapshot the soft ref policy to be used in this cycle
743
744 SATBMarkQueueSet& satb_mq_set = JavaThread::satb_mark_queue_set();
745 // This is the start of the marking cycle, we're expected all
746 // threads to have SATB queues with active set to false.
747 satb_mq_set.set_active_all_threads(true, /* new active value */
748 false /* expected_active */);
749
750 _root_regions.prepare_for_scan();
751
752 // update_g1_committed() will be called at the end of an evac pause
753 // when marking is on. So, it's also called at the end of the
754 // initial-mark pause to update the heap end, if the heap expands
755 // during it. No need to call it here.
756 }
757
758 /*
759 * Notice that in the next two methods, we actually leave the STS
760 * during the barrier sync and join it immediately afterwards. If we
761 * do not do this, the following deadlock can occur: one thread could
762 * be in the barrier sync code, waiting for the other thread to also
763 * sync up, whereas another one could be trying to yield, while also
764 * waiting for the other threads to sync up too.
765 *
766 * Note, however, that this code is also used during remark and in
767 * this case we should not attempt to leave / enter the STS, otherwise
768 * we'll either hit an assert (debug / fastdebug) or deadlock
769 * (product). So we should only leave / enter the STS if we are
770 * operating concurrently.
771 *
772 * Because the thread that does the sync barrier has left the STS, it
773 * is possible to be suspended for a Full GC or an evacuation pause
774 * could occur. This is actually safe, since the entering the sync
775 * barrier is one of the last things do_marking_step() does, and it
776 * doesn't manipulate any data structures afterwards.
777 */
778
779 void G1ConcurrentMark::enter_first_sync_barrier(uint worker_id) {
780 bool barrier_aborted;
781 {
782 SuspendibleThreadSetLeaver sts_leave(concurrent());
783 barrier_aborted = !_first_overflow_barrier_sync.enter();
784 }
785
786 // at this point everyone should have synced up and not be doing any
787 // more work
788
789 if (barrier_aborted) {
790 // If the barrier aborted we ignore the overflow condition and
791 // just abort the whole marking phase as quickly as possible.
792 return;
793 }
794
795 // If we're executing the concurrent phase of marking, reset the marking
796 // state; otherwise the marking state is reset after reference processing,
797 // during the remark pause.
798 // If we reset here as a result of an overflow during the remark we will
799 // see assertion failures from any subsequent set_concurrency_and_phase()
800 // calls.
801 if (concurrent()) {
802 // let the task associated with with worker 0 do this
803 if (worker_id == 0) {
804 // task 0 is responsible for clearing the global data structures
805 // We should be here because of an overflow. During STW we should
806 // not clear the overflow flag since we rely on it being true when
807 // we exit this method to abort the pause and restart concurrent
808 // marking.
809 reset_marking_state();
810
811 log_info(gc, marking)("Concurrent Mark reset for overflow");
812 }
813 }
814
815 // after this, each task should reset its own data structures then
816 // then go into the second barrier
817 }
818
819 void G1ConcurrentMark::enter_second_sync_barrier(uint worker_id) {
820 SuspendibleThreadSetLeaver sts_leave(concurrent());
821 _second_overflow_barrier_sync.enter();
822
823 // at this point everything should be re-initialized and ready to go
824 }
825
826 class G1CMConcurrentMarkingTask: public AbstractGangTask {
827 private:
828 G1ConcurrentMark* _cm;
829 ConcurrentMarkThread* _cmt;
830
831 public:
832 void work(uint worker_id) {
833 assert(Thread::current()->is_ConcurrentGC_thread(), "Not a concurrent GC thread");
834 ResourceMark rm;
835
836 double start_vtime = os::elapsedVTime();
837
838 {
839 SuspendibleThreadSetJoiner sts_join;
840
841 assert(worker_id < _cm->active_tasks(), "invariant");
842
843 G1CMTask* task = _cm->task(worker_id);
844 task->record_start_time();
845 if (!_cm->has_aborted()) {
846 do {
847 task->do_marking_step(G1ConcMarkStepDurationMillis,
848 true /* do_termination */,
849 false /* is_serial*/);
850
851 _cm->do_yield_check();
852 } while (!_cm->has_aborted() && task->has_aborted());
853 }
854 task->record_end_time();
855 guarantee(!task->has_aborted() || _cm->has_aborted(), "invariant");
856 }
857
858 double end_vtime = os::elapsedVTime();
859 _cm->update_accum_task_vtime(worker_id, end_vtime - start_vtime);
860 }
861
862 G1CMConcurrentMarkingTask(G1ConcurrentMark* cm,
863 ConcurrentMarkThread* cmt) :
864 AbstractGangTask("Concurrent Mark"), _cm(cm), _cmt(cmt) { }
865
866 ~G1CMConcurrentMarkingTask() { }
867 };
868
869 uint G1ConcurrentMark::calc_active_marking_workers() {
870 uint result = 0;
871 if (!UseDynamicNumberOfGCThreads ||
872 (!FLAG_IS_DEFAULT(ConcGCThreads) &&
873 !ForceDynamicNumberOfGCThreads)) {
874 result = _max_concurrent_workers;
875 } else {
876 result =
877 AdaptiveSizePolicy::calc_default_active_workers(_max_concurrent_workers,
878 1, /* Minimum workers */
879 _num_concurrent_workers,
880 Threads::number_of_non_daemon_threads());
881 // Don't scale the result down by scale_concurrent_workers() because
882 // that scaling has already gone into "_max_concurrent_workers".
883 }
884 assert(result > 0 && result <= _max_concurrent_workers,
885 "Calculated number of marking workers must be larger than zero and at most the maximum %u, but is %u",
886 _max_concurrent_workers, result);
887 return result;
888 }
889
890 void G1ConcurrentMark::scan_root_region(HeapRegion* hr) {
891 // Currently, only survivors can be root regions.
892 assert(hr->next_top_at_mark_start() == hr->bottom(), "invariant");
893 G1RootRegionScanClosure cl(_g1h, this);
894
895 const uintx interval = PrefetchScanIntervalInBytes;
896 HeapWord* curr = hr->bottom();
897 const HeapWord* end = hr->top();
898 while (curr < end) {
899 Prefetch::read(curr, interval);
900 oop obj = oop(curr);
901 int size = obj->oop_iterate_size(&cl);
902 assert(size == obj->size(), "sanity");
903 curr += size;
904 }
905 }
906
907 class G1CMRootRegionScanTask : public AbstractGangTask {
908 private:
909 G1ConcurrentMark* _cm;
910
911 public:
912 G1CMRootRegionScanTask(G1ConcurrentMark* cm) :
913 AbstractGangTask("G1 Root Region Scan"), _cm(cm) { }
914
915 void work(uint worker_id) {
916 assert(Thread::current()->is_ConcurrentGC_thread(),
917 "this should only be done by a conc GC thread");
918
919 G1CMRootRegions* root_regions = _cm->root_regions();
920 HeapRegion* hr = root_regions->claim_next();
921 while (hr != NULL) {
922 _cm->scan_root_region(hr);
923 hr = root_regions->claim_next();
924 }
925 }
926 };
927
928 void G1ConcurrentMark::scan_root_regions() {
929 // scan_in_progress() will have been set to true only if there was
930 // at least one root region to scan. So, if it's false, we
931 // should not attempt to do any further work.
932 if (root_regions()->scan_in_progress()) {
933 assert(!has_aborted(), "Aborting before root region scanning is finished not supported.");
934
935 _num_concurrent_workers = MIN2(calc_active_marking_workers(),
936 // We distribute work on a per-region basis, so starting
937 // more threads than that is useless.
938 root_regions()->num_root_regions());
939 assert(_num_concurrent_workers <= _max_concurrent_workers,
940 "Maximum number of marking threads exceeded");
941
942 G1CMRootRegionScanTask task(this);
943 log_debug(gc, ergo)("Running %s using %u workers for %u work units.",
944 task.name(), _num_concurrent_workers, root_regions()->num_root_regions());
945 _concurrent_workers->run_task(&task, _num_concurrent_workers);
946
947 // It's possible that has_aborted() is true here without actually
948 // aborting the survivor scan earlier. This is OK as it's
949 // mainly used for sanity checking.
950 root_regions()->scan_finished();
951 }
952 }
953
954 void G1ConcurrentMark::concurrent_cycle_start() {
955 _gc_timer_cm->register_gc_start();
956
957 _gc_tracer_cm->report_gc_start(GCCause::_no_gc /* first parameter is not used */, _gc_timer_cm->gc_start());
958
959 _g1h->trace_heap_before_gc(_gc_tracer_cm);
960 }
961
962 void G1ConcurrentMark::concurrent_cycle_end() {
963 _g1h->trace_heap_after_gc(_gc_tracer_cm);
964
965 if (has_aborted()) {
966 _gc_tracer_cm->report_concurrent_mode_failure();
967 }
968
969 _gc_timer_cm->register_gc_end();
970
971 _gc_tracer_cm->report_gc_end(_gc_timer_cm->gc_end(), _gc_timer_cm->time_partitions());
972 }
973
974 void G1ConcurrentMark::mark_from_roots() {
975 // we might be tempted to assert that:
976 // assert(asynch == !SafepointSynchronize::is_at_safepoint(),
977 // "inconsistent argument?");
978 // However that wouldn't be right, because it's possible that
979 // a safepoint is indeed in progress as a younger generation
980 // stop-the-world GC happens even as we mark in this generation.
981
982 _restart_for_overflow = false;
983
984 _num_concurrent_workers = calc_active_marking_workers();
985
986 uint active_workers = MAX2(1U, _num_concurrent_workers);
987
988 // Setting active workers is not guaranteed since fewer
989 // worker threads may currently exist and more may not be
990 // available.
991 active_workers = _concurrent_workers->update_active_workers(active_workers);
992 log_info(gc, task)("Using %u workers of %u for marking", active_workers, _concurrent_workers->total_workers());
993
994 // Parallel task terminator is set in "set_concurrency_and_phase()"
995 set_concurrency_and_phase(active_workers, true /* concurrent */);
996
997 G1CMConcurrentMarkingTask marking_task(this, cm_thread());
998 _concurrent_workers->run_task(&marking_task);
999 print_stats();
1000 }
1001
1002 void G1ConcurrentMark::checkpoint_roots_final(bool clear_all_soft_refs) {
1003 // world is stopped at this checkpoint
1004 assert(SafepointSynchronize::is_at_safepoint(),
1005 "world should be stopped");
1006
1007 G1CollectedHeap* g1h = G1CollectedHeap::heap();
1008
1009 // If a full collection has happened, we shouldn't do this.
1010 if (has_aborted()) {
1011 g1h->collector_state()->set_mark_in_progress(false); // So bitmap clearing isn't confused
1012 return;
1013 }
1014
1015 SvcGCMarker sgcm(SvcGCMarker::OTHER);
1016
1017 if (VerifyDuringGC) {
1018 g1h->verifier()->verify(G1HeapVerifier::G1VerifyRemark, VerifyOption_G1UsePrevMarking, "During GC (before)");
1019 }
1020 g1h->verifier()->check_bitmaps("Remark Start");
1021
1022 G1Policy* g1p = g1h->g1_policy();
1023 g1p->record_concurrent_mark_remark_start();
1024
1025 double start = os::elapsedTime();
1026
1027 checkpoint_roots_final_work();
1028
1029 double mark_work_end = os::elapsedTime();
1030
1031 weak_refs_work(clear_all_soft_refs);
1032
1033 if (has_overflown()) {
1034 // We overflowed. Restart concurrent marking.
1035 _restart_for_overflow = true;
1036
1037 // Verify the heap w.r.t. the previous marking bitmap.
1038 if (VerifyDuringGC) {
1039 g1h->verifier()->verify(G1HeapVerifier::G1VerifyRemark, VerifyOption_G1UsePrevMarking, "During GC (overflow)");
1040 }
1041
1042 // Clear the marking state because we will be restarting
1043 // marking due to overflowing the global mark stack.
1044 reset_marking_state();
1045 } else {
1046 SATBMarkQueueSet& satb_mq_set = JavaThread::satb_mark_queue_set();
1047 // We're done with marking.
1048 // This is the end of the marking cycle, we're expected all
1049 // threads to have SATB queues with active set to true.
1050 satb_mq_set.set_active_all_threads(false, /* new active value */
1051 true /* expected_active */);
1052
1053 if (VerifyDuringGC) {
1054 g1h->verifier()->verify(G1HeapVerifier::G1VerifyRemark, VerifyOption_G1UseNextMarking, "During GC (after)");
1055 }
1056 g1h->verifier()->check_bitmaps("Remark End");
1057 assert(!restart_for_overflow(), "sanity");
1058 // Completely reset the marking state since marking completed
1059 set_non_marking_state();
1060 }
1061
1062 // Statistics
1063 double now = os::elapsedTime();
1064 _remark_mark_times.add((mark_work_end - start) * 1000.0);
1065 _remark_weak_ref_times.add((now - mark_work_end) * 1000.0);
1066 _remark_times.add((now - start) * 1000.0);
1067
1068 g1p->record_concurrent_mark_remark_end();
1069
1070 G1CMIsAliveClosure is_alive(g1h);
1071 _gc_tracer_cm->report_object_count_after_gc(&is_alive);
1072 }
1073
1074 class G1NoteEndOfConcMarkClosure : public HeapRegionClosure {
1075 G1CollectedHeap* _g1;
1076 size_t _freed_bytes;
1077 FreeRegionList* _local_cleanup_list;
1078 uint _old_regions_removed;
1079 uint _humongous_regions_removed;
1080 HRRSCleanupTask* _hrrs_cleanup_task;
1081
1082 public:
1083 G1NoteEndOfConcMarkClosure(G1CollectedHeap* g1,
1084 FreeRegionList* local_cleanup_list,
1085 HRRSCleanupTask* hrrs_cleanup_task) :
1086 _g1(g1),
1087 _freed_bytes(0),
1088 _local_cleanup_list(local_cleanup_list),
1089 _old_regions_removed(0),
1090 _humongous_regions_removed(0),
1091 _hrrs_cleanup_task(hrrs_cleanup_task) { }
1092
1093 size_t freed_bytes() { return _freed_bytes; }
1094 const uint old_regions_removed() { return _old_regions_removed; }
1095 const uint humongous_regions_removed() { return _humongous_regions_removed; }
1096
1097 bool do_heap_region(HeapRegion *hr) {
1098 _g1->reset_gc_time_stamps(hr);
1099 hr->note_end_of_marking();
1100
1101 if (hr->used() > 0 && hr->max_live_bytes() == 0 && !hr->is_young() && !hr->is_archive()) {
1102 _freed_bytes += hr->used();
1103 hr->set_containing_set(NULL);
1104 if (hr->is_humongous()) {
1105 _humongous_regions_removed++;
1106 _g1->free_humongous_region(hr, _local_cleanup_list, true /* skip_remset */);
1107 } else {
1108 _old_regions_removed++;
1109 _g1->free_region(hr, _local_cleanup_list, true /* skip_remset */);
1110 }
1111 } else {
1112 hr->rem_set()->do_cleanup_work(_hrrs_cleanup_task);
1113 }
1114
1115 return false;
1116 }
1117 };
1118
1119 class G1ParNoteEndTask: public AbstractGangTask {
1120 friend class G1NoteEndOfConcMarkClosure;
1121
1122 protected:
1123 G1CollectedHeap* _g1h;
1124 FreeRegionList* _cleanup_list;
1125 HeapRegionClaimer _hrclaimer;
1126
1127 public:
1128 G1ParNoteEndTask(G1CollectedHeap* g1h, FreeRegionList* cleanup_list, uint n_workers) :
1129 AbstractGangTask("G1 note end"), _g1h(g1h), _cleanup_list(cleanup_list), _hrclaimer(n_workers) {
1130 }
1131
1132 void work(uint worker_id) {
1133 FreeRegionList local_cleanup_list("Local Cleanup List");
1134 HRRSCleanupTask hrrs_cleanup_task;
1135 G1NoteEndOfConcMarkClosure g1_note_end(_g1h, &local_cleanup_list,
1136 &hrrs_cleanup_task);
1137 _g1h->heap_region_par_iterate_from_worker_offset(&g1_note_end, &_hrclaimer, worker_id);
1138 assert(g1_note_end.is_complete(), "Shouldn't have yielded!");
1139
1140 // Now update the lists
1141 _g1h->remove_from_old_sets(g1_note_end.old_regions_removed(), g1_note_end.humongous_regions_removed());
1142 {
1143 MutexLockerEx x(ParGCRareEvent_lock, Mutex::_no_safepoint_check_flag);
1144 _g1h->decrement_summary_bytes(g1_note_end.freed_bytes());
1145
1146 // If we iterate over the global cleanup list at the end of
1147 // cleanup to do this printing we will not guarantee to only
1148 // generate output for the newly-reclaimed regions (the list
1149 // might not be empty at the beginning of cleanup; we might
1150 // still be working on its previous contents). So we do the
1151 // printing here, before we append the new regions to the global
1152 // cleanup list.
1153
1154 G1HRPrinter* hr_printer = _g1h->hr_printer();
1155 if (hr_printer->is_active()) {
1156 FreeRegionListIterator iter(&local_cleanup_list);
1157 while (iter.more_available()) {
1158 HeapRegion* hr = iter.get_next();
1159 hr_printer->cleanup(hr);
1160 }
1161 }
1162
1163 _cleanup_list->add_ordered(&local_cleanup_list);
1164 assert(local_cleanup_list.is_empty(), "post-condition");
1165
1166 HeapRegionRemSet::finish_cleanup_task(&hrrs_cleanup_task);
1167 }
1168 }
1169 };
1170
1171 void G1ConcurrentMark::cleanup() {
1172 // world is stopped at this checkpoint
1173 assert(SafepointSynchronize::is_at_safepoint(),
1174 "world should be stopped");
1175 G1CollectedHeap* g1h = G1CollectedHeap::heap();
1176
1177 // If a full collection has happened, we shouldn't do this.
1178 if (has_aborted()) {
1179 g1h->collector_state()->set_mark_in_progress(false); // So bitmap clearing isn't confused
1180 return;
1181 }
1182
1183 g1h->verifier()->verify_region_sets_optional();
1184
1185 if (VerifyDuringGC) {
1186 g1h->verifier()->verify(G1HeapVerifier::G1VerifyCleanup, VerifyOption_G1UsePrevMarking, "During GC (before)");
1187 }
1188 g1h->verifier()->check_bitmaps("Cleanup Start");
1189
1190 G1Policy* g1p = g1h->g1_policy();
1191 g1p->record_concurrent_mark_cleanup_start();
1192
1193 double start = os::elapsedTime();
1194
1195 HeapRegionRemSet::reset_for_cleanup_tasks();
1196
1197 {
1198 GCTraceTime(Debug, gc)("Finalize Live Data");
1199 finalize_live_data();
1200 }
1201
1202 if (VerifyDuringGC) {
1203 GCTraceTime(Debug, gc)("Verify Live Data");
1204 verify_live_data();
1205 }
1206
1207 g1h->collector_state()->set_mark_in_progress(false);
1208
1209 double count_end = os::elapsedTime();
1210 double this_final_counting_time = (count_end - start);
1211 _total_counting_time += this_final_counting_time;
1212
1213 if (log_is_enabled(Trace, gc, liveness)) {
1214 G1PrintRegionLivenessInfoClosure cl("Post-Marking");
1215 _g1h->heap_region_iterate(&cl);
1216 }
1217
1218 // Install newly created mark bitMap as "prev".
1219 swap_mark_bitmaps();
1220
1221 g1h->reset_gc_time_stamp();
1222
1223 uint n_workers = _g1h->workers()->active_workers();
1224
1225 // Note end of marking in all heap regions.
1226 G1ParNoteEndTask g1_par_note_end_task(g1h, &_cleanup_list, n_workers);
1227 g1h->workers()->run_task(&g1_par_note_end_task);
1228 g1h->check_gc_time_stamps();
1229
1230 if (!cleanup_list_is_empty()) {
1231 // The cleanup list is not empty, so we'll have to process it
1232 // concurrently. Notify anyone else that might be wanting free
1233 // regions that there will be more free regions coming soon.
1234 g1h->set_free_regions_coming();
1235 }
1236
1237 // call below, since it affects the metric by which we sort the heap
1238 // regions.
1239 if (G1ScrubRemSets) {
1240 double rs_scrub_start = os::elapsedTime();
1241 g1h->scrub_rem_set();
1242 _total_rs_scrub_time += (os::elapsedTime() - rs_scrub_start);
1243 }
1244
1245 // this will also free any regions totally full of garbage objects,
1246 // and sort the regions.
1247 g1h->g1_policy()->record_concurrent_mark_cleanup_end();
1248
1249 // Statistics.
1250 double end = os::elapsedTime();
1251 _cleanup_times.add((end - start) * 1000.0);
1252
1253 // Clean up will have freed any regions completely full of garbage.
1254 // Update the soft reference policy with the new heap occupancy.
1255 Universe::update_heap_info_at_gc();
1256
1257 if (VerifyDuringGC) {
1258 g1h->verifier()->verify(G1HeapVerifier::G1VerifyCleanup, VerifyOption_G1UsePrevMarking, "During GC (after)");
1259 }
1260
1261 g1h->verifier()->check_bitmaps("Cleanup End");
1262
1263 g1h->verifier()->verify_region_sets_optional();
1264
1265 // We need to make this be a "collection" so any collection pause that
1266 // races with it goes around and waits for completeCleanup to finish.
1267 g1h->increment_total_collections();
1268
1269 // Clean out dead classes and update Metaspace sizes.
1270 if (ClassUnloadingWithConcurrentMark) {
1271 ClassLoaderDataGraph::purge();
1272 }
1273 MetaspaceGC::compute_new_size();
1274
1275 // We reclaimed old regions so we should calculate the sizes to make
1276 // sure we update the old gen/space data.
1277 g1h->g1mm()->update_sizes();
1278 g1h->allocation_context_stats().update_after_mark();
1279 }
1280
1281 void G1ConcurrentMark::complete_cleanup() {
1282 if (has_aborted()) return;
1283
1284 G1CollectedHeap* g1h = G1CollectedHeap::heap();
1285
1286 _cleanup_list.verify_optional();
1287 FreeRegionList tmp_free_list("Tmp Free List");
1288
1289 log_develop_trace(gc, freelist)("G1ConcRegionFreeing [complete cleanup] : "
1290 "cleanup list has %u entries",
1291 _cleanup_list.length());
1292
1293 // No one else should be accessing the _cleanup_list at this point,
1294 // so it is not necessary to take any locks
1295 while (!_cleanup_list.is_empty()) {
1296 HeapRegion* hr = _cleanup_list.remove_region(true /* from_head */);
1297 assert(hr != NULL, "Got NULL from a non-empty list");
1298 hr->par_clear();
1299 tmp_free_list.add_ordered(hr);
1300
1301 // Instead of adding one region at a time to the secondary_free_list,
1302 // we accumulate them in the local list and move them a few at a
1303 // time. This also cuts down on the number of notify_all() calls
1304 // we do during this process. We'll also append the local list when
1305 // _cleanup_list is empty (which means we just removed the last
1306 // region from the _cleanup_list).
1307 if ((tmp_free_list.length() % G1SecondaryFreeListAppendLength == 0) ||
1308 _cleanup_list.is_empty()) {
1309 log_develop_trace(gc, freelist)("G1ConcRegionFreeing [complete cleanup] : "
1310 "appending %u entries to the secondary_free_list, "
1311 "cleanup list still has %u entries",
1312 tmp_free_list.length(),
1313 _cleanup_list.length());
1314
1315 {
1316 MutexLockerEx x(SecondaryFreeList_lock, Mutex::_no_safepoint_check_flag);
1317 g1h->secondary_free_list_add(&tmp_free_list);
1318 SecondaryFreeList_lock->notify_all();
1319 }
1320 #ifndef PRODUCT
1321 if (G1StressConcRegionFreeing) {
1322 for (uintx i = 0; i < G1StressConcRegionFreeingDelayMillis; ++i) {
1323 os::sleep(Thread::current(), (jlong) 1, false);
1324 }
1325 }
1326 #endif
1327 }
1328 }
1329 assert(tmp_free_list.is_empty(), "post-condition");
1330 }
1331
1332 // Supporting Object and Oop closures for reference discovery
1333 // and processing in during marking
1334
1335 bool G1CMIsAliveClosure::do_object_b(oop obj) {
1336 HeapWord* addr = (HeapWord*)obj;
1337 return addr != NULL &&
1338 (!_g1->is_in_g1_reserved(addr) || !_g1->is_obj_ill(obj));
1339 }
1340
1341 // 'Keep Alive' oop closure used by both serial parallel reference processing.
1342 // Uses the G1CMTask associated with a worker thread (for serial reference
1343 // processing the G1CMTask for worker 0 is used) to preserve (mark) and
1344 // trace referent objects.
1345 //
1346 // Using the G1CMTask and embedded local queues avoids having the worker
1347 // threads operating on the global mark stack. This reduces the risk
1348 // of overflowing the stack - which we would rather avoid at this late
1349 // state. Also using the tasks' local queues removes the potential
1350 // of the workers interfering with each other that could occur if
1351 // operating on the global stack.
1352
1353 class G1CMKeepAliveAndDrainClosure: public OopClosure {
1354 G1ConcurrentMark* _cm;
1355 G1CMTask* _task;
1356 int _ref_counter_limit;
1357 int _ref_counter;
1358 bool _is_serial;
1359 public:
1360 G1CMKeepAliveAndDrainClosure(G1ConcurrentMark* cm, G1CMTask* task, bool is_serial) :
1361 _cm(cm), _task(task), _is_serial(is_serial),
1362 _ref_counter_limit(G1RefProcDrainInterval) {
1363 assert(_ref_counter_limit > 0, "sanity");
1364 assert(!_is_serial || _task->worker_id() == 0, "only task 0 for serial code");
1365 _ref_counter = _ref_counter_limit;
1366 }
1367
1368 virtual void do_oop(narrowOop* p) { do_oop_work(p); }
1369 virtual void do_oop( oop* p) { do_oop_work(p); }
1370
1371 template <class T> void do_oop_work(T* p) {
1372 if (!_cm->has_overflown()) {
1373 oop obj = oopDesc::load_decode_heap_oop(p);
1374 _task->deal_with_reference(obj);
1375 _ref_counter--;
1376
1377 if (_ref_counter == 0) {
1378 // We have dealt with _ref_counter_limit references, pushing them
1379 // and objects reachable from them on to the local stack (and
1380 // possibly the global stack). Call G1CMTask::do_marking_step() to
1381 // process these entries.
1382 //
1383 // We call G1CMTask::do_marking_step() in a loop, which we'll exit if
1384 // there's nothing more to do (i.e. we're done with the entries that
1385 // were pushed as a result of the G1CMTask::deal_with_reference() calls
1386 // above) or we overflow.
1387 //
1388 // Note: G1CMTask::do_marking_step() can set the G1CMTask::has_aborted()
1389 // flag while there may still be some work to do. (See the comment at
1390 // the beginning of G1CMTask::do_marking_step() for those conditions -
1391 // one of which is reaching the specified time target.) It is only
1392 // when G1CMTask::do_marking_step() returns without setting the
1393 // has_aborted() flag that the marking step has completed.
1394 do {
1395 double mark_step_duration_ms = G1ConcMarkStepDurationMillis;
1396 _task->do_marking_step(mark_step_duration_ms,
1397 false /* do_termination */,
1398 _is_serial);
1399 } while (_task->has_aborted() && !_cm->has_overflown());
1400 _ref_counter = _ref_counter_limit;
1401 }
1402 }
1403 }
1404 };
1405
1406 // 'Drain' oop closure used by both serial and parallel reference processing.
1407 // Uses the G1CMTask associated with a given worker thread (for serial
1408 // reference processing the G1CMtask for worker 0 is used). Calls the
1409 // do_marking_step routine, with an unbelievably large timeout value,
1410 // to drain the marking data structures of the remaining entries
1411 // added by the 'keep alive' oop closure above.
1412
1413 class G1CMDrainMarkingStackClosure: public VoidClosure {
1414 G1ConcurrentMark* _cm;
1415 G1CMTask* _task;
1416 bool _is_serial;
1417 public:
1418 G1CMDrainMarkingStackClosure(G1ConcurrentMark* cm, G1CMTask* task, bool is_serial) :
1419 _cm(cm), _task(task), _is_serial(is_serial) {
1420 assert(!_is_serial || _task->worker_id() == 0, "only task 0 for serial code");
1421 }
1422
1423 void do_void() {
1424 do {
1425 // We call G1CMTask::do_marking_step() to completely drain the local
1426 // and global marking stacks of entries pushed by the 'keep alive'
1427 // oop closure (an instance of G1CMKeepAliveAndDrainClosure above).
1428 //
1429 // G1CMTask::do_marking_step() is called in a loop, which we'll exit
1430 // if there's nothing more to do (i.e. we've completely drained the
1431 // entries that were pushed as a a result of applying the 'keep alive'
1432 // closure to the entries on the discovered ref lists) or we overflow
1433 // the global marking stack.
1434 //
1435 // Note: G1CMTask::do_marking_step() can set the G1CMTask::has_aborted()
1436 // flag while there may still be some work to do. (See the comment at
1437 // the beginning of G1CMTask::do_marking_step() for those conditions -
1438 // one of which is reaching the specified time target.) It is only
1439 // when G1CMTask::do_marking_step() returns without setting the
1440 // has_aborted() flag that the marking step has completed.
1441
1442 _task->do_marking_step(1000000000.0 /* something very large */,
1443 true /* do_termination */,
1444 _is_serial);
1445 } while (_task->has_aborted() && !_cm->has_overflown());
1446 }
1447 };
1448
1449 // Implementation of AbstractRefProcTaskExecutor for parallel
1450 // reference processing at the end of G1 concurrent marking
1451
1452 class G1CMRefProcTaskExecutor: public AbstractRefProcTaskExecutor {
1453 private:
1454 G1CollectedHeap* _g1h;
1455 G1ConcurrentMark* _cm;
1456 WorkGang* _workers;
1457 uint _active_workers;
1458
1459 public:
1460 G1CMRefProcTaskExecutor(G1CollectedHeap* g1h,
1461 G1ConcurrentMark* cm,
1462 WorkGang* workers,
1463 uint n_workers) :
1464 _g1h(g1h), _cm(cm),
1465 _workers(workers), _active_workers(n_workers) { }
1466
1467 // Executes the given task using concurrent marking worker threads.
1468 virtual void execute(ProcessTask& task);
1469 virtual void execute(EnqueueTask& task);
1470 };
1471
1472 class G1CMRefProcTaskProxy: public AbstractGangTask {
1473 typedef AbstractRefProcTaskExecutor::ProcessTask ProcessTask;
1474 ProcessTask& _proc_task;
1475 G1CollectedHeap* _g1h;
1476 G1ConcurrentMark* _cm;
1477
1478 public:
1479 G1CMRefProcTaskProxy(ProcessTask& proc_task,
1480 G1CollectedHeap* g1h,
1481 G1ConcurrentMark* cm) :
1482 AbstractGangTask("Process reference objects in parallel"),
1483 _proc_task(proc_task), _g1h(g1h), _cm(cm) {
1484 ReferenceProcessor* rp = _g1h->ref_processor_cm();
1485 assert(rp->processing_is_mt(), "shouldn't be here otherwise");
1486 }
1487
1488 virtual void work(uint worker_id) {
1489 ResourceMark rm;
1490 HandleMark hm;
1491 G1CMTask* task = _cm->task(worker_id);
1492 G1CMIsAliveClosure g1_is_alive(_g1h);
1493 G1CMKeepAliveAndDrainClosure g1_par_keep_alive(_cm, task, false /* is_serial */);
1494 G1CMDrainMarkingStackClosure g1_par_drain(_cm, task, false /* is_serial */);
1495
1496 _proc_task.work(worker_id, g1_is_alive, g1_par_keep_alive, g1_par_drain);
1497 }
1498 };
1499
1500 void G1CMRefProcTaskExecutor::execute(ProcessTask& proc_task) {
1501 assert(_workers != NULL, "Need parallel worker threads.");
1502 assert(_g1h->ref_processor_cm()->processing_is_mt(), "processing is not MT");
1503
1504 G1CMRefProcTaskProxy proc_task_proxy(proc_task, _g1h, _cm);
1505
1506 // We need to reset the concurrency level before each
1507 // proxy task execution, so that the termination protocol
1508 // and overflow handling in G1CMTask::do_marking_step() knows
1509 // how many workers to wait for.
1510 _cm->set_concurrency(_active_workers);
1511 _workers->run_task(&proc_task_proxy);
1512 }
1513
1514 class G1CMRefEnqueueTaskProxy: public AbstractGangTask {
1515 typedef AbstractRefProcTaskExecutor::EnqueueTask EnqueueTask;
1516 EnqueueTask& _enq_task;
1517
1518 public:
1519 G1CMRefEnqueueTaskProxy(EnqueueTask& enq_task) :
1520 AbstractGangTask("Enqueue reference objects in parallel"),
1521 _enq_task(enq_task) { }
1522
1523 virtual void work(uint worker_id) {
1524 _enq_task.work(worker_id);
1525 }
1526 };
1527
1528 void G1CMRefProcTaskExecutor::execute(EnqueueTask& enq_task) {
1529 assert(_workers != NULL, "Need parallel worker threads.");
1530 assert(_g1h->ref_processor_cm()->processing_is_mt(), "processing is not MT");
1531
1532 G1CMRefEnqueueTaskProxy enq_task_proxy(enq_task);
1533
1534 // Not strictly necessary but...
1535 //
1536 // We need to reset the concurrency level before each
1537 // proxy task execution, so that the termination protocol
1538 // and overflow handling in G1CMTask::do_marking_step() knows
1539 // how many workers to wait for.
1540 _cm->set_concurrency(_active_workers);
1541 _workers->run_task(&enq_task_proxy);
1542 }
1543
1544 void G1ConcurrentMark::weak_refs_work(bool clear_all_soft_refs) {
1545 if (has_overflown()) {
1546 // Skip processing the discovered references if we have
1547 // overflown the global marking stack. Reference objects
1548 // only get discovered once so it is OK to not
1549 // de-populate the discovered reference lists. We could have,
1550 // but the only benefit would be that, when marking restarts,
1551 // less reference objects are discovered.
1552 return;
1553 }
1554
1555 ResourceMark rm;
1556 HandleMark hm;
1557
1558 G1CollectedHeap* g1h = G1CollectedHeap::heap();
1559
1560 // Is alive closure.
1561 G1CMIsAliveClosure g1_is_alive(g1h);
1562
1563 // Inner scope to exclude the cleaning of the string and symbol
1564 // tables from the displayed time.
1565 {
1566 GCTraceTime(Debug, gc, phases) trace("Reference Processing", _gc_timer_cm);
1567
1568 ReferenceProcessor* rp = g1h->ref_processor_cm();
1569
1570 // See the comment in G1CollectedHeap::ref_processing_init()
1571 // about how reference processing currently works in G1.
1572
1573 // Set the soft reference policy
1574 rp->setup_policy(clear_all_soft_refs);
1575 assert(_global_mark_stack.is_empty(), "mark stack should be empty");
1576
1577 // Instances of the 'Keep Alive' and 'Complete GC' closures used
1578 // in serial reference processing. Note these closures are also
1579 // used for serially processing (by the the current thread) the
1580 // JNI references during parallel reference processing.
1581 //
1582 // These closures do not need to synchronize with the worker
1583 // threads involved in parallel reference processing as these
1584 // instances are executed serially by the current thread (e.g.
1585 // reference processing is not multi-threaded and is thus
1586 // performed by the current thread instead of a gang worker).
1587 //
1588 // The gang tasks involved in parallel reference processing create
1589 // their own instances of these closures, which do their own
1590 // synchronization among themselves.
1591 G1CMKeepAliveAndDrainClosure g1_keep_alive(this, task(0), true /* is_serial */);
1592 G1CMDrainMarkingStackClosure g1_drain_mark_stack(this, task(0), true /* is_serial */);
1593
1594 // We need at least one active thread. If reference processing
1595 // is not multi-threaded we use the current (VMThread) thread,
1596 // otherwise we use the work gang from the G1CollectedHeap and
1597 // we utilize all the worker threads we can.
1598 bool processing_is_mt = rp->processing_is_mt();
1599 uint active_workers = (processing_is_mt ? g1h->workers()->active_workers() : 1U);
1600 active_workers = MAX2(MIN2(active_workers, _max_num_tasks), 1U);
1601
1602 // Parallel processing task executor.
1603 G1CMRefProcTaskExecutor par_task_executor(g1h, this,
1604 g1h->workers(), active_workers);
1605 AbstractRefProcTaskExecutor* executor = (processing_is_mt ? &par_task_executor : NULL);
1606
1607 // Set the concurrency level. The phase was already set prior to
1608 // executing the remark task.
1609 set_concurrency(active_workers);
1610
1611 // Set the degree of MT processing here. If the discovery was done MT,
1612 // the number of threads involved during discovery could differ from
1613 // the number of active workers. This is OK as long as the discovered
1614 // Reference lists are balanced (see balance_all_queues() and balance_queues()).
1615 rp->set_active_mt_degree(active_workers);
1616
1617 ReferenceProcessorPhaseTimes pt(_gc_timer_cm, rp->num_q());
1618
1619 // Process the weak references.
1620 const ReferenceProcessorStats& stats =
1621 rp->process_discovered_references(&g1_is_alive,
1622 &g1_keep_alive,
1623 &g1_drain_mark_stack,
1624 executor,
1625 &pt);
1626 _gc_tracer_cm->report_gc_reference_stats(stats);
1627 pt.print_all_references();
1628
1629 // The do_oop work routines of the keep_alive and drain_marking_stack
1630 // oop closures will set the has_overflown flag if we overflow the
1631 // global marking stack.
1632
1633 assert(has_overflown() || _global_mark_stack.is_empty(),
1634 "Mark stack should be empty (unless it has overflown)");
1635
1636 assert(rp->num_q() == active_workers, "why not");
1637
1638 rp->enqueue_discovered_references(executor, &pt);
1639
1640 rp->verify_no_references_recorded();
1641
1642 pt.print_enqueue_phase();
1643
1644 assert(!rp->discovery_enabled(), "Post condition");
1645 }
1646
1647 assert(has_overflown() || _global_mark_stack.is_empty(),
1648 "Mark stack should be empty (unless it has overflown)");
1649
1650 {
1651 GCTraceTime(Debug, gc, phases) debug("Weak Processing", _gc_timer_cm);
1652 WeakProcessor::weak_oops_do(&g1_is_alive, &do_nothing_cl);
1653 }
1654
1655 if (has_overflown()) {
1656 // We can not trust g1_is_alive if the marking stack overflowed
1657 return;
1658 }
1659
1660 assert(_global_mark_stack.is_empty(), "Marking should have completed");
1661
1662 // Unload Klasses, String, Symbols, Code Cache, etc.
1663 if (ClassUnloadingWithConcurrentMark) {
1664 GCTraceTime(Debug, gc, phases) debug("Class Unloading", _gc_timer_cm);
1665 bool purged_classes = SystemDictionary::do_unloading(&g1_is_alive, _gc_timer_cm, false /* Defer cleaning */);
1666 g1h->complete_cleaning(&g1_is_alive, purged_classes);
1667 } else {
1668 GCTraceTime(Debug, gc, phases) debug("Cleanup", _gc_timer_cm);
1669 // No need to clean string table and symbol table as they are treated as strong roots when
1670 // class unloading is disabled.
1671 g1h->partial_cleaning(&g1_is_alive, false, false, G1StringDedup::is_enabled());
1672
1673 }
1674 }
1675
1676 void G1ConcurrentMark::swap_mark_bitmaps() {
1677 G1CMBitMap* temp = _prev_mark_bitmap;
1678 _prev_mark_bitmap = _next_mark_bitmap;
1679 _next_mark_bitmap = temp;
1680 }
1681
1682 // Closure for marking entries in SATB buffers.
1683 class G1CMSATBBufferClosure : public SATBBufferClosure {
1684 private:
1685 G1CMTask* _task;
1686 G1CollectedHeap* _g1h;
1687
1688 // This is very similar to G1CMTask::deal_with_reference, but with
1689 // more relaxed requirements for the argument, so this must be more
1690 // circumspect about treating the argument as an object.
1691 void do_entry(void* entry) const {
1692 _task->increment_refs_reached();
1693 oop const obj = static_cast<oop>(entry);
1694 _task->make_reference_grey(obj);
1695 }
1696
1697 public:
1698 G1CMSATBBufferClosure(G1CMTask* task, G1CollectedHeap* g1h)
1699 : _task(task), _g1h(g1h) { }
1700
1701 virtual void do_buffer(void** buffer, size_t size) {
1702 for (size_t i = 0; i < size; ++i) {
1703 do_entry(buffer[i]);
1704 }
1705 }
1706 };
1707
1708 class G1RemarkThreadsClosure : public ThreadClosure {
1709 G1CMSATBBufferClosure _cm_satb_cl;
1710 G1CMOopClosure _cm_cl;
1711 MarkingCodeBlobClosure _code_cl;
1712 int _thread_parity;
1713
1714 public:
1715 G1RemarkThreadsClosure(G1CollectedHeap* g1h, G1CMTask* task) :
1716 _cm_satb_cl(task, g1h),
1717 _cm_cl(g1h, g1h->concurrent_mark(), task),
1718 _code_cl(&_cm_cl, !CodeBlobToOopClosure::FixRelocations),
1719 _thread_parity(Threads::thread_claim_parity()) {}
1720
1721 void do_thread(Thread* thread) {
1722 if (thread->is_Java_thread()) {
1723 if (thread->claim_oops_do(true, _thread_parity)) {
1724 JavaThread* jt = (JavaThread*)thread;
1725
1726 // In theory it should not be neccessary to explicitly walk the nmethods to find roots for concurrent marking
1727 // however the liveness of oops reachable from nmethods have very complex lifecycles:
1728 // * Alive if on the stack of an executing method
1729 // * Weakly reachable otherwise
1730 // Some objects reachable from nmethods, such as the class loader (or klass_holder) of the receiver should be
1731 // live by the SATB invariant but other oops recorded in nmethods may behave differently.
1732 jt->nmethods_do(&_code_cl);
1733
1734 jt->satb_mark_queue().apply_closure_and_empty(&_cm_satb_cl);
1735 }
1736 } else if (thread->is_VM_thread()) {
1737 if (thread->claim_oops_do(true, _thread_parity)) {
1738 JavaThread::satb_mark_queue_set().shared_satb_queue()->apply_closure_and_empty(&_cm_satb_cl);
1739 }
1740 }
1741 }
1742 };
1743
1744 class G1CMRemarkTask: public AbstractGangTask {
1745 private:
1746 G1ConcurrentMark* _cm;
1747 public:
1748 void work(uint worker_id) {
1749 G1CMTask* task = _cm->task(worker_id);
1750 task->record_start_time();
1751 {
1752 ResourceMark rm;
1753 HandleMark hm;
1754
1755 G1RemarkThreadsClosure threads_f(G1CollectedHeap::heap(), task);
1756 Threads::threads_do(&threads_f);
1757 }
1758
1759 do {
1760 task->do_marking_step(1000000000.0 /* something very large */,
1761 true /* do_termination */,
1762 false /* is_serial */);
1763 } while (task->has_aborted() && !_cm->has_overflown());
1764 // If we overflow, then we do not want to restart. We instead
1765 // want to abort remark and do concurrent marking again.
1766 task->record_end_time();
1767 }
1768
1769 G1CMRemarkTask(G1ConcurrentMark* cm, uint active_workers) :
1770 AbstractGangTask("Par Remark"), _cm(cm) {
1771 _cm->terminator()->reset_for_reuse(active_workers);
1772 }
1773 };
1774
1775 void G1ConcurrentMark::checkpoint_roots_final_work() {
1776 ResourceMark rm;
1777 HandleMark hm;
1778 G1CollectedHeap* g1h = G1CollectedHeap::heap();
1779
1780 GCTraceTime(Debug, gc, phases) trace("Finalize Marking", _gc_timer_cm);
1781
1782 g1h->ensure_parsability(false);
1783
1784 // this is remark, so we'll use up all active threads
1785 uint active_workers = g1h->workers()->active_workers();
1786 set_concurrency_and_phase(active_workers, false /* concurrent */);
1787 // Leave _parallel_marking_threads at it's
1788 // value originally calculated in the G1ConcurrentMark
1789 // constructor and pass values of the active workers
1790 // through the gang in the task.
1791
1792 {
1793 StrongRootsScope srs(active_workers);
1794
1795 G1CMRemarkTask remarkTask(this, active_workers);
1796 // We will start all available threads, even if we decide that the
1797 // active_workers will be fewer. The extra ones will just bail out
1798 // immediately.
1799 g1h->workers()->run_task(&remarkTask);
1800 }
1801
1802 SATBMarkQueueSet& satb_mq_set = JavaThread::satb_mark_queue_set();
1803 guarantee(has_overflown() ||
1804 satb_mq_set.completed_buffers_num() == 0,
1805 "Invariant: has_overflown = %s, num buffers = " SIZE_FORMAT,
1806 BOOL_TO_STR(has_overflown()),
1807 satb_mq_set.completed_buffers_num());
1808
1809 print_stats();
1810 }
1811
1812 void G1ConcurrentMark::clear_range_in_prev_bitmap(MemRegion mr) {
1813 _prev_mark_bitmap->clear_range(mr);
1814 }
1815
1816 HeapRegion*
1817 G1ConcurrentMark::claim_region(uint worker_id) {
1818 // "checkpoint" the finger
1819 HeapWord* finger = _finger;
1820
1821 // _heap_end will not change underneath our feet; it only changes at
1822 // yield points.
1823 while (finger < _heap_end) {
1824 assert(_g1h->is_in_g1_reserved(finger), "invariant");
1825
1826 HeapRegion* curr_region = _g1h->heap_region_containing(finger);
1827 // Make sure that the reads below do not float before loading curr_region.
1828 OrderAccess::loadload();
1829 // Above heap_region_containing may return NULL as we always scan claim
1830 // until the end of the heap. In this case, just jump to the next region.
1831 HeapWord* end = curr_region != NULL ? curr_region->end() : finger + HeapRegion::GrainWords;
1832
1833 // Is the gap between reading the finger and doing the CAS too long?
1834 HeapWord* res = Atomic::cmpxchg(end, &_finger, finger);
1835 if (res == finger && curr_region != NULL) {
1836 // we succeeded
1837 HeapWord* bottom = curr_region->bottom();
1838 HeapWord* limit = curr_region->next_top_at_mark_start();
1839
1840 // notice that _finger == end cannot be guaranteed here since,
1841 // someone else might have moved the finger even further
1842 assert(_finger >= end, "the finger should have moved forward");
1843
1844 if (limit > bottom) {
1845 return curr_region;
1846 } else {
1847 assert(limit == bottom,
1848 "the region limit should be at bottom");
1849 // we return NULL and the caller should try calling
1850 // claim_region() again.
1851 return NULL;
1852 }
1853 } else {
1854 assert(_finger > finger, "the finger should have moved forward");
1855 // read it again
1856 finger = _finger;
1857 }
1858 }
1859
1860 return NULL;
1861 }
1862
1863 #ifndef PRODUCT
1864 class VerifyNoCSetOops VALUE_OBJ_CLASS_SPEC {
1865 private:
1866 G1CollectedHeap* _g1h;
1867 const char* _phase;
1868 int _info;
1869
1870 public:
1871 VerifyNoCSetOops(const char* phase, int info = -1) :
1872 _g1h(G1CollectedHeap::heap()),
1873 _phase(phase),
1874 _info(info)
1875 { }
1876
1877 void operator()(G1TaskQueueEntry task_entry) const {
1878 if (task_entry.is_array_slice()) {
1879 guarantee(_g1h->is_in_reserved(task_entry.slice()), "Slice " PTR_FORMAT " must be in heap.", p2i(task_entry.slice()));
1880 return;
1881 }
1882 guarantee(oopDesc::is_oop(task_entry.obj()),
1883 "Non-oop " PTR_FORMAT ", phase: %s, info: %d",
1884 p2i(task_entry.obj()), _phase, _info);
1885 guarantee(!_g1h->is_in_cset(task_entry.obj()),
1886 "obj: " PTR_FORMAT " in CSet, phase: %s, info: %d",
1887 p2i(task_entry.obj()), _phase, _info);
1888 }
1889 };
1890
1891 void G1ConcurrentMark::verify_no_cset_oops() {
1892 assert(SafepointSynchronize::is_at_safepoint(), "should be at a safepoint");
1893 if (!G1CollectedHeap::heap()->collector_state()->mark_in_progress()) {
1894 return;
1895 }
1896
1897 // Verify entries on the global mark stack
1898 _global_mark_stack.iterate(VerifyNoCSetOops("Stack"));
1899
1900 // Verify entries on the task queues
1901 for (uint i = 0; i < _max_num_tasks; ++i) {
1902 G1CMTaskQueue* queue = _task_queues->queue(i);
1903 queue->iterate(VerifyNoCSetOops("Queue", i));
1904 }
1905
1906 // Verify the global finger
1907 HeapWord* global_finger = finger();
1908 if (global_finger != NULL && global_finger < _heap_end) {
1909 // Since we always iterate over all regions, we might get a NULL HeapRegion
1910 // here.
1911 HeapRegion* global_hr = _g1h->heap_region_containing(global_finger);
1912 guarantee(global_hr == NULL || global_finger == global_hr->bottom(),
1913 "global finger: " PTR_FORMAT " region: " HR_FORMAT,
1914 p2i(global_finger), HR_FORMAT_PARAMS(global_hr));
1915 }
1916
1917 // Verify the task fingers
1918 assert(_num_concurrent_workers <= _max_num_tasks, "sanity");
1919 for (uint i = 0; i < _num_concurrent_workers; ++i) {
1920 G1CMTask* task = _tasks[i];
1921 HeapWord* task_finger = task->finger();
1922 if (task_finger != NULL && task_finger < _heap_end) {
1923 // See above note on the global finger verification.
1924 HeapRegion* task_hr = _g1h->heap_region_containing(task_finger);
1925 guarantee(task_hr == NULL || task_finger == task_hr->bottom() ||
1926 !task_hr->in_collection_set(),
1927 "task finger: " PTR_FORMAT " region: " HR_FORMAT,
1928 p2i(task_finger), HR_FORMAT_PARAMS(task_hr));
1929 }
1930 }
1931 }
1932 #endif // PRODUCT
1933 void G1ConcurrentMark::create_live_data() {
1934 _g1h->g1_rem_set()->create_card_live_data(_concurrent_workers, _next_mark_bitmap);
1935 }
1936
1937 void G1ConcurrentMark::finalize_live_data() {
1938 _g1h->g1_rem_set()->finalize_card_live_data(_g1h->workers(), _next_mark_bitmap);
1939 }
1940
1941 void G1ConcurrentMark::verify_live_data() {
1942 _g1h->g1_rem_set()->verify_card_live_data(_g1h->workers(), _next_mark_bitmap);
1943 }
1944
1945 void G1ConcurrentMark::clear_live_data(WorkGang* workers) {
1946 _g1h->g1_rem_set()->clear_card_live_data(workers);
1947 }
1948
1949 #ifdef ASSERT
1950 void G1ConcurrentMark::verify_live_data_clear() {
1951 _g1h->g1_rem_set()->verify_card_live_data_is_clear();
1952 }
1953 #endif
1954
1955 void G1ConcurrentMark::print_stats() {
1956 if (!log_is_enabled(Debug, gc, stats)) {
1957 return;
1958 }
1959 log_debug(gc, stats)("---------------------------------------------------------------------");
1960 for (size_t i = 0; i < _num_active_tasks; ++i) {
1961 _tasks[i]->print_stats();
1962 log_debug(gc, stats)("---------------------------------------------------------------------");
1963 }
1964 }
1965
1966 void G1ConcurrentMark::abort() {
1967 if (!cm_thread()->during_cycle() || _has_aborted) {
1968 // We haven't started a concurrent cycle or we have already aborted it. No need to do anything.
1969 return;
1970 }
1971
1972 // Clear all marks in the next bitmap for the next marking cycle. This will allow us to skip the next
1973 // concurrent bitmap clearing.
1974 {
1975 GCTraceTime(Debug, gc)("Clear Next Bitmap");
1976 clear_bitmap(_next_mark_bitmap, _g1h->workers(), false);
1977 }
1978 // Note we cannot clear the previous marking bitmap here
1979 // since VerifyDuringGC verifies the objects marked during
1980 // a full GC against the previous bitmap.
1981
1982 {
1983 GCTraceTime(Debug, gc)("Clear Live Data");
1984 clear_live_data(_g1h->workers());
1985 }
1986 DEBUG_ONLY({
1987 GCTraceTime(Debug, gc)("Verify Live Data Clear");
1988 verify_live_data_clear();
1989 })
1990 // Empty mark stack
1991 reset_marking_state();
1992 for (uint i = 0; i < _max_num_tasks; ++i) {
1993 _tasks[i]->clear_region_fields();
1994 }
1995 _first_overflow_barrier_sync.abort();
1996 _second_overflow_barrier_sync.abort();
1997 _has_aborted = true;
1998
1999 SATBMarkQueueSet& satb_mq_set = JavaThread::satb_mark_queue_set();
2000 satb_mq_set.abandon_partial_marking();
2001 // This can be called either during or outside marking, we'll read
2002 // the expected_active value from the SATB queue set.
2003 satb_mq_set.set_active_all_threads(
2004 false, /* new active value */
2005 satb_mq_set.is_active() /* expected_active */);
2006 }
2007
2008 static void print_ms_time_info(const char* prefix, const char* name,
2009 NumberSeq& ns) {
2010 log_trace(gc, marking)("%s%5d %12s: total time = %8.2f s (avg = %8.2f ms).",
2011 prefix, ns.num(), name, ns.sum()/1000.0, ns.avg());
2012 if (ns.num() > 0) {
2013 log_trace(gc, marking)("%s [std. dev = %8.2f ms, max = %8.2f ms]",
2014 prefix, ns.sd(), ns.maximum());
2015 }
2016 }
2017
2018 void G1ConcurrentMark::print_summary_info() {
2019 Log(gc, marking) log;
2020 if (!log.is_trace()) {
2021 return;
2022 }
2023
2024 log.trace(" Concurrent marking:");
2025 print_ms_time_info(" ", "init marks", _init_times);
2026 print_ms_time_info(" ", "remarks", _remark_times);
2027 {
2028 print_ms_time_info(" ", "final marks", _remark_mark_times);
2029 print_ms_time_info(" ", "weak refs", _remark_weak_ref_times);
2030
2031 }
2032 print_ms_time_info(" ", "cleanups", _cleanup_times);
2033 log.trace(" Finalize live data total time = %8.2f s (avg = %8.2f ms).",
2034 _total_counting_time, (_cleanup_times.num() > 0 ? _total_counting_time * 1000.0 / (double)_cleanup_times.num() : 0.0));
2035 if (G1ScrubRemSets) {
2036 log.trace(" RS scrub total time = %8.2f s (avg = %8.2f ms).",
2037 _total_rs_scrub_time, (_cleanup_times.num() > 0 ? _total_rs_scrub_time * 1000.0 / (double)_cleanup_times.num() : 0.0));
2038 }
2039 log.trace(" Total stop_world time = %8.2f s.",
2040 (_init_times.sum() + _remark_times.sum() + _cleanup_times.sum())/1000.0);
2041 log.trace(" Total concurrent time = %8.2f s (%8.2f s marking).",
2042 cm_thread()->vtime_accum(), cm_thread()->vtime_mark_accum());
2043 }
2044
2045 void G1ConcurrentMark::print_worker_threads_on(outputStream* st) const {
2046 _concurrent_workers->print_worker_threads_on(st);
2047 }
2048
2049 void G1ConcurrentMark::threads_do(ThreadClosure* tc) const {
2050 _concurrent_workers->threads_do(tc);
2051 }
2052
2053 void G1ConcurrentMark::print_on_error(outputStream* st) const {
2054 st->print_cr("Marking Bits (Prev, Next): (CMBitMap*) " PTR_FORMAT ", (CMBitMap*) " PTR_FORMAT,
2055 p2i(_prev_mark_bitmap), p2i(_next_mark_bitmap));
2056 _prev_mark_bitmap->print_on_error(st, " Prev Bits: ");
2057 _next_mark_bitmap->print_on_error(st, " Next Bits: ");
2058 }
2059
2060 static ReferenceProcessor* get_cm_oop_closure_ref_processor(G1CollectedHeap* g1h) {
2061 ReferenceProcessor* result = g1h->ref_processor_cm();
2062 assert(result != NULL, "CM reference processor should not be NULL");
2063 return result;
2064 }
2065
2066 G1CMOopClosure::G1CMOopClosure(G1CollectedHeap* g1h,
2067 G1ConcurrentMark* cm,
2068 G1CMTask* task)
2069 : MetadataAwareOopClosure(get_cm_oop_closure_ref_processor(g1h)),
2070 _g1h(g1h), _cm(cm), _task(task)
2071 { }
2072
2073 void G1CMTask::setup_for_region(HeapRegion* hr) {
2074 assert(hr != NULL,
2075 "claim_region() should have filtered out NULL regions");
2076 _curr_region = hr;
2077 _finger = hr->bottom();
2078 update_region_limit();
2079 }
2080
2081 void G1CMTask::update_region_limit() {
2082 HeapRegion* hr = _curr_region;
2083 HeapWord* bottom = hr->bottom();
2084 HeapWord* limit = hr->next_top_at_mark_start();
2085
2086 if (limit == bottom) {
2087 // The region was collected underneath our feet.
2088 // We set the finger to bottom to ensure that the bitmap
2089 // iteration that will follow this will not do anything.
2090 // (this is not a condition that holds when we set the region up,
2091 // as the region is not supposed to be empty in the first place)
2092 _finger = bottom;
2093 } else if (limit >= _region_limit) {
2094 assert(limit >= _finger, "peace of mind");
2095 } else {
2096 assert(limit < _region_limit, "only way to get here");
2097 // This can happen under some pretty unusual circumstances. An
2098 // evacuation pause empties the region underneath our feet (NTAMS
2099 // at bottom). We then do some allocation in the region (NTAMS
2100 // stays at bottom), followed by the region being used as a GC
2101 // alloc region (NTAMS will move to top() and the objects
2102 // originally below it will be grayed). All objects now marked in
2103 // the region are explicitly grayed, if below the global finger,
2104 // and we do not need in fact to scan anything else. So, we simply
2105 // set _finger to be limit to ensure that the bitmap iteration
2106 // doesn't do anything.
2107 _finger = limit;
2108 }
2109
2110 _region_limit = limit;
2111 }
2112
2113 void G1CMTask::giveup_current_region() {
2114 assert(_curr_region != NULL, "invariant");
2115 clear_region_fields();
2116 }
2117
2118 void G1CMTask::clear_region_fields() {
2119 // Values for these three fields that indicate that we're not
2120 // holding on to a region.
2121 _curr_region = NULL;
2122 _finger = NULL;
2123 _region_limit = NULL;
2124 }
2125
2126 void G1CMTask::set_cm_oop_closure(G1CMOopClosure* cm_oop_closure) {
2127 if (cm_oop_closure == NULL) {
2128 assert(_cm_oop_closure != NULL, "invariant");
2129 } else {
2130 assert(_cm_oop_closure == NULL, "invariant");
2131 }
2132 _cm_oop_closure = cm_oop_closure;
2133 }
2134
2135 void G1CMTask::reset(G1CMBitMap* next_mark_bitmap) {
2136 guarantee(next_mark_bitmap != NULL, "invariant");
2137 _next_mark_bitmap = next_mark_bitmap;
2138 clear_region_fields();
2139
2140 _calls = 0;
2141 _elapsed_time_ms = 0.0;
2142 _termination_time_ms = 0.0;
2143 _termination_start_time_ms = 0.0;
2144 }
2145
2146 bool G1CMTask::should_exit_termination() {
2147 regular_clock_call();
2148 // This is called when we are in the termination protocol. We should
2149 // quit if, for some reason, this task wants to abort or the global
2150 // stack is not empty (this means that we can get work from it).
2151 return !_cm->mark_stack_empty() || has_aborted();
2152 }
2153
2154 void G1CMTask::reached_limit() {
2155 assert(_words_scanned >= _words_scanned_limit ||
2156 _refs_reached >= _refs_reached_limit ,
2157 "shouldn't have been called otherwise");
2158 regular_clock_call();
2159 }
2160
2161 void G1CMTask::regular_clock_call() {
2162 if (has_aborted()) return;
2163
2164 // First, we need to recalculate the words scanned and refs reached
2165 // limits for the next clock call.
2166 recalculate_limits();
2167
2168 // During the regular clock call we do the following
2169
2170 // (1) If an overflow has been flagged, then we abort.
2171 if (_cm->has_overflown()) {
2172 set_has_aborted();
2173 return;
2174 }
2175
2176 // If we are not concurrent (i.e. we're doing remark) we don't need
2177 // to check anything else. The other steps are only needed during
2178 // the concurrent marking phase.
2179 if (!_concurrent) {
2180 return;
2181 }
2182
2183 // (2) If marking has been aborted for Full GC, then we also abort.
2184 if (_cm->has_aborted()) {
2185 set_has_aborted();
2186 return;
2187 }
2188
2189 double curr_time_ms = os::elapsedVTime() * 1000.0;
2190
2191 // (4) We check whether we should yield. If we have to, then we abort.
2192 if (SuspendibleThreadSet::should_yield()) {
2193 // We should yield. To do this we abort the task. The caller is
2194 // responsible for yielding.
2195 set_has_aborted();
2196 return;
2197 }
2198
2199 // (5) We check whether we've reached our time quota. If we have,
2200 // then we abort.
2201 double elapsed_time_ms = curr_time_ms - _start_time_ms;
2202 if (elapsed_time_ms > _time_target_ms) {
2203 set_has_aborted();
2204 _has_timed_out = true;
2205 return;
2206 }
2207
2208 // (6) Finally, we check whether there are enough completed STAB
2209 // buffers available for processing. If there are, we abort.
2210 SATBMarkQueueSet& satb_mq_set = JavaThread::satb_mark_queue_set();
2211 if (!_draining_satb_buffers && satb_mq_set.process_completed_buffers()) {
2212 // we do need to process SATB buffers, we'll abort and restart
2213 // the marking task to do so
2214 set_has_aborted();
2215 return;
2216 }
2217 }
2218
2219 void G1CMTask::recalculate_limits() {
2220 _real_words_scanned_limit = _words_scanned + words_scanned_period;
2221 _words_scanned_limit = _real_words_scanned_limit;
2222
2223 _real_refs_reached_limit = _refs_reached + refs_reached_period;
2224 _refs_reached_limit = _real_refs_reached_limit;
2225 }
2226
2227 void G1CMTask::decrease_limits() {
2228 // This is called when we believe that we're going to do an infrequent
2229 // operation which will increase the per byte scanned cost (i.e. move
2230 // entries to/from the global stack). It basically tries to decrease the
2231 // scanning limit so that the clock is called earlier.
2232
2233 _words_scanned_limit = _real_words_scanned_limit - 3 * words_scanned_period / 4;
2234 _refs_reached_limit = _real_refs_reached_limit - 3 * refs_reached_period / 4;
2235 }
2236
2237 void G1CMTask::move_entries_to_global_stack() {
2238 // Local array where we'll store the entries that will be popped
2239 // from the local queue.
2240 G1TaskQueueEntry buffer[G1CMMarkStack::EntriesPerChunk];
2241
2242 size_t n = 0;
2243 G1TaskQueueEntry task_entry;
2244 while (n < G1CMMarkStack::EntriesPerChunk && _task_queue->pop_local(task_entry)) {
2245 buffer[n] = task_entry;
2246 ++n;
2247 }
2248 if (n < G1CMMarkStack::EntriesPerChunk) {
2249 buffer[n] = G1TaskQueueEntry();
2250 }
2251
2252 if (n > 0) {
2253 if (!_cm->mark_stack_push(buffer)) {
2254 set_has_aborted();
2255 }
2256 }
2257
2258 // This operation was quite expensive, so decrease the limits.
2259 decrease_limits();
2260 }
2261
2262 bool G1CMTask::get_entries_from_global_stack() {
2263 // Local array where we'll store the entries that will be popped
2264 // from the global stack.
2265 G1TaskQueueEntry buffer[G1CMMarkStack::EntriesPerChunk];
2266
2267 if (!_cm->mark_stack_pop(buffer)) {
2268 return false;
2269 }
2270
2271 // We did actually pop at least one entry.
2272 for (size_t i = 0; i < G1CMMarkStack::EntriesPerChunk; ++i) {
2273 G1TaskQueueEntry task_entry = buffer[i];
2274 if (task_entry.is_null()) {
2275 break;
2276 }
2277 assert(task_entry.is_array_slice() || oopDesc::is_oop(task_entry.obj()), "Element " PTR_FORMAT " must be an array slice or oop", p2i(task_entry.obj()));
2278 bool success = _task_queue->push(task_entry);
2279 // We only call this when the local queue is empty or under a
2280 // given target limit. So, we do not expect this push to fail.
2281 assert(success, "invariant");
2282 }
2283
2284 // This operation was quite expensive, so decrease the limits
2285 decrease_limits();
2286 return true;
2287 }
2288
2289 void G1CMTask::drain_local_queue(bool partially) {
2290 if (has_aborted()) {
2291 return;
2292 }
2293
2294 // Decide what the target size is, depending whether we're going to
2295 // drain it partially (so that other tasks can steal if they run out
2296 // of things to do) or totally (at the very end).
2297 size_t target_size;
2298 if (partially) {
2299 target_size = MIN2((size_t)_task_queue->max_elems()/3, GCDrainStackTargetSize);
2300 } else {
2301 target_size = 0;
2302 }
2303
2304 if (_task_queue->size() > target_size) {
2305 G1TaskQueueEntry entry;
2306 bool ret = _task_queue->pop_local(entry);
2307 while (ret) {
2308 scan_task_entry(entry);
2309 if (_task_queue->size() <= target_size || has_aborted()) {
2310 ret = false;
2311 } else {
2312 ret = _task_queue->pop_local(entry);
2313 }
2314 }
2315 }
2316 }
2317
2318 void G1CMTask::drain_global_stack(bool partially) {
2319 if (has_aborted()) return;
2320
2321 // We have a policy to drain the local queue before we attempt to
2322 // drain the global stack.
2323 assert(partially || _task_queue->size() == 0, "invariant");
2324
2325 // Decide what the target size is, depending whether we're going to
2326 // drain it partially (so that other tasks can steal if they run out
2327 // of things to do) or totally (at the very end).
2328 // Notice that when draining the global mark stack partially, due to the racyness
2329 // of the mark stack size update we might in fact drop below the target. But,
2330 // this is not a problem.
2331 // In case of total draining, we simply process until the global mark stack is
2332 // totally empty, disregarding the size counter.
2333 if (partially) {
2334 size_t const target_size = _cm->partial_mark_stack_size_target();
2335 while (!has_aborted() && _cm->mark_stack_size() > target_size) {
2336 if (get_entries_from_global_stack()) {
2337 drain_local_queue(partially);
2338 }
2339 }
2340 } else {
2341 while (!has_aborted() && get_entries_from_global_stack()) {
2342 drain_local_queue(partially);
2343 }
2344 }
2345 }
2346
2347 // SATB Queue has several assumptions on whether to call the par or
2348 // non-par versions of the methods. this is why some of the code is
2349 // replicated. We should really get rid of the single-threaded version
2350 // of the code to simplify things.
2351 void G1CMTask::drain_satb_buffers() {
2352 if (has_aborted()) return;
2353
2354 // We set this so that the regular clock knows that we're in the
2355 // middle of draining buffers and doesn't set the abort flag when it
2356 // notices that SATB buffers are available for draining. It'd be
2357 // very counter productive if it did that. :-)
2358 _draining_satb_buffers = true;
2359
2360 G1CMSATBBufferClosure satb_cl(this, _g1h);
2361 SATBMarkQueueSet& satb_mq_set = JavaThread::satb_mark_queue_set();
2362
2363 // This keeps claiming and applying the closure to completed buffers
2364 // until we run out of buffers or we need to abort.
2365 while (!has_aborted() &&
2366 satb_mq_set.apply_closure_to_completed_buffer(&satb_cl)) {
2367 regular_clock_call();
2368 }
2369
2370 _draining_satb_buffers = false;
2371
2372 assert(has_aborted() ||
2373 _concurrent ||
2374 satb_mq_set.completed_buffers_num() == 0, "invariant");
2375
2376 // again, this was a potentially expensive operation, decrease the
2377 // limits to get the regular clock call early
2378 decrease_limits();
2379 }
2380
2381 void G1CMTask::print_stats() {
2382 log_debug(gc, stats)("Marking Stats, task = %u, calls = %u",
2383 _worker_id, _calls);
2384 log_debug(gc, stats)(" Elapsed time = %1.2lfms, Termination time = %1.2lfms",
2385 _elapsed_time_ms, _termination_time_ms);
2386 log_debug(gc, stats)(" Step Times (cum): num = %d, avg = %1.2lfms, sd = %1.2lfms",
2387 _step_times_ms.num(), _step_times_ms.avg(),
2388 _step_times_ms.sd());
2389 log_debug(gc, stats)(" max = %1.2lfms, total = %1.2lfms",
2390 _step_times_ms.maximum(), _step_times_ms.sum());
2391 }
2392
2393 bool G1ConcurrentMark::try_stealing(uint worker_id, int* hash_seed, G1TaskQueueEntry& task_entry) {
2394 return _task_queues->steal(worker_id, hash_seed, task_entry);
2395 }
2396
2397 /*****************************************************************************
2398
2399 The do_marking_step(time_target_ms, ...) method is the building
2400 block of the parallel marking framework. It can be called in parallel
2401 with other invocations of do_marking_step() on different tasks
2402 (but only one per task, obviously) and concurrently with the
2403 mutator threads, or during remark, hence it eliminates the need
2404 for two versions of the code. When called during remark, it will
2405 pick up from where the task left off during the concurrent marking
2406 phase. Interestingly, tasks are also claimable during evacuation
2407 pauses too, since do_marking_step() ensures that it aborts before
2408 it needs to yield.
2409
2410 The data structures that it uses to do marking work are the
2411 following:
2412
2413 (1) Marking Bitmap. If there are gray objects that appear only
2414 on the bitmap (this happens either when dealing with an overflow
2415 or when the initial marking phase has simply marked the roots
2416 and didn't push them on the stack), then tasks claim heap
2417 regions whose bitmap they then scan to find gray objects. A
2418 global finger indicates where the end of the last claimed region
2419 is. A local finger indicates how far into the region a task has
2420 scanned. The two fingers are used to determine how to gray an
2421 object (i.e. whether simply marking it is OK, as it will be
2422 visited by a task in the future, or whether it needs to be also
2423 pushed on a stack).
2424
2425 (2) Local Queue. The local queue of the task which is accessed
2426 reasonably efficiently by the task. Other tasks can steal from
2427 it when they run out of work. Throughout the marking phase, a
2428 task attempts to keep its local queue short but not totally
2429 empty, so that entries are available for stealing by other
2430 tasks. Only when there is no more work, a task will totally
2431 drain its local queue.
2432
2433 (3) Global Mark Stack. This handles local queue overflow. During
2434 marking only sets of entries are moved between it and the local
2435 queues, as access to it requires a mutex and more fine-grain
2436 interaction with it which might cause contention. If it
2437 overflows, then the marking phase should restart and iterate
2438 over the bitmap to identify gray objects. Throughout the marking
2439 phase, tasks attempt to keep the global mark stack at a small
2440 length but not totally empty, so that entries are available for
2441 popping by other tasks. Only when there is no more work, tasks
2442 will totally drain the global mark stack.
2443
2444 (4) SATB Buffer Queue. This is where completed SATB buffers are
2445 made available. Buffers are regularly removed from this queue
2446 and scanned for roots, so that the queue doesn't get too
2447 long. During remark, all completed buffers are processed, as
2448 well as the filled in parts of any uncompleted buffers.
2449
2450 The do_marking_step() method tries to abort when the time target
2451 has been reached. There are a few other cases when the
2452 do_marking_step() method also aborts:
2453
2454 (1) When the marking phase has been aborted (after a Full GC).
2455
2456 (2) When a global overflow (on the global stack) has been
2457 triggered. Before the task aborts, it will actually sync up with
2458 the other tasks to ensure that all the marking data structures
2459 (local queues, stacks, fingers etc.) are re-initialized so that
2460 when do_marking_step() completes, the marking phase can
2461 immediately restart.
2462
2463 (3) When enough completed SATB buffers are available. The
2464 do_marking_step() method only tries to drain SATB buffers right
2465 at the beginning. So, if enough buffers are available, the
2466 marking step aborts and the SATB buffers are processed at
2467 the beginning of the next invocation.
2468
2469 (4) To yield. when we have to yield then we abort and yield
2470 right at the end of do_marking_step(). This saves us from a lot
2471 of hassle as, by yielding we might allow a Full GC. If this
2472 happens then objects will be compacted underneath our feet, the
2473 heap might shrink, etc. We save checking for this by just
2474 aborting and doing the yield right at the end.
2475
2476 From the above it follows that the do_marking_step() method should
2477 be called in a loop (or, otherwise, regularly) until it completes.
2478
2479 If a marking step completes without its has_aborted() flag being
2480 true, it means it has completed the current marking phase (and
2481 also all other marking tasks have done so and have all synced up).
2482
2483 A method called regular_clock_call() is invoked "regularly" (in
2484 sub ms intervals) throughout marking. It is this clock method that
2485 checks all the abort conditions which were mentioned above and
2486 decides when the task should abort. A work-based scheme is used to
2487 trigger this clock method: when the number of object words the
2488 marking phase has scanned or the number of references the marking
2489 phase has visited reach a given limit. Additional invocations to
2490 the method clock have been planted in a few other strategic places
2491 too. The initial reason for the clock method was to avoid calling
2492 vtime too regularly, as it is quite expensive. So, once it was in
2493 place, it was natural to piggy-back all the other conditions on it
2494 too and not constantly check them throughout the code.
2495
2496 If do_termination is true then do_marking_step will enter its
2497 termination protocol.
2498
2499 The value of is_serial must be true when do_marking_step is being
2500 called serially (i.e. by the VMThread) and do_marking_step should
2501 skip any synchronization in the termination and overflow code.
2502 Examples include the serial remark code and the serial reference
2503 processing closures.
2504
2505 The value of is_serial must be false when do_marking_step is
2506 being called by any of the worker threads in a work gang.
2507 Examples include the concurrent marking code (CMMarkingTask),
2508 the MT remark code, and the MT reference processing closures.
2509
2510 *****************************************************************************/
2511
2512 void G1CMTask::do_marking_step(double time_target_ms,
2513 bool do_termination,
2514 bool is_serial) {
2515 assert(time_target_ms >= 1.0, "minimum granularity is 1ms");
2516 assert(_concurrent == _cm->concurrent(), "they should be the same");
2517
2518 _start_time_ms = os::elapsedVTime() * 1000.0;
2519
2520 // If do_stealing is true then do_marking_step will attempt to
2521 // steal work from the other G1CMTasks. It only makes sense to
2522 // enable stealing when the termination protocol is enabled
2523 // and do_marking_step() is not being called serially.
2524 bool do_stealing = do_termination && !is_serial;
2525
2526 double diff_prediction_ms = _g1h->g1_policy()->predictor().get_new_prediction(&_marking_step_diffs_ms);
2527 _time_target_ms = time_target_ms - diff_prediction_ms;
2528
2529 // set up the variables that are used in the work-based scheme to
2530 // call the regular clock method
2531 _words_scanned = 0;
2532 _refs_reached = 0;
2533 recalculate_limits();
2534
2535 // clear all flags
2536 clear_has_aborted();
2537 _has_timed_out = false;
2538 _draining_satb_buffers = false;
2539
2540 ++_calls;
2541
2542 // Set up the bitmap and oop closures. Anything that uses them is
2543 // eventually called from this method, so it is OK to allocate these
2544 // statically.
2545 G1CMBitMapClosure bitmap_closure(this, _cm);
2546 G1CMOopClosure cm_oop_closure(_g1h, _cm, this);
2547 set_cm_oop_closure(&cm_oop_closure);
2548
2549 if (_cm->has_overflown()) {
2550 // This can happen if the mark stack overflows during a GC pause
2551 // and this task, after a yield point, restarts. We have to abort
2552 // as we need to get into the overflow protocol which happens
2553 // right at the end of this task.
2554 set_has_aborted();
2555 }
2556
2557 // First drain any available SATB buffers. After this, we will not
2558 // look at SATB buffers before the next invocation of this method.
2559 // If enough completed SATB buffers are queued up, the regular clock
2560 // will abort this task so that it restarts.
2561 drain_satb_buffers();
2562 // ...then partially drain the local queue and the global stack
2563 drain_local_queue(true);
2564 drain_global_stack(true);
2565
2566 do {
2567 if (!has_aborted() && _curr_region != NULL) {
2568 // This means that we're already holding on to a region.
2569 assert(_finger != NULL, "if region is not NULL, then the finger "
2570 "should not be NULL either");
2571
2572 // We might have restarted this task after an evacuation pause
2573 // which might have evacuated the region we're holding on to
2574 // underneath our feet. Let's read its limit again to make sure
2575 // that we do not iterate over a region of the heap that
2576 // contains garbage (update_region_limit() will also move
2577 // _finger to the start of the region if it is found empty).
2578 update_region_limit();
2579 // We will start from _finger not from the start of the region,
2580 // as we might be restarting this task after aborting half-way
2581 // through scanning this region. In this case, _finger points to
2582 // the address where we last found a marked object. If this is a
2583 // fresh region, _finger points to start().
2584 MemRegion mr = MemRegion(_finger, _region_limit);
2585
2586 assert(!_curr_region->is_humongous() || mr.start() == _curr_region->bottom(),
2587 "humongous regions should go around loop once only");
2588
2589 // Some special cases:
2590 // If the memory region is empty, we can just give up the region.
2591 // If the current region is humongous then we only need to check
2592 // the bitmap for the bit associated with the start of the object,
2593 // scan the object if it's live, and give up the region.
2594 // Otherwise, let's iterate over the bitmap of the part of the region
2595 // that is left.
2596 // If the iteration is successful, give up the region.
2597 if (mr.is_empty()) {
2598 giveup_current_region();
2599 regular_clock_call();
2600 } else if (_curr_region->is_humongous() && mr.start() == _curr_region->bottom()) {
2601 if (_next_mark_bitmap->is_marked(mr.start())) {
2602 // The object is marked - apply the closure
2603 bitmap_closure.do_addr(mr.start());
2604 }
2605 // Even if this task aborted while scanning the humongous object
2606 // we can (and should) give up the current region.
2607 giveup_current_region();
2608 regular_clock_call();
2609 } else if (_next_mark_bitmap->iterate(&bitmap_closure, mr)) {
2610 giveup_current_region();
2611 regular_clock_call();
2612 } else {
2613 assert(has_aborted(), "currently the only way to do so");
2614 // The only way to abort the bitmap iteration is to return
2615 // false from the do_bit() method. However, inside the
2616 // do_bit() method we move the _finger to point to the
2617 // object currently being looked at. So, if we bail out, we
2618 // have definitely set _finger to something non-null.
2619 assert(_finger != NULL, "invariant");
2620
2621 // Region iteration was actually aborted. So now _finger
2622 // points to the address of the object we last scanned. If we
2623 // leave it there, when we restart this task, we will rescan
2624 // the object. It is easy to avoid this. We move the finger by
2625 // enough to point to the next possible object header.
2626 assert(_finger < _region_limit, "invariant");
2627 HeapWord* const new_finger = _finger + ((oop)_finger)->size();
2628 // Check if bitmap iteration was aborted while scanning the last object
2629 if (new_finger >= _region_limit) {
2630 giveup_current_region();
2631 } else {
2632 move_finger_to(new_finger);
2633 }
2634 }
2635 }
2636 // At this point we have either completed iterating over the
2637 // region we were holding on to, or we have aborted.
2638
2639 // We then partially drain the local queue and the global stack.
2640 // (Do we really need this?)
2641 drain_local_queue(true);
2642 drain_global_stack(true);
2643
2644 // Read the note on the claim_region() method on why it might
2645 // return NULL with potentially more regions available for
2646 // claiming and why we have to check out_of_regions() to determine
2647 // whether we're done or not.
2648 while (!has_aborted() && _curr_region == NULL && !_cm->out_of_regions()) {
2649 // We are going to try to claim a new region. We should have
2650 // given up on the previous one.
2651 // Separated the asserts so that we know which one fires.
2652 assert(_curr_region == NULL, "invariant");
2653 assert(_finger == NULL, "invariant");
2654 assert(_region_limit == NULL, "invariant");
2655 HeapRegion* claimed_region = _cm->claim_region(_worker_id);
2656 if (claimed_region != NULL) {
2657 // Yes, we managed to claim one
2658 setup_for_region(claimed_region);
2659 assert(_curr_region == claimed_region, "invariant");
2660 }
2661 // It is important to call the regular clock here. It might take
2662 // a while to claim a region if, for example, we hit a large
2663 // block of empty regions. So we need to call the regular clock
2664 // method once round the loop to make sure it's called
2665 // frequently enough.
2666 regular_clock_call();
2667 }
2668
2669 if (!has_aborted() && _curr_region == NULL) {
2670 assert(_cm->out_of_regions(),
2671 "at this point we should be out of regions");
2672 }
2673 } while ( _curr_region != NULL && !has_aborted());
2674
2675 if (!has_aborted()) {
2676 // We cannot check whether the global stack is empty, since other
2677 // tasks might be pushing objects to it concurrently.
2678 assert(_cm->out_of_regions(),
2679 "at this point we should be out of regions");
2680 // Try to reduce the number of available SATB buffers so that
2681 // remark has less work to do.
2682 drain_satb_buffers();
2683 }
2684
2685 // Since we've done everything else, we can now totally drain the
2686 // local queue and global stack.
2687 drain_local_queue(false);
2688 drain_global_stack(false);
2689
2690 // Attempt at work stealing from other task's queues.
2691 if (do_stealing && !has_aborted()) {
2692 // We have not aborted. This means that we have finished all that
2693 // we could. Let's try to do some stealing...
2694
2695 // We cannot check whether the global stack is empty, since other
2696 // tasks might be pushing objects to it concurrently.
2697 assert(_cm->out_of_regions() && _task_queue->size() == 0,
2698 "only way to reach here");
2699 while (!has_aborted()) {
2700 G1TaskQueueEntry entry;
2701 if (_cm->try_stealing(_worker_id, &_hash_seed, entry)) {
2702 scan_task_entry(entry);
2703
2704 // And since we're towards the end, let's totally drain the
2705 // local queue and global stack.
2706 drain_local_queue(false);
2707 drain_global_stack(false);
2708 } else {
2709 break;
2710 }
2711 }
2712 }
2713
2714 // We still haven't aborted. Now, let's try to get into the
2715 // termination protocol.
2716 if (do_termination && !has_aborted()) {
2717 // We cannot check whether the global stack is empty, since other
2718 // tasks might be concurrently pushing objects on it.
2719 // Separated the asserts so that we know which one fires.
2720 assert(_cm->out_of_regions(), "only way to reach here");
2721 assert(_task_queue->size() == 0, "only way to reach here");
2722 _termination_start_time_ms = os::elapsedVTime() * 1000.0;
2723
2724 // The G1CMTask class also extends the TerminatorTerminator class,
2725 // hence its should_exit_termination() method will also decide
2726 // whether to exit the termination protocol or not.
2727 bool finished = (is_serial ||
2728 _cm->terminator()->offer_termination(this));
2729 double termination_end_time_ms = os::elapsedVTime() * 1000.0;
2730 _termination_time_ms +=
2731 termination_end_time_ms - _termination_start_time_ms;
2732
2733 if (finished) {
2734 // We're all done.
2735
2736 if (_worker_id == 0) {
2737 // Let's allow task 0 to do this
2738 if (_concurrent) {
2739 assert(_cm->concurrent_marking_in_progress(), "invariant");
2740 // We need to set this to false before the next
2741 // safepoint. This way we ensure that the marking phase
2742 // doesn't observe any more heap expansions.
2743 _cm->clear_concurrent_marking_in_progress();
2744 }
2745 }
2746
2747 // We can now guarantee that the global stack is empty, since
2748 // all other tasks have finished. We separated the guarantees so
2749 // that, if a condition is false, we can immediately find out
2750 // which one.
2751 guarantee(_cm->out_of_regions(), "only way to reach here");
2752 guarantee(_cm->mark_stack_empty(), "only way to reach here");
2753 guarantee(_task_queue->size() == 0, "only way to reach here");
2754 guarantee(!_cm->has_overflown(), "only way to reach here");
2755 } else {
2756 // Apparently there's more work to do. Let's abort this task. It
2757 // will restart it and we can hopefully find more things to do.
2758 set_has_aborted();
2759 }
2760 }
2761
2762 // Mainly for debugging purposes to make sure that a pointer to the
2763 // closure which was statically allocated in this frame doesn't
2764 // escape it by accident.
2765 set_cm_oop_closure(NULL);
2766 double end_time_ms = os::elapsedVTime() * 1000.0;
2767 double elapsed_time_ms = end_time_ms - _start_time_ms;
2768 // Update the step history.
2769 _step_times_ms.add(elapsed_time_ms);
2770
2771 if (has_aborted()) {
2772 // The task was aborted for some reason.
2773 if (_has_timed_out) {
2774 double diff_ms = elapsed_time_ms - _time_target_ms;
2775 // Keep statistics of how well we did with respect to hitting
2776 // our target only if we actually timed out (if we aborted for
2777 // other reasons, then the results might get skewed).
2778 _marking_step_diffs_ms.add(diff_ms);
2779 }
2780
2781 if (_cm->has_overflown()) {
2782 // This is the interesting one. We aborted because a global
2783 // overflow was raised. This means we have to restart the
2784 // marking phase and start iterating over regions. However, in
2785 // order to do this we have to make sure that all tasks stop
2786 // what they are doing and re-initialize in a safe manner. We
2787 // will achieve this with the use of two barrier sync points.
2788
2789 if (!is_serial) {
2790 // We only need to enter the sync barrier if being called
2791 // from a parallel context
2792 _cm->enter_first_sync_barrier(_worker_id);
2793
2794 // When we exit this sync barrier we know that all tasks have
2795 // stopped doing marking work. So, it's now safe to
2796 // re-initialize our data structures. At the end of this method,
2797 // task 0 will clear the global data structures.
2798 }
2799
2800 // We clear the local state of this task...
2801 clear_region_fields();
2802
2803 if (!is_serial) {
2804 // ...and enter the second barrier.
2805 _cm->enter_second_sync_barrier(_worker_id);
2806 }
2807 // At this point, if we're during the concurrent phase of
2808 // marking, everything has been re-initialized and we're
2809 // ready to restart.
2810 }
2811 }
2812 }
2813
2814 G1CMTask::G1CMTask(uint worker_id, G1ConcurrentMark* cm, G1CMTaskQueue* task_queue) :
2815 _objArray_processor(this),
2816 _worker_id(worker_id),
2817 _g1h(G1CollectedHeap::heap()),
2818 _cm(cm),
2819 _next_mark_bitmap(NULL),
2820 _task_queue(task_queue),
2821 _calls(0),
2822 _time_target_ms(0.0),
2823 _start_time_ms(0.0),
2824 _cm_oop_closure(NULL),
2825 _curr_region(NULL),
2826 _finger(NULL),
2827 _region_limit(NULL),
2828 _words_scanned(0),
2829 _words_scanned_limit(0),
2830 _real_words_scanned_limit(0),
2831 _refs_reached(0),
2832 _refs_reached_limit(0),
2833 _real_refs_reached_limit(0),
2834 _hash_seed(17),
2835 _has_aborted(false),
2836 _has_timed_out(false),
2837 _draining_satb_buffers(false),
2838 _step_times_ms(),
2839 _elapsed_time_ms(0.0),
2840 _termination_time_ms(0.0),
2841 _termination_start_time_ms(0.0),
2842 _concurrent(false),
2843 _marking_step_diffs_ms()
2844 {
2845 guarantee(task_queue != NULL, "invariant");
2846
2847 _marking_step_diffs_ms.add(0.5);
2848 }
2849
2850 // These are formatting macros that are used below to ensure
2851 // consistent formatting. The *_H_* versions are used to format the
2852 // header for a particular value and they should be kept consistent
2853 // with the corresponding macro. Also note that most of the macros add
2854 // the necessary white space (as a prefix) which makes them a bit
2855 // easier to compose.
2856
2857 // All the output lines are prefixed with this string to be able to
2858 // identify them easily in a large log file.
2859 #define G1PPRL_LINE_PREFIX "###"
2860
2861 #define G1PPRL_ADDR_BASE_FORMAT " " PTR_FORMAT "-" PTR_FORMAT
2862 #ifdef _LP64
2863 #define G1PPRL_ADDR_BASE_H_FORMAT " %37s"
2864 #else // _LP64
2865 #define G1PPRL_ADDR_BASE_H_FORMAT " %21s"
2866 #endif // _LP64
2867
2868 // For per-region info
2869 #define G1PPRL_TYPE_FORMAT " %-4s"
2870 #define G1PPRL_TYPE_H_FORMAT " %4s"
2871 #define G1PPRL_BYTE_FORMAT " " SIZE_FORMAT_W(9)
2872 #define G1PPRL_BYTE_H_FORMAT " %9s"
2873 #define G1PPRL_DOUBLE_FORMAT " %14.1f"
2874 #define G1PPRL_DOUBLE_H_FORMAT " %14s"
2875
2876 // For summary info
2877 #define G1PPRL_SUM_ADDR_FORMAT(tag) " " tag ":" G1PPRL_ADDR_BASE_FORMAT
2878 #define G1PPRL_SUM_BYTE_FORMAT(tag) " " tag ": " SIZE_FORMAT
2879 #define G1PPRL_SUM_MB_FORMAT(tag) " " tag ": %1.2f MB"
2880 #define G1PPRL_SUM_MB_PERC_FORMAT(tag) G1PPRL_SUM_MB_FORMAT(tag) " / %1.2f %%"
2881
2882 G1PrintRegionLivenessInfoClosure::G1PrintRegionLivenessInfoClosure(const char* phase_name) :
2883 _total_used_bytes(0), _total_capacity_bytes(0),
2884 _total_prev_live_bytes(0), _total_next_live_bytes(0),
2885 _total_remset_bytes(0), _total_strong_code_roots_bytes(0)
2886 {
2887 G1CollectedHeap* g1h = G1CollectedHeap::heap();
2888 MemRegion g1_reserved = g1h->g1_reserved();
2889 double now = os::elapsedTime();
2890
2891 // Print the header of the output.
2892 log_trace(gc, liveness)(G1PPRL_LINE_PREFIX" PHASE %s @ %1.3f", phase_name, now);
2893 log_trace(gc, liveness)(G1PPRL_LINE_PREFIX" HEAP"
2894 G1PPRL_SUM_ADDR_FORMAT("reserved")
2895 G1PPRL_SUM_BYTE_FORMAT("region-size"),
2896 p2i(g1_reserved.start()), p2i(g1_reserved.end()),
2897 HeapRegion::GrainBytes);
2898 log_trace(gc, liveness)(G1PPRL_LINE_PREFIX);
2899 log_trace(gc, liveness)(G1PPRL_LINE_PREFIX
2900 G1PPRL_TYPE_H_FORMAT
2901 G1PPRL_ADDR_BASE_H_FORMAT
2902 G1PPRL_BYTE_H_FORMAT
2903 G1PPRL_BYTE_H_FORMAT
2904 G1PPRL_BYTE_H_FORMAT
2905 G1PPRL_DOUBLE_H_FORMAT
2906 G1PPRL_BYTE_H_FORMAT
2907 G1PPRL_BYTE_H_FORMAT,
2908 "type", "address-range",
2909 "used", "prev-live", "next-live", "gc-eff",
2910 "remset", "code-roots");
2911 log_trace(gc, liveness)(G1PPRL_LINE_PREFIX
2912 G1PPRL_TYPE_H_FORMAT
2913 G1PPRL_ADDR_BASE_H_FORMAT
2914 G1PPRL_BYTE_H_FORMAT
2915 G1PPRL_BYTE_H_FORMAT
2916 G1PPRL_BYTE_H_FORMAT
2917 G1PPRL_DOUBLE_H_FORMAT
2918 G1PPRL_BYTE_H_FORMAT
2919 G1PPRL_BYTE_H_FORMAT,
2920 "", "",
2921 "(bytes)", "(bytes)", "(bytes)", "(bytes/ms)",
2922 "(bytes)", "(bytes)");
2923 }
2924
2925 bool G1PrintRegionLivenessInfoClosure::do_heap_region(HeapRegion* r) {
2926 const char* type = r->get_type_str();
2927 HeapWord* bottom = r->bottom();
2928 HeapWord* end = r->end();
2929 size_t capacity_bytes = r->capacity();
2930 size_t used_bytes = r->used();
2931 size_t prev_live_bytes = r->live_bytes();
2932 size_t next_live_bytes = r->next_live_bytes();
2933 double gc_eff = r->gc_efficiency();
2934 size_t remset_bytes = r->rem_set()->mem_size();
2935 size_t strong_code_roots_bytes = r->rem_set()->strong_code_roots_mem_size();
2936
2937 _total_used_bytes += used_bytes;
2938 _total_capacity_bytes += capacity_bytes;
2939 _total_prev_live_bytes += prev_live_bytes;
2940 _total_next_live_bytes += next_live_bytes;
2941 _total_remset_bytes += remset_bytes;
2942 _total_strong_code_roots_bytes += strong_code_roots_bytes;
2943
2944 // Print a line for this particular region.
2945 log_trace(gc, liveness)(G1PPRL_LINE_PREFIX
2946 G1PPRL_TYPE_FORMAT
2947 G1PPRL_ADDR_BASE_FORMAT
2948 G1PPRL_BYTE_FORMAT
2949 G1PPRL_BYTE_FORMAT
2950 G1PPRL_BYTE_FORMAT
2951 G1PPRL_DOUBLE_FORMAT
2952 G1PPRL_BYTE_FORMAT
2953 G1PPRL_BYTE_FORMAT,
2954 type, p2i(bottom), p2i(end),
2955 used_bytes, prev_live_bytes, next_live_bytes, gc_eff,
2956 remset_bytes, strong_code_roots_bytes);
2957
2958 return false;
2959 }
2960
2961 G1PrintRegionLivenessInfoClosure::~G1PrintRegionLivenessInfoClosure() {
2962 // add static memory usages to remembered set sizes
2963 _total_remset_bytes += HeapRegionRemSet::fl_mem_size() + HeapRegionRemSet::static_mem_size();
2964 // Print the footer of the output.
2965 log_trace(gc, liveness)(G1PPRL_LINE_PREFIX);
2966 log_trace(gc, liveness)(G1PPRL_LINE_PREFIX
2967 " SUMMARY"
2968 G1PPRL_SUM_MB_FORMAT("capacity")
2969 G1PPRL_SUM_MB_PERC_FORMAT("used")
2970 G1PPRL_SUM_MB_PERC_FORMAT("prev-live")
2971 G1PPRL_SUM_MB_PERC_FORMAT("next-live")
2972 G1PPRL_SUM_MB_FORMAT("remset")
2973 G1PPRL_SUM_MB_FORMAT("code-roots"),
2974 bytes_to_mb(_total_capacity_bytes),
2975 bytes_to_mb(_total_used_bytes),
2976 percent_of(_total_used_bytes, _total_capacity_bytes),
2977 bytes_to_mb(_total_prev_live_bytes),
2978 percent_of(_total_prev_live_bytes, _total_capacity_bytes),
2979 bytes_to_mb(_total_next_live_bytes),
2980 percent_of(_total_next_live_bytes, _total_capacity_bytes),
2981 bytes_to_mb(_total_remset_bytes),
2982 bytes_to_mb(_total_strong_code_roots_bytes));
2983 }