--- old/src/share/vm/gc_implementation/g1/g1CollectedHeap.cpp 2011-12-20 11:11:14.043440125 -0800 +++ new/src/share/vm/gc_implementation/g1/g1CollectedHeap.cpp 2011-12-20 11:11:13.806508438 -0800 @@ -4054,7 +4054,7 @@ assert(_cm->isPrevMarked(obj), "Should be marked!"); _prev_marked_bytes += (obj_size * HeapWordSize); if (_g1->mark_in_progress() && !_g1->is_obj_ill(obj)) { - _cm->markAndGrayObjectIfNecessary(obj); + _cm->markAndGrayObjectIfNecessary(obj, 0 /* worker_i */); } obj->set_mark(markOopDesc::prototype()); // While we were processing RSet buffers during the @@ -4173,7 +4173,8 @@ oop G1CollectedHeap::handle_evacuation_failure_par(OopsInHeapRegionClosure* cl, oop old, - bool should_mark_root) { + bool should_mark_root, + int worker_i) { assert(obj_in_cs(old), err_msg("obj: "PTR_FORMAT" should still be in the CSet", (HeapWord*) old)); @@ -4188,7 +4189,7 @@ // object is also responsible for marking the object. if (should_mark_root) { assert(!oopDesc::is_null(old), "shouldn't be"); - _cm->grayRoot(old); + _cm->grayRoot(old, worker_i); } if (_evac_failure_closure != cl) { @@ -4294,10 +4295,45 @@ } #endif // PRODUCT -G1ParGCAllocBuffer::G1ParGCAllocBuffer(size_t gclab_word_size) : +void GCLabBitMap::retire(int worker_i) { + guarantee(use_local_bitmaps, "invariant"); + assert(fields_well_formed(), "invariant"); + + if (_start_word != NULL) { + CMBitMap* mark_bitmap = _cm->nextMarkBitMap(); + + // this means that the bitmap was set up for the GCLab + assert(_real_start_word != NULL && _real_end_word != NULL, "invariant"); + + mark_bitmap->mostly_disjoint_range_union(this, + 0, // always start from the start of the bitmap + _start_word, + gclab_real_word_size()); + + // Note: Even though that not all objects copied into the LAB will + // have their bit set in the LAB bitmap (the LAB bitmap is used to + // propagate marks), we can just add the entire lab and its bitmap + // to the count of the marked data. It's OK (but inaccurate) to + // count a dead object but we can't miss counting a live object. + MemRegion lab_region(_real_start_word, _real_end_word); + _cm->count_region(lab_region, worker_i); + _cm->grayRegionIfNecessary(lab_region); + +#ifndef PRODUCT + if (use_local_bitmaps && verify_local_bitmaps) { + verify(); + } +#endif // PRODUCT + } else { + assert(_real_start_word == NULL && _real_end_word == NULL, "invariant"); + } +} + +G1ParGCAllocBuffer::G1ParGCAllocBuffer(size_t gclab_word_size, int worker_i) : ParGCAllocBuffer(gclab_word_size), _should_mark_objects(false), _bitmap(G1CollectedHeap::heap()->reserved_region().start(), gclab_word_size), + _worker_i(worker_i), _retired(false) { //_should_mark_objects is set to true when G1ParCopyHelper needs to @@ -4319,8 +4355,8 @@ _g1_rem(g1h->g1_rem_set()), _hash_seed(17), _queue_num(queue_num), _term_attempts(0), - _surviving_alloc_buffer(g1h->desired_plab_sz(GCAllocForSurvived)), - _tenured_alloc_buffer(g1h->desired_plab_sz(GCAllocForTenured)), + _surviving_alloc_buffer(g1h->desired_plab_sz(GCAllocForSurvived), queue_num), + _tenured_alloc_buffer(g1h->desired_plab_sz(GCAllocForTenured), queue_num), _age_table(false), _strong_roots_time(0), _term_time(0), _alloc_buffer_waste(0), _undo_waste(0) @@ -4432,6 +4468,7 @@ G1ParClosureSuper::G1ParClosureSuper(G1CollectedHeap* g1, G1ParScanThreadState* par_scan_state) : _g1(g1), _g1_rem(_g1->g1_rem_set()), _cm(_g1->concurrent_mark()), _par_scan_state(par_scan_state), + _worker_i(par_scan_state->queue_num()), _during_initial_mark(_g1->g1_policy()->during_initial_mark_pause()), _mark_in_progress(_g1->mark_in_progress()) { } @@ -4445,7 +4482,7 @@ oop obj = oopDesc::decode_heap_oop(heap_oop); HeapWord* addr = (HeapWord*)obj; if (_g1->is_in_g1_reserved(addr)) { - _cm->grayRoot(oop(addr)); + _cm->grayRoot(oop(addr), _worker_i); } } } @@ -4471,7 +4508,7 @@ // This will either forward-to-self, or detect that someone else has // installed a forwarding pointer. OopsInHeapRegionClosure* cl = _par_scan_state->evac_failure_closure(); - return _g1->handle_evacuation_failure_par(cl, old, should_mark_root); + return _g1->handle_evacuation_failure_par(cl, old, should_mark_root, _worker_i); } // We're going to allocate linearly, so might as well prefetch ahead. @@ -4514,13 +4551,29 @@ // if we couldn't mark it on the local bitmap (this happens when // the object was not allocated in the GCLab), we have to bite // the bullet and do the standard parallel mark - _cm->markAndGrayObjectIfNecessary(obj); + _cm->markAndGrayObjectIfNecessary(obj, _worker_i); } if (_g1->isMarkedNext(old)) { // Unmark the object's old location so that marking // doesn't think the old object is alive. _cm->nextMarkBitMap()->parClear((HeapWord*)old); + + // We could clear the count data for the old object here but + // currently we do not. Why don't we do this? The thread/task + // that marks a newly copied object is likely _not_ the thread/task + // that originally marked the old object. So, to clear the count + // data for the old object, we would have to scan the count + // data for all of the tasks (and clear the data for the old object + // in parallel with other threads adding to the count data). Even + // then we could clear a bit incorrectly (e.g. if the old object + // does not start or end on a card boundary). It's more important + // that we don't have missed bits that should've been set than + // having extra bits set. + // + // As a result the accumulated count data could be a superset + // of the data that is/would have been calculated by walking + // the marking bitmap. } } @@ -4607,7 +4660,8 @@ } // When scanning the RS, we only care about objs in CS. if (barrier == G1BarrierRS) { - _par_scan_state->update_rs(_from, p, _par_scan_state->queue_num()); + assert(_worker_i == _par_scan_state->queue_num(), "sanity"); + _par_scan_state->update_rs(_from, p, _worker_i); } } else { // The object is not in collection set. If we're a root scanning @@ -4619,7 +4673,8 @@ } if (barrier == G1BarrierEvac && obj != NULL) { - _par_scan_state->update_rs(_from, p, _par_scan_state->queue_num()); + assert(_worker_i == _par_scan_state->queue_num(), "sanity"); + _par_scan_state->update_rs(_from, p, _worker_i); } if (do_gen_barrier && obj != NULL) { @@ -5843,8 +5898,9 @@ if (mark_in_progress()) { assert(!g1_policy()->during_initial_mark_pause(), "sanity"); _cm->nextMarkBitMap()->clearRange(used_mr); + // Need to remove values from the count info + _cm->clear_count_data_for_heap_region(cur); } - free_region(cur, &pre_used, &local_free_list, false /* par */); } else { cur->uninstall_surv_rate_group();