--- old/src/share/vm/gc_implementation/g1/g1CollectedHeap.cpp 2014-02-18 10:51:50.519756662 +0100 +++ new/src/share/vm/gc_implementation/g1/g1CollectedHeap.cpp 2014-02-18 10:51:50.431756664 +0100 @@ -4534,7 +4534,96 @@ G1ParGCAllocBuffer::G1ParGCAllocBuffer(size_t gclab_word_size) : ParGCAllocBuffer(gclab_word_size), _retired(false) { } -G1ParScanThreadState::G1ParScanThreadState(G1CollectedHeap* g1h, uint queue_num) +inline oop G1ParScanThreadState::copy_to_survivor_space(oop old) { + size_t word_sz = old->size(); + HeapRegion* from_region = _g1h->heap_region_containing_raw(old); + // +1 to make the -1 indexes valid... + int young_index = from_region->young_index_in_cset()+1; + assert( (from_region->is_young() && young_index > 0) || + (!from_region->is_young() && young_index == 0), "invariant" ); + G1CollectorPolicy* g1p = _g1h->g1_policy(); + markOop m = old->mark(); + int age = m->has_displaced_mark_helper() ? m->displaced_mark_helper()->age() + : m->age(); + GCAllocPurpose alloc_purpose = g1p->evacuation_destination(from_region, age, + word_sz); + HeapWord* obj_ptr = allocate(alloc_purpose, word_sz); +#ifndef PRODUCT + // Should this evacuation fail? + if (_g1h->evacuation_should_fail()) { + if (obj_ptr != NULL) { + undo_allocation(alloc_purpose, obj_ptr, word_sz); + obj_ptr = NULL; + } + } +#endif // !PRODUCT + + if (obj_ptr == NULL) { + // This will either forward-to-self, or detect that someone else has + // installed a forwarding pointer. + return _g1h->handle_evacuation_failure_par(this, old); + } + + oop obj = oop(obj_ptr); + + // We're going to allocate linearly, so might as well prefetch ahead. + Prefetch::write(obj_ptr, PrefetchCopyIntervalInBytes); + + oop forward_ptr = old->forward_to_atomic(obj); + if (forward_ptr == NULL) { + Copy::aligned_disjoint_words((HeapWord*) old, obj_ptr, word_sz); + if (g1p->track_object_age(alloc_purpose)) { + // We could simply do obj->incr_age(). However, this causes a + // performance issue. obj->incr_age() will first check whether + // the object has a displaced mark by checking its mark word; + // getting the mark word from the new location of the object + // stalls. So, given that we already have the mark word and we + // are about to install it anyway, it's better to increase the + // age on the mark word, when the object does not have a + // displaced mark word. We're not expecting many objects to have + // a displaced marked word, so that case is not optimized + // further (it could be...) and we simply call obj->incr_age(). + + if (m->has_displaced_mark_helper()) { + // in this case, we have to install the mark word first, + // otherwise obj looks to be forwarded (the old mark word, + // which contains the forward pointer, was copied) + obj->set_mark(m); + obj->incr_age(); + } else { + m = m->incr_age(); + obj->set_mark(m); + } + age_table()->add(obj, word_sz); + } else { + obj->set_mark(m); + } + + size_t* surv_young_words = surviving_young_words(); + surv_young_words[young_index] += word_sz; + + if (!obj_needs_chunking(obj, word_sz)) { + // No point in using the slower heap_region_containing() method, + // given that we know obj is in the heap. + HeapRegion* obj_region = _g1h->heap_region_containing_raw(obj); + _scanner.set_region(obj_region); + obj->oop_iterate_backwards(&_scanner); + } else { + // We keep track of the next start index in the length field of + // the to-space object. The actual length can be found in the + // length field of the from-space object. + arrayOop(obj)->set_length(0); + oop* old_p = set_partial_array_mask(old); + push_on_queue(old_p); + } + } else { + undo_allocation(alloc_purpose, obj_ptr, word_sz); + obj = forward_ptr; + } + return obj; +} + +G1ParScanThreadState::G1ParScanThreadState(G1CollectedHeap* g1h, uint queue_num, ReferenceProcessor* rp) : _g1h(g1h), _refs(g1h->task_queue(queue_num)), _dcq(&g1h->dirty_card_queue_set()), @@ -4546,7 +4635,8 @@ _tenured_alloc_buffer(g1h->desired_plab_sz(GCAllocForTenured)), _age_table(false), _strong_roots_time(0), _term_time(0), - _alloc_buffer_waste(0), _undo_waste(0) { + _alloc_buffer_waste(0), _undo_waste(0), + _scanner(g1h, this, rp) { // we allocate G1YoungSurvRateNumRegions plus one entries, since // we "sacrifice" entry 0 to keep track of surviving bytes for // non-young regions (where the age is -1) @@ -4634,9 +4724,7 @@ #endif // ASSERT void G1ParScanThreadState::trim_queue() { - assert(_evac_cl != NULL, "not set"); assert(_evac_failure_cl != NULL, "not set"); - assert(_partial_scan_cl != NULL, "not set"); StarTask ref; do { @@ -4653,14 +4741,10 @@ G1ParClosureSuper::G1ParClosureSuper(G1CollectedHeap* g1, G1ParScanThreadState* par_scan_state) : - _g1(g1), _g1_rem(_g1->g1_rem_set()), _cm(_g1->concurrent_mark()), - _par_scan_state(par_scan_state), - _worker_id(par_scan_state->queue_num()), - _during_initial_mark(_g1->g1_policy()->during_initial_mark_pause()), - _mark_in_progress(_g1->mark_in_progress()) { } + _g1(g1), _par_scan_state(par_scan_state), + _worker_id(par_scan_state->queue_num()) { } -template -void G1ParCopyClosure::mark_object(oop obj) { +void G1ParCopyHelper::mark_object(oop obj) { #ifdef ASSERT HeapRegion* hr = _g1->heap_region_containing(obj); assert(hr != NULL, "sanity"); @@ -4671,9 +4755,7 @@ _cm->grayRoot(obj, (size_t) obj->size(), _worker_id); } -template -void G1ParCopyClosure - ::mark_forwarded_object(oop from_obj, oop to_obj) { +void G1ParCopyHelper::mark_forwarded_object(oop from_obj, oop to_obj) { #ifdef ASSERT assert(from_obj->is_forwarded(), "from obj should be forwarded"); assert(from_obj->forwardee() == to_obj, "to obj should be the forwardee"); @@ -4695,96 +4777,6 @@ _cm->grayRoot(to_obj, (size_t) from_obj->size(), _worker_id); } -template -oop G1ParCopyClosure - ::copy_to_survivor_space(oop old) { - size_t word_sz = old->size(); - HeapRegion* from_region = _g1->heap_region_containing_raw(old); - // +1 to make the -1 indexes valid... - int young_index = from_region->young_index_in_cset()+1; - assert( (from_region->is_young() && young_index > 0) || - (!from_region->is_young() && young_index == 0), "invariant" ); - G1CollectorPolicy* g1p = _g1->g1_policy(); - markOop m = old->mark(); - int age = m->has_displaced_mark_helper() ? m->displaced_mark_helper()->age() - : m->age(); - GCAllocPurpose alloc_purpose = g1p->evacuation_destination(from_region, age, - word_sz); - HeapWord* obj_ptr = _par_scan_state->allocate(alloc_purpose, word_sz); -#ifndef PRODUCT - // Should this evacuation fail? - if (_g1->evacuation_should_fail()) { - if (obj_ptr != NULL) { - _par_scan_state->undo_allocation(alloc_purpose, obj_ptr, word_sz); - obj_ptr = NULL; - } - } -#endif // !PRODUCT - - if (obj_ptr == NULL) { - // This will either forward-to-self, or detect that someone else has - // installed a forwarding pointer. - return _g1->handle_evacuation_failure_par(_par_scan_state, old); - } - - oop obj = oop(obj_ptr); - - // We're going to allocate linearly, so might as well prefetch ahead. - Prefetch::write(obj_ptr, PrefetchCopyIntervalInBytes); - - oop forward_ptr = old->forward_to_atomic(obj); - if (forward_ptr == NULL) { - Copy::aligned_disjoint_words((HeapWord*) old, obj_ptr, word_sz); - if (g1p->track_object_age(alloc_purpose)) { - // We could simply do obj->incr_age(). However, this causes a - // performance issue. obj->incr_age() will first check whether - // the object has a displaced mark by checking its mark word; - // getting the mark word from the new location of the object - // stalls. So, given that we already have the mark word and we - // are about to install it anyway, it's better to increase the - // age on the mark word, when the object does not have a - // displaced mark word. We're not expecting many objects to have - // a displaced marked word, so that case is not optimized - // further (it could be...) and we simply call obj->incr_age(). - - if (m->has_displaced_mark_helper()) { - // in this case, we have to install the mark word first, - // otherwise obj looks to be forwarded (the old mark word, - // which contains the forward pointer, was copied) - obj->set_mark(m); - obj->incr_age(); - } else { - m = m->incr_age(); - obj->set_mark(m); - } - _par_scan_state->age_table()->add(obj, word_sz); - } else { - obj->set_mark(m); - } - - size_t* surv_young_words = _par_scan_state->surviving_young_words(); - surv_young_words[young_index] += word_sz; - - if (obj->is_objArray() && arrayOop(obj)->length() >= ParGCArrayScanChunk) { - // We keep track of the next start index in the length field of - // the to-space object. The actual length can be found in the - // length field of the from-space object. - arrayOop(obj)->set_length(0); - oop* old_p = set_partial_array_mask(old); - _par_scan_state->push_on_queue(old_p); - } else { - // No point in using the slower heap_region_containing() method, - // given that we know obj is in the heap. - _scanner.set_region(_g1->heap_region_containing_raw(obj)); - obj->oop_iterate_backwards(&_scanner); - } - } else { - _par_scan_state->undo_allocation(alloc_purpose, obj_ptr, word_sz); - obj = forward_ptr; - } - return obj; -} - template void G1ParCopyHelper::do_klass_barrier(T* p, oop new_obj) { if (_g1->heap_region_containing_raw(new_obj)->is_young()) { @@ -4794,97 +4786,52 @@ template template -void G1ParCopyClosure -::do_oop_work(T* p) { - oop obj = oopDesc::load_decode_heap_oop(p); - - assert(_worker_id == _par_scan_state->queue_num(), "sanity"); - - // here the null check is implicit in the cset_fast_test() test - if (_g1->in_cset_fast_test(obj)) { - oop forwardee; - if (obj->is_forwarded()) { - forwardee = obj->forwardee(); +void G1ParCopyClosure::do_oop_work(T* p) { + T heap_oop = oopDesc::load_heap_oop(p); + + if (!oopDesc::is_null(heap_oop)) { + oop obj = oopDesc::decode_heap_oop_not_null(heap_oop); + + assert(_worker_id == _par_scan_state->queue_num(), "sanity"); + + if (_g1->in_cset_fast_test(obj)) { + oop forwardee; + if (obj->is_forwarded()) { + forwardee = obj->forwardee(); + } else { + forwardee = _par_scan_state->copy_to_survivor_space(obj); + } + assert(forwardee != NULL, "forwardee should not be NULL"); + oopDesc::encode_store_heap_oop(p, forwardee); + if (do_mark_object && forwardee != obj) { + // If the object is self-forwarded we don't need to explicitly + // mark it, the evacuation failure protocol will do so. + mark_forwarded_object(obj, forwardee); + } + + if (barrier == G1BarrierKlass) { + do_klass_barrier(p, forwardee); + } } else { - forwardee = copy_to_survivor_space(obj); - } - assert(forwardee != NULL, "forwardee should not be NULL"); - oopDesc::encode_store_heap_oop(p, forwardee); - if (do_mark_object && forwardee != obj) { - // If the object is self-forwarded we don't need to explicitly - // mark it, the evacuation failure protocol will do so. - mark_forwarded_object(obj, forwardee); + // The object is not in collection set. If we're a root scanning + // closure during an initial mark pause (i.e. do_mark_object will + // be true) then attempt to mark the object. + if (do_mark_object) { + assert(_g1->is_in_g1_reserved(obj), "Must reference an object within the heap"); + mark_object(obj); + } } - if (barrier == G1BarrierKlass) { - do_klass_barrier(p, forwardee); - } - } else { - // The object is not in collection set. If we're a root scanning - // closure during an initial mark pause (i.e. do_mark_object will - // be true) then attempt to mark the object. - if (do_mark_object && _g1->is_in_g1_reserved(obj)) { - mark_object(obj); + if (barrier == G1BarrierEvac) { + assert(obj != NULL, "Must be"); + _par_scan_state->update_rs(_from, p, _worker_id); } } - - if (barrier == G1BarrierEvac && obj != NULL) { - _par_scan_state->update_rs(_from, p, _worker_id); - } } template void G1ParCopyClosure::do_oop_work(oop* p); template void G1ParCopyClosure::do_oop_work(narrowOop* p); -template void G1ParScanPartialArrayClosure::do_oop_nv(T* p) { - assert(has_partial_array_mask(p), "invariant"); - oop from_obj = clear_partial_array_mask(p); - - assert(Universe::heap()->is_in_reserved(from_obj), "must be in heap."); - assert(from_obj->is_objArray(), "must be obj array"); - objArrayOop from_obj_array = objArrayOop(from_obj); - // The from-space object contains the real length. - int length = from_obj_array->length(); - - assert(from_obj->is_forwarded(), "must be forwarded"); - oop to_obj = from_obj->forwardee(); - assert(from_obj != to_obj, "should not be chunking self-forwarded objects"); - objArrayOop to_obj_array = objArrayOop(to_obj); - // We keep track of the next start index in the length field of the - // to-space object. - int next_index = to_obj_array->length(); - assert(0 <= next_index && next_index < length, - err_msg("invariant, next index: %d, length: %d", next_index, length)); - - int start = next_index; - int end = length; - int remainder = end - start; - // We'll try not to push a range that's smaller than ParGCArrayScanChunk. - if (remainder > 2 * ParGCArrayScanChunk) { - end = start + ParGCArrayScanChunk; - to_obj_array->set_length(end); - // Push the remainder before we process the range in case another - // worker has run out of things to do and can steal it. - oop* from_obj_p = set_partial_array_mask(from_obj); - _par_scan_state->push_on_queue(from_obj_p); - } else { - assert(length == end, "sanity"); - // We'll process the final range for this object. Restore the length - // so that the heap remains parsable in case of evacuation failure. - to_obj_array->set_length(end); - } - _scanner.set_region(_g1->heap_region_containing_raw(to_obj)); - // Process indexes [start,end). It will also process the header - // along with the first chunk (i.e., the chunk with start == 0). - // Note that at this point the length field of to_obj_array is not - // correct given that we are using it to keep track of the next - // start index. oop_iterate_range() (thankfully!) ignores the length - // field and only relies on the start / end parameters. It does - // however return the size of the object which will be incorrect. So - // we have to ignore it even if we wanted to use it. - to_obj_array->oop_iterate_range(&_scanner, start, end); -} - class G1ParEvacuateFollowersClosure : public VoidClosure { protected: G1CollectedHeap* _g1h; @@ -5025,14 +4972,10 @@ ReferenceProcessor* rp = _g1h->ref_processor_stw(); - G1ParScanThreadState pss(_g1h, worker_id); - G1ParScanHeapEvacClosure scan_evac_cl(_g1h, &pss, rp); + G1ParScanThreadState pss(_g1h, worker_id, rp); G1ParScanHeapEvacFailureClosure evac_failure_cl(_g1h, &pss, rp); - G1ParScanPartialArrayClosure partial_scan_cl(_g1h, &pss, rp); - pss.set_evac_closure(&scan_evac_cl); pss.set_evac_failure_closure(&evac_failure_cl); - pss.set_partial_scan_closure(&partial_scan_cl); G1ParScanExtRootClosure only_scan_root_cl(_g1h, &pss, rp); G1ParScanMetadataClosure only_scan_metadata_cl(_g1h, &pss, rp); @@ -5456,15 +5399,11 @@ G1STWIsAliveClosure is_alive(_g1h); - G1ParScanThreadState pss(_g1h, worker_id); + G1ParScanThreadState pss(_g1h, worker_id, NULL); - G1ParScanHeapEvacClosure scan_evac_cl(_g1h, &pss, NULL); G1ParScanHeapEvacFailureClosure evac_failure_cl(_g1h, &pss, NULL); - G1ParScanPartialArrayClosure partial_scan_cl(_g1h, &pss, NULL); - pss.set_evac_closure(&scan_evac_cl); pss.set_evac_failure_closure(&evac_failure_cl); - pss.set_partial_scan_closure(&partial_scan_cl); G1ParScanExtRootClosure only_copy_non_heap_cl(_g1h, &pss, NULL); G1ParScanMetadataClosure only_copy_metadata_cl(_g1h, &pss, NULL); @@ -5568,14 +5507,10 @@ ResourceMark rm; HandleMark hm; - G1ParScanThreadState pss(_g1h, worker_id); - G1ParScanHeapEvacClosure scan_evac_cl(_g1h, &pss, NULL); + G1ParScanThreadState pss(_g1h, worker_id, NULL); G1ParScanHeapEvacFailureClosure evac_failure_cl(_g1h, &pss, NULL); - G1ParScanPartialArrayClosure partial_scan_cl(_g1h, &pss, NULL); - pss.set_evac_closure(&scan_evac_cl); pss.set_evac_failure_closure(&evac_failure_cl); - pss.set_partial_scan_closure(&partial_scan_cl); assert(pss.refs()->is_empty(), "both queue and overflow should be empty"); @@ -5694,18 +5629,14 @@ // JNI refs. // Use only a single queue for this PSS. - G1ParScanThreadState pss(this, 0); + G1ParScanThreadState pss(this, 0, NULL); // We do not embed a reference processor in the copying/scanning // closures while we're actually processing the discovered // reference objects. - G1ParScanHeapEvacClosure scan_evac_cl(this, &pss, NULL); G1ParScanHeapEvacFailureClosure evac_failure_cl(this, &pss, NULL); - G1ParScanPartialArrayClosure partial_scan_cl(this, &pss, NULL); - pss.set_evac_closure(&scan_evac_cl); pss.set_evac_failure_closure(&evac_failure_cl); - pss.set_partial_scan_closure(&partial_scan_cl); assert(pss.refs()->is_empty(), "pre-condition");