--- old/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp 2020-06-01 12:25:21.725030095 +0200 +++ new/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp 2020-06-01 12:25:21.409030507 +0200 @@ -1660,7 +1660,7 @@ ShenandoahConcurrentRootsEvacUpdateTask(ShenandoahPhaseTimings::Phase phase) : AbstractGangTask("Shenandoah Evacuate/Update Concurrent Strong Roots Task"), _vm_roots(phase), - _cld_roots(phase) {} + _cld_roots(phase, ShenandoahHeap::heap()->workers()->active_workers()) {} void work(uint worker_id) { ShenandoahConcurrentWorkerSession worker_session(worker_id); @@ -1781,7 +1781,7 @@ _string_table_roots(OopStorageSet::string_table_weak(), phase, ShenandoahPhaseTimings::StringTableRoots), _resolved_method_table_roots(OopStorageSet::resolved_method_table_weak(), phase, ShenandoahPhaseTimings::ResolvedMethodTableRoots), _vm_roots(OopStorageSet::vm_weak(), phase, ShenandoahPhaseTimings::VMWeakRoots), - _cld_roots(phase), + _cld_roots(phase, ShenandoahHeap::heap()->workers()->active_workers()), _nmethod_itr(ShenandoahCodeRoots::table()), _concurrent_class_unloading(ShenandoahConcurrentRoots::should_do_concurrent_class_unloading()) { StringTable::reset_dead_counter(); --- old/src/hotspot/share/gc/shenandoah/shenandoahRootProcessor.cpp 2020-06-01 12:25:22.525029055 +0200 +++ new/src/hotspot/share/gc/shenandoah/shenandoahRootProcessor.cpp 2020-06-01 12:25:22.213029460 +0200 @@ -202,7 +202,7 @@ _code_roots(phase), _vm_roots(phase), _dedup_roots(phase), - _cld_roots(phase) { + _cld_roots(phase, n_workers) { } void ShenandoahRootScanner::roots_do(uint worker_id, OopClosure* oops) { @@ -234,9 +234,9 @@ // Process light-weight/limited parallel roots then _vm_roots.oops_do(oops, worker_id); _dedup_roots.oops_do(&always_true, oops, worker_id); + _cld_roots.cld_do(clds, worker_id); // Process heavy-weight/fully parallel roots the last - _cld_roots.cld_do(clds, worker_id); _thread_roots.threads_do(&tc_cl, worker_id); } @@ -251,9 +251,9 @@ // Process light-weight/limited parallel roots then _vm_roots.oops_do(oops, worker_id); + _cld_roots.always_strong_cld_do(clds, worker_id); // Process heavy-weight/fully parallel roots the last - _cld_roots.always_strong_cld_do(clds, worker_id); _thread_roots.threads_do(&tc_cl, worker_id); } @@ -264,7 +264,7 @@ ShenandoahRootProcessor(phase), _serial_roots(phase), _vm_roots(phase), - _cld_roots(phase), + _cld_roots(phase, n_workers), _thread_roots(phase, n_workers > 1), _serial_weak_roots(phase), _weak_roots(phase), @@ -292,11 +292,13 @@ _weak_roots.oops_do(oops, worker_id); _dedup_roots.oops_do(&always_true, oops, worker_id); } - - // Process heavy-weight/fully parallel roots the last if (_stw_class_unloading) { CLDToOopClosure clds(oops, ClassLoaderData::_claim_strong); _cld_roots.cld_do(&clds, worker_id); + } + + // Process heavy-weight/fully parallel roots the last + if (_stw_class_unloading) { _code_roots.code_blobs_do(codes_cl, worker_id); _thread_roots.oops_do(oops, NULL, worker_id); } else { @@ -308,7 +310,7 @@ ShenandoahRootProcessor(phase), _serial_roots(phase), _vm_roots(phase), - _cld_roots(phase), + _cld_roots(phase, n_workers), _thread_roots(phase, n_workers > 1), _serial_weak_roots(phase), _weak_roots(phase), @@ -320,7 +322,7 @@ ShenandoahRootProcessor(phase), _serial_roots(phase), _vm_roots(phase), - _cld_roots(phase), + _cld_roots(phase, n_workers), _thread_roots(phase, n_workers > 1), _serial_weak_roots(phase), _weak_roots(phase), @@ -346,9 +348,9 @@ _vm_roots.oops_do(oops, worker_id); _weak_roots.oops_do(oops, worker_id); _dedup_roots.oops_do(&always_true, oops, worker_id); + _cld_roots.cld_do(&adjust_cld_closure, worker_id); // Process heavy-weight/fully parallel roots the last - _cld_roots.cld_do(&adjust_cld_closure, worker_id); _code_roots.code_blobs_do(adjust_code_closure, worker_id); _thread_roots.oops_do(oops, NULL, worker_id); } @@ -358,7 +360,7 @@ _serial_roots(ShenandoahPhaseTimings::heap_iteration_roots), _thread_roots(ShenandoahPhaseTimings::heap_iteration_roots, false /*is par*/), _vm_roots(ShenandoahPhaseTimings::heap_iteration_roots), - _cld_roots(ShenandoahPhaseTimings::heap_iteration_roots), + _cld_roots(ShenandoahPhaseTimings::heap_iteration_roots, 1), _serial_weak_roots(ShenandoahPhaseTimings::heap_iteration_roots), _weak_roots(ShenandoahPhaseTimings::heap_iteration_roots), _code_roots(ShenandoahPhaseTimings::heap_iteration_roots) { @@ -382,9 +384,9 @@ _vm_roots.oops_do(oops, 0); _weak_roots.oops_do(oops, 0); _dedup_roots.oops_do(&always_true, oops, 0); + _cld_roots.cld_do(&clds, 0); // Process heavy-weight/fully parallel roots the last - _cld_roots.cld_do(&clds, 0); _code_roots.code_blobs_do(&code, 0); _thread_roots.threads_do(&tc_cl, 0); } --- old/src/hotspot/share/gc/shenandoah/shenandoahRootProcessor.hpp 2020-06-01 12:25:23.301028046 +0200 +++ new/src/hotspot/share/gc/shenandoah/shenandoahRootProcessor.hpp 2020-06-01 12:25:22.985028456 +0200 @@ -226,9 +226,18 @@ template class ShenandoahClassLoaderDataRoots { private: + ShenandoahSharedSemaphore _semaphore; ShenandoahPhaseTimings::Phase _phase; + + static uint worker_count(uint n_workers) { + // Limit concurrency a bit, otherwise it wastes resources when workers are tripping + // over each other. This also leaves free workers to process other parts of the root + // set, while admitted workers are busy with doing the CLDG walk. + return MAX2(1u, MIN2(ShenandoahSharedSemaphore::max_tokens(), n_workers / 2)); + } + public: - ShenandoahClassLoaderDataRoots(ShenandoahPhaseTimings::Phase phase); + ShenandoahClassLoaderDataRoots(ShenandoahPhaseTimings::Phase phase, uint n_workers); ~ShenandoahClassLoaderDataRoots(); void always_strong_cld_do(CLDClosure* clds, uint worker_id); --- old/src/hotspot/share/gc/shenandoah/shenandoahRootProcessor.inline.hpp 2020-06-01 12:25:24.069027049 +0200 +++ new/src/hotspot/share/gc/shenandoah/shenandoahRootProcessor.inline.hpp 2020-06-01 12:25:23.757027454 +0200 @@ -122,7 +122,8 @@ } template -ShenandoahClassLoaderDataRoots::ShenandoahClassLoaderDataRoots(ShenandoahPhaseTimings::Phase phase) : +ShenandoahClassLoaderDataRoots::ShenandoahClassLoaderDataRoots(ShenandoahPhaseTimings::Phase phase, uint n_workers) : + _semaphore(worker_count(n_workers)), _phase(phase) { if (!SINGLE_THREADED) { ClassLoaderDataGraph::clear_claimed_marks(); @@ -146,9 +147,10 @@ assert(SafepointSynchronize::is_at_safepoint(), "Must be at a safepoint"); assert(Thread::current()->is_VM_thread(), "Single threaded CLDG iteration can only be done by VM thread"); ClassLoaderDataGraph::always_strong_cld_do(clds); - } else { + } else if (_semaphore.try_acquire()) { ShenandoahWorkerTimingsTracker timer(_phase, ShenandoahPhaseTimings::CLDGRoots, worker_id); ClassLoaderDataGraph::always_strong_cld_do(clds); + _semaphore.claim_all(); } } @@ -158,9 +160,10 @@ assert(SafepointSynchronize::is_at_safepoint(), "Must be at a safepoint"); assert(Thread::current()->is_VM_thread(), "Single threaded CLDG iteration can only be done by VM thread"); ClassLoaderDataGraph::cld_do(clds); - } else { + } else if (_semaphore.try_acquire()) { ShenandoahWorkerTimingsTracker timer(_phase, ShenandoahPhaseTimings::CLDGRoots, worker_id); ClassLoaderDataGraph::cld_do(clds); + _semaphore.claim_all(); } } --- old/src/hotspot/share/gc/shenandoah/shenandoahSharedVariables.hpp 2020-06-01 12:25:24.849026038 +0200 +++ new/src/hotspot/share/gc/shenandoah/shenandoahSharedVariables.hpp 2020-06-01 12:25:24.533026448 +0200 @@ -245,4 +245,38 @@ }; +typedef struct ShenandoahSharedSemaphore { + shenandoah_padding(0); + volatile ShenandoahSharedValue value; + shenandoah_padding(1); + + static uint max_tokens() { + return sizeof(ShenandoahSharedValue) * CHAR_MAX; + } + + ShenandoahSharedSemaphore(uint tokens) { + assert(tokens <= max_tokens(), "sanity"); + Atomic::release_store_fence(&value, (ShenandoahSharedValue)tokens); + } + + bool try_acquire() { + while (true) { + ShenandoahSharedValue ov = Atomic::load_acquire(&value); + if (ov == 0) { + return false; + } + ShenandoahSharedValue nv = ov - 1; + if (Atomic::cmpxchg(&value, ov, nv) == ov) { + // successfully set + return true; + } + } + } + + void claim_all() { + Atomic::release_store_fence(&value, (ShenandoahSharedValue)0); + } + +} ShenandoahSharedSemaphore; + #endif // SHARE_GC_SHENANDOAH_SHENANDOAHSHAREDVARIABLES_HPP