--- old/src/share/vm/code/codeCache.cpp 2014-04-14 10:01:23.667015785 +0200 +++ new/src/share/vm/code/codeCache.cpp 2014-04-14 10:01:22.863015755 +0200 @@ -23,6 +23,7 @@ */ #include "precompiled.hpp" +#include "c1/c1_Compilation.hpp" #include "code/codeBlob.hpp" #include "code/codeCache.hpp" #include "code/compiledIC.hpp" @@ -39,17 +40,18 @@ #include "oops/method.hpp" #include "oops/objArrayOop.hpp" #include "oops/oop.inline.hpp" +#include "opto/compile.hpp" #include "runtime/handles.inline.hpp" #include "runtime/arguments.hpp" #include "runtime/icache.hpp" #include "runtime/java.hpp" #include "runtime/mutexLocker.hpp" +#include "runtime/compilationPolicy.hpp" #include "services/memoryService.hpp" #include "trace/tracing.hpp" #include "utilities/xmlstream.hpp" // Helper class for printing in CodeCache - class CodeBlob_sizes { private: int count; @@ -115,64 +117,200 @@ } }; -// CodeCache implementation +// Iterate over all CodeHeaps +#define FOR_ALL_HEAPS(it) for (GrowableArrayIterator it = _heaps->begin(); it != _heaps->end(); ++it) +// Iterate over all CodeHeaps containing nmethods +#define FOR_ALL_METHOD_HEAPS(it) for (GrowableArrayFilterIterator it(_heaps->begin(), IsMethodPredicate()); it != _heaps->end(); ++it) +// Iterate over all CodeBlobs (cb) on the given CodeHeap +#define FOR_ALL_BLOBS(cb, heap) for (CodeBlob* cb = first_blob(heap); cb != NULL; cb = next_blob(heap, cb)) +// Iterate over all alive CodeBlobs (cb) on the given CodeHeap +#define FOR_ALL_ALIVE_BLOBS(cb, heap) for (CodeBlob* cb = first_alive_blob(heap); cb != NULL; cb = next_alive_blob(heap, cb)) -CodeHeap * CodeCache::_heap = new CodeHeap(); +address CodeCache::_low_bound = 0; +address CodeCache::_high_bound = 0; int CodeCache::_number_of_blobs = 0; int CodeCache::_number_of_adapters = 0; int CodeCache::_number_of_nmethods = 0; int CodeCache::_number_of_nmethods_with_dependencies = 0; bool CodeCache::_needs_cache_clean = false; nmethod* CodeCache::_scavenge_root_nmethods = NULL; - int CodeCache::_codemem_full_count = 0; -CodeBlob* CodeCache::first() { - assert_locked_or_safepoint(CodeCache_lock); - return (CodeBlob*)_heap->first(); +// Initialize array of CodeHeaps +GrowableArray* CodeCache::_heaps = new(ResourceObj::C_HEAP, mtCode) GrowableArray (3, true); + +void CodeCache::initialize_heaps() { + // Calculate default CodeHeap sizes if not set by user + if (FLAG_IS_DEFAULT(NonMethodCodeHeapSize) && FLAG_IS_DEFAULT(ProfiledCodeHeapSize) + && FLAG_IS_DEFAULT(NonProfiledCodeHeapSize)) { + // Number of c1/c2 compiler threads + const int c1_count = CompilationPolicy::policy()->compiler_count(CompLevel_simple); + const int c2_count = CompilationPolicy::policy()->compiler_count(CompLevel_full_optimization); + + // C1 temporary code buffer size (see Compiler::init_buffer_blob()) + const int c1_buffer_size = Compilation::desired_max_code_buffer_size() + Compilation::desired_max_constant_size(); + + // C2 scratch buffer size (see Compile::init_scratch_buffer_blob()) + // Initial size of constant table (this may be increased if a compiled method needs more space) + const int constant_size = (4 * 1024); + const int c2_buffer_size = Compile::MAX_inst_size + Compile::MAX_locs_size + constant_size; + + // Increase default NonMethodCodeHeapSize to account for buffers + int total_buffer_size = c1_count * c1_buffer_size + c2_count * c2_buffer_size; + FLAG_SET_DEFAULT(NonMethodCodeHeapSize, NonMethodCodeHeapSize + total_buffer_size); + + // Check if we have enough space for the non-method code heap + if (ReservedCodeCacheSize > NonMethodCodeHeapSize) { + // Use the default value for NonMethodCodeHeapSize and one half of the + // remaining size for non-profiled methods and one half for profiled methods + size_t remaining_size = ReservedCodeCacheSize - NonMethodCodeHeapSize; + size_t profiled_size = remaining_size / 2; + size_t non_profiled_size = remaining_size - profiled_size; + FLAG_SET_DEFAULT(ProfiledCodeHeapSize, profiled_size); + FLAG_SET_DEFAULT(NonProfiledCodeHeapSize, non_profiled_size); + } else { + // Use all space for the non-method heap and set other heaps to minimal size + FLAG_SET_DEFAULT(NonMethodCodeHeapSize, ReservedCodeCacheSize - os::vm_page_size() * 2); + FLAG_SET_DEFAULT(ProfiledCodeHeapSize, os::vm_page_size()); + FLAG_SET_DEFAULT(NonProfiledCodeHeapSize, os::vm_page_size()); + } + } + + // We do not need the profiled CodeHeap, use all space for the non-profiled CodeHeap + if(!heap_available(CodeBlobType::MethodProfiled)) { + FLAG_SET_DEFAULT(NonProfiledCodeHeapSize, NonProfiledCodeHeapSize + ProfiledCodeHeapSize); + FLAG_SET_DEFAULT(ProfiledCodeHeapSize, 0); + } + + // Size check + guarantee(NonProfiledCodeHeapSize + ProfiledCodeHeapSize + NonMethodCodeHeapSize <= ReservedCodeCacheSize, "Size check"); + + // Align reserved sizes of CodeHeaps + size_t non_method_size = ReservedCodeSpace::allocation_align_size_up(NonMethodCodeHeapSize); + size_t profiled_size = ReservedCodeSpace::allocation_align_size_up(ProfiledCodeHeapSize); + size_t non_profiled_size = ReservedCodeSpace::allocation_align_size_up(NonProfiledCodeHeapSize); + + // Compute initial sizes of CodeHeaps + size_t init_non_method_size = MIN2(InitialCodeCacheSize, non_method_size); + size_t init_profiled_size = MIN2(InitialCodeCacheSize, profiled_size); + size_t init_non_profiled_size = MIN2(InitialCodeCacheSize, non_profiled_size); + + // Reserve one continuous chunk of memory for CodeHeaps and split it into + // parts for the individual heaps. The memory layout looks like this: + // ---------- high ----------- + // Non-profiled nmethods + // Profiled nmethods + // Non-methods + // ---------- low ------------ + ReservedCodeSpace rs = reserve_heap_memory(non_profiled_size + profiled_size + non_method_size); + ReservedSpace non_method_space = rs.first_part(non_method_size); + ReservedSpace rest = rs.last_part(non_method_size); + ReservedSpace profiled_space = rest.first_part(profiled_size); + ReservedSpace non_profiled_space = rest.last_part(profiled_size); + + // Non-methods (stubs, adapters, ...) + add_heap(non_method_space, "Non-methods", init_non_method_size, CodeBlobType::NonMethod); + // Tier 2 and tier 3 (profiled) methods + add_heap(profiled_space, "Profiled nmethods", init_profiled_size, CodeBlobType::MethodProfiled); + // Tier 1 and tier 4 (non-profiled) methods and native methods + add_heap(non_profiled_space, "Non-profiled nmethods", init_non_profiled_size, CodeBlobType::MethodNonProfiled); } +ReservedCodeSpace CodeCache::reserve_heap_memory(size_t size) { + // Determine alignment + const size_t page_size = os::can_execute_large_page_memory() ? + os::page_size_for_region(InitialCodeCacheSize, size, 8) : + os::vm_page_size(); + const size_t granularity = os::vm_allocation_granularity(); + const size_t r_align = MAX2(page_size, granularity); + const size_t r_size = align_size_up(size, r_align); + const size_t rs_align = page_size == (size_t) os::vm_page_size() ? 0 : + MAX2(page_size, granularity); -CodeBlob* CodeCache::next(CodeBlob* cb) { - assert_locked_or_safepoint(CodeCache_lock); - return (CodeBlob*)_heap->next(cb); + ReservedCodeSpace rs(r_size, rs_align, rs_align > 0); + + // Initialize bounds + _low_bound = (address)rs.base(); + _high_bound = _low_bound + rs.size(); + + return rs; } +bool CodeCache::heap_available(int code_blob_type) { + if (TieredCompilation || code_blob_type == CodeBlobType::NonMethod) { + // Use all heaps for TieredCompilation + return true; + } else { + // Without TieredCompilation we only need the non-profiled heap + return (code_blob_type == CodeBlobType::MethodNonProfiled); + } +} -CodeBlob* CodeCache::alive(CodeBlob *cb) { - assert_locked_or_safepoint(CodeCache_lock); - while (cb != NULL && !cb->is_alive()) cb = next(cb); - return cb; +void CodeCache::add_heap(ReservedSpace rs, const char* name, size_t size_initial, int code_blob_type) { + // Check if heap is needed + if (!heap_available(code_blob_type)) { + return; + } + + // Create CodeHeap + CodeHeap* heap = new CodeHeap(name, code_blob_type); + _heaps->append(heap); + + // Reserve Space + size_initial = round_to(size_initial, os::vm_page_size()); + + if (!heap->reserve(rs, size_initial, CodeCacheSegmentSize)) { + vm_exit_during_initialization("Could not reserve enough space for code cache"); + } + + // Register the CodeHeap + MemoryService::add_code_heap_memory_pool(heap, name); } +CodeHeap* CodeCache::get_code_heap(int code_blob_type) { + FOR_ALL_HEAPS(it) { + if ((*it)->accepts(code_blob_type)) { + return (*it); + } + } + return NULL; +} -nmethod* CodeCache::alive_nmethod(CodeBlob* cb) { +CodeBlob* CodeCache::first_blob(CodeHeap* heap) { assert_locked_or_safepoint(CodeCache_lock); - while (cb != NULL && (!cb->is_alive() || !cb->is_nmethod())) cb = next(cb); - return (nmethod*)cb; + if (heap != NULL) { + return (CodeBlob*)heap->first(); + } + return NULL; } -nmethod* CodeCache::first_nmethod() { +CodeBlob* CodeCache::next_blob(CodeHeap* heap, CodeBlob* cb) { assert_locked_or_safepoint(CodeCache_lock); - CodeBlob* cb = first(); - while (cb != NULL && !cb->is_nmethod()) { - cb = next(cb); + if (heap != NULL) { + return (CodeBlob*)heap->next(cb); } - return (nmethod*)cb; + return NULL; } -nmethod* CodeCache::next_nmethod (CodeBlob* cb) { +CodeBlob* CodeCache::first_alive_blob(CodeHeap* heap) { assert_locked_or_safepoint(CodeCache_lock); - cb = next(cb); - while (cb != NULL && !cb->is_nmethod()) { - cb = next(cb); + CodeBlob* cb = first_blob(heap); + while (cb != NULL && !cb->is_alive()) { + cb = next_blob(heap, cb); } - return (nmethod*)cb; + return cb; } -static size_t maxCodeCacheUsed = 0; +CodeBlob* CodeCache::next_alive_blob(CodeHeap* heap, CodeBlob* cb) { + assert_locked_or_safepoint(CodeCache_lock); + cb = next_blob(heap, cb); + while (cb != NULL && !cb->is_alive()) { + cb = next_blob(heap, cb); + } + return cb; +} -CodeBlob* CodeCache::allocate(int size, bool is_critical) { +CodeBlob* CodeCache::allocate(int size, int code_blob_type, bool is_critical) { // Do not seize the CodeCache lock here--if the caller has not // already done so, we are going to lose bigtime, since the code // cache will contain a garbage CodeBlob until the caller can @@ -182,27 +320,31 @@ assert_locked_or_safepoint(CodeCache_lock); CodeBlob* cb = NULL; _number_of_blobs++; + + // Get CodeHeap for the given CodeBlobType + CodeHeap* heap = get_code_heap(code_blob_type); + assert (heap != NULL, "Heap exists"); + while (true) { - cb = (CodeBlob*)_heap->allocate(size, is_critical); + cb = (CodeBlob*)heap->allocate(size, is_critical); if (cb != NULL) break; - if (!_heap->expand_by(CodeCacheExpansionSize)) { + if (!heap->expand_by(CodeCacheExpansionSize)) { // Expansion failed return NULL; } if (PrintCodeCacheExtension) { ResourceMark rm; - tty->print_cr("code cache extended to [" INTPTR_FORMAT ", " INTPTR_FORMAT "] (%d bytes)", - (intptr_t)_heap->low_boundary(), (intptr_t)_heap->high(), - (address)_heap->high() - (address)_heap->low_boundary()); + tty->print_cr("CodeHeap '%s' extended to [" INTPTR_FORMAT ", " INTPTR_FORMAT "] (%d bytes)", + heap->name(), (intptr_t)heap->low_boundary(), (intptr_t)heap->high(), + (address)heap->high() - (address)heap->low_boundary()); } } - maxCodeCacheUsed = MAX2(maxCodeCacheUsed, ((address)_heap->high_boundary() - - (address)_heap->low_boundary()) - unallocated_capacity()); print_trace("allocation", cb, size); + return cb; } -void CodeCache::free(CodeBlob* cb) { +void CodeCache::free(CodeBlob* cb, int code_blob_type) { assert_locked_or_safepoint(CodeCache_lock); print_trace("free", cb); @@ -217,12 +359,12 @@ } _number_of_blobs--; - _heap->deallocate(cb); + // Get heap for given CodeBlobType and deallocate + get_code_heap(code_blob_type)->deallocate(cb); assert(_number_of_blobs >= 0, "sanity check"); } - void CodeCache::commit(CodeBlob* cb) { // this is called by nmethod::nmethod, which must already own CodeCache_lock assert_locked_or_safepoint(CodeCache_lock); @@ -240,88 +382,115 @@ ICache::invalidate_range(cb->content_begin(), cb->content_size()); } - -// Iteration over CodeBlobs - -#define FOR_ALL_BLOBS(var) for (CodeBlob *var = first() ; var != NULL; var = next(var) ) -#define FOR_ALL_ALIVE_BLOBS(var) for (CodeBlob *var = alive(first()); var != NULL; var = alive(next(var))) -#define FOR_ALL_ALIVE_NMETHODS(var) for (nmethod *var = alive_nmethod(first()); var != NULL; var = alive_nmethod(next(var))) - - bool CodeCache::contains(void *p) { // It should be ok to call contains without holding a lock - return _heap->contains(p); + FOR_ALL_HEAPS(it) { + if ((*it)->contains(p)) { + return true; + } + } + return false; } - -// This method is safe to call without holding the CodeCache_lock, as long as a dead codeblob is not -// looked up (i.e., one that has been marked for deletion). It only dependes on the _segmap to contain +// This method is safe to call without holding the CodeCache_lock, as long as a dead CodeBlob is not +// looked up (i.e., one that has been marked for deletion). It only depends on the _segmap to contain // valid indices, which it will always do, as long as the CodeBlob is not in the process of being recycled. CodeBlob* CodeCache::find_blob(void* start) { CodeBlob* result = find_blob_unsafe(start); - if (result == NULL) return NULL; // We could potentially look up non_entrant methods - guarantee(!result->is_zombie() || result->is_locked_by_vm() || is_error_reported(), "unsafe access to zombie method"); + guarantee(result == NULL || !result->is_zombie() || result->is_locked_by_vm() || is_error_reported(), "unsafe access to zombie method"); return result; } +// Lookup that does not fail if you lookup a zombie method (if you call this, be sure to know +// what you are doing) +CodeBlob* CodeCache::find_blob_unsafe(void* start) { + // NMT can walk the stack before code cache is created + if (_heaps->is_empty()) return NULL; + + FOR_ALL_HEAPS(it) { + CodeBlob* result = (CodeBlob*) (*it)->find_start(start); + if (result != NULL && result->blob_contains((address)start)) { + return result; + } + } + return NULL; +} + nmethod* CodeCache::find_nmethod(void* start) { - CodeBlob *cb = find_blob(start); - assert(cb == NULL || cb->is_nmethod(), "did not find an nmethod"); + CodeBlob* cb = find_blob(start); + assert(cb->is_nmethod(), "did not find an nmethod"); return (nmethod*)cb; } +bool CodeCache::contains_nmethod(nmethod* nm) { + FOR_ALL_METHOD_HEAPS(it) { + if ((*it)->contains(nm)) { + return true; + } + } + return false; +} void CodeCache::blobs_do(void f(CodeBlob* nm)) { assert_locked_or_safepoint(CodeCache_lock); - FOR_ALL_BLOBS(p) { - f(p); + FOR_ALL_HEAPS(it) { + FOR_ALL_BLOBS(cb, *it) { + f(cb); + } } } - void CodeCache::nmethods_do(void f(nmethod* nm)) { assert_locked_or_safepoint(CodeCache_lock); - FOR_ALL_BLOBS(nm) { - if (nm->is_nmethod()) f((nmethod*)nm); + FOR_ALL_METHOD_HEAPS(it) { + FOR_ALL_BLOBS(cb, *it) { + f((nmethod*)cb); + } } } void CodeCache::alive_nmethods_do(void f(nmethod* nm)) { assert_locked_or_safepoint(CodeCache_lock); - FOR_ALL_ALIVE_NMETHODS(nm) { - f(nm); + FOR_ALL_METHOD_HEAPS(it) { + FOR_ALL_ALIVE_BLOBS(cb, *it) { + f((nmethod*)cb); + } } } int CodeCache::alignment_unit() { - return (int)_heap->alignment_unit(); + return (int)_heaps->first()->alignment_unit(); } - int CodeCache::alignment_offset() { - return (int)_heap->alignment_offset(); + return (int)_heaps->first()->alignment_offset(); } - -// Mark nmethods for unloading if they contain otherwise unreachable -// oops. +// Mark nmethods for unloading if they contain otherwise unreachable oops. void CodeCache::do_unloading(BoolObjectClosure* is_alive, bool unloading_occurred) { assert_locked_or_safepoint(CodeCache_lock); - FOR_ALL_ALIVE_NMETHODS(nm) { - nm->do_unloading(is_alive, unloading_occurred); + FOR_ALL_METHOD_HEAPS(it) { + FOR_ALL_ALIVE_BLOBS(cb, *it) { + nmethod* nm = (nmethod*)cb; + nm->do_unloading(is_alive, unloading_occurred); + } } } void CodeCache::blobs_do(CodeBlobClosure* f) { assert_locked_or_safepoint(CodeCache_lock); - FOR_ALL_ALIVE_BLOBS(cb) { - f->do_code_blob(cb); + FOR_ALL_HEAPS(it) { + FOR_ALL_BLOBS(cb, *it) { + if (cb->is_alive()) { + f->do_code_blob(cb); #ifdef ASSERT - if (cb->is_nmethod()) - ((nmethod*)cb)->verify_scavenge_root_oops(); + if (cb->is_nmethod()) + ((nmethod*)cb)->verify_scavenge_root_oops(); #endif //ASSERT + } + } } } @@ -425,9 +594,9 @@ // Temporarily mark nmethods that are claimed to be on the non-perm list. void CodeCache::mark_scavenge_root_nmethods() { - FOR_ALL_ALIVE_BLOBS(cb) { - if (cb->is_nmethod()) { - nmethod *nm = (nmethod*)cb; + FOR_ALL_METHOD_HEAPS(it) { + FOR_ALL_ALIVE_BLOBS(cb, *it) { + nmethod* nm = (nmethod*)cb; assert(nm->scavenge_root_not_marked(), "clean state"); if (nm->on_scavenge_root_list()) nm->set_scavenge_root_marked(); @@ -438,32 +607,29 @@ // If the closure is given, run it on the unlisted nmethods. // Also make sure that the effects of mark_scavenge_root_nmethods is gone. void CodeCache::verify_perm_nmethods(CodeBlobClosure* f_or_null) { - FOR_ALL_ALIVE_BLOBS(cb) { - bool call_f = (f_or_null != NULL); - if (cb->is_nmethod()) { - nmethod *nm = (nmethod*)cb; + FOR_ALL_METHOD_HEAPS(it) { + FOR_ALL_ALIVE_BLOBS(cb, *it) { + nmethod* nm = (nmethod*)cb; + bool call_f = (f_or_null != NULL); assert(nm->scavenge_root_not_marked(), "must be already processed"); if (nm->on_scavenge_root_list()) call_f = false; // don't show this one to the client nm->verify_scavenge_root_oops(); - } else { - call_f = false; // not an nmethod + if (call_f) f_or_null->do_code_blob(nm); } - if (call_f) f_or_null->do_code_blob(cb); } } #endif //PRODUCT - void CodeCache::gc_prologue() { assert(!nmethod::oops_do_marking_is_active(), "oops_do_marking_epilogue must be called"); } void CodeCache::gc_epilogue() { assert_locked_or_safepoint(CodeCache_lock); - FOR_ALL_ALIVE_BLOBS(cb) { - if (cb->is_nmethod()) { - nmethod *nm = (nmethod*)cb; + FOR_ALL_METHOD_HEAPS(it) { + FOR_ALL_ALIVE_BLOBS(cb, *it) { + nmethod* nm = (nmethod*)cb; assert(!nm->is_unloaded(), "Tautology"); if (needs_cache_clean()) { nm->cleanup_inline_caches(); @@ -479,8 +645,8 @@ #ifdef ASSERT // make sure that we aren't leaking icholders int count = 0; - FOR_ALL_BLOBS(cb) { - if (cb->is_nmethod()) { + FOR_ALL_METHOD_HEAPS(it) { + FOR_ALL_BLOBS(cb, *it) { RelocIterator iter((nmethod*)cb); while(iter.next()) { if (iter.type() == relocInfo::virtual_call_type) { @@ -503,41 +669,82 @@ #endif } - void CodeCache::verify_oops() { MutexLockerEx mu(CodeCache_lock, Mutex::_no_safepoint_check_flag); VerifyOopClosure voc; - FOR_ALL_ALIVE_BLOBS(cb) { - if (cb->is_nmethod()) { - nmethod *nm = (nmethod*)cb; + FOR_ALL_METHOD_HEAPS(it) { + FOR_ALL_ALIVE_BLOBS(cb, *it) { + nmethod* nm = (nmethod*)cb; nm->oops_do(&voc); nm->verify_oop_relocations(); } } } - -address CodeCache::first_address() { - assert_locked_or_safepoint(CodeCache_lock); - return (address)_heap->low_boundary(); +size_t CodeCache::capacity() { + size_t cap = 0; + FOR_ALL_HEAPS(it) { + cap += (*it)->capacity(); + } + return cap; } +size_t CodeCache::unallocated_capacity() { + size_t unallocated_cap = 0; + FOR_ALL_HEAPS(it) { + unallocated_cap += (*it)->unallocated_capacity(); + } + return unallocated_cap; +} -address CodeCache::last_address() { - assert_locked_or_safepoint(CodeCache_lock); - return (address)_heap->high(); +size_t CodeCache::max_capacity() { + size_t max_cap = 0; + FOR_ALL_HEAPS(it) { + max_cap += (*it)->max_capacity(); + } + return max_cap; } /** - * Returns the reverse free ratio. E.g., if 25% (1/4) of the code cache + * Returns the reverse free ratio. E.g., if 25% (1/4) of the code heap * is free, reverse_free_ratio() returns 4. */ -double CodeCache::reverse_free_ratio() { - double unallocated_capacity = (double)(CodeCache::unallocated_capacity() - CodeCacheMinimumFreeSpace); - double max_capacity = (double)CodeCache::max_capacity(); +double CodeCache::reverse_free_ratio(int code_blob_type) { + CodeHeap* heap = get_code_heap(code_blob_type); + if (heap == NULL) { + return 0; + } + IsMethodPredicate isMethodHeap; + // Subtract CodeCacheMinimumFreeSpace from capacity of the non-method heap + double unallocated_capacity = (double)(heap->unallocated_capacity() - (isMethodHeap(heap) ? 0 : CodeCacheMinimumFreeSpace)); + double max_capacity = (double)heap->max_capacity(); return max_capacity / unallocated_capacity; } +size_t CodeCache::bytes_allocated_in_freelists() { + size_t allocated_bytes = 0; + FOR_ALL_HEAPS(it) { + allocated_bytes += (*it)->allocated_in_freelist(); + } + return allocated_bytes; +} + +int CodeCache::allocated_segments() { + int number_of_segments = 0; + FOR_ALL_HEAPS(it) { + number_of_segments += (*it)->allocated_segments(); + } + return number_of_segments; +} + +size_t CodeCache::freelists_length() { + size_t length = 0; + FOR_ALL_HEAPS(it) { + length += (*it)->freelist_length(); + } + return length; +} + void icache_init(); void CodeCache::initialize() { @@ -550,13 +757,9 @@ // the code cache to the page size. In particular, Solaris is moving to a larger // default page size. CodeCacheExpansionSize = round_to(CodeCacheExpansionSize, os::vm_page_size()); - InitialCodeCacheSize = round_to(InitialCodeCacheSize, os::vm_page_size()); - ReservedCodeCacheSize = round_to(ReservedCodeCacheSize, os::vm_page_size()); - if (!_heap->reserve(ReservedCodeCacheSize, InitialCodeCacheSize, CodeCacheSegmentSize)) { - vm_exit_during_initialization("Could not reserve enough space for code cache"); - } - MemoryService::add_code_heap_memory_pool(_heap); + // Reserve space and create heaps + initialize_heaps(); // Initialize ICache flush mechanism // This service is needed for os::register_code_area @@ -565,10 +768,9 @@ // Give OS a chance to register generated code area. // This is used on Windows 64 bit platforms to register // Structured Exception Handlers for our generated code. - os::register_code_area(_heap->low_boundary(), _heap->high_boundary()); + os::register_code_area((char*)low_bound(), (char*)high_bound()); } - void codeCache_init() { CodeCache::initialize(); } @@ -581,8 +783,11 @@ void CodeCache::clear_inline_caches() { assert_locked_or_safepoint(CodeCache_lock); - FOR_ALL_ALIVE_NMETHODS(nm) { - nm->clear_inline_caches(); + FOR_ALL_METHOD_HEAPS(it) { + FOR_ALL_ALIVE_BLOBS(cb, *it) { + nmethod* nm = (nmethod*)cb; + nm->clear_inline_caches(); + } } } @@ -637,16 +842,19 @@ } } - FOR_ALL_ALIVE_NMETHODS(nm) { - if (nm->is_marked_for_deoptimization()) { - // ...Already marked in the previous pass; don't count it again. - } else if (nm->is_evol_dependent_on(dependee())) { - ResourceMark rm; - nm->mark_for_deoptimization(); - number_of_marked_CodeBlobs++; - } else { - // flush caches in case they refer to a redefined Method* - nm->clear_inline_caches(); + FOR_ALL_METHOD_HEAPS(it) { + FOR_ALL_ALIVE_BLOBS(cb, *it) { + nmethod* nm = (nmethod*)cb; + if (nm->is_marked_for_deoptimization()) { + // ...Already marked in the previous pass; don't count it again. + } else if (nm->is_evol_dependent_on(dependee())) { + ResourceMark rm; + nm->mark_for_deoptimization(); + number_of_marked_CodeBlobs++; + } else { + // flush caches in case they refer to a redefined Method* + nm->clear_inline_caches(); + } } } @@ -658,21 +866,26 @@ // Deoptimize all methods void CodeCache::mark_all_nmethods_for_deoptimization() { MutexLockerEx mu(CodeCache_lock, Mutex::_no_safepoint_check_flag); - FOR_ALL_ALIVE_NMETHODS(nm) { - nm->mark_for_deoptimization(); + FOR_ALL_METHOD_HEAPS(it) { + FOR_ALL_ALIVE_BLOBS(cb, *it) { + nmethod* nm = (nmethod*)cb; + nm->mark_for_deoptimization(); + } } } - int CodeCache::mark_for_deoptimization(Method* dependee) { MutexLockerEx mu(CodeCache_lock, Mutex::_no_safepoint_check_flag); int number_of_marked_CodeBlobs = 0; - FOR_ALL_ALIVE_NMETHODS(nm) { - if (nm->is_dependent_on_method(dependee)) { - ResourceMark rm; - nm->mark_for_deoptimization(); - number_of_marked_CodeBlobs++; + FOR_ALL_METHOD_HEAPS(it) { + FOR_ALL_ALIVE_BLOBS(cb, *it) { + nmethod* nm = (nmethod*)cb; + if (nm->is_dependent_on_method(dependee)) { + ResourceMark rm; + nm->mark_for_deoptimization(); + number_of_marked_CodeBlobs++; + } } } @@ -681,20 +894,23 @@ void CodeCache::make_marked_nmethods_zombies() { assert(SafepointSynchronize::is_at_safepoint(), "must be at a safepoint"); - FOR_ALL_ALIVE_NMETHODS(nm) { - if (nm->is_marked_for_deoptimization()) { - - // If the nmethod has already been made non-entrant and it can be converted - // then zombie it now. Otherwise make it non-entrant and it will eventually - // be zombied when it is no longer seen on the stack. Note that the nmethod - // might be "entrant" and not on the stack and so could be zombied immediately - // but we can't tell because we don't track it on stack until it becomes - // non-entrant. + FOR_ALL_METHOD_HEAPS(it) { + FOR_ALL_ALIVE_BLOBS(cb, *it) { + nmethod* nm = (nmethod*)cb; + if (nm->is_marked_for_deoptimization()) { - if (nm->is_not_entrant() && nm->can_not_entrant_be_converted()) { - nm->make_zombie(); - } else { - nm->make_not_entrant(); + // If the nmethod has already been made non-entrant and it can be converted + // then zombie it now. Otherwise make it non-entrant and it will eventually + // be zombied when it is no longer seen on the stack. Note that the nmethod + // might be "entrant" and not on the stack and so could be zombied immediately + // but we can't tell because we don't track it on stack until it becomes + // non-entrant. + + if (nm->is_not_entrant() && nm->can_not_entrant_be_converted()) { + nm->make_zombie(); + } else { + nm->make_not_entrant(); + } } } } @@ -702,31 +918,63 @@ void CodeCache::make_marked_nmethods_not_entrant() { assert_locked_or_safepoint(CodeCache_lock); - FOR_ALL_ALIVE_NMETHODS(nm) { - if (nm->is_marked_for_deoptimization()) { - nm->make_not_entrant(); + FOR_ALL_METHOD_HEAPS(it) { + FOR_ALL_ALIVE_BLOBS(cb, *it) { + nmethod* nm = (nmethod*)cb; + if (nm->is_marked_for_deoptimization()) { + nm->make_not_entrant(); + } } } } void CodeCache::verify() { - _heap->verify(); - FOR_ALL_ALIVE_BLOBS(p) { - p->verify(); + assert_locked_or_safepoint(CodeCache_lock); + FOR_ALL_HEAPS(it) { + CodeHeap* heap = *it; + heap->verify(); + FOR_ALL_BLOBS(cb, heap) { + if (cb->is_alive()) { + cb->verify(); + } + } } } -void CodeCache::report_codemem_full() { +// A CodeHeap is full. Print out warning and report event. +void CodeCache::report_codemem_full(int code_blob_type, bool print) { + // Get nmethod heap for the given CodeBlobType and build CodeCacheFull event + CodeHeap* heap = get_code_heap(code_blob_type); + + if (!heap->was_full() || print) { + // Not yet reported for this heap, report + heap->report_full(); + warning("CodeHeap for %s is full. Compiler has been disabled.", CodeCache::get_heap_name(code_blob_type)); + warning("Try increasing the code heap size using -XX:%s=", + (code_blob_type == CodeBlobType::MethodNonProfiled) ? "NonProfiledCodeHeapSize" : "ProfiledCodeHeapSize"); + + ResourceMark rm; + stringStream s; + // Dump CodeCache summary into a buffer before locking the tty + { + MutexLockerEx mu(CodeCache_lock, Mutex::_no_safepoint_check_flag); + print_summary(&s, true); + } + ttyLocker ttyl; + tty->print(s.as_string()); + } + _codemem_full_count++; EventCodeCacheFull event; if (event.should_commit()) { - event.set_startAddress((u8)low_bound()); - event.set_commitedTopAddress((u8)high()); - event.set_reservedTopAddress((u8)high_bound()); + event.set_codeBlobType(code_blob_type); + event.set_startAddress((u8)heap->low_boundary()); + event.set_commitedTopAddress((u8)heap->high()); + event.set_reservedTopAddress((u8)heap->high_boundary()); event.set_entryCount(nof_blobs()); event.set_methodCount(nof_nmethods()); event.set_adaptorCount(nof_adapters()); - event.set_unallocatedCapacity(unallocated_capacity()/K); + event.set_unallocatedCapacity(heap->unallocated_capacity()/K); event.set_fullCount(_codemem_full_count); event.commit(); } @@ -734,15 +982,19 @@ void CodeCache::print_memory_overhead() { size_t wasted_bytes = 0; - CodeBlob *cb; - for (cb = first(); cb != NULL; cb = next(cb)) { - HeapBlock* heap_block = ((HeapBlock*)cb) - 1; - wasted_bytes += heap_block->length() * CodeCacheSegmentSize - cb->size(); + + FOR_ALL_HEAPS(it) { + CodeHeap* heap = *it; + CodeBlob* cb; + for (cb = (CodeBlob*)heap->first(); cb != NULL; cb = (CodeBlob*)heap->next(cb)) { + HeapBlock* heap_block = ((HeapBlock*)cb) - 1; + wasted_bytes += heap_block->length() * CodeCacheSegmentSize - cb->size(); + } } // Print bytes that are allocated in the freelist ttyLocker ttl; - tty->print_cr("Number of elements in freelist: %d", freelist_length()); - tty->print_cr("Allocated in freelist: %dkB", bytes_allocated_in_freelist()/K); + tty->print_cr("Number of elements in freelist: %d", freelists_length()); + tty->print_cr("Allocated in freelist: %dkB", bytes_allocated_in_freelists()/K); tty->print_cr("Unused bytes in CodeBlobs: %dkB", (int)(wasted_bytes/K)); tty->print_cr("Segment map size: %dkB", allocated_segments()/K); // 1 byte per segment } @@ -777,43 +1029,48 @@ int max_nm_size = 0; ResourceMark rm; - CodeBlob *cb; - for (cb = first(); cb != NULL; cb = next(cb)) { - total++; - if (cb->is_nmethod()) { - nmethod* nm = (nmethod*)cb; - - if (Verbose && nm->method() != NULL) { - ResourceMark rm; - char *method_name = nm->method()->name_and_sig_as_C_string(); - tty->print("%s", method_name); - if(nm->is_alive()) { tty->print_cr(" alive"); } - if(nm->is_not_entrant()) { tty->print_cr(" not-entrant"); } - if(nm->is_zombie()) { tty->print_cr(" zombie"); } - } + int i = 0; + FOR_ALL_HEAPS(it) { + if (Verbose) { + tty->print_cr("## Heap '%s' ##", (*it)->name()); + } + FOR_ALL_BLOBS(cb, *it) { + total++; + if (cb->is_nmethod()) { + nmethod* nm = (nmethod*)cb; + + if (Verbose && nm->method() != NULL) { + ResourceMark rm; + char *method_name = nm->method()->name_and_sig_as_C_string(); + tty->print("%s %d", method_name, nm->comp_level()); + if(nm->is_alive()) { tty->print_cr(" alive"); } + if(nm->is_not_entrant()) { tty->print_cr(" not-entrant"); } + if(nm->is_zombie()) { tty->print_cr(" zombie"); } + } - nmethodCount++; - - if(nm->is_alive()) { nmethodAlive++; } - if(nm->is_not_entrant()) { nmethodNotEntrant++; } - if(nm->is_zombie()) { nmethodZombie++; } - if(nm->is_unloaded()) { nmethodUnloaded++; } - if(nm->method() != NULL && nm->is_native_method()) { nmethodNative++; } + nmethodCount++; - if(nm->method() != NULL && nm->is_java_method()) { - nmethodJava++; - max_nm_size = MAX2(max_nm_size, nm->size()); + if(nm->is_alive()) { nmethodAlive++; } + if(nm->is_not_entrant()) { nmethodNotEntrant++; } + if(nm->is_zombie()) { nmethodZombie++; } + if(nm->is_unloaded()) { nmethodUnloaded++; } + if(nm->method() != NULL && nm->is_native_method()) { nmethodNative++; } + + if(nm->method() != NULL && nm->is_java_method()) { + nmethodJava++; + max_nm_size = MAX2(max_nm_size, nm->size()); + } + } else if (cb->is_runtime_stub()) { + runtimeStubCount++; + } else if (cb->is_deoptimization_stub()) { + deoptimizationStubCount++; + } else if (cb->is_uncommon_trap_stub()) { + uncommonTrapStubCount++; + } else if (cb->is_adapter_blob()) { + adapterCount++; + } else if (cb->is_buffer_blob()) { + bufferBlobCount++; } - } else if (cb->is_runtime_stub()) { - runtimeStubCount++; - } else if (cb->is_deoptimization_stub()) { - deoptimizationStubCount++; - } else if (cb->is_uncommon_trap_stub()) { - uncommonTrapStubCount++; - } else if (cb->is_adapter_blob()) { - adapterCount++; - } else if (cb->is_buffer_blob()) { - bufferBlobCount++; } } @@ -822,12 +1079,12 @@ int *buckets = NEW_C_HEAP_ARRAY(int, bucketLimit, mtCode); memset(buckets, 0, sizeof(int) * bucketLimit); - for (cb = first(); cb != NULL; cb = next(cb)) { - if (cb->is_nmethod()) { + FOR_ALL_METHOD_HEAPS(it) { + FOR_ALL_BLOBS(cb, *it) { nmethod* nm = (nmethod*)cb; - if(nm->is_java_method()) { + if(nm->method() != NULL && nm->is_java_method()) { buckets[nm->size() / bucketSize]++; - } + } } } @@ -848,7 +1105,7 @@ tty->print_cr("\nnmethod size distribution (non-zombie java)"); tty->print_cr("-------------------------------------------------"); - for(int i=0; iprint("%d - %d bytes",i*bucketSize,(i+1)*bucketSize); tty->fill_to(40); @@ -871,11 +1128,13 @@ CodeBlob_sizes live; CodeBlob_sizes dead; - FOR_ALL_BLOBS(p) { - if (!p->is_alive()) { - dead.add(p); - } else { - live.add(p); + FOR_ALL_HEAPS(it) { + FOR_ALL_BLOBS(cb, *it) { + if (!cb->is_alive()) { + dead.add(cb); + } else { + live.add(cb); + } } } @@ -889,21 +1148,22 @@ dead.print("dead"); } - if (WizardMode) { // print the oop_map usage int code_size = 0; int number_of_blobs = 0; int number_of_oop_maps = 0; int map_size = 0; - FOR_ALL_BLOBS(p) { - if (p->is_alive()) { - number_of_blobs++; - code_size += p->code_size(); - OopMapSet* set = p->oop_maps(); - if (set != NULL) { - number_of_oop_maps += set->size(); - map_size += set->heap_size(); + FOR_ALL_HEAPS(it) { + FOR_ALL_BLOBS(cb, *it) { + if (cb->is_alive()) { + number_of_blobs++; + code_size += cb->code_size(); + OopMapSet* set = cb->oop_maps(); + if (set != NULL) { + number_of_oop_maps += set->size(); + map_size += set->heap_size(); + } } } } @@ -918,20 +1178,26 @@ } void CodeCache::print_summary(outputStream* st, bool detailed) { - size_t total = (_heap->high_boundary() - _heap->low_boundary()); - st->print_cr("CodeCache: size=" SIZE_FORMAT "Kb used=" SIZE_FORMAT - "Kb max_used=" SIZE_FORMAT "Kb free=" SIZE_FORMAT "Kb", - total/K, (total - unallocated_capacity())/K, - maxCodeCacheUsed/K, unallocated_capacity()/K); + st->print_cr("CodeCache Summary:"); + FOR_ALL_HEAPS(it) { + CodeHeap* heap = (*it); + size_t total = (heap->high_boundary() - heap->low_boundary()); + st->print_cr("Heap '%s': size=" SIZE_FORMAT "Kb used=" SIZE_FORMAT + "Kb max_used=" SIZE_FORMAT "Kb free=" SIZE_FORMAT "Kb", + heap->name(), total/K, (total - heap->unallocated_capacity())/K, + heap->max_allocated_capacity()/K, heap->unallocated_capacity()/K); + + if (detailed) { + st->print_cr(" bounds [" INTPTR_FORMAT ", " INTPTR_FORMAT ", " INTPTR_FORMAT "]", + heap->low_boundary(), + heap->high(), + heap->high_boundary()); + + } + } if (detailed) { - st->print_cr(" bounds [" INTPTR_FORMAT ", " INTPTR_FORMAT ", " INTPTR_FORMAT "]", - _heap->low_boundary(), - _heap->high(), - _heap->high_boundary()); - st->print_cr(" total_blobs=" UINT32_FORMAT " nmethods=" UINT32_FORMAT - " adapters=" UINT32_FORMAT, - nof_blobs(), nof_nmethods(), nof_adapters()); + log_state(st); st->print_cr(" compilation: %s", CompileBroker::should_compile_new_jobs() ? "enabled" : Arguments::mode() == Arguments::_int ? "disabled (interpreter mode)" : @@ -941,8 +1207,7 @@ void CodeCache::log_state(outputStream* st) { st->print(" total_blobs='" UINT32_FORMAT "' nmethods='" UINT32_FORMAT "'" - " adapters='" UINT32_FORMAT "' free_code_cache='" SIZE_FORMAT "'", - nof_blobs(), nof_nmethods(), nof_adapters(), - unallocated_capacity()); + " adapters='" UINT32_FORMAT "'", + nof_blobs(), nof_nmethods(), nof_adapters()); }