1 /* 2 * Copyright (c) 2019, 2020, Red Hat, Inc. All rights reserved. 3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 4 * 5 * This code is free software; you can redistribute it and/or modify it 6 * under the terms of the GNU General Public License version 2 only, as 7 * published by the Free Software Foundation. 8 * 9 * This code is distributed in the hope that it will be useful, but WITHOUT 10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 12 * version 2 for more details (a copy is included in the LICENSE file that 13 * accompanied this code). 14 * 15 * You should have received a copy of the GNU General Public License version 16 * 2 along with this work; if not, write to the Free Software Foundation, 17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 18 * 19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 20 * or visit www.oracle.com if you need additional information or have any 21 * questions. 22 * 23 */ 24 25 #include "precompiled.hpp" 26 27 #include "gc/shenandoah/shenandoahClosures.inline.hpp" 28 #include "gc/shenandoah/shenandoahConcurrentRoots.hpp" 29 #include "gc/shenandoah/shenandoahHeap.inline.hpp" 30 #include "gc/shenandoah/shenandoahNMethod.inline.hpp" 31 #include "memory/resourceArea.hpp" 32 33 ShenandoahNMethod::ShenandoahNMethod(nmethod* nm, GrowableArray<oop*>& oops, bool non_immediate_oops) : 34 _nm(nm), _oops(NULL), _oops_count(0), _unregistered(false) { 35 36 if (!oops.is_empty()) { 37 _oops_count = oops.length(); 38 _oops = NEW_C_HEAP_ARRAY(oop*, _oops_count, mtGC); 39 for (int c = 0; c < _oops_count; c++) { 40 _oops[c] = oops.at(c); 41 } 42 } 43 _has_non_immed_oops = non_immediate_oops; 44 45 assert_same_oops(); 46 } 47 48 ShenandoahNMethod::~ShenandoahNMethod() { 49 if (_oops != NULL) { 50 FREE_C_HEAP_ARRAY(oop*, _oops); 51 } 52 } 53 54 class ShenandoahHasCSetOopClosure : public OopClosure { 55 private: 56 ShenandoahHeap* const _heap; 57 bool _has_cset_oops; 58 59 public: 60 ShenandoahHasCSetOopClosure(ShenandoahHeap *heap) : 61 _heap(heap), 62 _has_cset_oops(false) { 63 } 64 65 bool has_cset_oops() const { 66 return _has_cset_oops; 67 } 68 69 void do_oop(oop* p) { 70 oop value = RawAccess<>::oop_load(p); 71 if (!_has_cset_oops && _heap->in_collection_set(value)) { 72 _has_cset_oops = true; 73 } 74 } 75 76 void do_oop(narrowOop* p) { 77 ShouldNotReachHere(); 78 } 79 }; 80 81 bool ShenandoahNMethod::has_cset_oops(ShenandoahHeap *heap) { 82 ShenandoahHasCSetOopClosure cl(heap); 83 oops_do(&cl); 84 return cl.has_cset_oops(); 85 } 86 87 void ShenandoahNMethod::update() { 88 ResourceMark rm; 89 bool non_immediate_oops = false; 90 GrowableArray<oop*> oops; 91 92 detect_reloc_oops(nm(), oops, non_immediate_oops); 93 if (oops.length() != _oops_count) { 94 if (_oops != NULL) { 95 FREE_C_HEAP_ARRAY(oop*, _oops); 96 _oops = NULL; 97 } 98 99 _oops_count = oops.length(); 100 if (_oops_count > 0) { 101 _oops = NEW_C_HEAP_ARRAY(oop*, _oops_count, mtGC); 102 } 103 } 104 105 for (int index = 0; index < _oops_count; index ++) { 106 _oops[index] = oops.at(index); 107 } 108 _has_non_immed_oops = non_immediate_oops; 109 110 assert_same_oops(); 111 } 112 113 void ShenandoahNMethod::detect_reloc_oops(nmethod* nm, GrowableArray<oop*>& oops, bool& has_non_immed_oops) { 114 has_non_immed_oops = false; 115 // Find all oops relocations 116 RelocIterator iter(nm); 117 while (iter.next()) { 118 if (iter.type() != relocInfo::oop_type) { 119 // Not an oop 120 continue; 121 } 122 123 oop_Relocation* r = iter.oop_reloc(); 124 if (!r->oop_is_immediate()) { 125 // Non-immediate oop found 126 has_non_immed_oops = true; 127 continue; 128 } 129 130 oop value = r->oop_value(); 131 if (value != NULL) { 132 oop* addr = r->oop_addr(); 133 shenandoah_assert_correct(addr, value); 134 shenandoah_assert_not_in_cset_except(addr, value, ShenandoahHeap::heap()->cancelled_gc()); 135 shenandoah_assert_not_forwarded(addr, value); 136 // Non-NULL immediate oop found. NULL oops can safely be 137 // ignored since the method will be re-registered if they 138 // are later patched to be non-NULL. 139 oops.push(addr); 140 } 141 } 142 } 143 144 ShenandoahNMethod* ShenandoahNMethod::for_nmethod(nmethod* nm) { 145 ResourceMark rm; 146 bool non_immediate_oops = false; 147 GrowableArray<oop*> oops; 148 149 detect_reloc_oops(nm, oops, non_immediate_oops); 150 151 // No embedded oops 152 if(!ShenandoahConcurrentRoots::can_do_concurrent_class_unloading() && 153 oops.is_empty() && nm->oops_begin() >= nm->oops_end()) { 154 return NULL; 155 } 156 157 return new ShenandoahNMethod(nm, oops, non_immediate_oops); 158 } 159 160 template <bool HAS_FWD> 161 class ShenandoahKeepNMethodMetadataAliveClosure : public OopClosure { 162 private: 163 ShenandoahBarrierSet* const _bs; 164 public: 165 ShenandoahKeepNMethodMetadataAliveClosure() : 166 _bs(static_cast<ShenandoahBarrierSet*>(BarrierSet::barrier_set())) { 167 } 168 169 virtual void do_oop(oop* p) { 170 oop obj = RawAccess<>::oop_load(p); 171 if (!CompressedOops::is_null(obj)) { 172 if (HAS_FWD) { 173 obj = ShenandoahBarrierSet::resolve_forwarded_not_null(obj); 174 } 175 _bs->enqueue(obj); 176 } 177 } 178 179 virtual void do_oop(narrowOop* p) { 180 ShouldNotReachHere(); 181 } 182 }; 183 184 void ShenandoahNMethod::heal_nmethod(nmethod* nm) { 185 ShenandoahNMethod* data = gc_data(nm); 186 assert(data != NULL, "Sanity"); 187 assert(data->lock()->owned_by_self(), "Must hold the lock"); 188 189 ShenandoahHeap* const heap = ShenandoahHeap::heap(); 190 if (heap->is_concurrent_mark_in_progress()) { 191 if (heap->has_forwarded_objects()) { 192 ShenandoahKeepNMethodMetadataAliveClosure<true> cl; 193 data->oops_do(&cl); 194 } else { 195 ShenandoahKeepNMethodMetadataAliveClosure<false> cl; 196 data->oops_do(&cl); 197 } 198 } else if (heap->is_concurrent_weak_root_in_progress()) { 199 ShenandoahEvacOOMScope evac_scope; 200 heal_nmethod_metadata(data); 201 } else { 202 // There is possibility that GC is cancelled when it arrives final mark. 203 // In this case, concurrent root phase is skipped and degenerated GC should be 204 // followed, where nmethods are disarmed. 205 assert(heap->cancelled_gc(), "What else?"); 206 } 207 } 208 209 #ifdef ASSERT 210 void ShenandoahNMethod::assert_alive_and_correct() { 211 assert(_nm->is_alive(), "only alive nmethods here"); 212 ShenandoahHeap* heap = ShenandoahHeap::heap(); 213 for (int c = 0; c < _oops_count; c++) { 214 oop *loc = _oops[c]; 215 assert(_nm->code_contains((address) loc) || _nm->oops_contains(loc), "nmethod should contain the oop*"); 216 oop o = RawAccess<>::oop_load(loc); 217 shenandoah_assert_correct_except(loc, o, o == NULL || heap->is_full_gc_move_in_progress()); 218 } 219 220 oop* const begin = _nm->oops_begin(); 221 oop* const end = _nm->oops_end(); 222 for (oop* p = begin; p < end; p++) { 223 if (*p != Universe::non_oop_word()) { 224 oop o = RawAccess<>::oop_load(p); 225 shenandoah_assert_correct_except(p, o, o == NULL || heap->is_full_gc_move_in_progress()); 226 } 227 } 228 } 229 230 class ShenandoahNMethodOopDetector : public OopClosure { 231 private: 232 ResourceMark rm; // For growable array allocation below. 233 GrowableArray<oop*> _oops; 234 235 public: 236 ShenandoahNMethodOopDetector() : _oops(10) {}; 237 238 void do_oop(oop* o) { 239 _oops.append(o); 240 } 241 void do_oop(narrowOop* o) { 242 fatal("NMethods should not have compressed oops embedded."); 243 } 244 245 GrowableArray<oop*>* oops() { 246 return &_oops; 247 } 248 249 bool has_oops() { 250 return !_oops.is_empty(); 251 } 252 }; 253 254 void ShenandoahNMethod::assert_same_oops(bool allow_dead) { 255 ShenandoahNMethodOopDetector detector; 256 nm()->oops_do(&detector, allow_dead); 257 258 GrowableArray<oop*>* oops = detector.oops(); 259 260 int count = _oops_count; 261 for (int index = 0; index < _oops_count; index ++) { 262 assert(oops->contains(_oops[index]), "Must contain this oop"); 263 } 264 265 for (oop* p = nm()->oops_begin(); p < nm()->oops_end(); p ++) { 266 if (*p == Universe::non_oop_word()) continue; 267 count++; 268 assert(oops->contains(p), "Must contain this oop"); 269 } 270 271 if (oops->length() < count) { 272 stringStream debug_stream; 273 debug_stream.print_cr("detected locs: %d", oops->length()); 274 for (int i = 0; i < oops->length(); i++) { 275 debug_stream.print_cr("-> " PTR_FORMAT, p2i(oops->at(i))); 276 } 277 debug_stream.print_cr("recorded oops: %d", _oops_count); 278 for (int i = 0; i < _oops_count; i++) { 279 debug_stream.print_cr("-> " PTR_FORMAT, p2i(_oops[i])); 280 } 281 GrowableArray<oop*> check; 282 bool non_immed; 283 detect_reloc_oops(nm(), check, non_immed); 284 debug_stream.print_cr("check oops: %d", check.length()); 285 for (int i = 0; i < check.length(); i++) { 286 debug_stream.print_cr("-> " PTR_FORMAT, p2i(check.at(i))); 287 } 288 fatal("Must match #detected: %d, #recorded: %d, #total: %d, begin: " PTR_FORMAT ", end: " PTR_FORMAT "\n%s", 289 oops->length(), _oops_count, count, p2i(nm()->oops_begin()), p2i(nm()->oops_end()), debug_stream.as_string()); 290 } 291 } 292 293 void ShenandoahNMethod::assert_no_oops(nmethod* nm, bool allow_dead) { 294 ShenandoahNMethodOopDetector detector; 295 nm->oops_do(&detector, allow_dead); 296 assert(detector.oops()->length() == 0, "Should not have oops"); 297 } 298 #endif 299 300 ShenandoahNMethodTable::ShenandoahNMethodTable() : 301 _heap(ShenandoahHeap::heap()), 302 _index(0), 303 _itr_cnt(0) { 304 _list = new ShenandoahNMethodList(minSize); 305 } 306 307 ShenandoahNMethodTable::~ShenandoahNMethodTable() { 308 assert(_list != NULL, "Sanity"); 309 _list->release(); 310 } 311 312 void ShenandoahNMethodTable::register_nmethod(nmethod* nm) { 313 assert(CodeCache_lock->owned_by_self(), "Must have CodeCache_lock held"); 314 assert(_index >= 0 && _index <= _list->size(), "Sanity"); 315 316 ShenandoahNMethod* data = ShenandoahNMethod::gc_data(nm); 317 ShenandoahReentrantLocker data_locker(data != NULL ? data->lock() : NULL); 318 319 if (data != NULL) { 320 assert(contain(nm), "Must have been registered"); 321 assert(nm == data->nm(), "Must be same nmethod"); 322 data->update(); 323 } else { 324 data = ShenandoahNMethod::for_nmethod(nm); 325 if (data == NULL) { 326 assert(!ShenandoahConcurrentRoots::can_do_concurrent_class_unloading(), 327 "Only possible when concurrent class unloading is off"); 328 return; 329 } 330 ShenandoahNMethod::attach_gc_data(nm, data); 331 ShenandoahLocker locker(&_lock); 332 log_register_nmethod(nm); 333 append(data); 334 } 335 // Disarm new nmethod 336 ShenandoahNMethod::disarm_nmethod(nm); 337 } 338 339 void ShenandoahNMethodTable::unregister_nmethod(nmethod* nm) { 340 assert_locked_or_safepoint(CodeCache_lock); 341 342 ShenandoahNMethod* data = ShenandoahNMethod::gc_data(nm); 343 if (data == NULL) { 344 assert(!ShenandoahConcurrentRoots::can_do_concurrent_class_unloading(), 345 "Only possible when concurrent class unloading is off"); 346 ShenandoahNMethod::assert_no_oops(nm, true /*allow_dead*/); 347 return; 348 } 349 350 if (Thread::current()->is_Code_cache_sweeper_thread()) { 351 wait_until_concurrent_iteration_done(); 352 } 353 log_unregister_nmethod(nm); 354 ShenandoahLocker locker(&_lock); 355 assert(contain(nm), "Must have been registered"); 356 357 ShenandoahReentrantLocker data_locker(data->lock()); 358 data->mark_unregistered(); 359 } 360 361 void ShenandoahNMethodTable::flush_nmethod(nmethod* nm) { 362 assert(CodeCache_lock->owned_by_self(), "Must have CodeCache_lock held"); 363 assert(Thread::current()->is_Code_cache_sweeper_thread(), "Must from Sweep thread"); 364 ShenandoahNMethod* data = ShenandoahNMethod::gc_data(nm); 365 assert(data != NULL || !ShenandoahConcurrentRoots::can_do_concurrent_class_unloading(), 366 "Only possible when concurrent class unloading is off"); 367 if (data == NULL) { 368 ShenandoahNMethod::assert_no_oops(nm, true /*allow_dead*/); 369 return; 370 } 371 372 // Can not alter the array when iteration is in progress 373 wait_until_concurrent_iteration_done(); 374 log_flush_nmethod(nm); 375 376 ShenandoahLocker locker(&_lock); 377 int idx = index_of(nm); 378 assert(idx >= 0 && idx < _index, "Invalid index"); 379 ShenandoahNMethod::attach_gc_data(nm, NULL); 380 remove(idx); 381 } 382 383 bool ShenandoahNMethodTable::contain(nmethod* nm) const { 384 return index_of(nm) != -1; 385 } 386 387 ShenandoahNMethod* ShenandoahNMethodTable::at(int index) const { 388 assert(index >= 0 && index < _index, "Out of bound"); 389 return _list->at(index); 390 } 391 392 int ShenandoahNMethodTable::index_of(nmethod* nm) const { 393 for (int index = 0; index < length(); index ++) { 394 if (at(index)->nm() == nm) { 395 return index; 396 } 397 } 398 return -1; 399 } 400 401 void ShenandoahNMethodTable::remove(int idx) { 402 shenandoah_assert_locked_or_safepoint(CodeCache_lock); 403 assert(!iteration_in_progress(), "Can not happen"); 404 assert(_index >= 0 && _index <= _list->size(), "Sanity"); 405 406 assert(idx >= 0 && idx < _index, "Out of bound"); 407 ShenandoahNMethod* snm = _list->at(idx); 408 ShenandoahNMethod* tmp = _list->at(_index - 1); 409 _list->set(idx, tmp); 410 _index --; 411 412 delete snm; 413 } 414 415 void ShenandoahNMethodTable::wait_until_concurrent_iteration_done() { 416 assert(CodeCache_lock->owned_by_self(), "Lock must be held"); 417 while (iteration_in_progress()) { 418 CodeCache_lock->wait_without_safepoint_check(); 419 } 420 } 421 422 void ShenandoahNMethodTable::append(ShenandoahNMethod* snm) { 423 if (is_full()) { 424 int new_size = 2 * _list->size(); 425 // Rebuild table and replace current one 426 rebuild(new_size); 427 } 428 429 _list->set(_index++, snm); 430 assert(_index >= 0 && _index <= _list->size(), "Sanity"); 431 } 432 433 void ShenandoahNMethodTable::rebuild(int size) { 434 ShenandoahNMethodList* new_list = new ShenandoahNMethodList(size); 435 new_list->transfer(_list, _index); 436 437 // Release old list 438 _list->release(); 439 _list = new_list; 440 } 441 442 ShenandoahNMethodTableSnapshot* ShenandoahNMethodTable::snapshot_for_iteration() { 443 _itr_cnt++; 444 return new ShenandoahNMethodTableSnapshot(this); 445 } 446 447 void ShenandoahNMethodTable::finish_iteration(ShenandoahNMethodTableSnapshot* snapshot) { 448 assert(iteration_in_progress(), "Why we here?"); 449 assert(snapshot != NULL, "No snapshot"); 450 _itr_cnt--; 451 452 delete snapshot; 453 } 454 455 void ShenandoahNMethodTable::log_register_nmethod(nmethod* nm) { 456 LogTarget(Debug, gc, nmethod) log; 457 if (!log.is_enabled()) { 458 return; 459 } 460 461 ResourceMark rm; 462 log.print("Register NMethod: %s.%s [" PTR_FORMAT "] (%s)", 463 nm->method()->method_holder()->external_name(), 464 nm->method()->name()->as_C_string(), 465 p2i(nm), 466 nm->compiler_name()); 467 } 468 469 void ShenandoahNMethodTable::log_unregister_nmethod(nmethod* nm) { 470 LogTarget(Debug, gc, nmethod) log; 471 if (!log.is_enabled()) { 472 return; 473 } 474 475 ResourceMark rm; 476 log.print("Unregister NMethod: %s.%s [" PTR_FORMAT "]", 477 nm->method()->method_holder()->external_name(), 478 nm->method()->name()->as_C_string(), 479 p2i(nm)); 480 } 481 482 void ShenandoahNMethodTable::log_flush_nmethod(nmethod* nm) { 483 LogTarget(Debug, gc, nmethod) log; 484 if (!log.is_enabled()) { 485 return; 486 } 487 488 ResourceMark rm; 489 log.print("Flush NMethod: (" PTR_FORMAT ")", p2i(nm)); 490 } 491 492 #ifdef ASSERT 493 void ShenandoahNMethodTable::assert_nmethods_alive_and_correct() { 494 assert_locked_or_safepoint(CodeCache_lock); 495 496 for (int index = 0; index < length(); index ++) { 497 ShenandoahNMethod* m = _list->at(index); 498 // Concurrent unloading may have dead nmethods to be cleaned by sweeper 499 if (m->is_unregistered()) continue; 500 m->assert_alive_and_correct(); 501 } 502 } 503 #endif 504 505 506 ShenandoahNMethodList::ShenandoahNMethodList(int size) : 507 _size(size), _ref_count(1) { 508 _list = NEW_C_HEAP_ARRAY(ShenandoahNMethod*, size, mtGC); 509 } 510 511 ShenandoahNMethodList::~ShenandoahNMethodList() { 512 assert(_list != NULL, "Sanity"); 513 assert(_ref_count == 0, "Must be"); 514 FREE_C_HEAP_ARRAY(ShenandoahNMethod*, _list); 515 } 516 517 void ShenandoahNMethodList::transfer(ShenandoahNMethodList* const list, int limit) { 518 assert(limit <= size(), "Sanity"); 519 ShenandoahNMethod** old_list = list->list(); 520 for (int index = 0; index < limit; index++) { 521 _list[index] = old_list[index]; 522 } 523 } 524 525 ShenandoahNMethodList* ShenandoahNMethodList::acquire() { 526 assert(CodeCache_lock->owned_by_self(), "Lock must be held"); 527 _ref_count++; 528 return this; 529 } 530 531 void ShenandoahNMethodList::release() { 532 assert(CodeCache_lock->owned_by_self(), "Lock must be held"); 533 _ref_count--; 534 if (_ref_count == 0) { 535 delete this; 536 } 537 } 538 539 ShenandoahNMethodTableSnapshot::ShenandoahNMethodTableSnapshot(ShenandoahNMethodTable* table) : 540 _heap(ShenandoahHeap::heap()), _list(table->_list->acquire()), _limit(table->_index), _claimed(0) { 541 } 542 543 ShenandoahNMethodTableSnapshot::~ShenandoahNMethodTableSnapshot() { 544 _list->release(); 545 } 546 547 void ShenandoahNMethodTableSnapshot::concurrent_nmethods_do(NMethodClosure* cl) { 548 size_t stride = 256; // educated guess 549 550 ShenandoahNMethod** list = _list->list(); 551 size_t max = (size_t)_limit; 552 while (_claimed < max) { 553 size_t cur = Atomic::fetch_and_add(&_claimed, stride); 554 size_t start = cur; 555 size_t end = MIN2(cur + stride, max); 556 if (start >= max) break; 557 558 for (size_t idx = start; idx < end; idx++) { 559 ShenandoahNMethod* data = list[idx]; 560 assert(data != NULL, "Should not be NULL"); 561 if (!data->is_unregistered()) { 562 cl->do_nmethod(data->nm()); 563 } 564 } 565 } 566 } 567 568 ShenandoahConcurrentNMethodIterator::ShenandoahConcurrentNMethodIterator(ShenandoahNMethodTable* table) : 569 _table(table), _table_snapshot(NULL) { 570 } 571 572 void ShenandoahConcurrentNMethodIterator::nmethods_do_begin() { 573 assert(CodeCache_lock->owned_by_self(), "Lock must be held"); 574 assert(ShenandoahConcurrentRoots::can_do_concurrent_class_unloading(), 575 "Only for concurrent class unloading"); 576 _table_snapshot = _table->snapshot_for_iteration(); 577 } 578 579 void ShenandoahConcurrentNMethodIterator::nmethods_do(NMethodClosure* cl) { 580 assert(_table_snapshot != NULL, "Must first call nmethod_do_begin()"); 581 _table_snapshot->concurrent_nmethods_do(cl); 582 } 583 584 void ShenandoahConcurrentNMethodIterator::nmethods_do_end() { 585 assert(CodeCache_lock->owned_by_self(), "Lock must be held"); 586 assert(ShenandoahConcurrentRoots::can_do_concurrent_class_unloading(), 587 "Only for concurrent class unloading"); 588 _table->finish_iteration(_table_snapshot); 589 CodeCache_lock->notify_all(); 590 }