1 /* 2 * Copyright (c) 2008, 2018, Oracle and/or its affiliates. All rights reserved. 3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 4 * 5 * This code is free software; you can redistribute it and/or modify it 6 * under the terms of the GNU General Public License version 2 only, as 7 * published by the Free Software Foundation. 8 * 9 * This code is distributed in the hope that it will be useful, but WITHOUT 10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 12 * version 2 for more details (a copy is included in the LICENSE file that 13 * accompanied this code). 14 * 15 * You should have received a copy of the GNU General Public License version 16 * 2 along with this work; if not, write to the Free Software Foundation, 17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 18 * 19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 20 * or visit www.oracle.com if you need additional information or have any 21 * questions. 22 * 23 */ 24 25 #include "precompiled.hpp" 26 #include "jvm.h" 27 #include "gc/shared/barrierSet.hpp" 28 #include "gc/shared/cardTable.hpp" 29 #include "gc/shared/cardTableBarrierSet.inline.hpp" 30 #include "gc/shared/collectedHeap.hpp" 31 #include "interp_masm_arm.hpp" 32 #include "interpreter/interpreter.hpp" 33 #include "interpreter/interpreterRuntime.hpp" 34 #include "logging/log.hpp" 35 #include "oops/arrayOop.hpp" 36 #include "oops/markOop.hpp" 37 #include "oops/method.hpp" 38 #include "oops/methodData.hpp" 39 #include "prims/jvmtiExport.hpp" 40 #include "prims/jvmtiThreadState.hpp" 41 #include "runtime/basicLock.hpp" 42 #include "runtime/biasedLocking.hpp" 43 #include "runtime/frame.inline.hpp" 44 #include "runtime/sharedRuntime.hpp" 45 46 #if INCLUDE_ALL_GCS 47 #include "gc/g1/g1BarrierSet.hpp" 48 #include "gc/g1/heapRegion.hpp" 49 #endif // INCLUDE_ALL_GCS 50 51 //-------------------------------------------------------------------- 52 // Implementation of InterpreterMacroAssembler 53 54 55 56 57 InterpreterMacroAssembler::InterpreterMacroAssembler(CodeBuffer* code) : MacroAssembler(code) { 58 } 59 60 void InterpreterMacroAssembler::call_VM_helper(Register oop_result, address entry_point, int number_of_arguments, bool check_exceptions) { 61 #if defined(ASSERT) && !defined(AARCH64) 62 // Ensure that last_sp is not filled. 63 { Label L; 64 ldr(Rtemp, Address(FP, frame::interpreter_frame_last_sp_offset * wordSize)); 65 cbz(Rtemp, L); 66 stop("InterpreterMacroAssembler::call_VM_helper: last_sp != NULL"); 67 bind(L); 68 } 69 #endif // ASSERT && !AARCH64 70 71 // Rbcp must be saved/restored since it may change due to GC. 72 save_bcp(); 73 74 #ifdef AARCH64 75 check_no_cached_stack_top(Rtemp); 76 save_stack_top(); 77 check_extended_sp(Rtemp); 78 cut_sp_before_call(); 79 #endif // AARCH64 80 81 // super call 82 MacroAssembler::call_VM_helper(oop_result, entry_point, number_of_arguments, check_exceptions); 83 84 #ifdef AARCH64 85 // Restore SP to extended SP 86 restore_sp_after_call(Rtemp); 87 check_stack_top(); 88 clear_cached_stack_top(); 89 #endif // AARCH64 90 91 // Restore interpreter specific registers. 92 restore_bcp(); 93 restore_method(); 94 } 95 96 void InterpreterMacroAssembler::jump_to_entry(address entry) { 97 assert(entry, "Entry must have been generated by now"); 98 b(entry); 99 } 100 101 void InterpreterMacroAssembler::check_and_handle_popframe() { 102 if (can_pop_frame()) { 103 Label L; 104 const Register popframe_cond = R2_tmp; 105 106 // Initiate popframe handling only if it is not already being processed. If the flag 107 // has the popframe_processing bit set, it means that this code is called *during* popframe 108 // handling - we don't want to reenter. 109 110 ldr_s32(popframe_cond, Address(Rthread, JavaThread::popframe_condition_offset())); 111 tbz(popframe_cond, exact_log2(JavaThread::popframe_pending_bit), L); 112 tbnz(popframe_cond, exact_log2(JavaThread::popframe_processing_bit), L); 113 114 // Call Interpreter::remove_activation_preserving_args_entry() to get the 115 // address of the same-named entrypoint in the generated interpreter code. 116 call_VM_leaf(CAST_FROM_FN_PTR(address, Interpreter::remove_activation_preserving_args_entry)); 117 118 // Call indirectly to avoid generation ordering problem. 119 jump(R0); 120 121 bind(L); 122 } 123 } 124 125 126 // Blows R2, Rtemp. Sets TOS cached value. 127 void InterpreterMacroAssembler::load_earlyret_value(TosState state) { 128 const Register thread_state = R2_tmp; 129 130 ldr(thread_state, Address(Rthread, JavaThread::jvmti_thread_state_offset())); 131 132 const Address tos_addr(thread_state, JvmtiThreadState::earlyret_tos_offset()); 133 const Address oop_addr(thread_state, JvmtiThreadState::earlyret_oop_offset()); 134 const Address val_addr(thread_state, JvmtiThreadState::earlyret_value_offset()); 135 #ifndef AARCH64 136 const Address val_addr_hi(thread_state, JvmtiThreadState::earlyret_value_offset() 137 + in_ByteSize(wordSize)); 138 #endif // !AARCH64 139 140 Register zero = zero_register(Rtemp); 141 142 switch (state) { 143 case atos: ldr(R0_tos, oop_addr); 144 str(zero, oop_addr); 145 interp_verify_oop(R0_tos, state, __FILE__, __LINE__); 146 break; 147 148 #ifdef AARCH64 149 case ltos: ldr(R0_tos, val_addr); break; 150 #else 151 case ltos: ldr(R1_tos_hi, val_addr_hi); // fall through 152 #endif // AARCH64 153 case btos: // fall through 154 case ztos: // fall through 155 case ctos: // fall through 156 case stos: // fall through 157 case itos: ldr_s32(R0_tos, val_addr); break; 158 #ifdef __SOFTFP__ 159 case dtos: ldr(R1_tos_hi, val_addr_hi); // fall through 160 case ftos: ldr(R0_tos, val_addr); break; 161 #else 162 case ftos: ldr_float (S0_tos, val_addr); break; 163 case dtos: ldr_double(D0_tos, val_addr); break; 164 #endif // __SOFTFP__ 165 case vtos: /* nothing to do */ break; 166 default : ShouldNotReachHere(); 167 } 168 // Clean up tos value in the thread object 169 str(zero, val_addr); 170 #ifndef AARCH64 171 str(zero, val_addr_hi); 172 #endif // !AARCH64 173 174 mov(Rtemp, (int) ilgl); 175 str_32(Rtemp, tos_addr); 176 } 177 178 179 // Blows R2, Rtemp. 180 void InterpreterMacroAssembler::check_and_handle_earlyret() { 181 if (can_force_early_return()) { 182 Label L; 183 const Register thread_state = R2_tmp; 184 185 ldr(thread_state, Address(Rthread, JavaThread::jvmti_thread_state_offset())); 186 cbz(thread_state, L); // if (thread->jvmti_thread_state() == NULL) exit; 187 188 // Initiate earlyret handling only if it is not already being processed. 189 // If the flag has the earlyret_processing bit set, it means that this code 190 // is called *during* earlyret handling - we don't want to reenter. 191 192 ldr_s32(Rtemp, Address(thread_state, JvmtiThreadState::earlyret_state_offset())); 193 cmp(Rtemp, JvmtiThreadState::earlyret_pending); 194 b(L, ne); 195 196 // Call Interpreter::remove_activation_early_entry() to get the address of the 197 // same-named entrypoint in the generated interpreter code. 198 199 ldr_s32(R0, Address(thread_state, JvmtiThreadState::earlyret_tos_offset())); 200 call_VM_leaf(CAST_FROM_FN_PTR(address, Interpreter::remove_activation_early_entry), R0); 201 202 jump(R0); 203 204 bind(L); 205 } 206 } 207 208 209 // Sets reg. Blows Rtemp. 210 void InterpreterMacroAssembler::get_unsigned_2_byte_index_at_bcp(Register reg, int bcp_offset) { 211 assert(bcp_offset >= 0, "bcp is still pointing to start of bytecode"); 212 assert(reg != Rtemp, "should be different registers"); 213 214 ldrb(Rtemp, Address(Rbcp, bcp_offset)); 215 ldrb(reg, Address(Rbcp, bcp_offset+1)); 216 orr(reg, reg, AsmOperand(Rtemp, lsl, BitsPerByte)); 217 } 218 219 void InterpreterMacroAssembler::get_index_at_bcp(Register index, int bcp_offset, Register tmp_reg, size_t index_size) { 220 assert_different_registers(index, tmp_reg); 221 if (index_size == sizeof(u2)) { 222 // load bytes of index separately to avoid unaligned access 223 ldrb(index, Address(Rbcp, bcp_offset+1)); 224 ldrb(tmp_reg, Address(Rbcp, bcp_offset)); 225 orr(index, tmp_reg, AsmOperand(index, lsl, BitsPerByte)); 226 } else if (index_size == sizeof(u4)) { 227 // TODO-AARCH64: consider using unaligned access here 228 ldrb(index, Address(Rbcp, bcp_offset+3)); 229 ldrb(tmp_reg, Address(Rbcp, bcp_offset+2)); 230 orr(index, tmp_reg, AsmOperand(index, lsl, BitsPerByte)); 231 ldrb(tmp_reg, Address(Rbcp, bcp_offset+1)); 232 orr(index, tmp_reg, AsmOperand(index, lsl, BitsPerByte)); 233 ldrb(tmp_reg, Address(Rbcp, bcp_offset)); 234 orr(index, tmp_reg, AsmOperand(index, lsl, BitsPerByte)); 235 // Check if the secondary index definition is still ~x, otherwise 236 // we have to change the following assembler code to calculate the 237 // plain index. 238 assert(ConstantPool::decode_invokedynamic_index(~123) == 123, "else change next line"); 239 mvn_32(index, index); // convert to plain index 240 } else if (index_size == sizeof(u1)) { 241 ldrb(index, Address(Rbcp, bcp_offset)); 242 } else { 243 ShouldNotReachHere(); 244 } 245 } 246 247 // Sets cache, index. 248 void InterpreterMacroAssembler::get_cache_and_index_at_bcp(Register cache, Register index, int bcp_offset, size_t index_size) { 249 assert(bcp_offset > 0, "bcp is still pointing to start of bytecode"); 250 assert_different_registers(cache, index); 251 252 get_index_at_bcp(index, bcp_offset, cache, index_size); 253 254 // load constant pool cache pointer 255 ldr(cache, Address(FP, frame::interpreter_frame_cache_offset * wordSize)); 256 257 // convert from field index to ConstantPoolCacheEntry index 258 assert(sizeof(ConstantPoolCacheEntry) == 4*wordSize, "adjust code below"); 259 // TODO-AARCH64 merge this shift with shift "add(..., Rcache, AsmOperand(Rindex, lsl, LogBytesPerWord))" after this method is called 260 logical_shift_left(index, index, 2); 261 } 262 263 // Sets cache, index, bytecode. 264 void InterpreterMacroAssembler::get_cache_and_index_and_bytecode_at_bcp(Register cache, Register index, Register bytecode, int byte_no, int bcp_offset, size_t index_size) { 265 get_cache_and_index_at_bcp(cache, index, bcp_offset, index_size); 266 // caution index and bytecode can be the same 267 add(bytecode, cache, AsmOperand(index, lsl, LogBytesPerWord)); 268 #ifdef AARCH64 269 add(bytecode, bytecode, (1 + byte_no) + in_bytes(ConstantPoolCache::base_offset() + ConstantPoolCacheEntry::indices_offset())); 270 ldarb(bytecode, bytecode); 271 #else 272 ldrb(bytecode, Address(bytecode, (1 + byte_no) + in_bytes(ConstantPoolCache::base_offset() + ConstantPoolCacheEntry::indices_offset()))); 273 TemplateTable::volatile_barrier(MacroAssembler::LoadLoad, noreg, true); 274 #endif // AARCH64 275 } 276 277 // Sets cache. Blows reg_tmp. 278 void InterpreterMacroAssembler::get_cache_entry_pointer_at_bcp(Register cache, Register reg_tmp, int bcp_offset, size_t index_size) { 279 assert(bcp_offset > 0, "bcp is still pointing to start of bytecode"); 280 assert_different_registers(cache, reg_tmp); 281 282 get_index_at_bcp(reg_tmp, bcp_offset, cache, index_size); 283 284 // load constant pool cache pointer 285 ldr(cache, Address(FP, frame::interpreter_frame_cache_offset * wordSize)); 286 287 // skip past the header 288 add(cache, cache, in_bytes(ConstantPoolCache::base_offset())); 289 // convert from field index to ConstantPoolCacheEntry index 290 // and from word offset to byte offset 291 assert(sizeof(ConstantPoolCacheEntry) == 4*wordSize, "adjust code below"); 292 add(cache, cache, AsmOperand(reg_tmp, lsl, 2 + LogBytesPerWord)); 293 } 294 295 // Load object from cpool->resolved_references(index) 296 void InterpreterMacroAssembler::load_resolved_reference_at_index( 297 Register result, Register index) { 298 assert_different_registers(result, index); 299 get_constant_pool(result); 300 301 Register cache = result; 302 // load pointer for resolved_references[] objArray 303 ldr(cache, Address(result, ConstantPool::cache_offset_in_bytes())); 304 ldr(cache, Address(result, ConstantPoolCache::resolved_references_offset_in_bytes())); 305 resolve_oop_handle(cache); 306 // Add in the index 307 // convert from field index to resolved_references() index and from 308 // word index to byte offset. Since this is a java object, it can be compressed 309 add(cache, cache, AsmOperand(index, lsl, LogBytesPerHeapOop)); 310 load_heap_oop(result, Address(cache, arrayOopDesc::base_offset_in_bytes(T_OBJECT))); 311 } 312 313 void InterpreterMacroAssembler::load_resolved_klass_at_offset( 314 Register Rcpool, Register Rindex, Register Rklass) { 315 add(Rtemp, Rcpool, AsmOperand(Rindex, lsl, LogBytesPerWord)); 316 ldrh(Rtemp, Address(Rtemp, sizeof(ConstantPool))); // Rtemp = resolved_klass_index 317 ldr(Rklass, Address(Rcpool, ConstantPool::resolved_klasses_offset_in_bytes())); // Rklass = cpool->_resolved_klasses 318 add(Rklass, Rklass, AsmOperand(Rtemp, lsl, LogBytesPerWord)); 319 ldr(Rklass, Address(Rklass, Array<Klass*>::base_offset_in_bytes())); 320 } 321 322 // Generate a subtype check: branch to not_subtype if sub_klass is 323 // not a subtype of super_klass. 324 // Profiling code for the subtype check failure (profile_typecheck_failed) 325 // should be explicitly generated by the caller in the not_subtype case. 326 // Blows Rtemp, tmp1, tmp2. 327 void InterpreterMacroAssembler::gen_subtype_check(Register Rsub_klass, 328 Register Rsuper_klass, 329 Label ¬_subtype, 330 Register tmp1, 331 Register tmp2) { 332 333 assert_different_registers(Rsub_klass, Rsuper_klass, tmp1, tmp2, Rtemp); 334 Label ok_is_subtype, loop, update_cache; 335 336 const Register super_check_offset = tmp1; 337 const Register cached_super = tmp2; 338 339 // Profile the not-null value's klass. 340 profile_typecheck(tmp1, Rsub_klass); 341 342 // Load the super-klass's check offset into 343 ldr_u32(super_check_offset, Address(Rsuper_klass, Klass::super_check_offset_offset())); 344 345 // Check for self 346 cmp(Rsub_klass, Rsuper_klass); 347 348 // Load from the sub-klass's super-class display list, or a 1-word cache of 349 // the secondary superclass list, or a failing value with a sentinel offset 350 // if the super-klass is an interface or exceptionally deep in the Java 351 // hierarchy and we have to scan the secondary superclass list the hard way. 352 // See if we get an immediate positive hit 353 ldr(cached_super, Address(Rsub_klass, super_check_offset)); 354 355 cond_cmp(Rsuper_klass, cached_super, ne); 356 b(ok_is_subtype, eq); 357 358 // Check for immediate negative hit 359 cmp(super_check_offset, in_bytes(Klass::secondary_super_cache_offset())); 360 b(not_subtype, ne); 361 362 // Now do a linear scan of the secondary super-klass chain. 363 const Register supers_arr = tmp1; 364 const Register supers_cnt = tmp2; 365 const Register cur_super = Rtemp; 366 367 // Load objArrayOop of secondary supers. 368 ldr(supers_arr, Address(Rsub_klass, Klass::secondary_supers_offset())); 369 370 ldr_u32(supers_cnt, Address(supers_arr, Array<Klass*>::length_offset_in_bytes())); // Load the array length 371 #ifdef AARCH64 372 cbz(supers_cnt, not_subtype); 373 add(supers_arr, supers_arr, Array<Klass*>::base_offset_in_bytes()); 374 #else 375 cmp(supers_cnt, 0); 376 377 // Skip to the start of array elements and prefetch the first super-klass. 378 ldr(cur_super, Address(supers_arr, Array<Klass*>::base_offset_in_bytes(), pre_indexed), ne); 379 b(not_subtype, eq); 380 #endif // AARCH64 381 382 bind(loop); 383 384 #ifdef AARCH64 385 ldr(cur_super, Address(supers_arr, wordSize, post_indexed)); 386 #endif // AARCH64 387 388 cmp(cur_super, Rsuper_klass); 389 b(update_cache, eq); 390 391 subs(supers_cnt, supers_cnt, 1); 392 393 #ifndef AARCH64 394 ldr(cur_super, Address(supers_arr, wordSize, pre_indexed), ne); 395 #endif // !AARCH64 396 397 b(loop, ne); 398 399 b(not_subtype); 400 401 bind(update_cache); 402 // Must be equal but missed in cache. Update cache. 403 str(Rsuper_klass, Address(Rsub_klass, Klass::secondary_super_cache_offset())); 404 405 bind(ok_is_subtype); 406 } 407 408 409 // The 1st part of the store check. 410 // Sets card_table_base register. 411 void InterpreterMacroAssembler::store_check_part1(Register card_table_base) { 412 // Check barrier set type (should be card table) and element size 413 BarrierSet* bs = BarrierSet::barrier_set(); 414 assert(bs->kind() == BarrierSet::CardTableBarrierSet, 415 "Wrong barrier set kind"); 416 417 CardTableBarrierSet* ctbs = barrier_set_cast<CardTableBarrierSet>(bs); 418 CardTable* ct = ctbs->card_table(); 419 assert(sizeof(*ct->byte_map_base()) == sizeof(jbyte), "Adjust store check code"); 420 421 // Load card table base address. 422 423 /* Performance note. 424 425 There is an alternative way of loading card table base address 426 from thread descriptor, which may look more efficient: 427 428 ldr(card_table_base, Address(Rthread, JavaThread::card_table_base_offset())); 429 430 However, performance measurements of micro benchmarks and specJVM98 431 showed that loading of card table base from thread descriptor is 432 7-18% slower compared to loading of literal embedded into the code. 433 Possible cause is a cache miss (card table base address resides in a 434 rarely accessed area of thread descriptor). 435 */ 436 // TODO-AARCH64 Investigate if mov_slow is faster than ldr from Rthread on AArch64 437 mov_address(card_table_base, (address)ct->byte_map_base(), symbolic_Relocation::card_table_reference); 438 } 439 440 // The 2nd part of the store check. 441 void InterpreterMacroAssembler::store_check_part2(Register obj, Register card_table_base, Register tmp) { 442 assert_different_registers(obj, card_table_base, tmp); 443 444 assert(CardTable::dirty_card_val() == 0, "Dirty card value must be 0 due to optimizations."); 445 #ifdef AARCH64 446 add(card_table_base, card_table_base, AsmOperand(obj, lsr, CardTable::card_shift)); 447 Address card_table_addr(card_table_base); 448 #else 449 Address card_table_addr(card_table_base, obj, lsr, CardTable::card_shift); 450 #endif 451 452 if (UseCondCardMark) { 453 #if INCLUDE_ALL_GCS 454 if (UseConcMarkSweepGC) { 455 membar(MacroAssembler::Membar_mask_bits(MacroAssembler::StoreLoad), noreg); 456 } 457 #endif 458 Label already_dirty; 459 460 ldrb(tmp, card_table_addr); 461 cbz(tmp, already_dirty); 462 463 set_card(card_table_base, card_table_addr, tmp); 464 bind(already_dirty); 465 466 } else { 467 #if INCLUDE_ALL_GCS 468 if (UseConcMarkSweepGC && CMSPrecleaningEnabled) { 469 membar(MacroAssembler::Membar_mask_bits(MacroAssembler::StoreStore), noreg); 470 } 471 #endif 472 set_card(card_table_base, card_table_addr, tmp); 473 } 474 } 475 476 void InterpreterMacroAssembler::set_card(Register card_table_base, Address card_table_addr, Register tmp) { 477 #ifdef AARCH64 478 strb(ZR, card_table_addr); 479 #else 480 CardTableBarrierSet* ctbs = barrier_set_cast<CardTableBarrierSet>(BarrierSet::barrier_set()); 481 CardTable* ct = ctbs->card_table(); 482 if ((((uintptr_t)ct->byte_map_base() & 0xff) == 0)) { 483 // Card table is aligned so the lowest byte of the table address base is zero. 484 // This works only if the code is not saved for later use, possibly 485 // in a context where the base would no longer be aligned. 486 strb(card_table_base, card_table_addr); 487 } else { 488 mov(tmp, 0); 489 strb(tmp, card_table_addr); 490 } 491 #endif // AARCH64 492 } 493 494 ////////////////////////////////////////////////////////////////////////////////// 495 496 497 // Java Expression Stack 498 499 void InterpreterMacroAssembler::pop_ptr(Register r) { 500 assert(r != Rstack_top, "unpredictable instruction"); 501 ldr(r, Address(Rstack_top, wordSize, post_indexed)); 502 } 503 504 void InterpreterMacroAssembler::pop_i(Register r) { 505 assert(r != Rstack_top, "unpredictable instruction"); 506 ldr_s32(r, Address(Rstack_top, wordSize, post_indexed)); 507 zap_high_non_significant_bits(r); 508 } 509 510 #ifdef AARCH64 511 void InterpreterMacroAssembler::pop_l(Register r) { 512 assert(r != Rstack_top, "unpredictable instruction"); 513 ldr(r, Address(Rstack_top, 2*wordSize, post_indexed)); 514 } 515 #else 516 void InterpreterMacroAssembler::pop_l(Register lo, Register hi) { 517 assert_different_registers(lo, hi); 518 assert(lo < hi, "lo must be < hi"); 519 pop(RegisterSet(lo) | RegisterSet(hi)); 520 } 521 #endif // AARCH64 522 523 void InterpreterMacroAssembler::pop_f(FloatRegister fd) { 524 #ifdef AARCH64 525 ldr_s(fd, Address(Rstack_top, wordSize, post_indexed)); 526 #else 527 fpops(fd); 528 #endif // AARCH64 529 } 530 531 void InterpreterMacroAssembler::pop_d(FloatRegister fd) { 532 #ifdef AARCH64 533 ldr_d(fd, Address(Rstack_top, 2*wordSize, post_indexed)); 534 #else 535 fpopd(fd); 536 #endif // AARCH64 537 } 538 539 540 // Transition vtos -> state. Blows R0, R1. Sets TOS cached value. 541 void InterpreterMacroAssembler::pop(TosState state) { 542 switch (state) { 543 case atos: pop_ptr(R0_tos); break; 544 case btos: // fall through 545 case ztos: // fall through 546 case ctos: // fall through 547 case stos: // fall through 548 case itos: pop_i(R0_tos); break; 549 #ifdef AARCH64 550 case ltos: pop_l(R0_tos); break; 551 #else 552 case ltos: pop_l(R0_tos_lo, R1_tos_hi); break; 553 #endif // AARCH64 554 #ifdef __SOFTFP__ 555 case ftos: pop_i(R0_tos); break; 556 case dtos: pop_l(R0_tos_lo, R1_tos_hi); break; 557 #else 558 case ftos: pop_f(S0_tos); break; 559 case dtos: pop_d(D0_tos); break; 560 #endif // __SOFTFP__ 561 case vtos: /* nothing to do */ break; 562 default : ShouldNotReachHere(); 563 } 564 interp_verify_oop(R0_tos, state, __FILE__, __LINE__); 565 } 566 567 void InterpreterMacroAssembler::push_ptr(Register r) { 568 assert(r != Rstack_top, "unpredictable instruction"); 569 str(r, Address(Rstack_top, -wordSize, pre_indexed)); 570 check_stack_top_on_expansion(); 571 } 572 573 void InterpreterMacroAssembler::push_i(Register r) { 574 assert(r != Rstack_top, "unpredictable instruction"); 575 str_32(r, Address(Rstack_top, -wordSize, pre_indexed)); 576 check_stack_top_on_expansion(); 577 } 578 579 #ifdef AARCH64 580 void InterpreterMacroAssembler::push_l(Register r) { 581 assert(r != Rstack_top, "unpredictable instruction"); 582 stp(r, ZR, Address(Rstack_top, -2*wordSize, pre_indexed)); 583 check_stack_top_on_expansion(); 584 } 585 #else 586 void InterpreterMacroAssembler::push_l(Register lo, Register hi) { 587 assert_different_registers(lo, hi); 588 assert(lo < hi, "lo must be < hi"); 589 push(RegisterSet(lo) | RegisterSet(hi)); 590 } 591 #endif // AARCH64 592 593 void InterpreterMacroAssembler::push_f() { 594 #ifdef AARCH64 595 str_s(S0_tos, Address(Rstack_top, -wordSize, pre_indexed)); 596 check_stack_top_on_expansion(); 597 #else 598 fpushs(S0_tos); 599 #endif // AARCH64 600 } 601 602 void InterpreterMacroAssembler::push_d() { 603 #ifdef AARCH64 604 str_d(D0_tos, Address(Rstack_top, -2*wordSize, pre_indexed)); 605 check_stack_top_on_expansion(); 606 #else 607 fpushd(D0_tos); 608 #endif // AARCH64 609 } 610 611 // Transition state -> vtos. Blows Rtemp. 612 void InterpreterMacroAssembler::push(TosState state) { 613 interp_verify_oop(R0_tos, state, __FILE__, __LINE__); 614 switch (state) { 615 case atos: push_ptr(R0_tos); break; 616 case btos: // fall through 617 case ztos: // fall through 618 case ctos: // fall through 619 case stos: // fall through 620 case itos: push_i(R0_tos); break; 621 #ifdef AARCH64 622 case ltos: push_l(R0_tos); break; 623 #else 624 case ltos: push_l(R0_tos_lo, R1_tos_hi); break; 625 #endif // AARCH64 626 #ifdef __SOFTFP__ 627 case ftos: push_i(R0_tos); break; 628 case dtos: push_l(R0_tos_lo, R1_tos_hi); break; 629 #else 630 case ftos: push_f(); break; 631 case dtos: push_d(); break; 632 #endif // __SOFTFP__ 633 case vtos: /* nothing to do */ break; 634 default : ShouldNotReachHere(); 635 } 636 } 637 638 639 #ifndef AARCH64 640 641 // Converts return value in R0/R1 (interpreter calling conventions) to TOS cached value. 642 void InterpreterMacroAssembler::convert_retval_to_tos(TosState state) { 643 #if (!defined __SOFTFP__ && !defined __ABI_HARD__) 644 // According to interpreter calling conventions, result is returned in R0/R1, 645 // but templates expect ftos in S0, and dtos in D0. 646 if (state == ftos) { 647 fmsr(S0_tos, R0); 648 } else if (state == dtos) { 649 fmdrr(D0_tos, R0, R1); 650 } 651 #endif // !__SOFTFP__ && !__ABI_HARD__ 652 } 653 654 // Converts TOS cached value to return value in R0/R1 (according to interpreter calling conventions). 655 void InterpreterMacroAssembler::convert_tos_to_retval(TosState state) { 656 #if (!defined __SOFTFP__ && !defined __ABI_HARD__) 657 // According to interpreter calling conventions, result is returned in R0/R1, 658 // so ftos (S0) and dtos (D0) are moved to R0/R1. 659 if (state == ftos) { 660 fmrs(R0, S0_tos); 661 } else if (state == dtos) { 662 fmrrd(R0, R1, D0_tos); 663 } 664 #endif // !__SOFTFP__ && !__ABI_HARD__ 665 } 666 667 #endif // !AARCH64 668 669 670 // Helpers for swap and dup 671 void InterpreterMacroAssembler::load_ptr(int n, Register val) { 672 ldr(val, Address(Rstack_top, Interpreter::expr_offset_in_bytes(n))); 673 } 674 675 void InterpreterMacroAssembler::store_ptr(int n, Register val) { 676 str(val, Address(Rstack_top, Interpreter::expr_offset_in_bytes(n))); 677 } 678 679 680 void InterpreterMacroAssembler::prepare_to_jump_from_interpreted() { 681 #ifdef AARCH64 682 check_no_cached_stack_top(Rtemp); 683 save_stack_top(); 684 cut_sp_before_call(); 685 mov(Rparams, Rstack_top); 686 #endif // AARCH64 687 688 // set sender sp 689 mov(Rsender_sp, SP); 690 691 #ifndef AARCH64 692 // record last_sp 693 str(Rsender_sp, Address(FP, frame::interpreter_frame_last_sp_offset * wordSize)); 694 #endif // !AARCH64 695 } 696 697 // Jump to from_interpreted entry of a call unless single stepping is possible 698 // in this thread in which case we must call the i2i entry 699 void InterpreterMacroAssembler::jump_from_interpreted(Register method) { 700 assert_different_registers(method, Rtemp); 701 702 prepare_to_jump_from_interpreted(); 703 704 if (can_post_interpreter_events()) { 705 // JVMTI events, such as single-stepping, are implemented partly by avoiding running 706 // compiled code in threads for which the event is enabled. Check here for 707 // interp_only_mode if these events CAN be enabled. 708 709 ldr_s32(Rtemp, Address(Rthread, JavaThread::interp_only_mode_offset())); 710 #ifdef AARCH64 711 { 712 Label not_interp_only_mode; 713 714 cbz(Rtemp, not_interp_only_mode); 715 indirect_jump(Address(method, Method::interpreter_entry_offset()), Rtemp); 716 717 bind(not_interp_only_mode); 718 } 719 #else 720 cmp(Rtemp, 0); 721 ldr(PC, Address(method, Method::interpreter_entry_offset()), ne); 722 #endif // AARCH64 723 } 724 725 indirect_jump(Address(method, Method::from_interpreted_offset()), Rtemp); 726 } 727 728 729 void InterpreterMacroAssembler::restore_dispatch() { 730 mov_slow(RdispatchTable, (address)Interpreter::dispatch_table(vtos)); 731 } 732 733 734 // The following two routines provide a hook so that an implementation 735 // can schedule the dispatch in two parts. 736 void InterpreterMacroAssembler::dispatch_prolog(TosState state, int step) { 737 // Nothing ARM-specific to be done here. 738 } 739 740 void InterpreterMacroAssembler::dispatch_epilog(TosState state, int step) { 741 dispatch_next(state, step); 742 } 743 744 void InterpreterMacroAssembler::dispatch_base(TosState state, 745 DispatchTableMode table_mode, 746 bool verifyoop) { 747 if (VerifyActivationFrameSize) { 748 Label L; 749 #ifdef AARCH64 750 mov(Rtemp, SP); 751 sub(Rtemp, FP, Rtemp); 752 #else 753 sub(Rtemp, FP, SP); 754 #endif // AARCH64 755 int min_frame_size = (frame::link_offset - frame::interpreter_frame_initial_sp_offset) * wordSize; 756 cmp(Rtemp, min_frame_size); 757 b(L, ge); 758 stop("broken stack frame"); 759 bind(L); 760 } 761 762 if (verifyoop) { 763 interp_verify_oop(R0_tos, state, __FILE__, __LINE__); 764 } 765 766 if((state == itos) || (state == btos) || (state == ztos) || (state == ctos) || (state == stos)) { 767 zap_high_non_significant_bits(R0_tos); 768 } 769 770 #ifdef ASSERT 771 Label L; 772 mov_slow(Rtemp, (address)Interpreter::dispatch_table(vtos)); 773 cmp(Rtemp, RdispatchTable); 774 b(L, eq); 775 stop("invalid RdispatchTable"); 776 bind(L); 777 #endif 778 779 if (table_mode == DispatchDefault) { 780 if (state == vtos) { 781 indirect_jump(Address::indexed_ptr(RdispatchTable, R3_bytecode), Rtemp); 782 } else { 783 #ifdef AARCH64 784 sub(Rtemp, R3_bytecode, (Interpreter::distance_from_dispatch_table(vtos) - 785 Interpreter::distance_from_dispatch_table(state))); 786 indirect_jump(Address::indexed_ptr(RdispatchTable, Rtemp), Rtemp); 787 #else 788 // on 32-bit ARM this method is faster than the one above. 789 sub(Rtemp, RdispatchTable, (Interpreter::distance_from_dispatch_table(vtos) - 790 Interpreter::distance_from_dispatch_table(state)) * wordSize); 791 indirect_jump(Address::indexed_ptr(Rtemp, R3_bytecode), Rtemp); 792 #endif 793 } 794 } else { 795 assert(table_mode == DispatchNormal, "invalid dispatch table mode"); 796 address table = (address) Interpreter::normal_table(state); 797 mov_slow(Rtemp, table); 798 indirect_jump(Address::indexed_ptr(Rtemp, R3_bytecode), Rtemp); 799 } 800 801 nop(); // to avoid filling CPU pipeline with invalid instructions 802 nop(); 803 } 804 805 void InterpreterMacroAssembler::dispatch_only(TosState state) { 806 dispatch_base(state, DispatchDefault); 807 } 808 809 810 void InterpreterMacroAssembler::dispatch_only_normal(TosState state) { 811 dispatch_base(state, DispatchNormal); 812 } 813 814 void InterpreterMacroAssembler::dispatch_only_noverify(TosState state) { 815 dispatch_base(state, DispatchNormal, false); 816 } 817 818 void InterpreterMacroAssembler::dispatch_next(TosState state, int step) { 819 // load next bytecode and advance Rbcp 820 ldrb(R3_bytecode, Address(Rbcp, step, pre_indexed)); 821 dispatch_base(state, DispatchDefault); 822 } 823 824 void InterpreterMacroAssembler::narrow(Register result) { 825 // mask integer result to narrower return type. 826 const Register Rtmp = R2; 827 828 // get method type 829 ldr(Rtmp, Address(Rmethod, Method::const_offset())); 830 ldrb(Rtmp, Address(Rtmp, ConstMethod::result_type_offset())); 831 832 Label notBool, notByte, notChar, done; 833 cmp(Rtmp, T_INT); 834 b(done, eq); 835 836 cmp(Rtmp, T_BOOLEAN); 837 b(notBool, ne); 838 and_32(result, result, 1); 839 b(done); 840 841 bind(notBool); 842 cmp(Rtmp, T_BYTE); 843 b(notByte, ne); 844 sign_extend(result, result, 8); 845 b(done); 846 847 bind(notByte); 848 cmp(Rtmp, T_CHAR); 849 b(notChar, ne); 850 zero_extend(result, result, 16); 851 b(done); 852 853 bind(notChar); 854 // cmp(Rtmp, T_SHORT); 855 // b(done, ne); 856 sign_extend(result, result, 16); 857 858 // Nothing to do 859 bind(done); 860 } 861 862 // remove activation 863 // 864 // Unlock the receiver if this is a synchronized method. 865 // Unlock any Java monitors from syncronized blocks. 866 // Remove the activation from the stack. 867 // 868 // If there are locked Java monitors 869 // If throw_monitor_exception 870 // throws IllegalMonitorStateException 871 // Else if install_monitor_exception 872 // installs IllegalMonitorStateException 873 // Else 874 // no error processing 875 void InterpreterMacroAssembler::remove_activation(TosState state, Register ret_addr, 876 bool throw_monitor_exception, 877 bool install_monitor_exception, 878 bool notify_jvmdi) { 879 Label unlock, unlocked, no_unlock; 880 881 // Note: Registers R0, R1, S0 and D0 (TOS cached value) may be in use for the result. 882 883 const Address do_not_unlock_if_synchronized(Rthread, 884 JavaThread::do_not_unlock_if_synchronized_offset()); 885 886 const Register Rflag = R2; 887 const Register Raccess_flags = R3; 888 889 restore_method(); 890 891 ldrb(Rflag, do_not_unlock_if_synchronized); 892 893 // get method access flags 894 ldr_u32(Raccess_flags, Address(Rmethod, Method::access_flags_offset())); 895 896 strb(zero_register(Rtemp), do_not_unlock_if_synchronized); // reset the flag 897 898 // check if method is synchronized 899 900 tbz(Raccess_flags, JVM_ACC_SYNCHRONIZED_BIT, unlocked); 901 902 // Don't unlock anything if the _do_not_unlock_if_synchronized flag is set. 903 cbnz(Rflag, no_unlock); 904 905 // unlock monitor 906 push(state); // save result 907 908 // BasicObjectLock will be first in list, since this is a synchronized method. However, need 909 // to check that the object has not been unlocked by an explicit monitorexit bytecode. 910 911 const Register Rmonitor = R1; // fixed in unlock_object() 912 const Register Robj = R2; 913 914 // address of first monitor 915 sub(Rmonitor, FP, - frame::interpreter_frame_monitor_block_bottom_offset * wordSize + (int)sizeof(BasicObjectLock)); 916 917 ldr(Robj, Address(Rmonitor, BasicObjectLock::obj_offset_in_bytes())); 918 cbnz(Robj, unlock); 919 920 pop(state); 921 922 if (throw_monitor_exception) { 923 // Entry already unlocked, need to throw exception 924 call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::throw_illegal_monitor_state_exception)); 925 should_not_reach_here(); 926 } else { 927 // Monitor already unlocked during a stack unroll. 928 // If requested, install an illegal_monitor_state_exception. 929 // Continue with stack unrolling. 930 if (install_monitor_exception) { 931 call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::new_illegal_monitor_state_exception)); 932 } 933 b(unlocked); 934 } 935 936 937 // Exception case for the check that all monitors are unlocked. 938 const Register Rcur = R2; 939 Label restart_check_monitors_unlocked, exception_monitor_is_still_locked; 940 941 bind(exception_monitor_is_still_locked); 942 // Monitor entry is still locked, need to throw exception. 943 // Rcur: monitor entry. 944 945 if (throw_monitor_exception) { 946 // Throw exception 947 call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::throw_illegal_monitor_state_exception)); 948 should_not_reach_here(); 949 } else { 950 // Stack unrolling. Unlock object and install illegal_monitor_exception 951 // Unlock does not block, so don't have to worry about the frame 952 953 push(state); 954 mov(R1, Rcur); 955 unlock_object(R1); 956 957 if (install_monitor_exception) { 958 call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::new_illegal_monitor_state_exception)); 959 } 960 961 pop(state); 962 b(restart_check_monitors_unlocked); 963 } 964 965 bind(unlock); 966 unlock_object(Rmonitor); 967 pop(state); 968 969 // Check that for block-structured locking (i.e., that all locked objects has been unlocked) 970 bind(unlocked); 971 972 // Check that all monitors are unlocked 973 { 974 Label loop; 975 976 const int entry_size = frame::interpreter_frame_monitor_size() * wordSize; 977 const Register Rbottom = R3; 978 const Register Rcur_obj = Rtemp; 979 980 bind(restart_check_monitors_unlocked); 981 982 ldr(Rcur, Address(FP, frame::interpreter_frame_monitor_block_top_offset * wordSize)); 983 // points to current entry, starting with top-most entry 984 sub(Rbottom, FP, -frame::interpreter_frame_monitor_block_bottom_offset * wordSize); 985 // points to word before bottom of monitor block 986 987 cmp(Rcur, Rbottom); // check if there are no monitors 988 #ifndef AARCH64 989 ldr(Rcur_obj, Address(Rcur, BasicObjectLock::obj_offset_in_bytes()), ne); 990 // prefetch monitor's object 991 #endif // !AARCH64 992 b(no_unlock, eq); 993 994 bind(loop); 995 #ifdef AARCH64 996 ldr(Rcur_obj, Address(Rcur, BasicObjectLock::obj_offset_in_bytes())); 997 #endif // AARCH64 998 // check if current entry is used 999 cbnz(Rcur_obj, exception_monitor_is_still_locked); 1000 1001 add(Rcur, Rcur, entry_size); // otherwise advance to next entry 1002 cmp(Rcur, Rbottom); // check if bottom reached 1003 #ifndef AARCH64 1004 ldr(Rcur_obj, Address(Rcur, BasicObjectLock::obj_offset_in_bytes()), ne); 1005 // prefetch monitor's object 1006 #endif // !AARCH64 1007 b(loop, ne); // if not at bottom then check this entry 1008 } 1009 1010 bind(no_unlock); 1011 1012 // jvmti support 1013 if (notify_jvmdi) { 1014 notify_method_exit(state, NotifyJVMTI); // preserve TOSCA 1015 } else { 1016 notify_method_exit(state, SkipNotifyJVMTI); // preserve TOSCA 1017 } 1018 1019 // remove activation 1020 #ifdef AARCH64 1021 ldr(Rtemp, Address(FP, frame::interpreter_frame_sender_sp_offset * wordSize)); 1022 ldp(FP, LR, Address(FP)); 1023 mov(SP, Rtemp); 1024 #else 1025 mov(Rtemp, FP); 1026 ldmia(FP, RegisterSet(FP) | RegisterSet(LR)); 1027 ldr(SP, Address(Rtemp, frame::interpreter_frame_sender_sp_offset * wordSize)); 1028 #endif 1029 1030 if (ret_addr != LR) { 1031 mov(ret_addr, LR); 1032 } 1033 } 1034 1035 1036 // At certain points in the method invocation the monitor of 1037 // synchronized methods hasn't been entered yet. 1038 // To correctly handle exceptions at these points, we set the thread local 1039 // variable _do_not_unlock_if_synchronized to true. The remove_activation will 1040 // check this flag. 1041 void InterpreterMacroAssembler::set_do_not_unlock_if_synchronized(bool flag, Register tmp) { 1042 const Address do_not_unlock_if_synchronized(Rthread, 1043 JavaThread::do_not_unlock_if_synchronized_offset()); 1044 if (flag) { 1045 mov(tmp, 1); 1046 strb(tmp, do_not_unlock_if_synchronized); 1047 } else { 1048 strb(zero_register(tmp), do_not_unlock_if_synchronized); 1049 } 1050 } 1051 1052 // Lock object 1053 // 1054 // Argument: R1 : Points to BasicObjectLock to be used for locking. 1055 // Must be initialized with object to lock. 1056 // Blows volatile registers (R0-R3 on 32-bit ARM, R0-R18 on AArch64), Rtemp, LR. Calls VM. 1057 void InterpreterMacroAssembler::lock_object(Register Rlock) { 1058 assert(Rlock == R1, "the second argument"); 1059 1060 if (UseHeavyMonitors) { 1061 call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::monitorenter), Rlock); 1062 } else { 1063 Label done; 1064 1065 const Register Robj = R2; 1066 const Register Rmark = R3; 1067 assert_different_registers(Robj, Rmark, Rlock, R0, Rtemp); 1068 1069 const int obj_offset = BasicObjectLock::obj_offset_in_bytes(); 1070 const int lock_offset = BasicObjectLock::lock_offset_in_bytes (); 1071 const int mark_offset = lock_offset + BasicLock::displaced_header_offset_in_bytes(); 1072 1073 Label already_locked, slow_case; 1074 1075 // Load object pointer 1076 ldr(Robj, Address(Rlock, obj_offset)); 1077 1078 if (UseBiasedLocking) { 1079 biased_locking_enter(Robj, Rmark/*scratched*/, R0, false, Rtemp, done, slow_case); 1080 } 1081 1082 #ifdef AARCH64 1083 assert(oopDesc::mark_offset_in_bytes() == 0, "must be"); 1084 ldr(Rmark, Robj); 1085 1086 // Test if object is already locked 1087 assert(markOopDesc::unlocked_value == 1, "adjust this code"); 1088 tbz(Rmark, exact_log2(markOopDesc::unlocked_value), already_locked); 1089 1090 #else // AARCH64 1091 1092 // On MP platforms the next load could return a 'stale' value if the memory location has been modified by another thread. 1093 // That would be acceptable as ether CAS or slow case path is taken in that case. 1094 // Exception to that is if the object is locked by the calling thread, then the recursive test will pass (guaranteed as 1095 // loads are satisfied from a store queue if performed on the same processor). 1096 1097 assert(oopDesc::mark_offset_in_bytes() == 0, "must be"); 1098 ldr(Rmark, Address(Robj, oopDesc::mark_offset_in_bytes())); 1099 1100 // Test if object is already locked 1101 tst(Rmark, markOopDesc::unlocked_value); 1102 b(already_locked, eq); 1103 1104 #endif // !AARCH64 1105 // Save old object->mark() into BasicLock's displaced header 1106 str(Rmark, Address(Rlock, mark_offset)); 1107 1108 cas_for_lock_acquire(Rmark, Rlock, Robj, Rtemp, slow_case); 1109 1110 #ifndef PRODUCT 1111 if (PrintBiasedLockingStatistics) { 1112 cond_atomic_inc32(al, BiasedLocking::fast_path_entry_count_addr()); 1113 } 1114 #endif //!PRODUCT 1115 1116 b(done); 1117 1118 // If we got here that means the object is locked by ether calling thread or another thread. 1119 bind(already_locked); 1120 // Handling of locked objects: recursive locks and slow case. 1121 1122 // Fast check for recursive lock. 1123 // 1124 // Can apply the optimization only if this is a stack lock 1125 // allocated in this thread. For efficiency, we can focus on 1126 // recently allocated stack locks (instead of reading the stack 1127 // base and checking whether 'mark' points inside the current 1128 // thread stack): 1129 // 1) (mark & 3) == 0 1130 // 2) SP <= mark < SP + os::pagesize() 1131 // 1132 // Warning: SP + os::pagesize can overflow the stack base. We must 1133 // neither apply the optimization for an inflated lock allocated 1134 // just above the thread stack (this is why condition 1 matters) 1135 // nor apply the optimization if the stack lock is inside the stack 1136 // of another thread. The latter is avoided even in case of overflow 1137 // because we have guard pages at the end of all stacks. Hence, if 1138 // we go over the stack base and hit the stack of another thread, 1139 // this should not be in a writeable area that could contain a 1140 // stack lock allocated by that thread. As a consequence, a stack 1141 // lock less than page size away from SP is guaranteed to be 1142 // owned by the current thread. 1143 // 1144 // Note: assuming SP is aligned, we can check the low bits of 1145 // (mark-SP) instead of the low bits of mark. In that case, 1146 // assuming page size is a power of 2, we can merge the two 1147 // conditions into a single test: 1148 // => ((mark - SP) & (3 - os::pagesize())) == 0 1149 1150 #ifdef AARCH64 1151 // Use the single check since the immediate is OK for AARCH64 1152 sub(R0, Rmark, Rstack_top); 1153 intptr_t mask = ((intptr_t)3) - ((intptr_t)os::vm_page_size()); 1154 Assembler::LogicalImmediate imm(mask, false); 1155 ands(R0, R0, imm); 1156 1157 // For recursive case store 0 into lock record. 1158 // It is harmless to store it unconditionally as lock record contains some garbage 1159 // value in its _displaced_header field by this moment. 1160 str(ZR, Address(Rlock, mark_offset)); 1161 1162 #else // AARCH64 1163 // (3 - os::pagesize()) cannot be encoded as an ARM immediate operand. 1164 // Check independently the low bits and the distance to SP. 1165 // -1- test low 2 bits 1166 movs(R0, AsmOperand(Rmark, lsl, 30)); 1167 // -2- test (mark - SP) if the low two bits are 0 1168 sub(R0, Rmark, SP, eq); 1169 movs(R0, AsmOperand(R0, lsr, exact_log2(os::vm_page_size())), eq); 1170 // If still 'eq' then recursive locking OK: store 0 into lock record 1171 str(R0, Address(Rlock, mark_offset), eq); 1172 1173 #endif // AARCH64 1174 1175 #ifndef PRODUCT 1176 if (PrintBiasedLockingStatistics) { 1177 cond_atomic_inc32(eq, BiasedLocking::fast_path_entry_count_addr()); 1178 } 1179 #endif // !PRODUCT 1180 1181 b(done, eq); 1182 1183 bind(slow_case); 1184 1185 // Call the runtime routine for slow case 1186 call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::monitorenter), Rlock); 1187 1188 bind(done); 1189 } 1190 } 1191 1192 1193 // Unlocks an object. Used in monitorexit bytecode and remove_activation. 1194 // 1195 // Argument: R1: Points to BasicObjectLock structure for lock 1196 // Throw an IllegalMonitorException if object is not locked by current thread 1197 // Blows volatile registers (R0-R3 on 32-bit ARM, R0-R18 on AArch64), Rtemp, LR. Calls VM. 1198 void InterpreterMacroAssembler::unlock_object(Register Rlock) { 1199 assert(Rlock == R1, "the second argument"); 1200 1201 if (UseHeavyMonitors) { 1202 call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::monitorexit), Rlock); 1203 } else { 1204 Label done, slow_case; 1205 1206 const Register Robj = R2; 1207 const Register Rmark = R3; 1208 const Register Rresult = R0; 1209 assert_different_registers(Robj, Rmark, Rlock, R0, Rtemp); 1210 1211 const int obj_offset = BasicObjectLock::obj_offset_in_bytes(); 1212 const int lock_offset = BasicObjectLock::lock_offset_in_bytes (); 1213 const int mark_offset = lock_offset + BasicLock::displaced_header_offset_in_bytes(); 1214 1215 const Register Rzero = zero_register(Rtemp); 1216 1217 // Load oop into Robj 1218 ldr(Robj, Address(Rlock, obj_offset)); 1219 1220 // Free entry 1221 str(Rzero, Address(Rlock, obj_offset)); 1222 1223 if (UseBiasedLocking) { 1224 biased_locking_exit(Robj, Rmark, done); 1225 } 1226 1227 // Load the old header from BasicLock structure 1228 ldr(Rmark, Address(Rlock, mark_offset)); 1229 1230 // Test for recursion (zero mark in BasicLock) 1231 cbz(Rmark, done); 1232 1233 bool allow_fallthrough_on_failure = true; 1234 1235 cas_for_lock_release(Rlock, Rmark, Robj, Rtemp, slow_case, allow_fallthrough_on_failure); 1236 1237 b(done, eq); 1238 1239 bind(slow_case); 1240 1241 // Call the runtime routine for slow case. 1242 str(Robj, Address(Rlock, obj_offset)); // restore obj 1243 call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::monitorexit), Rlock); 1244 1245 bind(done); 1246 } 1247 } 1248 1249 1250 // Test ImethodDataPtr. If it is null, continue at the specified label 1251 void InterpreterMacroAssembler::test_method_data_pointer(Register mdp, Label& zero_continue) { 1252 assert(ProfileInterpreter, "must be profiling interpreter"); 1253 ldr(mdp, Address(FP, frame::interpreter_frame_mdp_offset * wordSize)); 1254 cbz(mdp, zero_continue); 1255 } 1256 1257 1258 // Set the method data pointer for the current bcp. 1259 // Blows volatile registers (R0-R3 on 32-bit ARM, R0-R18 on AArch64), Rtemp, LR. 1260 void InterpreterMacroAssembler::set_method_data_pointer_for_bcp() { 1261 assert(ProfileInterpreter, "must be profiling interpreter"); 1262 Label set_mdp; 1263 1264 // Test MDO to avoid the call if it is NULL. 1265 ldr(Rtemp, Address(Rmethod, Method::method_data_offset())); 1266 cbz(Rtemp, set_mdp); 1267 1268 mov(R0, Rmethod); 1269 mov(R1, Rbcp); 1270 call_VM_leaf(CAST_FROM_FN_PTR(address, InterpreterRuntime::bcp_to_di), R0, R1); 1271 // R0/W0: mdi 1272 1273 // mdo is guaranteed to be non-zero here, we checked for it before the call. 1274 ldr(Rtemp, Address(Rmethod, Method::method_data_offset())); 1275 add(Rtemp, Rtemp, in_bytes(MethodData::data_offset())); 1276 add_ptr_scaled_int32(Rtemp, Rtemp, R0, 0); 1277 1278 bind(set_mdp); 1279 str(Rtemp, Address(FP, frame::interpreter_frame_mdp_offset * wordSize)); 1280 } 1281 1282 1283 void InterpreterMacroAssembler::verify_method_data_pointer() { 1284 assert(ProfileInterpreter, "must be profiling interpreter"); 1285 #ifdef ASSERT 1286 Label verify_continue; 1287 save_caller_save_registers(); 1288 1289 const Register Rmdp = R2; 1290 test_method_data_pointer(Rmdp, verify_continue); // If mdp is zero, continue 1291 1292 // If the mdp is valid, it will point to a DataLayout header which is 1293 // consistent with the bcp. The converse is highly probable also. 1294 1295 ldrh(R3, Address(Rmdp, DataLayout::bci_offset())); 1296 ldr(Rtemp, Address(Rmethod, Method::const_offset())); 1297 add(R3, R3, Rtemp); 1298 add(R3, R3, in_bytes(ConstMethod::codes_offset())); 1299 cmp(R3, Rbcp); 1300 b(verify_continue, eq); 1301 1302 mov(R0, Rmethod); 1303 mov(R1, Rbcp); 1304 call_VM_leaf(CAST_FROM_FN_PTR(address, InterpreterRuntime::verify_mdp), R0, R1, Rmdp); 1305 1306 bind(verify_continue); 1307 restore_caller_save_registers(); 1308 #endif // ASSERT 1309 } 1310 1311 1312 void InterpreterMacroAssembler::set_mdp_data_at(Register mdp_in, int offset, Register value) { 1313 assert(ProfileInterpreter, "must be profiling interpreter"); 1314 assert_different_registers(mdp_in, value); 1315 str(value, Address(mdp_in, offset)); 1316 } 1317 1318 1319 // Increments mdp data. Sets bumped_count register to adjusted counter. 1320 void InterpreterMacroAssembler::increment_mdp_data_at(Register mdp_in, 1321 int offset, 1322 Register bumped_count, 1323 bool decrement) { 1324 assert(ProfileInterpreter, "must be profiling interpreter"); 1325 1326 // Counter address 1327 Address data(mdp_in, offset); 1328 assert_different_registers(mdp_in, bumped_count); 1329 1330 increment_mdp_data_at(data, bumped_count, decrement); 1331 } 1332 1333 void InterpreterMacroAssembler::set_mdp_flag_at(Register mdp_in, int flag_byte_constant) { 1334 assert_different_registers(mdp_in, Rtemp); 1335 assert(ProfileInterpreter, "must be profiling interpreter"); 1336 assert((0 < flag_byte_constant) && (flag_byte_constant < (1 << BitsPerByte)), "flag mask is out of range"); 1337 1338 // Set the flag 1339 ldrb(Rtemp, Address(mdp_in, in_bytes(DataLayout::flags_offset()))); 1340 orr(Rtemp, Rtemp, (unsigned)flag_byte_constant); 1341 strb(Rtemp, Address(mdp_in, in_bytes(DataLayout::flags_offset()))); 1342 } 1343 1344 1345 // Increments mdp data. Sets bumped_count register to adjusted counter. 1346 void InterpreterMacroAssembler::increment_mdp_data_at(Address data, 1347 Register bumped_count, 1348 bool decrement) { 1349 assert(ProfileInterpreter, "must be profiling interpreter"); 1350 1351 ldr(bumped_count, data); 1352 if (decrement) { 1353 // Decrement the register. Set condition codes. 1354 subs(bumped_count, bumped_count, DataLayout::counter_increment); 1355 // Avoid overflow. 1356 #ifdef AARCH64 1357 assert(DataLayout::counter_increment == 1, "required for cinc"); 1358 cinc(bumped_count, bumped_count, pl); 1359 #else 1360 add(bumped_count, bumped_count, DataLayout::counter_increment, pl); 1361 #endif // AARCH64 1362 } else { 1363 // Increment the register. Set condition codes. 1364 adds(bumped_count, bumped_count, DataLayout::counter_increment); 1365 // Avoid overflow. 1366 #ifdef AARCH64 1367 assert(DataLayout::counter_increment == 1, "required for cinv"); 1368 cinv(bumped_count, bumped_count, mi); // inverts 0x80..00 back to 0x7f..ff 1369 #else 1370 sub(bumped_count, bumped_count, DataLayout::counter_increment, mi); 1371 #endif // AARCH64 1372 } 1373 str(bumped_count, data); 1374 } 1375 1376 1377 void InterpreterMacroAssembler::test_mdp_data_at(Register mdp_in, 1378 int offset, 1379 Register value, 1380 Register test_value_out, 1381 Label& not_equal_continue) { 1382 assert(ProfileInterpreter, "must be profiling interpreter"); 1383 assert_different_registers(mdp_in, test_value_out, value); 1384 1385 ldr(test_value_out, Address(mdp_in, offset)); 1386 cmp(test_value_out, value); 1387 1388 b(not_equal_continue, ne); 1389 } 1390 1391 1392 void InterpreterMacroAssembler::update_mdp_by_offset(Register mdp_in, int offset_of_disp, Register reg_temp) { 1393 assert(ProfileInterpreter, "must be profiling interpreter"); 1394 assert_different_registers(mdp_in, reg_temp); 1395 1396 ldr(reg_temp, Address(mdp_in, offset_of_disp)); 1397 add(mdp_in, mdp_in, reg_temp); 1398 str(mdp_in, Address(FP, frame::interpreter_frame_mdp_offset * wordSize)); 1399 } 1400 1401 1402 void InterpreterMacroAssembler::update_mdp_by_offset(Register mdp_in, Register reg_offset, Register reg_tmp) { 1403 assert(ProfileInterpreter, "must be profiling interpreter"); 1404 assert_different_registers(mdp_in, reg_offset, reg_tmp); 1405 1406 ldr(reg_tmp, Address(mdp_in, reg_offset)); 1407 add(mdp_in, mdp_in, reg_tmp); 1408 str(mdp_in, Address(FP, frame::interpreter_frame_mdp_offset * wordSize)); 1409 } 1410 1411 1412 void InterpreterMacroAssembler::update_mdp_by_constant(Register mdp_in, int constant) { 1413 assert(ProfileInterpreter, "must be profiling interpreter"); 1414 add(mdp_in, mdp_in, constant); 1415 str(mdp_in, Address(FP, frame::interpreter_frame_mdp_offset * wordSize)); 1416 } 1417 1418 1419 // Blows volatile registers (R0-R3 on 32-bit ARM, R0-R18 on AArch64, Rtemp, LR). 1420 void InterpreterMacroAssembler::update_mdp_for_ret(Register return_bci) { 1421 assert(ProfileInterpreter, "must be profiling interpreter"); 1422 assert_different_registers(return_bci, R0, R1, R2, R3, Rtemp); 1423 1424 mov(R1, return_bci); 1425 call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::update_mdp_for_ret), R1); 1426 } 1427 1428 1429 // Sets mdp, bumped_count registers, blows Rtemp. 1430 void InterpreterMacroAssembler::profile_taken_branch(Register mdp, Register bumped_count) { 1431 assert_different_registers(mdp, bumped_count); 1432 1433 if (ProfileInterpreter) { 1434 Label profile_continue; 1435 1436 // If no method data exists, go to profile_continue. 1437 // Otherwise, assign to mdp 1438 test_method_data_pointer(mdp, profile_continue); 1439 1440 // We are taking a branch. Increment the taken count. 1441 increment_mdp_data_at(mdp, in_bytes(JumpData::taken_offset()), bumped_count); 1442 1443 // The method data pointer needs to be updated to reflect the new target. 1444 update_mdp_by_offset(mdp, in_bytes(JumpData::displacement_offset()), Rtemp); 1445 1446 bind (profile_continue); 1447 } 1448 } 1449 1450 1451 // Sets mdp, blows Rtemp. 1452 void InterpreterMacroAssembler::profile_not_taken_branch(Register mdp) { 1453 assert_different_registers(mdp, Rtemp); 1454 1455 if (ProfileInterpreter) { 1456 Label profile_continue; 1457 1458 // If no method data exists, go to profile_continue. 1459 test_method_data_pointer(mdp, profile_continue); 1460 1461 // We are taking a branch. Increment the not taken count. 1462 increment_mdp_data_at(mdp, in_bytes(BranchData::not_taken_offset()), Rtemp); 1463 1464 // The method data pointer needs to be updated to correspond to the next bytecode 1465 update_mdp_by_constant(mdp, in_bytes(BranchData::branch_data_size())); 1466 1467 bind (profile_continue); 1468 } 1469 } 1470 1471 1472 // Sets mdp, blows Rtemp. 1473 void InterpreterMacroAssembler::profile_call(Register mdp) { 1474 assert_different_registers(mdp, Rtemp); 1475 1476 if (ProfileInterpreter) { 1477 Label profile_continue; 1478 1479 // If no method data exists, go to profile_continue. 1480 test_method_data_pointer(mdp, profile_continue); 1481 1482 // We are making a call. Increment the count. 1483 increment_mdp_data_at(mdp, in_bytes(CounterData::count_offset()), Rtemp); 1484 1485 // The method data pointer needs to be updated to reflect the new target. 1486 update_mdp_by_constant(mdp, in_bytes(CounterData::counter_data_size())); 1487 1488 bind (profile_continue); 1489 } 1490 } 1491 1492 1493 // Sets mdp, blows Rtemp. 1494 void InterpreterMacroAssembler::profile_final_call(Register mdp) { 1495 if (ProfileInterpreter) { 1496 Label profile_continue; 1497 1498 // If no method data exists, go to profile_continue. 1499 test_method_data_pointer(mdp, profile_continue); 1500 1501 // We are making a call. Increment the count. 1502 increment_mdp_data_at(mdp, in_bytes(CounterData::count_offset()), Rtemp); 1503 1504 // The method data pointer needs to be updated to reflect the new target. 1505 update_mdp_by_constant(mdp, in_bytes(VirtualCallData::virtual_call_data_size())); 1506 1507 bind (profile_continue); 1508 } 1509 } 1510 1511 1512 // Sets mdp, blows Rtemp. 1513 void InterpreterMacroAssembler::profile_virtual_call(Register mdp, Register receiver, bool receiver_can_be_null) { 1514 assert_different_registers(mdp, receiver, Rtemp); 1515 1516 if (ProfileInterpreter) { 1517 Label profile_continue; 1518 1519 // If no method data exists, go to profile_continue. 1520 test_method_data_pointer(mdp, profile_continue); 1521 1522 Label skip_receiver_profile; 1523 if (receiver_can_be_null) { 1524 Label not_null; 1525 cbnz(receiver, not_null); 1526 // We are making a call. Increment the count for null receiver. 1527 increment_mdp_data_at(mdp, in_bytes(CounterData::count_offset()), Rtemp); 1528 b(skip_receiver_profile); 1529 bind(not_null); 1530 } 1531 1532 // Record the receiver type. 1533 record_klass_in_profile(receiver, mdp, Rtemp, true); 1534 bind(skip_receiver_profile); 1535 1536 // The method data pointer needs to be updated to reflect the new target. 1537 update_mdp_by_constant(mdp, in_bytes(VirtualCallData::virtual_call_data_size())); 1538 bind(profile_continue); 1539 } 1540 } 1541 1542 1543 void InterpreterMacroAssembler::record_klass_in_profile_helper( 1544 Register receiver, Register mdp, 1545 Register reg_tmp, 1546 int start_row, Label& done, bool is_virtual_call) { 1547 if (TypeProfileWidth == 0) 1548 return; 1549 1550 assert_different_registers(receiver, mdp, reg_tmp); 1551 1552 int last_row = VirtualCallData::row_limit() - 1; 1553 assert(start_row <= last_row, "must be work left to do"); 1554 // Test this row for both the receiver and for null. 1555 // Take any of three different outcomes: 1556 // 1. found receiver => increment count and goto done 1557 // 2. found null => keep looking for case 1, maybe allocate this cell 1558 // 3. found something else => keep looking for cases 1 and 2 1559 // Case 3 is handled by a recursive call. 1560 for (int row = start_row; row <= last_row; row++) { 1561 Label next_test; 1562 1563 // See if the receiver is receiver[n]. 1564 int recvr_offset = in_bytes(VirtualCallData::receiver_offset(row)); 1565 1566 test_mdp_data_at(mdp, recvr_offset, receiver, reg_tmp, next_test); 1567 1568 // The receiver is receiver[n]. Increment count[n]. 1569 int count_offset = in_bytes(VirtualCallData::receiver_count_offset(row)); 1570 increment_mdp_data_at(mdp, count_offset, reg_tmp); 1571 b(done); 1572 1573 bind(next_test); 1574 // reg_tmp now contains the receiver from the CallData. 1575 1576 if (row == start_row) { 1577 Label found_null; 1578 // Failed the equality check on receiver[n]... Test for null. 1579 if (start_row == last_row) { 1580 // The only thing left to do is handle the null case. 1581 if (is_virtual_call) { 1582 cbz(reg_tmp, found_null); 1583 // Receiver did not match any saved receiver and there is no empty row for it. 1584 // Increment total counter to indicate polymorphic case. 1585 increment_mdp_data_at(mdp, in_bytes(CounterData::count_offset()), reg_tmp); 1586 b(done); 1587 bind(found_null); 1588 } else { 1589 cbnz(reg_tmp, done); 1590 } 1591 break; 1592 } 1593 // Since null is rare, make it be the branch-taken case. 1594 cbz(reg_tmp, found_null); 1595 1596 // Put all the "Case 3" tests here. 1597 record_klass_in_profile_helper(receiver, mdp, reg_tmp, start_row + 1, done, is_virtual_call); 1598 1599 // Found a null. Keep searching for a matching receiver, 1600 // but remember that this is an empty (unused) slot. 1601 bind(found_null); 1602 } 1603 } 1604 1605 // In the fall-through case, we found no matching receiver, but we 1606 // observed the receiver[start_row] is NULL. 1607 1608 // Fill in the receiver field and increment the count. 1609 int recvr_offset = in_bytes(VirtualCallData::receiver_offset(start_row)); 1610 set_mdp_data_at(mdp, recvr_offset, receiver); 1611 int count_offset = in_bytes(VirtualCallData::receiver_count_offset(start_row)); 1612 mov(reg_tmp, DataLayout::counter_increment); 1613 set_mdp_data_at(mdp, count_offset, reg_tmp); 1614 if (start_row > 0) { 1615 b(done); 1616 } 1617 } 1618 1619 void InterpreterMacroAssembler::record_klass_in_profile(Register receiver, 1620 Register mdp, 1621 Register reg_tmp, 1622 bool is_virtual_call) { 1623 assert(ProfileInterpreter, "must be profiling"); 1624 assert_different_registers(receiver, mdp, reg_tmp); 1625 1626 Label done; 1627 1628 record_klass_in_profile_helper(receiver, mdp, reg_tmp, 0, done, is_virtual_call); 1629 1630 bind (done); 1631 } 1632 1633 // Sets mdp, blows volatile registers (R0-R3 on 32-bit ARM, R0-R18 on AArch64, Rtemp, LR). 1634 void InterpreterMacroAssembler::profile_ret(Register mdp, Register return_bci) { 1635 assert_different_registers(mdp, return_bci, Rtemp, R0, R1, R2, R3); 1636 1637 if (ProfileInterpreter) { 1638 Label profile_continue; 1639 uint row; 1640 1641 // If no method data exists, go to profile_continue. 1642 test_method_data_pointer(mdp, profile_continue); 1643 1644 // Update the total ret count. 1645 increment_mdp_data_at(mdp, in_bytes(CounterData::count_offset()), Rtemp); 1646 1647 for (row = 0; row < RetData::row_limit(); row++) { 1648 Label next_test; 1649 1650 // See if return_bci is equal to bci[n]: 1651 test_mdp_data_at(mdp, in_bytes(RetData::bci_offset(row)), return_bci, 1652 Rtemp, next_test); 1653 1654 // return_bci is equal to bci[n]. Increment the count. 1655 increment_mdp_data_at(mdp, in_bytes(RetData::bci_count_offset(row)), Rtemp); 1656 1657 // The method data pointer needs to be updated to reflect the new target. 1658 update_mdp_by_offset(mdp, in_bytes(RetData::bci_displacement_offset(row)), Rtemp); 1659 b(profile_continue); 1660 bind(next_test); 1661 } 1662 1663 update_mdp_for_ret(return_bci); 1664 1665 bind(profile_continue); 1666 } 1667 } 1668 1669 1670 // Sets mdp. 1671 void InterpreterMacroAssembler::profile_null_seen(Register mdp) { 1672 if (ProfileInterpreter) { 1673 Label profile_continue; 1674 1675 // If no method data exists, go to profile_continue. 1676 test_method_data_pointer(mdp, profile_continue); 1677 1678 set_mdp_flag_at(mdp, BitData::null_seen_byte_constant()); 1679 1680 // The method data pointer needs to be updated. 1681 int mdp_delta = in_bytes(BitData::bit_data_size()); 1682 if (TypeProfileCasts) { 1683 mdp_delta = in_bytes(VirtualCallData::virtual_call_data_size()); 1684 } 1685 update_mdp_by_constant(mdp, mdp_delta); 1686 1687 bind (profile_continue); 1688 } 1689 } 1690 1691 1692 // Sets mdp, blows Rtemp. 1693 void InterpreterMacroAssembler::profile_typecheck_failed(Register mdp) { 1694 assert_different_registers(mdp, Rtemp); 1695 1696 if (ProfileInterpreter && TypeProfileCasts) { 1697 Label profile_continue; 1698 1699 // If no method data exists, go to profile_continue. 1700 test_method_data_pointer(mdp, profile_continue); 1701 1702 int count_offset = in_bytes(CounterData::count_offset()); 1703 // Back up the address, since we have already bumped the mdp. 1704 count_offset -= in_bytes(VirtualCallData::virtual_call_data_size()); 1705 1706 // *Decrement* the counter. We expect to see zero or small negatives. 1707 increment_mdp_data_at(mdp, count_offset, Rtemp, true); 1708 1709 bind (profile_continue); 1710 } 1711 } 1712 1713 1714 // Sets mdp, blows Rtemp. 1715 void InterpreterMacroAssembler::profile_typecheck(Register mdp, Register klass) 1716 { 1717 assert_different_registers(mdp, klass, Rtemp); 1718 1719 if (ProfileInterpreter) { 1720 Label profile_continue; 1721 1722 // If no method data exists, go to profile_continue. 1723 test_method_data_pointer(mdp, profile_continue); 1724 1725 // The method data pointer needs to be updated. 1726 int mdp_delta = in_bytes(BitData::bit_data_size()); 1727 if (TypeProfileCasts) { 1728 mdp_delta = in_bytes(VirtualCallData::virtual_call_data_size()); 1729 1730 // Record the object type. 1731 record_klass_in_profile(klass, mdp, Rtemp, false); 1732 } 1733 update_mdp_by_constant(mdp, mdp_delta); 1734 1735 bind(profile_continue); 1736 } 1737 } 1738 1739 1740 // Sets mdp, blows Rtemp. 1741 void InterpreterMacroAssembler::profile_switch_default(Register mdp) { 1742 assert_different_registers(mdp, Rtemp); 1743 1744 if (ProfileInterpreter) { 1745 Label profile_continue; 1746 1747 // If no method data exists, go to profile_continue. 1748 test_method_data_pointer(mdp, profile_continue); 1749 1750 // Update the default case count 1751 increment_mdp_data_at(mdp, in_bytes(MultiBranchData::default_count_offset()), Rtemp); 1752 1753 // The method data pointer needs to be updated. 1754 update_mdp_by_offset(mdp, in_bytes(MultiBranchData::default_displacement_offset()), Rtemp); 1755 1756 bind(profile_continue); 1757 } 1758 } 1759 1760 1761 // Sets mdp. Blows reg_tmp1, reg_tmp2. Index could be the same as reg_tmp2. 1762 void InterpreterMacroAssembler::profile_switch_case(Register mdp, Register index, Register reg_tmp1, Register reg_tmp2) { 1763 assert_different_registers(mdp, reg_tmp1, reg_tmp2); 1764 assert_different_registers(mdp, reg_tmp1, index); 1765 1766 if (ProfileInterpreter) { 1767 Label profile_continue; 1768 1769 const int count_offset = in_bytes(MultiBranchData::case_array_offset()) + 1770 in_bytes(MultiBranchData::relative_count_offset()); 1771 1772 const int displacement_offset = in_bytes(MultiBranchData::case_array_offset()) + 1773 in_bytes(MultiBranchData::relative_displacement_offset()); 1774 1775 // If no method data exists, go to profile_continue. 1776 test_method_data_pointer(mdp, profile_continue); 1777 1778 // Build the base (index * per_case_size_in_bytes()) 1779 logical_shift_left(reg_tmp1, index, exact_log2(in_bytes(MultiBranchData::per_case_size()))); 1780 1781 // Update the case count 1782 add(reg_tmp1, reg_tmp1, count_offset); 1783 increment_mdp_data_at(Address(mdp, reg_tmp1), reg_tmp2); 1784 1785 // The method data pointer needs to be updated. 1786 add(reg_tmp1, reg_tmp1, displacement_offset - count_offset); 1787 update_mdp_by_offset(mdp, reg_tmp1, reg_tmp2); 1788 1789 bind (profile_continue); 1790 } 1791 } 1792 1793 1794 void InterpreterMacroAssembler::byteswap_u32(Register r, Register rtmp1, Register rtmp2) { 1795 #ifdef AARCH64 1796 rev_w(r, r); 1797 #else 1798 if (VM_Version::supports_rev()) { 1799 rev(r, r); 1800 } else { 1801 eor(rtmp1, r, AsmOperand(r, ror, 16)); 1802 mvn(rtmp2, 0x0000ff00); 1803 andr(rtmp1, rtmp2, AsmOperand(rtmp1, lsr, 8)); 1804 eor(r, rtmp1, AsmOperand(r, ror, 8)); 1805 } 1806 #endif // AARCH64 1807 } 1808 1809 1810 void InterpreterMacroAssembler::inc_global_counter(address address_of_counter, int offset, Register tmp1, Register tmp2, bool avoid_overflow) { 1811 const intx addr = (intx) (address_of_counter + offset); 1812 1813 assert ((addr & 0x3) == 0, "address of counter should be aligned"); 1814 const intx offset_mask = right_n_bits(AARCH64_ONLY(12 + 2) NOT_AARCH64(12)); 1815 1816 const address base = (address) (addr & ~offset_mask); 1817 const int offs = (int) (addr & offset_mask); 1818 1819 const Register addr_base = tmp1; 1820 const Register val = tmp2; 1821 1822 mov_slow(addr_base, base); 1823 ldr_s32(val, Address(addr_base, offs)); 1824 1825 if (avoid_overflow) { 1826 adds_32(val, val, 1); 1827 #ifdef AARCH64 1828 Label L; 1829 b(L, mi); 1830 str_32(val, Address(addr_base, offs)); 1831 bind(L); 1832 #else 1833 str(val, Address(addr_base, offs), pl); 1834 #endif // AARCH64 1835 } else { 1836 add_32(val, val, 1); 1837 str_32(val, Address(addr_base, offs)); 1838 } 1839 } 1840 1841 void InterpreterMacroAssembler::interp_verify_oop(Register reg, TosState state, const char *file, int line) { 1842 if (state == atos) { MacroAssembler::_verify_oop(reg, "broken oop", file, line); } 1843 } 1844 1845 // Inline assembly for: 1846 // 1847 // if (thread is in interp_only_mode) { 1848 // InterpreterRuntime::post_method_entry(); 1849 // } 1850 // if (DTraceMethodProbes) { 1851 // SharedRuntime::dtrace_method_entry(method, receiver); 1852 // } 1853 // if (RC_TRACE_IN_RANGE(0x00001000, 0x00002000)) { 1854 // SharedRuntime::rc_trace_method_entry(method, receiver); 1855 // } 1856 1857 void InterpreterMacroAssembler::notify_method_entry() { 1858 // Whenever JVMTI is interp_only_mode, method entry/exit events are sent to 1859 // track stack depth. If it is possible to enter interp_only_mode we add 1860 // the code to check if the event should be sent. 1861 if (can_post_interpreter_events()) { 1862 Label L; 1863 1864 ldr_s32(Rtemp, Address(Rthread, JavaThread::interp_only_mode_offset())); 1865 cbz(Rtemp, L); 1866 1867 call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::post_method_entry)); 1868 1869 bind(L); 1870 } 1871 1872 // Note: Disable DTrace runtime check for now to eliminate overhead on each method entry 1873 if (DTraceMethodProbes) { 1874 Label Lcontinue; 1875 1876 ldrb_global(Rtemp, (address)&DTraceMethodProbes); 1877 cbz(Rtemp, Lcontinue); 1878 1879 mov(R0, Rthread); 1880 mov(R1, Rmethod); 1881 call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_method_entry), R0, R1); 1882 1883 bind(Lcontinue); 1884 } 1885 // RedefineClasses() tracing support for obsolete method entry 1886 if (log_is_enabled(Trace, redefine, class, obsolete)) { 1887 mov(R0, Rthread); 1888 mov(R1, Rmethod); 1889 call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::rc_trace_method_entry), 1890 R0, R1); 1891 } 1892 } 1893 1894 1895 void InterpreterMacroAssembler::notify_method_exit( 1896 TosState state, NotifyMethodExitMode mode, 1897 bool native, Register result_lo, Register result_hi, FloatRegister result_fp) { 1898 // Whenever JVMTI is interp_only_mode, method entry/exit events are sent to 1899 // track stack depth. If it is possible to enter interp_only_mode we add 1900 // the code to check if the event should be sent. 1901 if (mode == NotifyJVMTI && can_post_interpreter_events()) { 1902 Label L; 1903 // Note: frame::interpreter_frame_result has a dependency on how the 1904 // method result is saved across the call to post_method_exit. If this 1905 // is changed then the interpreter_frame_result implementation will 1906 // need to be updated too. 1907 1908 ldr_s32(Rtemp, Address(Rthread, JavaThread::interp_only_mode_offset())); 1909 cbz(Rtemp, L); 1910 1911 if (native) { 1912 // For c++ and template interpreter push both result registers on the 1913 // stack in native, we don't know the state. 1914 // On AArch64 result registers are stored into the frame at known locations. 1915 // See frame::interpreter_frame_result for code that gets the result values from here. 1916 assert(result_lo != noreg, "result registers should be defined"); 1917 1918 #ifdef AARCH64 1919 assert(result_hi == noreg, "result_hi is not used on AArch64"); 1920 assert(result_fp != fnoreg, "FP result register must be defined"); 1921 1922 str_d(result_fp, Address(FP, frame::interpreter_frame_fp_saved_result_offset * wordSize)); 1923 str(result_lo, Address(FP, frame::interpreter_frame_gp_saved_result_offset * wordSize)); 1924 #else 1925 assert(result_hi != noreg, "result registers should be defined"); 1926 1927 #ifdef __ABI_HARD__ 1928 assert(result_fp != fnoreg, "FP result register must be defined"); 1929 sub(SP, SP, 2 * wordSize); 1930 fstd(result_fp, Address(SP)); 1931 #endif // __ABI_HARD__ 1932 1933 push(RegisterSet(result_lo) | RegisterSet(result_hi)); 1934 #endif // AARCH64 1935 1936 call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::post_method_exit)); 1937 1938 #ifdef AARCH64 1939 ldr_d(result_fp, Address(FP, frame::interpreter_frame_fp_saved_result_offset * wordSize)); 1940 ldr(result_lo, Address(FP, frame::interpreter_frame_gp_saved_result_offset * wordSize)); 1941 #else 1942 pop(RegisterSet(result_lo) | RegisterSet(result_hi)); 1943 #ifdef __ABI_HARD__ 1944 fldd(result_fp, Address(SP)); 1945 add(SP, SP, 2 * wordSize); 1946 #endif // __ABI_HARD__ 1947 #endif // AARCH64 1948 1949 } else { 1950 // For the template interpreter, the value on tos is the size of the 1951 // state. (c++ interpreter calls jvmti somewhere else). 1952 push(state); 1953 call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::post_method_exit)); 1954 pop(state); 1955 } 1956 1957 bind(L); 1958 } 1959 1960 // Note: Disable DTrace runtime check for now to eliminate overhead on each method exit 1961 if (DTraceMethodProbes) { 1962 Label Lcontinue; 1963 1964 ldrb_global(Rtemp, (address)&DTraceMethodProbes); 1965 cbz(Rtemp, Lcontinue); 1966 1967 push(state); 1968 1969 mov(R0, Rthread); 1970 mov(R1, Rmethod); 1971 1972 call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_method_exit), R0, R1); 1973 1974 pop(state); 1975 1976 bind(Lcontinue); 1977 } 1978 } 1979 1980 1981 #ifndef PRODUCT 1982 1983 void InterpreterMacroAssembler::trace_state(const char* msg) { 1984 int push_size = save_caller_save_registers(); 1985 1986 Label Lcontinue; 1987 InlinedString Lmsg0("%s: FP=" INTPTR_FORMAT ", SP=" INTPTR_FORMAT "\n"); 1988 InlinedString Lmsg(msg); 1989 InlinedAddress Lprintf((address)printf); 1990 1991 ldr_literal(R0, Lmsg0); 1992 ldr_literal(R1, Lmsg); 1993 mov(R2, FP); 1994 add(R3, SP, push_size); // original SP (without saved registers) 1995 ldr_literal(Rtemp, Lprintf); 1996 call(Rtemp); 1997 1998 b(Lcontinue); 1999 2000 bind_literal(Lmsg0); 2001 bind_literal(Lmsg); 2002 bind_literal(Lprintf); 2003 2004 2005 bind(Lcontinue); 2006 2007 restore_caller_save_registers(); 2008 } 2009 2010 #endif 2011 2012 // Jump if ((*counter_addr += increment) & mask) satisfies the condition. 2013 void InterpreterMacroAssembler::increment_mask_and_jump(Address counter_addr, 2014 int increment, Address mask_addr, 2015 Register scratch, Register scratch2, 2016 AsmCondition cond, Label* where) { 2017 // caution: scratch2 and base address of counter_addr can be the same 2018 assert_different_registers(scratch, scratch2); 2019 ldr_u32(scratch, counter_addr); 2020 add(scratch, scratch, increment); 2021 str_32(scratch, counter_addr); 2022 2023 #ifdef AARCH64 2024 ldr_u32(scratch2, mask_addr); 2025 ands_w(ZR, scratch, scratch2); 2026 #else 2027 ldr(scratch2, mask_addr); 2028 andrs(scratch, scratch, scratch2); 2029 #endif // AARCH64 2030 b(*where, cond); 2031 } 2032 2033 void InterpreterMacroAssembler::get_method_counters(Register method, 2034 Register Rcounters, 2035 Label& skip, 2036 bool saveRegs, 2037 Register reg1, 2038 Register reg2, 2039 Register reg3) { 2040 const Address method_counters(method, Method::method_counters_offset()); 2041 Label has_counters; 2042 2043 ldr(Rcounters, method_counters); 2044 cbnz(Rcounters, has_counters); 2045 2046 if (saveRegs) { 2047 // Save and restore in use caller-saved registers since they will be trashed by call_VM 2048 assert(reg1 != noreg, "must specify reg1"); 2049 assert(reg2 != noreg, "must specify reg2"); 2050 #ifdef AARCH64 2051 assert(reg3 != noreg, "must specify reg3"); 2052 stp(reg1, reg2, Address(Rstack_top, -2*wordSize, pre_indexed)); 2053 stp(reg3, ZR, Address(Rstack_top, -2*wordSize, pre_indexed)); 2054 #else 2055 assert(reg3 == noreg, "must not specify reg3"); 2056 push(RegisterSet(reg1) | RegisterSet(reg2)); 2057 #endif 2058 } 2059 2060 mov(R1, method); 2061 call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::build_method_counters), R1); 2062 2063 if (saveRegs) { 2064 #ifdef AARCH64 2065 ldp(reg3, ZR, Address(Rstack_top, 2*wordSize, post_indexed)); 2066 ldp(reg1, reg2, Address(Rstack_top, 2*wordSize, post_indexed)); 2067 #else 2068 pop(RegisterSet(reg1) | RegisterSet(reg2)); 2069 #endif 2070 } 2071 2072 ldr(Rcounters, method_counters); 2073 cbz(Rcounters, skip); // No MethodCounters created, OutOfMemory 2074 2075 bind(has_counters); 2076 }