1 /* 2 * Copyright (c) 2008, 2018, Oracle and/or its affiliates. All rights reserved. 3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 4 * 5 * This code is free software; you can redistribute it and/or modify it 6 * under the terms of the GNU General Public License version 2 only, as 7 * published by the Free Software Foundation. 8 * 9 * This code is distributed in the hope that it will be useful, but WITHOUT 10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 12 * version 2 for more details (a copy is included in the LICENSE file that 13 * accompanied this code). 14 * 15 * You should have received a copy of the GNU General Public License version 16 * 2 along with this work; if not, write to the Free Software Foundation, 17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 18 * 19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 20 * or visit www.oracle.com if you need additional information or have any 21 * questions. 22 * 23 */ 24 25 #include "precompiled.hpp" 26 #include "jvm.h" 27 #include "gc/shared/barrierSet.hpp" 28 #include "gc/shared/cardTable.hpp" 29 #include "gc/shared/cardTableBarrierSet.inline.hpp" 30 #include "gc/shared/collectedHeap.hpp" 31 #include "interp_masm_arm.hpp" 32 #include "interpreter/interpreter.hpp" 33 #include "interpreter/interpreterRuntime.hpp" 34 #include "logging/log.hpp" 35 #include "oops/arrayOop.hpp" 36 #include "oops/markOop.hpp" 37 #include "oops/method.hpp" 38 #include "oops/methodData.hpp" 39 #include "prims/jvmtiExport.hpp" 40 #include "prims/jvmtiThreadState.hpp" 41 #include "runtime/basicLock.hpp" 42 #include "runtime/biasedLocking.hpp" 43 #include "runtime/frame.inline.hpp" 44 #include "runtime/sharedRuntime.hpp" 45 46 //-------------------------------------------------------------------- 47 // Implementation of InterpreterMacroAssembler 48 49 50 51 52 InterpreterMacroAssembler::InterpreterMacroAssembler(CodeBuffer* code) : MacroAssembler(code) { 53 } 54 55 void InterpreterMacroAssembler::call_VM_helper(Register oop_result, address entry_point, int number_of_arguments, bool check_exceptions) { 56 #if defined(ASSERT) && !defined(AARCH64) 57 // Ensure that last_sp is not filled. 58 { Label L; 59 ldr(Rtemp, Address(FP, frame::interpreter_frame_last_sp_offset * wordSize)); 60 cbz(Rtemp, L); 61 stop("InterpreterMacroAssembler::call_VM_helper: last_sp != NULL"); 62 bind(L); 63 } 64 #endif // ASSERT && !AARCH64 65 66 // Rbcp must be saved/restored since it may change due to GC. 67 save_bcp(); 68 69 #ifdef AARCH64 70 check_no_cached_stack_top(Rtemp); 71 save_stack_top(); 72 check_extended_sp(Rtemp); 73 cut_sp_before_call(); 74 #endif // AARCH64 75 76 // super call 77 MacroAssembler::call_VM_helper(oop_result, entry_point, number_of_arguments, check_exceptions); 78 79 #ifdef AARCH64 80 // Restore SP to extended SP 81 restore_sp_after_call(Rtemp); 82 check_stack_top(); 83 clear_cached_stack_top(); 84 #endif // AARCH64 85 86 // Restore interpreter specific registers. 87 restore_bcp(); 88 restore_method(); 89 } 90 91 void InterpreterMacroAssembler::jump_to_entry(address entry) { 92 assert(entry, "Entry must have been generated by now"); 93 b(entry); 94 } 95 96 void InterpreterMacroAssembler::check_and_handle_popframe() { 97 if (can_pop_frame()) { 98 Label L; 99 const Register popframe_cond = R2_tmp; 100 101 // Initiate popframe handling only if it is not already being processed. If the flag 102 // has the popframe_processing bit set, it means that this code is called *during* popframe 103 // handling - we don't want to reenter. 104 105 ldr_s32(popframe_cond, Address(Rthread, JavaThread::popframe_condition_offset())); 106 tbz(popframe_cond, exact_log2(JavaThread::popframe_pending_bit), L); 107 tbnz(popframe_cond, exact_log2(JavaThread::popframe_processing_bit), L); 108 109 // Call Interpreter::remove_activation_preserving_args_entry() to get the 110 // address of the same-named entrypoint in the generated interpreter code. 111 call_VM_leaf(CAST_FROM_FN_PTR(address, Interpreter::remove_activation_preserving_args_entry)); 112 113 // Call indirectly to avoid generation ordering problem. 114 jump(R0); 115 116 bind(L); 117 } 118 } 119 120 121 // Blows R2, Rtemp. Sets TOS cached value. 122 void InterpreterMacroAssembler::load_earlyret_value(TosState state) { 123 const Register thread_state = R2_tmp; 124 125 ldr(thread_state, Address(Rthread, JavaThread::jvmti_thread_state_offset())); 126 127 const Address tos_addr(thread_state, JvmtiThreadState::earlyret_tos_offset()); 128 const Address oop_addr(thread_state, JvmtiThreadState::earlyret_oop_offset()); 129 const Address val_addr(thread_state, JvmtiThreadState::earlyret_value_offset()); 130 #ifndef AARCH64 131 const Address val_addr_hi(thread_state, JvmtiThreadState::earlyret_value_offset() 132 + in_ByteSize(wordSize)); 133 #endif // !AARCH64 134 135 Register zero = zero_register(Rtemp); 136 137 switch (state) { 138 case atos: ldr(R0_tos, oop_addr); 139 str(zero, oop_addr); 140 interp_verify_oop(R0_tos, state, __FILE__, __LINE__); 141 break; 142 143 #ifdef AARCH64 144 case ltos: ldr(R0_tos, val_addr); break; 145 #else 146 case ltos: ldr(R1_tos_hi, val_addr_hi); // fall through 147 #endif // AARCH64 148 case btos: // fall through 149 case ztos: // fall through 150 case ctos: // fall through 151 case stos: // fall through 152 case itos: ldr_s32(R0_tos, val_addr); break; 153 #ifdef __SOFTFP__ 154 case dtos: ldr(R1_tos_hi, val_addr_hi); // fall through 155 case ftos: ldr(R0_tos, val_addr); break; 156 #else 157 case ftos: ldr_float (S0_tos, val_addr); break; 158 case dtos: ldr_double(D0_tos, val_addr); break; 159 #endif // __SOFTFP__ 160 case vtos: /* nothing to do */ break; 161 default : ShouldNotReachHere(); 162 } 163 // Clean up tos value in the thread object 164 str(zero, val_addr); 165 #ifndef AARCH64 166 str(zero, val_addr_hi); 167 #endif // !AARCH64 168 169 mov(Rtemp, (int) ilgl); 170 str_32(Rtemp, tos_addr); 171 } 172 173 174 // Blows R2, Rtemp. 175 void InterpreterMacroAssembler::check_and_handle_earlyret() { 176 if (can_force_early_return()) { 177 Label L; 178 const Register thread_state = R2_tmp; 179 180 ldr(thread_state, Address(Rthread, JavaThread::jvmti_thread_state_offset())); 181 cbz(thread_state, L); // if (thread->jvmti_thread_state() == NULL) exit; 182 183 // Initiate earlyret handling only if it is not already being processed. 184 // If the flag has the earlyret_processing bit set, it means that this code 185 // is called *during* earlyret handling - we don't want to reenter. 186 187 ldr_s32(Rtemp, Address(thread_state, JvmtiThreadState::earlyret_state_offset())); 188 cmp(Rtemp, JvmtiThreadState::earlyret_pending); 189 b(L, ne); 190 191 // Call Interpreter::remove_activation_early_entry() to get the address of the 192 // same-named entrypoint in the generated interpreter code. 193 194 ldr_s32(R0, Address(thread_state, JvmtiThreadState::earlyret_tos_offset())); 195 call_VM_leaf(CAST_FROM_FN_PTR(address, Interpreter::remove_activation_early_entry), R0); 196 197 jump(R0); 198 199 bind(L); 200 } 201 } 202 203 204 // Sets reg. Blows Rtemp. 205 void InterpreterMacroAssembler::get_unsigned_2_byte_index_at_bcp(Register reg, int bcp_offset) { 206 assert(bcp_offset >= 0, "bcp is still pointing to start of bytecode"); 207 assert(reg != Rtemp, "should be different registers"); 208 209 ldrb(Rtemp, Address(Rbcp, bcp_offset)); 210 ldrb(reg, Address(Rbcp, bcp_offset+1)); 211 orr(reg, reg, AsmOperand(Rtemp, lsl, BitsPerByte)); 212 } 213 214 void InterpreterMacroAssembler::get_index_at_bcp(Register index, int bcp_offset, Register tmp_reg, size_t index_size) { 215 assert_different_registers(index, tmp_reg); 216 if (index_size == sizeof(u2)) { 217 // load bytes of index separately to avoid unaligned access 218 ldrb(index, Address(Rbcp, bcp_offset+1)); 219 ldrb(tmp_reg, Address(Rbcp, bcp_offset)); 220 orr(index, tmp_reg, AsmOperand(index, lsl, BitsPerByte)); 221 } else if (index_size == sizeof(u4)) { 222 // TODO-AARCH64: consider using unaligned access here 223 ldrb(index, Address(Rbcp, bcp_offset+3)); 224 ldrb(tmp_reg, Address(Rbcp, bcp_offset+2)); 225 orr(index, tmp_reg, AsmOperand(index, lsl, BitsPerByte)); 226 ldrb(tmp_reg, Address(Rbcp, bcp_offset+1)); 227 orr(index, tmp_reg, AsmOperand(index, lsl, BitsPerByte)); 228 ldrb(tmp_reg, Address(Rbcp, bcp_offset)); 229 orr(index, tmp_reg, AsmOperand(index, lsl, BitsPerByte)); 230 // Check if the secondary index definition is still ~x, otherwise 231 // we have to change the following assembler code to calculate the 232 // plain index. 233 assert(ConstantPool::decode_invokedynamic_index(~123) == 123, "else change next line"); 234 mvn_32(index, index); // convert to plain index 235 } else if (index_size == sizeof(u1)) { 236 ldrb(index, Address(Rbcp, bcp_offset)); 237 } else { 238 ShouldNotReachHere(); 239 } 240 } 241 242 // Sets cache, index. 243 void InterpreterMacroAssembler::get_cache_and_index_at_bcp(Register cache, Register index, int bcp_offset, size_t index_size) { 244 assert(bcp_offset > 0, "bcp is still pointing to start of bytecode"); 245 assert_different_registers(cache, index); 246 247 get_index_at_bcp(index, bcp_offset, cache, index_size); 248 249 // load constant pool cache pointer 250 ldr(cache, Address(FP, frame::interpreter_frame_cache_offset * wordSize)); 251 252 // convert from field index to ConstantPoolCacheEntry index 253 assert(sizeof(ConstantPoolCacheEntry) == 4*wordSize, "adjust code below"); 254 // TODO-AARCH64 merge this shift with shift "add(..., Rcache, AsmOperand(Rindex, lsl, LogBytesPerWord))" after this method is called 255 logical_shift_left(index, index, 2); 256 } 257 258 // Sets cache, index, bytecode. 259 void InterpreterMacroAssembler::get_cache_and_index_and_bytecode_at_bcp(Register cache, Register index, Register bytecode, int byte_no, int bcp_offset, size_t index_size) { 260 get_cache_and_index_at_bcp(cache, index, bcp_offset, index_size); 261 // caution index and bytecode can be the same 262 add(bytecode, cache, AsmOperand(index, lsl, LogBytesPerWord)); 263 #ifdef AARCH64 264 add(bytecode, bytecode, (1 + byte_no) + in_bytes(ConstantPoolCache::base_offset() + ConstantPoolCacheEntry::indices_offset())); 265 ldarb(bytecode, bytecode); 266 #else 267 ldrb(bytecode, Address(bytecode, (1 + byte_no) + in_bytes(ConstantPoolCache::base_offset() + ConstantPoolCacheEntry::indices_offset()))); 268 TemplateTable::volatile_barrier(MacroAssembler::LoadLoad, noreg, true); 269 #endif // AARCH64 270 } 271 272 // Sets cache. Blows reg_tmp. 273 void InterpreterMacroAssembler::get_cache_entry_pointer_at_bcp(Register cache, Register reg_tmp, int bcp_offset, size_t index_size) { 274 assert(bcp_offset > 0, "bcp is still pointing to start of bytecode"); 275 assert_different_registers(cache, reg_tmp); 276 277 get_index_at_bcp(reg_tmp, bcp_offset, cache, index_size); 278 279 // load constant pool cache pointer 280 ldr(cache, Address(FP, frame::interpreter_frame_cache_offset * wordSize)); 281 282 // skip past the header 283 add(cache, cache, in_bytes(ConstantPoolCache::base_offset())); 284 // convert from field index to ConstantPoolCacheEntry index 285 // and from word offset to byte offset 286 assert(sizeof(ConstantPoolCacheEntry) == 4*wordSize, "adjust code below"); 287 add(cache, cache, AsmOperand(reg_tmp, lsl, 2 + LogBytesPerWord)); 288 } 289 290 // Load object from cpool->resolved_references(index) 291 void InterpreterMacroAssembler::load_resolved_reference_at_index( 292 Register result, Register index) { 293 assert_different_registers(result, index); 294 get_constant_pool(result); 295 296 Register cache = result; 297 // load pointer for resolved_references[] objArray 298 ldr(cache, Address(result, ConstantPool::cache_offset_in_bytes())); 299 ldr(cache, Address(result, ConstantPoolCache::resolved_references_offset_in_bytes())); 300 resolve_oop_handle(cache); 301 // Add in the index 302 // convert from field index to resolved_references() index and from 303 // word index to byte offset. Since this is a java object, it can be compressed 304 add(cache, cache, AsmOperand(index, lsl, LogBytesPerHeapOop)); 305 load_heap_oop(result, Address(cache, arrayOopDesc::base_offset_in_bytes(T_OBJECT))); 306 } 307 308 void InterpreterMacroAssembler::load_resolved_klass_at_offset( 309 Register Rcpool, Register Rindex, Register Rklass) { 310 add(Rtemp, Rcpool, AsmOperand(Rindex, lsl, LogBytesPerWord)); 311 ldrh(Rtemp, Address(Rtemp, sizeof(ConstantPool))); // Rtemp = resolved_klass_index 312 ldr(Rklass, Address(Rcpool, ConstantPool::resolved_klasses_offset_in_bytes())); // Rklass = cpool->_resolved_klasses 313 add(Rklass, Rklass, AsmOperand(Rtemp, lsl, LogBytesPerWord)); 314 ldr(Rklass, Address(Rklass, Array<Klass*>::base_offset_in_bytes())); 315 } 316 317 // Generate a subtype check: branch to not_subtype if sub_klass is 318 // not a subtype of super_klass. 319 // Profiling code for the subtype check failure (profile_typecheck_failed) 320 // should be explicitly generated by the caller in the not_subtype case. 321 // Blows Rtemp, tmp1, tmp2. 322 void InterpreterMacroAssembler::gen_subtype_check(Register Rsub_klass, 323 Register Rsuper_klass, 324 Label ¬_subtype, 325 Register tmp1, 326 Register tmp2) { 327 328 assert_different_registers(Rsub_klass, Rsuper_klass, tmp1, tmp2, Rtemp); 329 Label ok_is_subtype, loop, update_cache; 330 331 const Register super_check_offset = tmp1; 332 const Register cached_super = tmp2; 333 334 // Profile the not-null value's klass. 335 profile_typecheck(tmp1, Rsub_klass); 336 337 // Load the super-klass's check offset into 338 ldr_u32(super_check_offset, Address(Rsuper_klass, Klass::super_check_offset_offset())); 339 340 // Check for self 341 cmp(Rsub_klass, Rsuper_klass); 342 343 // Load from the sub-klass's super-class display list, or a 1-word cache of 344 // the secondary superclass list, or a failing value with a sentinel offset 345 // if the super-klass is an interface or exceptionally deep in the Java 346 // hierarchy and we have to scan the secondary superclass list the hard way. 347 // See if we get an immediate positive hit 348 ldr(cached_super, Address(Rsub_klass, super_check_offset)); 349 350 cond_cmp(Rsuper_klass, cached_super, ne); 351 b(ok_is_subtype, eq); 352 353 // Check for immediate negative hit 354 cmp(super_check_offset, in_bytes(Klass::secondary_super_cache_offset())); 355 b(not_subtype, ne); 356 357 // Now do a linear scan of the secondary super-klass chain. 358 const Register supers_arr = tmp1; 359 const Register supers_cnt = tmp2; 360 const Register cur_super = Rtemp; 361 362 // Load objArrayOop of secondary supers. 363 ldr(supers_arr, Address(Rsub_klass, Klass::secondary_supers_offset())); 364 365 ldr_u32(supers_cnt, Address(supers_arr, Array<Klass*>::length_offset_in_bytes())); // Load the array length 366 #ifdef AARCH64 367 cbz(supers_cnt, not_subtype); 368 add(supers_arr, supers_arr, Array<Klass*>::base_offset_in_bytes()); 369 #else 370 cmp(supers_cnt, 0); 371 372 // Skip to the start of array elements and prefetch the first super-klass. 373 ldr(cur_super, Address(supers_arr, Array<Klass*>::base_offset_in_bytes(), pre_indexed), ne); 374 b(not_subtype, eq); 375 #endif // AARCH64 376 377 bind(loop); 378 379 #ifdef AARCH64 380 ldr(cur_super, Address(supers_arr, wordSize, post_indexed)); 381 #endif // AARCH64 382 383 cmp(cur_super, Rsuper_klass); 384 b(update_cache, eq); 385 386 subs(supers_cnt, supers_cnt, 1); 387 388 #ifndef AARCH64 389 ldr(cur_super, Address(supers_arr, wordSize, pre_indexed), ne); 390 #endif // !AARCH64 391 392 b(loop, ne); 393 394 b(not_subtype); 395 396 bind(update_cache); 397 // Must be equal but missed in cache. Update cache. 398 str(Rsuper_klass, Address(Rsub_klass, Klass::secondary_super_cache_offset())); 399 400 bind(ok_is_subtype); 401 } 402 403 404 // The 1st part of the store check. 405 // Sets card_table_base register. 406 void InterpreterMacroAssembler::store_check_part1(Register card_table_base) { 407 // Check barrier set type (should be card table) and element size 408 BarrierSet* bs = BarrierSet::barrier_set(); 409 assert(bs->kind() == BarrierSet::CardTableBarrierSet, 410 "Wrong barrier set kind"); 411 412 CardTableBarrierSet* ctbs = barrier_set_cast<CardTableBarrierSet>(bs); 413 CardTable* ct = ctbs->card_table(); 414 assert(sizeof(*ct->byte_map_base()) == sizeof(jbyte), "Adjust store check code"); 415 416 // Load card table base address. 417 418 /* Performance note. 419 420 There is an alternative way of loading card table base address 421 from thread descriptor, which may look more efficient: 422 423 ldr(card_table_base, Address(Rthread, JavaThread::card_table_base_offset())); 424 425 However, performance measurements of micro benchmarks and specJVM98 426 showed that loading of card table base from thread descriptor is 427 7-18% slower compared to loading of literal embedded into the code. 428 Possible cause is a cache miss (card table base address resides in a 429 rarely accessed area of thread descriptor). 430 */ 431 // TODO-AARCH64 Investigate if mov_slow is faster than ldr from Rthread on AArch64 432 mov_address(card_table_base, (address)ct->byte_map_base(), symbolic_Relocation::card_table_reference); 433 } 434 435 // The 2nd part of the store check. 436 void InterpreterMacroAssembler::store_check_part2(Register obj, Register card_table_base, Register tmp) { 437 assert_different_registers(obj, card_table_base, tmp); 438 439 assert(CardTable::dirty_card_val() == 0, "Dirty card value must be 0 due to optimizations."); 440 #ifdef AARCH64 441 add(card_table_base, card_table_base, AsmOperand(obj, lsr, CardTable::card_shift)); 442 Address card_table_addr(card_table_base); 443 #else 444 Address card_table_addr(card_table_base, obj, lsr, CardTable::card_shift); 445 #endif 446 447 if (UseCondCardMark) { 448 #if INCLUDE_ALL_GCS 449 if (UseConcMarkSweepGC) { 450 membar(MacroAssembler::Membar_mask_bits(MacroAssembler::StoreLoad), noreg); 451 } 452 #endif 453 Label already_dirty; 454 455 ldrb(tmp, card_table_addr); 456 cbz(tmp, already_dirty); 457 458 set_card(card_table_base, card_table_addr, tmp); 459 bind(already_dirty); 460 461 } else { 462 #if INCLUDE_ALL_GCS 463 if (UseConcMarkSweepGC && CMSPrecleaningEnabled) { 464 membar(MacroAssembler::Membar_mask_bits(MacroAssembler::StoreStore), noreg); 465 } 466 #endif 467 set_card(card_table_base, card_table_addr, tmp); 468 } 469 } 470 471 void InterpreterMacroAssembler::set_card(Register card_table_base, Address card_table_addr, Register tmp) { 472 #ifdef AARCH64 473 strb(ZR, card_table_addr); 474 #else 475 CardTableBarrierSet* ctbs = barrier_set_cast<CardTableBarrierSet>(BarrierSet::barrier_set()); 476 CardTable* ct = ctbs->card_table(); 477 if ((((uintptr_t)ct->byte_map_base() & 0xff) == 0)) { 478 // Card table is aligned so the lowest byte of the table address base is zero. 479 // This works only if the code is not saved for later use, possibly 480 // in a context where the base would no longer be aligned. 481 strb(card_table_base, card_table_addr); 482 } else { 483 mov(tmp, 0); 484 strb(tmp, card_table_addr); 485 } 486 #endif // AARCH64 487 } 488 489 ////////////////////////////////////////////////////////////////////////////////// 490 491 492 // Java Expression Stack 493 494 void InterpreterMacroAssembler::pop_ptr(Register r) { 495 assert(r != Rstack_top, "unpredictable instruction"); 496 ldr(r, Address(Rstack_top, wordSize, post_indexed)); 497 } 498 499 void InterpreterMacroAssembler::pop_i(Register r) { 500 assert(r != Rstack_top, "unpredictable instruction"); 501 ldr_s32(r, Address(Rstack_top, wordSize, post_indexed)); 502 zap_high_non_significant_bits(r); 503 } 504 505 #ifdef AARCH64 506 void InterpreterMacroAssembler::pop_l(Register r) { 507 assert(r != Rstack_top, "unpredictable instruction"); 508 ldr(r, Address(Rstack_top, 2*wordSize, post_indexed)); 509 } 510 #else 511 void InterpreterMacroAssembler::pop_l(Register lo, Register hi) { 512 assert_different_registers(lo, hi); 513 assert(lo < hi, "lo must be < hi"); 514 pop(RegisterSet(lo) | RegisterSet(hi)); 515 } 516 #endif // AARCH64 517 518 void InterpreterMacroAssembler::pop_f(FloatRegister fd) { 519 #ifdef AARCH64 520 ldr_s(fd, Address(Rstack_top, wordSize, post_indexed)); 521 #else 522 fpops(fd); 523 #endif // AARCH64 524 } 525 526 void InterpreterMacroAssembler::pop_d(FloatRegister fd) { 527 #ifdef AARCH64 528 ldr_d(fd, Address(Rstack_top, 2*wordSize, post_indexed)); 529 #else 530 fpopd(fd); 531 #endif // AARCH64 532 } 533 534 535 // Transition vtos -> state. Blows R0, R1. Sets TOS cached value. 536 void InterpreterMacroAssembler::pop(TosState state) { 537 switch (state) { 538 case atos: pop_ptr(R0_tos); break; 539 case btos: // fall through 540 case ztos: // fall through 541 case ctos: // fall through 542 case stos: // fall through 543 case itos: pop_i(R0_tos); break; 544 #ifdef AARCH64 545 case ltos: pop_l(R0_tos); break; 546 #else 547 case ltos: pop_l(R0_tos_lo, R1_tos_hi); break; 548 #endif // AARCH64 549 #ifdef __SOFTFP__ 550 case ftos: pop_i(R0_tos); break; 551 case dtos: pop_l(R0_tos_lo, R1_tos_hi); break; 552 #else 553 case ftos: pop_f(S0_tos); break; 554 case dtos: pop_d(D0_tos); break; 555 #endif // __SOFTFP__ 556 case vtos: /* nothing to do */ break; 557 default : ShouldNotReachHere(); 558 } 559 interp_verify_oop(R0_tos, state, __FILE__, __LINE__); 560 } 561 562 void InterpreterMacroAssembler::push_ptr(Register r) { 563 assert(r != Rstack_top, "unpredictable instruction"); 564 str(r, Address(Rstack_top, -wordSize, pre_indexed)); 565 check_stack_top_on_expansion(); 566 } 567 568 void InterpreterMacroAssembler::push_i(Register r) { 569 assert(r != Rstack_top, "unpredictable instruction"); 570 str_32(r, Address(Rstack_top, -wordSize, pre_indexed)); 571 check_stack_top_on_expansion(); 572 } 573 574 #ifdef AARCH64 575 void InterpreterMacroAssembler::push_l(Register r) { 576 assert(r != Rstack_top, "unpredictable instruction"); 577 stp(r, ZR, Address(Rstack_top, -2*wordSize, pre_indexed)); 578 check_stack_top_on_expansion(); 579 } 580 #else 581 void InterpreterMacroAssembler::push_l(Register lo, Register hi) { 582 assert_different_registers(lo, hi); 583 assert(lo < hi, "lo must be < hi"); 584 push(RegisterSet(lo) | RegisterSet(hi)); 585 } 586 #endif // AARCH64 587 588 void InterpreterMacroAssembler::push_f() { 589 #ifdef AARCH64 590 str_s(S0_tos, Address(Rstack_top, -wordSize, pre_indexed)); 591 check_stack_top_on_expansion(); 592 #else 593 fpushs(S0_tos); 594 #endif // AARCH64 595 } 596 597 void InterpreterMacroAssembler::push_d() { 598 #ifdef AARCH64 599 str_d(D0_tos, Address(Rstack_top, -2*wordSize, pre_indexed)); 600 check_stack_top_on_expansion(); 601 #else 602 fpushd(D0_tos); 603 #endif // AARCH64 604 } 605 606 // Transition state -> vtos. Blows Rtemp. 607 void InterpreterMacroAssembler::push(TosState state) { 608 interp_verify_oop(R0_tos, state, __FILE__, __LINE__); 609 switch (state) { 610 case atos: push_ptr(R0_tos); break; 611 case btos: // fall through 612 case ztos: // fall through 613 case ctos: // fall through 614 case stos: // fall through 615 case itos: push_i(R0_tos); break; 616 #ifdef AARCH64 617 case ltos: push_l(R0_tos); break; 618 #else 619 case ltos: push_l(R0_tos_lo, R1_tos_hi); break; 620 #endif // AARCH64 621 #ifdef __SOFTFP__ 622 case ftos: push_i(R0_tos); break; 623 case dtos: push_l(R0_tos_lo, R1_tos_hi); break; 624 #else 625 case ftos: push_f(); break; 626 case dtos: push_d(); break; 627 #endif // __SOFTFP__ 628 case vtos: /* nothing to do */ break; 629 default : ShouldNotReachHere(); 630 } 631 } 632 633 634 #ifndef AARCH64 635 636 // Converts return value in R0/R1 (interpreter calling conventions) to TOS cached value. 637 void InterpreterMacroAssembler::convert_retval_to_tos(TosState state) { 638 #if (!defined __SOFTFP__ && !defined __ABI_HARD__) 639 // According to interpreter calling conventions, result is returned in R0/R1, 640 // but templates expect ftos in S0, and dtos in D0. 641 if (state == ftos) { 642 fmsr(S0_tos, R0); 643 } else if (state == dtos) { 644 fmdrr(D0_tos, R0, R1); 645 } 646 #endif // !__SOFTFP__ && !__ABI_HARD__ 647 } 648 649 // Converts TOS cached value to return value in R0/R1 (according to interpreter calling conventions). 650 void InterpreterMacroAssembler::convert_tos_to_retval(TosState state) { 651 #if (!defined __SOFTFP__ && !defined __ABI_HARD__) 652 // According to interpreter calling conventions, result is returned in R0/R1, 653 // so ftos (S0) and dtos (D0) are moved to R0/R1. 654 if (state == ftos) { 655 fmrs(R0, S0_tos); 656 } else if (state == dtos) { 657 fmrrd(R0, R1, D0_tos); 658 } 659 #endif // !__SOFTFP__ && !__ABI_HARD__ 660 } 661 662 #endif // !AARCH64 663 664 665 // Helpers for swap and dup 666 void InterpreterMacroAssembler::load_ptr(int n, Register val) { 667 ldr(val, Address(Rstack_top, Interpreter::expr_offset_in_bytes(n))); 668 } 669 670 void InterpreterMacroAssembler::store_ptr(int n, Register val) { 671 str(val, Address(Rstack_top, Interpreter::expr_offset_in_bytes(n))); 672 } 673 674 675 void InterpreterMacroAssembler::prepare_to_jump_from_interpreted() { 676 #ifdef AARCH64 677 check_no_cached_stack_top(Rtemp); 678 save_stack_top(); 679 cut_sp_before_call(); 680 mov(Rparams, Rstack_top); 681 #endif // AARCH64 682 683 // set sender sp 684 mov(Rsender_sp, SP); 685 686 #ifndef AARCH64 687 // record last_sp 688 str(Rsender_sp, Address(FP, frame::interpreter_frame_last_sp_offset * wordSize)); 689 #endif // !AARCH64 690 } 691 692 // Jump to from_interpreted entry of a call unless single stepping is possible 693 // in this thread in which case we must call the i2i entry 694 void InterpreterMacroAssembler::jump_from_interpreted(Register method) { 695 assert_different_registers(method, Rtemp); 696 697 prepare_to_jump_from_interpreted(); 698 699 if (can_post_interpreter_events()) { 700 // JVMTI events, such as single-stepping, are implemented partly by avoiding running 701 // compiled code in threads for which the event is enabled. Check here for 702 // interp_only_mode if these events CAN be enabled. 703 704 ldr_s32(Rtemp, Address(Rthread, JavaThread::interp_only_mode_offset())); 705 #ifdef AARCH64 706 { 707 Label not_interp_only_mode; 708 709 cbz(Rtemp, not_interp_only_mode); 710 indirect_jump(Address(method, Method::interpreter_entry_offset()), Rtemp); 711 712 bind(not_interp_only_mode); 713 } 714 #else 715 cmp(Rtemp, 0); 716 ldr(PC, Address(method, Method::interpreter_entry_offset()), ne); 717 #endif // AARCH64 718 } 719 720 indirect_jump(Address(method, Method::from_interpreted_offset()), Rtemp); 721 } 722 723 724 void InterpreterMacroAssembler::restore_dispatch() { 725 mov_slow(RdispatchTable, (address)Interpreter::dispatch_table(vtos)); 726 } 727 728 729 // The following two routines provide a hook so that an implementation 730 // can schedule the dispatch in two parts. 731 void InterpreterMacroAssembler::dispatch_prolog(TosState state, int step) { 732 // Nothing ARM-specific to be done here. 733 } 734 735 void InterpreterMacroAssembler::dispatch_epilog(TosState state, int step) { 736 dispatch_next(state, step); 737 } 738 739 void InterpreterMacroAssembler::dispatch_base(TosState state, 740 DispatchTableMode table_mode, 741 bool verifyoop) { 742 if (VerifyActivationFrameSize) { 743 Label L; 744 #ifdef AARCH64 745 mov(Rtemp, SP); 746 sub(Rtemp, FP, Rtemp); 747 #else 748 sub(Rtemp, FP, SP); 749 #endif // AARCH64 750 int min_frame_size = (frame::link_offset - frame::interpreter_frame_initial_sp_offset) * wordSize; 751 cmp(Rtemp, min_frame_size); 752 b(L, ge); 753 stop("broken stack frame"); 754 bind(L); 755 } 756 757 if (verifyoop) { 758 interp_verify_oop(R0_tos, state, __FILE__, __LINE__); 759 } 760 761 if((state == itos) || (state == btos) || (state == ztos) || (state == ctos) || (state == stos)) { 762 zap_high_non_significant_bits(R0_tos); 763 } 764 765 #ifdef ASSERT 766 Label L; 767 mov_slow(Rtemp, (address)Interpreter::dispatch_table(vtos)); 768 cmp(Rtemp, RdispatchTable); 769 b(L, eq); 770 stop("invalid RdispatchTable"); 771 bind(L); 772 #endif 773 774 if (table_mode == DispatchDefault) { 775 if (state == vtos) { 776 indirect_jump(Address::indexed_ptr(RdispatchTable, R3_bytecode), Rtemp); 777 } else { 778 #ifdef AARCH64 779 sub(Rtemp, R3_bytecode, (Interpreter::distance_from_dispatch_table(vtos) - 780 Interpreter::distance_from_dispatch_table(state))); 781 indirect_jump(Address::indexed_ptr(RdispatchTable, Rtemp), Rtemp); 782 #else 783 // on 32-bit ARM this method is faster than the one above. 784 sub(Rtemp, RdispatchTable, (Interpreter::distance_from_dispatch_table(vtos) - 785 Interpreter::distance_from_dispatch_table(state)) * wordSize); 786 indirect_jump(Address::indexed_ptr(Rtemp, R3_bytecode), Rtemp); 787 #endif 788 } 789 } else { 790 assert(table_mode == DispatchNormal, "invalid dispatch table mode"); 791 address table = (address) Interpreter::normal_table(state); 792 mov_slow(Rtemp, table); 793 indirect_jump(Address::indexed_ptr(Rtemp, R3_bytecode), Rtemp); 794 } 795 796 nop(); // to avoid filling CPU pipeline with invalid instructions 797 nop(); 798 } 799 800 void InterpreterMacroAssembler::dispatch_only(TosState state) { 801 dispatch_base(state, DispatchDefault); 802 } 803 804 805 void InterpreterMacroAssembler::dispatch_only_normal(TosState state) { 806 dispatch_base(state, DispatchNormal); 807 } 808 809 void InterpreterMacroAssembler::dispatch_only_noverify(TosState state) { 810 dispatch_base(state, DispatchNormal, false); 811 } 812 813 void InterpreterMacroAssembler::dispatch_next(TosState state, int step) { 814 // load next bytecode and advance Rbcp 815 ldrb(R3_bytecode, Address(Rbcp, step, pre_indexed)); 816 dispatch_base(state, DispatchDefault); 817 } 818 819 void InterpreterMacroAssembler::narrow(Register result) { 820 // mask integer result to narrower return type. 821 const Register Rtmp = R2; 822 823 // get method type 824 ldr(Rtmp, Address(Rmethod, Method::const_offset())); 825 ldrb(Rtmp, Address(Rtmp, ConstMethod::result_type_offset())); 826 827 Label notBool, notByte, notChar, done; 828 cmp(Rtmp, T_INT); 829 b(done, eq); 830 831 cmp(Rtmp, T_BOOLEAN); 832 b(notBool, ne); 833 and_32(result, result, 1); 834 b(done); 835 836 bind(notBool); 837 cmp(Rtmp, T_BYTE); 838 b(notByte, ne); 839 sign_extend(result, result, 8); 840 b(done); 841 842 bind(notByte); 843 cmp(Rtmp, T_CHAR); 844 b(notChar, ne); 845 zero_extend(result, result, 16); 846 b(done); 847 848 bind(notChar); 849 // cmp(Rtmp, T_SHORT); 850 // b(done, ne); 851 sign_extend(result, result, 16); 852 853 // Nothing to do 854 bind(done); 855 } 856 857 // remove activation 858 // 859 // Unlock the receiver if this is a synchronized method. 860 // Unlock any Java monitors from syncronized blocks. 861 // Remove the activation from the stack. 862 // 863 // If there are locked Java monitors 864 // If throw_monitor_exception 865 // throws IllegalMonitorStateException 866 // Else if install_monitor_exception 867 // installs IllegalMonitorStateException 868 // Else 869 // no error processing 870 void InterpreterMacroAssembler::remove_activation(TosState state, Register ret_addr, 871 bool throw_monitor_exception, 872 bool install_monitor_exception, 873 bool notify_jvmdi) { 874 Label unlock, unlocked, no_unlock; 875 876 // Note: Registers R0, R1, S0 and D0 (TOS cached value) may be in use for the result. 877 878 const Address do_not_unlock_if_synchronized(Rthread, 879 JavaThread::do_not_unlock_if_synchronized_offset()); 880 881 const Register Rflag = R2; 882 const Register Raccess_flags = R3; 883 884 restore_method(); 885 886 ldrb(Rflag, do_not_unlock_if_synchronized); 887 888 // get method access flags 889 ldr_u32(Raccess_flags, Address(Rmethod, Method::access_flags_offset())); 890 891 strb(zero_register(Rtemp), do_not_unlock_if_synchronized); // reset the flag 892 893 // check if method is synchronized 894 895 tbz(Raccess_flags, JVM_ACC_SYNCHRONIZED_BIT, unlocked); 896 897 // Don't unlock anything if the _do_not_unlock_if_synchronized flag is set. 898 cbnz(Rflag, no_unlock); 899 900 // unlock monitor 901 push(state); // save result 902 903 // BasicObjectLock will be first in list, since this is a synchronized method. However, need 904 // to check that the object has not been unlocked by an explicit monitorexit bytecode. 905 906 const Register Rmonitor = R1; // fixed in unlock_object() 907 const Register Robj = R2; 908 909 // address of first monitor 910 sub(Rmonitor, FP, - frame::interpreter_frame_monitor_block_bottom_offset * wordSize + (int)sizeof(BasicObjectLock)); 911 912 ldr(Robj, Address(Rmonitor, BasicObjectLock::obj_offset_in_bytes())); 913 cbnz(Robj, unlock); 914 915 pop(state); 916 917 if (throw_monitor_exception) { 918 // Entry already unlocked, need to throw exception 919 call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::throw_illegal_monitor_state_exception)); 920 should_not_reach_here(); 921 } else { 922 // Monitor already unlocked during a stack unroll. 923 // If requested, install an illegal_monitor_state_exception. 924 // Continue with stack unrolling. 925 if (install_monitor_exception) { 926 call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::new_illegal_monitor_state_exception)); 927 } 928 b(unlocked); 929 } 930 931 932 // Exception case for the check that all monitors are unlocked. 933 const Register Rcur = R2; 934 Label restart_check_monitors_unlocked, exception_monitor_is_still_locked; 935 936 bind(exception_monitor_is_still_locked); 937 // Monitor entry is still locked, need to throw exception. 938 // Rcur: monitor entry. 939 940 if (throw_monitor_exception) { 941 // Throw exception 942 call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::throw_illegal_monitor_state_exception)); 943 should_not_reach_here(); 944 } else { 945 // Stack unrolling. Unlock object and install illegal_monitor_exception 946 // Unlock does not block, so don't have to worry about the frame 947 948 push(state); 949 mov(R1, Rcur); 950 unlock_object(R1); 951 952 if (install_monitor_exception) { 953 call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::new_illegal_monitor_state_exception)); 954 } 955 956 pop(state); 957 b(restart_check_monitors_unlocked); 958 } 959 960 bind(unlock); 961 unlock_object(Rmonitor); 962 pop(state); 963 964 // Check that for block-structured locking (i.e., that all locked objects has been unlocked) 965 bind(unlocked); 966 967 // Check that all monitors are unlocked 968 { 969 Label loop; 970 971 const int entry_size = frame::interpreter_frame_monitor_size() * wordSize; 972 const Register Rbottom = R3; 973 const Register Rcur_obj = Rtemp; 974 975 bind(restart_check_monitors_unlocked); 976 977 ldr(Rcur, Address(FP, frame::interpreter_frame_monitor_block_top_offset * wordSize)); 978 // points to current entry, starting with top-most entry 979 sub(Rbottom, FP, -frame::interpreter_frame_monitor_block_bottom_offset * wordSize); 980 // points to word before bottom of monitor block 981 982 cmp(Rcur, Rbottom); // check if there are no monitors 983 #ifndef AARCH64 984 ldr(Rcur_obj, Address(Rcur, BasicObjectLock::obj_offset_in_bytes()), ne); 985 // prefetch monitor's object 986 #endif // !AARCH64 987 b(no_unlock, eq); 988 989 bind(loop); 990 #ifdef AARCH64 991 ldr(Rcur_obj, Address(Rcur, BasicObjectLock::obj_offset_in_bytes())); 992 #endif // AARCH64 993 // check if current entry is used 994 cbnz(Rcur_obj, exception_monitor_is_still_locked); 995 996 add(Rcur, Rcur, entry_size); // otherwise advance to next entry 997 cmp(Rcur, Rbottom); // check if bottom reached 998 #ifndef AARCH64 999 ldr(Rcur_obj, Address(Rcur, BasicObjectLock::obj_offset_in_bytes()), ne); 1000 // prefetch monitor's object 1001 #endif // !AARCH64 1002 b(loop, ne); // if not at bottom then check this entry 1003 } 1004 1005 bind(no_unlock); 1006 1007 // jvmti support 1008 if (notify_jvmdi) { 1009 notify_method_exit(state, NotifyJVMTI); // preserve TOSCA 1010 } else { 1011 notify_method_exit(state, SkipNotifyJVMTI); // preserve TOSCA 1012 } 1013 1014 // remove activation 1015 #ifdef AARCH64 1016 ldr(Rtemp, Address(FP, frame::interpreter_frame_sender_sp_offset * wordSize)); 1017 ldp(FP, LR, Address(FP)); 1018 mov(SP, Rtemp); 1019 #else 1020 mov(Rtemp, FP); 1021 ldmia(FP, RegisterSet(FP) | RegisterSet(LR)); 1022 ldr(SP, Address(Rtemp, frame::interpreter_frame_sender_sp_offset * wordSize)); 1023 #endif 1024 1025 if (ret_addr != LR) { 1026 mov(ret_addr, LR); 1027 } 1028 } 1029 1030 1031 // At certain points in the method invocation the monitor of 1032 // synchronized methods hasn't been entered yet. 1033 // To correctly handle exceptions at these points, we set the thread local 1034 // variable _do_not_unlock_if_synchronized to true. The remove_activation will 1035 // check this flag. 1036 void InterpreterMacroAssembler::set_do_not_unlock_if_synchronized(bool flag, Register tmp) { 1037 const Address do_not_unlock_if_synchronized(Rthread, 1038 JavaThread::do_not_unlock_if_synchronized_offset()); 1039 if (flag) { 1040 mov(tmp, 1); 1041 strb(tmp, do_not_unlock_if_synchronized); 1042 } else { 1043 strb(zero_register(tmp), do_not_unlock_if_synchronized); 1044 } 1045 } 1046 1047 // Lock object 1048 // 1049 // Argument: R1 : Points to BasicObjectLock to be used for locking. 1050 // Must be initialized with object to lock. 1051 // Blows volatile registers (R0-R3 on 32-bit ARM, R0-R18 on AArch64), Rtemp, LR. Calls VM. 1052 void InterpreterMacroAssembler::lock_object(Register Rlock) { 1053 assert(Rlock == R1, "the second argument"); 1054 1055 if (UseHeavyMonitors) { 1056 call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::monitorenter), Rlock); 1057 } else { 1058 Label done; 1059 1060 const Register Robj = R2; 1061 const Register Rmark = R3; 1062 assert_different_registers(Robj, Rmark, Rlock, R0, Rtemp); 1063 1064 const int obj_offset = BasicObjectLock::obj_offset_in_bytes(); 1065 const int lock_offset = BasicObjectLock::lock_offset_in_bytes (); 1066 const int mark_offset = lock_offset + BasicLock::displaced_header_offset_in_bytes(); 1067 1068 Label already_locked, slow_case; 1069 1070 // Load object pointer 1071 ldr(Robj, Address(Rlock, obj_offset)); 1072 1073 if (UseBiasedLocking) { 1074 biased_locking_enter(Robj, Rmark/*scratched*/, R0, false, Rtemp, done, slow_case); 1075 } 1076 1077 #ifdef AARCH64 1078 assert(oopDesc::mark_offset_in_bytes() == 0, "must be"); 1079 ldr(Rmark, Robj); 1080 1081 // Test if object is already locked 1082 assert(markOopDesc::unlocked_value == 1, "adjust this code"); 1083 tbz(Rmark, exact_log2(markOopDesc::unlocked_value), already_locked); 1084 1085 #else // AARCH64 1086 1087 // On MP platforms the next load could return a 'stale' value if the memory location has been modified by another thread. 1088 // That would be acceptable as ether CAS or slow case path is taken in that case. 1089 // Exception to that is if the object is locked by the calling thread, then the recursive test will pass (guaranteed as 1090 // loads are satisfied from a store queue if performed on the same processor). 1091 1092 assert(oopDesc::mark_offset_in_bytes() == 0, "must be"); 1093 ldr(Rmark, Address(Robj, oopDesc::mark_offset_in_bytes())); 1094 1095 // Test if object is already locked 1096 tst(Rmark, markOopDesc::unlocked_value); 1097 b(already_locked, eq); 1098 1099 #endif // !AARCH64 1100 // Save old object->mark() into BasicLock's displaced header 1101 str(Rmark, Address(Rlock, mark_offset)); 1102 1103 cas_for_lock_acquire(Rmark, Rlock, Robj, Rtemp, slow_case); 1104 1105 #ifndef PRODUCT 1106 if (PrintBiasedLockingStatistics) { 1107 cond_atomic_inc32(al, BiasedLocking::fast_path_entry_count_addr()); 1108 } 1109 #endif //!PRODUCT 1110 1111 b(done); 1112 1113 // If we got here that means the object is locked by ether calling thread or another thread. 1114 bind(already_locked); 1115 // Handling of locked objects: recursive locks and slow case. 1116 1117 // Fast check for recursive lock. 1118 // 1119 // Can apply the optimization only if this is a stack lock 1120 // allocated in this thread. For efficiency, we can focus on 1121 // recently allocated stack locks (instead of reading the stack 1122 // base and checking whether 'mark' points inside the current 1123 // thread stack): 1124 // 1) (mark & 3) == 0 1125 // 2) SP <= mark < SP + os::pagesize() 1126 // 1127 // Warning: SP + os::pagesize can overflow the stack base. We must 1128 // neither apply the optimization for an inflated lock allocated 1129 // just above the thread stack (this is why condition 1 matters) 1130 // nor apply the optimization if the stack lock is inside the stack 1131 // of another thread. The latter is avoided even in case of overflow 1132 // because we have guard pages at the end of all stacks. Hence, if 1133 // we go over the stack base and hit the stack of another thread, 1134 // this should not be in a writeable area that could contain a 1135 // stack lock allocated by that thread. As a consequence, a stack 1136 // lock less than page size away from SP is guaranteed to be 1137 // owned by the current thread. 1138 // 1139 // Note: assuming SP is aligned, we can check the low bits of 1140 // (mark-SP) instead of the low bits of mark. In that case, 1141 // assuming page size is a power of 2, we can merge the two 1142 // conditions into a single test: 1143 // => ((mark - SP) & (3 - os::pagesize())) == 0 1144 1145 #ifdef AARCH64 1146 // Use the single check since the immediate is OK for AARCH64 1147 sub(R0, Rmark, Rstack_top); 1148 intptr_t mask = ((intptr_t)3) - ((intptr_t)os::vm_page_size()); 1149 Assembler::LogicalImmediate imm(mask, false); 1150 ands(R0, R0, imm); 1151 1152 // For recursive case store 0 into lock record. 1153 // It is harmless to store it unconditionally as lock record contains some garbage 1154 // value in its _displaced_header field by this moment. 1155 str(ZR, Address(Rlock, mark_offset)); 1156 1157 #else // AARCH64 1158 // (3 - os::pagesize()) cannot be encoded as an ARM immediate operand. 1159 // Check independently the low bits and the distance to SP. 1160 // -1- test low 2 bits 1161 movs(R0, AsmOperand(Rmark, lsl, 30)); 1162 // -2- test (mark - SP) if the low two bits are 0 1163 sub(R0, Rmark, SP, eq); 1164 movs(R0, AsmOperand(R0, lsr, exact_log2(os::vm_page_size())), eq); 1165 // If still 'eq' then recursive locking OK: store 0 into lock record 1166 str(R0, Address(Rlock, mark_offset), eq); 1167 1168 #endif // AARCH64 1169 1170 #ifndef PRODUCT 1171 if (PrintBiasedLockingStatistics) { 1172 cond_atomic_inc32(eq, BiasedLocking::fast_path_entry_count_addr()); 1173 } 1174 #endif // !PRODUCT 1175 1176 b(done, eq); 1177 1178 bind(slow_case); 1179 1180 // Call the runtime routine for slow case 1181 call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::monitorenter), Rlock); 1182 1183 bind(done); 1184 } 1185 } 1186 1187 1188 // Unlocks an object. Used in monitorexit bytecode and remove_activation. 1189 // 1190 // Argument: R1: Points to BasicObjectLock structure for lock 1191 // Throw an IllegalMonitorException if object is not locked by current thread 1192 // Blows volatile registers (R0-R3 on 32-bit ARM, R0-R18 on AArch64), Rtemp, LR. Calls VM. 1193 void InterpreterMacroAssembler::unlock_object(Register Rlock) { 1194 assert(Rlock == R1, "the second argument"); 1195 1196 if (UseHeavyMonitors) { 1197 call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::monitorexit), Rlock); 1198 } else { 1199 Label done, slow_case; 1200 1201 const Register Robj = R2; 1202 const Register Rmark = R3; 1203 const Register Rresult = R0; 1204 assert_different_registers(Robj, Rmark, Rlock, R0, Rtemp); 1205 1206 const int obj_offset = BasicObjectLock::obj_offset_in_bytes(); 1207 const int lock_offset = BasicObjectLock::lock_offset_in_bytes (); 1208 const int mark_offset = lock_offset + BasicLock::displaced_header_offset_in_bytes(); 1209 1210 const Register Rzero = zero_register(Rtemp); 1211 1212 // Load oop into Robj 1213 ldr(Robj, Address(Rlock, obj_offset)); 1214 1215 // Free entry 1216 str(Rzero, Address(Rlock, obj_offset)); 1217 1218 if (UseBiasedLocking) { 1219 biased_locking_exit(Robj, Rmark, done); 1220 } 1221 1222 // Load the old header from BasicLock structure 1223 ldr(Rmark, Address(Rlock, mark_offset)); 1224 1225 // Test for recursion (zero mark in BasicLock) 1226 cbz(Rmark, done); 1227 1228 bool allow_fallthrough_on_failure = true; 1229 1230 cas_for_lock_release(Rlock, Rmark, Robj, Rtemp, slow_case, allow_fallthrough_on_failure); 1231 1232 b(done, eq); 1233 1234 bind(slow_case); 1235 1236 // Call the runtime routine for slow case. 1237 str(Robj, Address(Rlock, obj_offset)); // restore obj 1238 call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::monitorexit), Rlock); 1239 1240 bind(done); 1241 } 1242 } 1243 1244 1245 // Test ImethodDataPtr. If it is null, continue at the specified label 1246 void InterpreterMacroAssembler::test_method_data_pointer(Register mdp, Label& zero_continue) { 1247 assert(ProfileInterpreter, "must be profiling interpreter"); 1248 ldr(mdp, Address(FP, frame::interpreter_frame_mdp_offset * wordSize)); 1249 cbz(mdp, zero_continue); 1250 } 1251 1252 1253 // Set the method data pointer for the current bcp. 1254 // Blows volatile registers (R0-R3 on 32-bit ARM, R0-R18 on AArch64), Rtemp, LR. 1255 void InterpreterMacroAssembler::set_method_data_pointer_for_bcp() { 1256 assert(ProfileInterpreter, "must be profiling interpreter"); 1257 Label set_mdp; 1258 1259 // Test MDO to avoid the call if it is NULL. 1260 ldr(Rtemp, Address(Rmethod, Method::method_data_offset())); 1261 cbz(Rtemp, set_mdp); 1262 1263 mov(R0, Rmethod); 1264 mov(R1, Rbcp); 1265 call_VM_leaf(CAST_FROM_FN_PTR(address, InterpreterRuntime::bcp_to_di), R0, R1); 1266 // R0/W0: mdi 1267 1268 // mdo is guaranteed to be non-zero here, we checked for it before the call. 1269 ldr(Rtemp, Address(Rmethod, Method::method_data_offset())); 1270 add(Rtemp, Rtemp, in_bytes(MethodData::data_offset())); 1271 add_ptr_scaled_int32(Rtemp, Rtemp, R0, 0); 1272 1273 bind(set_mdp); 1274 str(Rtemp, Address(FP, frame::interpreter_frame_mdp_offset * wordSize)); 1275 } 1276 1277 1278 void InterpreterMacroAssembler::verify_method_data_pointer() { 1279 assert(ProfileInterpreter, "must be profiling interpreter"); 1280 #ifdef ASSERT 1281 Label verify_continue; 1282 save_caller_save_registers(); 1283 1284 const Register Rmdp = R2; 1285 test_method_data_pointer(Rmdp, verify_continue); // If mdp is zero, continue 1286 1287 // If the mdp is valid, it will point to a DataLayout header which is 1288 // consistent with the bcp. The converse is highly probable also. 1289 1290 ldrh(R3, Address(Rmdp, DataLayout::bci_offset())); 1291 ldr(Rtemp, Address(Rmethod, Method::const_offset())); 1292 add(R3, R3, Rtemp); 1293 add(R3, R3, in_bytes(ConstMethod::codes_offset())); 1294 cmp(R3, Rbcp); 1295 b(verify_continue, eq); 1296 1297 mov(R0, Rmethod); 1298 mov(R1, Rbcp); 1299 call_VM_leaf(CAST_FROM_FN_PTR(address, InterpreterRuntime::verify_mdp), R0, R1, Rmdp); 1300 1301 bind(verify_continue); 1302 restore_caller_save_registers(); 1303 #endif // ASSERT 1304 } 1305 1306 1307 void InterpreterMacroAssembler::set_mdp_data_at(Register mdp_in, int offset, Register value) { 1308 assert(ProfileInterpreter, "must be profiling interpreter"); 1309 assert_different_registers(mdp_in, value); 1310 str(value, Address(mdp_in, offset)); 1311 } 1312 1313 1314 // Increments mdp data. Sets bumped_count register to adjusted counter. 1315 void InterpreterMacroAssembler::increment_mdp_data_at(Register mdp_in, 1316 int offset, 1317 Register bumped_count, 1318 bool decrement) { 1319 assert(ProfileInterpreter, "must be profiling interpreter"); 1320 1321 // Counter address 1322 Address data(mdp_in, offset); 1323 assert_different_registers(mdp_in, bumped_count); 1324 1325 increment_mdp_data_at(data, bumped_count, decrement); 1326 } 1327 1328 void InterpreterMacroAssembler::set_mdp_flag_at(Register mdp_in, int flag_byte_constant) { 1329 assert_different_registers(mdp_in, Rtemp); 1330 assert(ProfileInterpreter, "must be profiling interpreter"); 1331 assert((0 < flag_byte_constant) && (flag_byte_constant < (1 << BitsPerByte)), "flag mask is out of range"); 1332 1333 // Set the flag 1334 ldrb(Rtemp, Address(mdp_in, in_bytes(DataLayout::flags_offset()))); 1335 orr(Rtemp, Rtemp, (unsigned)flag_byte_constant); 1336 strb(Rtemp, Address(mdp_in, in_bytes(DataLayout::flags_offset()))); 1337 } 1338 1339 1340 // Increments mdp data. Sets bumped_count register to adjusted counter. 1341 void InterpreterMacroAssembler::increment_mdp_data_at(Address data, 1342 Register bumped_count, 1343 bool decrement) { 1344 assert(ProfileInterpreter, "must be profiling interpreter"); 1345 1346 ldr(bumped_count, data); 1347 if (decrement) { 1348 // Decrement the register. Set condition codes. 1349 subs(bumped_count, bumped_count, DataLayout::counter_increment); 1350 // Avoid overflow. 1351 #ifdef AARCH64 1352 assert(DataLayout::counter_increment == 1, "required for cinc"); 1353 cinc(bumped_count, bumped_count, pl); 1354 #else 1355 add(bumped_count, bumped_count, DataLayout::counter_increment, pl); 1356 #endif // AARCH64 1357 } else { 1358 // Increment the register. Set condition codes. 1359 adds(bumped_count, bumped_count, DataLayout::counter_increment); 1360 // Avoid overflow. 1361 #ifdef AARCH64 1362 assert(DataLayout::counter_increment == 1, "required for cinv"); 1363 cinv(bumped_count, bumped_count, mi); // inverts 0x80..00 back to 0x7f..ff 1364 #else 1365 sub(bumped_count, bumped_count, DataLayout::counter_increment, mi); 1366 #endif // AARCH64 1367 } 1368 str(bumped_count, data); 1369 } 1370 1371 1372 void InterpreterMacroAssembler::test_mdp_data_at(Register mdp_in, 1373 int offset, 1374 Register value, 1375 Register test_value_out, 1376 Label& not_equal_continue) { 1377 assert(ProfileInterpreter, "must be profiling interpreter"); 1378 assert_different_registers(mdp_in, test_value_out, value); 1379 1380 ldr(test_value_out, Address(mdp_in, offset)); 1381 cmp(test_value_out, value); 1382 1383 b(not_equal_continue, ne); 1384 } 1385 1386 1387 void InterpreterMacroAssembler::update_mdp_by_offset(Register mdp_in, int offset_of_disp, Register reg_temp) { 1388 assert(ProfileInterpreter, "must be profiling interpreter"); 1389 assert_different_registers(mdp_in, reg_temp); 1390 1391 ldr(reg_temp, Address(mdp_in, offset_of_disp)); 1392 add(mdp_in, mdp_in, reg_temp); 1393 str(mdp_in, Address(FP, frame::interpreter_frame_mdp_offset * wordSize)); 1394 } 1395 1396 1397 void InterpreterMacroAssembler::update_mdp_by_offset(Register mdp_in, Register reg_offset, Register reg_tmp) { 1398 assert(ProfileInterpreter, "must be profiling interpreter"); 1399 assert_different_registers(mdp_in, reg_offset, reg_tmp); 1400 1401 ldr(reg_tmp, Address(mdp_in, reg_offset)); 1402 add(mdp_in, mdp_in, reg_tmp); 1403 str(mdp_in, Address(FP, frame::interpreter_frame_mdp_offset * wordSize)); 1404 } 1405 1406 1407 void InterpreterMacroAssembler::update_mdp_by_constant(Register mdp_in, int constant) { 1408 assert(ProfileInterpreter, "must be profiling interpreter"); 1409 add(mdp_in, mdp_in, constant); 1410 str(mdp_in, Address(FP, frame::interpreter_frame_mdp_offset * wordSize)); 1411 } 1412 1413 1414 // Blows volatile registers (R0-R3 on 32-bit ARM, R0-R18 on AArch64, Rtemp, LR). 1415 void InterpreterMacroAssembler::update_mdp_for_ret(Register return_bci) { 1416 assert(ProfileInterpreter, "must be profiling interpreter"); 1417 assert_different_registers(return_bci, R0, R1, R2, R3, Rtemp); 1418 1419 mov(R1, return_bci); 1420 call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::update_mdp_for_ret), R1); 1421 } 1422 1423 1424 // Sets mdp, bumped_count registers, blows Rtemp. 1425 void InterpreterMacroAssembler::profile_taken_branch(Register mdp, Register bumped_count) { 1426 assert_different_registers(mdp, bumped_count); 1427 1428 if (ProfileInterpreter) { 1429 Label profile_continue; 1430 1431 // If no method data exists, go to profile_continue. 1432 // Otherwise, assign to mdp 1433 test_method_data_pointer(mdp, profile_continue); 1434 1435 // We are taking a branch. Increment the taken count. 1436 increment_mdp_data_at(mdp, in_bytes(JumpData::taken_offset()), bumped_count); 1437 1438 // The method data pointer needs to be updated to reflect the new target. 1439 update_mdp_by_offset(mdp, in_bytes(JumpData::displacement_offset()), Rtemp); 1440 1441 bind (profile_continue); 1442 } 1443 } 1444 1445 1446 // Sets mdp, blows Rtemp. 1447 void InterpreterMacroAssembler::profile_not_taken_branch(Register mdp) { 1448 assert_different_registers(mdp, Rtemp); 1449 1450 if (ProfileInterpreter) { 1451 Label profile_continue; 1452 1453 // If no method data exists, go to profile_continue. 1454 test_method_data_pointer(mdp, profile_continue); 1455 1456 // We are taking a branch. Increment the not taken count. 1457 increment_mdp_data_at(mdp, in_bytes(BranchData::not_taken_offset()), Rtemp); 1458 1459 // The method data pointer needs to be updated to correspond to the next bytecode 1460 update_mdp_by_constant(mdp, in_bytes(BranchData::branch_data_size())); 1461 1462 bind (profile_continue); 1463 } 1464 } 1465 1466 1467 // Sets mdp, blows Rtemp. 1468 void InterpreterMacroAssembler::profile_call(Register mdp) { 1469 assert_different_registers(mdp, Rtemp); 1470 1471 if (ProfileInterpreter) { 1472 Label profile_continue; 1473 1474 // If no method data exists, go to profile_continue. 1475 test_method_data_pointer(mdp, profile_continue); 1476 1477 // We are making a call. Increment the count. 1478 increment_mdp_data_at(mdp, in_bytes(CounterData::count_offset()), Rtemp); 1479 1480 // The method data pointer needs to be updated to reflect the new target. 1481 update_mdp_by_constant(mdp, in_bytes(CounterData::counter_data_size())); 1482 1483 bind (profile_continue); 1484 } 1485 } 1486 1487 1488 // Sets mdp, blows Rtemp. 1489 void InterpreterMacroAssembler::profile_final_call(Register mdp) { 1490 if (ProfileInterpreter) { 1491 Label profile_continue; 1492 1493 // If no method data exists, go to profile_continue. 1494 test_method_data_pointer(mdp, profile_continue); 1495 1496 // We are making a call. Increment the count. 1497 increment_mdp_data_at(mdp, in_bytes(CounterData::count_offset()), Rtemp); 1498 1499 // The method data pointer needs to be updated to reflect the new target. 1500 update_mdp_by_constant(mdp, in_bytes(VirtualCallData::virtual_call_data_size())); 1501 1502 bind (profile_continue); 1503 } 1504 } 1505 1506 1507 // Sets mdp, blows Rtemp. 1508 void InterpreterMacroAssembler::profile_virtual_call(Register mdp, Register receiver, bool receiver_can_be_null) { 1509 assert_different_registers(mdp, receiver, Rtemp); 1510 1511 if (ProfileInterpreter) { 1512 Label profile_continue; 1513 1514 // If no method data exists, go to profile_continue. 1515 test_method_data_pointer(mdp, profile_continue); 1516 1517 Label skip_receiver_profile; 1518 if (receiver_can_be_null) { 1519 Label not_null; 1520 cbnz(receiver, not_null); 1521 // We are making a call. Increment the count for null receiver. 1522 increment_mdp_data_at(mdp, in_bytes(CounterData::count_offset()), Rtemp); 1523 b(skip_receiver_profile); 1524 bind(not_null); 1525 } 1526 1527 // Record the receiver type. 1528 record_klass_in_profile(receiver, mdp, Rtemp, true); 1529 bind(skip_receiver_profile); 1530 1531 // The method data pointer needs to be updated to reflect the new target. 1532 update_mdp_by_constant(mdp, in_bytes(VirtualCallData::virtual_call_data_size())); 1533 bind(profile_continue); 1534 } 1535 } 1536 1537 1538 void InterpreterMacroAssembler::record_klass_in_profile_helper( 1539 Register receiver, Register mdp, 1540 Register reg_tmp, 1541 int start_row, Label& done, bool is_virtual_call) { 1542 if (TypeProfileWidth == 0) 1543 return; 1544 1545 assert_different_registers(receiver, mdp, reg_tmp); 1546 1547 int last_row = VirtualCallData::row_limit() - 1; 1548 assert(start_row <= last_row, "must be work left to do"); 1549 // Test this row for both the receiver and for null. 1550 // Take any of three different outcomes: 1551 // 1. found receiver => increment count and goto done 1552 // 2. found null => keep looking for case 1, maybe allocate this cell 1553 // 3. found something else => keep looking for cases 1 and 2 1554 // Case 3 is handled by a recursive call. 1555 for (int row = start_row; row <= last_row; row++) { 1556 Label next_test; 1557 1558 // See if the receiver is receiver[n]. 1559 int recvr_offset = in_bytes(VirtualCallData::receiver_offset(row)); 1560 1561 test_mdp_data_at(mdp, recvr_offset, receiver, reg_tmp, next_test); 1562 1563 // The receiver is receiver[n]. Increment count[n]. 1564 int count_offset = in_bytes(VirtualCallData::receiver_count_offset(row)); 1565 increment_mdp_data_at(mdp, count_offset, reg_tmp); 1566 b(done); 1567 1568 bind(next_test); 1569 // reg_tmp now contains the receiver from the CallData. 1570 1571 if (row == start_row) { 1572 Label found_null; 1573 // Failed the equality check on receiver[n]... Test for null. 1574 if (start_row == last_row) { 1575 // The only thing left to do is handle the null case. 1576 if (is_virtual_call) { 1577 cbz(reg_tmp, found_null); 1578 // Receiver did not match any saved receiver and there is no empty row for it. 1579 // Increment total counter to indicate polymorphic case. 1580 increment_mdp_data_at(mdp, in_bytes(CounterData::count_offset()), reg_tmp); 1581 b(done); 1582 bind(found_null); 1583 } else { 1584 cbnz(reg_tmp, done); 1585 } 1586 break; 1587 } 1588 // Since null is rare, make it be the branch-taken case. 1589 cbz(reg_tmp, found_null); 1590 1591 // Put all the "Case 3" tests here. 1592 record_klass_in_profile_helper(receiver, mdp, reg_tmp, start_row + 1, done, is_virtual_call); 1593 1594 // Found a null. Keep searching for a matching receiver, 1595 // but remember that this is an empty (unused) slot. 1596 bind(found_null); 1597 } 1598 } 1599 1600 // In the fall-through case, we found no matching receiver, but we 1601 // observed the receiver[start_row] is NULL. 1602 1603 // Fill in the receiver field and increment the count. 1604 int recvr_offset = in_bytes(VirtualCallData::receiver_offset(start_row)); 1605 set_mdp_data_at(mdp, recvr_offset, receiver); 1606 int count_offset = in_bytes(VirtualCallData::receiver_count_offset(start_row)); 1607 mov(reg_tmp, DataLayout::counter_increment); 1608 set_mdp_data_at(mdp, count_offset, reg_tmp); 1609 if (start_row > 0) { 1610 b(done); 1611 } 1612 } 1613 1614 void InterpreterMacroAssembler::record_klass_in_profile(Register receiver, 1615 Register mdp, 1616 Register reg_tmp, 1617 bool is_virtual_call) { 1618 assert(ProfileInterpreter, "must be profiling"); 1619 assert_different_registers(receiver, mdp, reg_tmp); 1620 1621 Label done; 1622 1623 record_klass_in_profile_helper(receiver, mdp, reg_tmp, 0, done, is_virtual_call); 1624 1625 bind (done); 1626 } 1627 1628 // Sets mdp, blows volatile registers (R0-R3 on 32-bit ARM, R0-R18 on AArch64, Rtemp, LR). 1629 void InterpreterMacroAssembler::profile_ret(Register mdp, Register return_bci) { 1630 assert_different_registers(mdp, return_bci, Rtemp, R0, R1, R2, R3); 1631 1632 if (ProfileInterpreter) { 1633 Label profile_continue; 1634 uint row; 1635 1636 // If no method data exists, go to profile_continue. 1637 test_method_data_pointer(mdp, profile_continue); 1638 1639 // Update the total ret count. 1640 increment_mdp_data_at(mdp, in_bytes(CounterData::count_offset()), Rtemp); 1641 1642 for (row = 0; row < RetData::row_limit(); row++) { 1643 Label next_test; 1644 1645 // See if return_bci is equal to bci[n]: 1646 test_mdp_data_at(mdp, in_bytes(RetData::bci_offset(row)), return_bci, 1647 Rtemp, next_test); 1648 1649 // return_bci is equal to bci[n]. Increment the count. 1650 increment_mdp_data_at(mdp, in_bytes(RetData::bci_count_offset(row)), Rtemp); 1651 1652 // The method data pointer needs to be updated to reflect the new target. 1653 update_mdp_by_offset(mdp, in_bytes(RetData::bci_displacement_offset(row)), Rtemp); 1654 b(profile_continue); 1655 bind(next_test); 1656 } 1657 1658 update_mdp_for_ret(return_bci); 1659 1660 bind(profile_continue); 1661 } 1662 } 1663 1664 1665 // Sets mdp. 1666 void InterpreterMacroAssembler::profile_null_seen(Register mdp) { 1667 if (ProfileInterpreter) { 1668 Label profile_continue; 1669 1670 // If no method data exists, go to profile_continue. 1671 test_method_data_pointer(mdp, profile_continue); 1672 1673 set_mdp_flag_at(mdp, BitData::null_seen_byte_constant()); 1674 1675 // The method data pointer needs to be updated. 1676 int mdp_delta = in_bytes(BitData::bit_data_size()); 1677 if (TypeProfileCasts) { 1678 mdp_delta = in_bytes(VirtualCallData::virtual_call_data_size()); 1679 } 1680 update_mdp_by_constant(mdp, mdp_delta); 1681 1682 bind (profile_continue); 1683 } 1684 } 1685 1686 1687 // Sets mdp, blows Rtemp. 1688 void InterpreterMacroAssembler::profile_typecheck_failed(Register mdp) { 1689 assert_different_registers(mdp, Rtemp); 1690 1691 if (ProfileInterpreter && TypeProfileCasts) { 1692 Label profile_continue; 1693 1694 // If no method data exists, go to profile_continue. 1695 test_method_data_pointer(mdp, profile_continue); 1696 1697 int count_offset = in_bytes(CounterData::count_offset()); 1698 // Back up the address, since we have already bumped the mdp. 1699 count_offset -= in_bytes(VirtualCallData::virtual_call_data_size()); 1700 1701 // *Decrement* the counter. We expect to see zero or small negatives. 1702 increment_mdp_data_at(mdp, count_offset, Rtemp, true); 1703 1704 bind (profile_continue); 1705 } 1706 } 1707 1708 1709 // Sets mdp, blows Rtemp. 1710 void InterpreterMacroAssembler::profile_typecheck(Register mdp, Register klass) 1711 { 1712 assert_different_registers(mdp, klass, Rtemp); 1713 1714 if (ProfileInterpreter) { 1715 Label profile_continue; 1716 1717 // If no method data exists, go to profile_continue. 1718 test_method_data_pointer(mdp, profile_continue); 1719 1720 // The method data pointer needs to be updated. 1721 int mdp_delta = in_bytes(BitData::bit_data_size()); 1722 if (TypeProfileCasts) { 1723 mdp_delta = in_bytes(VirtualCallData::virtual_call_data_size()); 1724 1725 // Record the object type. 1726 record_klass_in_profile(klass, mdp, Rtemp, false); 1727 } 1728 update_mdp_by_constant(mdp, mdp_delta); 1729 1730 bind(profile_continue); 1731 } 1732 } 1733 1734 1735 // Sets mdp, blows Rtemp. 1736 void InterpreterMacroAssembler::profile_switch_default(Register mdp) { 1737 assert_different_registers(mdp, Rtemp); 1738 1739 if (ProfileInterpreter) { 1740 Label profile_continue; 1741 1742 // If no method data exists, go to profile_continue. 1743 test_method_data_pointer(mdp, profile_continue); 1744 1745 // Update the default case count 1746 increment_mdp_data_at(mdp, in_bytes(MultiBranchData::default_count_offset()), Rtemp); 1747 1748 // The method data pointer needs to be updated. 1749 update_mdp_by_offset(mdp, in_bytes(MultiBranchData::default_displacement_offset()), Rtemp); 1750 1751 bind(profile_continue); 1752 } 1753 } 1754 1755 1756 // Sets mdp. Blows reg_tmp1, reg_tmp2. Index could be the same as reg_tmp2. 1757 void InterpreterMacroAssembler::profile_switch_case(Register mdp, Register index, Register reg_tmp1, Register reg_tmp2) { 1758 assert_different_registers(mdp, reg_tmp1, reg_tmp2); 1759 assert_different_registers(mdp, reg_tmp1, index); 1760 1761 if (ProfileInterpreter) { 1762 Label profile_continue; 1763 1764 const int count_offset = in_bytes(MultiBranchData::case_array_offset()) + 1765 in_bytes(MultiBranchData::relative_count_offset()); 1766 1767 const int displacement_offset = in_bytes(MultiBranchData::case_array_offset()) + 1768 in_bytes(MultiBranchData::relative_displacement_offset()); 1769 1770 // If no method data exists, go to profile_continue. 1771 test_method_data_pointer(mdp, profile_continue); 1772 1773 // Build the base (index * per_case_size_in_bytes()) 1774 logical_shift_left(reg_tmp1, index, exact_log2(in_bytes(MultiBranchData::per_case_size()))); 1775 1776 // Update the case count 1777 add(reg_tmp1, reg_tmp1, count_offset); 1778 increment_mdp_data_at(Address(mdp, reg_tmp1), reg_tmp2); 1779 1780 // The method data pointer needs to be updated. 1781 add(reg_tmp1, reg_tmp1, displacement_offset - count_offset); 1782 update_mdp_by_offset(mdp, reg_tmp1, reg_tmp2); 1783 1784 bind (profile_continue); 1785 } 1786 } 1787 1788 1789 void InterpreterMacroAssembler::byteswap_u32(Register r, Register rtmp1, Register rtmp2) { 1790 #ifdef AARCH64 1791 rev_w(r, r); 1792 #else 1793 if (VM_Version::supports_rev()) { 1794 rev(r, r); 1795 } else { 1796 eor(rtmp1, r, AsmOperand(r, ror, 16)); 1797 mvn(rtmp2, 0x0000ff00); 1798 andr(rtmp1, rtmp2, AsmOperand(rtmp1, lsr, 8)); 1799 eor(r, rtmp1, AsmOperand(r, ror, 8)); 1800 } 1801 #endif // AARCH64 1802 } 1803 1804 1805 void InterpreterMacroAssembler::inc_global_counter(address address_of_counter, int offset, Register tmp1, Register tmp2, bool avoid_overflow) { 1806 const intx addr = (intx) (address_of_counter + offset); 1807 1808 assert ((addr & 0x3) == 0, "address of counter should be aligned"); 1809 const intx offset_mask = right_n_bits(AARCH64_ONLY(12 + 2) NOT_AARCH64(12)); 1810 1811 const address base = (address) (addr & ~offset_mask); 1812 const int offs = (int) (addr & offset_mask); 1813 1814 const Register addr_base = tmp1; 1815 const Register val = tmp2; 1816 1817 mov_slow(addr_base, base); 1818 ldr_s32(val, Address(addr_base, offs)); 1819 1820 if (avoid_overflow) { 1821 adds_32(val, val, 1); 1822 #ifdef AARCH64 1823 Label L; 1824 b(L, mi); 1825 str_32(val, Address(addr_base, offs)); 1826 bind(L); 1827 #else 1828 str(val, Address(addr_base, offs), pl); 1829 #endif // AARCH64 1830 } else { 1831 add_32(val, val, 1); 1832 str_32(val, Address(addr_base, offs)); 1833 } 1834 } 1835 1836 void InterpreterMacroAssembler::interp_verify_oop(Register reg, TosState state, const char *file, int line) { 1837 if (state == atos) { MacroAssembler::_verify_oop(reg, "broken oop", file, line); } 1838 } 1839 1840 // Inline assembly for: 1841 // 1842 // if (thread is in interp_only_mode) { 1843 // InterpreterRuntime::post_method_entry(); 1844 // } 1845 // if (DTraceMethodProbes) { 1846 // SharedRuntime::dtrace_method_entry(method, receiver); 1847 // } 1848 // if (RC_TRACE_IN_RANGE(0x00001000, 0x00002000)) { 1849 // SharedRuntime::rc_trace_method_entry(method, receiver); 1850 // } 1851 1852 void InterpreterMacroAssembler::notify_method_entry() { 1853 // Whenever JVMTI is interp_only_mode, method entry/exit events are sent to 1854 // track stack depth. If it is possible to enter interp_only_mode we add 1855 // the code to check if the event should be sent. 1856 if (can_post_interpreter_events()) { 1857 Label L; 1858 1859 ldr_s32(Rtemp, Address(Rthread, JavaThread::interp_only_mode_offset())); 1860 cbz(Rtemp, L); 1861 1862 call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::post_method_entry)); 1863 1864 bind(L); 1865 } 1866 1867 // Note: Disable DTrace runtime check for now to eliminate overhead on each method entry 1868 if (DTraceMethodProbes) { 1869 Label Lcontinue; 1870 1871 ldrb_global(Rtemp, (address)&DTraceMethodProbes); 1872 cbz(Rtemp, Lcontinue); 1873 1874 mov(R0, Rthread); 1875 mov(R1, Rmethod); 1876 call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_method_entry), R0, R1); 1877 1878 bind(Lcontinue); 1879 } 1880 // RedefineClasses() tracing support for obsolete method entry 1881 if (log_is_enabled(Trace, redefine, class, obsolete)) { 1882 mov(R0, Rthread); 1883 mov(R1, Rmethod); 1884 call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::rc_trace_method_entry), 1885 R0, R1); 1886 } 1887 } 1888 1889 1890 void InterpreterMacroAssembler::notify_method_exit( 1891 TosState state, NotifyMethodExitMode mode, 1892 bool native, Register result_lo, Register result_hi, FloatRegister result_fp) { 1893 // Whenever JVMTI is interp_only_mode, method entry/exit events are sent to 1894 // track stack depth. If it is possible to enter interp_only_mode we add 1895 // the code to check if the event should be sent. 1896 if (mode == NotifyJVMTI && can_post_interpreter_events()) { 1897 Label L; 1898 // Note: frame::interpreter_frame_result has a dependency on how the 1899 // method result is saved across the call to post_method_exit. If this 1900 // is changed then the interpreter_frame_result implementation will 1901 // need to be updated too. 1902 1903 ldr_s32(Rtemp, Address(Rthread, JavaThread::interp_only_mode_offset())); 1904 cbz(Rtemp, L); 1905 1906 if (native) { 1907 // For c++ and template interpreter push both result registers on the 1908 // stack in native, we don't know the state. 1909 // On AArch64 result registers are stored into the frame at known locations. 1910 // See frame::interpreter_frame_result for code that gets the result values from here. 1911 assert(result_lo != noreg, "result registers should be defined"); 1912 1913 #ifdef AARCH64 1914 assert(result_hi == noreg, "result_hi is not used on AArch64"); 1915 assert(result_fp != fnoreg, "FP result register must be defined"); 1916 1917 str_d(result_fp, Address(FP, frame::interpreter_frame_fp_saved_result_offset * wordSize)); 1918 str(result_lo, Address(FP, frame::interpreter_frame_gp_saved_result_offset * wordSize)); 1919 #else 1920 assert(result_hi != noreg, "result registers should be defined"); 1921 1922 #ifdef __ABI_HARD__ 1923 assert(result_fp != fnoreg, "FP result register must be defined"); 1924 sub(SP, SP, 2 * wordSize); 1925 fstd(result_fp, Address(SP)); 1926 #endif // __ABI_HARD__ 1927 1928 push(RegisterSet(result_lo) | RegisterSet(result_hi)); 1929 #endif // AARCH64 1930 1931 call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::post_method_exit)); 1932 1933 #ifdef AARCH64 1934 ldr_d(result_fp, Address(FP, frame::interpreter_frame_fp_saved_result_offset * wordSize)); 1935 ldr(result_lo, Address(FP, frame::interpreter_frame_gp_saved_result_offset * wordSize)); 1936 #else 1937 pop(RegisterSet(result_lo) | RegisterSet(result_hi)); 1938 #ifdef __ABI_HARD__ 1939 fldd(result_fp, Address(SP)); 1940 add(SP, SP, 2 * wordSize); 1941 #endif // __ABI_HARD__ 1942 #endif // AARCH64 1943 1944 } else { 1945 // For the template interpreter, the value on tos is the size of the 1946 // state. (c++ interpreter calls jvmti somewhere else). 1947 push(state); 1948 call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::post_method_exit)); 1949 pop(state); 1950 } 1951 1952 bind(L); 1953 } 1954 1955 // Note: Disable DTrace runtime check for now to eliminate overhead on each method exit 1956 if (DTraceMethodProbes) { 1957 Label Lcontinue; 1958 1959 ldrb_global(Rtemp, (address)&DTraceMethodProbes); 1960 cbz(Rtemp, Lcontinue); 1961 1962 push(state); 1963 1964 mov(R0, Rthread); 1965 mov(R1, Rmethod); 1966 1967 call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_method_exit), R0, R1); 1968 1969 pop(state); 1970 1971 bind(Lcontinue); 1972 } 1973 } 1974 1975 1976 #ifndef PRODUCT 1977 1978 void InterpreterMacroAssembler::trace_state(const char* msg) { 1979 int push_size = save_caller_save_registers(); 1980 1981 Label Lcontinue; 1982 InlinedString Lmsg0("%s: FP=" INTPTR_FORMAT ", SP=" INTPTR_FORMAT "\n"); 1983 InlinedString Lmsg(msg); 1984 InlinedAddress Lprintf((address)printf); 1985 1986 ldr_literal(R0, Lmsg0); 1987 ldr_literal(R1, Lmsg); 1988 mov(R2, FP); 1989 add(R3, SP, push_size); // original SP (without saved registers) 1990 ldr_literal(Rtemp, Lprintf); 1991 call(Rtemp); 1992 1993 b(Lcontinue); 1994 1995 bind_literal(Lmsg0); 1996 bind_literal(Lmsg); 1997 bind_literal(Lprintf); 1998 1999 2000 bind(Lcontinue); 2001 2002 restore_caller_save_registers(); 2003 } 2004 2005 #endif 2006 2007 // Jump if ((*counter_addr += increment) & mask) satisfies the condition. 2008 void InterpreterMacroAssembler::increment_mask_and_jump(Address counter_addr, 2009 int increment, Address mask_addr, 2010 Register scratch, Register scratch2, 2011 AsmCondition cond, Label* where) { 2012 // caution: scratch2 and base address of counter_addr can be the same 2013 assert_different_registers(scratch, scratch2); 2014 ldr_u32(scratch, counter_addr); 2015 add(scratch, scratch, increment); 2016 str_32(scratch, counter_addr); 2017 2018 #ifdef AARCH64 2019 ldr_u32(scratch2, mask_addr); 2020 ands_w(ZR, scratch, scratch2); 2021 #else 2022 ldr(scratch2, mask_addr); 2023 andrs(scratch, scratch, scratch2); 2024 #endif // AARCH64 2025 b(*where, cond); 2026 } 2027 2028 void InterpreterMacroAssembler::get_method_counters(Register method, 2029 Register Rcounters, 2030 Label& skip, 2031 bool saveRegs, 2032 Register reg1, 2033 Register reg2, 2034 Register reg3) { 2035 const Address method_counters(method, Method::method_counters_offset()); 2036 Label has_counters; 2037 2038 ldr(Rcounters, method_counters); 2039 cbnz(Rcounters, has_counters); 2040 2041 if (saveRegs) { 2042 // Save and restore in use caller-saved registers since they will be trashed by call_VM 2043 assert(reg1 != noreg, "must specify reg1"); 2044 assert(reg2 != noreg, "must specify reg2"); 2045 #ifdef AARCH64 2046 assert(reg3 != noreg, "must specify reg3"); 2047 stp(reg1, reg2, Address(Rstack_top, -2*wordSize, pre_indexed)); 2048 stp(reg3, ZR, Address(Rstack_top, -2*wordSize, pre_indexed)); 2049 #else 2050 assert(reg3 == noreg, "must not specify reg3"); 2051 push(RegisterSet(reg1) | RegisterSet(reg2)); 2052 #endif 2053 } 2054 2055 mov(R1, method); 2056 call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::build_method_counters), R1); 2057 2058 if (saveRegs) { 2059 #ifdef AARCH64 2060 ldp(reg3, ZR, Address(Rstack_top, 2*wordSize, post_indexed)); 2061 ldp(reg1, reg2, Address(Rstack_top, 2*wordSize, post_indexed)); 2062 #else 2063 pop(RegisterSet(reg1) | RegisterSet(reg2)); 2064 #endif 2065 } 2066 2067 ldr(Rcounters, method_counters); 2068 cbz(Rcounters, skip); // No MethodCounters created, OutOfMemory 2069 2070 bind(has_counters); 2071 }