1 /*
   2  * Copyright (c) 1997, 2012, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  *
  23  */
  24 
  25 #include "precompiled.hpp"
  26 #include "interp_masm_sparc.hpp"
  27 #include "interpreter/interpreter.hpp"
  28 #include "interpreter/interpreterRuntime.hpp"
  29 #include "oops/arrayOop.hpp"
  30 #include "oops/markOop.hpp"
  31 #include "oops/methodDataOop.hpp"
  32 #include "oops/methodOop.hpp"
  33 #include "prims/jvmtiExport.hpp"
  34 #include "prims/jvmtiRedefineClassesTrace.hpp"
  35 #include "prims/jvmtiThreadState.hpp"
  36 #include "runtime/basicLock.hpp"
  37 #include "runtime/biasedLocking.hpp"
  38 #include "runtime/sharedRuntime.hpp"
  39 #ifdef TARGET_OS_FAMILY_linux
  40 # include "thread_linux.inline.hpp"
  41 #endif
  42 #ifdef TARGET_OS_FAMILY_solaris
  43 # include "thread_solaris.inline.hpp"
  44 #endif
  45 
  46 #ifndef CC_INTERP
  47 #ifndef FAST_DISPATCH
  48 #define FAST_DISPATCH 1
  49 #endif
  50 #undef FAST_DISPATCH
  51 
  52 // Implementation of InterpreterMacroAssembler
  53 
  54 // This file specializes the assember with interpreter-specific macros
  55 
  56 const Address InterpreterMacroAssembler::l_tmp(FP, (frame::interpreter_frame_l_scratch_fp_offset * wordSize) + STACK_BIAS);
  57 const Address InterpreterMacroAssembler::d_tmp(FP, (frame::interpreter_frame_d_scratch_fp_offset * wordSize) + STACK_BIAS);
  58 
  59 #else // CC_INTERP
  60 #ifndef STATE
  61 #define STATE(field_name) Lstate, in_bytes(byte_offset_of(BytecodeInterpreter, field_name))
  62 #endif // STATE
  63 
  64 #endif // CC_INTERP
  65 
  66 void InterpreterMacroAssembler::compute_extra_locals_size_in_bytes(Register args_size, Register locals_size, Register delta) {
  67   // Note: this algorithm is also used by C1's OSR entry sequence.
  68   // Any changes should also be applied to CodeEmitter::emit_osr_entry().
  69   assert_different_registers(args_size, locals_size);
  70   // max_locals*2 for TAGS.  Assumes that args_size has already been adjusted.
  71   subcc(locals_size, args_size, delta);// extra space for non-arguments locals in words
  72   // Use br/mov combination because it works on both V8 and V9 and is
  73   // faster.
  74   Label skip_move;
  75   br(Assembler::negative, true, Assembler::pt, skip_move);
  76   delayed()->mov(G0, delta);
  77   bind(skip_move);
  78   round_to(delta, WordsPerLong);       // make multiple of 2 (SP must be 2-word aligned)
  79   sll(delta, LogBytesPerWord, delta);  // extra space for locals in bytes
  80 }
  81 
  82 #ifndef CC_INTERP
  83 
  84 // Dispatch code executed in the prolog of a bytecode which does not do it's
  85 // own dispatch. The dispatch address is computed and placed in IdispatchAddress
  86 void InterpreterMacroAssembler::dispatch_prolog(TosState state, int bcp_incr) {
  87   assert_not_delayed();
  88 #ifdef FAST_DISPATCH
  89   // FAST_DISPATCH and ProfileInterpreter are mutually exclusive since
  90   // they both use I2.
  91   assert(!ProfileInterpreter, "FAST_DISPATCH and +ProfileInterpreter are mutually exclusive");
  92   ldub(Lbcp, bcp_incr, Lbyte_code);                     // load next bytecode
  93   add(Lbyte_code, Interpreter::distance_from_dispatch_table(state), Lbyte_code);
  94                                                         // add offset to correct dispatch table
  95   sll(Lbyte_code, LogBytesPerWord, Lbyte_code);         // multiply by wordSize
  96   ld_ptr(IdispatchTables, Lbyte_code, IdispatchAddress);// get entry addr
  97 #else
  98   ldub( Lbcp, bcp_incr, Lbyte_code);                    // load next bytecode
  99   // dispatch table to use
 100   AddressLiteral tbl(Interpreter::dispatch_table(state));
 101   sll(Lbyte_code, LogBytesPerWord, Lbyte_code);         // multiply by wordSize
 102   set(tbl, G3_scratch);                                 // compute addr of table
 103   ld_ptr(G3_scratch, Lbyte_code, IdispatchAddress);     // get entry addr
 104 #endif
 105 }
 106 
 107 
 108 // Dispatch code executed in the epilog of a bytecode which does not do it's
 109 // own dispatch. The dispatch address in IdispatchAddress is used for the
 110 // dispatch.
 111 void InterpreterMacroAssembler::dispatch_epilog(TosState state, int bcp_incr) {
 112   assert_not_delayed();
 113   verify_FPU(1, state);
 114   interp_verify_oop(Otos_i, state, __FILE__, __LINE__);
 115   jmp( IdispatchAddress, 0 );
 116   if (bcp_incr != 0)  delayed()->inc(Lbcp, bcp_incr);
 117   else                delayed()->nop();
 118 }
 119 
 120 
 121 void InterpreterMacroAssembler::dispatch_next(TosState state, int bcp_incr) {
 122   // %%%% consider branching to a single shared dispatch stub (for each bcp_incr)
 123   assert_not_delayed();
 124   ldub( Lbcp, bcp_incr, Lbyte_code);               // load next bytecode
 125   dispatch_Lbyte_code(state, Interpreter::dispatch_table(state), bcp_incr);
 126 }
 127 
 128 
 129 void InterpreterMacroAssembler::dispatch_next_noverify_oop(TosState state, int bcp_incr) {
 130   // %%%% consider branching to a single shared dispatch stub (for each bcp_incr)
 131   assert_not_delayed();
 132   ldub( Lbcp, bcp_incr, Lbyte_code);               // load next bytecode
 133   dispatch_Lbyte_code(state, Interpreter::dispatch_table(state), bcp_incr, false);
 134 }
 135 
 136 
 137 void InterpreterMacroAssembler::dispatch_via(TosState state, address* table) {
 138   // load current bytecode
 139   assert_not_delayed();
 140   ldub( Lbcp, 0, Lbyte_code);               // load next bytecode
 141   dispatch_base(state, table);
 142 }
 143 
 144 
 145 void InterpreterMacroAssembler::call_VM_leaf_base(
 146   Register java_thread,
 147   address  entry_point,
 148   int      number_of_arguments
 149 ) {
 150   if (!java_thread->is_valid())
 151     java_thread = L7_thread_cache;
 152   // super call
 153   MacroAssembler::call_VM_leaf_base(java_thread, entry_point, number_of_arguments);
 154 }
 155 
 156 
 157 void InterpreterMacroAssembler::call_VM_base(
 158   Register        oop_result,
 159   Register        java_thread,
 160   Register        last_java_sp,
 161   address         entry_point,
 162   int             number_of_arguments,
 163   bool            check_exception
 164 ) {
 165   if (!java_thread->is_valid())
 166     java_thread = L7_thread_cache;
 167   // See class ThreadInVMfromInterpreter, which assumes that the interpreter
 168   // takes responsibility for setting its own thread-state on call-out.
 169   // However, ThreadInVMfromInterpreter resets the state to "in_Java".
 170 
 171   //save_bcp();                                  // save bcp
 172   MacroAssembler::call_VM_base(oop_result, java_thread, last_java_sp, entry_point, number_of_arguments, check_exception);
 173   //restore_bcp();                               // restore bcp
 174   //restore_locals();                            // restore locals pointer
 175 }
 176 
 177 
 178 void InterpreterMacroAssembler::check_and_handle_popframe(Register scratch_reg) {
 179   if (JvmtiExport::can_pop_frame()) {
 180     Label L;
 181 
 182     // Check the "pending popframe condition" flag in the current thread
 183     ld(G2_thread, JavaThread::popframe_condition_offset(), scratch_reg);
 184 
 185     // Initiate popframe handling only if it is not already being processed.  If the flag
 186     // has the popframe_processing bit set, it means that this code is called *during* popframe
 187     // handling - we don't want to reenter.
 188     btst(JavaThread::popframe_pending_bit, scratch_reg);
 189     br(zero, false, pt, L);
 190     delayed()->nop();
 191     btst(JavaThread::popframe_processing_bit, scratch_reg);
 192     br(notZero, false, pt, L);
 193     delayed()->nop();
 194 
 195     // Call Interpreter::remove_activation_preserving_args_entry() to get the
 196     // address of the same-named entrypoint in the generated interpreter code.
 197     call_VM_leaf(noreg, CAST_FROM_FN_PTR(address, Interpreter::remove_activation_preserving_args_entry));
 198 
 199     // Jump to Interpreter::_remove_activation_preserving_args_entry
 200     jmpl(O0, G0, G0);
 201     delayed()->nop();
 202     bind(L);
 203   }
 204 }
 205 
 206 
 207 void InterpreterMacroAssembler::load_earlyret_value(TosState state) {
 208   Register thr_state = G4_scratch;
 209   ld_ptr(G2_thread, JavaThread::jvmti_thread_state_offset(), thr_state);
 210   const Address tos_addr(thr_state, JvmtiThreadState::earlyret_tos_offset());
 211   const Address oop_addr(thr_state, JvmtiThreadState::earlyret_oop_offset());
 212   const Address val_addr(thr_state, JvmtiThreadState::earlyret_value_offset());
 213   switch (state) {
 214   case ltos: ld_long(val_addr, Otos_l);                   break;
 215   case atos: ld_ptr(oop_addr, Otos_l);
 216              st_ptr(G0, oop_addr);                        break;
 217   case btos:                                           // fall through
 218   case ctos:                                           // fall through
 219   case stos:                                           // fall through
 220   case itos: ld(val_addr, Otos_l1);                       break;
 221   case ftos: ldf(FloatRegisterImpl::S, val_addr, Ftos_f); break;
 222   case dtos: ldf(FloatRegisterImpl::D, val_addr, Ftos_d); break;
 223   case vtos: /* nothing to do */                          break;
 224   default  : ShouldNotReachHere();
 225   }
 226   // Clean up tos value in the jvmti thread state
 227   or3(G0, ilgl, G3_scratch);
 228   stw(G3_scratch, tos_addr);
 229   st_long(G0, val_addr);
 230   interp_verify_oop(Otos_i, state, __FILE__, __LINE__);
 231 }
 232 
 233 
 234 void InterpreterMacroAssembler::check_and_handle_earlyret(Register scratch_reg) {
 235   if (JvmtiExport::can_force_early_return()) {
 236     Label L;
 237     Register thr_state = G3_scratch;
 238     ld_ptr(G2_thread, JavaThread::jvmti_thread_state_offset(), thr_state);
 239     br_null_short(thr_state, pt, L); // if (thread->jvmti_thread_state() == NULL) exit;
 240 
 241     // Initiate earlyret handling only if it is not already being processed.
 242     // If the flag has the earlyret_processing bit set, it means that this code
 243     // is called *during* earlyret handling - we don't want to reenter.
 244     ld(thr_state, JvmtiThreadState::earlyret_state_offset(), G4_scratch);
 245     cmp_and_br_short(G4_scratch, JvmtiThreadState::earlyret_pending, Assembler::notEqual, pt, L);
 246 
 247     // Call Interpreter::remove_activation_early_entry() to get the address of the
 248     // same-named entrypoint in the generated interpreter code
 249     ld(thr_state, JvmtiThreadState::earlyret_tos_offset(), Otos_l1);
 250     call_VM_leaf(noreg, CAST_FROM_FN_PTR(address, Interpreter::remove_activation_early_entry), Otos_l1);
 251 
 252     // Jump to Interpreter::_remove_activation_early_entry
 253     jmpl(O0, G0, G0);
 254     delayed()->nop();
 255     bind(L);
 256   }
 257 }
 258 
 259 
 260 void InterpreterMacroAssembler::super_call_VM_leaf(Register thread_cache, address entry_point, Register arg_1, Register arg_2) {
 261   mov(arg_1, O0);
 262   mov(arg_2, O1);
 263   MacroAssembler::call_VM_leaf_base(thread_cache, entry_point, 2);
 264 }
 265 #endif /* CC_INTERP */
 266 
 267 
 268 #ifndef CC_INTERP
 269 
 270 void InterpreterMacroAssembler::dispatch_base(TosState state, address* table) {
 271   assert_not_delayed();
 272   dispatch_Lbyte_code(state, table);
 273 }
 274 
 275 
 276 void InterpreterMacroAssembler::dispatch_normal(TosState state) {
 277   dispatch_base(state, Interpreter::normal_table(state));
 278 }
 279 
 280 
 281 void InterpreterMacroAssembler::dispatch_only(TosState state) {
 282   dispatch_base(state, Interpreter::dispatch_table(state));
 283 }
 284 
 285 
 286 // common code to dispatch and dispatch_only
 287 // dispatch value in Lbyte_code and increment Lbcp
 288 
 289 void InterpreterMacroAssembler::dispatch_Lbyte_code(TosState state, address* table, int bcp_incr, bool verify) {
 290   verify_FPU(1, state);
 291   // %%%%% maybe implement +VerifyActivationFrameSize here
 292   //verify_thread(); //too slow; we will just verify on method entry & exit
 293   if (verify) interp_verify_oop(Otos_i, state, __FILE__, __LINE__);
 294 #ifdef FAST_DISPATCH
 295   if (table == Interpreter::dispatch_table(state)) {
 296     // use IdispatchTables
 297     add(Lbyte_code, Interpreter::distance_from_dispatch_table(state), Lbyte_code);
 298                                                         // add offset to correct dispatch table
 299     sll(Lbyte_code, LogBytesPerWord, Lbyte_code);       // multiply by wordSize
 300     ld_ptr(IdispatchTables, Lbyte_code, G3_scratch);    // get entry addr
 301   } else {
 302 #endif
 303     // dispatch table to use
 304     AddressLiteral tbl(table);
 305     sll(Lbyte_code, LogBytesPerWord, Lbyte_code);       // multiply by wordSize
 306     set(tbl, G3_scratch);                               // compute addr of table
 307     ld_ptr(G3_scratch, Lbyte_code, G3_scratch);         // get entry addr
 308 #ifdef FAST_DISPATCH
 309   }
 310 #endif
 311   jmp( G3_scratch, 0 );
 312   if (bcp_incr != 0)  delayed()->inc(Lbcp, bcp_incr);
 313   else                delayed()->nop();
 314 }
 315 
 316 
 317 // Helpers for expression stack
 318 
 319 // Longs and doubles are Category 2 computational types in the
 320 // JVM specification (section 3.11.1) and take 2 expression stack or
 321 // local slots.
 322 // Aligning them on 32 bit with tagged stacks is hard because the code generated
 323 // for the dup* bytecodes depends on what types are already on the stack.
 324 // If the types are split into the two stack/local slots, that is much easier
 325 // (and we can use 0 for non-reference tags).
 326 
 327 // Known good alignment in _LP64 but unknown otherwise
 328 void InterpreterMacroAssembler::load_unaligned_double(Register r1, int offset, FloatRegister d) {
 329   assert_not_delayed();
 330 
 331 #ifdef _LP64
 332   ldf(FloatRegisterImpl::D, r1, offset, d);
 333 #else
 334   ldf(FloatRegisterImpl::S, r1, offset, d);
 335   ldf(FloatRegisterImpl::S, r1, offset + Interpreter::stackElementSize, d->successor());
 336 #endif
 337 }
 338 
 339 // Known good alignment in _LP64 but unknown otherwise
 340 void InterpreterMacroAssembler::store_unaligned_double(FloatRegister d, Register r1, int offset) {
 341   assert_not_delayed();
 342 
 343 #ifdef _LP64
 344   stf(FloatRegisterImpl::D, d, r1, offset);
 345   // store something more useful here
 346   debug_only(stx(G0, r1, offset+Interpreter::stackElementSize);)
 347 #else
 348   stf(FloatRegisterImpl::S, d, r1, offset);
 349   stf(FloatRegisterImpl::S, d->successor(), r1, offset + Interpreter::stackElementSize);
 350 #endif
 351 }
 352 
 353 
 354 // Known good alignment in _LP64 but unknown otherwise
 355 void InterpreterMacroAssembler::load_unaligned_long(Register r1, int offset, Register rd) {
 356   assert_not_delayed();
 357 #ifdef _LP64
 358   ldx(r1, offset, rd);
 359 #else
 360   ld(r1, offset, rd);
 361   ld(r1, offset + Interpreter::stackElementSize, rd->successor());
 362 #endif
 363 }
 364 
 365 // Known good alignment in _LP64 but unknown otherwise
 366 void InterpreterMacroAssembler::store_unaligned_long(Register l, Register r1, int offset) {
 367   assert_not_delayed();
 368 
 369 #ifdef _LP64
 370   stx(l, r1, offset);
 371   // store something more useful here
 372   debug_only(stx(G0, r1, offset+Interpreter::stackElementSize);)
 373 #else
 374   st(l, r1, offset);
 375   st(l->successor(), r1, offset + Interpreter::stackElementSize);
 376 #endif
 377 }
 378 
 379 void InterpreterMacroAssembler::pop_i(Register r) {
 380   assert_not_delayed();
 381   ld(Lesp, Interpreter::expr_offset_in_bytes(0), r);
 382   inc(Lesp, Interpreter::stackElementSize);
 383   debug_only(verify_esp(Lesp));
 384 }
 385 
 386 void InterpreterMacroAssembler::pop_ptr(Register r, Register scratch) {
 387   assert_not_delayed();
 388   ld_ptr(Lesp, Interpreter::expr_offset_in_bytes(0), r);
 389   inc(Lesp, Interpreter::stackElementSize);
 390   debug_only(verify_esp(Lesp));
 391 }
 392 
 393 void InterpreterMacroAssembler::pop_l(Register r) {
 394   assert_not_delayed();
 395   load_unaligned_long(Lesp, Interpreter::expr_offset_in_bytes(0), r);
 396   inc(Lesp, 2*Interpreter::stackElementSize);
 397   debug_only(verify_esp(Lesp));
 398 }
 399 
 400 
 401 void InterpreterMacroAssembler::pop_f(FloatRegister f, Register scratch) {
 402   assert_not_delayed();
 403   ldf(FloatRegisterImpl::S, Lesp, Interpreter::expr_offset_in_bytes(0), f);
 404   inc(Lesp, Interpreter::stackElementSize);
 405   debug_only(verify_esp(Lesp));
 406 }
 407 
 408 
 409 void InterpreterMacroAssembler::pop_d(FloatRegister f, Register scratch) {
 410   assert_not_delayed();
 411   load_unaligned_double(Lesp, Interpreter::expr_offset_in_bytes(0), f);
 412   inc(Lesp, 2*Interpreter::stackElementSize);
 413   debug_only(verify_esp(Lesp));
 414 }
 415 
 416 
 417 void InterpreterMacroAssembler::push_i(Register r) {
 418   assert_not_delayed();
 419   debug_only(verify_esp(Lesp));
 420   st(r, Lesp, 0);
 421   dec(Lesp, Interpreter::stackElementSize);
 422 }
 423 
 424 void InterpreterMacroAssembler::push_ptr(Register r) {
 425   assert_not_delayed();
 426   st_ptr(r, Lesp, 0);
 427   dec(Lesp, Interpreter::stackElementSize);
 428 }
 429 
 430 // remember: our convention for longs in SPARC is:
 431 // O0 (Otos_l1) has high-order part in first word,
 432 // O1 (Otos_l2) has low-order part in second word
 433 
 434 void InterpreterMacroAssembler::push_l(Register r) {
 435   assert_not_delayed();
 436   debug_only(verify_esp(Lesp));
 437   // Longs are stored in memory-correct order, even if unaligned.
 438   int offset = -Interpreter::stackElementSize;
 439   store_unaligned_long(r, Lesp, offset);
 440   dec(Lesp, 2 * Interpreter::stackElementSize);
 441 }
 442 
 443 
 444 void InterpreterMacroAssembler::push_f(FloatRegister f) {
 445   assert_not_delayed();
 446   debug_only(verify_esp(Lesp));
 447   stf(FloatRegisterImpl::S, f, Lesp, 0);
 448   dec(Lesp, Interpreter::stackElementSize);
 449 }
 450 
 451 
 452 void InterpreterMacroAssembler::push_d(FloatRegister d)   {
 453   assert_not_delayed();
 454   debug_only(verify_esp(Lesp));
 455   // Longs are stored in memory-correct order, even if unaligned.
 456   int offset = -Interpreter::stackElementSize;
 457   store_unaligned_double(d, Lesp, offset);
 458   dec(Lesp, 2 * Interpreter::stackElementSize);
 459 }
 460 
 461 
 462 void InterpreterMacroAssembler::push(TosState state) {
 463   interp_verify_oop(Otos_i, state, __FILE__, __LINE__);
 464   switch (state) {
 465     case atos: push_ptr();            break;
 466     case btos: push_i();              break;
 467     case ctos:
 468     case stos: push_i();              break;
 469     case itos: push_i();              break;
 470     case ltos: push_l();              break;
 471     case ftos: push_f();              break;
 472     case dtos: push_d();              break;
 473     case vtos: /* nothing to do */    break;
 474     default  : ShouldNotReachHere();
 475   }
 476 }
 477 
 478 
 479 void InterpreterMacroAssembler::pop(TosState state) {
 480   switch (state) {
 481     case atos: pop_ptr();            break;
 482     case btos: pop_i();              break;
 483     case ctos:
 484     case stos: pop_i();              break;
 485     case itos: pop_i();              break;
 486     case ltos: pop_l();              break;
 487     case ftos: pop_f();              break;
 488     case dtos: pop_d();              break;
 489     case vtos: /* nothing to do */   break;
 490     default  : ShouldNotReachHere();
 491   }
 492   interp_verify_oop(Otos_i, state, __FILE__, __LINE__);
 493 }
 494 
 495 
 496 // Helpers for swap and dup
 497 void InterpreterMacroAssembler::load_ptr(int n, Register val) {
 498   ld_ptr(Lesp, Interpreter::expr_offset_in_bytes(n), val);
 499 }
 500 void InterpreterMacroAssembler::store_ptr(int n, Register val) {
 501   st_ptr(val, Lesp, Interpreter::expr_offset_in_bytes(n));
 502 }
 503 
 504 
 505 void InterpreterMacroAssembler::load_receiver(Register param_count,
 506                                               Register recv) {
 507   sll(param_count, Interpreter::logStackElementSize, param_count);
 508   ld_ptr(Lesp, param_count, recv);                      // gets receiver Oop
 509 }
 510 
 511 void InterpreterMacroAssembler::empty_expression_stack() {
 512   // Reset Lesp.
 513   sub( Lmonitors, wordSize, Lesp );
 514 
 515   // Reset SP by subtracting more space from Lesp.
 516   Label done;
 517   verify_oop(Lmethod);
 518   assert(G4_scratch != Gframe_size, "Only you can prevent register aliasing!");
 519 
 520   // A native does not need to do this, since its callee does not change SP.
 521   ld(Lmethod, methodOopDesc::access_flags_offset(), Gframe_size);  // Load access flags.
 522   btst(JVM_ACC_NATIVE, Gframe_size);
 523   br(Assembler::notZero, false, Assembler::pt, done);
 524   delayed()->nop();
 525 
 526   // Compute max expression stack+register save area
 527   lduh(Lmethod, in_bytes(methodOopDesc::max_stack_offset()), Gframe_size);  // Load max stack.
 528   add( Gframe_size, frame::memory_parameter_word_sp_offset, Gframe_size );
 529 
 530   //
 531   // now set up a stack frame with the size computed above
 532   //
 533   //round_to( Gframe_size, WordsPerLong ); // -- moved down to the "and" below
 534   sll( Gframe_size, LogBytesPerWord, Gframe_size );
 535   sub( Lesp, Gframe_size, Gframe_size );
 536   and3( Gframe_size, -(2 * wordSize), Gframe_size );          // align SP (downwards) to an 8/16-byte boundary
 537   debug_only(verify_sp(Gframe_size, G4_scratch));
 538 #ifdef _LP64
 539   sub(Gframe_size, STACK_BIAS, Gframe_size );
 540 #endif
 541   mov(Gframe_size, SP);
 542 
 543   bind(done);
 544 }
 545 
 546 
 547 #ifdef ASSERT
 548 void InterpreterMacroAssembler::verify_sp(Register Rsp, Register Rtemp) {
 549   Label Bad, OK;
 550 
 551   // Saved SP must be aligned.
 552 #ifdef _LP64
 553   btst(2*BytesPerWord-1, Rsp);
 554 #else
 555   btst(LongAlignmentMask, Rsp);
 556 #endif
 557   br(Assembler::notZero, false, Assembler::pn, Bad);
 558   delayed()->nop();
 559 
 560   // Saved SP, plus register window size, must not be above FP.
 561   add(Rsp, frame::register_save_words * wordSize, Rtemp);
 562 #ifdef _LP64
 563   sub(Rtemp, STACK_BIAS, Rtemp);  // Bias Rtemp before cmp to FP
 564 #endif
 565   cmp_and_brx_short(Rtemp, FP, Assembler::greaterUnsigned, Assembler::pn, Bad);
 566 
 567   // Saved SP must not be ridiculously below current SP.
 568   size_t maxstack = MAX2(JavaThread::stack_size_at_create(), (size_t) 4*K*K);
 569   set(maxstack, Rtemp);
 570   sub(SP, Rtemp, Rtemp);
 571 #ifdef _LP64
 572   add(Rtemp, STACK_BIAS, Rtemp);  // Unbias Rtemp before cmp to Rsp
 573 #endif
 574   cmp_and_brx_short(Rsp, Rtemp, Assembler::lessUnsigned, Assembler::pn, Bad);
 575 
 576   ba_short(OK);
 577 
 578   bind(Bad);
 579   stop("on return to interpreted call, restored SP is corrupted");
 580 
 581   bind(OK);
 582 }
 583 
 584 
 585 void InterpreterMacroAssembler::verify_esp(Register Resp) {
 586   // about to read or write Resp[0]
 587   // make sure it is not in the monitors or the register save area
 588   Label OK1, OK2;
 589 
 590   cmp(Resp, Lmonitors);
 591   brx(Assembler::lessUnsigned, true, Assembler::pt, OK1);
 592   delayed()->sub(Resp, frame::memory_parameter_word_sp_offset * wordSize, Resp);
 593   stop("too many pops:  Lesp points into monitor area");
 594   bind(OK1);
 595 #ifdef _LP64
 596   sub(Resp, STACK_BIAS, Resp);
 597 #endif
 598   cmp(Resp, SP);
 599   brx(Assembler::greaterEqualUnsigned, false, Assembler::pt, OK2);
 600   delayed()->add(Resp, STACK_BIAS + frame::memory_parameter_word_sp_offset * wordSize, Resp);
 601   stop("too many pushes:  Lesp points into register window");
 602   bind(OK2);
 603 }
 604 #endif // ASSERT
 605 
 606 // Load compiled (i2c) or interpreter entry when calling from interpreted and
 607 // do the call. Centralized so that all interpreter calls will do the same actions.
 608 // If jvmti single stepping is on for a thread we must not call compiled code.
 609 void InterpreterMacroAssembler::call_from_interpreter(Register target, Register scratch, Register Rret) {
 610 
 611   // Assume we want to go compiled if available
 612 
 613   ld_ptr(G5_method, in_bytes(methodOopDesc::from_interpreted_offset()), target);
 614 
 615   if (JvmtiExport::can_post_interpreter_events()) {
 616     // JVMTI events, such as single-stepping, are implemented partly by avoiding running
 617     // compiled code in threads for which the event is enabled.  Check here for
 618     // interp_only_mode if these events CAN be enabled.
 619     verify_thread();
 620     Label skip_compiled_code;
 621 
 622     const Address interp_only(G2_thread, JavaThread::interp_only_mode_offset());
 623     ld(interp_only, scratch);
 624     cmp_zero_and_br(Assembler::notZero, scratch, skip_compiled_code, true, Assembler::pn);
 625     delayed()->ld_ptr(G5_method, in_bytes(methodOopDesc::interpreter_entry_offset()), target);
 626     bind(skip_compiled_code);
 627   }
 628 
 629   // the i2c_adapters need methodOop in G5_method (right? %%%)
 630   // do the call
 631 #ifdef ASSERT
 632   {
 633     Label ok;
 634     br_notnull_short(target, Assembler::pt, ok);
 635     stop("null entry point");
 636     bind(ok);
 637   }
 638 #endif // ASSERT
 639 
 640   // Adjust Rret first so Llast_SP can be same as Rret
 641   add(Rret, -frame::pc_return_offset, O7);
 642   add(Lesp, BytesPerWord, Gargs); // setup parameter pointer
 643   // Record SP so we can remove any stack space allocated by adapter transition
 644   jmp(target, 0);
 645   delayed()->mov(SP, Llast_SP);
 646 }
 647 
 648 void InterpreterMacroAssembler::if_cmp(Condition cc, bool ptr_compare) {
 649   assert_not_delayed();
 650 
 651   Label not_taken;
 652   if (ptr_compare) brx(cc, false, Assembler::pn, not_taken);
 653   else             br (cc, false, Assembler::pn, not_taken);
 654   delayed()->nop();
 655 
 656   TemplateTable::branch(false,false);
 657 
 658   bind(not_taken);
 659 
 660   profile_not_taken_branch(G3_scratch);
 661 }
 662 
 663 
 664 void InterpreterMacroAssembler::get_2_byte_integer_at_bcp(
 665                                   int         bcp_offset,
 666                                   Register    Rtmp,
 667                                   Register    Rdst,
 668                                   signedOrNot is_signed,
 669                                   setCCOrNot  should_set_CC ) {
 670   assert(Rtmp != Rdst, "need separate temp register");
 671   assert_not_delayed();
 672   switch (is_signed) {
 673    default: ShouldNotReachHere();
 674 
 675    case   Signed:  ldsb( Lbcp, bcp_offset, Rdst  );  break; // high byte
 676    case Unsigned:  ldub( Lbcp, bcp_offset, Rdst  );  break; // high byte
 677   }
 678   ldub( Lbcp, bcp_offset + 1, Rtmp ); // low byte
 679   sll( Rdst, BitsPerByte, Rdst);
 680   switch (should_set_CC ) {
 681    default: ShouldNotReachHere();
 682 
 683    case      set_CC:  orcc( Rdst, Rtmp, Rdst ); break;
 684    case dont_set_CC:  or3(  Rdst, Rtmp, Rdst ); break;
 685   }
 686 }
 687 
 688 
 689 void InterpreterMacroAssembler::get_4_byte_integer_at_bcp(
 690                                   int        bcp_offset,
 691                                   Register   Rtmp,
 692                                   Register   Rdst,
 693                                   setCCOrNot should_set_CC ) {
 694   assert(Rtmp != Rdst, "need separate temp register");
 695   assert_not_delayed();
 696   add( Lbcp, bcp_offset, Rtmp);
 697   andcc( Rtmp, 3, G0);
 698   Label aligned;
 699   switch (should_set_CC ) {
 700    default: ShouldNotReachHere();
 701 
 702    case      set_CC: break;
 703    case dont_set_CC: break;
 704   }
 705 
 706   br(Assembler::zero, true, Assembler::pn, aligned);
 707 #ifdef _LP64
 708   delayed()->ldsw(Rtmp, 0, Rdst);
 709 #else
 710   delayed()->ld(Rtmp, 0, Rdst);
 711 #endif
 712 
 713   ldub(Lbcp, bcp_offset + 3, Rdst);
 714   ldub(Lbcp, bcp_offset + 2, Rtmp);  sll(Rtmp,  8, Rtmp);  or3(Rtmp, Rdst, Rdst);
 715   ldub(Lbcp, bcp_offset + 1, Rtmp);  sll(Rtmp, 16, Rtmp);  or3(Rtmp, Rdst, Rdst);
 716 #ifdef _LP64
 717   ldsb(Lbcp, bcp_offset + 0, Rtmp);  sll(Rtmp, 24, Rtmp);
 718 #else
 719   // Unsigned load is faster than signed on some implementations
 720   ldub(Lbcp, bcp_offset + 0, Rtmp);  sll(Rtmp, 24, Rtmp);
 721 #endif
 722   or3(Rtmp, Rdst, Rdst );
 723 
 724   bind(aligned);
 725   if (should_set_CC == set_CC) tst(Rdst);
 726 }
 727 
 728 
 729 void InterpreterMacroAssembler::get_cache_index_at_bcp(Register cache, Register tmp,
 730                                                        int bcp_offset, size_t index_size) {
 731   assert(bcp_offset > 0, "bcp is still pointing to start of bytecode");
 732   if (index_size == sizeof(u2)) {
 733     get_2_byte_integer_at_bcp(bcp_offset, cache, tmp, Unsigned);
 734   } else if (index_size == sizeof(u4)) {
 735     assert(EnableInvokeDynamic, "giant index used only for JSR 292");
 736     get_4_byte_integer_at_bcp(bcp_offset, cache, tmp);
 737     assert(constantPoolCacheOopDesc::decode_secondary_index(~123) == 123, "else change next line");
 738     xor3(tmp, -1, tmp);  // convert to plain index
 739   } else if (index_size == sizeof(u1)) {
 740     assert(EnableInvokeDynamic, "tiny index used only for JSR 292");
 741     ldub(Lbcp, bcp_offset, tmp);
 742   } else {
 743     ShouldNotReachHere();
 744   }
 745 }
 746 
 747 
 748 void InterpreterMacroAssembler::get_cache_and_index_at_bcp(Register cache, Register tmp,
 749                                                            int bcp_offset, size_t index_size) {
 750   assert(bcp_offset > 0, "bcp is still pointing to start of bytecode");
 751   assert_different_registers(cache, tmp);
 752   assert_not_delayed();
 753   get_cache_index_at_bcp(cache, tmp, bcp_offset, index_size);
 754   // convert from field index to ConstantPoolCacheEntry index and from
 755   // word index to byte offset
 756   sll(tmp, exact_log2(in_words(ConstantPoolCacheEntry::size()) * BytesPerWord), tmp);
 757   add(LcpoolCache, tmp, cache);
 758 }
 759 
 760 
 761 void InterpreterMacroAssembler::get_cache_and_index_and_bytecode_at_bcp(Register cache,
 762                                                                         Register temp,
 763                                                                         Register bytecode,
 764                                                                         int byte_no,
 765                                                                         int bcp_offset,
 766                                                                         size_t index_size) {
 767   get_cache_and_index_at_bcp(cache, temp, bcp_offset, index_size);
 768   ld_ptr(cache, constantPoolCacheOopDesc::base_offset() + ConstantPoolCacheEntry::indices_offset(), bytecode);
 769   const int shift_count = (1 + byte_no) * BitsPerByte;
 770   srl( bytecode, shift_count, bytecode);
 771   and3(bytecode,        0xFF, bytecode);
 772 }
 773 
 774 
 775 void InterpreterMacroAssembler::get_cache_entry_pointer_at_bcp(Register cache, Register tmp,
 776                                                                int bcp_offset, size_t index_size) {
 777   assert(bcp_offset > 0, "bcp is still pointing to start of bytecode");
 778   assert_different_registers(cache, tmp);
 779   assert_not_delayed();
 780   if (index_size == sizeof(u2)) {
 781     get_2_byte_integer_at_bcp(bcp_offset, cache, tmp, Unsigned);
 782   } else {
 783     ShouldNotReachHere();  // other sizes not supported here
 784   }
 785               // convert from field index to ConstantPoolCacheEntry index
 786               // and from word index to byte offset
 787   sll(tmp, exact_log2(in_words(ConstantPoolCacheEntry::size()) * BytesPerWord), tmp);
 788               // skip past the header
 789   add(tmp, in_bytes(constantPoolCacheOopDesc::base_offset()), tmp);
 790               // construct pointer to cache entry
 791   add(LcpoolCache, tmp, cache);
 792 }
 793 
 794 
 795 // Generate a subtype check: branch to ok_is_subtype if sub_klass is
 796 // a subtype of super_klass.  Blows registers Rsuper_klass, Rsub_klass, tmp1, tmp2.
 797 void InterpreterMacroAssembler::gen_subtype_check(Register Rsub_klass,
 798                                                   Register Rsuper_klass,
 799                                                   Register Rtmp1,
 800                                                   Register Rtmp2,
 801                                                   Register Rtmp3,
 802                                                   Label &ok_is_subtype ) {
 803   Label not_subtype;
 804 
 805   // Profile the not-null value's klass.
 806   profile_typecheck(Rsub_klass, Rtmp1);
 807 
 808   check_klass_subtype_fast_path(Rsub_klass, Rsuper_klass,
 809                                 Rtmp1, Rtmp2,
 810                                 &ok_is_subtype, &not_subtype, NULL);
 811 
 812   check_klass_subtype_slow_path(Rsub_klass, Rsuper_klass,
 813                                 Rtmp1, Rtmp2, Rtmp3, /*hack:*/ noreg,
 814                                 &ok_is_subtype, NULL);
 815 
 816   bind(not_subtype);
 817   profile_typecheck_failed(Rtmp1);
 818 }
 819 
 820 // Separate these two to allow for delay slot in middle
 821 // These are used to do a test and full jump to exception-throwing code.
 822 
 823 // %%%%% Could possibly reoptimize this by testing to see if could use
 824 // a single conditional branch (i.e. if span is small enough.
 825 // If you go that route, than get rid of the split and give up
 826 // on the delay-slot hack.
 827 
 828 void InterpreterMacroAssembler::throw_if_not_1_icc( Condition ok_condition,
 829                                                     Label&    ok ) {
 830   assert_not_delayed();
 831   br(ok_condition, true, pt, ok);
 832   // DELAY SLOT
 833 }
 834 
 835 void InterpreterMacroAssembler::throw_if_not_1_xcc( Condition ok_condition,
 836                                                     Label&    ok ) {
 837   assert_not_delayed();
 838   bp( ok_condition, true, Assembler::xcc, pt, ok);
 839   // DELAY SLOT
 840 }
 841 
 842 void InterpreterMacroAssembler::throw_if_not_1_x( Condition ok_condition,
 843                                                   Label&    ok ) {
 844   assert_not_delayed();
 845   brx(ok_condition, true, pt, ok);
 846   // DELAY SLOT
 847 }
 848 
 849 void InterpreterMacroAssembler::throw_if_not_2( address  throw_entry_point,
 850                                                 Register Rscratch,
 851                                                 Label&   ok ) {
 852   assert(throw_entry_point != NULL, "entry point must be generated by now");
 853   AddressLiteral dest(throw_entry_point);
 854   jump_to(dest, Rscratch);
 855   delayed()->nop();
 856   bind(ok);
 857 }
 858 
 859 
 860 // And if you cannot use the delay slot, here is a shorthand:
 861 
 862 void InterpreterMacroAssembler::throw_if_not_icc( Condition ok_condition,
 863                                                   address   throw_entry_point,
 864                                                   Register  Rscratch ) {
 865   Label ok;
 866   if (ok_condition != never) {
 867     throw_if_not_1_icc( ok_condition, ok);
 868     delayed()->nop();
 869   }
 870   throw_if_not_2( throw_entry_point, Rscratch, ok);
 871 }
 872 void InterpreterMacroAssembler::throw_if_not_xcc( Condition ok_condition,
 873                                                   address   throw_entry_point,
 874                                                   Register  Rscratch ) {
 875   Label ok;
 876   if (ok_condition != never) {
 877     throw_if_not_1_xcc( ok_condition, ok);
 878     delayed()->nop();
 879   }
 880   throw_if_not_2( throw_entry_point, Rscratch, ok);
 881 }
 882 void InterpreterMacroAssembler::throw_if_not_x( Condition ok_condition,
 883                                                 address   throw_entry_point,
 884                                                 Register  Rscratch ) {
 885   Label ok;
 886   if (ok_condition != never) {
 887     throw_if_not_1_x( ok_condition, ok);
 888     delayed()->nop();
 889   }
 890   throw_if_not_2( throw_entry_point, Rscratch, ok);
 891 }
 892 
 893 // Check that index is in range for array, then shift index by index_shift, and put arrayOop + shifted_index into res
 894 // Note: res is still shy of address by array offset into object.
 895 
 896 void InterpreterMacroAssembler::index_check_without_pop(Register array, Register index, int index_shift, Register tmp, Register res) {
 897   assert_not_delayed();
 898 
 899   verify_oop(array);
 900 #ifdef _LP64
 901   // sign extend since tos (index) can be a 32bit value
 902   sra(index, G0, index);
 903 #endif // _LP64
 904 
 905   // check array
 906   Label ptr_ok;
 907   tst(array);
 908   throw_if_not_1_x( notZero, ptr_ok );
 909   delayed()->ld( array, arrayOopDesc::length_offset_in_bytes(), tmp ); // check index
 910   throw_if_not_2( Interpreter::_throw_NullPointerException_entry, G3_scratch, ptr_ok);
 911 
 912   Label index_ok;
 913   cmp(index, tmp);
 914   throw_if_not_1_icc( lessUnsigned, index_ok );
 915   if (index_shift > 0)  delayed()->sll(index, index_shift, index);
 916   else                  delayed()->add(array, index, res); // addr - const offset in index
 917   // convention: move aberrant index into G3_scratch for exception message
 918   mov(index, G3_scratch);
 919   throw_if_not_2( Interpreter::_throw_ArrayIndexOutOfBoundsException_entry, G4_scratch, index_ok);
 920 
 921   // add offset if didn't do it in delay slot
 922   if (index_shift > 0)   add(array, index, res); // addr - const offset in index
 923 }
 924 
 925 
 926 void InterpreterMacroAssembler::index_check(Register array, Register index, int index_shift, Register tmp, Register res) {
 927   assert_not_delayed();
 928 
 929   // pop array
 930   pop_ptr(array);
 931 
 932   // check array
 933   index_check_without_pop(array, index, index_shift, tmp, res);
 934 }
 935 
 936 
 937 void InterpreterMacroAssembler::get_const(Register Rdst) {
 938   ld_ptr(Lmethod, in_bytes(methodOopDesc::const_offset()), Rdst);
 939 }
 940 
 941 
 942 void InterpreterMacroAssembler::get_constant_pool(Register Rdst) {
 943   get_const(Rdst);
 944   ld_ptr(Rdst, in_bytes(constMethodOopDesc::constants_offset()), Rdst);
 945 }
 946 
 947 
 948 void InterpreterMacroAssembler::get_constant_pool_cache(Register Rdst) {
 949   get_constant_pool(Rdst);
 950   ld_ptr(Rdst, constantPoolOopDesc::cache_offset_in_bytes(), Rdst);
 951 }
 952 
 953 
 954 void InterpreterMacroAssembler::get_cpool_and_tags(Register Rcpool, Register Rtags) {
 955   get_constant_pool(Rcpool);
 956   ld_ptr(Rcpool, constantPoolOopDesc::tags_offset_in_bytes(), Rtags);
 957 }
 958 
 959 
 960 // unlock if synchronized method
 961 //
 962 // Unlock the receiver if this is a synchronized method.
 963 // Unlock any Java monitors from syncronized blocks.
 964 //
 965 // If there are locked Java monitors
 966 //    If throw_monitor_exception
 967 //       throws IllegalMonitorStateException
 968 //    Else if install_monitor_exception
 969 //       installs IllegalMonitorStateException
 970 //    Else
 971 //       no error processing
 972 void InterpreterMacroAssembler::unlock_if_synchronized_method(TosState state,
 973                                                               bool throw_monitor_exception,
 974                                                               bool install_monitor_exception) {
 975   Label unlocked, unlock, no_unlock;
 976 
 977   // get the value of _do_not_unlock_if_synchronized into G1_scratch
 978   const Address do_not_unlock_if_synchronized(G2_thread,
 979     JavaThread::do_not_unlock_if_synchronized_offset());
 980   ldbool(do_not_unlock_if_synchronized, G1_scratch);
 981   stbool(G0, do_not_unlock_if_synchronized); // reset the flag
 982 
 983   // check if synchronized method
 984   const Address access_flags(Lmethod, methodOopDesc::access_flags_offset());
 985   interp_verify_oop(Otos_i, state, __FILE__, __LINE__);
 986   push(state); // save tos
 987   ld(access_flags, G3_scratch); // Load access flags.
 988   btst(JVM_ACC_SYNCHRONIZED, G3_scratch);
 989   br(zero, false, pt, unlocked);
 990   delayed()->nop();
 991 
 992   // Don't unlock anything if the _do_not_unlock_if_synchronized flag
 993   // is set.
 994   cmp_zero_and_br(Assembler::notZero, G1_scratch, no_unlock);
 995   delayed()->nop();
 996 
 997   // BasicObjectLock will be first in list, since this is a synchronized method. However, need
 998   // to check that the object has not been unlocked by an explicit monitorexit bytecode.
 999 
1000   //Intel: if (throw_monitor_exception) ... else ...
1001   // Entry already unlocked, need to throw exception
1002   //...
1003 
1004   // pass top-most monitor elem
1005   add( top_most_monitor(), O1 );
1006 
1007   ld_ptr(O1, BasicObjectLock::obj_offset_in_bytes(), G3_scratch);
1008   br_notnull_short(G3_scratch, pt, unlock);
1009 
1010   if (throw_monitor_exception) {
1011     // Entry already unlocked need to throw an exception
1012     MacroAssembler::call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::throw_illegal_monitor_state_exception));
1013     should_not_reach_here();
1014   } else {
1015     // Monitor already unlocked during a stack unroll.
1016     // If requested, install an illegal_monitor_state_exception.
1017     // Continue with stack unrolling.
1018     if (install_monitor_exception) {
1019       MacroAssembler::call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::new_illegal_monitor_state_exception));
1020     }
1021     ba_short(unlocked);
1022   }
1023 
1024   bind(unlock);
1025 
1026   unlock_object(O1);
1027 
1028   bind(unlocked);
1029 
1030   // I0, I1: Might contain return value
1031 
1032   // Check that all monitors are unlocked
1033   { Label loop, exception, entry, restart;
1034 
1035     Register Rmptr   = O0;
1036     Register Rtemp   = O1;
1037     Register Rlimit  = Lmonitors;
1038     const jint delta = frame::interpreter_frame_monitor_size() * wordSize;
1039     assert( (delta & LongAlignmentMask) == 0,
1040             "sizeof BasicObjectLock must be even number of doublewords");
1041 
1042     #ifdef ASSERT
1043     add(top_most_monitor(), Rmptr, delta);
1044     { Label L;
1045       // ensure that Rmptr starts out above (or at) Rlimit
1046       cmp_and_brx_short(Rmptr, Rlimit, Assembler::greaterEqualUnsigned, pn, L);
1047       stop("monitor stack has negative size");
1048       bind(L);
1049     }
1050     #endif
1051     bind(restart);
1052     ba(entry);
1053     delayed()->
1054     add(top_most_monitor(), Rmptr, delta);      // points to current entry, starting with bottom-most entry
1055 
1056     // Entry is still locked, need to throw exception
1057     bind(exception);
1058     if (throw_monitor_exception) {
1059       MacroAssembler::call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::throw_illegal_monitor_state_exception));
1060       should_not_reach_here();
1061     } else {
1062       // Stack unrolling. Unlock object and if requested, install illegal_monitor_exception.
1063       // Unlock does not block, so don't have to worry about the frame
1064       unlock_object(Rmptr);
1065       if (install_monitor_exception) {
1066         MacroAssembler::call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::new_illegal_monitor_state_exception));
1067       }
1068       ba_short(restart);
1069     }
1070 
1071     bind(loop);
1072     cmp(Rtemp, G0);                             // check if current entry is used
1073     brx(Assembler::notEqual, false, pn, exception);
1074     delayed()->
1075     dec(Rmptr, delta);                          // otherwise advance to next entry
1076     #ifdef ASSERT
1077     { Label L;
1078       // ensure that Rmptr has not somehow stepped below Rlimit
1079       cmp_and_brx_short(Rmptr, Rlimit, Assembler::greaterEqualUnsigned, pn, L);
1080       stop("ran off the end of the monitor stack");
1081       bind(L);
1082     }
1083     #endif
1084     bind(entry);
1085     cmp(Rmptr, Rlimit);                         // check if bottom reached
1086     brx(Assembler::notEqual, true, pn, loop);   // if not at bottom then check this entry
1087     delayed()->
1088     ld_ptr(Rmptr, BasicObjectLock::obj_offset_in_bytes() - delta, Rtemp);
1089   }
1090 
1091   bind(no_unlock);
1092   pop(state);
1093   interp_verify_oop(Otos_i, state, __FILE__, __LINE__);
1094 }
1095 
1096 
1097 // remove activation
1098 //
1099 // Unlock the receiver if this is a synchronized method.
1100 // Unlock any Java monitors from syncronized blocks.
1101 // Remove the activation from the stack.
1102 //
1103 // If there are locked Java monitors
1104 //    If throw_monitor_exception
1105 //       throws IllegalMonitorStateException
1106 //    Else if install_monitor_exception
1107 //       installs IllegalMonitorStateException
1108 //    Else
1109 //       no error processing
1110 void InterpreterMacroAssembler::remove_activation(TosState state,
1111                                                   bool throw_monitor_exception,
1112                                                   bool install_monitor_exception) {
1113 
1114   unlock_if_synchronized_method(state, throw_monitor_exception, install_monitor_exception);
1115 
1116   // save result (push state before jvmti call and pop it afterwards) and notify jvmti
1117   notify_method_exit(false, state, NotifyJVMTI);
1118 
1119   interp_verify_oop(Otos_i, state, __FILE__, __LINE__);
1120   verify_oop(Lmethod);
1121   verify_thread();
1122 
1123   // return tos
1124   assert(Otos_l1 == Otos_i, "adjust code below");
1125   switch (state) {
1126 #ifdef _LP64
1127   case ltos: mov(Otos_l, Otos_l->after_save()); break; // O0 -> I0
1128 #else
1129   case ltos: mov(Otos_l2, Otos_l2->after_save()); // fall through  // O1 -> I1
1130 #endif
1131   case btos:                                      // fall through
1132   case ctos:
1133   case stos:                                      // fall through
1134   case atos:                                      // fall through
1135   case itos: mov(Otos_l1, Otos_l1->after_save());    break;        // O0 -> I0
1136   case ftos:                                      // fall through
1137   case dtos:                                      // fall through
1138   case vtos: /* nothing to do */                     break;
1139   default  : ShouldNotReachHere();
1140   }
1141 
1142 #if defined(COMPILER2) && !defined(_LP64)
1143   if (state == ltos) {
1144     // C2 expects long results in G1 we can't tell if we're returning to interpreted
1145     // or compiled so just be safe use G1 and O0/O1
1146 
1147     // Shift bits into high (msb) of G1
1148     sllx(Otos_l1->after_save(), 32, G1);
1149     // Zero extend low bits
1150     srl (Otos_l2->after_save(), 0, Otos_l2->after_save());
1151     or3 (Otos_l2->after_save(), G1, G1);
1152   }
1153 #endif /* COMPILER2 */
1154 
1155 }
1156 #endif /* CC_INTERP */
1157 
1158 
1159 // Lock object
1160 //
1161 // Argument - lock_reg points to the BasicObjectLock to be used for locking,
1162 //            it must be initialized with the object to lock
1163 void InterpreterMacroAssembler::lock_object(Register lock_reg, Register Object) {
1164   if (UseHeavyMonitors) {
1165     call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::monitorenter), lock_reg);
1166   }
1167   else {
1168     Register obj_reg = Object;
1169     Register mark_reg = G4_scratch;
1170     Register temp_reg = G1_scratch;
1171     Address  lock_addr(lock_reg, BasicObjectLock::lock_offset_in_bytes());
1172     Address  mark_addr(obj_reg, oopDesc::mark_offset_in_bytes());
1173     Label    done;
1174 
1175     Label slow_case;
1176 
1177     assert_different_registers(lock_reg, obj_reg, mark_reg, temp_reg);
1178 
1179     // load markOop from object into mark_reg
1180     ld_ptr(mark_addr, mark_reg);
1181 
1182     if (UseBiasedLocking) {
1183       biased_locking_enter(obj_reg, mark_reg, temp_reg, done, &slow_case);
1184     }
1185 
1186     // get the address of basicLock on stack that will be stored in the object
1187     // we need a temporary register here as we do not want to clobber lock_reg
1188     // (cas clobbers the destination register)
1189     mov(lock_reg, temp_reg);
1190     // set mark reg to be (markOop of object | UNLOCK_VALUE)
1191     or3(mark_reg, markOopDesc::unlocked_value, mark_reg);
1192     // initialize the box  (Must happen before we update the object mark!)
1193     st_ptr(mark_reg, lock_addr, BasicLock::displaced_header_offset_in_bytes());
1194     // compare and exchange object_addr, markOop | 1, stack address of basicLock
1195     assert(mark_addr.disp() == 0, "cas must take a zero displacement");
1196     casx_under_lock(mark_addr.base(), mark_reg, temp_reg,
1197       (address)StubRoutines::Sparc::atomic_memory_operation_lock_addr());
1198 
1199     // if the compare and exchange succeeded we are done (we saw an unlocked object)
1200     cmp_and_brx_short(mark_reg, temp_reg, Assembler::equal, Assembler::pt, done);
1201 
1202     // We did not see an unlocked object so try the fast recursive case
1203 
1204     // Check if owner is self by comparing the value in the markOop of object
1205     // with the stack pointer
1206     sub(temp_reg, SP, temp_reg);
1207 #ifdef _LP64
1208     sub(temp_reg, STACK_BIAS, temp_reg);
1209 #endif
1210     assert(os::vm_page_size() > 0xfff, "page size too small - change the constant");
1211 
1212     // Composite "andcc" test:
1213     // (a) %sp -vs- markword proximity check, and,
1214     // (b) verify mark word LSBs == 0 (Stack-locked).
1215     //
1216     // FFFFF003/FFFFFFFFFFFF003 is (markOopDesc::lock_mask_in_place | -os::vm_page_size())
1217     // Note that the page size used for %sp proximity testing is arbitrary and is
1218     // unrelated to the actual MMU page size.  We use a 'logical' page size of
1219     // 4096 bytes.   F..FFF003 is designed to fit conveniently in the SIMM13 immediate
1220     // field of the andcc instruction.
1221     andcc (temp_reg, 0xFFFFF003, G0) ;
1222 
1223     // if condition is true we are done and hence we can store 0 in the displaced
1224     // header indicating it is a recursive lock and be done
1225     brx(Assembler::zero, true, Assembler::pt, done);
1226     delayed()->st_ptr(G0, lock_addr, BasicLock::displaced_header_offset_in_bytes());
1227 
1228     // none of the above fast optimizations worked so we have to get into the
1229     // slow case of monitor enter
1230     bind(slow_case);
1231     call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::monitorenter), lock_reg);
1232 
1233     bind(done);
1234   }
1235 }
1236 
1237 // Unlocks an object. Used in monitorexit bytecode and remove_activation.
1238 //
1239 // Argument - lock_reg points to the BasicObjectLock for lock
1240 // Throw IllegalMonitorException if object is not locked by current thread
1241 void InterpreterMacroAssembler::unlock_object(Register lock_reg) {
1242   if (UseHeavyMonitors) {
1243     call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::monitorexit), lock_reg);
1244   } else {
1245     Register obj_reg = G3_scratch;
1246     Register mark_reg = G4_scratch;
1247     Register displaced_header_reg = G1_scratch;
1248     Address  lockobj_addr(lock_reg, BasicObjectLock::obj_offset_in_bytes());
1249     Address  mark_addr(obj_reg, oopDesc::mark_offset_in_bytes());
1250     Label    done;
1251 
1252     if (UseBiasedLocking) {
1253       // load the object out of the BasicObjectLock
1254       ld_ptr(lockobj_addr, obj_reg);
1255       biased_locking_exit(mark_addr, mark_reg, done, true);
1256       st_ptr(G0, lockobj_addr);  // free entry
1257     }
1258 
1259     // Test first if we are in the fast recursive case
1260     Address lock_addr(lock_reg, BasicObjectLock::lock_offset_in_bytes() + BasicLock::displaced_header_offset_in_bytes());
1261     ld_ptr(lock_addr, displaced_header_reg);
1262     br_null(displaced_header_reg, true, Assembler::pn, done);
1263     delayed()->st_ptr(G0, lockobj_addr);  // free entry
1264 
1265     // See if it is still a light weight lock, if so we just unlock
1266     // the object and we are done
1267 
1268     if (!UseBiasedLocking) {
1269       // load the object out of the BasicObjectLock
1270       ld_ptr(lockobj_addr, obj_reg);
1271     }
1272 
1273     // we have the displaced header in displaced_header_reg
1274     // we expect to see the stack address of the basicLock in case the
1275     // lock is still a light weight lock (lock_reg)
1276     assert(mark_addr.disp() == 0, "cas must take a zero displacement");
1277     casx_under_lock(mark_addr.base(), lock_reg, displaced_header_reg,
1278       (address)StubRoutines::Sparc::atomic_memory_operation_lock_addr());
1279     cmp(lock_reg, displaced_header_reg);
1280     brx(Assembler::equal, true, Assembler::pn, done);
1281     delayed()->st_ptr(G0, lockobj_addr);  // free entry
1282 
1283     // The lock has been converted into a heavy lock and hence
1284     // we need to get into the slow case
1285 
1286     call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::monitorexit), lock_reg);
1287 
1288     bind(done);
1289   }
1290 }
1291 
1292 #ifndef CC_INTERP
1293 
1294 // Get the method data pointer from the methodOop and set the
1295 // specified register to its value.
1296 
1297 void InterpreterMacroAssembler::set_method_data_pointer() {
1298   assert(ProfileInterpreter, "must be profiling interpreter");
1299   Label get_continue;
1300 
1301   ld_ptr(Lmethod, in_bytes(methodOopDesc::method_data_offset()), ImethodDataPtr);
1302   test_method_data_pointer(get_continue);
1303   add(ImethodDataPtr, in_bytes(methodDataOopDesc::data_offset()), ImethodDataPtr);
1304   bind(get_continue);
1305 }
1306 
1307 // Set the method data pointer for the current bcp.
1308 
1309 void InterpreterMacroAssembler::set_method_data_pointer_for_bcp() {
1310   assert(ProfileInterpreter, "must be profiling interpreter");
1311   Label zero_continue;
1312 
1313   // Test MDO to avoid the call if it is NULL.
1314   ld_ptr(Lmethod, in_bytes(methodOopDesc::method_data_offset()), ImethodDataPtr);
1315   test_method_data_pointer(zero_continue);
1316   call_VM_leaf(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::bcp_to_di), Lmethod, Lbcp);
1317   add(ImethodDataPtr, in_bytes(methodDataOopDesc::data_offset()), ImethodDataPtr);
1318   add(ImethodDataPtr, O0, ImethodDataPtr);
1319   bind(zero_continue);
1320 }
1321 
1322 // Test ImethodDataPtr.  If it is null, continue at the specified label
1323 
1324 void InterpreterMacroAssembler::test_method_data_pointer(Label& zero_continue) {
1325   assert(ProfileInterpreter, "must be profiling interpreter");
1326   br_null_short(ImethodDataPtr, Assembler::pn, zero_continue);
1327 }
1328 
1329 void InterpreterMacroAssembler::verify_method_data_pointer() {
1330   assert(ProfileInterpreter, "must be profiling interpreter");
1331 #ifdef ASSERT
1332   Label verify_continue;
1333   test_method_data_pointer(verify_continue);
1334 
1335   // If the mdp is valid, it will point to a DataLayout header which is
1336   // consistent with the bcp.  The converse is highly probable also.
1337   lduh(ImethodDataPtr, in_bytes(DataLayout::bci_offset()), G3_scratch);
1338   ld_ptr(Lmethod, methodOopDesc::const_offset(), O5);
1339   add(G3_scratch, in_bytes(constMethodOopDesc::codes_offset()), G3_scratch);
1340   add(G3_scratch, O5, G3_scratch);
1341   cmp(Lbcp, G3_scratch);
1342   brx(Assembler::equal, false, Assembler::pt, verify_continue);
1343 
1344   Register temp_reg = O5;
1345   delayed()->mov(ImethodDataPtr, temp_reg);
1346   // %%% should use call_VM_leaf here?
1347   //call_VM_leaf(noreg, ..., Lmethod, Lbcp, ImethodDataPtr);
1348   save_frame_and_mov(sizeof(jdouble) / wordSize, Lmethod, O0, Lbcp, O1);
1349   Address d_save(FP, -sizeof(jdouble) + STACK_BIAS);
1350   stf(FloatRegisterImpl::D, Ftos_d, d_save);
1351   mov(temp_reg->after_save(), O2);
1352   save_thread(L7_thread_cache);
1353   call(CAST_FROM_FN_PTR(address, InterpreterRuntime::verify_mdp), relocInfo::none);
1354   delayed()->nop();
1355   restore_thread(L7_thread_cache);
1356   ldf(FloatRegisterImpl::D, d_save, Ftos_d);
1357   restore();
1358   bind(verify_continue);
1359 #endif // ASSERT
1360 }
1361 
1362 void InterpreterMacroAssembler::test_invocation_counter_for_mdp(Register invocation_count,
1363                                                                 Register Rtmp,
1364                                                                 Label &profile_continue) {
1365   assert(ProfileInterpreter, "must be profiling interpreter");
1366   // Control will flow to "profile_continue" if the counter is less than the
1367   // limit or if we call profile_method()
1368 
1369   Label done;
1370 
1371   // if no method data exists, and the counter is high enough, make one
1372   br_notnull_short(ImethodDataPtr, Assembler::pn, done);
1373 
1374   // Test to see if we should create a method data oop
1375   AddressLiteral profile_limit((address) &InvocationCounter::InterpreterProfileLimit);
1376   sethi(profile_limit, Rtmp);
1377   ld(Rtmp, profile_limit.low10(), Rtmp);
1378   cmp_and_br_short(invocation_count, Rtmp, Assembler::lessUnsigned, Assembler::pn, profile_continue);
1379 
1380   // Build it now.
1381   call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::profile_method));
1382   set_method_data_pointer_for_bcp();
1383   ba_short(profile_continue);
1384   bind(done);
1385 }
1386 
1387 // Store a value at some constant offset from the method data pointer.
1388 
1389 void InterpreterMacroAssembler::set_mdp_data_at(int constant, Register value) {
1390   assert(ProfileInterpreter, "must be profiling interpreter");
1391   st_ptr(value, ImethodDataPtr, constant);
1392 }
1393 
1394 void InterpreterMacroAssembler::increment_mdp_data_at(Address counter,
1395                                                       Register bumped_count,
1396                                                       bool decrement) {
1397   assert(ProfileInterpreter, "must be profiling interpreter");
1398 
1399   // Load the counter.
1400   ld_ptr(counter, bumped_count);
1401 
1402   if (decrement) {
1403     // Decrement the register.  Set condition codes.
1404     subcc(bumped_count, DataLayout::counter_increment, bumped_count);
1405 
1406     // If the decrement causes the counter to overflow, stay negative
1407     Label L;
1408     brx(Assembler::negative, true, Assembler::pn, L);
1409 
1410     // Store the decremented counter, if it is still negative.
1411     delayed()->st_ptr(bumped_count, counter);
1412     bind(L);
1413   } else {
1414     // Increment the register.  Set carry flag.
1415     addcc(bumped_count, DataLayout::counter_increment, bumped_count);
1416 
1417     // If the increment causes the counter to overflow, pull back by 1.
1418     assert(DataLayout::counter_increment == 1, "subc works");
1419     subc(bumped_count, G0, bumped_count);
1420 
1421     // Store the incremented counter.
1422     st_ptr(bumped_count, counter);
1423   }
1424 }
1425 
1426 // Increment the value at some constant offset from the method data pointer.
1427 
1428 void InterpreterMacroAssembler::increment_mdp_data_at(int constant,
1429                                                       Register bumped_count,
1430                                                       bool decrement) {
1431   // Locate the counter at a fixed offset from the mdp:
1432   Address counter(ImethodDataPtr, constant);
1433   increment_mdp_data_at(counter, bumped_count, decrement);
1434 }
1435 
1436 // Increment the value at some non-fixed (reg + constant) offset from
1437 // the method data pointer.
1438 
1439 void InterpreterMacroAssembler::increment_mdp_data_at(Register reg,
1440                                                       int constant,
1441                                                       Register bumped_count,
1442                                                       Register scratch2,
1443                                                       bool decrement) {
1444   // Add the constant to reg to get the offset.
1445   add(ImethodDataPtr, reg, scratch2);
1446   Address counter(scratch2, constant);
1447   increment_mdp_data_at(counter, bumped_count, decrement);
1448 }
1449 
1450 // Set a flag value at the current method data pointer position.
1451 // Updates a single byte of the header, to avoid races with other header bits.
1452 
1453 void InterpreterMacroAssembler::set_mdp_flag_at(int flag_constant,
1454                                                 Register scratch) {
1455   assert(ProfileInterpreter, "must be profiling interpreter");
1456   // Load the data header
1457   ldub(ImethodDataPtr, in_bytes(DataLayout::flags_offset()), scratch);
1458 
1459   // Set the flag
1460   or3(scratch, flag_constant, scratch);
1461 
1462   // Store the modified header.
1463   stb(scratch, ImethodDataPtr, in_bytes(DataLayout::flags_offset()));
1464 }
1465 
1466 // Test the location at some offset from the method data pointer.
1467 // If it is not equal to value, branch to the not_equal_continue Label.
1468 // Set condition codes to match the nullness of the loaded value.
1469 
1470 void InterpreterMacroAssembler::test_mdp_data_at(int offset,
1471                                                  Register value,
1472                                                  Label& not_equal_continue,
1473                                                  Register scratch) {
1474   assert(ProfileInterpreter, "must be profiling interpreter");
1475   ld_ptr(ImethodDataPtr, offset, scratch);
1476   cmp(value, scratch);
1477   brx(Assembler::notEqual, false, Assembler::pn, not_equal_continue);
1478   delayed()->tst(scratch);
1479 }
1480 
1481 // Update the method data pointer by the displacement located at some fixed
1482 // offset from the method data pointer.
1483 
1484 void InterpreterMacroAssembler::update_mdp_by_offset(int offset_of_disp,
1485                                                      Register scratch) {
1486   assert(ProfileInterpreter, "must be profiling interpreter");
1487   ld_ptr(ImethodDataPtr, offset_of_disp, scratch);
1488   add(ImethodDataPtr, scratch, ImethodDataPtr);
1489 }
1490 
1491 // Update the method data pointer by the displacement located at the
1492 // offset (reg + offset_of_disp).
1493 
1494 void InterpreterMacroAssembler::update_mdp_by_offset(Register reg,
1495                                                      int offset_of_disp,
1496                                                      Register scratch) {
1497   assert(ProfileInterpreter, "must be profiling interpreter");
1498   add(reg, offset_of_disp, scratch);
1499   ld_ptr(ImethodDataPtr, scratch, scratch);
1500   add(ImethodDataPtr, scratch, ImethodDataPtr);
1501 }
1502 
1503 // Update the method data pointer by a simple constant displacement.
1504 
1505 void InterpreterMacroAssembler::update_mdp_by_constant(int constant) {
1506   assert(ProfileInterpreter, "must be profiling interpreter");
1507   add(ImethodDataPtr, constant, ImethodDataPtr);
1508 }
1509 
1510 // Update the method data pointer for a _ret bytecode whose target
1511 // was not among our cached targets.
1512 
1513 void InterpreterMacroAssembler::update_mdp_for_ret(TosState state,
1514                                                    Register return_bci) {
1515   assert(ProfileInterpreter, "must be profiling interpreter");
1516   push(state);
1517   st_ptr(return_bci, l_tmp);  // protect return_bci, in case it is volatile
1518   call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::update_mdp_for_ret), return_bci);
1519   ld_ptr(l_tmp, return_bci);
1520   pop(state);
1521 }
1522 
1523 // Count a taken branch in the bytecodes.
1524 
1525 void InterpreterMacroAssembler::profile_taken_branch(Register scratch, Register bumped_count) {
1526   if (ProfileInterpreter) {
1527     Label profile_continue;
1528 
1529     // If no method data exists, go to profile_continue.
1530     test_method_data_pointer(profile_continue);
1531 
1532     // We are taking a branch.  Increment the taken count.
1533     increment_mdp_data_at(in_bytes(JumpData::taken_offset()), bumped_count);
1534 
1535     // The method data pointer needs to be updated to reflect the new target.
1536     update_mdp_by_offset(in_bytes(JumpData::displacement_offset()), scratch);
1537     bind (profile_continue);
1538   }
1539 }
1540 
1541 
1542 // Count a not-taken branch in the bytecodes.
1543 
1544 void InterpreterMacroAssembler::profile_not_taken_branch(Register scratch) {
1545   if (ProfileInterpreter) {
1546     Label profile_continue;
1547 
1548     // If no method data exists, go to profile_continue.
1549     test_method_data_pointer(profile_continue);
1550 
1551     // We are taking a branch.  Increment the not taken count.
1552     increment_mdp_data_at(in_bytes(BranchData::not_taken_offset()), scratch);
1553 
1554     // The method data pointer needs to be updated to correspond to the
1555     // next bytecode.
1556     update_mdp_by_constant(in_bytes(BranchData::branch_data_size()));
1557     bind (profile_continue);
1558   }
1559 }
1560 
1561 
1562 // Count a non-virtual call in the bytecodes.
1563 
1564 void InterpreterMacroAssembler::profile_call(Register scratch) {
1565   if (ProfileInterpreter) {
1566     Label profile_continue;
1567 
1568     // If no method data exists, go to profile_continue.
1569     test_method_data_pointer(profile_continue);
1570 
1571     // We are making a call.  Increment the count.
1572     increment_mdp_data_at(in_bytes(CounterData::count_offset()), scratch);
1573 
1574     // The method data pointer needs to be updated to reflect the new target.
1575     update_mdp_by_constant(in_bytes(CounterData::counter_data_size()));
1576     bind (profile_continue);
1577   }
1578 }
1579 
1580 
1581 // Count a final call in the bytecodes.
1582 
1583 void InterpreterMacroAssembler::profile_final_call(Register scratch) {
1584   if (ProfileInterpreter) {
1585     Label profile_continue;
1586 
1587     // If no method data exists, go to profile_continue.
1588     test_method_data_pointer(profile_continue);
1589 
1590     // We are making a call.  Increment the count.
1591     increment_mdp_data_at(in_bytes(CounterData::count_offset()), scratch);
1592 
1593     // The method data pointer needs to be updated to reflect the new target.
1594     update_mdp_by_constant(in_bytes(VirtualCallData::virtual_call_data_size()));
1595     bind (profile_continue);
1596   }
1597 }
1598 
1599 
1600 // Count a virtual call in the bytecodes.
1601 
1602 void InterpreterMacroAssembler::profile_virtual_call(Register receiver,
1603                                                      Register scratch,
1604                                                      bool receiver_can_be_null) {
1605   if (ProfileInterpreter) {
1606     Label profile_continue;
1607 
1608     // If no method data exists, go to profile_continue.
1609     test_method_data_pointer(profile_continue);
1610 
1611 
1612     Label skip_receiver_profile;
1613     if (receiver_can_be_null) {
1614       Label not_null;
1615       br_notnull_short(receiver, Assembler::pt, not_null);
1616       // We are making a call.  Increment the count for null receiver.
1617       increment_mdp_data_at(in_bytes(CounterData::count_offset()), scratch);
1618       ba_short(skip_receiver_profile);
1619       bind(not_null);
1620     }
1621 
1622     // Record the receiver type.
1623     record_klass_in_profile(receiver, scratch, true);
1624     bind(skip_receiver_profile);
1625 
1626     // The method data pointer needs to be updated to reflect the new target.
1627     update_mdp_by_constant(in_bytes(VirtualCallData::virtual_call_data_size()));
1628     bind (profile_continue);
1629   }
1630 }
1631 
1632 void InterpreterMacroAssembler::record_klass_in_profile_helper(
1633                                         Register receiver, Register scratch,
1634                                         int start_row, Label& done, bool is_virtual_call) {
1635   if (TypeProfileWidth == 0) {
1636     if (is_virtual_call) {
1637       increment_mdp_data_at(in_bytes(CounterData::count_offset()), scratch);
1638     }
1639     return;
1640   }
1641 
1642   int last_row = VirtualCallData::row_limit() - 1;
1643   assert(start_row <= last_row, "must be work left to do");
1644   // Test this row for both the receiver and for null.
1645   // Take any of three different outcomes:
1646   //   1. found receiver => increment count and goto done
1647   //   2. found null => keep looking for case 1, maybe allocate this cell
1648   //   3. found something else => keep looking for cases 1 and 2
1649   // Case 3 is handled by a recursive call.
1650   for (int row = start_row; row <= last_row; row++) {
1651     Label next_test;
1652     bool test_for_null_also = (row == start_row);
1653 
1654     // See if the receiver is receiver[n].
1655     int recvr_offset = in_bytes(VirtualCallData::receiver_offset(row));
1656     test_mdp_data_at(recvr_offset, receiver, next_test, scratch);
1657     // delayed()->tst(scratch);
1658 
1659     // The receiver is receiver[n].  Increment count[n].
1660     int count_offset = in_bytes(VirtualCallData::receiver_count_offset(row));
1661     increment_mdp_data_at(count_offset, scratch);
1662     ba_short(done);
1663     bind(next_test);
1664 
1665     if (test_for_null_also) {
1666       Label found_null;
1667       // Failed the equality check on receiver[n]...  Test for null.
1668       if (start_row == last_row) {
1669         // The only thing left to do is handle the null case.
1670         if (is_virtual_call) {
1671           brx(Assembler::zero, false, Assembler::pn, found_null);
1672           delayed()->nop();
1673           // Receiver did not match any saved receiver and there is no empty row for it.
1674           // Increment total counter to indicate polymorphic case.
1675           increment_mdp_data_at(in_bytes(CounterData::count_offset()), scratch);
1676           ba_short(done);
1677           bind(found_null);
1678         } else {
1679           brx(Assembler::notZero, false, Assembler::pt, done);
1680           delayed()->nop();
1681         }
1682         break;
1683       }
1684       // Since null is rare, make it be the branch-taken case.
1685       brx(Assembler::zero, false, Assembler::pn, found_null);
1686       delayed()->nop();
1687 
1688       // Put all the "Case 3" tests here.
1689       record_klass_in_profile_helper(receiver, scratch, start_row + 1, done, is_virtual_call);
1690 
1691       // Found a null.  Keep searching for a matching receiver,
1692       // but remember that this is an empty (unused) slot.
1693       bind(found_null);
1694     }
1695   }
1696 
1697   // In the fall-through case, we found no matching receiver, but we
1698   // observed the receiver[start_row] is NULL.
1699 
1700   // Fill in the receiver field and increment the count.
1701   int recvr_offset = in_bytes(VirtualCallData::receiver_offset(start_row));
1702   set_mdp_data_at(recvr_offset, receiver);
1703   int count_offset = in_bytes(VirtualCallData::receiver_count_offset(start_row));
1704   mov(DataLayout::counter_increment, scratch);
1705   set_mdp_data_at(count_offset, scratch);
1706   if (start_row > 0) {
1707     ba_short(done);
1708   }
1709 }
1710 
1711 void InterpreterMacroAssembler::record_klass_in_profile(Register receiver,
1712                                                         Register scratch, bool is_virtual_call) {
1713   assert(ProfileInterpreter, "must be profiling");
1714   Label done;
1715 
1716   record_klass_in_profile_helper(receiver, scratch, 0, done, is_virtual_call);
1717 
1718   bind (done);
1719 }
1720 
1721 
1722 // Count a ret in the bytecodes.
1723 
1724 void InterpreterMacroAssembler::profile_ret(TosState state,
1725                                             Register return_bci,
1726                                             Register scratch) {
1727   if (ProfileInterpreter) {
1728     Label profile_continue;
1729     uint row;
1730 
1731     // If no method data exists, go to profile_continue.
1732     test_method_data_pointer(profile_continue);
1733 
1734     // Update the total ret count.
1735     increment_mdp_data_at(in_bytes(CounterData::count_offset()), scratch);
1736 
1737     for (row = 0; row < RetData::row_limit(); row++) {
1738       Label next_test;
1739 
1740       // See if return_bci is equal to bci[n]:
1741       test_mdp_data_at(in_bytes(RetData::bci_offset(row)),
1742                        return_bci, next_test, scratch);
1743 
1744       // return_bci is equal to bci[n].  Increment the count.
1745       increment_mdp_data_at(in_bytes(RetData::bci_count_offset(row)), scratch);
1746 
1747       // The method data pointer needs to be updated to reflect the new target.
1748       update_mdp_by_offset(in_bytes(RetData::bci_displacement_offset(row)), scratch);
1749       ba_short(profile_continue);
1750       bind(next_test);
1751     }
1752 
1753     update_mdp_for_ret(state, return_bci);
1754 
1755     bind (profile_continue);
1756   }
1757 }
1758 
1759 // Profile an unexpected null in the bytecodes.
1760 void InterpreterMacroAssembler::profile_null_seen(Register scratch) {
1761   if (ProfileInterpreter) {
1762     Label profile_continue;
1763 
1764     // If no method data exists, go to profile_continue.
1765     test_method_data_pointer(profile_continue);
1766 
1767     set_mdp_flag_at(BitData::null_seen_byte_constant(), scratch);
1768 
1769     // The method data pointer needs to be updated.
1770     int mdp_delta = in_bytes(BitData::bit_data_size());
1771     if (TypeProfileCasts) {
1772       mdp_delta = in_bytes(VirtualCallData::virtual_call_data_size());
1773     }
1774     update_mdp_by_constant(mdp_delta);
1775 
1776     bind (profile_continue);
1777   }
1778 }
1779 
1780 void InterpreterMacroAssembler::profile_typecheck(Register klass,
1781                                                   Register scratch) {
1782   if (ProfileInterpreter) {
1783     Label profile_continue;
1784 
1785     // If no method data exists, go to profile_continue.
1786     test_method_data_pointer(profile_continue);
1787 
1788     int mdp_delta = in_bytes(BitData::bit_data_size());
1789     if (TypeProfileCasts) {
1790       mdp_delta = in_bytes(VirtualCallData::virtual_call_data_size());
1791 
1792       // Record the object type.
1793       record_klass_in_profile(klass, scratch, false);
1794     }
1795 
1796     // The method data pointer needs to be updated.
1797     update_mdp_by_constant(mdp_delta);
1798 
1799     bind (profile_continue);
1800   }
1801 }
1802 
1803 void InterpreterMacroAssembler::profile_typecheck_failed(Register scratch) {
1804   if (ProfileInterpreter && TypeProfileCasts) {
1805     Label profile_continue;
1806 
1807     // If no method data exists, go to profile_continue.
1808     test_method_data_pointer(profile_continue);
1809 
1810     int count_offset = in_bytes(CounterData::count_offset());
1811     // Back up the address, since we have already bumped the mdp.
1812     count_offset -= in_bytes(VirtualCallData::virtual_call_data_size());
1813 
1814     // *Decrement* the counter.  We expect to see zero or small negatives.
1815     increment_mdp_data_at(count_offset, scratch, true);
1816 
1817     bind (profile_continue);
1818   }
1819 }
1820 
1821 // Count the default case of a switch construct.
1822 
1823 void InterpreterMacroAssembler::profile_switch_default(Register scratch) {
1824   if (ProfileInterpreter) {
1825     Label profile_continue;
1826 
1827     // If no method data exists, go to profile_continue.
1828     test_method_data_pointer(profile_continue);
1829 
1830     // Update the default case count
1831     increment_mdp_data_at(in_bytes(MultiBranchData::default_count_offset()),
1832                           scratch);
1833 
1834     // The method data pointer needs to be updated.
1835     update_mdp_by_offset(
1836                     in_bytes(MultiBranchData::default_displacement_offset()),
1837                     scratch);
1838 
1839     bind (profile_continue);
1840   }
1841 }
1842 
1843 // Count the index'th case of a switch construct.
1844 
1845 void InterpreterMacroAssembler::profile_switch_case(Register index,
1846                                                     Register scratch,
1847                                                     Register scratch2,
1848                                                     Register scratch3) {
1849   if (ProfileInterpreter) {
1850     Label profile_continue;
1851 
1852     // If no method data exists, go to profile_continue.
1853     test_method_data_pointer(profile_continue);
1854 
1855     // Build the base (index * per_case_size_in_bytes()) + case_array_offset_in_bytes()
1856     set(in_bytes(MultiBranchData::per_case_size()), scratch);
1857     smul(index, scratch, scratch);
1858     add(scratch, in_bytes(MultiBranchData::case_array_offset()), scratch);
1859 
1860     // Update the case count
1861     increment_mdp_data_at(scratch,
1862                           in_bytes(MultiBranchData::relative_count_offset()),
1863                           scratch2,
1864                           scratch3);
1865 
1866     // The method data pointer needs to be updated.
1867     update_mdp_by_offset(scratch,
1868                      in_bytes(MultiBranchData::relative_displacement_offset()),
1869                      scratch2);
1870 
1871     bind (profile_continue);
1872   }
1873 }
1874 
1875 // add a InterpMonitorElem to stack (see frame_sparc.hpp)
1876 
1877 void InterpreterMacroAssembler::add_monitor_to_stack( bool stack_is_empty,
1878                                                       Register Rtemp,
1879                                                       Register Rtemp2 ) {
1880 
1881   Register Rlimit = Lmonitors;
1882   const jint delta = frame::interpreter_frame_monitor_size() * wordSize;
1883   assert( (delta & LongAlignmentMask) == 0,
1884           "sizeof BasicObjectLock must be even number of doublewords");
1885 
1886   sub( SP,        delta, SP);
1887   sub( Lesp,      delta, Lesp);
1888   sub( Lmonitors, delta, Lmonitors);
1889 
1890   if (!stack_is_empty) {
1891 
1892     // must copy stack contents down
1893 
1894     Label start_copying, next;
1895 
1896     // untested("monitor stack expansion");
1897     compute_stack_base(Rtemp);
1898     ba(start_copying);
1899     delayed()->cmp(Rtemp, Rlimit); // done? duplicated below
1900 
1901     // note: must copy from low memory upwards
1902     // On entry to loop,
1903     // Rtemp points to new base of stack, Lesp points to new end of stack (1 past TOS)
1904     // Loop mutates Rtemp
1905 
1906     bind( next);
1907 
1908     st_ptr(Rtemp2, Rtemp, 0);
1909     inc(Rtemp, wordSize);
1910     cmp(Rtemp, Rlimit); // are we done? (duplicated above)
1911 
1912     bind( start_copying );
1913 
1914     brx( notEqual, true, pn, next );
1915     delayed()->ld_ptr( Rtemp, delta, Rtemp2 );
1916 
1917     // done copying stack
1918   }
1919 }
1920 
1921 // Locals
1922 void InterpreterMacroAssembler::access_local_ptr( Register index, Register dst ) {
1923   assert_not_delayed();
1924   sll(index, Interpreter::logStackElementSize, index);
1925   sub(Llocals, index, index);
1926   ld_ptr(index, 0, dst);
1927   // Note:  index must hold the effective address--the iinc template uses it
1928 }
1929 
1930 // Just like access_local_ptr but the tag is a returnAddress
1931 void InterpreterMacroAssembler::access_local_returnAddress(Register index,
1932                                                            Register dst ) {
1933   assert_not_delayed();
1934   sll(index, Interpreter::logStackElementSize, index);
1935   sub(Llocals, index, index);
1936   ld_ptr(index, 0, dst);
1937 }
1938 
1939 void InterpreterMacroAssembler::access_local_int( Register index, Register dst ) {
1940   assert_not_delayed();
1941   sll(index, Interpreter::logStackElementSize, index);
1942   sub(Llocals, index, index);
1943   ld(index, 0, dst);
1944   // Note:  index must hold the effective address--the iinc template uses it
1945 }
1946 
1947 
1948 void InterpreterMacroAssembler::access_local_long( Register index, Register dst ) {
1949   assert_not_delayed();
1950   sll(index, Interpreter::logStackElementSize, index);
1951   sub(Llocals, index, index);
1952   // First half stored at index n+1 (which grows down from Llocals[n])
1953   load_unaligned_long(index, Interpreter::local_offset_in_bytes(1), dst);
1954 }
1955 
1956 
1957 void InterpreterMacroAssembler::access_local_float( Register index, FloatRegister dst ) {
1958   assert_not_delayed();
1959   sll(index, Interpreter::logStackElementSize, index);
1960   sub(Llocals, index, index);
1961   ldf(FloatRegisterImpl::S, index, 0, dst);
1962 }
1963 
1964 
1965 void InterpreterMacroAssembler::access_local_double( Register index, FloatRegister dst ) {
1966   assert_not_delayed();
1967   sll(index, Interpreter::logStackElementSize, index);
1968   sub(Llocals, index, index);
1969   load_unaligned_double(index, Interpreter::local_offset_in_bytes(1), dst);
1970 }
1971 
1972 
1973 #ifdef ASSERT
1974 void InterpreterMacroAssembler::check_for_regarea_stomp(Register Rindex, int offset, Register Rlimit, Register Rscratch, Register Rscratch1) {
1975   Label L;
1976 
1977   assert(Rindex != Rscratch, "Registers cannot be same");
1978   assert(Rindex != Rscratch1, "Registers cannot be same");
1979   assert(Rlimit != Rscratch, "Registers cannot be same");
1980   assert(Rlimit != Rscratch1, "Registers cannot be same");
1981   assert(Rscratch1 != Rscratch, "Registers cannot be same");
1982 
1983   // untested("reg area corruption");
1984   add(Rindex, offset, Rscratch);
1985   add(Rlimit, 64 + STACK_BIAS, Rscratch1);
1986   cmp_and_brx_short(Rscratch, Rscratch1, Assembler::greaterEqualUnsigned, pn, L);
1987   stop("regsave area is being clobbered");
1988   bind(L);
1989 }
1990 #endif // ASSERT
1991 
1992 
1993 void InterpreterMacroAssembler::store_local_int( Register index, Register src ) {
1994   assert_not_delayed();
1995   sll(index, Interpreter::logStackElementSize, index);
1996   sub(Llocals, index, index);
1997   debug_only(check_for_regarea_stomp(index, 0, FP, G1_scratch, G4_scratch);)
1998   st(src, index, 0);
1999 }
2000 
2001 void InterpreterMacroAssembler::store_local_ptr( Register index, Register src ) {
2002   assert_not_delayed();
2003   sll(index, Interpreter::logStackElementSize, index);
2004   sub(Llocals, index, index);
2005 #ifdef ASSERT
2006   check_for_regarea_stomp(index, 0, FP, G1_scratch, G4_scratch);
2007 #endif
2008   st_ptr(src, index, 0);
2009 }
2010 
2011 
2012 
2013 void InterpreterMacroAssembler::store_local_ptr( int n, Register src ) {
2014   st_ptr(src, Llocals, Interpreter::local_offset_in_bytes(n));
2015 }
2016 
2017 void InterpreterMacroAssembler::store_local_long( Register index, Register src ) {
2018   assert_not_delayed();
2019   sll(index, Interpreter::logStackElementSize, index);
2020   sub(Llocals, index, index);
2021 #ifdef ASSERT
2022   check_for_regarea_stomp(index, Interpreter::local_offset_in_bytes(1), FP, G1_scratch, G4_scratch);
2023 #endif
2024   store_unaligned_long(src, index, Interpreter::local_offset_in_bytes(1)); // which is n+1
2025 }
2026 
2027 
2028 void InterpreterMacroAssembler::store_local_float( Register index, FloatRegister src ) {
2029   assert_not_delayed();
2030   sll(index, Interpreter::logStackElementSize, index);
2031   sub(Llocals, index, index);
2032 #ifdef ASSERT
2033   check_for_regarea_stomp(index, 0, FP, G1_scratch, G4_scratch);
2034 #endif
2035   stf(FloatRegisterImpl::S, src, index, 0);
2036 }
2037 
2038 
2039 void InterpreterMacroAssembler::store_local_double( Register index, FloatRegister src ) {
2040   assert_not_delayed();
2041   sll(index, Interpreter::logStackElementSize, index);
2042   sub(Llocals, index, index);
2043 #ifdef ASSERT
2044   check_for_regarea_stomp(index, Interpreter::local_offset_in_bytes(1), FP, G1_scratch, G4_scratch);
2045 #endif
2046   store_unaligned_double(src, index, Interpreter::local_offset_in_bytes(1));
2047 }
2048 
2049 
2050 int InterpreterMacroAssembler::top_most_monitor_byte_offset() {
2051   const jint delta = frame::interpreter_frame_monitor_size() * wordSize;
2052   int rounded_vm_local_words = ::round_to(frame::interpreter_frame_vm_local_words, WordsPerLong);
2053   return ((-rounded_vm_local_words * wordSize) - delta ) + STACK_BIAS;
2054 }
2055 
2056 
2057 Address InterpreterMacroAssembler::top_most_monitor() {
2058   return Address(FP, top_most_monitor_byte_offset());
2059 }
2060 
2061 
2062 void InterpreterMacroAssembler::compute_stack_base( Register Rdest ) {
2063   add( Lesp,      wordSize,                                    Rdest );
2064 }
2065 
2066 #endif /* CC_INTERP */
2067 
2068 void InterpreterMacroAssembler::increment_invocation_counter( Register Rtmp, Register Rtmp2 ) {
2069   assert(UseCompiler, "incrementing must be useful");
2070 #ifdef CC_INTERP
2071   Address inv_counter(G5_method, methodOopDesc::invocation_counter_offset() +
2072                                  InvocationCounter::counter_offset());
2073   Address be_counter (G5_method, methodOopDesc::backedge_counter_offset() +
2074                                  InvocationCounter::counter_offset());
2075 #else
2076   Address inv_counter(Lmethod, methodOopDesc::invocation_counter_offset() +
2077                                InvocationCounter::counter_offset());
2078   Address be_counter (Lmethod, methodOopDesc::backedge_counter_offset() +
2079                                InvocationCounter::counter_offset());
2080 #endif /* CC_INTERP */
2081   int delta = InvocationCounter::count_increment;
2082 
2083   // Load each counter in a register
2084   ld( inv_counter, Rtmp );
2085   ld( be_counter, Rtmp2 );
2086 
2087   assert( is_simm13( delta ), " delta too large.");
2088 
2089   // Add the delta to the invocation counter and store the result
2090   add( Rtmp, delta, Rtmp );
2091 
2092   // Mask the backedge counter
2093   and3( Rtmp2, InvocationCounter::count_mask_value, Rtmp2 );
2094 
2095   // Store value
2096   st( Rtmp, inv_counter);
2097 
2098   // Add invocation counter + backedge counter
2099   add( Rtmp, Rtmp2, Rtmp);
2100 
2101   // Note that this macro must leave the backedge_count + invocation_count in Rtmp!
2102 }
2103 
2104 void InterpreterMacroAssembler::increment_backedge_counter( Register Rtmp, Register Rtmp2 ) {
2105   assert(UseCompiler, "incrementing must be useful");
2106 #ifdef CC_INTERP
2107   Address be_counter (G5_method, methodOopDesc::backedge_counter_offset() +
2108                                  InvocationCounter::counter_offset());
2109   Address inv_counter(G5_method, methodOopDesc::invocation_counter_offset() +
2110                                  InvocationCounter::counter_offset());
2111 #else
2112   Address be_counter (Lmethod, methodOopDesc::backedge_counter_offset() +
2113                                InvocationCounter::counter_offset());
2114   Address inv_counter(Lmethod, methodOopDesc::invocation_counter_offset() +
2115                                InvocationCounter::counter_offset());
2116 #endif /* CC_INTERP */
2117   int delta = InvocationCounter::count_increment;
2118   // Load each counter in a register
2119   ld( be_counter, Rtmp );
2120   ld( inv_counter, Rtmp2 );
2121 
2122   // Add the delta to the backedge counter
2123   add( Rtmp, delta, Rtmp );
2124 
2125   // Mask the invocation counter, add to backedge counter
2126   and3( Rtmp2, InvocationCounter::count_mask_value, Rtmp2 );
2127 
2128   // and store the result to memory
2129   st( Rtmp, be_counter );
2130 
2131   // Add backedge + invocation counter
2132   add( Rtmp, Rtmp2, Rtmp );
2133 
2134   // Note that this macro must leave backedge_count + invocation_count in Rtmp!
2135 }
2136 
2137 #ifndef CC_INTERP
2138 void InterpreterMacroAssembler::test_backedge_count_for_osr( Register backedge_count,
2139                                                              Register branch_bcp,
2140                                                              Register Rtmp ) {
2141   Label did_not_overflow;
2142   Label overflow_with_error;
2143   assert_different_registers(backedge_count, Rtmp, branch_bcp);
2144   assert(UseOnStackReplacement,"Must UseOnStackReplacement to test_backedge_count_for_osr");
2145 
2146   AddressLiteral limit(&InvocationCounter::InterpreterBackwardBranchLimit);
2147   load_contents(limit, Rtmp);
2148   cmp_and_br_short(backedge_count, Rtmp, Assembler::lessUnsigned, Assembler::pt, did_not_overflow);
2149 
2150   // When ProfileInterpreter is on, the backedge_count comes from the
2151   // methodDataOop, which value does not get reset on the call to
2152   // frequency_counter_overflow().  To avoid excessive calls to the overflow
2153   // routine while the method is being compiled, add a second test to make sure
2154   // the overflow function is called only once every overflow_frequency.
2155   if (ProfileInterpreter) {
2156     const int overflow_frequency = 1024;
2157     andcc(backedge_count, overflow_frequency-1, Rtmp);
2158     brx(Assembler::notZero, false, Assembler::pt, did_not_overflow);
2159     delayed()->nop();
2160   }
2161 
2162   // overflow in loop, pass branch bytecode
2163   set(6,Rtmp);
2164   call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::frequency_counter_overflow), branch_bcp, Rtmp);
2165 
2166   // Was an OSR adapter generated?
2167   // O0 = osr nmethod
2168   br_null_short(O0, Assembler::pn, overflow_with_error);
2169 
2170   // Has the nmethod been invalidated already?
2171   ld(O0, nmethod::entry_bci_offset(), O2);
2172   cmp_and_br_short(O2, InvalidOSREntryBci, Assembler::equal, Assembler::pn, overflow_with_error);
2173 
2174   // migrate the interpreter frame off of the stack
2175 
2176   mov(G2_thread, L7);
2177   // save nmethod
2178   mov(O0, L6);
2179   set_last_Java_frame(SP, noreg);
2180   call_VM_leaf(noreg, CAST_FROM_FN_PTR(address, SharedRuntime::OSR_migration_begin), L7);
2181   reset_last_Java_frame();
2182   mov(L7, G2_thread);
2183 
2184   // move OSR nmethod to I1
2185   mov(L6, I1);
2186 
2187   // OSR buffer to I0
2188   mov(O0, I0);
2189 
2190   // remove the interpreter frame
2191   restore(I5_savedSP, 0, SP);
2192 
2193   // Jump to the osr code.
2194   ld_ptr(O1, nmethod::osr_entry_point_offset(), O2);
2195   jmp(O2, G0);
2196   delayed()->nop();
2197 
2198   bind(overflow_with_error);
2199 
2200   bind(did_not_overflow);
2201 }
2202 
2203 
2204 
2205 void InterpreterMacroAssembler::interp_verify_oop(Register reg, TosState state, const char * file, int line) {
2206   if (state == atos) { MacroAssembler::_verify_oop(reg, "broken oop ", file, line); }
2207 }
2208 
2209 
2210 // local helper function for the verify_oop_or_return_address macro
2211 static bool verify_return_address(methodOopDesc* m, int bci) {
2212 #ifndef PRODUCT
2213   address pc = (address)(m->constMethod())
2214              + in_bytes(constMethodOopDesc::codes_offset()) + bci;
2215   // assume it is a valid return address if it is inside m and is preceded by a jsr
2216   if (!m->contains(pc))                                          return false;
2217   address jsr_pc;
2218   jsr_pc = pc - Bytecodes::length_for(Bytecodes::_jsr);
2219   if (*jsr_pc == Bytecodes::_jsr   && jsr_pc >= m->code_base())    return true;
2220   jsr_pc = pc - Bytecodes::length_for(Bytecodes::_jsr_w);
2221   if (*jsr_pc == Bytecodes::_jsr_w && jsr_pc >= m->code_base())    return true;
2222 #endif // PRODUCT
2223   return false;
2224 }
2225 
2226 
2227 void InterpreterMacroAssembler::verify_oop_or_return_address(Register reg, Register Rtmp) {
2228   if (!VerifyOops)  return;
2229   // the VM documentation for the astore[_wide] bytecode allows
2230   // the TOS to be not only an oop but also a return address
2231   Label test;
2232   Label skip;
2233   // See if it is an address (in the current method):
2234 
2235   mov(reg, Rtmp);
2236   const int log2_bytecode_size_limit = 16;
2237   srl(Rtmp, log2_bytecode_size_limit, Rtmp);
2238   br_notnull_short( Rtmp, pt, test );
2239 
2240   // %%% should use call_VM_leaf here?
2241   save_frame_and_mov(0, Lmethod, O0, reg, O1);
2242   save_thread(L7_thread_cache);
2243   call(CAST_FROM_FN_PTR(address,verify_return_address), relocInfo::none);
2244   delayed()->nop();
2245   restore_thread(L7_thread_cache);
2246   br_notnull( O0, false, pt, skip );
2247   delayed()->restore();
2248 
2249   // Perform a more elaborate out-of-line call
2250   // Not an address; verify it:
2251   bind(test);
2252   verify_oop(reg);
2253   bind(skip);
2254 }
2255 
2256 
2257 void InterpreterMacroAssembler::verify_FPU(int stack_depth, TosState state) {
2258   if (state == ftos || state == dtos) MacroAssembler::verify_FPU(stack_depth);
2259 }
2260 #endif /* CC_INTERP */
2261 
2262 // Inline assembly for:
2263 //
2264 // if (thread is in interp_only_mode) {
2265 //   InterpreterRuntime::post_method_entry();
2266 // }
2267 // if (DTraceMethodProbes) {
2268 //   SharedRuntime::dtrace_method_entry(method, receiver);
2269 // }
2270 // if (RC_TRACE_IN_RANGE(0x00001000, 0x00002000)) {
2271 //   SharedRuntime::rc_trace_method_entry(method, receiver);
2272 // }
2273 
2274 void InterpreterMacroAssembler::notify_method_entry() {
2275 
2276   // C++ interpreter only uses this for native methods.
2277 
2278   // Whenever JVMTI puts a thread in interp_only_mode, method
2279   // entry/exit events are sent for that thread to track stack
2280   // depth.  If it is possible to enter interp_only_mode we add
2281   // the code to check if the event should be sent.
2282   if (JvmtiExport::can_post_interpreter_events()) {
2283     Label L;
2284     Register temp_reg = O5;
2285     const Address interp_only(G2_thread, JavaThread::interp_only_mode_offset());
2286     ld(interp_only, temp_reg);
2287     cmp_and_br_short(temp_reg, 0, equal, pt, L);
2288     call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::post_method_entry));
2289     bind(L);
2290   }
2291 
2292   {
2293     Register temp_reg = O5;
2294     SkipIfEqual skip_if(this, temp_reg, &DTraceMethodProbes, zero);
2295     call_VM_leaf(noreg,
2296       CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_method_entry),
2297       G2_thread, Lmethod);
2298   }
2299 
2300   // RedefineClasses() tracing support for obsolete method entry
2301   if (RC_TRACE_IN_RANGE(0x00001000, 0x00002000)) {
2302     call_VM_leaf(noreg,
2303       CAST_FROM_FN_PTR(address, SharedRuntime::rc_trace_method_entry),
2304       G2_thread, Lmethod);
2305   }
2306 }
2307 
2308 
2309 // Inline assembly for:
2310 //
2311 // if (thread is in interp_only_mode) {
2312 //   // save result
2313 //   InterpreterRuntime::post_method_exit();
2314 //   // restore result
2315 // }
2316 // if (DTraceMethodProbes) {
2317 //   SharedRuntime::dtrace_method_exit(thread, method);
2318 // }
2319 //
2320 // Native methods have their result stored in d_tmp and l_tmp
2321 // Java methods have their result stored in the expression stack
2322 
2323 void InterpreterMacroAssembler::notify_method_exit(bool is_native_method,
2324                                                    TosState state,
2325                                                    NotifyMethodExitMode mode) {
2326   // C++ interpreter only uses this for native methods.
2327 
2328   // Whenever JVMTI puts a thread in interp_only_mode, method
2329   // entry/exit events are sent for that thread to track stack
2330   // depth.  If it is possible to enter interp_only_mode we add
2331   // the code to check if the event should be sent.
2332   if (mode == NotifyJVMTI && JvmtiExport::can_post_interpreter_events()) {
2333     Label L;
2334     Register temp_reg = O5;
2335     const Address interp_only(G2_thread, JavaThread::interp_only_mode_offset());
2336     ld(interp_only, temp_reg);
2337     cmp_and_br_short(temp_reg, 0, equal, pt, L);
2338 
2339     // Note: frame::interpreter_frame_result has a dependency on how the
2340     // method result is saved across the call to post_method_exit. For
2341     // native methods it assumes the result registers are saved to
2342     // l_scratch and d_scratch. If this changes then the interpreter_frame_result
2343     // implementation will need to be updated too.
2344 
2345     save_return_value(state, is_native_method);
2346     call_VM(noreg,
2347             CAST_FROM_FN_PTR(address, InterpreterRuntime::post_method_exit));
2348     restore_return_value(state, is_native_method);
2349     bind(L);
2350   }
2351 
2352   {
2353     Register temp_reg = O5;
2354     // Dtrace notification
2355     SkipIfEqual skip_if(this, temp_reg, &DTraceMethodProbes, zero);
2356     save_return_value(state, is_native_method);
2357     call_VM_leaf(
2358       noreg,
2359       CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_method_exit),
2360       G2_thread, Lmethod);
2361     restore_return_value(state, is_native_method);
2362   }
2363 }
2364 
2365 void InterpreterMacroAssembler::save_return_value(TosState state, bool is_native_call) {
2366 #ifdef CC_INTERP
2367   // result potentially in O0/O1: save it across calls
2368   stf(FloatRegisterImpl::D, F0, STATE(_native_fresult));
2369 #ifdef _LP64
2370   stx(O0, STATE(_native_lresult));
2371 #else
2372   std(O0, STATE(_native_lresult));
2373 #endif
2374 #else // CC_INTERP
2375   if (is_native_call) {
2376     stf(FloatRegisterImpl::D, F0, d_tmp);
2377 #ifdef _LP64
2378     stx(O0, l_tmp);
2379 #else
2380     std(O0, l_tmp);
2381 #endif
2382   } else {
2383     push(state);
2384   }
2385 #endif // CC_INTERP
2386 }
2387 
2388 void InterpreterMacroAssembler::restore_return_value( TosState state, bool is_native_call) {
2389 #ifdef CC_INTERP
2390   ldf(FloatRegisterImpl::D, STATE(_native_fresult), F0);
2391 #ifdef _LP64
2392   ldx(STATE(_native_lresult), O0);
2393 #else
2394   ldd(STATE(_native_lresult), O0);
2395 #endif
2396 #else // CC_INTERP
2397   if (is_native_call) {
2398     ldf(FloatRegisterImpl::D, d_tmp, F0);
2399 #ifdef _LP64
2400     ldx(l_tmp, O0);
2401 #else
2402     ldd(l_tmp, O0);
2403 #endif
2404   } else {
2405     pop(state);
2406   }
2407 #endif // CC_INTERP
2408 }
2409 
2410 // Jump if ((*counter_addr += increment) & mask) satisfies the condition.
2411 void InterpreterMacroAssembler::increment_mask_and_jump(Address counter_addr,
2412                                                         int increment, int mask,
2413                                                         Register scratch1, Register scratch2,
2414                                                         Condition cond, Label *where) {
2415   ld(counter_addr, scratch1);
2416   add(scratch1, increment, scratch1);
2417   if (is_simm13(mask)) {
2418     andcc(scratch1, mask, G0);
2419   } else {
2420     set(mask, scratch2);
2421     andcc(scratch1, scratch2,  G0);
2422   }
2423   br(cond, false, Assembler::pn, *where);
2424   delayed()->st(scratch1, counter_addr);
2425 }