1 /*
   2  * Copyright (c) 2003, 2016, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  *
  23  */
  24 
  25 #include "precompiled.hpp"
  26 #include "asm/macroAssembler.hpp"
  27 #include "asm/macroAssembler.inline.hpp"
  28 #include "interpreter/interpreter.hpp"
  29 #include "nativeInst_x86.hpp"
  30 #include "oops/instanceOop.hpp"
  31 #include "oops/method.hpp"
  32 #include "oops/objArrayKlass.hpp"
  33 #include "oops/oop.inline.hpp"
  34 #include "prims/methodHandles.hpp"
  35 #include "runtime/frame.inline.hpp"
  36 #include "runtime/handles.inline.hpp"
  37 #include "runtime/sharedRuntime.hpp"
  38 #include "runtime/stubCodeGenerator.hpp"
  39 #include "runtime/stubRoutines.hpp"
  40 #include "runtime/thread.inline.hpp"
  41 #ifdef COMPILER2
  42 #include "opto/runtime.hpp"
  43 #endif
  44 
  45 // Declaration and definition of StubGenerator (no .hpp file).
  46 // For a more detailed description of the stub routine structure
  47 // see the comment in stubRoutines.hpp
  48 
  49 #define __ _masm->
  50 #define TIMES_OOP (UseCompressedOops ? Address::times_4 : Address::times_8)
  51 #define a__ ((Assembler*)_masm)->
  52 
  53 #ifdef PRODUCT
  54 #define BLOCK_COMMENT(str) /* nothing */
  55 #else
  56 #define BLOCK_COMMENT(str) __ block_comment(str)
  57 #endif
  58 
  59 #define BIND(label) bind(label); BLOCK_COMMENT(#label ":")
  60 const int MXCSR_MASK = 0xFFC0;  // Mask out any pending exceptions
  61 
  62 // Stub Code definitions
  63 
  64 class StubGenerator: public StubCodeGenerator {
  65  private:
  66 
  67 #ifdef PRODUCT
  68 #define inc_counter_np(counter) ((void)0)
  69 #else
  70   void inc_counter_np_(int& counter) {
  71     // This can destroy rscratch1 if counter is far from the code cache
  72     __ incrementl(ExternalAddress((address)&counter));
  73   }
  74 #define inc_counter_np(counter) \
  75   BLOCK_COMMENT("inc_counter " #counter); \
  76   inc_counter_np_(counter);
  77 #endif
  78 
  79   // Call stubs are used to call Java from C
  80   //
  81   // Linux Arguments:
  82   //    c_rarg0:   call wrapper address                   address
  83   //    c_rarg1:   result                                 address
  84   //    c_rarg2:   result type                            BasicType
  85   //    c_rarg3:   method                                 Method*
  86   //    c_rarg4:   (interpreter) entry point              address
  87   //    c_rarg5:   parameters                             intptr_t*
  88   //    16(rbp): parameter size (in words)              int
  89   //    24(rbp): thread                                 Thread*
  90   //
  91   //     [ return_from_Java     ] <--- rsp
  92   //     [ argument word n      ]
  93   //      ...
  94   // -12 [ argument word 1      ]
  95   // -11 [ saved r15            ] <--- rsp_after_call
  96   // -10 [ saved r14            ]
  97   //  -9 [ saved r13            ]
  98   //  -8 [ saved r12            ]
  99   //  -7 [ saved rbx            ]
 100   //  -6 [ call wrapper         ]
 101   //  -5 [ result               ]
 102   //  -4 [ result type          ]
 103   //  -3 [ method               ]
 104   //  -2 [ entry point          ]
 105   //  -1 [ parameters           ]
 106   //   0 [ saved rbp            ] <--- rbp
 107   //   1 [ return address       ]
 108   //   2 [ parameter size       ]
 109   //   3 [ thread               ]
 110   //
 111   // Windows Arguments:
 112   //    c_rarg0:   call wrapper address                   address
 113   //    c_rarg1:   result                                 address
 114   //    c_rarg2:   result type                            BasicType
 115   //    c_rarg3:   method                                 Method*
 116   //    48(rbp): (interpreter) entry point              address
 117   //    56(rbp): parameters                             intptr_t*
 118   //    64(rbp): parameter size (in words)              int
 119   //    72(rbp): thread                                 Thread*
 120   //
 121   //     [ return_from_Java     ] <--- rsp
 122   //     [ argument word n      ]
 123   //      ...
 124   // -60 [ argument word 1      ]
 125   // -59 [ saved xmm31          ] <--- rsp after_call
 126   //     [ saved xmm16-xmm30    ] (EVEX enabled, else the space is blank)
 127   // -27 [ saved xmm15          ]
 128   //     [ saved xmm7-xmm14     ]
 129   //  -9 [ saved xmm6           ] (each xmm register takes 2 slots)
 130   //  -7 [ saved r15            ]
 131   //  -6 [ saved r14            ]
 132   //  -5 [ saved r13            ]
 133   //  -4 [ saved r12            ]
 134   //  -3 [ saved rdi            ]
 135   //  -2 [ saved rsi            ]
 136   //  -1 [ saved rbx            ]
 137   //   0 [ saved rbp            ] <--- rbp
 138   //   1 [ return address       ]
 139   //   2 [ call wrapper         ]
 140   //   3 [ result               ]
 141   //   4 [ result type          ]
 142   //   5 [ method               ]
 143   //   6 [ entry point          ]
 144   //   7 [ parameters           ]
 145   //   8 [ parameter size       ]
 146   //   9 [ thread               ]
 147   //
 148   //    Windows reserves the callers stack space for arguments 1-4.
 149   //    We spill c_rarg0-c_rarg3 to this space.
 150 
 151   // Call stub stack layout word offsets from rbp
 152   enum call_stub_layout {
 153 #ifdef _WIN64
 154     xmm_save_first     = 6,  // save from xmm6
 155     xmm_save_last      = 31, // to xmm31
 156     xmm_save_base      = -9,
 157     rsp_after_call_off = xmm_save_base - 2 * (xmm_save_last - xmm_save_first), // -27
 158     r15_off            = -7,
 159     r14_off            = -6,
 160     r13_off            = -5,
 161     r12_off            = -4,
 162     rdi_off            = -3,
 163     rsi_off            = -2,
 164     rbx_off            = -1,
 165     rbp_off            =  0,
 166     retaddr_off        =  1,
 167     call_wrapper_off   =  2,
 168     result_off         =  3,
 169     result_type_off    =  4,
 170     method_off         =  5,
 171     entry_point_off    =  6,
 172     parameters_off     =  7,
 173     parameter_size_off =  8,
 174     thread_off         =  9
 175 #else
 176     rsp_after_call_off = -12,
 177     mxcsr_off          = rsp_after_call_off,
 178     r15_off            = -11,
 179     r14_off            = -10,
 180     r13_off            = -9,
 181     r12_off            = -8,
 182     rbx_off            = -7,
 183     call_wrapper_off   = -6,
 184     result_off         = -5,
 185     result_type_off    = -4,
 186     method_off         = -3,
 187     entry_point_off    = -2,
 188     parameters_off     = -1,
 189     rbp_off            =  0,
 190     retaddr_off        =  1,
 191     parameter_size_off =  2,
 192     thread_off         =  3
 193 #endif
 194   };
 195 
 196 #ifdef _WIN64
 197   Address xmm_save(int reg) {
 198     assert(reg >= xmm_save_first && reg <= xmm_save_last, "XMM register number out of range");
 199     return Address(rbp, (xmm_save_base - (reg - xmm_save_first) * 2) * wordSize);
 200   }
 201 #endif
 202 
 203   address generate_call_stub(address& return_address) {
 204     assert((int)frame::entry_frame_after_call_words == -(int)rsp_after_call_off + 1 &&
 205            (int)frame::entry_frame_call_wrapper_offset == (int)call_wrapper_off,
 206            "adjust this code");
 207     StubCodeMark mark(this, "StubRoutines", "call_stub");
 208     address start = __ pc();
 209 
 210     // same as in generate_catch_exception()!
 211     const Address rsp_after_call(rbp, rsp_after_call_off * wordSize);
 212 
 213     const Address call_wrapper  (rbp, call_wrapper_off   * wordSize);
 214     const Address result        (rbp, result_off         * wordSize);
 215     const Address result_type   (rbp, result_type_off    * wordSize);
 216     const Address method        (rbp, method_off         * wordSize);
 217     const Address entry_point   (rbp, entry_point_off    * wordSize);
 218     const Address parameters    (rbp, parameters_off     * wordSize);
 219     const Address parameter_size(rbp, parameter_size_off * wordSize);
 220 
 221     // same as in generate_catch_exception()!
 222     const Address thread        (rbp, thread_off         * wordSize);
 223 
 224     const Address r15_save(rbp, r15_off * wordSize);
 225     const Address r14_save(rbp, r14_off * wordSize);
 226     const Address r13_save(rbp, r13_off * wordSize);
 227     const Address r12_save(rbp, r12_off * wordSize);
 228     const Address rbx_save(rbp, rbx_off * wordSize);
 229 
 230     // stub code
 231     __ enter();
 232     __ subptr(rsp, -rsp_after_call_off * wordSize);
 233 
 234     // save register parameters
 235 #ifndef _WIN64
 236     __ movptr(parameters,   c_rarg5); // parameters
 237     __ movptr(entry_point,  c_rarg4); // entry_point
 238 #endif
 239 
 240     __ movptr(method,       c_rarg3); // method
 241     __ movl(result_type,  c_rarg2);   // result type
 242     __ movptr(result,       c_rarg1); // result
 243     __ movptr(call_wrapper, c_rarg0); // call wrapper
 244 
 245     // save regs belonging to calling function
 246     __ movptr(rbx_save, rbx);
 247     __ movptr(r12_save, r12);
 248     __ movptr(r13_save, r13);
 249     __ movptr(r14_save, r14);
 250     __ movptr(r15_save, r15);
 251     if (UseAVX > 2) {
 252       __ movl(rbx, 0xffff);
 253       __ kmovwl(k1, rbx);
 254     }
 255 #ifdef _WIN64
 256     int last_reg = 15;
 257     if (UseAVX > 2) {
 258       last_reg = 31;
 259     }
 260     if (VM_Version::supports_evex()) {
 261       for (int i = xmm_save_first; i <= last_reg; i++) {
 262         __ vextractf32x4(xmm_save(i), as_XMMRegister(i), 0);
 263       }
 264     } else {
 265       for (int i = xmm_save_first; i <= last_reg; i++) {
 266         __ movdqu(xmm_save(i), as_XMMRegister(i));
 267       }
 268     }
 269 
 270     const Address rdi_save(rbp, rdi_off * wordSize);
 271     const Address rsi_save(rbp, rsi_off * wordSize);
 272 
 273     __ movptr(rsi_save, rsi);
 274     __ movptr(rdi_save, rdi);
 275 #else
 276     const Address mxcsr_save(rbp, mxcsr_off * wordSize);
 277     {
 278       Label skip_ldmx;
 279       __ stmxcsr(mxcsr_save);
 280       __ movl(rax, mxcsr_save);
 281       __ andl(rax, MXCSR_MASK);    // Only check control and mask bits
 282       ExternalAddress mxcsr_std(StubRoutines::addr_mxcsr_std());
 283       __ cmp32(rax, mxcsr_std);
 284       __ jcc(Assembler::equal, skip_ldmx);
 285       __ ldmxcsr(mxcsr_std);
 286       __ bind(skip_ldmx);
 287     }
 288 #endif
 289 
 290     // Load up thread register
 291     __ movptr(r15_thread, thread);
 292     __ reinit_heapbase();
 293 
 294 #ifdef ASSERT
 295     // make sure we have no pending exceptions
 296     {
 297       Label L;
 298       __ cmpptr(Address(r15_thread, Thread::pending_exception_offset()), (int32_t)NULL_WORD);
 299       __ jcc(Assembler::equal, L);
 300       __ stop("StubRoutines::call_stub: entered with pending exception");
 301       __ bind(L);
 302     }
 303 #endif
 304 
 305     // pass parameters if any
 306     BLOCK_COMMENT("pass parameters if any");
 307     Label parameters_done;
 308     __ movl(c_rarg3, parameter_size);
 309     __ testl(c_rarg3, c_rarg3);
 310     __ jcc(Assembler::zero, parameters_done);
 311 
 312     Label loop;
 313     __ movptr(c_rarg2, parameters);       // parameter pointer
 314     __ movl(c_rarg1, c_rarg3);            // parameter counter is in c_rarg1
 315     __ BIND(loop);
 316     __ movptr(rax, Address(c_rarg2, 0));// get parameter
 317     __ addptr(c_rarg2, wordSize);       // advance to next parameter
 318     __ decrementl(c_rarg1);             // decrement counter
 319     __ push(rax);                       // pass parameter
 320     __ jcc(Assembler::notZero, loop);
 321 
 322     // call Java function
 323     __ BIND(parameters_done);
 324     __ movptr(rbx, method);             // get Method*
 325     __ movptr(c_rarg1, entry_point);    // get entry_point
 326     __ mov(r13, rsp);                   // set sender sp
 327     BLOCK_COMMENT("call Java function");
 328     __ call(c_rarg1);
 329 
 330     BLOCK_COMMENT("call_stub_return_address:");
 331     return_address = __ pc();
 332 
 333     // store result depending on type (everything that is not
 334     // T_OBJECT, T_LONG, T_FLOAT or T_DOUBLE is treated as T_INT)
 335     __ movptr(c_rarg0, result);
 336     Label is_long, is_float, is_double, exit;
 337     __ movl(c_rarg1, result_type);
 338     __ cmpl(c_rarg1, T_OBJECT);
 339     __ jcc(Assembler::equal, is_long);
 340     __ cmpl(c_rarg1, T_LONG);
 341     __ jcc(Assembler::equal, is_long);
 342     __ cmpl(c_rarg1, T_FLOAT);
 343     __ jcc(Assembler::equal, is_float);
 344     __ cmpl(c_rarg1, T_DOUBLE);
 345     __ jcc(Assembler::equal, is_double);
 346 
 347     // handle T_INT case
 348     __ movl(Address(c_rarg0, 0), rax);
 349 
 350     __ BIND(exit);
 351 
 352     // pop parameters
 353     __ lea(rsp, rsp_after_call);
 354 
 355 #ifdef ASSERT
 356     // verify that threads correspond
 357     {
 358      Label L1, L2, L3;
 359       __ cmpptr(r15_thread, thread);
 360       __ jcc(Assembler::equal, L1);
 361       __ stop("StubRoutines::call_stub: r15_thread is corrupted");
 362       __ bind(L1);
 363       __ get_thread(rbx);
 364       __ cmpptr(r15_thread, thread);
 365       __ jcc(Assembler::equal, L2);
 366       __ stop("StubRoutines::call_stub: r15_thread is modified by call");
 367       __ bind(L2);
 368       __ cmpptr(r15_thread, rbx);
 369       __ jcc(Assembler::equal, L3);
 370       __ stop("StubRoutines::call_stub: threads must correspond");
 371       __ bind(L3);
 372     }
 373 #endif
 374 
 375     // restore regs belonging to calling function
 376 #ifdef _WIN64
 377     // emit the restores for xmm regs
 378     if (VM_Version::supports_evex()) {
 379       for (int i = xmm_save_first; i <= last_reg; i++) {
 380         __ vinsertf32x4(as_XMMRegister(i), as_XMMRegister(i), xmm_save(i), 0);
 381       }
 382     } else {
 383       for (int i = xmm_save_first; i <= last_reg; i++) {
 384         __ movdqu(as_XMMRegister(i), xmm_save(i));
 385       }
 386     }
 387 #endif
 388     __ movptr(r15, r15_save);
 389     __ movptr(r14, r14_save);
 390     __ movptr(r13, r13_save);
 391     __ movptr(r12, r12_save);
 392     __ movptr(rbx, rbx_save);
 393 
 394 #ifdef _WIN64
 395     __ movptr(rdi, rdi_save);
 396     __ movptr(rsi, rsi_save);
 397 #else
 398     __ ldmxcsr(mxcsr_save);
 399 #endif
 400 
 401     // restore rsp
 402     __ addptr(rsp, -rsp_after_call_off * wordSize);
 403 
 404     // return
 405     __ pop(rbp);
 406     __ ret(0);
 407 
 408     // handle return types different from T_INT
 409     __ BIND(is_long);
 410     __ movq(Address(c_rarg0, 0), rax);
 411     __ jmp(exit);
 412 
 413     __ BIND(is_float);
 414     __ movflt(Address(c_rarg0, 0), xmm0);
 415     __ jmp(exit);
 416 
 417     __ BIND(is_double);
 418     __ movdbl(Address(c_rarg0, 0), xmm0);
 419     __ jmp(exit);
 420 
 421     return start;
 422   }
 423 
 424   // Return point for a Java call if there's an exception thrown in
 425   // Java code.  The exception is caught and transformed into a
 426   // pending exception stored in JavaThread that can be tested from
 427   // within the VM.
 428   //
 429   // Note: Usually the parameters are removed by the callee. In case
 430   // of an exception crossing an activation frame boundary, that is
 431   // not the case if the callee is compiled code => need to setup the
 432   // rsp.
 433   //
 434   // rax: exception oop
 435 
 436   address generate_catch_exception() {
 437     StubCodeMark mark(this, "StubRoutines", "catch_exception");
 438     address start = __ pc();
 439 
 440     // same as in generate_call_stub():
 441     const Address rsp_after_call(rbp, rsp_after_call_off * wordSize);
 442     const Address thread        (rbp, thread_off         * wordSize);
 443 
 444 #ifdef ASSERT
 445     // verify that threads correspond
 446     {
 447       Label L1, L2, L3;
 448       __ cmpptr(r15_thread, thread);
 449       __ jcc(Assembler::equal, L1);
 450       __ stop("StubRoutines::catch_exception: r15_thread is corrupted");
 451       __ bind(L1);
 452       __ get_thread(rbx);
 453       __ cmpptr(r15_thread, thread);
 454       __ jcc(Assembler::equal, L2);
 455       __ stop("StubRoutines::catch_exception: r15_thread is modified by call");
 456       __ bind(L2);
 457       __ cmpptr(r15_thread, rbx);
 458       __ jcc(Assembler::equal, L3);
 459       __ stop("StubRoutines::catch_exception: threads must correspond");
 460       __ bind(L3);
 461     }
 462 #endif
 463 
 464     // set pending exception
 465     __ verify_oop(rax);
 466 
 467     __ movptr(Address(r15_thread, Thread::pending_exception_offset()), rax);
 468     __ lea(rscratch1, ExternalAddress((address)__FILE__));
 469     __ movptr(Address(r15_thread, Thread::exception_file_offset()), rscratch1);
 470     __ movl(Address(r15_thread, Thread::exception_line_offset()), (int)  __LINE__);
 471 
 472     // complete return to VM
 473     assert(StubRoutines::_call_stub_return_address != NULL,
 474            "_call_stub_return_address must have been generated before");
 475     __ jump(RuntimeAddress(StubRoutines::_call_stub_return_address));
 476 
 477     return start;
 478   }
 479 
 480   // Continuation point for runtime calls returning with a pending
 481   // exception.  The pending exception check happened in the runtime
 482   // or native call stub.  The pending exception in Thread is
 483   // converted into a Java-level exception.
 484   //
 485   // Contract with Java-level exception handlers:
 486   // rax: exception
 487   // rdx: throwing pc
 488   //
 489   // NOTE: At entry of this stub, exception-pc must be on stack !!
 490 
 491   address generate_forward_exception() {
 492     StubCodeMark mark(this, "StubRoutines", "forward exception");
 493     address start = __ pc();
 494 
 495     // Upon entry, the sp points to the return address returning into
 496     // Java (interpreted or compiled) code; i.e., the return address
 497     // becomes the throwing pc.
 498     //
 499     // Arguments pushed before the runtime call are still on the stack
 500     // but the exception handler will reset the stack pointer ->
 501     // ignore them.  A potential result in registers can be ignored as
 502     // well.
 503 
 504 #ifdef ASSERT
 505     // make sure this code is only executed if there is a pending exception
 506     {
 507       Label L;
 508       __ cmpptr(Address(r15_thread, Thread::pending_exception_offset()), (int32_t) NULL);
 509       __ jcc(Assembler::notEqual, L);
 510       __ stop("StubRoutines::forward exception: no pending exception (1)");
 511       __ bind(L);
 512     }
 513 #endif
 514 
 515     // compute exception handler into rbx
 516     __ movptr(c_rarg0, Address(rsp, 0));
 517     BLOCK_COMMENT("call exception_handler_for_return_address");
 518     __ call_VM_leaf(CAST_FROM_FN_PTR(address,
 519                          SharedRuntime::exception_handler_for_return_address),
 520                     r15_thread, c_rarg0);
 521     __ mov(rbx, rax);
 522 
 523     // setup rax & rdx, remove return address & clear pending exception
 524     __ pop(rdx);
 525     __ movptr(rax, Address(r15_thread, Thread::pending_exception_offset()));
 526     __ movptr(Address(r15_thread, Thread::pending_exception_offset()), (int32_t)NULL_WORD);
 527 
 528 #ifdef ASSERT
 529     // make sure exception is set
 530     {
 531       Label L;
 532       __ testptr(rax, rax);
 533       __ jcc(Assembler::notEqual, L);
 534       __ stop("StubRoutines::forward exception: no pending exception (2)");
 535       __ bind(L);
 536     }
 537 #endif
 538 
 539     // continue at exception handler (return address removed)
 540     // rax: exception
 541     // rbx: exception handler
 542     // rdx: throwing pc
 543     __ verify_oop(rax);
 544     __ jmp(rbx);
 545 
 546     return start;
 547   }
 548 
 549   // Support for jint atomic::xchg(jint exchange_value, volatile jint* dest)
 550   //
 551   // Arguments :
 552   //    c_rarg0: exchange_value
 553   //    c_rarg0: dest
 554   //
 555   // Result:
 556   //    *dest <- ex, return (orig *dest)
 557   address generate_atomic_xchg() {
 558     StubCodeMark mark(this, "StubRoutines", "atomic_xchg");
 559     address start = __ pc();
 560 
 561     __ movl(rax, c_rarg0); // Copy to eax we need a return value anyhow
 562     __ xchgl(rax, Address(c_rarg1, 0)); // automatic LOCK
 563     __ ret(0);
 564 
 565     return start;
 566   }
 567 
 568   // Support for intptr_t atomic::xchg_ptr(intptr_t exchange_value, volatile intptr_t* dest)
 569   //
 570   // Arguments :
 571   //    c_rarg0: exchange_value
 572   //    c_rarg1: dest
 573   //
 574   // Result:
 575   //    *dest <- ex, return (orig *dest)
 576   address generate_atomic_xchg_ptr() {
 577     StubCodeMark mark(this, "StubRoutines", "atomic_xchg_ptr");
 578     address start = __ pc();
 579 
 580     __ movptr(rax, c_rarg0); // Copy to eax we need a return value anyhow
 581     __ xchgptr(rax, Address(c_rarg1, 0)); // automatic LOCK
 582     __ ret(0);
 583 
 584     return start;
 585   }
 586 
 587   // Support for jint atomic::atomic_cmpxchg(jint exchange_value, volatile jint* dest,
 588   //                                         jint compare_value)
 589   //
 590   // Arguments :
 591   //    c_rarg0: exchange_value
 592   //    c_rarg1: dest
 593   //    c_rarg2: compare_value
 594   //
 595   // Result:
 596   //    if ( compare_value == *dest ) {
 597   //       *dest = exchange_value
 598   //       return compare_value;
 599   //    else
 600   //       return *dest;
 601   address generate_atomic_cmpxchg() {
 602     StubCodeMark mark(this, "StubRoutines", "atomic_cmpxchg");
 603     address start = __ pc();
 604 
 605     __ movl(rax, c_rarg2);
 606    if ( os::is_MP() ) __ lock();
 607     __ cmpxchgl(c_rarg0, Address(c_rarg1, 0));
 608     __ ret(0);
 609 
 610     return start;
 611   }
 612 
 613   // Support for jbyte atomic::atomic_cmpxchg(jbyte exchange_value, volatile jbyte* dest,
 614   //                                          jbyte compare_value)
 615   //
 616   // Arguments :
 617   //    c_rarg0: exchange_value
 618   //    c_rarg1: dest
 619   //    c_rarg2: compare_value
 620   //
 621   // Result:
 622   //    if ( compare_value == *dest ) {
 623   //       *dest = exchange_value
 624   //       return compare_value;
 625   //    else
 626   //       return *dest;
 627   address generate_atomic_cmpxchg_byte() {
 628     StubCodeMark mark(this, "StubRoutines", "atomic_cmpxchg_byte");
 629     address start = __ pc();
 630 
 631     __ movsbq(rax, c_rarg2);
 632    if ( os::is_MP() ) __ lock();
 633     __ cmpxchgb(c_rarg0, Address(c_rarg1, 0));
 634     __ ret(0);
 635 
 636     return start;
 637   }
 638 
 639   // Support for jlong atomic::atomic_cmpxchg(jlong exchange_value,
 640   //                                          volatile jlong* dest,
 641   //                                          jlong compare_value)
 642   // Arguments :
 643   //    c_rarg0: exchange_value
 644   //    c_rarg1: dest
 645   //    c_rarg2: compare_value
 646   //
 647   // Result:
 648   //    if ( compare_value == *dest ) {
 649   //       *dest = exchange_value
 650   //       return compare_value;
 651   //    else
 652   //       return *dest;
 653   address generate_atomic_cmpxchg_long() {
 654     StubCodeMark mark(this, "StubRoutines", "atomic_cmpxchg_long");
 655     address start = __ pc();
 656 
 657     __ movq(rax, c_rarg2);
 658    if ( os::is_MP() ) __ lock();
 659     __ cmpxchgq(c_rarg0, Address(c_rarg1, 0));
 660     __ ret(0);
 661 
 662     return start;
 663   }
 664 
 665   // Support for jint atomic::add(jint add_value, volatile jint* dest)
 666   //
 667   // Arguments :
 668   //    c_rarg0: add_value
 669   //    c_rarg1: dest
 670   //
 671   // Result:
 672   //    *dest += add_value
 673   //    return *dest;
 674   address generate_atomic_add() {
 675     StubCodeMark mark(this, "StubRoutines", "atomic_add");
 676     address start = __ pc();
 677 
 678     __ movl(rax, c_rarg0);
 679    if ( os::is_MP() ) __ lock();
 680     __ xaddl(Address(c_rarg1, 0), c_rarg0);
 681     __ addl(rax, c_rarg0);
 682     __ ret(0);
 683 
 684     return start;
 685   }
 686 
 687   // Support for intptr_t atomic::add_ptr(intptr_t add_value, volatile intptr_t* dest)
 688   //
 689   // Arguments :
 690   //    c_rarg0: add_value
 691   //    c_rarg1: dest
 692   //
 693   // Result:
 694   //    *dest += add_value
 695   //    return *dest;
 696   address generate_atomic_add_ptr() {
 697     StubCodeMark mark(this, "StubRoutines", "atomic_add_ptr");
 698     address start = __ pc();
 699 
 700     __ movptr(rax, c_rarg0); // Copy to eax we need a return value anyhow
 701    if ( os::is_MP() ) __ lock();
 702     __ xaddptr(Address(c_rarg1, 0), c_rarg0);
 703     __ addptr(rax, c_rarg0);
 704     __ ret(0);
 705 
 706     return start;
 707   }
 708 
 709   // Support for intptr_t OrderAccess::fence()
 710   //
 711   // Arguments :
 712   //
 713   // Result:
 714   address generate_orderaccess_fence() {
 715     StubCodeMark mark(this, "StubRoutines", "orderaccess_fence");
 716     address start = __ pc();
 717     __ membar(Assembler::StoreLoad);
 718     __ ret(0);
 719 
 720     return start;
 721   }
 722 
 723   // Support for intptr_t get_previous_fp()
 724   //
 725   // This routine is used to find the previous frame pointer for the
 726   // caller (current_frame_guess). This is used as part of debugging
 727   // ps() is seemingly lost trying to find frames.
 728   // This code assumes that caller current_frame_guess) has a frame.
 729   address generate_get_previous_fp() {
 730     StubCodeMark mark(this, "StubRoutines", "get_previous_fp");
 731     const Address old_fp(rbp, 0);
 732     const Address older_fp(rax, 0);
 733     address start = __ pc();
 734 
 735     __ enter();
 736     __ movptr(rax, old_fp); // callers fp
 737     __ movptr(rax, older_fp); // the frame for ps()
 738     __ pop(rbp);
 739     __ ret(0);
 740 
 741     return start;
 742   }
 743 
 744   // Support for intptr_t get_previous_sp()
 745   //
 746   // This routine is used to find the previous stack pointer for the
 747   // caller.
 748   address generate_get_previous_sp() {
 749     StubCodeMark mark(this, "StubRoutines", "get_previous_sp");
 750     address start = __ pc();
 751 
 752     __ movptr(rax, rsp);
 753     __ addptr(rax, 8); // return address is at the top of the stack.
 754     __ ret(0);
 755 
 756     return start;
 757   }
 758 
 759   //----------------------------------------------------------------------------------------------------
 760   // Support for void verify_mxcsr()
 761   //
 762   // This routine is used with -Xcheck:jni to verify that native
 763   // JNI code does not return to Java code without restoring the
 764   // MXCSR register to our expected state.
 765 
 766   address generate_verify_mxcsr() {
 767     StubCodeMark mark(this, "StubRoutines", "verify_mxcsr");
 768     address start = __ pc();
 769 
 770     const Address mxcsr_save(rsp, 0);
 771 
 772     if (CheckJNICalls) {
 773       Label ok_ret;
 774       ExternalAddress mxcsr_std(StubRoutines::addr_mxcsr_std());
 775       __ push(rax);
 776       __ subptr(rsp, wordSize);      // allocate a temp location
 777       __ stmxcsr(mxcsr_save);
 778       __ movl(rax, mxcsr_save);
 779       __ andl(rax, MXCSR_MASK);    // Only check control and mask bits
 780       __ cmp32(rax, mxcsr_std);
 781       __ jcc(Assembler::equal, ok_ret);
 782 
 783       __ warn("MXCSR changed by native JNI code, use -XX:+RestoreMXCSROnJNICall");
 784 
 785       __ ldmxcsr(mxcsr_std);
 786 
 787       __ bind(ok_ret);
 788       __ addptr(rsp, wordSize);
 789       __ pop(rax);
 790     }
 791 
 792     __ ret(0);
 793 
 794     return start;
 795   }
 796 
 797   address generate_f2i_fixup() {
 798     StubCodeMark mark(this, "StubRoutines", "f2i_fixup");
 799     Address inout(rsp, 5 * wordSize); // return address + 4 saves
 800 
 801     address start = __ pc();
 802 
 803     Label L;
 804 
 805     __ push(rax);
 806     __ push(c_rarg3);
 807     __ push(c_rarg2);
 808     __ push(c_rarg1);
 809 
 810     __ movl(rax, 0x7f800000);
 811     __ xorl(c_rarg3, c_rarg3);
 812     __ movl(c_rarg2, inout);
 813     __ movl(c_rarg1, c_rarg2);
 814     __ andl(c_rarg1, 0x7fffffff);
 815     __ cmpl(rax, c_rarg1); // NaN? -> 0
 816     __ jcc(Assembler::negative, L);
 817     __ testl(c_rarg2, c_rarg2); // signed ? min_jint : max_jint
 818     __ movl(c_rarg3, 0x80000000);
 819     __ movl(rax, 0x7fffffff);
 820     __ cmovl(Assembler::positive, c_rarg3, rax);
 821 
 822     __ bind(L);
 823     __ movptr(inout, c_rarg3);
 824 
 825     __ pop(c_rarg1);
 826     __ pop(c_rarg2);
 827     __ pop(c_rarg3);
 828     __ pop(rax);
 829 
 830     __ ret(0);
 831 
 832     return start;
 833   }
 834 
 835   address generate_f2l_fixup() {
 836     StubCodeMark mark(this, "StubRoutines", "f2l_fixup");
 837     Address inout(rsp, 5 * wordSize); // return address + 4 saves
 838     address start = __ pc();
 839 
 840     Label L;
 841 
 842     __ push(rax);
 843     __ push(c_rarg3);
 844     __ push(c_rarg2);
 845     __ push(c_rarg1);
 846 
 847     __ movl(rax, 0x7f800000);
 848     __ xorl(c_rarg3, c_rarg3);
 849     __ movl(c_rarg2, inout);
 850     __ movl(c_rarg1, c_rarg2);
 851     __ andl(c_rarg1, 0x7fffffff);
 852     __ cmpl(rax, c_rarg1); // NaN? -> 0
 853     __ jcc(Assembler::negative, L);
 854     __ testl(c_rarg2, c_rarg2); // signed ? min_jlong : max_jlong
 855     __ mov64(c_rarg3, 0x8000000000000000);
 856     __ mov64(rax, 0x7fffffffffffffff);
 857     __ cmov(Assembler::positive, c_rarg3, rax);
 858 
 859     __ bind(L);
 860     __ movptr(inout, c_rarg3);
 861 
 862     __ pop(c_rarg1);
 863     __ pop(c_rarg2);
 864     __ pop(c_rarg3);
 865     __ pop(rax);
 866 
 867     __ ret(0);
 868 
 869     return start;
 870   }
 871 
 872   address generate_d2i_fixup() {
 873     StubCodeMark mark(this, "StubRoutines", "d2i_fixup");
 874     Address inout(rsp, 6 * wordSize); // return address + 5 saves
 875 
 876     address start = __ pc();
 877 
 878     Label L;
 879 
 880     __ push(rax);
 881     __ push(c_rarg3);
 882     __ push(c_rarg2);
 883     __ push(c_rarg1);
 884     __ push(c_rarg0);
 885 
 886     __ movl(rax, 0x7ff00000);
 887     __ movq(c_rarg2, inout);
 888     __ movl(c_rarg3, c_rarg2);
 889     __ mov(c_rarg1, c_rarg2);
 890     __ mov(c_rarg0, c_rarg2);
 891     __ negl(c_rarg3);
 892     __ shrptr(c_rarg1, 0x20);
 893     __ orl(c_rarg3, c_rarg2);
 894     __ andl(c_rarg1, 0x7fffffff);
 895     __ xorl(c_rarg2, c_rarg2);
 896     __ shrl(c_rarg3, 0x1f);
 897     __ orl(c_rarg1, c_rarg3);
 898     __ cmpl(rax, c_rarg1);
 899     __ jcc(Assembler::negative, L); // NaN -> 0
 900     __ testptr(c_rarg0, c_rarg0); // signed ? min_jint : max_jint
 901     __ movl(c_rarg2, 0x80000000);
 902     __ movl(rax, 0x7fffffff);
 903     __ cmov(Assembler::positive, c_rarg2, rax);
 904 
 905     __ bind(L);
 906     __ movptr(inout, c_rarg2);
 907 
 908     __ pop(c_rarg0);
 909     __ pop(c_rarg1);
 910     __ pop(c_rarg2);
 911     __ pop(c_rarg3);
 912     __ pop(rax);
 913 
 914     __ ret(0);
 915 
 916     return start;
 917   }
 918 
 919   address generate_d2l_fixup() {
 920     StubCodeMark mark(this, "StubRoutines", "d2l_fixup");
 921     Address inout(rsp, 6 * wordSize); // return address + 5 saves
 922 
 923     address start = __ pc();
 924 
 925     Label L;
 926 
 927     __ push(rax);
 928     __ push(c_rarg3);
 929     __ push(c_rarg2);
 930     __ push(c_rarg1);
 931     __ push(c_rarg0);
 932 
 933     __ movl(rax, 0x7ff00000);
 934     __ movq(c_rarg2, inout);
 935     __ movl(c_rarg3, c_rarg2);
 936     __ mov(c_rarg1, c_rarg2);
 937     __ mov(c_rarg0, c_rarg2);
 938     __ negl(c_rarg3);
 939     __ shrptr(c_rarg1, 0x20);
 940     __ orl(c_rarg3, c_rarg2);
 941     __ andl(c_rarg1, 0x7fffffff);
 942     __ xorl(c_rarg2, c_rarg2);
 943     __ shrl(c_rarg3, 0x1f);
 944     __ orl(c_rarg1, c_rarg3);
 945     __ cmpl(rax, c_rarg1);
 946     __ jcc(Assembler::negative, L); // NaN -> 0
 947     __ testq(c_rarg0, c_rarg0); // signed ? min_jlong : max_jlong
 948     __ mov64(c_rarg2, 0x8000000000000000);
 949     __ mov64(rax, 0x7fffffffffffffff);
 950     __ cmovq(Assembler::positive, c_rarg2, rax);
 951 
 952     __ bind(L);
 953     __ movq(inout, c_rarg2);
 954 
 955     __ pop(c_rarg0);
 956     __ pop(c_rarg1);
 957     __ pop(c_rarg2);
 958     __ pop(c_rarg3);
 959     __ pop(rax);
 960 
 961     __ ret(0);
 962 
 963     return start;
 964   }
 965 
 966   address generate_fp_mask(const char *stub_name, int64_t mask) {
 967     __ align(CodeEntryAlignment);
 968     StubCodeMark mark(this, "StubRoutines", stub_name);
 969     address start = __ pc();
 970 
 971     __ emit_data64( mask, relocInfo::none );
 972     __ emit_data64( mask, relocInfo::none );
 973 
 974     return start;
 975   }
 976 
 977   // Non-destructive plausibility checks for oops
 978   //
 979   // Arguments:
 980   //    all args on stack!
 981   //
 982   // Stack after saving c_rarg3:
 983   //    [tos + 0]: saved c_rarg3
 984   //    [tos + 1]: saved c_rarg2
 985   //    [tos + 2]: saved r12 (several TemplateTable methods use it)
 986   //    [tos + 3]: saved flags
 987   //    [tos + 4]: return address
 988   //  * [tos + 5]: error message (char*)
 989   //  * [tos + 6]: object to verify (oop)
 990   //  * [tos + 7]: saved rax - saved by caller and bashed
 991   //  * [tos + 8]: saved r10 (rscratch1) - saved by caller
 992   //  * = popped on exit
 993   address generate_verify_oop() {
 994     StubCodeMark mark(this, "StubRoutines", "verify_oop");
 995     address start = __ pc();
 996 
 997     Label exit, error;
 998 
 999     __ pushf();
1000     __ incrementl(ExternalAddress((address) StubRoutines::verify_oop_count_addr()));
1001 
1002     __ push(r12);
1003 
1004     // save c_rarg2 and c_rarg3
1005     __ push(c_rarg2);
1006     __ push(c_rarg3);
1007 
1008     enum {
1009            // After previous pushes.
1010            oop_to_verify = 6 * wordSize,
1011            saved_rax     = 7 * wordSize,
1012            saved_r10     = 8 * wordSize,
1013 
1014            // Before the call to MacroAssembler::debug(), see below.
1015            return_addr   = 16 * wordSize,
1016            error_msg     = 17 * wordSize
1017     };
1018 
1019     // get object
1020     __ movptr(rax, Address(rsp, oop_to_verify));
1021 
1022     // make sure object is 'reasonable'
1023     __ testptr(rax, rax);
1024     __ jcc(Assembler::zero, exit); // if obj is NULL it is OK
1025     // Check if the oop is in the right area of memory
1026     __ movptr(c_rarg2, rax);
1027     __ movptr(c_rarg3, (intptr_t) Universe::verify_oop_mask());
1028     __ andptr(c_rarg2, c_rarg3);
1029     __ movptr(c_rarg3, (intptr_t) Universe::verify_oop_bits());
1030     __ cmpptr(c_rarg2, c_rarg3);
1031     __ jcc(Assembler::notZero, error);
1032 
1033     // set r12 to heapbase for load_klass()
1034     __ reinit_heapbase();
1035 
1036     // make sure klass is 'reasonable', which is not zero.
1037     __ load_klass(rax, rax);  // get klass
1038     __ testptr(rax, rax);
1039     __ jcc(Assembler::zero, error); // if klass is NULL it is broken
1040 
1041     // return if everything seems ok
1042     __ bind(exit);
1043     __ movptr(rax, Address(rsp, saved_rax));     // get saved rax back
1044     __ movptr(rscratch1, Address(rsp, saved_r10)); // get saved r10 back
1045     __ pop(c_rarg3);                             // restore c_rarg3
1046     __ pop(c_rarg2);                             // restore c_rarg2
1047     __ pop(r12);                                 // restore r12
1048     __ popf();                                   // restore flags
1049     __ ret(4 * wordSize);                        // pop caller saved stuff
1050 
1051     // handle errors
1052     __ bind(error);
1053     __ movptr(rax, Address(rsp, saved_rax));     // get saved rax back
1054     __ movptr(rscratch1, Address(rsp, saved_r10)); // get saved r10 back
1055     __ pop(c_rarg3);                             // get saved c_rarg3 back
1056     __ pop(c_rarg2);                             // get saved c_rarg2 back
1057     __ pop(r12);                                 // get saved r12 back
1058     __ popf();                                   // get saved flags off stack --
1059                                                  // will be ignored
1060 
1061     __ pusha();                                  // push registers
1062                                                  // (rip is already
1063                                                  // already pushed)
1064     // debug(char* msg, int64_t pc, int64_t regs[])
1065     // We've popped the registers we'd saved (c_rarg3, c_rarg2 and flags), and
1066     // pushed all the registers, so now the stack looks like:
1067     //     [tos +  0] 16 saved registers
1068     //     [tos + 16] return address
1069     //   * [tos + 17] error message (char*)
1070     //   * [tos + 18] object to verify (oop)
1071     //   * [tos + 19] saved rax - saved by caller and bashed
1072     //   * [tos + 20] saved r10 (rscratch1) - saved by caller
1073     //   * = popped on exit
1074 
1075     __ movptr(c_rarg0, Address(rsp, error_msg));    // pass address of error message
1076     __ movptr(c_rarg1, Address(rsp, return_addr));  // pass return address
1077     __ movq(c_rarg2, rsp);                          // pass address of regs on stack
1078     __ mov(r12, rsp);                               // remember rsp
1079     __ subptr(rsp, frame::arg_reg_save_area_bytes); // windows
1080     __ andptr(rsp, -16);                            // align stack as required by ABI
1081     BLOCK_COMMENT("call MacroAssembler::debug");
1082     __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, MacroAssembler::debug64)));
1083     __ mov(rsp, r12);                               // restore rsp
1084     __ popa();                                      // pop registers (includes r12)
1085     __ ret(4 * wordSize);                           // pop caller saved stuff
1086 
1087     return start;
1088   }
1089 
1090   //
1091   // Verify that a register contains clean 32-bits positive value
1092   // (high 32-bits are 0) so it could be used in 64-bits shifts.
1093   //
1094   //  Input:
1095   //    Rint  -  32-bits value
1096   //    Rtmp  -  scratch
1097   //
1098   void assert_clean_int(Register Rint, Register Rtmp) {
1099 #ifdef ASSERT
1100     Label L;
1101     assert_different_registers(Rtmp, Rint);
1102     __ movslq(Rtmp, Rint);
1103     __ cmpq(Rtmp, Rint);
1104     __ jcc(Assembler::equal, L);
1105     __ stop("high 32-bits of int value are not 0");
1106     __ bind(L);
1107 #endif
1108   }
1109 
1110   //  Generate overlap test for array copy stubs
1111   //
1112   //  Input:
1113   //     c_rarg0 - from
1114   //     c_rarg1 - to
1115   //     c_rarg2 - element count
1116   //
1117   //  Output:
1118   //     rax   - &from[element count - 1]
1119   //
1120   void array_overlap_test(address no_overlap_target, Address::ScaleFactor sf) {
1121     assert(no_overlap_target != NULL, "must be generated");
1122     array_overlap_test(no_overlap_target, NULL, sf);
1123   }
1124   void array_overlap_test(Label& L_no_overlap, Address::ScaleFactor sf) {
1125     array_overlap_test(NULL, &L_no_overlap, sf);
1126   }
1127   void array_overlap_test(address no_overlap_target, Label* NOLp, Address::ScaleFactor sf) {
1128     const Register from     = c_rarg0;
1129     const Register to       = c_rarg1;
1130     const Register count    = c_rarg2;
1131     const Register end_from = rax;
1132 
1133     __ cmpptr(to, from);
1134     __ lea(end_from, Address(from, count, sf, 0));
1135     if (NOLp == NULL) {
1136       ExternalAddress no_overlap(no_overlap_target);
1137       __ jump_cc(Assembler::belowEqual, no_overlap);
1138       __ cmpptr(to, end_from);
1139       __ jump_cc(Assembler::aboveEqual, no_overlap);
1140     } else {
1141       __ jcc(Assembler::belowEqual, (*NOLp));
1142       __ cmpptr(to, end_from);
1143       __ jcc(Assembler::aboveEqual, (*NOLp));
1144     }
1145   }
1146 
1147   // Shuffle first three arg regs on Windows into Linux/Solaris locations.
1148   //
1149   // Outputs:
1150   //    rdi - rcx
1151   //    rsi - rdx
1152   //    rdx - r8
1153   //    rcx - r9
1154   //
1155   // Registers r9 and r10 are used to save rdi and rsi on Windows, which latter
1156   // are non-volatile.  r9 and r10 should not be used by the caller.
1157   //
1158   void setup_arg_regs(int nargs = 3) {
1159     const Register saved_rdi = r9;
1160     const Register saved_rsi = r10;
1161     assert(nargs == 3 || nargs == 4, "else fix");
1162 #ifdef _WIN64
1163     assert(c_rarg0 == rcx && c_rarg1 == rdx && c_rarg2 == r8 && c_rarg3 == r9,
1164            "unexpected argument registers");
1165     if (nargs >= 4)
1166       __ mov(rax, r9);  // r9 is also saved_rdi
1167     __ movptr(saved_rdi, rdi);
1168     __ movptr(saved_rsi, rsi);
1169     __ mov(rdi, rcx); // c_rarg0
1170     __ mov(rsi, rdx); // c_rarg1
1171     __ mov(rdx, r8);  // c_rarg2
1172     if (nargs >= 4)
1173       __ mov(rcx, rax); // c_rarg3 (via rax)
1174 #else
1175     assert(c_rarg0 == rdi && c_rarg1 == rsi && c_rarg2 == rdx && c_rarg3 == rcx,
1176            "unexpected argument registers");
1177 #endif
1178   }
1179 
1180   void restore_arg_regs() {
1181     const Register saved_rdi = r9;
1182     const Register saved_rsi = r10;
1183 #ifdef _WIN64
1184     __ movptr(rdi, saved_rdi);
1185     __ movptr(rsi, saved_rsi);
1186 #endif
1187   }
1188 
1189   // Generate code for an array write pre barrier
1190   //
1191   //     addr    -  starting address
1192   //     count   -  element count
1193   //     tmp     - scratch register
1194   //
1195   //     Destroy no registers!
1196   //
1197   void  gen_write_ref_array_pre_barrier(Register addr, Register count, bool dest_uninitialized) {
1198     BarrierSet* bs = Universe::heap()->barrier_set();
1199     switch (bs->kind()) {
1200       case BarrierSet::G1SATBCTLogging:
1201         // With G1, don't generate the call if we statically know that the target in uninitialized
1202         if (!dest_uninitialized) {
1203            __ pusha();                      // push registers
1204            if (count == c_rarg0) {
1205              if (addr == c_rarg1) {
1206                // exactly backwards!!
1207                __ xchgptr(c_rarg1, c_rarg0);
1208              } else {
1209                __ movptr(c_rarg1, count);
1210                __ movptr(c_rarg0, addr);
1211              }
1212            } else {
1213              __ movptr(c_rarg0, addr);
1214              __ movptr(c_rarg1, count);
1215            }
1216            __ call_VM_leaf(CAST_FROM_FN_PTR(address, BarrierSet::static_write_ref_array_pre), 2);
1217            __ popa();
1218         }
1219          break;
1220       case BarrierSet::CardTableForRS:
1221       case BarrierSet::CardTableExtension:
1222       case BarrierSet::ModRef:
1223         break;
1224       default:
1225         ShouldNotReachHere();
1226 
1227     }
1228   }
1229 
1230   //
1231   // Generate code for an array write post barrier
1232   //
1233   //  Input:
1234   //     start    - register containing starting address of destination array
1235   //     count    - elements count
1236   //     scratch  - scratch register
1237   //
1238   //  The input registers are overwritten.
1239   //
1240   void  gen_write_ref_array_post_barrier(Register start, Register count, Register scratch) {
1241     assert_different_registers(start, count, scratch);
1242     BarrierSet* bs = Universe::heap()->barrier_set();
1243     switch (bs->kind()) {
1244       case BarrierSet::G1SATBCTLogging:
1245         {
1246           __ pusha();             // push registers (overkill)
1247           if (c_rarg0 == count) { // On win64 c_rarg0 == rcx
1248             assert_different_registers(c_rarg1, start);
1249             __ mov(c_rarg1, count);
1250             __ mov(c_rarg0, start);
1251           } else {
1252             assert_different_registers(c_rarg0, count);
1253             __ mov(c_rarg0, start);
1254             __ mov(c_rarg1, count);
1255           }
1256           __ call_VM_leaf(CAST_FROM_FN_PTR(address, BarrierSet::static_write_ref_array_post), 2);
1257           __ popa();
1258         }
1259         break;
1260       case BarrierSet::CardTableForRS:
1261       case BarrierSet::CardTableExtension:
1262         {
1263           CardTableModRefBS* ct = barrier_set_cast<CardTableModRefBS>(bs);
1264           assert(sizeof(*ct->byte_map_base) == sizeof(jbyte), "adjust this code");
1265 
1266           Label L_loop;
1267           const Register end = count;
1268 
1269           __ leaq(end, Address(start, count, TIMES_OOP, 0));  // end == start+count*oop_size
1270           __ subptr(end, BytesPerHeapOop); // end - 1 to make inclusive
1271           __ shrptr(start, CardTableModRefBS::card_shift);
1272           __ shrptr(end,   CardTableModRefBS::card_shift);
1273           __ subptr(end, start); // end --> cards count
1274 
1275           int64_t disp = (int64_t) ct->byte_map_base;
1276           __ mov64(scratch, disp);
1277           __ addptr(start, scratch);
1278         __ BIND(L_loop);
1279           __ movb(Address(start, count, Address::times_1), 0);
1280           __ decrement(count);
1281           __ jcc(Assembler::greaterEqual, L_loop);
1282         }
1283         break;
1284       default:
1285         ShouldNotReachHere();
1286 
1287     }
1288   }
1289 
1290 
1291   // Copy big chunks forward
1292   //
1293   // Inputs:
1294   //   end_from     - source arrays end address
1295   //   end_to       - destination array end address
1296   //   qword_count  - 64-bits element count, negative
1297   //   to           - scratch
1298   //   L_copy_bytes - entry label
1299   //   L_copy_8_bytes  - exit  label
1300   //
1301   void copy_bytes_forward(Register end_from, Register end_to,
1302                              Register qword_count, Register to,
1303                              Label& L_copy_bytes, Label& L_copy_8_bytes) {
1304     DEBUG_ONLY(__ stop("enter at entry label, not here"));
1305     Label L_loop;
1306     __ align(OptoLoopAlignment);
1307     if (UseUnalignedLoadStores) {
1308       Label L_end;
1309       if (UseAVX > 2) {
1310         __ movl(to, 0xffff);
1311         __ kmovwl(k1, to);
1312       }
1313       // Copy 64-bytes per iteration
1314       __ BIND(L_loop);
1315       if (UseAVX > 2) {
1316         __ evmovdqul(xmm0, Address(end_from, qword_count, Address::times_8, -56), Assembler::AVX_512bit);
1317         __ evmovdqul(Address(end_to, qword_count, Address::times_8, -56), xmm0, Assembler::AVX_512bit);
1318       } else if (UseAVX == 2) {
1319         __ vmovdqu(xmm0, Address(end_from, qword_count, Address::times_8, -56));
1320         __ vmovdqu(Address(end_to, qword_count, Address::times_8, -56), xmm0);
1321         __ vmovdqu(xmm1, Address(end_from, qword_count, Address::times_8, -24));
1322         __ vmovdqu(Address(end_to, qword_count, Address::times_8, -24), xmm1);
1323       } else {
1324         __ movdqu(xmm0, Address(end_from, qword_count, Address::times_8, -56));
1325         __ movdqu(Address(end_to, qword_count, Address::times_8, -56), xmm0);
1326         __ movdqu(xmm1, Address(end_from, qword_count, Address::times_8, -40));
1327         __ movdqu(Address(end_to, qword_count, Address::times_8, -40), xmm1);
1328         __ movdqu(xmm2, Address(end_from, qword_count, Address::times_8, -24));
1329         __ movdqu(Address(end_to, qword_count, Address::times_8, -24), xmm2);
1330         __ movdqu(xmm3, Address(end_from, qword_count, Address::times_8, - 8));
1331         __ movdqu(Address(end_to, qword_count, Address::times_8, - 8), xmm3);
1332       }
1333       __ BIND(L_copy_bytes);
1334       __ addptr(qword_count, 8);
1335       __ jcc(Assembler::lessEqual, L_loop);
1336       __ subptr(qword_count, 4);  // sub(8) and add(4)
1337       __ jccb(Assembler::greater, L_end);
1338       // Copy trailing 32 bytes
1339       if (UseAVX >= 2) {
1340         __ vmovdqu(xmm0, Address(end_from, qword_count, Address::times_8, -24));
1341         __ vmovdqu(Address(end_to, qword_count, Address::times_8, -24), xmm0);
1342       } else {
1343         __ movdqu(xmm0, Address(end_from, qword_count, Address::times_8, -24));
1344         __ movdqu(Address(end_to, qword_count, Address::times_8, -24), xmm0);
1345         __ movdqu(xmm1, Address(end_from, qword_count, Address::times_8, - 8));
1346         __ movdqu(Address(end_to, qword_count, Address::times_8, - 8), xmm1);
1347       }
1348       __ addptr(qword_count, 4);
1349       __ BIND(L_end);
1350       if (UseAVX >= 2) {
1351         // clean upper bits of YMM registers
1352         __ vpxor(xmm0, xmm0);
1353         __ vpxor(xmm1, xmm1);
1354       }
1355     } else {
1356       // Copy 32-bytes per iteration
1357       __ BIND(L_loop);
1358       __ movq(to, Address(end_from, qword_count, Address::times_8, -24));
1359       __ movq(Address(end_to, qword_count, Address::times_8, -24), to);
1360       __ movq(to, Address(end_from, qword_count, Address::times_8, -16));
1361       __ movq(Address(end_to, qword_count, Address::times_8, -16), to);
1362       __ movq(to, Address(end_from, qword_count, Address::times_8, - 8));
1363       __ movq(Address(end_to, qword_count, Address::times_8, - 8), to);
1364       __ movq(to, Address(end_from, qword_count, Address::times_8, - 0));
1365       __ movq(Address(end_to, qword_count, Address::times_8, - 0), to);
1366 
1367       __ BIND(L_copy_bytes);
1368       __ addptr(qword_count, 4);
1369       __ jcc(Assembler::lessEqual, L_loop);
1370     }
1371     __ subptr(qword_count, 4);
1372     __ jcc(Assembler::less, L_copy_8_bytes); // Copy trailing qwords
1373   }
1374 
1375   // Copy big chunks backward
1376   //
1377   // Inputs:
1378   //   from         - source arrays address
1379   //   dest         - destination array address
1380   //   qword_count  - 64-bits element count
1381   //   to           - scratch
1382   //   L_copy_bytes - entry label
1383   //   L_copy_8_bytes  - exit  label
1384   //
1385   void copy_bytes_backward(Register from, Register dest,
1386                               Register qword_count, Register to,
1387                               Label& L_copy_bytes, Label& L_copy_8_bytes) {
1388     DEBUG_ONLY(__ stop("enter at entry label, not here"));
1389     Label L_loop;
1390     __ align(OptoLoopAlignment);
1391     if (UseUnalignedLoadStores) {
1392       Label L_end;
1393       if (UseAVX > 2) {
1394         __ movl(to, 0xffff);
1395         __ kmovwl(k1, to);
1396       }
1397       // Copy 64-bytes per iteration
1398       __ BIND(L_loop);
1399       if (UseAVX > 2) {
1400         __ evmovdqul(xmm0, Address(from, qword_count, Address::times_8, 0), Assembler::AVX_512bit);
1401         __ evmovdqul(Address(dest, qword_count, Address::times_8, 0), xmm0, Assembler::AVX_512bit);
1402       } else if (UseAVX == 2) {
1403         __ vmovdqu(xmm0, Address(from, qword_count, Address::times_8, 32));
1404         __ vmovdqu(Address(dest, qword_count, Address::times_8, 32), xmm0);
1405         __ vmovdqu(xmm1, Address(from, qword_count, Address::times_8,  0));
1406         __ vmovdqu(Address(dest, qword_count, Address::times_8,  0), xmm1);
1407       } else {
1408         __ movdqu(xmm0, Address(from, qword_count, Address::times_8, 48));
1409         __ movdqu(Address(dest, qword_count, Address::times_8, 48), xmm0);
1410         __ movdqu(xmm1, Address(from, qword_count, Address::times_8, 32));
1411         __ movdqu(Address(dest, qword_count, Address::times_8, 32), xmm1);
1412         __ movdqu(xmm2, Address(from, qword_count, Address::times_8, 16));
1413         __ movdqu(Address(dest, qword_count, Address::times_8, 16), xmm2);
1414         __ movdqu(xmm3, Address(from, qword_count, Address::times_8,  0));
1415         __ movdqu(Address(dest, qword_count, Address::times_8,  0), xmm3);
1416       }
1417       __ BIND(L_copy_bytes);
1418       __ subptr(qword_count, 8);
1419       __ jcc(Assembler::greaterEqual, L_loop);
1420 
1421       __ addptr(qword_count, 4);  // add(8) and sub(4)
1422       __ jccb(Assembler::less, L_end);
1423       // Copy trailing 32 bytes
1424       if (UseAVX >= 2) {
1425         __ vmovdqu(xmm0, Address(from, qword_count, Address::times_8, 0));
1426         __ vmovdqu(Address(dest, qword_count, Address::times_8, 0), xmm0);
1427       } else {
1428         __ movdqu(xmm0, Address(from, qword_count, Address::times_8, 16));
1429         __ movdqu(Address(dest, qword_count, Address::times_8, 16), xmm0);
1430         __ movdqu(xmm1, Address(from, qword_count, Address::times_8,  0));
1431         __ movdqu(Address(dest, qword_count, Address::times_8,  0), xmm1);
1432       }
1433       __ subptr(qword_count, 4);
1434       __ BIND(L_end);
1435       if (UseAVX >= 2) {
1436         // clean upper bits of YMM registers
1437         __ vpxor(xmm0, xmm0);
1438         __ vpxor(xmm1, xmm1);
1439       }
1440     } else {
1441       // Copy 32-bytes per iteration
1442       __ BIND(L_loop);
1443       __ movq(to, Address(from, qword_count, Address::times_8, 24));
1444       __ movq(Address(dest, qword_count, Address::times_8, 24), to);
1445       __ movq(to, Address(from, qword_count, Address::times_8, 16));
1446       __ movq(Address(dest, qword_count, Address::times_8, 16), to);
1447       __ movq(to, Address(from, qword_count, Address::times_8,  8));
1448       __ movq(Address(dest, qword_count, Address::times_8,  8), to);
1449       __ movq(to, Address(from, qword_count, Address::times_8,  0));
1450       __ movq(Address(dest, qword_count, Address::times_8,  0), to);
1451 
1452       __ BIND(L_copy_bytes);
1453       __ subptr(qword_count, 4);
1454       __ jcc(Assembler::greaterEqual, L_loop);
1455     }
1456     __ addptr(qword_count, 4);
1457     __ jcc(Assembler::greater, L_copy_8_bytes); // Copy trailing qwords
1458   }
1459 
1460 
1461   // Arguments:
1462   //   aligned - true => Input and output aligned on a HeapWord == 8-byte boundary
1463   //             ignored
1464   //   name    - stub name string
1465   //
1466   // Inputs:
1467   //   c_rarg0   - source array address
1468   //   c_rarg1   - destination array address
1469   //   c_rarg2   - element count, treated as ssize_t, can be zero
1470   //
1471   // If 'from' and/or 'to' are aligned on 4-, 2-, or 1-byte boundaries,
1472   // we let the hardware handle it.  The one to eight bytes within words,
1473   // dwords or qwords that span cache line boundaries will still be loaded
1474   // and stored atomically.
1475   //
1476   // Side Effects:
1477   //   disjoint_byte_copy_entry is set to the no-overlap entry point
1478   //   used by generate_conjoint_byte_copy().
1479   //
1480   address generate_disjoint_byte_copy(bool aligned, address* entry, const char *name) {
1481     __ align(CodeEntryAlignment);
1482     StubCodeMark mark(this, "StubRoutines", name);
1483     address start = __ pc();
1484 
1485     Label L_copy_bytes, L_copy_8_bytes, L_copy_4_bytes, L_copy_2_bytes;
1486     Label L_copy_byte, L_exit;
1487     const Register from        = rdi;  // source array address
1488     const Register to          = rsi;  // destination array address
1489     const Register count       = rdx;  // elements count
1490     const Register byte_count  = rcx;
1491     const Register qword_count = count;
1492     const Register end_from    = from; // source array end address
1493     const Register end_to      = to;   // destination array end address
1494     // End pointers are inclusive, and if count is not zero they point
1495     // to the last unit copied:  end_to[0] := end_from[0]
1496 
1497     __ enter(); // required for proper stackwalking of RuntimeStub frame
1498     assert_clean_int(c_rarg2, rax);    // Make sure 'count' is clean int.
1499 
1500     if (entry != NULL) {
1501       *entry = __ pc();
1502        // caller can pass a 64-bit byte count here (from Unsafe.copyMemory)
1503       BLOCK_COMMENT("Entry:");
1504     }
1505 
1506     setup_arg_regs(); // from => rdi, to => rsi, count => rdx
1507                       // r9 and r10 may be used to save non-volatile registers
1508 
1509     // 'from', 'to' and 'count' are now valid
1510     __ movptr(byte_count, count);
1511     __ shrptr(count, 3); // count => qword_count
1512 
1513     // Copy from low to high addresses.  Use 'to' as scratch.
1514     __ lea(end_from, Address(from, qword_count, Address::times_8, -8));
1515     __ lea(end_to,   Address(to,   qword_count, Address::times_8, -8));
1516     __ negptr(qword_count); // make the count negative
1517     __ jmp(L_copy_bytes);
1518 
1519     // Copy trailing qwords
1520   __ BIND(L_copy_8_bytes);
1521     __ movq(rax, Address(end_from, qword_count, Address::times_8, 8));
1522     __ movq(Address(end_to, qword_count, Address::times_8, 8), rax);
1523     __ increment(qword_count);
1524     __ jcc(Assembler::notZero, L_copy_8_bytes);
1525 
1526     // Check for and copy trailing dword
1527   __ BIND(L_copy_4_bytes);
1528     __ testl(byte_count, 4);
1529     __ jccb(Assembler::zero, L_copy_2_bytes);
1530     __ movl(rax, Address(end_from, 8));
1531     __ movl(Address(end_to, 8), rax);
1532 
1533     __ addptr(end_from, 4);
1534     __ addptr(end_to, 4);
1535 
1536     // Check for and copy trailing word
1537   __ BIND(L_copy_2_bytes);
1538     __ testl(byte_count, 2);
1539     __ jccb(Assembler::zero, L_copy_byte);
1540     __ movw(rax, Address(end_from, 8));
1541     __ movw(Address(end_to, 8), rax);
1542 
1543     __ addptr(end_from, 2);
1544     __ addptr(end_to, 2);
1545 
1546     // Check for and copy trailing byte
1547   __ BIND(L_copy_byte);
1548     __ testl(byte_count, 1);
1549     __ jccb(Assembler::zero, L_exit);
1550     __ movb(rax, Address(end_from, 8));
1551     __ movb(Address(end_to, 8), rax);
1552 
1553   __ BIND(L_exit);
1554     restore_arg_regs();
1555     inc_counter_np(SharedRuntime::_jbyte_array_copy_ctr); // Update counter after rscratch1 is free
1556     __ xorptr(rax, rax); // return 0
1557     __ leave(); // required for proper stackwalking of RuntimeStub frame
1558     __ ret(0);
1559 
1560     // Copy in multi-bytes chunks
1561     copy_bytes_forward(end_from, end_to, qword_count, rax, L_copy_bytes, L_copy_8_bytes);
1562     __ jmp(L_copy_4_bytes);
1563 
1564     return start;
1565   }
1566 
1567   // Arguments:
1568   //   aligned - true => Input and output aligned on a HeapWord == 8-byte boundary
1569   //             ignored
1570   //   name    - stub name string
1571   //
1572   // Inputs:
1573   //   c_rarg0   - source array address
1574   //   c_rarg1   - destination array address
1575   //   c_rarg2   - element count, treated as ssize_t, can be zero
1576   //
1577   // If 'from' and/or 'to' are aligned on 4-, 2-, or 1-byte boundaries,
1578   // we let the hardware handle it.  The one to eight bytes within words,
1579   // dwords or qwords that span cache line boundaries will still be loaded
1580   // and stored atomically.
1581   //
1582   address generate_conjoint_byte_copy(bool aligned, address nooverlap_target,
1583                                       address* entry, const char *name) {
1584     __ align(CodeEntryAlignment);
1585     StubCodeMark mark(this, "StubRoutines", name);
1586     address start = __ pc();
1587 
1588     Label L_copy_bytes, L_copy_8_bytes, L_copy_4_bytes, L_copy_2_bytes;
1589     const Register from        = rdi;  // source array address
1590     const Register to          = rsi;  // destination array address
1591     const Register count       = rdx;  // elements count
1592     const Register byte_count  = rcx;
1593     const Register qword_count = count;
1594 
1595     __ enter(); // required for proper stackwalking of RuntimeStub frame
1596     assert_clean_int(c_rarg2, rax);    // Make sure 'count' is clean int.
1597 
1598     if (entry != NULL) {
1599       *entry = __ pc();
1600       // caller can pass a 64-bit byte count here (from Unsafe.copyMemory)
1601       BLOCK_COMMENT("Entry:");
1602     }
1603 
1604     array_overlap_test(nooverlap_target, Address::times_1);
1605     setup_arg_regs(); // from => rdi, to => rsi, count => rdx
1606                       // r9 and r10 may be used to save non-volatile registers
1607 
1608     // 'from', 'to' and 'count' are now valid
1609     __ movptr(byte_count, count);
1610     __ shrptr(count, 3);   // count => qword_count
1611 
1612     // Copy from high to low addresses.
1613 
1614     // Check for and copy trailing byte
1615     __ testl(byte_count, 1);
1616     __ jcc(Assembler::zero, L_copy_2_bytes);
1617     __ movb(rax, Address(from, byte_count, Address::times_1, -1));
1618     __ movb(Address(to, byte_count, Address::times_1, -1), rax);
1619     __ decrement(byte_count); // Adjust for possible trailing word
1620 
1621     // Check for and copy trailing word
1622   __ BIND(L_copy_2_bytes);
1623     __ testl(byte_count, 2);
1624     __ jcc(Assembler::zero, L_copy_4_bytes);
1625     __ movw(rax, Address(from, byte_count, Address::times_1, -2));
1626     __ movw(Address(to, byte_count, Address::times_1, -2), rax);
1627 
1628     // Check for and copy trailing dword
1629   __ BIND(L_copy_4_bytes);
1630     __ testl(byte_count, 4);
1631     __ jcc(Assembler::zero, L_copy_bytes);
1632     __ movl(rax, Address(from, qword_count, Address::times_8));
1633     __ movl(Address(to, qword_count, Address::times_8), rax);
1634     __ jmp(L_copy_bytes);
1635 
1636     // Copy trailing qwords
1637   __ BIND(L_copy_8_bytes);
1638     __ movq(rax, Address(from, qword_count, Address::times_8, -8));
1639     __ movq(Address(to, qword_count, Address::times_8, -8), rax);
1640     __ decrement(qword_count);
1641     __ jcc(Assembler::notZero, L_copy_8_bytes);
1642 
1643     restore_arg_regs();
1644     inc_counter_np(SharedRuntime::_jbyte_array_copy_ctr); // Update counter after rscratch1 is free
1645     __ xorptr(rax, rax); // return 0
1646     __ leave(); // required for proper stackwalking of RuntimeStub frame
1647     __ ret(0);
1648 
1649     // Copy in multi-bytes chunks
1650     copy_bytes_backward(from, to, qword_count, rax, L_copy_bytes, L_copy_8_bytes);
1651 
1652     restore_arg_regs();
1653     inc_counter_np(SharedRuntime::_jbyte_array_copy_ctr); // Update counter after rscratch1 is free
1654     __ xorptr(rax, rax); // return 0
1655     __ leave(); // required for proper stackwalking of RuntimeStub frame
1656     __ ret(0);
1657 
1658     return start;
1659   }
1660 
1661   // Arguments:
1662   //   aligned - true => Input and output aligned on a HeapWord == 8-byte boundary
1663   //             ignored
1664   //   name    - stub name string
1665   //
1666   // Inputs:
1667   //   c_rarg0   - source array address
1668   //   c_rarg1   - destination array address
1669   //   c_rarg2   - element count, treated as ssize_t, can be zero
1670   //
1671   // If 'from' and/or 'to' are aligned on 4- or 2-byte boundaries, we
1672   // let the hardware handle it.  The two or four words within dwords
1673   // or qwords that span cache line boundaries will still be loaded
1674   // and stored atomically.
1675   //
1676   // Side Effects:
1677   //   disjoint_short_copy_entry is set to the no-overlap entry point
1678   //   used by generate_conjoint_short_copy().
1679   //
1680   address generate_disjoint_short_copy(bool aligned, address *entry, const char *name) {
1681     __ align(CodeEntryAlignment);
1682     StubCodeMark mark(this, "StubRoutines", name);
1683     address start = __ pc();
1684 
1685     Label L_copy_bytes, L_copy_8_bytes, L_copy_4_bytes,L_copy_2_bytes,L_exit;
1686     const Register from        = rdi;  // source array address
1687     const Register to          = rsi;  // destination array address
1688     const Register count       = rdx;  // elements count
1689     const Register word_count  = rcx;
1690     const Register qword_count = count;
1691     const Register end_from    = from; // source array end address
1692     const Register end_to      = to;   // destination array end address
1693     // End pointers are inclusive, and if count is not zero they point
1694     // to the last unit copied:  end_to[0] := end_from[0]
1695 
1696     __ enter(); // required for proper stackwalking of RuntimeStub frame
1697     assert_clean_int(c_rarg2, rax);    // Make sure 'count' is clean int.
1698 
1699     if (entry != NULL) {
1700       *entry = __ pc();
1701       // caller can pass a 64-bit byte count here (from Unsafe.copyMemory)
1702       BLOCK_COMMENT("Entry:");
1703     }
1704 
1705     setup_arg_regs(); // from => rdi, to => rsi, count => rdx
1706                       // r9 and r10 may be used to save non-volatile registers
1707 
1708     // 'from', 'to' and 'count' are now valid
1709     __ movptr(word_count, count);
1710     __ shrptr(count, 2); // count => qword_count
1711 
1712     // Copy from low to high addresses.  Use 'to' as scratch.
1713     __ lea(end_from, Address(from, qword_count, Address::times_8, -8));
1714     __ lea(end_to,   Address(to,   qword_count, Address::times_8, -8));
1715     __ negptr(qword_count);
1716     __ jmp(L_copy_bytes);
1717 
1718     // Copy trailing qwords
1719   __ BIND(L_copy_8_bytes);
1720     __ movq(rax, Address(end_from, qword_count, Address::times_8, 8));
1721     __ movq(Address(end_to, qword_count, Address::times_8, 8), rax);
1722     __ increment(qword_count);
1723     __ jcc(Assembler::notZero, L_copy_8_bytes);
1724 
1725     // Original 'dest' is trashed, so we can't use it as a
1726     // base register for a possible trailing word copy
1727 
1728     // Check for and copy trailing dword
1729   __ BIND(L_copy_4_bytes);
1730     __ testl(word_count, 2);
1731     __ jccb(Assembler::zero, L_copy_2_bytes);
1732     __ movl(rax, Address(end_from, 8));
1733     __ movl(Address(end_to, 8), rax);
1734 
1735     __ addptr(end_from, 4);
1736     __ addptr(end_to, 4);
1737 
1738     // Check for and copy trailing word
1739   __ BIND(L_copy_2_bytes);
1740     __ testl(word_count, 1);
1741     __ jccb(Assembler::zero, L_exit);
1742     __ movw(rax, Address(end_from, 8));
1743     __ movw(Address(end_to, 8), rax);
1744 
1745   __ BIND(L_exit);
1746     restore_arg_regs();
1747     inc_counter_np(SharedRuntime::_jshort_array_copy_ctr); // Update counter after rscratch1 is free
1748     __ xorptr(rax, rax); // return 0
1749     __ leave(); // required for proper stackwalking of RuntimeStub frame
1750     __ ret(0);
1751 
1752     // Copy in multi-bytes chunks
1753     copy_bytes_forward(end_from, end_to, qword_count, rax, L_copy_bytes, L_copy_8_bytes);
1754     __ jmp(L_copy_4_bytes);
1755 
1756     return start;
1757   }
1758 
1759   address generate_fill(BasicType t, bool aligned, const char *name) {
1760     __ align(CodeEntryAlignment);
1761     StubCodeMark mark(this, "StubRoutines", name);
1762     address start = __ pc();
1763 
1764     BLOCK_COMMENT("Entry:");
1765 
1766     const Register to       = c_rarg0;  // source array address
1767     const Register value    = c_rarg1;  // value
1768     const Register count    = c_rarg2;  // elements count
1769 
1770     __ enter(); // required for proper stackwalking of RuntimeStub frame
1771 
1772     __ generate_fill(t, aligned, to, value, count, rax, xmm0);
1773 
1774     __ leave(); // required for proper stackwalking of RuntimeStub frame
1775     __ ret(0);
1776     return start;
1777   }
1778 
1779   // Arguments:
1780   //   aligned - true => Input and output aligned on a HeapWord == 8-byte boundary
1781   //             ignored
1782   //   name    - stub name string
1783   //
1784   // Inputs:
1785   //   c_rarg0   - source array address
1786   //   c_rarg1   - destination array address
1787   //   c_rarg2   - element count, treated as ssize_t, can be zero
1788   //
1789   // If 'from' and/or 'to' are aligned on 4- or 2-byte boundaries, we
1790   // let the hardware handle it.  The two or four words within dwords
1791   // or qwords that span cache line boundaries will still be loaded
1792   // and stored atomically.
1793   //
1794   address generate_conjoint_short_copy(bool aligned, address nooverlap_target,
1795                                        address *entry, const char *name) {
1796     __ align(CodeEntryAlignment);
1797     StubCodeMark mark(this, "StubRoutines", name);
1798     address start = __ pc();
1799 
1800     Label L_copy_bytes, L_copy_8_bytes, L_copy_4_bytes;
1801     const Register from        = rdi;  // source array address
1802     const Register to          = rsi;  // destination array address
1803     const Register count       = rdx;  // elements count
1804     const Register word_count  = rcx;
1805     const Register qword_count = count;
1806 
1807     __ enter(); // required for proper stackwalking of RuntimeStub frame
1808     assert_clean_int(c_rarg2, rax);    // Make sure 'count' is clean int.
1809 
1810     if (entry != NULL) {
1811       *entry = __ pc();
1812       // caller can pass a 64-bit byte count here (from Unsafe.copyMemory)
1813       BLOCK_COMMENT("Entry:");
1814     }
1815 
1816     array_overlap_test(nooverlap_target, Address::times_2);
1817     setup_arg_regs(); // from => rdi, to => rsi, count => rdx
1818                       // r9 and r10 may be used to save non-volatile registers
1819 
1820     // 'from', 'to' and 'count' are now valid
1821     __ movptr(word_count, count);
1822     __ shrptr(count, 2); // count => qword_count
1823 
1824     // Copy from high to low addresses.  Use 'to' as scratch.
1825 
1826     // Check for and copy trailing word
1827     __ testl(word_count, 1);
1828     __ jccb(Assembler::zero, L_copy_4_bytes);
1829     __ movw(rax, Address(from, word_count, Address::times_2, -2));
1830     __ movw(Address(to, word_count, Address::times_2, -2), rax);
1831 
1832     // Check for and copy trailing dword
1833   __ BIND(L_copy_4_bytes);
1834     __ testl(word_count, 2);
1835     __ jcc(Assembler::zero, L_copy_bytes);
1836     __ movl(rax, Address(from, qword_count, Address::times_8));
1837     __ movl(Address(to, qword_count, Address::times_8), rax);
1838     __ jmp(L_copy_bytes);
1839 
1840     // Copy trailing qwords
1841   __ BIND(L_copy_8_bytes);
1842     __ movq(rax, Address(from, qword_count, Address::times_8, -8));
1843     __ movq(Address(to, qword_count, Address::times_8, -8), rax);
1844     __ decrement(qword_count);
1845     __ jcc(Assembler::notZero, L_copy_8_bytes);
1846 
1847     restore_arg_regs();
1848     inc_counter_np(SharedRuntime::_jshort_array_copy_ctr); // Update counter after rscratch1 is free
1849     __ xorptr(rax, rax); // return 0
1850     __ leave(); // required for proper stackwalking of RuntimeStub frame
1851     __ ret(0);
1852 
1853     // Copy in multi-bytes chunks
1854     copy_bytes_backward(from, to, qword_count, rax, L_copy_bytes, L_copy_8_bytes);
1855 
1856     restore_arg_regs();
1857     inc_counter_np(SharedRuntime::_jshort_array_copy_ctr); // Update counter after rscratch1 is free
1858     __ xorptr(rax, rax); // return 0
1859     __ leave(); // required for proper stackwalking of RuntimeStub frame
1860     __ ret(0);
1861 
1862     return start;
1863   }
1864 
1865   // Arguments:
1866   //   aligned - true => Input and output aligned on a HeapWord == 8-byte boundary
1867   //             ignored
1868   //   is_oop  - true => oop array, so generate store check code
1869   //   name    - stub name string
1870   //
1871   // Inputs:
1872   //   c_rarg0   - source array address
1873   //   c_rarg1   - destination array address
1874   //   c_rarg2   - element count, treated as ssize_t, can be zero
1875   //
1876   // If 'from' and/or 'to' are aligned on 4-byte boundaries, we let
1877   // the hardware handle it.  The two dwords within qwords that span
1878   // cache line boundaries will still be loaded and stored atomicly.
1879   //
1880   // Side Effects:
1881   //   disjoint_int_copy_entry is set to the no-overlap entry point
1882   //   used by generate_conjoint_int_oop_copy().
1883   //
1884   address generate_disjoint_int_oop_copy(bool aligned, bool is_oop, address* entry,
1885                                          const char *name, bool dest_uninitialized = false) {
1886     __ align(CodeEntryAlignment);
1887     StubCodeMark mark(this, "StubRoutines", name);
1888     address start = __ pc();
1889 
1890     Label L_copy_bytes, L_copy_8_bytes, L_copy_4_bytes, L_exit;
1891     const Register from        = rdi;  // source array address
1892     const Register to          = rsi;  // destination array address
1893     const Register count       = rdx;  // elements count
1894     const Register dword_count = rcx;
1895     const Register qword_count = count;
1896     const Register end_from    = from; // source array end address
1897     const Register end_to      = to;   // destination array end address
1898     const Register saved_to    = r11;  // saved destination array address
1899     // End pointers are inclusive, and if count is not zero they point
1900     // to the last unit copied:  end_to[0] := end_from[0]
1901 
1902     __ enter(); // required for proper stackwalking of RuntimeStub frame
1903     assert_clean_int(c_rarg2, rax);    // Make sure 'count' is clean int.
1904 
1905     if (entry != NULL) {
1906       *entry = __ pc();
1907       // caller can pass a 64-bit byte count here (from Unsafe.copyMemory)
1908       BLOCK_COMMENT("Entry:");
1909     }
1910 
1911     setup_arg_regs(); // from => rdi, to => rsi, count => rdx
1912                       // r9 and r10 may be used to save non-volatile registers
1913     if (is_oop) {
1914       __ movq(saved_to, to);
1915       gen_write_ref_array_pre_barrier(to, count, dest_uninitialized);
1916     }
1917 
1918     // 'from', 'to' and 'count' are now valid
1919     __ movptr(dword_count, count);
1920     __ shrptr(count, 1); // count => qword_count
1921 
1922     // Copy from low to high addresses.  Use 'to' as scratch.
1923     __ lea(end_from, Address(from, qword_count, Address::times_8, -8));
1924     __ lea(end_to,   Address(to,   qword_count, Address::times_8, -8));
1925     __ negptr(qword_count);
1926     __ jmp(L_copy_bytes);
1927 
1928     // Copy trailing qwords
1929   __ BIND(L_copy_8_bytes);
1930     __ movq(rax, Address(end_from, qword_count, Address::times_8, 8));
1931     __ movq(Address(end_to, qword_count, Address::times_8, 8), rax);
1932     __ increment(qword_count);
1933     __ jcc(Assembler::notZero, L_copy_8_bytes);
1934 
1935     // Check for and copy trailing dword
1936   __ BIND(L_copy_4_bytes);
1937     __ testl(dword_count, 1); // Only byte test since the value is 0 or 1
1938     __ jccb(Assembler::zero, L_exit);
1939     __ movl(rax, Address(end_from, 8));
1940     __ movl(Address(end_to, 8), rax);
1941 
1942   __ BIND(L_exit);
1943     if (is_oop) {
1944       gen_write_ref_array_post_barrier(saved_to, dword_count, rax);
1945     }
1946     restore_arg_regs();
1947     inc_counter_np(SharedRuntime::_jint_array_copy_ctr); // Update counter after rscratch1 is free
1948     __ xorptr(rax, rax); // return 0
1949     __ leave(); // required for proper stackwalking of RuntimeStub frame
1950     __ ret(0);
1951 
1952     // Copy in multi-bytes chunks
1953     copy_bytes_forward(end_from, end_to, qword_count, rax, L_copy_bytes, L_copy_8_bytes);
1954     __ jmp(L_copy_4_bytes);
1955 
1956     return start;
1957   }
1958 
1959   // Arguments:
1960   //   aligned - true => Input and output aligned on a HeapWord == 8-byte boundary
1961   //             ignored
1962   //   is_oop  - true => oop array, so generate store check code
1963   //   name    - stub name string
1964   //
1965   // Inputs:
1966   //   c_rarg0   - source array address
1967   //   c_rarg1   - destination array address
1968   //   c_rarg2   - element count, treated as ssize_t, can be zero
1969   //
1970   // If 'from' and/or 'to' are aligned on 4-byte boundaries, we let
1971   // the hardware handle it.  The two dwords within qwords that span
1972   // cache line boundaries will still be loaded and stored atomicly.
1973   //
1974   address generate_conjoint_int_oop_copy(bool aligned, bool is_oop, address nooverlap_target,
1975                                          address *entry, const char *name,
1976                                          bool dest_uninitialized = false) {
1977     __ align(CodeEntryAlignment);
1978     StubCodeMark mark(this, "StubRoutines", name);
1979     address start = __ pc();
1980 
1981     Label L_copy_bytes, L_copy_8_bytes, L_copy_2_bytes, L_exit;
1982     const Register from        = rdi;  // source array address
1983     const Register to          = rsi;  // destination array address
1984     const Register count       = rdx;  // elements count
1985     const Register dword_count = rcx;
1986     const Register qword_count = count;
1987 
1988     __ enter(); // required for proper stackwalking of RuntimeStub frame
1989     assert_clean_int(c_rarg2, rax);    // Make sure 'count' is clean int.
1990 
1991     if (entry != NULL) {
1992       *entry = __ pc();
1993        // caller can pass a 64-bit byte count here (from Unsafe.copyMemory)
1994       BLOCK_COMMENT("Entry:");
1995     }
1996 
1997     array_overlap_test(nooverlap_target, Address::times_4);
1998     setup_arg_regs(); // from => rdi, to => rsi, count => rdx
1999                       // r9 and r10 may be used to save non-volatile registers
2000 
2001     if (is_oop) {
2002       // no registers are destroyed by this call
2003       gen_write_ref_array_pre_barrier(to, count, dest_uninitialized);
2004     }
2005 
2006     assert_clean_int(count, rax); // Make sure 'count' is clean int.
2007     // 'from', 'to' and 'count' are now valid
2008     __ movptr(dword_count, count);
2009     __ shrptr(count, 1); // count => qword_count
2010 
2011     // Copy from high to low addresses.  Use 'to' as scratch.
2012 
2013     // Check for and copy trailing dword
2014     __ testl(dword_count, 1);
2015     __ jcc(Assembler::zero, L_copy_bytes);
2016     __ movl(rax, Address(from, dword_count, Address::times_4, -4));
2017     __ movl(Address(to, dword_count, Address::times_4, -4), rax);
2018     __ jmp(L_copy_bytes);
2019 
2020     // Copy trailing qwords
2021   __ BIND(L_copy_8_bytes);
2022     __ movq(rax, Address(from, qword_count, Address::times_8, -8));
2023     __ movq(Address(to, qword_count, Address::times_8, -8), rax);
2024     __ decrement(qword_count);
2025     __ jcc(Assembler::notZero, L_copy_8_bytes);
2026 
2027     if (is_oop) {
2028       __ jmp(L_exit);
2029     }
2030     restore_arg_regs();
2031     inc_counter_np(SharedRuntime::_jint_array_copy_ctr); // Update counter after rscratch1 is free
2032     __ xorptr(rax, rax); // return 0
2033     __ leave(); // required for proper stackwalking of RuntimeStub frame
2034     __ ret(0);
2035 
2036     // Copy in multi-bytes chunks
2037     copy_bytes_backward(from, to, qword_count, rax, L_copy_bytes, L_copy_8_bytes);
2038 
2039   __ BIND(L_exit);
2040     if (is_oop) {
2041       gen_write_ref_array_post_barrier(to, dword_count, rax);
2042     }
2043     restore_arg_regs();
2044     inc_counter_np(SharedRuntime::_jint_array_copy_ctr); // Update counter after rscratch1 is free
2045     __ xorptr(rax, rax); // return 0
2046     __ leave(); // required for proper stackwalking of RuntimeStub frame
2047     __ ret(0);
2048 
2049     return start;
2050   }
2051 
2052   // Arguments:
2053   //   aligned - true => Input and output aligned on a HeapWord boundary == 8 bytes
2054   //             ignored
2055   //   is_oop  - true => oop array, so generate store check code
2056   //   name    - stub name string
2057   //
2058   // Inputs:
2059   //   c_rarg0   - source array address
2060   //   c_rarg1   - destination array address
2061   //   c_rarg2   - element count, treated as ssize_t, can be zero
2062   //
2063  // Side Effects:
2064   //   disjoint_oop_copy_entry or disjoint_long_copy_entry is set to the
2065   //   no-overlap entry point used by generate_conjoint_long_oop_copy().
2066   //
2067   address generate_disjoint_long_oop_copy(bool aligned, bool is_oop, address *entry,
2068                                           const char *name, bool dest_uninitialized = false) {
2069     __ align(CodeEntryAlignment);
2070     StubCodeMark mark(this, "StubRoutines", name);
2071     address start = __ pc();
2072 
2073     Label L_copy_bytes, L_copy_8_bytes, L_exit;
2074     const Register from        = rdi;  // source array address
2075     const Register to          = rsi;  // destination array address
2076     const Register qword_count = rdx;  // elements count
2077     const Register end_from    = from; // source array end address
2078     const Register end_to      = rcx;  // destination array end address
2079     const Register saved_to    = to;
2080     const Register saved_count = r11;
2081     // End pointers are inclusive, and if count is not zero they point
2082     // to the last unit copied:  end_to[0] := end_from[0]
2083 
2084     __ enter(); // required for proper stackwalking of RuntimeStub frame
2085     // Save no-overlap entry point for generate_conjoint_long_oop_copy()
2086     assert_clean_int(c_rarg2, rax);    // Make sure 'count' is clean int.
2087 
2088     if (entry != NULL) {
2089       *entry = __ pc();
2090       // caller can pass a 64-bit byte count here (from Unsafe.copyMemory)
2091       BLOCK_COMMENT("Entry:");
2092     }
2093 
2094     setup_arg_regs(); // from => rdi, to => rsi, count => rdx
2095                       // r9 and r10 may be used to save non-volatile registers
2096     // 'from', 'to' and 'qword_count' are now valid
2097     if (is_oop) {
2098       // Save to and count for store barrier
2099       __ movptr(saved_count, qword_count);
2100       // no registers are destroyed by this call
2101       gen_write_ref_array_pre_barrier(to, qword_count, dest_uninitialized);
2102     }
2103 
2104     // Copy from low to high addresses.  Use 'to' as scratch.
2105     __ lea(end_from, Address(from, qword_count, Address::times_8, -8));
2106     __ lea(end_to,   Address(to,   qword_count, Address::times_8, -8));
2107     __ negptr(qword_count);
2108     __ jmp(L_copy_bytes);
2109 
2110     // Copy trailing qwords
2111   __ BIND(L_copy_8_bytes);
2112     __ movq(rax, Address(end_from, qword_count, Address::times_8, 8));
2113     __ movq(Address(end_to, qword_count, Address::times_8, 8), rax);
2114     __ increment(qword_count);
2115     __ jcc(Assembler::notZero, L_copy_8_bytes);
2116 
2117     if (is_oop) {
2118       __ jmp(L_exit);
2119     } else {
2120       restore_arg_regs();
2121       inc_counter_np(SharedRuntime::_jlong_array_copy_ctr); // Update counter after rscratch1 is free
2122       __ xorptr(rax, rax); // return 0
2123       __ leave(); // required for proper stackwalking of RuntimeStub frame
2124       __ ret(0);
2125     }
2126 
2127     // Copy in multi-bytes chunks
2128     copy_bytes_forward(end_from, end_to, qword_count, rax, L_copy_bytes, L_copy_8_bytes);
2129 
2130     if (is_oop) {
2131     __ BIND(L_exit);
2132       gen_write_ref_array_post_barrier(saved_to, saved_count, rax);
2133     }
2134     restore_arg_regs();
2135     if (is_oop) {
2136       inc_counter_np(SharedRuntime::_oop_array_copy_ctr); // Update counter after rscratch1 is free
2137     } else {
2138       inc_counter_np(SharedRuntime::_jlong_array_copy_ctr); // Update counter after rscratch1 is free
2139     }
2140     __ xorptr(rax, rax); // return 0
2141     __ leave(); // required for proper stackwalking of RuntimeStub frame
2142     __ ret(0);
2143 
2144     return start;
2145   }
2146 
2147   // Arguments:
2148   //   aligned - true => Input and output aligned on a HeapWord boundary == 8 bytes
2149   //             ignored
2150   //   is_oop  - true => oop array, so generate store check code
2151   //   name    - stub name string
2152   //
2153   // Inputs:
2154   //   c_rarg0   - source array address
2155   //   c_rarg1   - destination array address
2156   //   c_rarg2   - element count, treated as ssize_t, can be zero
2157   //
2158   address generate_conjoint_long_oop_copy(bool aligned, bool is_oop,
2159                                           address nooverlap_target, address *entry,
2160                                           const char *name, bool dest_uninitialized = false) {
2161     __ align(CodeEntryAlignment);
2162     StubCodeMark mark(this, "StubRoutines", name);
2163     address start = __ pc();
2164 
2165     Label L_copy_bytes, L_copy_8_bytes, L_exit;
2166     const Register from        = rdi;  // source array address
2167     const Register to          = rsi;  // destination array address
2168     const Register qword_count = rdx;  // elements count
2169     const Register saved_count = rcx;
2170 
2171     __ enter(); // required for proper stackwalking of RuntimeStub frame
2172     assert_clean_int(c_rarg2, rax);    // Make sure 'count' is clean int.
2173 
2174     if (entry != NULL) {
2175       *entry = __ pc();
2176       // caller can pass a 64-bit byte count here (from Unsafe.copyMemory)
2177       BLOCK_COMMENT("Entry:");
2178     }
2179 
2180     array_overlap_test(nooverlap_target, Address::times_8);
2181     setup_arg_regs(); // from => rdi, to => rsi, count => rdx
2182                       // r9 and r10 may be used to save non-volatile registers
2183     // 'from', 'to' and 'qword_count' are now valid
2184     if (is_oop) {
2185       // Save to and count for store barrier
2186       __ movptr(saved_count, qword_count);
2187       // No registers are destroyed by this call
2188       gen_write_ref_array_pre_barrier(to, saved_count, dest_uninitialized);
2189     }
2190 
2191     __ jmp(L_copy_bytes);
2192 
2193     // Copy trailing qwords
2194   __ BIND(L_copy_8_bytes);
2195     __ movq(rax, Address(from, qword_count, Address::times_8, -8));
2196     __ movq(Address(to, qword_count, Address::times_8, -8), rax);
2197     __ decrement(qword_count);
2198     __ jcc(Assembler::notZero, L_copy_8_bytes);
2199 
2200     if (is_oop) {
2201       __ jmp(L_exit);
2202     } else {
2203       restore_arg_regs();
2204       inc_counter_np(SharedRuntime::_jlong_array_copy_ctr); // Update counter after rscratch1 is free
2205       __ xorptr(rax, rax); // return 0
2206       __ leave(); // required for proper stackwalking of RuntimeStub frame
2207       __ ret(0);
2208     }
2209 
2210     // Copy in multi-bytes chunks
2211     copy_bytes_backward(from, to, qword_count, rax, L_copy_bytes, L_copy_8_bytes);
2212 
2213     if (is_oop) {
2214     __ BIND(L_exit);
2215       gen_write_ref_array_post_barrier(to, saved_count, rax);
2216     }
2217     restore_arg_regs();
2218     if (is_oop) {
2219       inc_counter_np(SharedRuntime::_oop_array_copy_ctr); // Update counter after rscratch1 is free
2220     } else {
2221       inc_counter_np(SharedRuntime::_jlong_array_copy_ctr); // Update counter after rscratch1 is free
2222     }
2223     __ xorptr(rax, rax); // return 0
2224     __ leave(); // required for proper stackwalking of RuntimeStub frame
2225     __ ret(0);
2226 
2227     return start;
2228   }
2229 
2230 
2231   // Helper for generating a dynamic type check.
2232   // Smashes no registers.
2233   void generate_type_check(Register sub_klass,
2234                            Register super_check_offset,
2235                            Register super_klass,
2236                            Label& L_success) {
2237     assert_different_registers(sub_klass, super_check_offset, super_klass);
2238 
2239     BLOCK_COMMENT("type_check:");
2240 
2241     Label L_miss;
2242 
2243     __ check_klass_subtype_fast_path(sub_klass, super_klass, noreg,        &L_success, &L_miss, NULL,
2244                                      super_check_offset);
2245     __ check_klass_subtype_slow_path(sub_klass, super_klass, noreg, noreg, &L_success, NULL);
2246 
2247     // Fall through on failure!
2248     __ BIND(L_miss);
2249   }
2250 
2251   //
2252   //  Generate checkcasting array copy stub
2253   //
2254   //  Input:
2255   //    c_rarg0   - source array address
2256   //    c_rarg1   - destination array address
2257   //    c_rarg2   - element count, treated as ssize_t, can be zero
2258   //    c_rarg3   - size_t ckoff (super_check_offset)
2259   // not Win64
2260   //    c_rarg4   - oop ckval (super_klass)
2261   // Win64
2262   //    rsp+40    - oop ckval (super_klass)
2263   //
2264   //  Output:
2265   //    rax ==  0  -  success
2266   //    rax == -1^K - failure, where K is partial transfer count
2267   //
2268   address generate_checkcast_copy(const char *name, address *entry,
2269                                   bool dest_uninitialized = false) {
2270 
2271     Label L_load_element, L_store_element, L_do_card_marks, L_done;
2272 
2273     // Input registers (after setup_arg_regs)
2274     const Register from        = rdi;   // source array address
2275     const Register to          = rsi;   // destination array address
2276     const Register length      = rdx;   // elements count
2277     const Register ckoff       = rcx;   // super_check_offset
2278     const Register ckval       = r8;    // super_klass
2279 
2280     // Registers used as temps (r13, r14 are save-on-entry)
2281     const Register end_from    = from;  // source array end address
2282     const Register end_to      = r13;   // destination array end address
2283     const Register count       = rdx;   // -(count_remaining)
2284     const Register r14_length  = r14;   // saved copy of length
2285     // End pointers are inclusive, and if length is not zero they point
2286     // to the last unit copied:  end_to[0] := end_from[0]
2287 
2288     const Register rax_oop    = rax;    // actual oop copied
2289     const Register r11_klass  = r11;    // oop._klass
2290 
2291     //---------------------------------------------------------------
2292     // Assembler stub will be used for this call to arraycopy
2293     // if the two arrays are subtypes of Object[] but the
2294     // destination array type is not equal to or a supertype
2295     // of the source type.  Each element must be separately
2296     // checked.
2297 
2298     __ align(CodeEntryAlignment);
2299     StubCodeMark mark(this, "StubRoutines", name);
2300     address start = __ pc();
2301 
2302     __ enter(); // required for proper stackwalking of RuntimeStub frame
2303 
2304 #ifdef ASSERT
2305     // caller guarantees that the arrays really are different
2306     // otherwise, we would have to make conjoint checks
2307     { Label L;
2308       array_overlap_test(L, TIMES_OOP);
2309       __ stop("checkcast_copy within a single array");
2310       __ bind(L);
2311     }
2312 #endif //ASSERT
2313 
2314     setup_arg_regs(4); // from => rdi, to => rsi, length => rdx
2315                        // ckoff => rcx, ckval => r8
2316                        // r9 and r10 may be used to save non-volatile registers
2317 #ifdef _WIN64
2318     // last argument (#4) is on stack on Win64
2319     __ movptr(ckval, Address(rsp, 6 * wordSize));
2320 #endif
2321 
2322     // Caller of this entry point must set up the argument registers.
2323     if (entry != NULL) {
2324       *entry = __ pc();
2325       BLOCK_COMMENT("Entry:");
2326     }
2327 
2328     // allocate spill slots for r13, r14
2329     enum {
2330       saved_r13_offset,
2331       saved_r14_offset,
2332       saved_rbp_offset
2333     };
2334     __ subptr(rsp, saved_rbp_offset * wordSize);
2335     __ movptr(Address(rsp, saved_r13_offset * wordSize), r13);
2336     __ movptr(Address(rsp, saved_r14_offset * wordSize), r14);
2337 
2338     // check that int operands are properly extended to size_t
2339     assert_clean_int(length, rax);
2340     assert_clean_int(ckoff, rax);
2341 
2342 #ifdef ASSERT
2343     BLOCK_COMMENT("assert consistent ckoff/ckval");
2344     // The ckoff and ckval must be mutually consistent,
2345     // even though caller generates both.
2346     { Label L;
2347       int sco_offset = in_bytes(Klass::super_check_offset_offset());
2348       __ cmpl(ckoff, Address(ckval, sco_offset));
2349       __ jcc(Assembler::equal, L);
2350       __ stop("super_check_offset inconsistent");
2351       __ bind(L);
2352     }
2353 #endif //ASSERT
2354 
2355     // Loop-invariant addresses.  They are exclusive end pointers.
2356     Address end_from_addr(from, length, TIMES_OOP, 0);
2357     Address   end_to_addr(to,   length, TIMES_OOP, 0);
2358     // Loop-variant addresses.  They assume post-incremented count < 0.
2359     Address from_element_addr(end_from, count, TIMES_OOP, 0);
2360     Address   to_element_addr(end_to,   count, TIMES_OOP, 0);
2361 
2362     gen_write_ref_array_pre_barrier(to, count, dest_uninitialized);
2363 
2364     // Copy from low to high addresses, indexed from the end of each array.
2365     __ lea(end_from, end_from_addr);
2366     __ lea(end_to,   end_to_addr);
2367     __ movptr(r14_length, length);        // save a copy of the length
2368     assert(length == count, "");          // else fix next line:
2369     __ negptr(count);                     // negate and test the length
2370     __ jcc(Assembler::notZero, L_load_element);
2371 
2372     // Empty array:  Nothing to do.
2373     __ xorptr(rax, rax);                  // return 0 on (trivial) success
2374     __ jmp(L_done);
2375 
2376     // ======== begin loop ========
2377     // (Loop is rotated; its entry is L_load_element.)
2378     // Loop control:
2379     //   for (count = -count; count != 0; count++)
2380     // Base pointers src, dst are biased by 8*(count-1),to last element.
2381     __ align(OptoLoopAlignment);
2382 
2383     __ BIND(L_store_element);
2384     __ store_heap_oop(to_element_addr, rax_oop);  // store the oop
2385     __ increment(count);               // increment the count toward zero
2386     __ jcc(Assembler::zero, L_do_card_marks);
2387 
2388     // ======== loop entry is here ========
2389     __ BIND(L_load_element);
2390     __ load_heap_oop(rax_oop, from_element_addr); // load the oop
2391     __ testptr(rax_oop, rax_oop);
2392     __ jcc(Assembler::zero, L_store_element);
2393 
2394     __ load_klass(r11_klass, rax_oop);// query the object klass
2395     generate_type_check(r11_klass, ckoff, ckval, L_store_element);
2396     // ======== end loop ========
2397 
2398     // It was a real error; we must depend on the caller to finish the job.
2399     // Register rdx = -1 * number of *remaining* oops, r14 = *total* oops.
2400     // Emit GC store barriers for the oops we have copied (r14 + rdx),
2401     // and report their number to the caller.
2402     assert_different_registers(rax, r14_length, count, to, end_to, rcx, rscratch1);
2403     Label L_post_barrier;
2404     __ addptr(r14_length, count);     // K = (original - remaining) oops
2405     __ movptr(rax, r14_length);       // save the value
2406     __ notptr(rax);                   // report (-1^K) to caller (does not affect flags)
2407     __ jccb(Assembler::notZero, L_post_barrier);
2408     __ jmp(L_done); // K == 0, nothing was copied, skip post barrier
2409 
2410     // Come here on success only.
2411     __ BIND(L_do_card_marks);
2412     __ xorptr(rax, rax);              // return 0 on success
2413 
2414     __ BIND(L_post_barrier);
2415     gen_write_ref_array_post_barrier(to, r14_length, rscratch1);
2416 
2417     // Common exit point (success or failure).
2418     __ BIND(L_done);
2419     __ movptr(r13, Address(rsp, saved_r13_offset * wordSize));
2420     __ movptr(r14, Address(rsp, saved_r14_offset * wordSize));
2421     restore_arg_regs();
2422     inc_counter_np(SharedRuntime::_checkcast_array_copy_ctr); // Update counter after rscratch1 is free
2423     __ leave(); // required for proper stackwalking of RuntimeStub frame
2424     __ ret(0);
2425 
2426     return start;
2427   }
2428 
2429   //
2430   //  Generate 'unsafe' array copy stub
2431   //  Though just as safe as the other stubs, it takes an unscaled
2432   //  size_t argument instead of an element count.
2433   //
2434   //  Input:
2435   //    c_rarg0   - source array address
2436   //    c_rarg1   - destination array address
2437   //    c_rarg2   - byte count, treated as ssize_t, can be zero
2438   //
2439   // Examines the alignment of the operands and dispatches
2440   // to a long, int, short, or byte copy loop.
2441   //
2442   address generate_unsafe_copy(const char *name,
2443                                address byte_copy_entry, address short_copy_entry,
2444                                address int_copy_entry, address long_copy_entry) {
2445 
2446     Label L_long_aligned, L_int_aligned, L_short_aligned;
2447 
2448     // Input registers (before setup_arg_regs)
2449     const Register from        = c_rarg0;  // source array address
2450     const Register to          = c_rarg1;  // destination array address
2451     const Register size        = c_rarg2;  // byte count (size_t)
2452 
2453     // Register used as a temp
2454     const Register bits        = rax;      // test copy of low bits
2455 
2456     __ align(CodeEntryAlignment);
2457     StubCodeMark mark(this, "StubRoutines", name);
2458     address start = __ pc();
2459 
2460     __ enter(); // required for proper stackwalking of RuntimeStub frame
2461 
2462     // bump this on entry, not on exit:
2463     inc_counter_np(SharedRuntime::_unsafe_array_copy_ctr);
2464 
2465     __ mov(bits, from);
2466     __ orptr(bits, to);
2467     __ orptr(bits, size);
2468 
2469     __ testb(bits, BytesPerLong-1);
2470     __ jccb(Assembler::zero, L_long_aligned);
2471 
2472     __ testb(bits, BytesPerInt-1);
2473     __ jccb(Assembler::zero, L_int_aligned);
2474 
2475     __ testb(bits, BytesPerShort-1);
2476     __ jump_cc(Assembler::notZero, RuntimeAddress(byte_copy_entry));
2477 
2478     __ BIND(L_short_aligned);
2479     __ shrptr(size, LogBytesPerShort); // size => short_count
2480     __ jump(RuntimeAddress(short_copy_entry));
2481 
2482     __ BIND(L_int_aligned);
2483     __ shrptr(size, LogBytesPerInt); // size => int_count
2484     __ jump(RuntimeAddress(int_copy_entry));
2485 
2486     __ BIND(L_long_aligned);
2487     __ shrptr(size, LogBytesPerLong); // size => qword_count
2488     __ jump(RuntimeAddress(long_copy_entry));
2489 
2490     return start;
2491   }
2492 
2493   // Perform range checks on the proposed arraycopy.
2494   // Kills temp, but nothing else.
2495   // Also, clean the sign bits of src_pos and dst_pos.
2496   void arraycopy_range_checks(Register src,     // source array oop (c_rarg0)
2497                               Register src_pos, // source position (c_rarg1)
2498                               Register dst,     // destination array oo (c_rarg2)
2499                               Register dst_pos, // destination position (c_rarg3)
2500                               Register length,
2501                               Register temp,
2502                               Label& L_failed) {
2503     BLOCK_COMMENT("arraycopy_range_checks:");
2504 
2505     //  if (src_pos + length > arrayOop(src)->length())  FAIL;
2506     __ movl(temp, length);
2507     __ addl(temp, src_pos);             // src_pos + length
2508     __ cmpl(temp, Address(src, arrayOopDesc::length_offset_in_bytes()));
2509     __ jcc(Assembler::above, L_failed);
2510 
2511     //  if (dst_pos + length > arrayOop(dst)->length())  FAIL;
2512     __ movl(temp, length);
2513     __ addl(temp, dst_pos);             // dst_pos + length
2514     __ cmpl(temp, Address(dst, arrayOopDesc::length_offset_in_bytes()));
2515     __ jcc(Assembler::above, L_failed);
2516 
2517     // Have to clean up high 32-bits of 'src_pos' and 'dst_pos'.
2518     // Move with sign extension can be used since they are positive.
2519     __ movslq(src_pos, src_pos);
2520     __ movslq(dst_pos, dst_pos);
2521 
2522     BLOCK_COMMENT("arraycopy_range_checks done");
2523   }
2524 
2525   //
2526   //  Generate generic array copy stubs
2527   //
2528   //  Input:
2529   //    c_rarg0    -  src oop
2530   //    c_rarg1    -  src_pos (32-bits)
2531   //    c_rarg2    -  dst oop
2532   //    c_rarg3    -  dst_pos (32-bits)
2533   // not Win64
2534   //    c_rarg4    -  element count (32-bits)
2535   // Win64
2536   //    rsp+40     -  element count (32-bits)
2537   //
2538   //  Output:
2539   //    rax ==  0  -  success
2540   //    rax == -1^K - failure, where K is partial transfer count
2541   //
2542   address generate_generic_copy(const char *name,
2543                                 address byte_copy_entry, address short_copy_entry,
2544                                 address int_copy_entry, address oop_copy_entry,
2545                                 address long_copy_entry, address checkcast_copy_entry) {
2546 
2547     Label L_failed, L_failed_0, L_objArray;
2548     Label L_copy_bytes, L_copy_shorts, L_copy_ints, L_copy_longs;
2549 
2550     // Input registers
2551     const Register src        = c_rarg0;  // source array oop
2552     const Register src_pos    = c_rarg1;  // source position
2553     const Register dst        = c_rarg2;  // destination array oop
2554     const Register dst_pos    = c_rarg3;  // destination position
2555 #ifndef _WIN64
2556     const Register length     = c_rarg4;
2557 #else
2558     const Address  length(rsp, 6 * wordSize);  // elements count is on stack on Win64
2559 #endif
2560 
2561     { int modulus = CodeEntryAlignment;
2562       int target  = modulus - 5; // 5 = sizeof jmp(L_failed)
2563       int advance = target - (__ offset() % modulus);
2564       if (advance < 0)  advance += modulus;
2565       if (advance > 0)  __ nop(advance);
2566     }
2567     StubCodeMark mark(this, "StubRoutines", name);
2568 
2569     // Short-hop target to L_failed.  Makes for denser prologue code.
2570     __ BIND(L_failed_0);
2571     __ jmp(L_failed);
2572     assert(__ offset() % CodeEntryAlignment == 0, "no further alignment needed");
2573 
2574     __ align(CodeEntryAlignment);
2575     address start = __ pc();
2576 
2577     __ enter(); // required for proper stackwalking of RuntimeStub frame
2578 
2579     // bump this on entry, not on exit:
2580     inc_counter_np(SharedRuntime::_generic_array_copy_ctr);
2581 
2582     //-----------------------------------------------------------------------
2583     // Assembler stub will be used for this call to arraycopy
2584     // if the following conditions are met:
2585     //
2586     // (1) src and dst must not be null.
2587     // (2) src_pos must not be negative.
2588     // (3) dst_pos must not be negative.
2589     // (4) length  must not be negative.
2590     // (5) src klass and dst klass should be the same and not NULL.
2591     // (6) src and dst should be arrays.
2592     // (7) src_pos + length must not exceed length of src.
2593     // (8) dst_pos + length must not exceed length of dst.
2594     //
2595 
2596     //  if (src == NULL) return -1;
2597     __ testptr(src, src);         // src oop
2598     size_t j1off = __ offset();
2599     __ jccb(Assembler::zero, L_failed_0);
2600 
2601     //  if (src_pos < 0) return -1;
2602     __ testl(src_pos, src_pos); // src_pos (32-bits)
2603     __ jccb(Assembler::negative, L_failed_0);
2604 
2605     //  if (dst == NULL) return -1;
2606     __ testptr(dst, dst);         // dst oop
2607     __ jccb(Assembler::zero, L_failed_0);
2608 
2609     //  if (dst_pos < 0) return -1;
2610     __ testl(dst_pos, dst_pos); // dst_pos (32-bits)
2611     size_t j4off = __ offset();
2612     __ jccb(Assembler::negative, L_failed_0);
2613 
2614     // The first four tests are very dense code,
2615     // but not quite dense enough to put four
2616     // jumps in a 16-byte instruction fetch buffer.
2617     // That's good, because some branch predicters
2618     // do not like jumps so close together.
2619     // Make sure of this.
2620     guarantee(((j1off ^ j4off) & ~15) != 0, "I$ line of 1st & 4th jumps");
2621 
2622     // registers used as temp
2623     const Register r11_length    = r11; // elements count to copy
2624     const Register r10_src_klass = r10; // array klass
2625 
2626     //  if (length < 0) return -1;
2627     __ movl(r11_length, length);        // length (elements count, 32-bits value)
2628     __ testl(r11_length, r11_length);
2629     __ jccb(Assembler::negative, L_failed_0);
2630 
2631     __ load_klass(r10_src_klass, src);
2632 #ifdef ASSERT
2633     //  assert(src->klass() != NULL);
2634     {
2635       BLOCK_COMMENT("assert klasses not null {");
2636       Label L1, L2;
2637       __ testptr(r10_src_klass, r10_src_klass);
2638       __ jcc(Assembler::notZero, L2);   // it is broken if klass is NULL
2639       __ bind(L1);
2640       __ stop("broken null klass");
2641       __ bind(L2);
2642       __ load_klass(rax, dst);
2643       __ cmpq(rax, 0);
2644       __ jcc(Assembler::equal, L1);     // this would be broken also
2645       BLOCK_COMMENT("} assert klasses not null done");
2646     }
2647 #endif
2648 
2649     // Load layout helper (32-bits)
2650     //
2651     //  |array_tag|     | header_size | element_type |     |log2_element_size|
2652     // 32        30    24            16              8     2                 0
2653     //
2654     //   array_tag: typeArray = 0x3, objArray = 0x2, non-array = 0x0
2655     //
2656 
2657     const int lh_offset = in_bytes(Klass::layout_helper_offset());
2658 
2659     // Handle objArrays completely differently...
2660     const jint objArray_lh = Klass::array_layout_helper(T_OBJECT);
2661     __ cmpl(Address(r10_src_klass, lh_offset), objArray_lh);
2662     __ jcc(Assembler::equal, L_objArray);
2663 
2664     //  if (src->klass() != dst->klass()) return -1;
2665     __ load_klass(rax, dst);
2666     __ cmpq(r10_src_klass, rax);
2667     __ jcc(Assembler::notEqual, L_failed);
2668 
2669     const Register rax_lh = rax;  // layout helper
2670     __ movl(rax_lh, Address(r10_src_klass, lh_offset));
2671 
2672     //  if (!src->is_Array()) return -1;
2673     __ cmpl(rax_lh, Klass::_lh_neutral_value);
2674     __ jcc(Assembler::greaterEqual, L_failed);
2675 
2676     // At this point, it is known to be a typeArray (array_tag 0x3).
2677 #ifdef ASSERT
2678     {
2679       BLOCK_COMMENT("assert primitive array {");
2680       Label L;
2681       __ cmpl(rax_lh, (Klass::_lh_array_tag_type_value << Klass::_lh_array_tag_shift));
2682       __ jcc(Assembler::greaterEqual, L);
2683       __ stop("must be a primitive array");
2684       __ bind(L);
2685       BLOCK_COMMENT("} assert primitive array done");
2686     }
2687 #endif
2688 
2689     arraycopy_range_checks(src, src_pos, dst, dst_pos, r11_length,
2690                            r10, L_failed);
2691 
2692     // TypeArrayKlass
2693     //
2694     // src_addr = (src + array_header_in_bytes()) + (src_pos << log2elemsize);
2695     // dst_addr = (dst + array_header_in_bytes()) + (dst_pos << log2elemsize);
2696     //
2697 
2698     const Register r10_offset = r10;    // array offset
2699     const Register rax_elsize = rax_lh; // element size
2700 
2701     __ movl(r10_offset, rax_lh);
2702     __ shrl(r10_offset, Klass::_lh_header_size_shift);
2703     __ andptr(r10_offset, Klass::_lh_header_size_mask);   // array_offset
2704     __ addptr(src, r10_offset);           // src array offset
2705     __ addptr(dst, r10_offset);           // dst array offset
2706     BLOCK_COMMENT("choose copy loop based on element size");
2707     __ andl(rax_lh, Klass::_lh_log2_element_size_mask); // rax_lh -> rax_elsize
2708 
2709     // next registers should be set before the jump to corresponding stub
2710     const Register from     = c_rarg0;  // source array address
2711     const Register to       = c_rarg1;  // destination array address
2712     const Register count    = c_rarg2;  // elements count
2713 
2714     // 'from', 'to', 'count' registers should be set in such order
2715     // since they are the same as 'src', 'src_pos', 'dst'.
2716 
2717   __ BIND(L_copy_bytes);
2718     __ cmpl(rax_elsize, 0);
2719     __ jccb(Assembler::notEqual, L_copy_shorts);
2720     __ lea(from, Address(src, src_pos, Address::times_1, 0));// src_addr
2721     __ lea(to,   Address(dst, dst_pos, Address::times_1, 0));// dst_addr
2722     __ movl2ptr(count, r11_length); // length
2723     __ jump(RuntimeAddress(byte_copy_entry));
2724 
2725   __ BIND(L_copy_shorts);
2726     __ cmpl(rax_elsize, LogBytesPerShort);
2727     __ jccb(Assembler::notEqual, L_copy_ints);
2728     __ lea(from, Address(src, src_pos, Address::times_2, 0));// src_addr
2729     __ lea(to,   Address(dst, dst_pos, Address::times_2, 0));// dst_addr
2730     __ movl2ptr(count, r11_length); // length
2731     __ jump(RuntimeAddress(short_copy_entry));
2732 
2733   __ BIND(L_copy_ints);
2734     __ cmpl(rax_elsize, LogBytesPerInt);
2735     __ jccb(Assembler::notEqual, L_copy_longs);
2736     __ lea(from, Address(src, src_pos, Address::times_4, 0));// src_addr
2737     __ lea(to,   Address(dst, dst_pos, Address::times_4, 0));// dst_addr
2738     __ movl2ptr(count, r11_length); // length
2739     __ jump(RuntimeAddress(int_copy_entry));
2740 
2741   __ BIND(L_copy_longs);
2742 #ifdef ASSERT
2743     {
2744       BLOCK_COMMENT("assert long copy {");
2745       Label L;
2746       __ cmpl(rax_elsize, LogBytesPerLong);
2747       __ jcc(Assembler::equal, L);
2748       __ stop("must be long copy, but elsize is wrong");
2749       __ bind(L);
2750       BLOCK_COMMENT("} assert long copy done");
2751     }
2752 #endif
2753     __ lea(from, Address(src, src_pos, Address::times_8, 0));// src_addr
2754     __ lea(to,   Address(dst, dst_pos, Address::times_8, 0));// dst_addr
2755     __ movl2ptr(count, r11_length); // length
2756     __ jump(RuntimeAddress(long_copy_entry));
2757 
2758     // ObjArrayKlass
2759   __ BIND(L_objArray);
2760     // live at this point:  r10_src_klass, r11_length, src[_pos], dst[_pos]
2761 
2762     Label L_plain_copy, L_checkcast_copy;
2763     //  test array classes for subtyping
2764     __ load_klass(rax, dst);
2765     __ cmpq(r10_src_klass, rax); // usual case is exact equality
2766     __ jcc(Assembler::notEqual, L_checkcast_copy);
2767 
2768     // Identically typed arrays can be copied without element-wise checks.
2769     arraycopy_range_checks(src, src_pos, dst, dst_pos, r11_length,
2770                            r10, L_failed);
2771 
2772     __ lea(from, Address(src, src_pos, TIMES_OOP,
2773                  arrayOopDesc::base_offset_in_bytes(T_OBJECT))); // src_addr
2774     __ lea(to,   Address(dst, dst_pos, TIMES_OOP,
2775                  arrayOopDesc::base_offset_in_bytes(T_OBJECT))); // dst_addr
2776     __ movl2ptr(count, r11_length); // length
2777   __ BIND(L_plain_copy);
2778     __ jump(RuntimeAddress(oop_copy_entry));
2779 
2780   __ BIND(L_checkcast_copy);
2781     // live at this point:  r10_src_klass, r11_length, rax (dst_klass)
2782     {
2783       // Before looking at dst.length, make sure dst is also an objArray.
2784       __ cmpl(Address(rax, lh_offset), objArray_lh);
2785       __ jcc(Assembler::notEqual, L_failed);
2786 
2787       // It is safe to examine both src.length and dst.length.
2788       arraycopy_range_checks(src, src_pos, dst, dst_pos, r11_length,
2789                              rax, L_failed);
2790 
2791       const Register r11_dst_klass = r11;
2792       __ load_klass(r11_dst_klass, dst); // reload
2793 
2794       // Marshal the base address arguments now, freeing registers.
2795       __ lea(from, Address(src, src_pos, TIMES_OOP,
2796                    arrayOopDesc::base_offset_in_bytes(T_OBJECT)));
2797       __ lea(to,   Address(dst, dst_pos, TIMES_OOP,
2798                    arrayOopDesc::base_offset_in_bytes(T_OBJECT)));
2799       __ movl(count, length);           // length (reloaded)
2800       Register sco_temp = c_rarg3;      // this register is free now
2801       assert_different_registers(from, to, count, sco_temp,
2802                                  r11_dst_klass, r10_src_klass);
2803       assert_clean_int(count, sco_temp);
2804 
2805       // Generate the type check.
2806       const int sco_offset = in_bytes(Klass::super_check_offset_offset());
2807       __ movl(sco_temp, Address(r11_dst_klass, sco_offset));
2808       assert_clean_int(sco_temp, rax);
2809       generate_type_check(r10_src_klass, sco_temp, r11_dst_klass, L_plain_copy);
2810 
2811       // Fetch destination element klass from the ObjArrayKlass header.
2812       int ek_offset = in_bytes(ObjArrayKlass::element_klass_offset());
2813       __ movptr(r11_dst_klass, Address(r11_dst_klass, ek_offset));
2814       __ movl(  sco_temp,      Address(r11_dst_klass, sco_offset));
2815       assert_clean_int(sco_temp, rax);
2816 
2817       // the checkcast_copy loop needs two extra arguments:
2818       assert(c_rarg3 == sco_temp, "#3 already in place");
2819       // Set up arguments for checkcast_copy_entry.
2820       setup_arg_regs(4);
2821       __ movptr(r8, r11_dst_klass);  // dst.klass.element_klass, r8 is c_rarg4 on Linux/Solaris
2822       __ jump(RuntimeAddress(checkcast_copy_entry));
2823     }
2824 
2825   __ BIND(L_failed);
2826     __ xorptr(rax, rax);
2827     __ notptr(rax); // return -1
2828     __ leave();   // required for proper stackwalking of RuntimeStub frame
2829     __ ret(0);
2830 
2831     return start;
2832   }
2833 
2834   void generate_arraycopy_stubs() {
2835     address entry;
2836     address entry_jbyte_arraycopy;
2837     address entry_jshort_arraycopy;
2838     address entry_jint_arraycopy;
2839     address entry_oop_arraycopy;
2840     address entry_jlong_arraycopy;
2841     address entry_checkcast_arraycopy;
2842 
2843     StubRoutines::_jbyte_disjoint_arraycopy  = generate_disjoint_byte_copy(false, &entry,
2844                                                                            "jbyte_disjoint_arraycopy");
2845     StubRoutines::_jbyte_arraycopy           = generate_conjoint_byte_copy(false, entry, &entry_jbyte_arraycopy,
2846                                                                            "jbyte_arraycopy");
2847 
2848     StubRoutines::_jshort_disjoint_arraycopy = generate_disjoint_short_copy(false, &entry,
2849                                                                             "jshort_disjoint_arraycopy");
2850     StubRoutines::_jshort_arraycopy          = generate_conjoint_short_copy(false, entry, &entry_jshort_arraycopy,
2851                                                                             "jshort_arraycopy");
2852 
2853     StubRoutines::_jint_disjoint_arraycopy   = generate_disjoint_int_oop_copy(false, false, &entry,
2854                                                                               "jint_disjoint_arraycopy");
2855     StubRoutines::_jint_arraycopy            = generate_conjoint_int_oop_copy(false, false, entry,
2856                                                                               &entry_jint_arraycopy, "jint_arraycopy");
2857 
2858     StubRoutines::_jlong_disjoint_arraycopy  = generate_disjoint_long_oop_copy(false, false, &entry,
2859                                                                                "jlong_disjoint_arraycopy");
2860     StubRoutines::_jlong_arraycopy           = generate_conjoint_long_oop_copy(false, false, entry,
2861                                                                                &entry_jlong_arraycopy, "jlong_arraycopy");
2862 
2863 
2864     if (UseCompressedOops) {
2865       StubRoutines::_oop_disjoint_arraycopy  = generate_disjoint_int_oop_copy(false, true, &entry,
2866                                                                               "oop_disjoint_arraycopy");
2867       StubRoutines::_oop_arraycopy           = generate_conjoint_int_oop_copy(false, true, entry,
2868                                                                               &entry_oop_arraycopy, "oop_arraycopy");
2869       StubRoutines::_oop_disjoint_arraycopy_uninit  = generate_disjoint_int_oop_copy(false, true, &entry,
2870                                                                                      "oop_disjoint_arraycopy_uninit",
2871                                                                                      /*dest_uninitialized*/true);
2872       StubRoutines::_oop_arraycopy_uninit           = generate_conjoint_int_oop_copy(false, true, entry,
2873                                                                                      NULL, "oop_arraycopy_uninit",
2874                                                                                      /*dest_uninitialized*/true);
2875     } else {
2876       StubRoutines::_oop_disjoint_arraycopy  = generate_disjoint_long_oop_copy(false, true, &entry,
2877                                                                                "oop_disjoint_arraycopy");
2878       StubRoutines::_oop_arraycopy           = generate_conjoint_long_oop_copy(false, true, entry,
2879                                                                                &entry_oop_arraycopy, "oop_arraycopy");
2880       StubRoutines::_oop_disjoint_arraycopy_uninit  = generate_disjoint_long_oop_copy(false, true, &entry,
2881                                                                                       "oop_disjoint_arraycopy_uninit",
2882                                                                                       /*dest_uninitialized*/true);
2883       StubRoutines::_oop_arraycopy_uninit           = generate_conjoint_long_oop_copy(false, true, entry,
2884                                                                                       NULL, "oop_arraycopy_uninit",
2885                                                                                       /*dest_uninitialized*/true);
2886     }
2887 
2888     StubRoutines::_checkcast_arraycopy        = generate_checkcast_copy("checkcast_arraycopy", &entry_checkcast_arraycopy);
2889     StubRoutines::_checkcast_arraycopy_uninit = generate_checkcast_copy("checkcast_arraycopy_uninit", NULL,
2890                                                                         /*dest_uninitialized*/true);
2891 
2892     StubRoutines::_unsafe_arraycopy    = generate_unsafe_copy("unsafe_arraycopy",
2893                                                               entry_jbyte_arraycopy,
2894                                                               entry_jshort_arraycopy,
2895                                                               entry_jint_arraycopy,
2896                                                               entry_jlong_arraycopy);
2897     StubRoutines::_generic_arraycopy   = generate_generic_copy("generic_arraycopy",
2898                                                                entry_jbyte_arraycopy,
2899                                                                entry_jshort_arraycopy,
2900                                                                entry_jint_arraycopy,
2901                                                                entry_oop_arraycopy,
2902                                                                entry_jlong_arraycopy,
2903                                                                entry_checkcast_arraycopy);
2904 
2905     StubRoutines::_jbyte_fill = generate_fill(T_BYTE, false, "jbyte_fill");
2906     StubRoutines::_jshort_fill = generate_fill(T_SHORT, false, "jshort_fill");
2907     StubRoutines::_jint_fill = generate_fill(T_INT, false, "jint_fill");
2908     StubRoutines::_arrayof_jbyte_fill = generate_fill(T_BYTE, true, "arrayof_jbyte_fill");
2909     StubRoutines::_arrayof_jshort_fill = generate_fill(T_SHORT, true, "arrayof_jshort_fill");
2910     StubRoutines::_arrayof_jint_fill = generate_fill(T_INT, true, "arrayof_jint_fill");
2911 
2912     // We don't generate specialized code for HeapWord-aligned source
2913     // arrays, so just use the code we've already generated
2914     StubRoutines::_arrayof_jbyte_disjoint_arraycopy  = StubRoutines::_jbyte_disjoint_arraycopy;
2915     StubRoutines::_arrayof_jbyte_arraycopy           = StubRoutines::_jbyte_arraycopy;
2916 
2917     StubRoutines::_arrayof_jshort_disjoint_arraycopy = StubRoutines::_jshort_disjoint_arraycopy;
2918     StubRoutines::_arrayof_jshort_arraycopy          = StubRoutines::_jshort_arraycopy;
2919 
2920     StubRoutines::_arrayof_jint_disjoint_arraycopy   = StubRoutines::_jint_disjoint_arraycopy;
2921     StubRoutines::_arrayof_jint_arraycopy            = StubRoutines::_jint_arraycopy;
2922 
2923     StubRoutines::_arrayof_jlong_disjoint_arraycopy  = StubRoutines::_jlong_disjoint_arraycopy;
2924     StubRoutines::_arrayof_jlong_arraycopy           = StubRoutines::_jlong_arraycopy;
2925 
2926     StubRoutines::_arrayof_oop_disjoint_arraycopy    = StubRoutines::_oop_disjoint_arraycopy;
2927     StubRoutines::_arrayof_oop_arraycopy             = StubRoutines::_oop_arraycopy;
2928 
2929     StubRoutines::_arrayof_oop_disjoint_arraycopy_uninit    = StubRoutines::_oop_disjoint_arraycopy_uninit;
2930     StubRoutines::_arrayof_oop_arraycopy_uninit             = StubRoutines::_oop_arraycopy_uninit;
2931   }
2932 
2933   // AES intrinsic stubs
2934   enum {AESBlockSize = 16};
2935 
2936   address generate_key_shuffle_mask() {
2937     __ align(16);
2938     StubCodeMark mark(this, "StubRoutines", "key_shuffle_mask");
2939     address start = __ pc();
2940     __ emit_data64( 0x0405060700010203, relocInfo::none );
2941     __ emit_data64( 0x0c0d0e0f08090a0b, relocInfo::none );
2942     return start;
2943   }
2944 
2945   address generate_counter_shuffle_mask() {
2946     __ align(16);
2947     StubCodeMark mark(this, "StubRoutines", "counter_shuffle_mask");
2948     address start = __ pc();
2949     __ emit_data64(0x08090a0b0c0d0e0f, relocInfo::none);
2950     __ emit_data64(0x0001020304050607, relocInfo::none);
2951     return start;
2952   }
2953 
2954   // Utility routine for loading a 128-bit key word in little endian format
2955   // can optionally specify that the shuffle mask is already in an xmmregister
2956   void load_key(XMMRegister xmmdst, Register key, int offset, XMMRegister xmm_shuf_mask=NULL) {
2957     __ movdqu(xmmdst, Address(key, offset));
2958     if (xmm_shuf_mask != NULL) {
2959       __ pshufb(xmmdst, xmm_shuf_mask);
2960     } else {
2961       __ pshufb(xmmdst, ExternalAddress(StubRoutines::x86::key_shuffle_mask_addr()));
2962     }
2963   }
2964 
2965   // Utility routine for increase 128bit counter (iv in CTR mode)
2966   void inc_counter(Register reg, XMMRegister xmmdst, int inc_delta, Label& next_block) {
2967     __ pextrq(reg, xmmdst, 0x0);
2968     __ addq(reg, inc_delta);
2969     __ pinsrq(xmmdst, reg, 0x0);
2970     __ jcc(Assembler::carryClear, next_block); // jump if no carry
2971     __ pextrq(reg, xmmdst, 0x01); // Carry
2972     __ addq(reg, 0x01);
2973     __ pinsrq(xmmdst, reg, 0x01); //Carry end
2974     __ BIND(next_block);          // next instruction
2975   }
2976 
2977   // Arguments:
2978   //
2979   // Inputs:
2980   //   c_rarg0   - source byte array address
2981   //   c_rarg1   - destination byte array address
2982   //   c_rarg2   - K (key) in little endian int array
2983   //
2984   address generate_aescrypt_encryptBlock() {
2985     assert(UseAES, "need AES instructions and misaligned SSE support");
2986     __ align(CodeEntryAlignment);
2987     StubCodeMark mark(this, "StubRoutines", "aescrypt_encryptBlock");
2988     Label L_doLast;
2989     address start = __ pc();
2990 
2991     const Register from        = c_rarg0;  // source array address
2992     const Register to          = c_rarg1;  // destination array address
2993     const Register key         = c_rarg2;  // key array address
2994     const Register keylen      = rax;
2995 
2996     const XMMRegister xmm_result = xmm0;
2997     const XMMRegister xmm_key_shuf_mask = xmm1;
2998     // On win64 xmm6-xmm15 must be preserved so don't use them.
2999     const XMMRegister xmm_temp1  = xmm2;
3000     const XMMRegister xmm_temp2  = xmm3;
3001     const XMMRegister xmm_temp3  = xmm4;
3002     const XMMRegister xmm_temp4  = xmm5;
3003 
3004     __ enter(); // required for proper stackwalking of RuntimeStub frame
3005 
3006     // For EVEX with VL and BW, provide a standard mask, VL = 128 will guide the merge
3007     // context for the registers used, where all instructions below are using 128-bit mode
3008     // On EVEX without VL and BW, these instructions will all be AVX.
3009     if (VM_Version::supports_avx512vlbw()) {
3010       __ movl(rax, 0xffff);
3011       __ kmovql(k1, rax);
3012     }
3013 
3014     // keylen could be only {11, 13, 15} * 4 = {44, 52, 60}
3015     __ movl(keylen, Address(key, arrayOopDesc::length_offset_in_bytes() - arrayOopDesc::base_offset_in_bytes(T_INT)));
3016 
3017     __ movdqu(xmm_key_shuf_mask, ExternalAddress(StubRoutines::x86::key_shuffle_mask_addr()));
3018     __ movdqu(xmm_result, Address(from, 0));  // get 16 bytes of input
3019 
3020     // For encryption, the java expanded key ordering is just what we need
3021     // we don't know if the key is aligned, hence not using load-execute form
3022 
3023     load_key(xmm_temp1, key, 0x00, xmm_key_shuf_mask);
3024     __ pxor(xmm_result, xmm_temp1);
3025 
3026     load_key(xmm_temp1, key, 0x10, xmm_key_shuf_mask);
3027     load_key(xmm_temp2, key, 0x20, xmm_key_shuf_mask);
3028     load_key(xmm_temp3, key, 0x30, xmm_key_shuf_mask);
3029     load_key(xmm_temp4, key, 0x40, xmm_key_shuf_mask);
3030 
3031     __ aesenc(xmm_result, xmm_temp1);
3032     __ aesenc(xmm_result, xmm_temp2);
3033     __ aesenc(xmm_result, xmm_temp3);
3034     __ aesenc(xmm_result, xmm_temp4);
3035 
3036     load_key(xmm_temp1, key, 0x50, xmm_key_shuf_mask);
3037     load_key(xmm_temp2, key, 0x60, xmm_key_shuf_mask);
3038     load_key(xmm_temp3, key, 0x70, xmm_key_shuf_mask);
3039     load_key(xmm_temp4, key, 0x80, xmm_key_shuf_mask);
3040 
3041     __ aesenc(xmm_result, xmm_temp1);
3042     __ aesenc(xmm_result, xmm_temp2);
3043     __ aesenc(xmm_result, xmm_temp3);
3044     __ aesenc(xmm_result, xmm_temp4);
3045 
3046     load_key(xmm_temp1, key, 0x90, xmm_key_shuf_mask);
3047     load_key(xmm_temp2, key, 0xa0, xmm_key_shuf_mask);
3048 
3049     __ cmpl(keylen, 44);
3050     __ jccb(Assembler::equal, L_doLast);
3051 
3052     __ aesenc(xmm_result, xmm_temp1);
3053     __ aesenc(xmm_result, xmm_temp2);
3054 
3055     load_key(xmm_temp1, key, 0xb0, xmm_key_shuf_mask);
3056     load_key(xmm_temp2, key, 0xc0, xmm_key_shuf_mask);
3057 
3058     __ cmpl(keylen, 52);
3059     __ jccb(Assembler::equal, L_doLast);
3060 
3061     __ aesenc(xmm_result, xmm_temp1);
3062     __ aesenc(xmm_result, xmm_temp2);
3063 
3064     load_key(xmm_temp1, key, 0xd0, xmm_key_shuf_mask);
3065     load_key(xmm_temp2, key, 0xe0, xmm_key_shuf_mask);
3066 
3067     __ BIND(L_doLast);
3068     __ aesenc(xmm_result, xmm_temp1);
3069     __ aesenclast(xmm_result, xmm_temp2);
3070     __ movdqu(Address(to, 0), xmm_result);        // store the result
3071     __ xorptr(rax, rax); // return 0
3072     __ leave(); // required for proper stackwalking of RuntimeStub frame
3073     __ ret(0);
3074 
3075     return start;
3076   }
3077 
3078 
3079   // Arguments:
3080   //
3081   // Inputs:
3082   //   c_rarg0   - source byte array address
3083   //   c_rarg1   - destination byte array address
3084   //   c_rarg2   - K (key) in little endian int array
3085   //
3086   address generate_aescrypt_decryptBlock() {
3087     assert(UseAES, "need AES instructions and misaligned SSE support");
3088     __ align(CodeEntryAlignment);
3089     StubCodeMark mark(this, "StubRoutines", "aescrypt_decryptBlock");
3090     Label L_doLast;
3091     address start = __ pc();
3092 
3093     const Register from        = c_rarg0;  // source array address
3094     const Register to          = c_rarg1;  // destination array address
3095     const Register key         = c_rarg2;  // key array address
3096     const Register keylen      = rax;
3097 
3098     const XMMRegister xmm_result = xmm0;
3099     const XMMRegister xmm_key_shuf_mask = xmm1;
3100     // On win64 xmm6-xmm15 must be preserved so don't use them.
3101     const XMMRegister xmm_temp1  = xmm2;
3102     const XMMRegister xmm_temp2  = xmm3;
3103     const XMMRegister xmm_temp3  = xmm4;
3104     const XMMRegister xmm_temp4  = xmm5;
3105 
3106     __ enter(); // required for proper stackwalking of RuntimeStub frame
3107 
3108     // For EVEX with VL and BW, provide a standard mask, VL = 128 will guide the merge
3109     // context for the registers used, where all instructions below are using 128-bit mode
3110     // On EVEX without VL and BW, these instructions will all be AVX.
3111     if (VM_Version::supports_avx512vlbw()) {
3112       __ movl(rax, 0xffff);
3113       __ kmovql(k1, rax);
3114     }
3115 
3116     // keylen could be only {11, 13, 15} * 4 = {44, 52, 60}
3117     __ movl(keylen, Address(key, arrayOopDesc::length_offset_in_bytes() - arrayOopDesc::base_offset_in_bytes(T_INT)));
3118 
3119     __ movdqu(xmm_key_shuf_mask, ExternalAddress(StubRoutines::x86::key_shuffle_mask_addr()));
3120     __ movdqu(xmm_result, Address(from, 0));
3121 
3122     // for decryption java expanded key ordering is rotated one position from what we want
3123     // so we start from 0x10 here and hit 0x00 last
3124     // we don't know if the key is aligned, hence not using load-execute form
3125     load_key(xmm_temp1, key, 0x10, xmm_key_shuf_mask);
3126     load_key(xmm_temp2, key, 0x20, xmm_key_shuf_mask);
3127     load_key(xmm_temp3, key, 0x30, xmm_key_shuf_mask);
3128     load_key(xmm_temp4, key, 0x40, xmm_key_shuf_mask);
3129 
3130     __ pxor  (xmm_result, xmm_temp1);
3131     __ aesdec(xmm_result, xmm_temp2);
3132     __ aesdec(xmm_result, xmm_temp3);
3133     __ aesdec(xmm_result, xmm_temp4);
3134 
3135     load_key(xmm_temp1, key, 0x50, xmm_key_shuf_mask);
3136     load_key(xmm_temp2, key, 0x60, xmm_key_shuf_mask);
3137     load_key(xmm_temp3, key, 0x70, xmm_key_shuf_mask);
3138     load_key(xmm_temp4, key, 0x80, xmm_key_shuf_mask);
3139 
3140     __ aesdec(xmm_result, xmm_temp1);
3141     __ aesdec(xmm_result, xmm_temp2);
3142     __ aesdec(xmm_result, xmm_temp3);
3143     __ aesdec(xmm_result, xmm_temp4);
3144 
3145     load_key(xmm_temp1, key, 0x90, xmm_key_shuf_mask);
3146     load_key(xmm_temp2, key, 0xa0, xmm_key_shuf_mask);
3147     load_key(xmm_temp3, key, 0x00, xmm_key_shuf_mask);
3148 
3149     __ cmpl(keylen, 44);
3150     __ jccb(Assembler::equal, L_doLast);
3151 
3152     __ aesdec(xmm_result, xmm_temp1);
3153     __ aesdec(xmm_result, xmm_temp2);
3154 
3155     load_key(xmm_temp1, key, 0xb0, xmm_key_shuf_mask);
3156     load_key(xmm_temp2, key, 0xc0, xmm_key_shuf_mask);
3157 
3158     __ cmpl(keylen, 52);
3159     __ jccb(Assembler::equal, L_doLast);
3160 
3161     __ aesdec(xmm_result, xmm_temp1);
3162     __ aesdec(xmm_result, xmm_temp2);
3163 
3164     load_key(xmm_temp1, key, 0xd0, xmm_key_shuf_mask);
3165     load_key(xmm_temp2, key, 0xe0, xmm_key_shuf_mask);
3166 
3167     __ BIND(L_doLast);
3168     __ aesdec(xmm_result, xmm_temp1);
3169     __ aesdec(xmm_result, xmm_temp2);
3170 
3171     // for decryption the aesdeclast operation is always on key+0x00
3172     __ aesdeclast(xmm_result, xmm_temp3);
3173     __ movdqu(Address(to, 0), xmm_result);  // store the result
3174     __ xorptr(rax, rax); // return 0
3175     __ leave(); // required for proper stackwalking of RuntimeStub frame
3176     __ ret(0);
3177 
3178     return start;
3179   }
3180 
3181 
3182   // Arguments:
3183   //
3184   // Inputs:
3185   //   c_rarg0   - source byte array address
3186   //   c_rarg1   - destination byte array address
3187   //   c_rarg2   - K (key) in little endian int array
3188   //   c_rarg3   - r vector byte array address
3189   //   c_rarg4   - input length
3190   //
3191   // Output:
3192   //   rax       - input length
3193   //
3194   address generate_cipherBlockChaining_encryptAESCrypt() {
3195     assert(UseAES, "need AES instructions and misaligned SSE support");
3196     __ align(CodeEntryAlignment);
3197     StubCodeMark mark(this, "StubRoutines", "cipherBlockChaining_encryptAESCrypt");
3198     address start = __ pc();
3199 
3200     Label L_exit, L_key_192_256, L_key_256, L_loopTop_128, L_loopTop_192, L_loopTop_256;
3201     const Register from        = c_rarg0;  // source array address
3202     const Register to          = c_rarg1;  // destination array address
3203     const Register key         = c_rarg2;  // key array address
3204     const Register rvec        = c_rarg3;  // r byte array initialized from initvector array address
3205                                            // and left with the results of the last encryption block
3206 #ifndef _WIN64
3207     const Register len_reg     = c_rarg4;  // src len (must be multiple of blocksize 16)
3208 #else
3209     const Address  len_mem(rbp, 6 * wordSize);  // length is on stack on Win64
3210     const Register len_reg     = r10;      // pick the first volatile windows register
3211 #endif
3212     const Register pos         = rax;
3213 
3214     // xmm register assignments for the loops below
3215     const XMMRegister xmm_result = xmm0;
3216     const XMMRegister xmm_temp   = xmm1;
3217     // keys 0-10 preloaded into xmm2-xmm12
3218     const int XMM_REG_NUM_KEY_FIRST = 2;
3219     const int XMM_REG_NUM_KEY_LAST  = 15;
3220     const XMMRegister xmm_key0   = as_XMMRegister(XMM_REG_NUM_KEY_FIRST);
3221     const XMMRegister xmm_key10  = as_XMMRegister(XMM_REG_NUM_KEY_FIRST+10);
3222     const XMMRegister xmm_key11  = as_XMMRegister(XMM_REG_NUM_KEY_FIRST+11);
3223     const XMMRegister xmm_key12  = as_XMMRegister(XMM_REG_NUM_KEY_FIRST+12);
3224     const XMMRegister xmm_key13  = as_XMMRegister(XMM_REG_NUM_KEY_FIRST+13);
3225 
3226     __ enter(); // required for proper stackwalking of RuntimeStub frame
3227 
3228     // For EVEX with VL and BW, provide a standard mask, VL = 128 will guide the merge
3229     // context for the registers used, where all instructions below are using 128-bit mode
3230     // On EVEX without VL and BW, these instructions will all be AVX.
3231     if (VM_Version::supports_avx512vlbw()) {
3232       __ movl(rax, 0xffff);
3233       __ kmovql(k1, rax);
3234     }
3235 
3236 #ifdef _WIN64
3237     // on win64, fill len_reg from stack position
3238     __ movl(len_reg, len_mem);
3239 #else
3240     __ push(len_reg); // Save
3241 #endif
3242 
3243     const XMMRegister xmm_key_shuf_mask = xmm_temp;  // used temporarily to swap key bytes up front
3244     __ movdqu(xmm_key_shuf_mask, ExternalAddress(StubRoutines::x86::key_shuffle_mask_addr()));
3245     // load up xmm regs xmm2 thru xmm12 with key 0x00 - 0xa0
3246     for (int rnum = XMM_REG_NUM_KEY_FIRST, offset = 0x00; rnum <= XMM_REG_NUM_KEY_FIRST+10; rnum++) {
3247       load_key(as_XMMRegister(rnum), key, offset, xmm_key_shuf_mask);
3248       offset += 0x10;
3249     }
3250     __ movdqu(xmm_result, Address(rvec, 0x00));   // initialize xmm_result with r vec
3251 
3252     // now split to different paths depending on the keylen (len in ints of AESCrypt.KLE array (52=192, or 60=256))
3253     __ movl(rax, Address(key, arrayOopDesc::length_offset_in_bytes() - arrayOopDesc::base_offset_in_bytes(T_INT)));
3254     __ cmpl(rax, 44);
3255     __ jcc(Assembler::notEqual, L_key_192_256);
3256 
3257     // 128 bit code follows here
3258     __ movptr(pos, 0);
3259     __ align(OptoLoopAlignment);
3260 
3261     __ BIND(L_loopTop_128);
3262     __ movdqu(xmm_temp, Address(from, pos, Address::times_1, 0));   // get next 16 bytes of input
3263     __ pxor  (xmm_result, xmm_temp);               // xor with the current r vector
3264     __ pxor  (xmm_result, xmm_key0);               // do the aes rounds
3265     for (int rnum = XMM_REG_NUM_KEY_FIRST + 1; rnum <= XMM_REG_NUM_KEY_FIRST + 9; rnum++) {
3266       __ aesenc(xmm_result, as_XMMRegister(rnum));
3267     }
3268     __ aesenclast(xmm_result, xmm_key10);
3269     __ movdqu(Address(to, pos, Address::times_1, 0), xmm_result);     // store into the next 16 bytes of output
3270     // no need to store r to memory until we exit
3271     __ addptr(pos, AESBlockSize);
3272     __ subptr(len_reg, AESBlockSize);
3273     __ jcc(Assembler::notEqual, L_loopTop_128);
3274 
3275     __ BIND(L_exit);
3276     __ movdqu(Address(rvec, 0), xmm_result);     // final value of r stored in rvec of CipherBlockChaining object
3277 
3278 #ifdef _WIN64
3279     __ movl(rax, len_mem);
3280 #else
3281     __ pop(rax); // return length
3282 #endif
3283     __ leave(); // required for proper stackwalking of RuntimeStub frame
3284     __ ret(0);
3285 
3286     __ BIND(L_key_192_256);
3287     // here rax = len in ints of AESCrypt.KLE array (52=192, or 60=256)
3288     load_key(xmm_key11, key, 0xb0, xmm_key_shuf_mask);
3289     load_key(xmm_key12, key, 0xc0, xmm_key_shuf_mask);
3290     __ cmpl(rax, 52);
3291     __ jcc(Assembler::notEqual, L_key_256);
3292 
3293     // 192-bit code follows here (could be changed to use more xmm registers)
3294     __ movptr(pos, 0);
3295     __ align(OptoLoopAlignment);
3296 
3297     __ BIND(L_loopTop_192);
3298     __ movdqu(xmm_temp, Address(from, pos, Address::times_1, 0));   // get next 16 bytes of input
3299     __ pxor  (xmm_result, xmm_temp);               // xor with the current r vector
3300     __ pxor  (xmm_result, xmm_key0);               // do the aes rounds
3301     for (int rnum = XMM_REG_NUM_KEY_FIRST + 1; rnum  <= XMM_REG_NUM_KEY_FIRST + 11; rnum++) {
3302       __ aesenc(xmm_result, as_XMMRegister(rnum));
3303     }
3304     __ aesenclast(xmm_result, xmm_key12);
3305     __ movdqu(Address(to, pos, Address::times_1, 0), xmm_result);     // store into the next 16 bytes of output
3306     // no need to store r to memory until we exit
3307     __ addptr(pos, AESBlockSize);
3308     __ subptr(len_reg, AESBlockSize);
3309     __ jcc(Assembler::notEqual, L_loopTop_192);
3310     __ jmp(L_exit);
3311 
3312     __ BIND(L_key_256);
3313     // 256-bit code follows here (could be changed to use more xmm registers)
3314     load_key(xmm_key13, key, 0xd0, xmm_key_shuf_mask);
3315     __ movptr(pos, 0);
3316     __ align(OptoLoopAlignment);
3317 
3318     __ BIND(L_loopTop_256);
3319     __ movdqu(xmm_temp, Address(from, pos, Address::times_1, 0));   // get next 16 bytes of input
3320     __ pxor  (xmm_result, xmm_temp);               // xor with the current r vector
3321     __ pxor  (xmm_result, xmm_key0);               // do the aes rounds
3322     for (int rnum = XMM_REG_NUM_KEY_FIRST + 1; rnum  <= XMM_REG_NUM_KEY_FIRST + 13; rnum++) {
3323       __ aesenc(xmm_result, as_XMMRegister(rnum));
3324     }
3325     load_key(xmm_temp, key, 0xe0);
3326     __ aesenclast(xmm_result, xmm_temp);
3327     __ movdqu(Address(to, pos, Address::times_1, 0), xmm_result);     // store into the next 16 bytes of output
3328     // no need to store r to memory until we exit
3329     __ addptr(pos, AESBlockSize);
3330     __ subptr(len_reg, AESBlockSize);
3331     __ jcc(Assembler::notEqual, L_loopTop_256);
3332     __ jmp(L_exit);
3333 
3334     return start;
3335   }
3336 
3337   // Safefetch stubs.
3338   void generate_safefetch(const char* name, int size, address* entry,
3339                           address* fault_pc, address* continuation_pc) {
3340     // safefetch signatures:
3341     //   int      SafeFetch32(int*      adr, int      errValue);
3342     //   intptr_t SafeFetchN (intptr_t* adr, intptr_t errValue);
3343     //
3344     // arguments:
3345     //   c_rarg0 = adr
3346     //   c_rarg1 = errValue
3347     //
3348     // result:
3349     //   PPC_RET  = *adr or errValue
3350 
3351     StubCodeMark mark(this, "StubRoutines", name);
3352 
3353     // Entry point, pc or function descriptor.
3354     *entry = __ pc();
3355 
3356     // Load *adr into c_rarg1, may fault.
3357     *fault_pc = __ pc();
3358     switch (size) {
3359       case 4:
3360         // int32_t
3361         __ movl(c_rarg1, Address(c_rarg0, 0));
3362         break;
3363       case 8:
3364         // int64_t
3365         __ movq(c_rarg1, Address(c_rarg0, 0));
3366         break;
3367       default:
3368         ShouldNotReachHere();
3369     }
3370 
3371     // return errValue or *adr
3372     *continuation_pc = __ pc();
3373     __ movq(rax, c_rarg1);
3374     __ ret(0);
3375   }
3376 
3377   // This is a version of CBC/AES Decrypt which does 4 blocks in a loop at a time
3378   // to hide instruction latency
3379   //
3380   // Arguments:
3381   //
3382   // Inputs:
3383   //   c_rarg0   - source byte array address
3384   //   c_rarg1   - destination byte array address
3385   //   c_rarg2   - K (key) in little endian int array
3386   //   c_rarg3   - r vector byte array address
3387   //   c_rarg4   - input length
3388   //
3389   // Output:
3390   //   rax       - input length
3391   //
3392   address generate_cipherBlockChaining_decryptAESCrypt_Parallel() {
3393     assert(UseAES, "need AES instructions and misaligned SSE support");
3394     __ align(CodeEntryAlignment);
3395     StubCodeMark mark(this, "StubRoutines", "cipherBlockChaining_decryptAESCrypt");
3396     address start = __ pc();
3397 
3398     const Register from        = c_rarg0;  // source array address
3399     const Register to          = c_rarg1;  // destination array address
3400     const Register key         = c_rarg2;  // key array address
3401     const Register rvec        = c_rarg3;  // r byte array initialized from initvector array address
3402                                            // and left with the results of the last encryption block
3403 #ifndef _WIN64
3404     const Register len_reg     = c_rarg4;  // src len (must be multiple of blocksize 16)
3405 #else
3406     const Address  len_mem(rbp, 6 * wordSize);  // length is on stack on Win64
3407     const Register len_reg     = r10;      // pick the first volatile windows register
3408 #endif
3409     const Register pos         = rax;
3410 
3411     const int PARALLEL_FACTOR = 4;
3412     const int ROUNDS[3] = { 10, 12, 14 }; // aes rounds for key128, key192, key256
3413 
3414     Label L_exit;
3415     Label L_singleBlock_loopTopHead[3]; // 128, 192, 256
3416     Label L_singleBlock_loopTopHead2[3]; // 128, 192, 256
3417     Label L_singleBlock_loopTop[3]; // 128, 192, 256
3418     Label L_multiBlock_loopTopHead[3]; // 128, 192, 256
3419     Label L_multiBlock_loopTop[3]; // 128, 192, 256
3420 
3421     // keys 0-10 preloaded into xmm5-xmm15
3422     const int XMM_REG_NUM_KEY_FIRST = 5;
3423     const int XMM_REG_NUM_KEY_LAST  = 15;
3424     const XMMRegister xmm_key_first = as_XMMRegister(XMM_REG_NUM_KEY_FIRST);
3425     const XMMRegister xmm_key_last  = as_XMMRegister(XMM_REG_NUM_KEY_LAST);
3426 
3427     __ enter(); // required for proper stackwalking of RuntimeStub frame
3428 
3429     // For EVEX with VL and BW, provide a standard mask, VL = 128 will guide the merge
3430     // context for the registers used, where all instructions below are using 128-bit mode
3431     // On EVEX without VL and BW, these instructions will all be AVX.
3432     if (VM_Version::supports_avx512vlbw()) {
3433       __ movl(rax, 0xffff);
3434       __ kmovql(k1, rax);
3435     }
3436 
3437 #ifdef _WIN64
3438     // on win64, fill len_reg from stack position
3439     __ movl(len_reg, len_mem);
3440 #else
3441     __ push(len_reg); // Save
3442 #endif
3443     __ push(rbx);
3444     // the java expanded key ordering is rotated one position from what we want
3445     // so we start from 0x10 here and hit 0x00 last
3446     const XMMRegister xmm_key_shuf_mask = xmm1;  // used temporarily to swap key bytes up front
3447     __ movdqu(xmm_key_shuf_mask, ExternalAddress(StubRoutines::x86::key_shuffle_mask_addr()));
3448     // load up xmm regs 5 thru 15 with key 0x10 - 0xa0 - 0x00
3449     for (int rnum = XMM_REG_NUM_KEY_FIRST, offset = 0x10; rnum < XMM_REG_NUM_KEY_LAST; rnum++) {
3450       load_key(as_XMMRegister(rnum), key, offset, xmm_key_shuf_mask);
3451       offset += 0x10;
3452     }
3453     load_key(xmm_key_last, key, 0x00, xmm_key_shuf_mask);
3454 
3455     const XMMRegister xmm_prev_block_cipher = xmm1;  // holds cipher of previous block
3456 
3457     // registers holding the four results in the parallelized loop
3458     const XMMRegister xmm_result0 = xmm0;
3459     const XMMRegister xmm_result1 = xmm2;
3460     const XMMRegister xmm_result2 = xmm3;
3461     const XMMRegister xmm_result3 = xmm4;
3462 
3463     __ movdqu(xmm_prev_block_cipher, Address(rvec, 0x00));   // initialize with initial rvec
3464 
3465     __ xorptr(pos, pos);
3466 
3467     // now split to different paths depending on the keylen (len in ints of AESCrypt.KLE array (52=192, or 60=256))
3468     __ movl(rbx, Address(key, arrayOopDesc::length_offset_in_bytes() - arrayOopDesc::base_offset_in_bytes(T_INT)));
3469     __ cmpl(rbx, 52);
3470     __ jcc(Assembler::equal, L_multiBlock_loopTopHead[1]);
3471     __ cmpl(rbx, 60);
3472     __ jcc(Assembler::equal, L_multiBlock_loopTopHead[2]);
3473 
3474 #define DoFour(opc, src_reg)           \
3475   __ opc(xmm_result0, src_reg);         \
3476   __ opc(xmm_result1, src_reg);         \
3477   __ opc(xmm_result2, src_reg);         \
3478   __ opc(xmm_result3, src_reg);         \
3479 
3480     for (int k = 0; k < 3; ++k) {
3481       __ BIND(L_multiBlock_loopTopHead[k]);
3482       if (k != 0) {
3483         __ cmpptr(len_reg, PARALLEL_FACTOR * AESBlockSize); // see if at least 4 blocks left
3484         __ jcc(Assembler::less, L_singleBlock_loopTopHead2[k]);
3485       }
3486       if (k == 1) {
3487         __ subptr(rsp, 6 * wordSize);
3488         __ movdqu(Address(rsp, 0), xmm15); //save last_key from xmm15
3489         load_key(xmm15, key, 0xb0); // 0xb0; 192-bit key goes up to 0xc0
3490         __ movdqu(Address(rsp, 2 * wordSize), xmm15);
3491         load_key(xmm1, key, 0xc0);  // 0xc0;
3492         __ movdqu(Address(rsp, 4 * wordSize), xmm1);
3493       } else if (k == 2) {
3494         __ subptr(rsp, 10 * wordSize);
3495         __ movdqu(Address(rsp, 0), xmm15); //save last_key from xmm15
3496         load_key(xmm15, key, 0xd0); // 0xd0; 256-bit key goes upto 0xe0
3497         __ movdqu(Address(rsp, 6 * wordSize), xmm15);
3498         load_key(xmm1, key, 0xe0);  // 0xe0;
3499         __ movdqu(Address(rsp, 8 * wordSize), xmm1);
3500         load_key(xmm15, key, 0xb0); // 0xb0;
3501         __ movdqu(Address(rsp, 2 * wordSize), xmm15);
3502         load_key(xmm1, key, 0xc0);  // 0xc0;
3503         __ movdqu(Address(rsp, 4 * wordSize), xmm1);
3504       }
3505       __ align(OptoLoopAlignment);
3506       __ BIND(L_multiBlock_loopTop[k]);
3507       __ cmpptr(len_reg, PARALLEL_FACTOR * AESBlockSize); // see if at least 4 blocks left
3508       __ jcc(Assembler::less, L_singleBlock_loopTopHead[k]);
3509 
3510       if  (k != 0) {
3511         __ movdqu(xmm15, Address(rsp, 2 * wordSize));
3512         __ movdqu(xmm1, Address(rsp, 4 * wordSize));
3513       }
3514 
3515       __ movdqu(xmm_result0, Address(from, pos, Address::times_1, 0 * AESBlockSize)); // get next 4 blocks into xmmresult registers
3516       __ movdqu(xmm_result1, Address(from, pos, Address::times_1, 1 * AESBlockSize));
3517       __ movdqu(xmm_result2, Address(from, pos, Address::times_1, 2 * AESBlockSize));
3518       __ movdqu(xmm_result3, Address(from, pos, Address::times_1, 3 * AESBlockSize));
3519 
3520       DoFour(pxor, xmm_key_first);
3521       if (k == 0) {
3522         for (int rnum = 1; rnum < ROUNDS[k]; rnum++) {
3523           DoFour(aesdec, as_XMMRegister(rnum + XMM_REG_NUM_KEY_FIRST));
3524         }
3525         DoFour(aesdeclast, xmm_key_last);
3526       } else if (k == 1) {
3527         for (int rnum = 1; rnum <= ROUNDS[k]-2; rnum++) {
3528           DoFour(aesdec, as_XMMRegister(rnum + XMM_REG_NUM_KEY_FIRST));
3529         }
3530         __ movdqu(xmm_key_last, Address(rsp, 0)); // xmm15 needs to be loaded again.
3531         DoFour(aesdec, xmm1);  // key : 0xc0
3532         __ movdqu(xmm_prev_block_cipher, Address(rvec, 0x00));  // xmm1 needs to be loaded again
3533         DoFour(aesdeclast, xmm_key_last);
3534       } else if (k == 2) {
3535         for (int rnum = 1; rnum <= ROUNDS[k] - 4; rnum++) {
3536           DoFour(aesdec, as_XMMRegister(rnum + XMM_REG_NUM_KEY_FIRST));
3537         }
3538         DoFour(aesdec, xmm1);  // key : 0xc0
3539         __ movdqu(xmm15, Address(rsp, 6 * wordSize));
3540         __ movdqu(xmm1, Address(rsp, 8 * wordSize));
3541         DoFour(aesdec, xmm15);  // key : 0xd0
3542         __ movdqu(xmm_key_last, Address(rsp, 0)); // xmm15 needs to be loaded again.
3543         DoFour(aesdec, xmm1);  // key : 0xe0
3544         __ movdqu(xmm_prev_block_cipher, Address(rvec, 0x00));  // xmm1 needs to be loaded again
3545         DoFour(aesdeclast, xmm_key_last);
3546       }
3547 
3548       // for each result, xor with the r vector of previous cipher block
3549       __ pxor(xmm_result0, xmm_prev_block_cipher);
3550       __ movdqu(xmm_prev_block_cipher, Address(from, pos, Address::times_1, 0 * AESBlockSize));
3551       __ pxor(xmm_result1, xmm_prev_block_cipher);
3552       __ movdqu(xmm_prev_block_cipher, Address(from, pos, Address::times_1, 1 * AESBlockSize));
3553       __ pxor(xmm_result2, xmm_prev_block_cipher);
3554       __ movdqu(xmm_prev_block_cipher, Address(from, pos, Address::times_1, 2 * AESBlockSize));
3555       __ pxor(xmm_result3, xmm_prev_block_cipher);
3556       __ movdqu(xmm_prev_block_cipher, Address(from, pos, Address::times_1, 3 * AESBlockSize));   // this will carry over to next set of blocks
3557       if (k != 0) {
3558         __ movdqu(Address(rvec, 0x00), xmm_prev_block_cipher);
3559       }
3560 
3561       __ movdqu(Address(to, pos, Address::times_1, 0 * AESBlockSize), xmm_result0);     // store 4 results into the next 64 bytes of output
3562       __ movdqu(Address(to, pos, Address::times_1, 1 * AESBlockSize), xmm_result1);
3563       __ movdqu(Address(to, pos, Address::times_1, 2 * AESBlockSize), xmm_result2);
3564       __ movdqu(Address(to, pos, Address::times_1, 3 * AESBlockSize), xmm_result3);
3565 
3566       __ addptr(pos, PARALLEL_FACTOR * AESBlockSize);
3567       __ subptr(len_reg, PARALLEL_FACTOR * AESBlockSize);
3568       __ jmp(L_multiBlock_loopTop[k]);
3569 
3570       // registers used in the non-parallelized loops
3571       // xmm register assignments for the loops below
3572       const XMMRegister xmm_result = xmm0;
3573       const XMMRegister xmm_prev_block_cipher_save = xmm2;
3574       const XMMRegister xmm_key11 = xmm3;
3575       const XMMRegister xmm_key12 = xmm4;
3576       const XMMRegister key_tmp = xmm4;
3577 
3578       __ BIND(L_singleBlock_loopTopHead[k]);
3579       if (k == 1) {
3580         __ addptr(rsp, 6 * wordSize);
3581       } else if (k == 2) {
3582         __ addptr(rsp, 10 * wordSize);
3583       }
3584       __ cmpptr(len_reg, 0); // any blocks left??
3585       __ jcc(Assembler::equal, L_exit);
3586       __ BIND(L_singleBlock_loopTopHead2[k]);
3587       if (k == 1) {
3588         load_key(xmm_key11, key, 0xb0); // 0xb0; 192-bit key goes upto 0xc0
3589         load_key(xmm_key12, key, 0xc0); // 0xc0; 192-bit key goes upto 0xc0
3590       }
3591       if (k == 2) {
3592         load_key(xmm_key11, key, 0xb0); // 0xb0; 256-bit key goes upto 0xe0
3593       }
3594       __ align(OptoLoopAlignment);
3595       __ BIND(L_singleBlock_loopTop[k]);
3596       __ movdqu(xmm_result, Address(from, pos, Address::times_1, 0)); // get next 16 bytes of cipher input
3597       __ movdqa(xmm_prev_block_cipher_save, xmm_result); // save for next r vector
3598       __ pxor(xmm_result, xmm_key_first); // do the aes dec rounds
3599       for (int rnum = 1; rnum <= 9 ; rnum++) {
3600           __ aesdec(xmm_result, as_XMMRegister(rnum + XMM_REG_NUM_KEY_FIRST));
3601       }
3602       if (k == 1) {
3603         __ aesdec(xmm_result, xmm_key11);
3604         __ aesdec(xmm_result, xmm_key12);
3605       }
3606       if (k == 2) {
3607         __ aesdec(xmm_result, xmm_key11);
3608         load_key(key_tmp, key, 0xc0);
3609         __ aesdec(xmm_result, key_tmp);
3610         load_key(key_tmp, key, 0xd0);
3611         __ aesdec(xmm_result, key_tmp);
3612         load_key(key_tmp, key, 0xe0);
3613         __ aesdec(xmm_result, key_tmp);
3614       }
3615 
3616       __ aesdeclast(xmm_result, xmm_key_last); // xmm15 always came from key+0
3617       __ pxor(xmm_result, xmm_prev_block_cipher); // xor with the current r vector
3618       __ movdqu(Address(to, pos, Address::times_1, 0), xmm_result); // store into the next 16 bytes of output
3619       // no need to store r to memory until we exit
3620       __ movdqa(xmm_prev_block_cipher, xmm_prev_block_cipher_save); // set up next r vector with cipher input from this block
3621       __ addptr(pos, AESBlockSize);
3622       __ subptr(len_reg, AESBlockSize);
3623       __ jcc(Assembler::notEqual, L_singleBlock_loopTop[k]);
3624       if (k != 2) {
3625         __ jmp(L_exit);
3626       }
3627     } //for 128/192/256
3628 
3629     __ BIND(L_exit);
3630     __ movdqu(Address(rvec, 0), xmm_prev_block_cipher);     // final value of r stored in rvec of CipherBlockChaining object
3631     __ pop(rbx);
3632 #ifdef _WIN64
3633     __ movl(rax, len_mem);
3634 #else
3635     __ pop(rax); // return length
3636 #endif
3637     __ leave(); // required for proper stackwalking of RuntimeStub frame
3638     __ ret(0);
3639     return start;
3640 }
3641 
3642   address generate_upper_word_mask() {
3643     __ align(64);
3644     StubCodeMark mark(this, "StubRoutines", "upper_word_mask");
3645     address start = __ pc();
3646     __ emit_data64(0x0000000000000000, relocInfo::none);
3647     __ emit_data64(0xFFFFFFFF00000000, relocInfo::none);
3648     return start;
3649   }
3650 
3651   address generate_shuffle_byte_flip_mask() {
3652     __ align(64);
3653     StubCodeMark mark(this, "StubRoutines", "shuffle_byte_flip_mask");
3654     address start = __ pc();
3655     __ emit_data64(0x08090a0b0c0d0e0f, relocInfo::none);
3656     __ emit_data64(0x0001020304050607, relocInfo::none);
3657     return start;
3658   }
3659 
3660   // ofs and limit are use for multi-block byte array.
3661   // int com.sun.security.provider.DigestBase.implCompressMultiBlock(byte[] b, int ofs, int limit)
3662   address generate_sha1_implCompress(bool multi_block, const char *name) {
3663     __ align(CodeEntryAlignment);
3664     StubCodeMark mark(this, "StubRoutines", name);
3665     address start = __ pc();
3666 
3667     Register buf = c_rarg0;
3668     Register state = c_rarg1;
3669     Register ofs = c_rarg2;
3670     Register limit = c_rarg3;
3671 
3672     const XMMRegister abcd = xmm0;
3673     const XMMRegister e0 = xmm1;
3674     const XMMRegister e1 = xmm2;
3675     const XMMRegister msg0 = xmm3;
3676 
3677     const XMMRegister msg1 = xmm4;
3678     const XMMRegister msg2 = xmm5;
3679     const XMMRegister msg3 = xmm6;
3680     const XMMRegister shuf_mask = xmm7;
3681 
3682     __ enter();
3683 
3684     __ subptr(rsp, 4 * wordSize);
3685 
3686     __ fast_sha1(abcd, e0, e1, msg0, msg1, msg2, msg3, shuf_mask,
3687       buf, state, ofs, limit, rsp, multi_block);
3688 
3689     __ addptr(rsp, 4 * wordSize);
3690 
3691     __ leave();
3692     __ ret(0);
3693     return start;
3694   }
3695 
3696   address generate_pshuffle_byte_flip_mask() {
3697     __ align(64);
3698     StubCodeMark mark(this, "StubRoutines", "pshuffle_byte_flip_mask");
3699     address start = __ pc();
3700     __ emit_data64(0x0405060700010203, relocInfo::none);
3701     __ emit_data64(0x0c0d0e0f08090a0b, relocInfo::none);
3702 
3703     if (VM_Version::supports_avx2()) {
3704       __ emit_data64(0x0405060700010203, relocInfo::none); // second copy
3705       __ emit_data64(0x0c0d0e0f08090a0b, relocInfo::none);
3706       // _SHUF_00BA
3707       __ emit_data64(0x0b0a090803020100, relocInfo::none);
3708       __ emit_data64(0xFFFFFFFFFFFFFFFF, relocInfo::none);
3709       __ emit_data64(0x0b0a090803020100, relocInfo::none);
3710       __ emit_data64(0xFFFFFFFFFFFFFFFF, relocInfo::none);
3711       // _SHUF_DC00
3712       __ emit_data64(0xFFFFFFFFFFFFFFFF, relocInfo::none);
3713       __ emit_data64(0x0b0a090803020100, relocInfo::none);
3714       __ emit_data64(0xFFFFFFFFFFFFFFFF, relocInfo::none);
3715       __ emit_data64(0x0b0a090803020100, relocInfo::none);
3716     }
3717 
3718     return start;
3719   }
3720 
3721   //Mask for byte-swapping a couple of qwords in an XMM register using (v)pshufb.
3722   address generate_pshuffle_byte_flip_mask_sha512() {
3723     __ align(32);
3724     StubCodeMark mark(this, "StubRoutines", "pshuffle_byte_flip_mask_sha512");
3725     address start = __ pc();
3726     if (VM_Version::supports_avx2()) {
3727       __ emit_data64(0x0001020304050607, relocInfo::none); // PSHUFFLE_BYTE_FLIP_MASK
3728       __ emit_data64(0x08090a0b0c0d0e0f, relocInfo::none);
3729       __ emit_data64(0x1011121314151617, relocInfo::none);
3730       __ emit_data64(0x18191a1b1c1d1e1f, relocInfo::none);
3731       __ emit_data64(0x0000000000000000, relocInfo::none); //MASK_YMM_LO
3732       __ emit_data64(0x0000000000000000, relocInfo::none);
3733       __ emit_data64(0xFFFFFFFFFFFFFFFF, relocInfo::none);
3734       __ emit_data64(0xFFFFFFFFFFFFFFFF, relocInfo::none);
3735     }
3736 
3737     return start;
3738   }
3739 
3740 // ofs and limit are use for multi-block byte array.
3741 // int com.sun.security.provider.DigestBase.implCompressMultiBlock(byte[] b, int ofs, int limit)
3742   address generate_sha256_implCompress(bool multi_block, const char *name) {
3743     assert(VM_Version::supports_sha() || VM_Version::supports_avx2(), "");
3744     __ align(CodeEntryAlignment);
3745     StubCodeMark mark(this, "StubRoutines", name);
3746     address start = __ pc();
3747 
3748     Register buf = c_rarg0;
3749     Register state = c_rarg1;
3750     Register ofs = c_rarg2;
3751     Register limit = c_rarg3;
3752 
3753     const XMMRegister msg = xmm0;
3754     const XMMRegister state0 = xmm1;
3755     const XMMRegister state1 = xmm2;
3756     const XMMRegister msgtmp0 = xmm3;
3757 
3758     const XMMRegister msgtmp1 = xmm4;
3759     const XMMRegister msgtmp2 = xmm5;
3760     const XMMRegister msgtmp3 = xmm6;
3761     const XMMRegister msgtmp4 = xmm7;
3762 
3763     const XMMRegister shuf_mask = xmm8;
3764 
3765     __ enter();
3766 
3767     __ subptr(rsp, 4 * wordSize);
3768 
3769     if (VM_Version::supports_sha()) {
3770       __ fast_sha256(msg, state0, state1, msgtmp0, msgtmp1, msgtmp2, msgtmp3, msgtmp4,
3771         buf, state, ofs, limit, rsp, multi_block, shuf_mask);
3772     } else if (VM_Version::supports_avx2()) {
3773       __ sha256_AVX2(msg, state0, state1, msgtmp0, msgtmp1, msgtmp2, msgtmp3, msgtmp4,
3774         buf, state, ofs, limit, rsp, multi_block, shuf_mask);
3775     }
3776     __ addptr(rsp, 4 * wordSize);
3777 
3778     __ leave();
3779     __ ret(0);
3780     return start;
3781   }
3782 
3783   address generate_sha512_implCompress(bool multi_block, const char *name) {
3784     assert(VM_Version::supports_avx2(), "");
3785     assert(VM_Version::supports_bmi2(), "");
3786     __ align(CodeEntryAlignment);
3787     StubCodeMark mark(this, "StubRoutines", name);
3788     address start = __ pc();
3789 
3790     Register buf = c_rarg0;
3791     Register state = c_rarg1;
3792     Register ofs = c_rarg2;
3793     Register limit = c_rarg3;
3794 
3795     const XMMRegister msg = xmm0;
3796     const XMMRegister state0 = xmm1;
3797     const XMMRegister state1 = xmm2;
3798     const XMMRegister msgtmp0 = xmm3;
3799     const XMMRegister msgtmp1 = xmm4;
3800     const XMMRegister msgtmp2 = xmm5;
3801     const XMMRegister msgtmp3 = xmm6;
3802     const XMMRegister msgtmp4 = xmm7;
3803 
3804     const XMMRegister shuf_mask = xmm8;
3805 
3806     __ enter();
3807 
3808     __ sha512_AVX2(msg, state0, state1, msgtmp0, msgtmp1, msgtmp2, msgtmp3, msgtmp4,
3809     buf, state, ofs, limit, rsp, multi_block, shuf_mask);
3810 
3811     __ leave();
3812     __ ret(0);
3813     return start;
3814   }
3815 
3816   // This is a version of CTR/AES crypt which does 6 blocks in a loop at a time
3817   // to hide instruction latency
3818   //
3819   // Arguments:
3820   //
3821   // Inputs:
3822   //   c_rarg0   - source byte array address
3823   //   c_rarg1   - destination byte array address
3824   //   c_rarg2   - K (key) in little endian int array
3825   //   c_rarg3   - counter vector byte array address
3826   //   Linux
3827   //     c_rarg4   -          input length
3828   //     c_rarg5   -          saved encryptedCounter start
3829   //     rbp + 6 * wordSize - saved used length
3830   //   Windows
3831   //     rbp + 6 * wordSize - input length
3832   //     rbp + 7 * wordSize - saved encryptedCounter start
3833   //     rbp + 8 * wordSize - saved used length
3834   //
3835   // Output:
3836   //   rax       - input length
3837   //
3838   address generate_counterMode_AESCrypt_Parallel() {
3839     assert(UseAES, "need AES instructions and misaligned SSE support");
3840     __ align(CodeEntryAlignment);
3841     StubCodeMark mark(this, "StubRoutines", "counterMode_AESCrypt");
3842     address start = __ pc();
3843     const Register from = c_rarg0; // source array address
3844     const Register to = c_rarg1; // destination array address
3845     const Register key = c_rarg2; // key array address
3846     const Register counter = c_rarg3; // counter byte array initialized from counter array address
3847                                       // and updated with the incremented counter in the end
3848 #ifndef _WIN64
3849     const Register len_reg = c_rarg4;
3850     const Register saved_encCounter_start = c_rarg5;
3851     const Register used_addr = r10;
3852     const Address  used_mem(rbp, 2 * wordSize);
3853     const Register used = r11;
3854 #else
3855     const Address len_mem(rbp, 6 * wordSize); // length is on stack on Win64
3856     const Address saved_encCounter_mem(rbp, 7 * wordSize); // length is on stack on Win64
3857     const Address used_mem(rbp, 8 * wordSize); // length is on stack on Win64
3858     const Register len_reg = r10; // pick the first volatile windows register
3859     const Register saved_encCounter_start = r11;
3860     const Register used_addr = r13;
3861     const Register used = r14;
3862 #endif
3863     const Register pos = rax;
3864 
3865     const int PARALLEL_FACTOR = 6;
3866     const XMMRegister xmm_counter_shuf_mask = xmm0;
3867     const XMMRegister xmm_key_shuf_mask = xmm1; // used temporarily to swap key bytes up front
3868     const XMMRegister xmm_curr_counter = xmm2;
3869 
3870     const XMMRegister xmm_key_tmp0 = xmm3;
3871     const XMMRegister xmm_key_tmp1 = xmm4;
3872 
3873     // registers holding the four results in the parallelized loop
3874     const XMMRegister xmm_result0 = xmm5;
3875     const XMMRegister xmm_result1 = xmm6;
3876     const XMMRegister xmm_result2 = xmm7;
3877     const XMMRegister xmm_result3 = xmm8;
3878     const XMMRegister xmm_result4 = xmm9;
3879     const XMMRegister xmm_result5 = xmm10;
3880 
3881     const XMMRegister xmm_from0 = xmm11;
3882     const XMMRegister xmm_from1 = xmm12;
3883     const XMMRegister xmm_from2 = xmm13;
3884     const XMMRegister xmm_from3 = xmm14; //the last one is xmm14. we have to preserve it on WIN64.
3885     const XMMRegister xmm_from4 = xmm3; //reuse xmm3~4. Because xmm_key_tmp0~1 are useless when loading input text
3886     const XMMRegister xmm_from5 = xmm4;
3887 
3888     //for key_128, key_192, key_256
3889     const int rounds[3] = {10, 12, 14};
3890     Label L_exit_preLoop, L_preLoop_start;
3891     Label L_multiBlock_loopTop[3];
3892     Label L_singleBlockLoopTop[3];
3893     Label L__incCounter[3][6]; //for 6 blocks
3894     Label L__incCounter_single[3]; //for single block, key128, key192, key256
3895     Label L_processTail_insr[3], L_processTail_4_insr[3], L_processTail_2_insr[3], L_processTail_1_insr[3], L_processTail_exit_insr[3];
3896     Label L_processTail_extr[3], L_processTail_4_extr[3], L_processTail_2_extr[3], L_processTail_1_extr[3], L_processTail_exit_extr[3];
3897 
3898     Label L_exit;
3899 
3900     __ enter(); // required for proper stackwalking of RuntimeStub frame
3901 
3902     // For EVEX with VL and BW, provide a standard mask, VL = 128 will guide the merge
3903     // context for the registers used, where all instructions below are using 128-bit mode
3904     // On EVEX without VL and BW, these instructions will all be AVX.
3905     if (VM_Version::supports_avx512vlbw()) {
3906         __ movl(rax, 0xffff);
3907         __ kmovql(k1, rax);
3908     }
3909 
3910 #ifdef _WIN64
3911     // allocate spill slots for r13, r14
3912     enum {
3913         saved_r13_offset,
3914         saved_r14_offset
3915     };
3916     __ subptr(rsp, 2 * wordSize);
3917     __ movptr(Address(rsp, saved_r13_offset * wordSize), r13);
3918     __ movptr(Address(rsp, saved_r14_offset * wordSize), r14);
3919 
3920     // on win64, fill len_reg from stack position
3921     __ movl(len_reg, len_mem);
3922     __ movptr(saved_encCounter_start, saved_encCounter_mem);
3923     __ movptr(used_addr, used_mem);
3924     __ movl(used, Address(used_addr, 0));
3925 #else
3926     __ push(len_reg); // Save
3927     __ movptr(used_addr, used_mem);
3928     __ movl(used, Address(used_addr, 0));
3929 #endif
3930 
3931     __ push(rbx); // Save RBX
3932     __ movdqu(xmm_curr_counter, Address(counter, 0x00)); // initialize counter with initial counter
3933     __ movdqu(xmm_counter_shuf_mask, ExternalAddress(StubRoutines::x86::counter_shuffle_mask_addr()));
3934     __ pshufb(xmm_curr_counter, xmm_counter_shuf_mask); //counter is shuffled
3935     __ movptr(pos, 0);
3936 
3937     // Use the partially used encrpyted counter from last invocation
3938     __ BIND(L_preLoop_start);
3939     __ cmpptr(used, 16);
3940     __ jcc(Assembler::aboveEqual, L_exit_preLoop);
3941       __ cmpptr(len_reg, 0);
3942       __ jcc(Assembler::lessEqual, L_exit_preLoop);
3943       __ movb(rbx, Address(saved_encCounter_start, used));
3944       __ xorb(rbx, Address(from, pos));
3945       __ movb(Address(to, pos), rbx);
3946       __ addptr(pos, 1);
3947       __ addptr(used, 1);
3948       __ subptr(len_reg, 1);
3949 
3950     __ jmp(L_preLoop_start);
3951 
3952     __ BIND(L_exit_preLoop);
3953     __ movl(Address(used_addr, 0), used);
3954 
3955     // key length could be only {11, 13, 15} * 4 = {44, 52, 60}
3956     __ movdqu(xmm_key_shuf_mask, ExternalAddress(StubRoutines::x86::key_shuffle_mask_addr()));
3957     __ movl(rbx, Address(key, arrayOopDesc::length_offset_in_bytes() - arrayOopDesc::base_offset_in_bytes(T_INT)));
3958     __ cmpl(rbx, 52);
3959     __ jcc(Assembler::equal, L_multiBlock_loopTop[1]);
3960     __ cmpl(rbx, 60);
3961     __ jcc(Assembler::equal, L_multiBlock_loopTop[2]);
3962 
3963 #define CTR_DoSix(opc, src_reg)                \
3964     __ opc(xmm_result0, src_reg);              \
3965     __ opc(xmm_result1, src_reg);              \
3966     __ opc(xmm_result2, src_reg);              \
3967     __ opc(xmm_result3, src_reg);              \
3968     __ opc(xmm_result4, src_reg);              \
3969     __ opc(xmm_result5, src_reg);
3970 
3971     // k == 0 :  generate code for key_128
3972     // k == 1 :  generate code for key_192
3973     // k == 2 :  generate code for key_256
3974     for (int k = 0; k < 3; ++k) {
3975       //multi blocks starts here
3976       __ align(OptoLoopAlignment);
3977       __ BIND(L_multiBlock_loopTop[k]);
3978       __ cmpptr(len_reg, PARALLEL_FACTOR * AESBlockSize); // see if at least PARALLEL_FACTOR blocks left
3979       __ jcc(Assembler::less, L_singleBlockLoopTop[k]);
3980       load_key(xmm_key_tmp0, key, 0x00, xmm_key_shuf_mask);
3981 
3982       //load, then increase counters
3983       CTR_DoSix(movdqa, xmm_curr_counter);
3984       inc_counter(rbx, xmm_result1, 0x01, L__incCounter[k][0]);
3985       inc_counter(rbx, xmm_result2, 0x02, L__incCounter[k][1]);
3986       inc_counter(rbx, xmm_result3, 0x03, L__incCounter[k][2]);
3987       inc_counter(rbx, xmm_result4, 0x04, L__incCounter[k][3]);
3988       inc_counter(rbx, xmm_result5,  0x05, L__incCounter[k][4]);
3989       inc_counter(rbx, xmm_curr_counter, 0x06, L__incCounter[k][5]);
3990       CTR_DoSix(pshufb, xmm_counter_shuf_mask); // after increased, shuffled counters back for PXOR
3991       CTR_DoSix(pxor, xmm_key_tmp0);   //PXOR with Round 0 key
3992 
3993       //load two ROUND_KEYs at a time
3994       for (int i = 1; i < rounds[k]; ) {
3995         load_key(xmm_key_tmp1, key, (0x10 * i), xmm_key_shuf_mask);
3996         load_key(xmm_key_tmp0, key, (0x10 * (i+1)), xmm_key_shuf_mask);
3997         CTR_DoSix(aesenc, xmm_key_tmp1);
3998         i++;
3999         if (i != rounds[k]) {
4000           CTR_DoSix(aesenc, xmm_key_tmp0);
4001         } else {
4002           CTR_DoSix(aesenclast, xmm_key_tmp0);
4003         }
4004         i++;
4005       }
4006 
4007       // get next PARALLEL_FACTOR blocks into xmm_result registers
4008       __ movdqu(xmm_from0, Address(from, pos, Address::times_1, 0 * AESBlockSize));
4009       __ movdqu(xmm_from1, Address(from, pos, Address::times_1, 1 * AESBlockSize));
4010       __ movdqu(xmm_from2, Address(from, pos, Address::times_1, 2 * AESBlockSize));
4011       __ movdqu(xmm_from3, Address(from, pos, Address::times_1, 3 * AESBlockSize));
4012       __ movdqu(xmm_from4, Address(from, pos, Address::times_1, 4 * AESBlockSize));
4013       __ movdqu(xmm_from5, Address(from, pos, Address::times_1, 5 * AESBlockSize));
4014 
4015       __ pxor(xmm_result0, xmm_from0);
4016       __ pxor(xmm_result1, xmm_from1);
4017       __ pxor(xmm_result2, xmm_from2);
4018       __ pxor(xmm_result3, xmm_from3);
4019       __ pxor(xmm_result4, xmm_from4);
4020       __ pxor(xmm_result5, xmm_from5);
4021 
4022       // store 6 results into the next 64 bytes of output
4023       __ movdqu(Address(to, pos, Address::times_1, 0 * AESBlockSize), xmm_result0);
4024       __ movdqu(Address(to, pos, Address::times_1, 1 * AESBlockSize), xmm_result1);
4025       __ movdqu(Address(to, pos, Address::times_1, 2 * AESBlockSize), xmm_result2);
4026       __ movdqu(Address(to, pos, Address::times_1, 3 * AESBlockSize), xmm_result3);
4027       __ movdqu(Address(to, pos, Address::times_1, 4 * AESBlockSize), xmm_result4);
4028       __ movdqu(Address(to, pos, Address::times_1, 5 * AESBlockSize), xmm_result5);
4029 
4030       __ addptr(pos, PARALLEL_FACTOR * AESBlockSize); // increase the length of crypt text
4031       __ subptr(len_reg, PARALLEL_FACTOR * AESBlockSize); // decrease the remaining length
4032       __ jmp(L_multiBlock_loopTop[k]);
4033 
4034       // singleBlock starts here
4035       __ align(OptoLoopAlignment);
4036       __ BIND(L_singleBlockLoopTop[k]);
4037       __ cmpptr(len_reg, 0);
4038       __ jcc(Assembler::lessEqual, L_exit);
4039       load_key(xmm_key_tmp0, key, 0x00, xmm_key_shuf_mask);
4040       __ movdqa(xmm_result0, xmm_curr_counter);
4041       inc_counter(rbx, xmm_curr_counter, 0x01, L__incCounter_single[k]);
4042       __ pshufb(xmm_result0, xmm_counter_shuf_mask);
4043       __ pxor(xmm_result0, xmm_key_tmp0);
4044       for (int i = 1; i < rounds[k]; i++) {
4045         load_key(xmm_key_tmp0, key, (0x10 * i), xmm_key_shuf_mask);
4046         __ aesenc(xmm_result0, xmm_key_tmp0);
4047       }
4048       load_key(xmm_key_tmp0, key, (rounds[k] * 0x10), xmm_key_shuf_mask);
4049       __ aesenclast(xmm_result0, xmm_key_tmp0);
4050       __ cmpptr(len_reg, AESBlockSize);
4051       __ jcc(Assembler::less, L_processTail_insr[k]);
4052         __ movdqu(xmm_from0, Address(from, pos, Address::times_1, 0 * AESBlockSize));
4053         __ pxor(xmm_result0, xmm_from0);
4054         __ movdqu(Address(to, pos, Address::times_1, 0 * AESBlockSize), xmm_result0);
4055         __ addptr(pos, AESBlockSize);
4056         __ subptr(len_reg, AESBlockSize);
4057         __ jmp(L_singleBlockLoopTop[k]);
4058       __ BIND(L_processTail_insr[k]);                               // Process the tail part of the input array
4059         __ addptr(pos, len_reg);                                    // 1. Insert bytes from src array into xmm_from0 register
4060         __ testptr(len_reg, 8);
4061         __ jcc(Assembler::zero, L_processTail_4_insr[k]);
4062           __ subptr(pos,8);
4063           __ pinsrq(xmm_from0, Address(from, pos), 0);
4064         __ BIND(L_processTail_4_insr[k]);
4065         __ testptr(len_reg, 4);
4066         __ jcc(Assembler::zero, L_processTail_2_insr[k]);
4067           __ subptr(pos,4);
4068           __ pslldq(xmm_from0, 4);
4069           __ pinsrd(xmm_from0, Address(from, pos), 0);
4070         __ BIND(L_processTail_2_insr[k]);
4071         __ testptr(len_reg, 2);
4072         __ jcc(Assembler::zero, L_processTail_1_insr[k]);
4073           __ subptr(pos, 2);
4074           __ pslldq(xmm_from0, 2);
4075           __ pinsrw(xmm_from0, Address(from, pos), 0);
4076         __ BIND(L_processTail_1_insr[k]);
4077         __ testptr(len_reg, 1);
4078         __ jcc(Assembler::zero, L_processTail_exit_insr[k]);
4079           __ subptr(pos, 1);
4080           __ pslldq(xmm_from0, 1);
4081           __ pinsrb(xmm_from0, Address(from, pos), 0);
4082         __ BIND(L_processTail_exit_insr[k]);
4083 
4084         __ movdqu(Address(saved_encCounter_start, 0), xmm_result0);  // 2. Perform pxor of the encrypted counter and plaintext Bytes.
4085         __ pxor(xmm_result0, xmm_from0);                             //    Also the encrypted counter is saved for next invocation.
4086 
4087         __ testptr(len_reg, 8);
4088         __ jcc(Assembler::zero, L_processTail_4_extr[k]);            // 3. Extract bytes from xmm_result0 into the dest. array
4089           __ pextrq(Address(to, pos), xmm_result0, 0);
4090           __ psrldq(xmm_result0, 8);
4091           __ addptr(pos, 8);
4092         __ BIND(L_processTail_4_extr[k]);
4093         __ testptr(len_reg, 4);
4094         __ jcc(Assembler::zero, L_processTail_2_extr[k]);
4095           __ pextrd(Address(to, pos), xmm_result0, 0);
4096           __ psrldq(xmm_result0, 4);
4097           __ addptr(pos, 4);
4098         __ BIND(L_processTail_2_extr[k]);
4099         __ testptr(len_reg, 2);
4100         __ jcc(Assembler::zero, L_processTail_1_extr[k]);
4101           __ pextrw(Address(to, pos), xmm_result0, 0);
4102           __ psrldq(xmm_result0, 2);
4103           __ addptr(pos, 2);
4104         __ BIND(L_processTail_1_extr[k]);
4105         __ testptr(len_reg, 1);
4106         __ jcc(Assembler::zero, L_processTail_exit_extr[k]);
4107           __ pextrb(Address(to, pos), xmm_result0, 0);
4108 
4109         __ BIND(L_processTail_exit_extr[k]);
4110         __ movl(Address(used_addr, 0), len_reg);
4111         __ jmp(L_exit);
4112 
4113     }
4114 
4115     __ BIND(L_exit);
4116     __ pshufb(xmm_curr_counter, xmm_counter_shuf_mask); //counter is shuffled back.
4117     __ movdqu(Address(counter, 0), xmm_curr_counter); //save counter back
4118     __ pop(rbx); // pop the saved RBX.
4119 #ifdef _WIN64
4120     __ movl(rax, len_mem);
4121     __ movptr(r13, Address(rsp, saved_r13_offset * wordSize));
4122     __ movptr(r14, Address(rsp, saved_r14_offset * wordSize));
4123     __ addptr(rsp, 2 * wordSize);
4124 #else
4125     __ pop(rax); // return 'len'
4126 #endif
4127     __ leave(); // required for proper stackwalking of RuntimeStub frame
4128     __ ret(0);
4129     return start;
4130   }
4131 
4132   // byte swap x86 long
4133   address generate_ghash_long_swap_mask() {
4134     __ align(CodeEntryAlignment);
4135     StubCodeMark mark(this, "StubRoutines", "ghash_long_swap_mask");
4136     address start = __ pc();
4137     __ emit_data64(0x0f0e0d0c0b0a0908, relocInfo::none );
4138     __ emit_data64(0x0706050403020100, relocInfo::none );
4139   return start;
4140   }
4141 
4142   // byte swap x86 byte array
4143   address generate_ghash_byte_swap_mask() {
4144     __ align(CodeEntryAlignment);
4145     StubCodeMark mark(this, "StubRoutines", "ghash_byte_swap_mask");
4146     address start = __ pc();
4147     __ emit_data64(0x08090a0b0c0d0e0f, relocInfo::none );
4148     __ emit_data64(0x0001020304050607, relocInfo::none );
4149   return start;
4150   }
4151 
4152   /* Single and multi-block ghash operations */
4153   address generate_ghash_processBlocks() {
4154     __ align(CodeEntryAlignment);
4155     Label L_ghash_loop, L_exit;
4156     StubCodeMark mark(this, "StubRoutines", "ghash_processBlocks");
4157     address start = __ pc();
4158 
4159     const Register state        = c_rarg0;
4160     const Register subkeyH      = c_rarg1;
4161     const Register data         = c_rarg2;
4162     const Register blocks       = c_rarg3;
4163 
4164     const XMMRegister xmm_temp0 = xmm0;
4165     const XMMRegister xmm_temp1 = xmm1;
4166     const XMMRegister xmm_temp2 = xmm2;
4167     const XMMRegister xmm_temp3 = xmm3;
4168     const XMMRegister xmm_temp4 = xmm4;
4169     const XMMRegister xmm_temp5 = xmm5;
4170     const XMMRegister xmm_temp6 = xmm6;
4171     const XMMRegister xmm_temp7 = xmm7;
4172     const XMMRegister xmm_temp8 = xmm8;
4173     const XMMRegister xmm_temp9 = xmm9;
4174     const XMMRegister xmm_temp10 = xmm10;
4175 
4176     __ enter();
4177 
4178     // For EVEX with VL and BW, provide a standard mask, VL = 128 will guide the merge
4179     // context for the registers used, where all instructions below are using 128-bit mode
4180     // On EVEX without VL and BW, these instructions will all be AVX.
4181     if (VM_Version::supports_avx512vlbw()) {
4182       __ movl(rax, 0xffff);
4183       __ kmovql(k1, rax);
4184     }
4185 
4186     __ movdqu(xmm_temp10, ExternalAddress(StubRoutines::x86::ghash_long_swap_mask_addr()));
4187 
4188     __ movdqu(xmm_temp0, Address(state, 0));
4189     __ pshufb(xmm_temp0, xmm_temp10);
4190 
4191 
4192     __ BIND(L_ghash_loop);
4193     __ movdqu(xmm_temp2, Address(data, 0));
4194     __ pshufb(xmm_temp2, ExternalAddress(StubRoutines::x86::ghash_byte_swap_mask_addr()));
4195 
4196     __ movdqu(xmm_temp1, Address(subkeyH, 0));
4197     __ pshufb(xmm_temp1, xmm_temp10);
4198 
4199     __ pxor(xmm_temp0, xmm_temp2);
4200 
4201     //
4202     // Multiply with the hash key
4203     //
4204     __ movdqu(xmm_temp3, xmm_temp0);
4205     __ pclmulqdq(xmm_temp3, xmm_temp1, 0);      // xmm3 holds a0*b0
4206     __ movdqu(xmm_temp4, xmm_temp0);
4207     __ pclmulqdq(xmm_temp4, xmm_temp1, 16);     // xmm4 holds a0*b1
4208 
4209     __ movdqu(xmm_temp5, xmm_temp0);
4210     __ pclmulqdq(xmm_temp5, xmm_temp1, 1);      // xmm5 holds a1*b0
4211     __ movdqu(xmm_temp6, xmm_temp0);
4212     __ pclmulqdq(xmm_temp6, xmm_temp1, 17);     // xmm6 holds a1*b1
4213 
4214     __ pxor(xmm_temp4, xmm_temp5);      // xmm4 holds a0*b1 + a1*b0
4215 
4216     __ movdqu(xmm_temp5, xmm_temp4);    // move the contents of xmm4 to xmm5
4217     __ psrldq(xmm_temp4, 8);    // shift by xmm4 64 bits to the right
4218     __ pslldq(xmm_temp5, 8);    // shift by xmm5 64 bits to the left
4219     __ pxor(xmm_temp3, xmm_temp5);
4220     __ pxor(xmm_temp6, xmm_temp4);      // Register pair <xmm6:xmm3> holds the result
4221                                         // of the carry-less multiplication of
4222                                         // xmm0 by xmm1.
4223 
4224     // We shift the result of the multiplication by one bit position
4225     // to the left to cope for the fact that the bits are reversed.
4226     __ movdqu(xmm_temp7, xmm_temp3);
4227     __ movdqu(xmm_temp8, xmm_temp6);
4228     __ pslld(xmm_temp3, 1);
4229     __ pslld(xmm_temp6, 1);
4230     __ psrld(xmm_temp7, 31);
4231     __ psrld(xmm_temp8, 31);
4232     __ movdqu(xmm_temp9, xmm_temp7);
4233     __ pslldq(xmm_temp8, 4);
4234     __ pslldq(xmm_temp7, 4);
4235     __ psrldq(xmm_temp9, 12);
4236     __ por(xmm_temp3, xmm_temp7);
4237     __ por(xmm_temp6, xmm_temp8);
4238     __ por(xmm_temp6, xmm_temp9);
4239 
4240     //
4241     // First phase of the reduction
4242     //
4243     // Move xmm3 into xmm7, xmm8, xmm9 in order to perform the shifts
4244     // independently.
4245     __ movdqu(xmm_temp7, xmm_temp3);
4246     __ movdqu(xmm_temp8, xmm_temp3);
4247     __ movdqu(xmm_temp9, xmm_temp3);
4248     __ pslld(xmm_temp7, 31);    // packed right shift shifting << 31
4249     __ pslld(xmm_temp8, 30);    // packed right shift shifting << 30
4250     __ pslld(xmm_temp9, 25);    // packed right shift shifting << 25
4251     __ pxor(xmm_temp7, xmm_temp8);      // xor the shifted versions
4252     __ pxor(xmm_temp7, xmm_temp9);
4253     __ movdqu(xmm_temp8, xmm_temp7);
4254     __ pslldq(xmm_temp7, 12);
4255     __ psrldq(xmm_temp8, 4);
4256     __ pxor(xmm_temp3, xmm_temp7);      // first phase of the reduction complete
4257 
4258     //
4259     // Second phase of the reduction
4260     //
4261     // Make 3 copies of xmm3 in xmm2, xmm4, xmm5 for doing these
4262     // shift operations.
4263     __ movdqu(xmm_temp2, xmm_temp3);
4264     __ movdqu(xmm_temp4, xmm_temp3);
4265     __ movdqu(xmm_temp5, xmm_temp3);
4266     __ psrld(xmm_temp2, 1);     // packed left shifting >> 1
4267     __ psrld(xmm_temp4, 2);     // packed left shifting >> 2
4268     __ psrld(xmm_temp5, 7);     // packed left shifting >> 7
4269     __ pxor(xmm_temp2, xmm_temp4);      // xor the shifted versions
4270     __ pxor(xmm_temp2, xmm_temp5);
4271     __ pxor(xmm_temp2, xmm_temp8);
4272     __ pxor(xmm_temp3, xmm_temp2);
4273     __ pxor(xmm_temp6, xmm_temp3);      // the result is in xmm6
4274 
4275     __ decrement(blocks);
4276     __ jcc(Assembler::zero, L_exit);
4277     __ movdqu(xmm_temp0, xmm_temp6);
4278     __ addptr(data, 16);
4279     __ jmp(L_ghash_loop);
4280 
4281     __ BIND(L_exit);
4282     __ pshufb(xmm_temp6, xmm_temp10);          // Byte swap 16-byte result
4283     __ movdqu(Address(state, 0), xmm_temp6);   // store the result
4284 
4285     __ leave();
4286     __ ret(0);
4287     return start;
4288   }
4289 
4290   /**
4291    *  Arguments:
4292    *
4293    * Inputs:
4294    *   c_rarg0   - int crc
4295    *   c_rarg1   - byte* buf
4296    *   c_rarg2   - int length
4297    *
4298    * Ouput:
4299    *       rax   - int crc result
4300    */
4301   address generate_updateBytesCRC32() {
4302     assert(UseCRC32Intrinsics, "need AVX and CLMUL instructions");
4303 
4304     __ align(CodeEntryAlignment);
4305     StubCodeMark mark(this, "StubRoutines", "updateBytesCRC32");
4306 
4307     address start = __ pc();
4308     // Win64: rcx, rdx, r8, r9 (c_rarg0, c_rarg1, ...)
4309     // Unix:  rdi, rsi, rdx, rcx, r8, r9 (c_rarg0, c_rarg1, ...)
4310     // rscratch1: r10
4311     const Register crc   = c_rarg0;  // crc
4312     const Register buf   = c_rarg1;  // source java byte array address
4313     const Register len   = c_rarg2;  // length
4314     const Register table = c_rarg3;  // crc_table address (reuse register)
4315     const Register tmp   = r11;
4316     assert_different_registers(crc, buf, len, table, tmp, rax);
4317 
4318     BLOCK_COMMENT("Entry:");
4319     __ enter(); // required for proper stackwalking of RuntimeStub frame
4320 
4321     __ kernel_crc32(crc, buf, len, table, tmp);
4322 
4323     __ movl(rax, crc);
4324     __ leave(); // required for proper stackwalking of RuntimeStub frame
4325     __ ret(0);
4326 
4327     return start;
4328   }
4329 
4330   /**
4331   *  Arguments:
4332   *
4333   * Inputs:
4334   *   c_rarg0   - int crc
4335   *   c_rarg1   - byte* buf
4336   *   c_rarg2   - long length
4337   *   c_rarg3   - table_start - optional (present only when doing a library_call,
4338   *              not used by x86 algorithm)
4339   *
4340   * Ouput:
4341   *       rax   - int crc result
4342   */
4343   address generate_updateBytesCRC32C(bool is_pclmulqdq_supported) {
4344       assert(UseCRC32CIntrinsics, "need SSE4_2");
4345       __ align(CodeEntryAlignment);
4346       StubCodeMark mark(this, "StubRoutines", "updateBytesCRC32C");
4347       address start = __ pc();
4348       //reg.arg        int#0        int#1        int#2        int#3        int#4        int#5        float regs
4349       //Windows        RCX          RDX          R8           R9           none         none         XMM0..XMM3
4350       //Lin / Sol      RDI          RSI          RDX          RCX          R8           R9           XMM0..XMM7
4351       const Register crc = c_rarg0;  // crc
4352       const Register buf = c_rarg1;  // source java byte array address
4353       const Register len = c_rarg2;  // length
4354       const Register a = rax;
4355       const Register j = r9;
4356       const Register k = r10;
4357       const Register l = r11;
4358 #ifdef _WIN64
4359       const Register y = rdi;
4360       const Register z = rsi;
4361 #else
4362       const Register y = rcx;
4363       const Register z = r8;
4364 #endif
4365       assert_different_registers(crc, buf, len, a, j, k, l, y, z);
4366 
4367       BLOCK_COMMENT("Entry:");
4368       __ enter(); // required for proper stackwalking of RuntimeStub frame
4369 #ifdef _WIN64
4370       __ push(y);
4371       __ push(z);
4372 #endif
4373       __ crc32c_ipl_alg2_alt2(crc, buf, len,
4374                               a, j, k,
4375                               l, y, z,
4376                               c_farg0, c_farg1, c_farg2,
4377                               is_pclmulqdq_supported);
4378       __ movl(rax, crc);
4379 #ifdef _WIN64
4380       __ pop(z);
4381       __ pop(y);
4382 #endif
4383       __ leave(); // required for proper stackwalking of RuntimeStub frame
4384       __ ret(0);
4385 
4386       return start;
4387   }
4388 
4389   /**
4390    *  Arguments:
4391    *
4392    *  Input:
4393    *    c_rarg0   - x address
4394    *    c_rarg1   - x length
4395    *    c_rarg2   - y address
4396    *    c_rarg3   - y lenth
4397    * not Win64
4398    *    c_rarg4   - z address
4399    *    c_rarg5   - z length
4400    * Win64
4401    *    rsp+40    - z address
4402    *    rsp+48    - z length
4403    */
4404   address generate_multiplyToLen() {
4405     __ align(CodeEntryAlignment);
4406     StubCodeMark mark(this, "StubRoutines", "multiplyToLen");
4407 
4408     address start = __ pc();
4409     // Win64: rcx, rdx, r8, r9 (c_rarg0, c_rarg1, ...)
4410     // Unix:  rdi, rsi, rdx, rcx, r8, r9 (c_rarg0, c_rarg1, ...)
4411     const Register x     = rdi;
4412     const Register xlen  = rax;
4413     const Register y     = rsi;
4414     const Register ylen  = rcx;
4415     const Register z     = r8;
4416     const Register zlen  = r11;
4417 
4418     // Next registers will be saved on stack in multiply_to_len().
4419     const Register tmp1  = r12;
4420     const Register tmp2  = r13;
4421     const Register tmp3  = r14;
4422     const Register tmp4  = r15;
4423     const Register tmp5  = rbx;
4424 
4425     BLOCK_COMMENT("Entry:");
4426     __ enter(); // required for proper stackwalking of RuntimeStub frame
4427 
4428 #ifndef _WIN64
4429     __ movptr(zlen, r9); // Save r9 in r11 - zlen
4430 #endif
4431     setup_arg_regs(4); // x => rdi, xlen => rsi, y => rdx
4432                        // ylen => rcx, z => r8, zlen => r11
4433                        // r9 and r10 may be used to save non-volatile registers
4434 #ifdef _WIN64
4435     // last 2 arguments (#4, #5) are on stack on Win64
4436     __ movptr(z, Address(rsp, 6 * wordSize));
4437     __ movptr(zlen, Address(rsp, 7 * wordSize));
4438 #endif
4439 
4440     __ movptr(xlen, rsi);
4441     __ movptr(y,    rdx);
4442     __ multiply_to_len(x, xlen, y, ylen, z, zlen, tmp1, tmp2, tmp3, tmp4, tmp5);
4443 
4444     restore_arg_regs();
4445 
4446     __ leave(); // required for proper stackwalking of RuntimeStub frame
4447     __ ret(0);
4448 
4449     return start;
4450   }
4451 
4452   /**
4453   *  Arguments:
4454   *
4455   *  Input:
4456   *    c_rarg0   - obja     address
4457   *    c_rarg1   - objb     address
4458   *    c_rarg3   - length   length
4459   *    c_rarg4   - scale    log2_array_indxscale
4460   *
4461   *  Output:
4462   *        rax   - int >= mismatched index, < 0 bitwise complement of tail
4463   */
4464   address generate_vectorizedMismatch() {
4465     __ align(CodeEntryAlignment);
4466     StubCodeMark mark(this, "StubRoutines", "vectorizedMismatch");
4467     address start = __ pc();
4468 
4469     BLOCK_COMMENT("Entry:");
4470     __ enter();
4471 
4472 #ifdef _WIN64  // Win64: rcx, rdx, r8, r9 (c_rarg0, c_rarg1, ...)
4473     const Register scale = c_rarg0;  //rcx, will exchange with r9
4474     const Register objb = c_rarg1;   //rdx
4475     const Register length = c_rarg2; //r8
4476     const Register obja = c_rarg3;   //r9
4477     __ xchgq(obja, scale);  //now obja and scale contains the correct contents
4478 
4479     const Register tmp1 = r10;
4480     const Register tmp2 = r11;
4481 #endif
4482 #ifndef _WIN64 // Unix:  rdi, rsi, rdx, rcx, r8, r9 (c_rarg0, c_rarg1, ...)
4483     const Register obja = c_rarg0;   //U:rdi
4484     const Register objb = c_rarg1;   //U:rsi
4485     const Register length = c_rarg2; //U:rdx
4486     const Register scale = c_rarg3;  //U:rcx
4487     const Register tmp1 = r8;
4488     const Register tmp2 = r9;
4489 #endif
4490     const Register result = rax; //return value
4491     const XMMRegister vec0 = xmm0;
4492     const XMMRegister vec1 = xmm1;
4493     const XMMRegister vec2 = xmm2;
4494 
4495     __ vectorized_mismatch(obja, objb, length, scale, result, tmp1, tmp2, vec0, vec1, vec2);
4496 
4497     __ leave();
4498     __ ret(0);
4499 
4500     return start;
4501   }
4502 
4503 /**
4504    *  Arguments:
4505    *
4506   //  Input:
4507   //    c_rarg0   - x address
4508   //    c_rarg1   - x length
4509   //    c_rarg2   - z address
4510   //    c_rarg3   - z lenth
4511    *
4512    */
4513   address generate_squareToLen() {
4514 
4515     __ align(CodeEntryAlignment);
4516     StubCodeMark mark(this, "StubRoutines", "squareToLen");
4517 
4518     address start = __ pc();
4519     // Win64: rcx, rdx, r8, r9 (c_rarg0, c_rarg1, ...)
4520     // Unix:  rdi, rsi, rdx, rcx (c_rarg0, c_rarg1, ...)
4521     const Register x      = rdi;
4522     const Register len    = rsi;
4523     const Register z      = r8;
4524     const Register zlen   = rcx;
4525 
4526    const Register tmp1      = r12;
4527    const Register tmp2      = r13;
4528    const Register tmp3      = r14;
4529    const Register tmp4      = r15;
4530    const Register tmp5      = rbx;
4531 
4532     BLOCK_COMMENT("Entry:");
4533     __ enter(); // required for proper stackwalking of RuntimeStub frame
4534 
4535        setup_arg_regs(4); // x => rdi, len => rsi, z => rdx
4536                           // zlen => rcx
4537                           // r9 and r10 may be used to save non-volatile registers
4538     __ movptr(r8, rdx);
4539     __ square_to_len(x, len, z, zlen, tmp1, tmp2, tmp3, tmp4, tmp5, rdx, rax);
4540 
4541     restore_arg_regs();
4542 
4543     __ leave(); // required for proper stackwalking of RuntimeStub frame
4544     __ ret(0);
4545 
4546     return start;
4547   }
4548 
4549    /**
4550    *  Arguments:
4551    *
4552    *  Input:
4553    *    c_rarg0   - out address
4554    *    c_rarg1   - in address
4555    *    c_rarg2   - offset
4556    *    c_rarg3   - len
4557    * not Win64
4558    *    c_rarg4   - k
4559    * Win64
4560    *    rsp+40    - k
4561    */
4562   address generate_mulAdd() {
4563     __ align(CodeEntryAlignment);
4564     StubCodeMark mark(this, "StubRoutines", "mulAdd");
4565 
4566     address start = __ pc();
4567     // Win64: rcx, rdx, r8, r9 (c_rarg0, c_rarg1, ...)
4568     // Unix:  rdi, rsi, rdx, rcx, r8, r9 (c_rarg0, c_rarg1, ...)
4569     const Register out     = rdi;
4570     const Register in      = rsi;
4571     const Register offset  = r11;
4572     const Register len     = rcx;
4573     const Register k       = r8;
4574 
4575     // Next registers will be saved on stack in mul_add().
4576     const Register tmp1  = r12;
4577     const Register tmp2  = r13;
4578     const Register tmp3  = r14;
4579     const Register tmp4  = r15;
4580     const Register tmp5  = rbx;
4581 
4582     BLOCK_COMMENT("Entry:");
4583     __ enter(); // required for proper stackwalking of RuntimeStub frame
4584 
4585     setup_arg_regs(4); // out => rdi, in => rsi, offset => rdx
4586                        // len => rcx, k => r8
4587                        // r9 and r10 may be used to save non-volatile registers
4588 #ifdef _WIN64
4589     // last argument is on stack on Win64
4590     __ movl(k, Address(rsp, 6 * wordSize));
4591 #endif
4592     __ movptr(r11, rdx);  // move offset in rdx to offset(r11)
4593     __ mul_add(out, in, offset, len, k, tmp1, tmp2, tmp3, tmp4, tmp5, rdx, rax);
4594 
4595     restore_arg_regs();
4596 
4597     __ leave(); // required for proper stackwalking of RuntimeStub frame
4598     __ ret(0);
4599 
4600     return start;
4601   }
4602 
4603   address generate_libmExp() {
4604     address start = __ pc();
4605 
4606     const XMMRegister x0  = xmm0;
4607     const XMMRegister x1  = xmm1;
4608     const XMMRegister x2  = xmm2;
4609     const XMMRegister x3  = xmm3;
4610 
4611     const XMMRegister x4  = xmm4;
4612     const XMMRegister x5  = xmm5;
4613     const XMMRegister x6  = xmm6;
4614     const XMMRegister x7  = xmm7;
4615 
4616     const Register tmp   = r11;
4617 
4618     BLOCK_COMMENT("Entry:");
4619     __ enter(); // required for proper stackwalking of RuntimeStub frame
4620 
4621       __ fast_exp(x0, x1, x2, x3, x4, x5, x6, x7, rax, rcx, rdx, tmp);
4622 
4623     __ leave(); // required for proper stackwalking of RuntimeStub frame
4624     __ ret(0);
4625 
4626     return start;
4627 
4628   }
4629 
4630   address generate_libmLog() {
4631     address start = __ pc();
4632 
4633     const XMMRegister x0 = xmm0;
4634     const XMMRegister x1 = xmm1;
4635     const XMMRegister x2 = xmm2;
4636     const XMMRegister x3 = xmm3;
4637 
4638     const XMMRegister x4 = xmm4;
4639     const XMMRegister x5 = xmm5;
4640     const XMMRegister x6 = xmm6;
4641     const XMMRegister x7 = xmm7;
4642 
4643     const Register tmp1 = r11;
4644     const Register tmp2 = r8;
4645 
4646     BLOCK_COMMENT("Entry:");
4647     __ enter(); // required for proper stackwalking of RuntimeStub frame
4648 
4649     __ fast_log(x0, x1, x2, x3, x4, x5, x6, x7, rax, rcx, rdx, tmp1, tmp2);
4650 
4651     __ leave(); // required for proper stackwalking of RuntimeStub frame
4652     __ ret(0);
4653 
4654     return start;
4655 
4656   }
4657 
4658   address generate_libmLog10() {
4659     address start = __ pc();
4660 
4661     const XMMRegister x0 = xmm0;
4662     const XMMRegister x1 = xmm1;
4663     const XMMRegister x2 = xmm2;
4664     const XMMRegister x3 = xmm3;
4665 
4666     const XMMRegister x4 = xmm4;
4667     const XMMRegister x5 = xmm5;
4668     const XMMRegister x6 = xmm6;
4669     const XMMRegister x7 = xmm7;
4670 
4671     const Register tmp = r11;
4672 
4673     BLOCK_COMMENT("Entry:");
4674     __ enter(); // required for proper stackwalking of RuntimeStub frame
4675 
4676     __ fast_log10(x0, x1, x2, x3, x4, x5, x6, x7, rax, rcx, rdx, tmp);
4677 
4678     __ leave(); // required for proper stackwalking of RuntimeStub frame
4679     __ ret(0);
4680 
4681     return start;
4682 
4683   }
4684 
4685   address generate_libmPow() {
4686     address start = __ pc();
4687 
4688     const XMMRegister x0 = xmm0;
4689     const XMMRegister x1 = xmm1;
4690     const XMMRegister x2 = xmm2;
4691     const XMMRegister x3 = xmm3;
4692 
4693     const XMMRegister x4 = xmm4;
4694     const XMMRegister x5 = xmm5;
4695     const XMMRegister x6 = xmm6;
4696     const XMMRegister x7 = xmm7;
4697 
4698     const Register tmp1 = r8;
4699     const Register tmp2 = r9;
4700     const Register tmp3 = r10;
4701     const Register tmp4 = r11;
4702 
4703     BLOCK_COMMENT("Entry:");
4704     __ enter(); // required for proper stackwalking of RuntimeStub frame
4705 
4706     __ fast_pow(x0, x1, x2, x3, x4, x5, x6, x7, rax, rcx, rdx, tmp1, tmp2, tmp3, tmp4);
4707 
4708     __ leave(); // required for proper stackwalking of RuntimeStub frame
4709     __ ret(0);
4710 
4711     return start;
4712 
4713   }
4714 
4715   address generate_libmSin() {
4716     address start = __ pc();
4717 
4718     const XMMRegister x0 = xmm0;
4719     const XMMRegister x1 = xmm1;
4720     const XMMRegister x2 = xmm2;
4721     const XMMRegister x3 = xmm3;
4722 
4723     const XMMRegister x4 = xmm4;
4724     const XMMRegister x5 = xmm5;
4725     const XMMRegister x6 = xmm6;
4726     const XMMRegister x7 = xmm7;
4727 
4728     const Register tmp1 = r8;
4729     const Register tmp2 = r9;
4730     const Register tmp3 = r10;
4731     const Register tmp4 = r11;
4732 
4733     BLOCK_COMMENT("Entry:");
4734     __ enter(); // required for proper stackwalking of RuntimeStub frame
4735 
4736 #ifdef _WIN64
4737     __ push(rsi);
4738     __ push(rdi);
4739 #endif
4740     __ fast_sin(x0, x1, x2, x3, x4, x5, x6, x7, rax, rbx, rcx, rdx, tmp1, tmp2, tmp3, tmp4);
4741 
4742 #ifdef _WIN64
4743     __ pop(rdi);
4744     __ pop(rsi);
4745 #endif
4746 
4747     __ leave(); // required for proper stackwalking of RuntimeStub frame
4748     __ ret(0);
4749 
4750     return start;
4751 
4752   }
4753 
4754   address generate_libmCos() {
4755     address start = __ pc();
4756 
4757     const XMMRegister x0 = xmm0;
4758     const XMMRegister x1 = xmm1;
4759     const XMMRegister x2 = xmm2;
4760     const XMMRegister x3 = xmm3;
4761 
4762     const XMMRegister x4 = xmm4;
4763     const XMMRegister x5 = xmm5;
4764     const XMMRegister x6 = xmm6;
4765     const XMMRegister x7 = xmm7;
4766 
4767     const Register tmp1 = r8;
4768     const Register tmp2 = r9;
4769     const Register tmp3 = r10;
4770     const Register tmp4 = r11;
4771 
4772     BLOCK_COMMENT("Entry:");
4773     __ enter(); // required for proper stackwalking of RuntimeStub frame
4774 
4775 #ifdef _WIN64
4776     __ push(rsi);
4777     __ push(rdi);
4778 #endif
4779     __ fast_cos(x0, x1, x2, x3, x4, x5, x6, x7, rax, rcx, rdx, tmp1, tmp2, tmp3, tmp4);
4780 
4781 #ifdef _WIN64
4782     __ pop(rdi);
4783     __ pop(rsi);
4784 #endif
4785 
4786     __ leave(); // required for proper stackwalking of RuntimeStub frame
4787     __ ret(0);
4788 
4789     return start;
4790 
4791   }
4792 
4793   address generate_libmTan() {
4794     address start = __ pc();
4795 
4796     const XMMRegister x0 = xmm0;
4797     const XMMRegister x1 = xmm1;
4798     const XMMRegister x2 = xmm2;
4799     const XMMRegister x3 = xmm3;
4800 
4801     const XMMRegister x4 = xmm4;
4802     const XMMRegister x5 = xmm5;
4803     const XMMRegister x6 = xmm6;
4804     const XMMRegister x7 = xmm7;
4805 
4806     const Register tmp1 = r8;
4807     const Register tmp2 = r9;
4808     const Register tmp3 = r10;
4809     const Register tmp4 = r11;
4810 
4811     BLOCK_COMMENT("Entry:");
4812     __ enter(); // required for proper stackwalking of RuntimeStub frame
4813 
4814 #ifdef _WIN64
4815     __ push(rsi);
4816     __ push(rdi);
4817 #endif
4818     __ fast_tan(x0, x1, x2, x3, x4, x5, x6, x7, rax, rcx, rdx, tmp1, tmp2, tmp3, tmp4);
4819 
4820 #ifdef _WIN64
4821     __ pop(rdi);
4822     __ pop(rsi);
4823 #endif
4824 
4825     __ leave(); // required for proper stackwalking of RuntimeStub frame
4826     __ ret(0);
4827 
4828     return start;
4829 
4830   }
4831 
4832 #undef __
4833 #define __ masm->
4834 
4835   // Continuation point for throwing of implicit exceptions that are
4836   // not handled in the current activation. Fabricates an exception
4837   // oop and initiates normal exception dispatching in this
4838   // frame. Since we need to preserve callee-saved values (currently
4839   // only for C2, but done for C1 as well) we need a callee-saved oop
4840   // map and therefore have to make these stubs into RuntimeStubs
4841   // rather than BufferBlobs.  If the compiler needs all registers to
4842   // be preserved between the fault point and the exception handler
4843   // then it must assume responsibility for that in
4844   // AbstractCompiler::continuation_for_implicit_null_exception or
4845   // continuation_for_implicit_division_by_zero_exception. All other
4846   // implicit exceptions (e.g., NullPointerException or
4847   // AbstractMethodError on entry) are either at call sites or
4848   // otherwise assume that stack unwinding will be initiated, so
4849   // caller saved registers were assumed volatile in the compiler.
4850   address generate_throw_exception(const char* name,
4851                                    address runtime_entry,
4852                                    Register arg1 = noreg,
4853                                    Register arg2 = noreg) {
4854     // Information about frame layout at time of blocking runtime call.
4855     // Note that we only have to preserve callee-saved registers since
4856     // the compilers are responsible for supplying a continuation point
4857     // if they expect all registers to be preserved.
4858     enum layout {
4859       rbp_off = frame::arg_reg_save_area_bytes/BytesPerInt,
4860       rbp_off2,
4861       return_off,
4862       return_off2,
4863       framesize // inclusive of return address
4864     };
4865 
4866     int insts_size = 512;
4867     int locs_size  = 64;
4868 
4869     CodeBuffer code(name, insts_size, locs_size);
4870     OopMapSet* oop_maps  = new OopMapSet();
4871     MacroAssembler* masm = new MacroAssembler(&code);
4872 
4873     address start = __ pc();
4874 
4875     // This is an inlined and slightly modified version of call_VM
4876     // which has the ability to fetch the return PC out of
4877     // thread-local storage and also sets up last_Java_sp slightly
4878     // differently than the real call_VM
4879 
4880     __ enter(); // required for proper stackwalking of RuntimeStub frame
4881 
4882     assert(is_even(framesize/2), "sp not 16-byte aligned");
4883 
4884     // return address and rbp are already in place
4885     __ subptr(rsp, (framesize-4) << LogBytesPerInt); // prolog
4886 
4887     int frame_complete = __ pc() - start;
4888 
4889     // Set up last_Java_sp and last_Java_fp
4890     address the_pc = __ pc();
4891     __ set_last_Java_frame(rsp, rbp, the_pc);
4892     __ andptr(rsp, -(StackAlignmentInBytes));    // Align stack
4893 
4894     // Call runtime
4895     if (arg1 != noreg) {
4896       assert(arg2 != c_rarg1, "clobbered");
4897       __ movptr(c_rarg1, arg1);
4898     }
4899     if (arg2 != noreg) {
4900       __ movptr(c_rarg2, arg2);
4901     }
4902     __ movptr(c_rarg0, r15_thread);
4903     BLOCK_COMMENT("call runtime_entry");
4904     __ call(RuntimeAddress(runtime_entry));
4905 
4906     // Generate oop map
4907     OopMap* map = new OopMap(framesize, 0);
4908 
4909     oop_maps->add_gc_map(the_pc - start, map);
4910 
4911     __ reset_last_Java_frame(true);
4912 
4913     __ leave(); // required for proper stackwalking of RuntimeStub frame
4914 
4915     // check for pending exceptions
4916 #ifdef ASSERT
4917     Label L;
4918     __ cmpptr(Address(r15_thread, Thread::pending_exception_offset()),
4919             (int32_t) NULL_WORD);
4920     __ jcc(Assembler::notEqual, L);
4921     __ should_not_reach_here();
4922     __ bind(L);
4923 #endif // ASSERT
4924     __ jump(RuntimeAddress(StubRoutines::forward_exception_entry()));
4925 
4926 
4927     // codeBlob framesize is in words (not VMRegImpl::slot_size)
4928     RuntimeStub* stub =
4929       RuntimeStub::new_runtime_stub(name,
4930                                     &code,
4931                                     frame_complete,
4932                                     (framesize >> (LogBytesPerWord - LogBytesPerInt)),
4933                                     oop_maps, false);
4934     return stub->entry_point();
4935   }
4936 
4937   void create_control_words() {
4938     // Round to nearest, 53-bit mode, exceptions masked
4939     StubRoutines::_fpu_cntrl_wrd_std   = 0x027F;
4940     // Round to zero, 53-bit mode, exception mased
4941     StubRoutines::_fpu_cntrl_wrd_trunc = 0x0D7F;
4942     // Round to nearest, 24-bit mode, exceptions masked
4943     StubRoutines::_fpu_cntrl_wrd_24    = 0x007F;
4944     // Round to nearest, 64-bit mode, exceptions masked
4945     StubRoutines::_fpu_cntrl_wrd_64    = 0x037F;
4946     // Round to nearest, 64-bit mode, exceptions masked
4947     StubRoutines::_mxcsr_std           = 0x1F80;
4948     // Note: the following two constants are 80-bit values
4949     //       layout is critical for correct loading by FPU.
4950     // Bias for strict fp multiply/divide
4951     StubRoutines::_fpu_subnormal_bias1[0]= 0x00000000; // 2^(-15360) == 0x03ff 8000 0000 0000 0000
4952     StubRoutines::_fpu_subnormal_bias1[1]= 0x80000000;
4953     StubRoutines::_fpu_subnormal_bias1[2]= 0x03ff;
4954     // Un-Bias for strict fp multiply/divide
4955     StubRoutines::_fpu_subnormal_bias2[0]= 0x00000000; // 2^(+15360) == 0x7bff 8000 0000 0000 0000
4956     StubRoutines::_fpu_subnormal_bias2[1]= 0x80000000;
4957     StubRoutines::_fpu_subnormal_bias2[2]= 0x7bff;
4958   }
4959 
4960   // Initialization
4961   void generate_initial() {
4962     // Generates all stubs and initializes the entry points
4963 
4964     // This platform-specific settings are needed by generate_call_stub()
4965     create_control_words();
4966 
4967     // entry points that exist in all platforms Note: This is code
4968     // that could be shared among different platforms - however the
4969     // benefit seems to be smaller than the disadvantage of having a
4970     // much more complicated generator structure. See also comment in
4971     // stubRoutines.hpp.
4972 
4973     StubRoutines::_forward_exception_entry = generate_forward_exception();
4974 
4975     StubRoutines::_call_stub_entry =
4976       generate_call_stub(StubRoutines::_call_stub_return_address);
4977 
4978     // is referenced by megamorphic call
4979     StubRoutines::_catch_exception_entry = generate_catch_exception();
4980 
4981     // atomic calls
4982     StubRoutines::_atomic_xchg_entry         = generate_atomic_xchg();
4983     StubRoutines::_atomic_xchg_ptr_entry     = generate_atomic_xchg_ptr();
4984     StubRoutines::_atomic_cmpxchg_entry      = generate_atomic_cmpxchg();
4985     StubRoutines::_atomic_cmpxchg_byte_entry = generate_atomic_cmpxchg_byte();
4986     StubRoutines::_atomic_cmpxchg_long_entry = generate_atomic_cmpxchg_long();
4987     StubRoutines::_atomic_add_entry          = generate_atomic_add();
4988     StubRoutines::_atomic_add_ptr_entry      = generate_atomic_add_ptr();
4989     StubRoutines::_fence_entry               = generate_orderaccess_fence();
4990 
4991     // platform dependent
4992     StubRoutines::x86::_get_previous_fp_entry = generate_get_previous_fp();
4993     StubRoutines::x86::_get_previous_sp_entry = generate_get_previous_sp();
4994 
4995     StubRoutines::x86::_verify_mxcsr_entry    = generate_verify_mxcsr();
4996 
4997     // Build this early so it's available for the interpreter.
4998     StubRoutines::_throw_StackOverflowError_entry =
4999       generate_throw_exception("StackOverflowError throw_exception",
5000                                CAST_FROM_FN_PTR(address,
5001                                                 SharedRuntime::
5002                                                 throw_StackOverflowError));
5003     StubRoutines::_throw_delayed_StackOverflowError_entry =
5004       generate_throw_exception("delayed StackOverflowError throw_exception",
5005                                CAST_FROM_FN_PTR(address,
5006                                                 SharedRuntime::
5007                                                 throw_delayed_StackOverflowError));
5008     if (UseCRC32Intrinsics) {
5009       // set table address before stub generation which use it
5010       StubRoutines::_crc_table_adr = (address)StubRoutines::x86::_crc_table;
5011       StubRoutines::_updateBytesCRC32 = generate_updateBytesCRC32();
5012     }
5013 
5014     if (UseCRC32CIntrinsics) {
5015       bool supports_clmul = VM_Version::supports_clmul();
5016       StubRoutines::x86::generate_CRC32C_table(supports_clmul);
5017       StubRoutines::_crc32c_table_addr = (address)StubRoutines::x86::_crc32c_table;
5018       StubRoutines::_updateBytesCRC32C = generate_updateBytesCRC32C(supports_clmul);
5019     }
5020     if (VM_Version::supports_sse2() && UseLibmIntrinsic && InlineIntrinsics) {
5021       if (vmIntrinsics::is_intrinsic_available(vmIntrinsics::_dsin) ||
5022           vmIntrinsics::is_intrinsic_available(vmIntrinsics::_dcos) ||
5023           vmIntrinsics::is_intrinsic_available(vmIntrinsics::_dtan)) {
5024         StubRoutines::x86::_ONEHALF_adr = (address)StubRoutines::x86::_ONEHALF;
5025         StubRoutines::x86::_P_2_adr = (address)StubRoutines::x86::_P_2;
5026         StubRoutines::x86::_SC_4_adr = (address)StubRoutines::x86::_SC_4;
5027         StubRoutines::x86::_Ctable_adr = (address)StubRoutines::x86::_Ctable;
5028         StubRoutines::x86::_SC_2_adr = (address)StubRoutines::x86::_SC_2;
5029         StubRoutines::x86::_SC_3_adr = (address)StubRoutines::x86::_SC_3;
5030         StubRoutines::x86::_SC_1_adr = (address)StubRoutines::x86::_SC_1;
5031         StubRoutines::x86::_PI_INV_TABLE_adr = (address)StubRoutines::x86::_PI_INV_TABLE;
5032         StubRoutines::x86::_PI_4_adr = (address)StubRoutines::x86::_PI_4;
5033         StubRoutines::x86::_PI32INV_adr = (address)StubRoutines::x86::_PI32INV;
5034         StubRoutines::x86::_SIGN_MASK_adr = (address)StubRoutines::x86::_SIGN_MASK;
5035         StubRoutines::x86::_P_1_adr = (address)StubRoutines::x86::_P_1;
5036         StubRoutines::x86::_P_3_adr = (address)StubRoutines::x86::_P_3;
5037         StubRoutines::x86::_NEG_ZERO_adr = (address)StubRoutines::x86::_NEG_ZERO;
5038       }
5039       if (vmIntrinsics::is_intrinsic_available(vmIntrinsics::_dexp)) {
5040         StubRoutines::_dexp = generate_libmExp();
5041       }
5042       if (vmIntrinsics::is_intrinsic_available(vmIntrinsics::_dlog)) {
5043         StubRoutines::_dlog = generate_libmLog();
5044       }
5045       if (vmIntrinsics::is_intrinsic_available(vmIntrinsics::_dlog10)) {
5046         StubRoutines::_dlog10 = generate_libmLog10();
5047       }
5048       if (vmIntrinsics::is_intrinsic_available(vmIntrinsics::_dpow)) {
5049         StubRoutines::_dpow = generate_libmPow();
5050       }
5051       if (vmIntrinsics::is_intrinsic_available(vmIntrinsics::_dsin)) {
5052         StubRoutines::_dsin = generate_libmSin();
5053       }
5054       if (vmIntrinsics::is_intrinsic_available(vmIntrinsics::_dcos)) {
5055         StubRoutines::_dcos = generate_libmCos();
5056       }
5057       if (vmIntrinsics::is_intrinsic_available(vmIntrinsics::_dtan)) {
5058         StubRoutines::_dtan = generate_libmTan();
5059       }
5060     }
5061   }
5062 
5063   void generate_all() {
5064     // Generates all stubs and initializes the entry points
5065 
5066     // These entry points require SharedInfo::stack0 to be set up in
5067     // non-core builds and need to be relocatable, so they each
5068     // fabricate a RuntimeStub internally.
5069     StubRoutines::_throw_AbstractMethodError_entry =
5070       generate_throw_exception("AbstractMethodError throw_exception",
5071                                CAST_FROM_FN_PTR(address,
5072                                                 SharedRuntime::
5073                                                 throw_AbstractMethodError));
5074 
5075     StubRoutines::_throw_IncompatibleClassChangeError_entry =
5076       generate_throw_exception("IncompatibleClassChangeError throw_exception",
5077                                CAST_FROM_FN_PTR(address,
5078                                                 SharedRuntime::
5079                                                 throw_IncompatibleClassChangeError));
5080 
5081     StubRoutines::_throw_NullPointerException_at_call_entry =
5082       generate_throw_exception("NullPointerException at call throw_exception",
5083                                CAST_FROM_FN_PTR(address,
5084                                                 SharedRuntime::
5085                                                 throw_NullPointerException_at_call));
5086 
5087     // entry points that are platform specific
5088     StubRoutines::x86::_f2i_fixup = generate_f2i_fixup();
5089     StubRoutines::x86::_f2l_fixup = generate_f2l_fixup();
5090     StubRoutines::x86::_d2i_fixup = generate_d2i_fixup();
5091     StubRoutines::x86::_d2l_fixup = generate_d2l_fixup();
5092 
5093     StubRoutines::x86::_float_sign_mask  = generate_fp_mask("float_sign_mask",  0x7FFFFFFF7FFFFFFF);
5094     StubRoutines::x86::_float_sign_flip  = generate_fp_mask("float_sign_flip",  0x8000000080000000);
5095     StubRoutines::x86::_double_sign_mask = generate_fp_mask("double_sign_mask", 0x7FFFFFFFFFFFFFFF);
5096     StubRoutines::x86::_double_sign_flip = generate_fp_mask("double_sign_flip", 0x8000000000000000);
5097 
5098     // support for verify_oop (must happen after universe_init)
5099     StubRoutines::_verify_oop_subroutine_entry = generate_verify_oop();
5100 
5101     // arraycopy stubs used by compilers
5102     generate_arraycopy_stubs();
5103 
5104     // don't bother generating these AES intrinsic stubs unless global flag is set
5105     if (UseAESIntrinsics) {
5106       StubRoutines::x86::_key_shuffle_mask_addr = generate_key_shuffle_mask();  // needed by the others
5107       StubRoutines::_aescrypt_encryptBlock = generate_aescrypt_encryptBlock();
5108       StubRoutines::_aescrypt_decryptBlock = generate_aescrypt_decryptBlock();
5109       StubRoutines::_cipherBlockChaining_encryptAESCrypt = generate_cipherBlockChaining_encryptAESCrypt();
5110       StubRoutines::_cipherBlockChaining_decryptAESCrypt = generate_cipherBlockChaining_decryptAESCrypt_Parallel();
5111     }
5112     if (UseAESCTRIntrinsics){
5113       StubRoutines::x86::_counter_shuffle_mask_addr = generate_counter_shuffle_mask();
5114       StubRoutines::_counterMode_AESCrypt = generate_counterMode_AESCrypt_Parallel();
5115     }
5116 
5117     if (UseSHA1Intrinsics) {
5118       StubRoutines::x86::_upper_word_mask_addr = generate_upper_word_mask();
5119       StubRoutines::x86::_shuffle_byte_flip_mask_addr = generate_shuffle_byte_flip_mask();
5120       StubRoutines::_sha1_implCompress = generate_sha1_implCompress(false, "sha1_implCompress");
5121       StubRoutines::_sha1_implCompressMB = generate_sha1_implCompress(true, "sha1_implCompressMB");
5122     }
5123     if (UseSHA256Intrinsics) {
5124       StubRoutines::x86::_k256_adr = (address)StubRoutines::x86::_k256;
5125       char* dst = (char*)StubRoutines::x86::_k256_W;
5126       char* src = (char*)StubRoutines::x86::_k256;
5127       for (int ii = 0; ii < 16; ++ii) {
5128         memcpy(dst + 32 * ii,      src + 16 * ii, 16);
5129         memcpy(dst + 32 * ii + 16, src + 16 * ii, 16);
5130       }
5131       StubRoutines::x86::_k256_W_adr = (address)StubRoutines::x86::_k256_W;
5132       StubRoutines::x86::_pshuffle_byte_flip_mask_addr = generate_pshuffle_byte_flip_mask();
5133       StubRoutines::_sha256_implCompress = generate_sha256_implCompress(false, "sha256_implCompress");
5134       StubRoutines::_sha256_implCompressMB = generate_sha256_implCompress(true, "sha256_implCompressMB");
5135     }
5136     if (UseSHA512Intrinsics) {
5137       StubRoutines::x86::_k512_W_addr = (address)StubRoutines::x86::_k512_W;
5138       StubRoutines::x86::_pshuffle_byte_flip_mask_addr_sha512 = generate_pshuffle_byte_flip_mask_sha512();
5139       StubRoutines::_sha512_implCompress = generate_sha512_implCompress(false, "sha512_implCompress");
5140       StubRoutines::_sha512_implCompressMB = generate_sha512_implCompress(true, "sha512_implCompressMB");
5141     }
5142 
5143     // Generate GHASH intrinsics code
5144     if (UseGHASHIntrinsics) {
5145       StubRoutines::x86::_ghash_long_swap_mask_addr = generate_ghash_long_swap_mask();
5146       StubRoutines::x86::_ghash_byte_swap_mask_addr = generate_ghash_byte_swap_mask();
5147       StubRoutines::_ghash_processBlocks = generate_ghash_processBlocks();
5148     }
5149 
5150     // Safefetch stubs.
5151     generate_safefetch("SafeFetch32", sizeof(int),     &StubRoutines::_safefetch32_entry,
5152                                                        &StubRoutines::_safefetch32_fault_pc,
5153                                                        &StubRoutines::_safefetch32_continuation_pc);
5154     generate_safefetch("SafeFetchN", sizeof(intptr_t), &StubRoutines::_safefetchN_entry,
5155                                                        &StubRoutines::_safefetchN_fault_pc,
5156                                                        &StubRoutines::_safefetchN_continuation_pc);
5157 #ifdef COMPILER2
5158     if (UseMultiplyToLenIntrinsic) {
5159       StubRoutines::_multiplyToLen = generate_multiplyToLen();
5160     }
5161     if (UseSquareToLenIntrinsic) {
5162       StubRoutines::_squareToLen = generate_squareToLen();
5163     }
5164     if (UseMulAddIntrinsic) {
5165       StubRoutines::_mulAdd = generate_mulAdd();
5166     }
5167 #ifndef _WINDOWS
5168     if (UseMontgomeryMultiplyIntrinsic) {
5169       StubRoutines::_montgomeryMultiply
5170         = CAST_FROM_FN_PTR(address, SharedRuntime::montgomery_multiply);
5171     }
5172     if (UseMontgomerySquareIntrinsic) {
5173       StubRoutines::_montgomerySquare
5174         = CAST_FROM_FN_PTR(address, SharedRuntime::montgomery_square);
5175     }
5176 #endif // WINDOWS
5177 #endif // COMPILER2
5178 
5179     if (UseVectorizedMismatchIntrinsic) {
5180       StubRoutines::_vectorizedMismatch = generate_vectorizedMismatch();
5181     }
5182   }
5183 
5184  public:
5185   StubGenerator(CodeBuffer* code, bool all) : StubCodeGenerator(code) {
5186     if (all) {
5187       generate_all();
5188     } else {
5189       generate_initial();
5190     }
5191   }
5192 }; // end class declaration
5193 
5194 void StubGenerator_generate(CodeBuffer* code, bool all) {
5195   StubGenerator g(code, all);
5196 }