1 /*
2 * Copyright (c) 1997, 2015, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "precompiled.hpp"
26 #include "asm/assembler.hpp"
27 #include "asm/assembler.inline.hpp"
28 #include "gc/shared/cardTableModRefBS.hpp"
29 #include "gc/shared/collectedHeap.inline.hpp"
30 #include "interpreter/interpreter.hpp"
31 #include "memory/resourceArea.hpp"
32 #include "prims/methodHandles.hpp"
33 #include "runtime/biasedLocking.hpp"
34 #include "runtime/interfaceSupport.hpp"
35 #include "runtime/objectMonitor.hpp"
36 #include "runtime/os.hpp"
37 #include "runtime/sharedRuntime.hpp"
38 #include "runtime/stubRoutines.hpp"
39 #include "utilities/macros.hpp"
40 #if INCLUDE_ALL_GCS
41 #include "gc/g1/g1CollectedHeap.inline.hpp"
42 #include "gc/g1/g1SATBCardTableModRefBS.hpp"
43 #include "gc/g1/heapRegion.hpp"
44 #endif // INCLUDE_ALL_GCS
45
46 #ifdef PRODUCT
47 #define BLOCK_COMMENT(str) /* nothing */
48 #define STOP(error) stop(error)
49 #else
50 #define BLOCK_COMMENT(str) block_comment(str)
51 #define STOP(error) block_comment(error); stop(error)
52 #endif
53
54 #define BIND(label) bind(label); BLOCK_COMMENT(#label ":")
55 // Implementation of AddressLiteral
56
57 // A 2-D table for managing compressed displacement(disp8) on EVEX enabled platforms.
58 unsigned char tuple_table[Assembler::EVEX_ETUP + 1][Assembler::AVX_512bit + 1] = {
59 // -----------------Table 4.5 -------------------- //
60 16, 32, 64, // EVEX_FV(0)
61 4, 4, 4, // EVEX_FV(1) - with Evex.b
62 16, 32, 64, // EVEX_FV(2) - with Evex.w
63 8, 8, 8, // EVEX_FV(3) - with Evex.w and Evex.b
64 8, 16, 32, // EVEX_HV(0)
65 4, 4, 4, // EVEX_HV(1) - with Evex.b
66 // -----------------Table 4.6 -------------------- //
67 16, 32, 64, // EVEX_FVM(0)
68 1, 1, 1, // EVEX_T1S(0)
69 2, 2, 2, // EVEX_T1S(1)
70 4, 4, 4, // EVEX_T1S(2)
71 8, 8, 8, // EVEX_T1S(3)
72 4, 4, 4, // EVEX_T1F(0)
73 8, 8, 8, // EVEX_T1F(1)
74 8, 8, 8, // EVEX_T2(0)
75 0, 16, 16, // EVEX_T2(1)
76 0, 16, 16, // EVEX_T4(0)
77 0, 0, 32, // EVEX_T4(1)
78 0, 0, 32, // EVEX_T8(0)
79 8, 16, 32, // EVEX_HVM(0)
80 4, 8, 16, // EVEX_QVM(0)
81 2, 4, 8, // EVEX_OVM(0)
82 16, 16, 16, // EVEX_M128(0)
83 8, 32, 64, // EVEX_DUP(0)
84 0, 0, 0 // EVEX_NTUP
85 };
86
87 AddressLiteral::AddressLiteral(address target, relocInfo::relocType rtype) {
88 _is_lval = false;
89 _target = target;
90 switch (rtype) {
91 case relocInfo::oop_type:
92 case relocInfo::metadata_type:
93 // Oops are a special case. Normally they would be their own section
94 // but in cases like icBuffer they are literals in the code stream that
95 // we don't have a section for. We use none so that we get a literal address
96 // which is always patchable.
97 break;
98 case relocInfo::external_word_type:
99 _rspec = external_word_Relocation::spec(target);
100 break;
101 case relocInfo::internal_word_type:
102 _rspec = internal_word_Relocation::spec(target);
103 break;
104 case relocInfo::opt_virtual_call_type:
105 _rspec = opt_virtual_call_Relocation::spec();
106 break;
107 case relocInfo::static_call_type:
108 _rspec = static_call_Relocation::spec();
109 break;
110 case relocInfo::runtime_call_type:
111 _rspec = runtime_call_Relocation::spec();
112 break;
113 case relocInfo::poll_type:
114 case relocInfo::poll_return_type:
115 _rspec = Relocation::spec_simple(rtype);
116 break;
117 case relocInfo::none:
118 break;
119 default:
120 ShouldNotReachHere();
121 break;
122 }
123 }
124
125 // Implementation of Address
126
127 #ifdef _LP64
128
129 Address Address::make_array(ArrayAddress adr) {
130 // Not implementable on 64bit machines
131 // Should have been handled higher up the call chain.
132 ShouldNotReachHere();
133 return Address();
134 }
135
136 // exceedingly dangerous constructor
137 Address::Address(int disp, address loc, relocInfo::relocType rtype) {
138 _base = noreg;
139 _index = noreg;
140 _scale = no_scale;
141 _disp = disp;
142 switch (rtype) {
143 case relocInfo::external_word_type:
144 _rspec = external_word_Relocation::spec(loc);
145 break;
146 case relocInfo::internal_word_type:
147 _rspec = internal_word_Relocation::spec(loc);
148 break;
149 case relocInfo::runtime_call_type:
150 // HMM
151 _rspec = runtime_call_Relocation::spec();
152 break;
153 case relocInfo::poll_type:
154 case relocInfo::poll_return_type:
155 _rspec = Relocation::spec_simple(rtype);
156 break;
157 case relocInfo::none:
158 break;
159 default:
160 ShouldNotReachHere();
161 }
162 }
163 #else // LP64
164
165 Address Address::make_array(ArrayAddress adr) {
166 AddressLiteral base = adr.base();
167 Address index = adr.index();
168 assert(index._disp == 0, "must not have disp"); // maybe it can?
169 Address array(index._base, index._index, index._scale, (intptr_t) base.target());
170 array._rspec = base._rspec;
171 return array;
172 }
173
174 // exceedingly dangerous constructor
175 Address::Address(address loc, RelocationHolder spec) {
176 _base = noreg;
177 _index = noreg;
178 _scale = no_scale;
179 _disp = (intptr_t) loc;
180 _rspec = spec;
181 }
182
183 #endif // _LP64
184
185
186
187 // Convert the raw encoding form into the form expected by the constructor for
188 // Address. An index of 4 (rsp) corresponds to having no index, so convert
189 // that to noreg for the Address constructor.
190 Address Address::make_raw(int base, int index, int scale, int disp, relocInfo::relocType disp_reloc) {
191 RelocationHolder rspec;
192 if (disp_reloc != relocInfo::none) {
193 rspec = Relocation::spec_simple(disp_reloc);
194 }
195 bool valid_index = index != rsp->encoding();
196 if (valid_index) {
197 Address madr(as_Register(base), as_Register(index), (Address::ScaleFactor)scale, in_ByteSize(disp));
198 madr._rspec = rspec;
199 return madr;
200 } else {
201 Address madr(as_Register(base), noreg, Address::no_scale, in_ByteSize(disp));
202 madr._rspec = rspec;
203 return madr;
204 }
205 }
206
207 // Implementation of Assembler
208
209 int AbstractAssembler::code_fill_byte() {
210 return (u_char)'\xF4'; // hlt
211 }
212
213 // make this go away someday
214 void Assembler::emit_data(jint data, relocInfo::relocType rtype, int format) {
215 if (rtype == relocInfo::none)
216 emit_int32(data);
217 else
218 emit_data(data, Relocation::spec_simple(rtype), format);
219 }
220
221 void Assembler::emit_data(jint data, RelocationHolder const& rspec, int format) {
222 assert(imm_operand == 0, "default format must be immediate in this file");
223 assert(inst_mark() != NULL, "must be inside InstructionMark");
224 if (rspec.type() != relocInfo::none) {
225 #ifdef ASSERT
226 check_relocation(rspec, format);
227 #endif
228 // Do not use AbstractAssembler::relocate, which is not intended for
229 // embedded words. Instead, relocate to the enclosing instruction.
230
231 // hack. call32 is too wide for mask so use disp32
232 if (format == call32_operand)
233 code_section()->relocate(inst_mark(), rspec, disp32_operand);
234 else
235 code_section()->relocate(inst_mark(), rspec, format);
236 }
237 emit_int32(data);
238 }
239
240 static int encode(Register r) {
241 int enc = r->encoding();
242 if (enc >= 8) {
243 enc -= 8;
244 }
245 return enc;
246 }
247
248 void Assembler::emit_arith_b(int op1, int op2, Register dst, int imm8) {
249 assert(dst->has_byte_register(), "must have byte register");
250 assert(isByte(op1) && isByte(op2), "wrong opcode");
251 assert(isByte(imm8), "not a byte");
252 assert((op1 & 0x01) == 0, "should be 8bit operation");
253 emit_int8(op1);
254 emit_int8(op2 | encode(dst));
255 emit_int8(imm8);
256 }
257
258
259 void Assembler::emit_arith(int op1, int op2, Register dst, int32_t imm32) {
260 assert(isByte(op1) && isByte(op2), "wrong opcode");
261 assert((op1 & 0x01) == 1, "should be 32bit operation");
262 assert((op1 & 0x02) == 0, "sign-extension bit should not be set");
263 if (is8bit(imm32)) {
264 emit_int8(op1 | 0x02); // set sign bit
265 emit_int8(op2 | encode(dst));
266 emit_int8(imm32 & 0xFF);
267 } else {
268 emit_int8(op1);
269 emit_int8(op2 | encode(dst));
270 emit_int32(imm32);
271 }
272 }
273
274 // Force generation of a 4 byte immediate value even if it fits into 8bit
275 void Assembler::emit_arith_imm32(int op1, int op2, Register dst, int32_t imm32) {
276 assert(isByte(op1) && isByte(op2), "wrong opcode");
277 assert((op1 & 0x01) == 1, "should be 32bit operation");
278 assert((op1 & 0x02) == 0, "sign-extension bit should not be set");
279 emit_int8(op1);
280 emit_int8(op2 | encode(dst));
281 emit_int32(imm32);
282 }
283
284 // immediate-to-memory forms
285 void Assembler::emit_arith_operand(int op1, Register rm, Address adr, int32_t imm32) {
286 assert((op1 & 0x01) == 1, "should be 32bit operation");
287 assert((op1 & 0x02) == 0, "sign-extension bit should not be set");
288 if (is8bit(imm32)) {
289 emit_int8(op1 | 0x02); // set sign bit
290 emit_operand(rm, adr, 1);
291 emit_int8(imm32 & 0xFF);
292 } else {
293 emit_int8(op1);
294 emit_operand(rm, adr, 4);
295 emit_int32(imm32);
296 }
297 }
298
299
300 void Assembler::emit_arith(int op1, int op2, Register dst, Register src) {
301 assert(isByte(op1) && isByte(op2), "wrong opcode");
302 emit_int8(op1);
303 emit_int8(op2 | encode(dst) << 3 | encode(src));
304 }
305
306
307 bool Assembler::query_compressed_disp_byte(int disp, bool is_evex_inst, int vector_len,
308 int cur_tuple_type, int in_size_in_bits, int cur_encoding) {
309 int mod_idx = 0;
310 // We will test if the displacement fits the compressed format and if so
311 // apply the compression to the displacment iff the result is8bit.
312 if (VM_Version::supports_evex() && is_evex_inst) {
313 switch (cur_tuple_type) {
314 case EVEX_FV:
315 if ((cur_encoding & VEX_W) == VEX_W) {
316 mod_idx += 2 + ((cur_encoding & EVEX_Rb) == EVEX_Rb) ? 1 : 0;
317 } else {
318 mod_idx = ((cur_encoding & EVEX_Rb) == EVEX_Rb) ? 1 : 0;
319 }
320 break;
321
322 case EVEX_HV:
323 mod_idx = ((cur_encoding & EVEX_Rb) == EVEX_Rb) ? 1 : 0;
324 break;
325
326 case EVEX_FVM:
327 break;
328
329 case EVEX_T1S:
330 switch (in_size_in_bits) {
331 case EVEX_8bit:
332 break;
333
334 case EVEX_16bit:
335 mod_idx = 1;
336 break;
337
338 case EVEX_32bit:
339 mod_idx = 2;
340 break;
341
342 case EVEX_64bit:
343 mod_idx = 3;
344 break;
345 }
346 break;
347
348 case EVEX_T1F:
349 case EVEX_T2:
350 case EVEX_T4:
351 mod_idx = (in_size_in_bits == EVEX_64bit) ? 1 : 0;
352 break;
353
354 case EVEX_T8:
355 break;
356
357 case EVEX_HVM:
358 break;
359
360 case EVEX_QVM:
361 break;
362
363 case EVEX_OVM:
364 break;
365
366 case EVEX_M128:
367 break;
368
369 case EVEX_DUP:
370 break;
371
372 default:
373 assert(0, "no valid evex tuple_table entry");
374 break;
375 }
376
377 if (vector_len >= AVX_128bit && vector_len <= AVX_512bit) {
378 int disp_factor = tuple_table[cur_tuple_type + mod_idx][vector_len];
379 if ((disp % disp_factor) == 0) {
380 int new_disp = disp / disp_factor;
381 if ((-0x80 <= new_disp && new_disp < 0x80)) {
382 disp = new_disp;
383 }
384 } else {
385 return false;
386 }
387 }
388 }
389 return (-0x80 <= disp && disp < 0x80);
390 }
391
392
393 bool Assembler::emit_compressed_disp_byte(int &disp) {
394 int mod_idx = 0;
395 // We will test if the displacement fits the compressed format and if so
396 // apply the compression to the displacment iff the result is8bit.
397 if (VM_Version::supports_evex() && _is_evex_instruction) {
398 switch (_tuple_type) {
399 case EVEX_FV:
400 if ((_evex_encoding & VEX_W) == VEX_W) {
401 mod_idx += 2 + ((_evex_encoding & EVEX_Rb) == EVEX_Rb) ? 1 : 0;
402 } else {
403 mod_idx = ((_evex_encoding & EVEX_Rb) == EVEX_Rb) ? 1 : 0;
404 }
405 break;
406
407 case EVEX_HV:
408 mod_idx = ((_evex_encoding & EVEX_Rb) == EVEX_Rb) ? 1 : 0;
409 break;
410
411 case EVEX_FVM:
412 break;
413
414 case EVEX_T1S:
415 switch (_input_size_in_bits) {
416 case EVEX_8bit:
417 break;
418
419 case EVEX_16bit:
420 mod_idx = 1;
421 break;
422
423 case EVEX_32bit:
424 mod_idx = 2;
425 break;
426
427 case EVEX_64bit:
428 mod_idx = 3;
429 break;
430 }
431 break;
432
433 case EVEX_T1F:
434 case EVEX_T2:
435 case EVEX_T4:
436 mod_idx = (_input_size_in_bits == EVEX_64bit) ? 1 : 0;
437 break;
438
439 case EVEX_T8:
440 break;
441
442 case EVEX_HVM:
443 break;
444
445 case EVEX_QVM:
446 break;
447
448 case EVEX_OVM:
449 break;
450
451 case EVEX_M128:
452 break;
453
454 case EVEX_DUP:
455 break;
456
457 default:
458 assert(0, "no valid evex tuple_table entry");
459 break;
460 }
461
462 if (_avx_vector_len >= AVX_128bit && _avx_vector_len <= AVX_512bit) {
463 int disp_factor = tuple_table[_tuple_type + mod_idx][_avx_vector_len];
464 if ((disp % disp_factor) == 0) {
465 int new_disp = disp / disp_factor;
466 if (is8bit(new_disp)) {
467 disp = new_disp;
468 }
469 } else {
470 return false;
471 }
472 }
473 }
474 return is8bit(disp);
475 }
476
477
478 void Assembler::emit_operand(Register reg, Register base, Register index,
479 Address::ScaleFactor scale, int disp,
480 RelocationHolder const& rspec,
481 int rip_relative_correction) {
482 relocInfo::relocType rtype = (relocInfo::relocType) rspec.type();
483
484 // Encode the registers as needed in the fields they are used in
485
486 int regenc = encode(reg) << 3;
487 int indexenc = index->is_valid() ? encode(index) << 3 : 0;
488 int baseenc = base->is_valid() ? encode(base) : 0;
489
490 if (base->is_valid()) {
491 if (index->is_valid()) {
492 assert(scale != Address::no_scale, "inconsistent address");
493 // [base + index*scale + disp]
494 if (disp == 0 && rtype == relocInfo::none &&
495 base != rbp LP64_ONLY(&& base != r13)) {
496 // [base + index*scale]
497 // [00 reg 100][ss index base]
498 assert(index != rsp, "illegal addressing mode");
499 emit_int8(0x04 | regenc);
500 emit_int8(scale << 6 | indexenc | baseenc);
501 } else if (emit_compressed_disp_byte(disp) && rtype == relocInfo::none) {
502 // [base + index*scale + imm8]
503 // [01 reg 100][ss index base] imm8
504 assert(index != rsp, "illegal addressing mode");
505 emit_int8(0x44 | regenc);
506 emit_int8(scale << 6 | indexenc | baseenc);
507 emit_int8(disp & 0xFF);
508 } else {
509 // [base + index*scale + disp32]
510 // [10 reg 100][ss index base] disp32
511 assert(index != rsp, "illegal addressing mode");
512 emit_int8(0x84 | regenc);
513 emit_int8(scale << 6 | indexenc | baseenc);
514 emit_data(disp, rspec, disp32_operand);
515 }
516 } else if (base == rsp LP64_ONLY(|| base == r12)) {
517 // [rsp + disp]
518 if (disp == 0 && rtype == relocInfo::none) {
519 // [rsp]
520 // [00 reg 100][00 100 100]
521 emit_int8(0x04 | regenc);
522 emit_int8(0x24);
523 } else if (emit_compressed_disp_byte(disp) && rtype == relocInfo::none) {
524 // [rsp + imm8]
525 // [01 reg 100][00 100 100] disp8
526 emit_int8(0x44 | regenc);
527 emit_int8(0x24);
528 emit_int8(disp & 0xFF);
529 } else {
530 // [rsp + imm32]
531 // [10 reg 100][00 100 100] disp32
532 emit_int8(0x84 | regenc);
533 emit_int8(0x24);
534 emit_data(disp, rspec, disp32_operand);
535 }
536 } else {
537 // [base + disp]
538 assert(base != rsp LP64_ONLY(&& base != r12), "illegal addressing mode");
539 if (disp == 0 && rtype == relocInfo::none &&
540 base != rbp LP64_ONLY(&& base != r13)) {
541 // [base]
542 // [00 reg base]
543 emit_int8(0x00 | regenc | baseenc);
544 } else if (emit_compressed_disp_byte(disp) && rtype == relocInfo::none) {
545 // [base + disp8]
546 // [01 reg base] disp8
547 emit_int8(0x40 | regenc | baseenc);
548 emit_int8(disp & 0xFF);
549 } else {
550 // [base + disp32]
551 // [10 reg base] disp32
552 emit_int8(0x80 | regenc | baseenc);
553 emit_data(disp, rspec, disp32_operand);
554 }
555 }
556 } else {
557 if (index->is_valid()) {
558 assert(scale != Address::no_scale, "inconsistent address");
559 // [index*scale + disp]
560 // [00 reg 100][ss index 101] disp32
561 assert(index != rsp, "illegal addressing mode");
562 emit_int8(0x04 | regenc);
563 emit_int8(scale << 6 | indexenc | 0x05);
564 emit_data(disp, rspec, disp32_operand);
565 } else if (rtype != relocInfo::none ) {
566 // [disp] (64bit) RIP-RELATIVE (32bit) abs
567 // [00 000 101] disp32
568
569 emit_int8(0x05 | regenc);
570 // Note that the RIP-rel. correction applies to the generated
571 // disp field, but _not_ to the target address in the rspec.
572
573 // disp was created by converting the target address minus the pc
574 // at the start of the instruction. That needs more correction here.
575 // intptr_t disp = target - next_ip;
576 assert(inst_mark() != NULL, "must be inside InstructionMark");
577 address next_ip = pc() + sizeof(int32_t) + rip_relative_correction;
578 int64_t adjusted = disp;
579 // Do rip-rel adjustment for 64bit
580 LP64_ONLY(adjusted -= (next_ip - inst_mark()));
581 assert(is_simm32(adjusted),
582 "must be 32bit offset (RIP relative address)");
583 emit_data((int32_t) adjusted, rspec, disp32_operand);
584
585 } else {
586 // 32bit never did this, did everything as the rip-rel/disp code above
587 // [disp] ABSOLUTE
588 // [00 reg 100][00 100 101] disp32
589 emit_int8(0x04 | regenc);
590 emit_int8(0x25);
591 emit_data(disp, rspec, disp32_operand);
592 }
593 }
594 _is_evex_instruction = false;
595 }
596
597 void Assembler::emit_operand(XMMRegister reg, Register base, Register index,
598 Address::ScaleFactor scale, int disp,
599 RelocationHolder const& rspec) {
600 if (UseAVX > 2) {
601 int xreg_enc = reg->encoding();
602 if (xreg_enc > 15) {
603 XMMRegister new_reg = as_XMMRegister(xreg_enc & 0xf);
604 emit_operand((Register)new_reg, base, index, scale, disp, rspec);
605 return;
606 }
607 }
608 emit_operand((Register)reg, base, index, scale, disp, rspec);
609 }
610
611 // Secret local extension to Assembler::WhichOperand:
612 #define end_pc_operand (_WhichOperand_limit)
613
614 address Assembler::locate_operand(address inst, WhichOperand which) {
615 // Decode the given instruction, and return the address of
616 // an embedded 32-bit operand word.
617
618 // If "which" is disp32_operand, selects the displacement portion
619 // of an effective address specifier.
620 // If "which" is imm64_operand, selects the trailing immediate constant.
621 // If "which" is call32_operand, selects the displacement of a call or jump.
622 // Caller is responsible for ensuring that there is such an operand,
623 // and that it is 32/64 bits wide.
624
625 // If "which" is end_pc_operand, find the end of the instruction.
626
627 address ip = inst;
628 bool is_64bit = false;
629
630 debug_only(bool has_disp32 = false);
631 int tail_size = 0; // other random bytes (#32, #16, etc.) at end of insn
632
633 again_after_prefix:
634 switch (0xFF & *ip++) {
635
636 // These convenience macros generate groups of "case" labels for the switch.
637 #define REP4(x) (x)+0: case (x)+1: case (x)+2: case (x)+3
638 #define REP8(x) (x)+0: case (x)+1: case (x)+2: case (x)+3: \
639 case (x)+4: case (x)+5: case (x)+6: case (x)+7
640 #define REP16(x) REP8((x)+0): \
641 case REP8((x)+8)
642
643 case CS_segment:
644 case SS_segment:
645 case DS_segment:
646 case ES_segment:
647 case FS_segment:
648 case GS_segment:
649 // Seems dubious
650 LP64_ONLY(assert(false, "shouldn't have that prefix"));
651 assert(ip == inst+1, "only one prefix allowed");
652 goto again_after_prefix;
653
654 case 0x67:
655 case REX:
656 case REX_B:
657 case REX_X:
658 case REX_XB:
659 case REX_R:
660 case REX_RB:
661 case REX_RX:
662 case REX_RXB:
663 NOT_LP64(assert(false, "64bit prefixes"));
664 goto again_after_prefix;
665
666 case REX_W:
667 case REX_WB:
668 case REX_WX:
669 case REX_WXB:
670 case REX_WR:
671 case REX_WRB:
672 case REX_WRX:
673 case REX_WRXB:
674 NOT_LP64(assert(false, "64bit prefixes"));
675 is_64bit = true;
676 goto again_after_prefix;
677
678 case 0xFF: // pushq a; decl a; incl a; call a; jmp a
679 case 0x88: // movb a, r
680 case 0x89: // movl a, r
681 case 0x8A: // movb r, a
682 case 0x8B: // movl r, a
683 case 0x8F: // popl a
684 debug_only(has_disp32 = true);
685 break;
686
687 case 0x68: // pushq #32
688 if (which == end_pc_operand) {
689 return ip + 4;
690 }
691 assert(which == imm_operand && !is_64bit, "pushl has no disp32 or 64bit immediate");
692 return ip; // not produced by emit_operand
693
694 case 0x66: // movw ... (size prefix)
695 again_after_size_prefix2:
696 switch (0xFF & *ip++) {
697 case REX:
698 case REX_B:
699 case REX_X:
700 case REX_XB:
701 case REX_R:
702 case REX_RB:
703 case REX_RX:
704 case REX_RXB:
705 case REX_W:
706 case REX_WB:
707 case REX_WX:
708 case REX_WXB:
709 case REX_WR:
710 case REX_WRB:
711 case REX_WRX:
712 case REX_WRXB:
713 NOT_LP64(assert(false, "64bit prefix found"));
714 goto again_after_size_prefix2;
715 case 0x8B: // movw r, a
716 case 0x89: // movw a, r
717 debug_only(has_disp32 = true);
718 break;
719 case 0xC7: // movw a, #16
720 debug_only(has_disp32 = true);
721 tail_size = 2; // the imm16
722 break;
723 case 0x0F: // several SSE/SSE2 variants
724 ip--; // reparse the 0x0F
725 goto again_after_prefix;
726 default:
727 ShouldNotReachHere();
728 }
729 break;
730
731 case REP8(0xB8): // movl/q r, #32/#64(oop?)
732 if (which == end_pc_operand) return ip + (is_64bit ? 8 : 4);
733 // these asserts are somewhat nonsensical
734 #ifndef _LP64
735 assert(which == imm_operand || which == disp32_operand,
736 err_msg("which %d is_64_bit %d ip " INTPTR_FORMAT, which, is_64bit, p2i(ip)));
737 #else
738 assert((which == call32_operand || which == imm_operand) && is_64bit ||
739 which == narrow_oop_operand && !is_64bit,
740 err_msg("which %d is_64_bit %d ip " INTPTR_FORMAT, which, is_64bit, p2i(ip)));
741 #endif // _LP64
742 return ip;
743
744 case 0x69: // imul r, a, #32
745 case 0xC7: // movl a, #32(oop?)
746 tail_size = 4;
747 debug_only(has_disp32 = true); // has both kinds of operands!
748 break;
749
750 case 0x0F: // movx..., etc.
751 switch (0xFF & *ip++) {
752 case 0x3A: // pcmpestri
753 tail_size = 1;
754 case 0x38: // ptest, pmovzxbw
755 ip++; // skip opcode
756 debug_only(has_disp32 = true); // has both kinds of operands!
757 break;
758
759 case 0x70: // pshufd r, r/a, #8
760 debug_only(has_disp32 = true); // has both kinds of operands!
761 case 0x73: // psrldq r, #8
762 tail_size = 1;
763 break;
764
765 case 0x12: // movlps
766 case 0x28: // movaps
767 case 0x2E: // ucomiss
768 case 0x2F: // comiss
769 case 0x54: // andps
770 case 0x55: // andnps
771 case 0x56: // orps
772 case 0x57: // xorps
773 case 0x6E: // movd
774 case 0x7E: // movd
775 case 0xAE: // ldmxcsr, stmxcsr, fxrstor, fxsave, clflush
776 debug_only(has_disp32 = true);
777 break;
778
779 case 0xAD: // shrd r, a, %cl
780 case 0xAF: // imul r, a
781 case 0xBE: // movsbl r, a (movsxb)
782 case 0xBF: // movswl r, a (movsxw)
783 case 0xB6: // movzbl r, a (movzxb)
784 case 0xB7: // movzwl r, a (movzxw)
785 case REP16(0x40): // cmovl cc, r, a
786 case 0xB0: // cmpxchgb
787 case 0xB1: // cmpxchg
788 case 0xC1: // xaddl
789 case 0xC7: // cmpxchg8
790 case REP16(0x90): // setcc a
791 debug_only(has_disp32 = true);
792 // fall out of the switch to decode the address
793 break;
794
795 case 0xC4: // pinsrw r, a, #8
796 debug_only(has_disp32 = true);
797 case 0xC5: // pextrw r, r, #8
798 tail_size = 1; // the imm8
799 break;
800
801 case 0xAC: // shrd r, a, #8
802 debug_only(has_disp32 = true);
803 tail_size = 1; // the imm8
804 break;
805
806 case REP16(0x80): // jcc rdisp32
807 if (which == end_pc_operand) return ip + 4;
808 assert(which == call32_operand, "jcc has no disp32 or imm");
809 return ip;
810 default:
811 ShouldNotReachHere();
812 }
813 break;
814
815 case 0x81: // addl a, #32; addl r, #32
816 // also: orl, adcl, sbbl, andl, subl, xorl, cmpl
817 // on 32bit in the case of cmpl, the imm might be an oop
818 tail_size = 4;
819 debug_only(has_disp32 = true); // has both kinds of operands!
820 break;
821
822 case 0x83: // addl a, #8; addl r, #8
823 // also: orl, adcl, sbbl, andl, subl, xorl, cmpl
824 debug_only(has_disp32 = true); // has both kinds of operands!
825 tail_size = 1;
826 break;
827
828 case 0x9B:
829 switch (0xFF & *ip++) {
830 case 0xD9: // fnstcw a
831 debug_only(has_disp32 = true);
832 break;
833 default:
834 ShouldNotReachHere();
835 }
836 break;
837
838 case REP4(0x00): // addb a, r; addl a, r; addb r, a; addl r, a
839 case REP4(0x10): // adc...
840 case REP4(0x20): // and...
841 case REP4(0x30): // xor...
842 case REP4(0x08): // or...
843 case REP4(0x18): // sbb...
844 case REP4(0x28): // sub...
845 case 0xF7: // mull a
846 case 0x8D: // lea r, a
847 case 0x87: // xchg r, a
848 case REP4(0x38): // cmp...
849 case 0x85: // test r, a
850 debug_only(has_disp32 = true); // has both kinds of operands!
851 break;
852
853 case 0xC1: // sal a, #8; sar a, #8; shl a, #8; shr a, #8
854 case 0xC6: // movb a, #8
855 case 0x80: // cmpb a, #8
856 case 0x6B: // imul r, a, #8
857 debug_only(has_disp32 = true); // has both kinds of operands!
858 tail_size = 1; // the imm8
859 break;
860
861 case 0xC4: // VEX_3bytes
862 case 0xC5: // VEX_2bytes
863 assert((UseAVX > 0), "shouldn't have VEX prefix");
864 assert(ip == inst+1, "no prefixes allowed");
865 // C4 and C5 are also used as opcodes for PINSRW and PEXTRW instructions
866 // but they have prefix 0x0F and processed when 0x0F processed above.
867 //
868 // In 32-bit mode the VEX first byte C4 and C5 alias onto LDS and LES
869 // instructions (these instructions are not supported in 64-bit mode).
870 // To distinguish them bits [7:6] are set in the VEX second byte since
871 // ModRM byte can not be of the form 11xxxxxx in 32-bit mode. To set
872 // those VEX bits REX and vvvv bits are inverted.
873 //
874 // Fortunately C2 doesn't generate these instructions so we don't need
875 // to check for them in product version.
876
877 // Check second byte
878 NOT_LP64(assert((0xC0 & *ip) == 0xC0, "shouldn't have LDS and LES instructions"));
879
880 int vex_opcode;
881 // First byte
882 if ((0xFF & *inst) == VEX_3bytes) {
883 vex_opcode = VEX_OPCODE_MASK & *ip;
884 ip++; // third byte
885 is_64bit = ((VEX_W & *ip) == VEX_W);
886 } else {
887 vex_opcode = VEX_OPCODE_0F;
888 }
889 ip++; // opcode
890 // To find the end of instruction (which == end_pc_operand).
891 switch (vex_opcode) {
892 case VEX_OPCODE_0F:
893 switch (0xFF & *ip) {
894 case 0x70: // pshufd r, r/a, #8
895 case 0x71: // ps[rl|ra|ll]w r, #8
896 case 0x72: // ps[rl|ra|ll]d r, #8
897 case 0x73: // ps[rl|ra|ll]q r, #8
898 case 0xC2: // cmp[ps|pd|ss|sd] r, r, r/a, #8
899 case 0xC4: // pinsrw r, r, r/a, #8
900 case 0xC5: // pextrw r/a, r, #8
901 case 0xC6: // shufp[s|d] r, r, r/a, #8
902 tail_size = 1; // the imm8
903 break;
904 }
905 break;
906 case VEX_OPCODE_0F_3A:
907 tail_size = 1;
908 break;
909 }
910 ip++; // skip opcode
911 debug_only(has_disp32 = true); // has both kinds of operands!
912 break;
913
914 case 0x62: // EVEX_4bytes
915 assert((UseAVX > 0), "shouldn't have EVEX prefix");
916 assert(ip == inst+1, "no prefixes allowed");
917 // no EVEX collisions, all instructions that have 0x62 opcodes
918 // have EVEX versions and are subopcodes of 0x66
919 ip++; // skip P0 and exmaine W in P1
920 is_64bit = ((VEX_W & *ip) == VEX_W);
921 ip++; // move to P2
922 ip++; // skip P2, move to opcode
923 // To find the end of instruction (which == end_pc_operand).
924 switch (0xFF & *ip) {
925 case 0x61: // pcmpestri r, r/a, #8
926 case 0x70: // pshufd r, r/a, #8
927 case 0x73: // psrldq r, #8
928 tail_size = 1; // the imm8
929 break;
930 default:
931 break;
932 }
933 ip++; // skip opcode
934 debug_only(has_disp32 = true); // has both kinds of operands!
935 break;
936
937 case 0xD1: // sal a, 1; sar a, 1; shl a, 1; shr a, 1
938 case 0xD3: // sal a, %cl; sar a, %cl; shl a, %cl; shr a, %cl
939 case 0xD9: // fld_s a; fst_s a; fstp_s a; fldcw a
940 case 0xDD: // fld_d a; fst_d a; fstp_d a
941 case 0xDB: // fild_s a; fistp_s a; fld_x a; fstp_x a
942 case 0xDF: // fild_d a; fistp_d a
943 case 0xD8: // fadd_s a; fsubr_s a; fmul_s a; fdivr_s a; fcomp_s a
944 case 0xDC: // fadd_d a; fsubr_d a; fmul_d a; fdivr_d a; fcomp_d a
945 case 0xDE: // faddp_d a; fsubrp_d a; fmulp_d a; fdivrp_d a; fcompp_d a
946 debug_only(has_disp32 = true);
947 break;
948
949 case 0xE8: // call rdisp32
950 case 0xE9: // jmp rdisp32
951 if (which == end_pc_operand) return ip + 4;
952 assert(which == call32_operand, "call has no disp32 or imm");
953 return ip;
954
955 case 0xF0: // Lock
956 assert(os::is_MP(), "only on MP");
957 goto again_after_prefix;
958
959 case 0xF3: // For SSE
960 case 0xF2: // For SSE2
961 switch (0xFF & *ip++) {
962 case REX:
963 case REX_B:
964 case REX_X:
965 case REX_XB:
966 case REX_R:
967 case REX_RB:
968 case REX_RX:
969 case REX_RXB:
970 case REX_W:
971 case REX_WB:
972 case REX_WX:
973 case REX_WXB:
974 case REX_WR:
975 case REX_WRB:
976 case REX_WRX:
977 case REX_WRXB:
978 NOT_LP64(assert(false, "found 64bit prefix"));
979 ip++;
980 default:
981 ip++;
982 }
983 debug_only(has_disp32 = true); // has both kinds of operands!
984 break;
985
986 default:
987 ShouldNotReachHere();
988
989 #undef REP8
990 #undef REP16
991 }
992
993 assert(which != call32_operand, "instruction is not a call, jmp, or jcc");
994 #ifdef _LP64
995 assert(which != imm_operand, "instruction is not a movq reg, imm64");
996 #else
997 // assert(which != imm_operand || has_imm32, "instruction has no imm32 field");
998 assert(which != imm_operand || has_disp32, "instruction has no imm32 field");
999 #endif // LP64
1000 assert(which != disp32_operand || has_disp32, "instruction has no disp32 field");
1001
1002 // parse the output of emit_operand
1003 int op2 = 0xFF & *ip++;
1004 int base = op2 & 0x07;
1005 int op3 = -1;
1006 const int b100 = 4;
1007 const int b101 = 5;
1008 if (base == b100 && (op2 >> 6) != 3) {
1009 op3 = 0xFF & *ip++;
1010 base = op3 & 0x07; // refetch the base
1011 }
1012 // now ip points at the disp (if any)
1013
1014 switch (op2 >> 6) {
1015 case 0:
1016 // [00 reg 100][ss index base]
1017 // [00 reg 100][00 100 esp]
1018 // [00 reg base]
1019 // [00 reg 100][ss index 101][disp32]
1020 // [00 reg 101] [disp32]
1021
1022 if (base == b101) {
1023 if (which == disp32_operand)
1024 return ip; // caller wants the disp32
1025 ip += 4; // skip the disp32
1026 }
1027 break;
1028
1029 case 1:
1030 // [01 reg 100][ss index base][disp8]
1031 // [01 reg 100][00 100 esp][disp8]
1032 // [01 reg base] [disp8]
1033 ip += 1; // skip the disp8
1034 break;
1035
1036 case 2:
1037 // [10 reg 100][ss index base][disp32]
1038 // [10 reg 100][00 100 esp][disp32]
1039 // [10 reg base] [disp32]
1040 if (which == disp32_operand)
1041 return ip; // caller wants the disp32
1042 ip += 4; // skip the disp32
1043 break;
1044
1045 case 3:
1046 // [11 reg base] (not a memory addressing mode)
1047 break;
1048 }
1049
1050 if (which == end_pc_operand) {
1051 return ip + tail_size;
1052 }
1053
1054 #ifdef _LP64
1055 assert(which == narrow_oop_operand && !is_64bit, "instruction is not a movl adr, imm32");
1056 #else
1057 assert(which == imm_operand, "instruction has only an imm field");
1058 #endif // LP64
1059 return ip;
1060 }
1061
1062 address Assembler::locate_next_instruction(address inst) {
1063 // Secretly share code with locate_operand:
1064 return locate_operand(inst, end_pc_operand);
1065 }
1066
1067
1068 #ifdef ASSERT
1069 void Assembler::check_relocation(RelocationHolder const& rspec, int format) {
1070 address inst = inst_mark();
1071 assert(inst != NULL && inst < pc(), "must point to beginning of instruction");
1072 address opnd;
1073
1074 Relocation* r = rspec.reloc();
1075 if (r->type() == relocInfo::none) {
1076 return;
1077 } else if (r->is_call() || format == call32_operand) {
1078 // assert(format == imm32_operand, "cannot specify a nonzero format");
1079 opnd = locate_operand(inst, call32_operand);
1080 } else if (r->is_data()) {
1081 assert(format == imm_operand || format == disp32_operand
1082 LP64_ONLY(|| format == narrow_oop_operand), "format ok");
1083 opnd = locate_operand(inst, (WhichOperand)format);
1084 } else {
1085 assert(format == imm_operand, "cannot specify a format");
1086 return;
1087 }
1088 assert(opnd == pc(), "must put operand where relocs can find it");
1089 }
1090 #endif // ASSERT
1091
1092 void Assembler::emit_operand32(Register reg, Address adr) {
1093 assert(reg->encoding() < 8, "no extended registers");
1094 assert(!adr.base_needs_rex() && !adr.index_needs_rex(), "no extended registers");
1095 emit_operand(reg, adr._base, adr._index, adr._scale, adr._disp,
1096 adr._rspec);
1097 }
1098
1099 void Assembler::emit_operand(Register reg, Address adr,
1100 int rip_relative_correction) {
1101 emit_operand(reg, adr._base, adr._index, adr._scale, adr._disp,
1102 adr._rspec,
1103 rip_relative_correction);
1104 }
1105
1106 void Assembler::emit_operand(XMMRegister reg, Address adr) {
1107 emit_operand(reg, adr._base, adr._index, adr._scale, adr._disp,
1108 adr._rspec);
1109 }
1110
1111 // MMX operations
1112 void Assembler::emit_operand(MMXRegister reg, Address adr) {
1113 assert(!adr.base_needs_rex() && !adr.index_needs_rex(), "no extended registers");
1114 emit_operand((Register)reg, adr._base, adr._index, adr._scale, adr._disp, adr._rspec);
1115 }
1116
1117 // work around gcc (3.2.1-7a) bug
1118 void Assembler::emit_operand(Address adr, MMXRegister reg) {
1119 assert(!adr.base_needs_rex() && !adr.index_needs_rex(), "no extended registers");
1120 emit_operand((Register)reg, adr._base, adr._index, adr._scale, adr._disp, adr._rspec);
1121 }
1122
1123
1124 void Assembler::emit_farith(int b1, int b2, int i) {
1125 assert(isByte(b1) && isByte(b2), "wrong opcode");
1126 assert(0 <= i && i < 8, "illegal stack offset");
1127 emit_int8(b1);
1128 emit_int8(b2 + i);
1129 }
1130
1131
1132 // Now the Assembler instructions (identical for 32/64 bits)
1133
1134 void Assembler::adcl(Address dst, int32_t imm32) {
1135 InstructionMark im(this);
1136 prefix(dst);
1137 emit_arith_operand(0x81, rdx, dst, imm32);
1138 }
1139
1140 void Assembler::adcl(Address dst, Register src) {
1141 InstructionMark im(this);
1142 prefix(dst, src);
1143 emit_int8(0x11);
1144 emit_operand(src, dst);
1145 }
1146
1147 void Assembler::adcl(Register dst, int32_t imm32) {
1148 prefix(dst);
1149 emit_arith(0x81, 0xD0, dst, imm32);
1150 }
1151
1152 void Assembler::adcl(Register dst, Address src) {
1153 InstructionMark im(this);
1154 prefix(src, dst);
1155 emit_int8(0x13);
1156 emit_operand(dst, src);
1157 }
1158
1159 void Assembler::adcl(Register dst, Register src) {
1160 (void) prefix_and_encode(dst->encoding(), src->encoding());
1161 emit_arith(0x13, 0xC0, dst, src);
1162 }
1163
1164 void Assembler::addl(Address dst, int32_t imm32) {
1165 InstructionMark im(this);
1166 prefix(dst);
1167 emit_arith_operand(0x81, rax, dst, imm32);
1168 }
1169
1170 void Assembler::addl(Address dst, Register src) {
1171 InstructionMark im(this);
1172 prefix(dst, src);
1173 emit_int8(0x01);
1174 emit_operand(src, dst);
1175 }
1176
1177 void Assembler::addl(Register dst, int32_t imm32) {
1178 prefix(dst);
1179 emit_arith(0x81, 0xC0, dst, imm32);
1180 }
1181
1182 void Assembler::addl(Register dst, Address src) {
1183 InstructionMark im(this);
1184 prefix(src, dst);
1185 emit_int8(0x03);
1186 emit_operand(dst, src);
1187 }
1188
1189 void Assembler::addl(Register dst, Register src) {
1190 (void) prefix_and_encode(dst->encoding(), src->encoding());
1191 emit_arith(0x03, 0xC0, dst, src);
1192 }
1193
1194 void Assembler::addr_nop_4() {
1195 assert(UseAddressNop, "no CPU support");
1196 // 4 bytes: NOP DWORD PTR [EAX+0]
1197 emit_int8(0x0F);
1198 emit_int8(0x1F);
1199 emit_int8(0x40); // emit_rm(cbuf, 0x1, EAX_enc, EAX_enc);
1200 emit_int8(0); // 8-bits offset (1 byte)
1201 }
1202
1203 void Assembler::addr_nop_5() {
1204 assert(UseAddressNop, "no CPU support");
1205 // 5 bytes: NOP DWORD PTR [EAX+EAX*0+0] 8-bits offset
1206 emit_int8(0x0F);
1207 emit_int8(0x1F);
1208 emit_int8(0x44); // emit_rm(cbuf, 0x1, EAX_enc, 0x4);
1209 emit_int8(0x00); // emit_rm(cbuf, 0x0, EAX_enc, EAX_enc);
1210 emit_int8(0); // 8-bits offset (1 byte)
1211 }
1212
1213 void Assembler::addr_nop_7() {
1214 assert(UseAddressNop, "no CPU support");
1215 // 7 bytes: NOP DWORD PTR [EAX+0] 32-bits offset
1216 emit_int8(0x0F);
1217 emit_int8(0x1F);
1218 emit_int8((unsigned char)0x80);
1219 // emit_rm(cbuf, 0x2, EAX_enc, EAX_enc);
1220 emit_int32(0); // 32-bits offset (4 bytes)
1221 }
1222
1223 void Assembler::addr_nop_8() {
1224 assert(UseAddressNop, "no CPU support");
1225 // 8 bytes: NOP DWORD PTR [EAX+EAX*0+0] 32-bits offset
1226 emit_int8(0x0F);
1227 emit_int8(0x1F);
1228 emit_int8((unsigned char)0x84);
1229 // emit_rm(cbuf, 0x2, EAX_enc, 0x4);
1230 emit_int8(0x00); // emit_rm(cbuf, 0x0, EAX_enc, EAX_enc);
1231 emit_int32(0); // 32-bits offset (4 bytes)
1232 }
1233
1234 void Assembler::addsd(XMMRegister dst, XMMRegister src) {
1235 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1236 if (VM_Version::supports_evex()) {
1237 emit_simd_arith_q(0x58, dst, src, VEX_SIMD_F2);
1238 } else {
1239 emit_simd_arith(0x58, dst, src, VEX_SIMD_F2);
1240 }
1241 }
1242
1243 void Assembler::addsd(XMMRegister dst, Address src) {
1244 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1245 if (VM_Version::supports_evex()) {
1246 _tuple_type = EVEX_T1S;
1247 _input_size_in_bits = EVEX_64bit;
1248 emit_simd_arith_q(0x58, dst, src, VEX_SIMD_F2);
1249 } else {
1250 emit_simd_arith(0x58, dst, src, VEX_SIMD_F2);
1251 }
1252 }
1253
1254 void Assembler::addss(XMMRegister dst, XMMRegister src) {
1255 NOT_LP64(assert(VM_Version::supports_sse(), ""));
1256 emit_simd_arith(0x58, dst, src, VEX_SIMD_F3);
1257 }
1258
1259 void Assembler::addss(XMMRegister dst, Address src) {
1260 NOT_LP64(assert(VM_Version::supports_sse(), ""));
1261 if (VM_Version::supports_evex()) {
1262 _tuple_type = EVEX_T1S;
1263 _input_size_in_bits = EVEX_32bit;
1264 }
1265 emit_simd_arith(0x58, dst, src, VEX_SIMD_F3);
1266 }
1267
1268 void Assembler::aesdec(XMMRegister dst, Address src) {
1269 assert(VM_Version::supports_aes(), "");
1270 InstructionMark im(this);
1271 simd_prefix(dst, dst, src, VEX_SIMD_66, /* no_mask_reg */ false,
1272 VEX_OPCODE_0F_38, /* rex_w */ false, AVX_128bit, /* legacy_mode */ true);
1273 emit_int8((unsigned char)0xDE);
1274 emit_operand(dst, src);
1275 }
1276
1277 void Assembler::aesdec(XMMRegister dst, XMMRegister src) {
1278 assert(VM_Version::supports_aes(), "");
1279 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, /* no_mask_reg */ false,
1280 VEX_OPCODE_0F_38, /* rex_w */ false, AVX_128bit, /* legacy_mode */ true);
1281 emit_int8((unsigned char)0xDE);
1282 emit_int8(0xC0 | encode);
1283 }
1284
1285 void Assembler::aesdeclast(XMMRegister dst, Address src) {
1286 assert(VM_Version::supports_aes(), "");
1287 InstructionMark im(this);
1288 simd_prefix(dst, dst, src, VEX_SIMD_66, /* no_mask_reg */ false,
1289 VEX_OPCODE_0F_38, /* rex_w */ false, AVX_128bit, /* legacy_mode */ true);
1290 emit_int8((unsigned char)0xDF);
1291 emit_operand(dst, src);
1292 }
1293
1294 void Assembler::aesdeclast(XMMRegister dst, XMMRegister src) {
1295 assert(VM_Version::supports_aes(), "");
1296 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, /* no_mask_reg */ false,
1297 VEX_OPCODE_0F_38, /* rex_w */ false, AVX_128bit, /* legacy_mode */ true);
1298 emit_int8((unsigned char)0xDF);
1299 emit_int8((unsigned char)(0xC0 | encode));
1300 }
1301
1302 void Assembler::aesenc(XMMRegister dst, Address src) {
1303 assert(VM_Version::supports_aes(), "");
1304 InstructionMark im(this);
1305 simd_prefix(dst, dst, src, VEX_SIMD_66, /* no_mask_reg */ false,
1306 VEX_OPCODE_0F_38, /* rex_w */ false, AVX_128bit, /* legacy_mode */ true);
1307 emit_int8((unsigned char)0xDC);
1308 emit_operand(dst, src);
1309 }
1310
1311 void Assembler::aesenc(XMMRegister dst, XMMRegister src) {
1312 assert(VM_Version::supports_aes(), "");
1313 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, /* no_mask_reg */ false,
1314 VEX_OPCODE_0F_38, /* rex_w */ false, AVX_128bit, /* legacy_mode */ true);
1315 emit_int8((unsigned char)0xDC);
1316 emit_int8(0xC0 | encode);
1317 }
1318
1319 void Assembler::aesenclast(XMMRegister dst, Address src) {
1320 assert(VM_Version::supports_aes(), "");
1321 InstructionMark im(this);
1322 simd_prefix(dst, dst, src, VEX_SIMD_66, /* no_mask_reg */ false,
1323 VEX_OPCODE_0F_38, /* rex_w */ false, AVX_128bit, /* legacy_mode */ true);
1324 emit_int8((unsigned char)0xDD);
1325 emit_operand(dst, src);
1326 }
1327
1328 void Assembler::aesenclast(XMMRegister dst, XMMRegister src) {
1329 assert(VM_Version::supports_aes(), "");
1330 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, /* no_mask_reg */ false,
1331 VEX_OPCODE_0F_38, /* rex_w */ false, AVX_128bit, /* legacy_mode */ true);
1332 emit_int8((unsigned char)0xDD);
1333 emit_int8((unsigned char)(0xC0 | encode));
1334 }
1335
1336 void Assembler::andl(Address dst, int32_t imm32) {
1337 InstructionMark im(this);
1338 prefix(dst);
1339 emit_int8((unsigned char)0x81);
1340 emit_operand(rsp, dst, 4);
1341 emit_int32(imm32);
1342 }
1343
1344 void Assembler::andl(Register dst, int32_t imm32) {
1345 prefix(dst);
1346 emit_arith(0x81, 0xE0, dst, imm32);
1347 }
1348
1349 void Assembler::andl(Register dst, Address src) {
1350 InstructionMark im(this);
1351 prefix(src, dst);
1352 emit_int8(0x23);
1353 emit_operand(dst, src);
1354 }
1355
1356 void Assembler::andl(Register dst, Register src) {
1357 (void) prefix_and_encode(dst->encoding(), src->encoding());
1358 emit_arith(0x23, 0xC0, dst, src);
1359 }
1360
1361 void Assembler::andnl(Register dst, Register src1, Register src2) {
1362 assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
1363 int encode = vex_prefix_0F38_and_encode_legacy(dst, src1, src2);
1364 emit_int8((unsigned char)0xF2);
1365 emit_int8((unsigned char)(0xC0 | encode));
1366 }
1367
1368 void Assembler::andnl(Register dst, Register src1, Address src2) {
1369 InstructionMark im(this);
1370 assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
1371 vex_prefix_0F38_legacy(dst, src1, src2);
1372 emit_int8((unsigned char)0xF2);
1373 emit_operand(dst, src2);
1374 }
1375
1376 void Assembler::bsfl(Register dst, Register src) {
1377 int encode = prefix_and_encode(dst->encoding(), src->encoding());
1378 emit_int8(0x0F);
1379 emit_int8((unsigned char)0xBC);
1380 emit_int8((unsigned char)(0xC0 | encode));
1381 }
1382
1383 void Assembler::bsrl(Register dst, Register src) {
1384 int encode = prefix_and_encode(dst->encoding(), src->encoding());
1385 emit_int8(0x0F);
1386 emit_int8((unsigned char)0xBD);
1387 emit_int8((unsigned char)(0xC0 | encode));
1388 }
1389
1390 void Assembler::bswapl(Register reg) { // bswap
1391 int encode = prefix_and_encode(reg->encoding());
1392 emit_int8(0x0F);
1393 emit_int8((unsigned char)(0xC8 | encode));
1394 }
1395
1396 void Assembler::blsil(Register dst, Register src) {
1397 assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
1398 int encode = vex_prefix_0F38_and_encode_legacy(rbx, dst, src);
1399 emit_int8((unsigned char)0xF3);
1400 emit_int8((unsigned char)(0xC0 | encode));
1401 }
1402
1403 void Assembler::blsil(Register dst, Address src) {
1404 InstructionMark im(this);
1405 assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
1406 vex_prefix_0F38_legacy(rbx, dst, src);
1407 emit_int8((unsigned char)0xF3);
1408 emit_operand(rbx, src);
1409 }
1410
1411 void Assembler::blsmskl(Register dst, Register src) {
1412 assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
1413 int encode = vex_prefix_0F38_and_encode_legacy(rdx, dst, src);
1414 emit_int8((unsigned char)0xF3);
1415 emit_int8((unsigned char)(0xC0 | encode));
1416 }
1417
1418 void Assembler::blsmskl(Register dst, Address src) {
1419 InstructionMark im(this);
1420 assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
1421 vex_prefix_0F38_legacy(rdx, dst, src);
1422 emit_int8((unsigned char)0xF3);
1423 emit_operand(rdx, src);
1424 }
1425
1426 void Assembler::blsrl(Register dst, Register src) {
1427 assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
1428 int encode = vex_prefix_0F38_and_encode_legacy(rcx, dst, src);
1429 emit_int8((unsigned char)0xF3);
1430 emit_int8((unsigned char)(0xC0 | encode));
1431 }
1432
1433 void Assembler::blsrl(Register dst, Address src) {
1434 InstructionMark im(this);
1435 assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
1436 vex_prefix_0F38_legacy(rcx, dst, src);
1437 emit_int8((unsigned char)0xF3);
1438 emit_operand(rcx, src);
1439 }
1440
1441 void Assembler::call(Label& L, relocInfo::relocType rtype) {
1442 // suspect disp32 is always good
1443 int operand = LP64_ONLY(disp32_operand) NOT_LP64(imm_operand);
1444
1445 if (L.is_bound()) {
1446 const int long_size = 5;
1447 int offs = (int)( target(L) - pc() );
1448 assert(offs <= 0, "assembler error");
1449 InstructionMark im(this);
1450 // 1110 1000 #32-bit disp
1451 emit_int8((unsigned char)0xE8);
1452 emit_data(offs - long_size, rtype, operand);
1453 } else {
1454 InstructionMark im(this);
1455 // 1110 1000 #32-bit disp
1456 L.add_patch_at(code(), locator());
1457
1458 emit_int8((unsigned char)0xE8);
1459 emit_data(int(0), rtype, operand);
1460 }
1461 }
1462
1463 void Assembler::call(Register dst) {
1464 int encode = prefix_and_encode(dst->encoding());
1465 emit_int8((unsigned char)0xFF);
1466 emit_int8((unsigned char)(0xD0 | encode));
1467 }
1468
1469
1470 void Assembler::call(Address adr) {
1471 InstructionMark im(this);
1472 prefix(adr);
1473 emit_int8((unsigned char)0xFF);
1474 emit_operand(rdx, adr);
1475 }
1476
1477 void Assembler::call_literal(address entry, RelocationHolder const& rspec) {
1478 assert(entry != NULL, "call most probably wrong");
1479 InstructionMark im(this);
1480 emit_int8((unsigned char)0xE8);
1481 intptr_t disp = entry - (pc() + sizeof(int32_t));
1482 assert(is_simm32(disp), "must be 32bit offset (call2)");
1483 // Technically, should use call32_operand, but this format is
1484 // implied by the fact that we're emitting a call instruction.
1485
1486 int operand = LP64_ONLY(disp32_operand) NOT_LP64(call32_operand);
1487 emit_data((int) disp, rspec, operand);
1488 }
1489
1490 void Assembler::cdql() {
1491 emit_int8((unsigned char)0x99);
1492 }
1493
1494 void Assembler::cld() {
1495 emit_int8((unsigned char)0xFC);
1496 }
1497
1498 void Assembler::cmovl(Condition cc, Register dst, Register src) {
1499 NOT_LP64(guarantee(VM_Version::supports_cmov(), "illegal instruction"));
1500 int encode = prefix_and_encode(dst->encoding(), src->encoding());
1501 emit_int8(0x0F);
1502 emit_int8(0x40 | cc);
1503 emit_int8((unsigned char)(0xC0 | encode));
1504 }
1505
1506
1507 void Assembler::cmovl(Condition cc, Register dst, Address src) {
1508 NOT_LP64(guarantee(VM_Version::supports_cmov(), "illegal instruction"));
1509 prefix(src, dst);
1510 emit_int8(0x0F);
1511 emit_int8(0x40 | cc);
1512 emit_operand(dst, src);
1513 }
1514
1515 void Assembler::cmpb(Address dst, int imm8) {
1516 InstructionMark im(this);
1517 prefix(dst);
1518 emit_int8((unsigned char)0x80);
1519 emit_operand(rdi, dst, 1);
1520 emit_int8(imm8);
1521 }
1522
1523 void Assembler::cmpl(Address dst, int32_t imm32) {
1524 InstructionMark im(this);
1525 prefix(dst);
1526 emit_int8((unsigned char)0x81);
1527 emit_operand(rdi, dst, 4);
1528 emit_int32(imm32);
1529 }
1530
1531 void Assembler::cmpl(Register dst, int32_t imm32) {
1532 prefix(dst);
1533 emit_arith(0x81, 0xF8, dst, imm32);
1534 }
1535
1536 void Assembler::cmpl(Register dst, Register src) {
1537 (void) prefix_and_encode(dst->encoding(), src->encoding());
1538 emit_arith(0x3B, 0xC0, dst, src);
1539 }
1540
1541
1542 void Assembler::cmpl(Register dst, Address src) {
1543 InstructionMark im(this);
1544 prefix(src, dst);
1545 emit_int8((unsigned char)0x3B);
1546 emit_operand(dst, src);
1547 }
1548
1549 void Assembler::cmpw(Address dst, int imm16) {
1550 InstructionMark im(this);
1551 assert(!dst.base_needs_rex() && !dst.index_needs_rex(), "no extended registers");
1552 emit_int8(0x66);
1553 emit_int8((unsigned char)0x81);
1554 emit_operand(rdi, dst, 2);
1555 emit_int16(imm16);
1556 }
1557
1558 // The 32-bit cmpxchg compares the value at adr with the contents of rax,
1559 // and stores reg into adr if so; otherwise, the value at adr is loaded into rax,.
1560 // The ZF is set if the compared values were equal, and cleared otherwise.
1561 void Assembler::cmpxchgl(Register reg, Address adr) { // cmpxchg
1562 InstructionMark im(this);
1563 prefix(adr, reg);
1564 emit_int8(0x0F);
1565 emit_int8((unsigned char)0xB1);
1566 emit_operand(reg, adr);
1567 }
1568
1569 // The 8-bit cmpxchg compares the value at adr with the contents of rax,
1570 // and stores reg into adr if so; otherwise, the value at adr is loaded into rax,.
1571 // The ZF is set if the compared values were equal, and cleared otherwise.
1572 void Assembler::cmpxchgb(Register reg, Address adr) { // cmpxchg
1573 InstructionMark im(this);
1574 prefix(adr, reg, true);
1575 emit_int8(0x0F);
1576 emit_int8((unsigned char)0xB0);
1577 emit_operand(reg, adr);
1578 }
1579
1580 void Assembler::comisd(XMMRegister dst, Address src) {
1581 // NOTE: dbx seems to decode this as comiss even though the
1582 // 0x66 is there. Strangly ucomisd comes out correct
1583 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1584 if (VM_Version::supports_evex()) {
1585 _tuple_type = EVEX_T1S;
1586 _input_size_in_bits = EVEX_64bit;
1587 emit_simd_arith_nonds_q(0x2F, dst, src, VEX_SIMD_66, /* no_mask_reg */ true);
1588 } else {
1589 emit_simd_arith_nonds(0x2F, dst, src, VEX_SIMD_66);
1590 }
1591 }
1592
1593 void Assembler::comisd(XMMRegister dst, XMMRegister src) {
1594 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1595 if (VM_Version::supports_evex()) {
1596 emit_simd_arith_nonds_q(0x2F, dst, src, VEX_SIMD_66, /* no_mask_reg */ true);
1597 } else {
1598 emit_simd_arith_nonds(0x2F, dst, src, VEX_SIMD_66);
1599 }
1600 }
1601
1602 void Assembler::comiss(XMMRegister dst, Address src) {
1603 if (VM_Version::supports_evex()) {
1604 _tuple_type = EVEX_T1S;
1605 _input_size_in_bits = EVEX_32bit;
1606 }
1607 NOT_LP64(assert(VM_Version::supports_sse(), ""));
1608 emit_simd_arith_nonds(0x2F, dst, src, VEX_SIMD_NONE, /* no_mask_reg */ true);
1609 }
1610
1611 void Assembler::comiss(XMMRegister dst, XMMRegister src) {
1612 NOT_LP64(assert(VM_Version::supports_sse(), ""));
1613 emit_simd_arith_nonds(0x2F, dst, src, VEX_SIMD_NONE, /* no_mask_reg */ true);
1614 }
1615
1616 void Assembler::cpuid() {
1617 emit_int8(0x0F);
1618 emit_int8((unsigned char)0xA2);
1619 }
1620
1621 // Opcode / Instruction Op / En 64 - Bit Mode Compat / Leg Mode Description Implemented
1622 // F2 0F 38 F0 / r CRC32 r32, r / m8 RM Valid Valid Accumulate CRC32 on r / m8. v
1623 // F2 REX 0F 38 F0 / r CRC32 r32, r / m8* RM Valid N.E. Accumulate CRC32 on r / m8. -
1624 // F2 REX.W 0F 38 F0 / r CRC32 r64, r / m8 RM Valid N.E. Accumulate CRC32 on r / m8. -
1625 //
1626 // F2 0F 38 F1 / r CRC32 r32, r / m16 RM Valid Valid Accumulate CRC32 on r / m16. v
1627 //
1628 // F2 0F 38 F1 / r CRC32 r32, r / m32 RM Valid Valid Accumulate CRC32 on r / m32. v
1629 //
1630 // F2 REX.W 0F 38 F1 / r CRC32 r64, r / m64 RM Valid N.E. Accumulate CRC32 on r / m64. v
1631 void Assembler::crc32(Register crc, Register v, int8_t sizeInBytes) {
1632 assert(VM_Version::supports_sse4_2(), "");
1633 int8_t w = 0x01;
1634 Prefix p = Prefix_EMPTY;
1635
1636 emit_int8((int8_t)0xF2);
1637 switch (sizeInBytes) {
1638 case 1:
1639 w = 0;
1640 break;
1641 case 2:
1642 case 4:
1643 break;
1644 LP64_ONLY(case 8:)
1645 // This instruction is not valid in 32 bits
1646 // Note:
1647 // http://www.intel.com/content/dam/www/public/us/en/documents/manuals/64-ia-32-architectures-software-developer-instruction-set-reference-manual-325383.pdf
1648 //
1649 // Page B - 72 Vol. 2C says
1650 // qwreg2 to qwreg 1111 0010 : 0100 1R0B : 0000 1111 : 0011 1000 : 1111 0000 : 11 qwreg1 qwreg2
1651 // mem64 to qwreg 1111 0010 : 0100 1R0B : 0000 1111 : 0011 1000 : 1111 0000 : mod qwreg r / m
1652 // F0!!!
1653 // while 3 - 208 Vol. 2A
1654 // F2 REX.W 0F 38 F1 / r CRC32 r64, r / m64 RM Valid N.E.Accumulate CRC32 on r / m64.
1655 //
1656 // the 0 on a last bit is reserved for a different flavor of this instruction :
1657 // F2 REX.W 0F 38 F0 / r CRC32 r64, r / m8 RM Valid N.E.Accumulate CRC32 on r / m8.
1658 p = REX_W;
1659 break;
1660 default:
1661 assert(0, "Unsupported value for a sizeInBytes argument");
1662 break;
1663 }
1664 LP64_ONLY(prefix(crc, v, p);)
1665 emit_int8((int8_t)0x0F);
1666 emit_int8(0x38);
1667 emit_int8((int8_t)(0xF0 | w));
1668 emit_int8(0xC0 | ((crc->encoding() & 0x7) << 3) | (v->encoding() & 7));
1669 }
1670
1671 void Assembler::crc32(Register crc, Address adr, int8_t sizeInBytes) {
1672 assert(VM_Version::supports_sse4_2(), "");
1673 InstructionMark im(this);
1674 int8_t w = 0x01;
1675 Prefix p = Prefix_EMPTY;
1676
1677 emit_int8((int8_t)0xF2);
1678 switch (sizeInBytes) {
1679 case 1:
1680 w = 0;
1681 break;
1682 case 2:
1683 case 4:
1684 break;
1685 LP64_ONLY(case 8:)
1686 // This instruction is not valid in 32 bits
1687 p = REX_W;
1688 break;
1689 default:
1690 assert(0, "Unsupported value for a sizeInBytes argument");
1691 break;
1692 }
1693 LP64_ONLY(prefix(crc, adr, p);)
1694 emit_int8((int8_t)0x0F);
1695 emit_int8(0x38);
1696 emit_int8((int8_t)(0xF0 | w));
1697 emit_operand(crc, adr);
1698 }
1699
1700 void Assembler::cvtdq2pd(XMMRegister dst, XMMRegister src) {
1701 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1702 emit_simd_arith_nonds(0xE6, dst, src, VEX_SIMD_F3, /* no_mask_reg */ false, /* legacy_mode */ true);
1703 }
1704
1705 void Assembler::cvtdq2ps(XMMRegister dst, XMMRegister src) {
1706 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1707 emit_simd_arith_nonds(0x5B, dst, src, VEX_SIMD_NONE, /* no_mask_reg */ false, /* legacy_mode */ true);
1708 }
1709
1710 void Assembler::cvtsd2ss(XMMRegister dst, XMMRegister src) {
1711 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1712 if (VM_Version::supports_evex()) {
1713 emit_simd_arith_q(0x5A, dst, src, VEX_SIMD_F2);
1714 } else {
1715 emit_simd_arith(0x5A, dst, src, VEX_SIMD_F2);
1716 }
1717 }
1718
1719 void Assembler::cvtsd2ss(XMMRegister dst, Address src) {
1720 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1721 if (VM_Version::supports_evex()) {
1722 _tuple_type = EVEX_T1F;
1723 _input_size_in_bits = EVEX_64bit;
1724 emit_simd_arith_q(0x5A, dst, src, VEX_SIMD_F2);
1725 } else {
1726 emit_simd_arith(0x5A, dst, src, VEX_SIMD_F2);
1727 }
1728 }
1729
1730 void Assembler::cvtsi2sdl(XMMRegister dst, Register src) {
1731 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1732 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VM_Version::supports_evex());
1733 emit_int8(0x2A);
1734 emit_int8((unsigned char)(0xC0 | encode));
1735 }
1736
1737 void Assembler::cvtsi2sdl(XMMRegister dst, Address src) {
1738 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1739 if (VM_Version::supports_evex()) {
1740 _tuple_type = EVEX_T1S;
1741 _input_size_in_bits = EVEX_32bit;
1742 emit_simd_arith(0x2A, dst, src, VEX_SIMD_F2, /* no_mask_reg */ true);
1743 } else {
1744 emit_simd_arith(0x2A, dst, src, VEX_SIMD_F2);
1745 }
1746 }
1747
1748 void Assembler::cvtsi2ssl(XMMRegister dst, Register src) {
1749 NOT_LP64(assert(VM_Version::supports_sse(), ""));
1750 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, /* no_mask_reg */ true);
1751 emit_int8(0x2A);
1752 emit_int8((unsigned char)(0xC0 | encode));
1753 }
1754
1755 void Assembler::cvtsi2ssl(XMMRegister dst, Address src) {
1756 if (VM_Version::supports_evex()) {
1757 _tuple_type = EVEX_T1S;
1758 _input_size_in_bits = EVEX_32bit;
1759 }
1760 NOT_LP64(assert(VM_Version::supports_sse(), ""));
1761 emit_simd_arith(0x2A, dst, src, VEX_SIMD_F3, /* no_mask_reg */ true);
1762 }
1763
1764 void Assembler::cvtsi2ssq(XMMRegister dst, Register src) {
1765 NOT_LP64(assert(VM_Version::supports_sse(), ""));
1766 int encode = simd_prefix_and_encode_q(dst, dst, src, VEX_SIMD_F3, /* no_mask_reg */ true);
1767 emit_int8(0x2A);
1768 emit_int8((unsigned char)(0xC0 | encode));
1769 }
1770
1771 void Assembler::cvtss2sd(XMMRegister dst, XMMRegister src) {
1772 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1773 emit_simd_arith(0x5A, dst, src, VEX_SIMD_F3);
1774 }
1775
1776 void Assembler::cvtss2sd(XMMRegister dst, Address src) {
1777 if (VM_Version::supports_evex()) {
1778 _tuple_type = EVEX_T1S;
1779 _input_size_in_bits = EVEX_32bit;
1780 }
1781 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1782 emit_simd_arith(0x5A, dst, src, VEX_SIMD_F3);
1783 }
1784
1785
1786 void Assembler::cvttsd2sil(Register dst, XMMRegister src) {
1787 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1788 int encode = simd_prefix_and_encode(dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, /* no_mask_reg */ true);
1789 emit_int8(0x2C);
1790 emit_int8((unsigned char)(0xC0 | encode));
1791 }
1792
1793 void Assembler::cvttss2sil(Register dst, XMMRegister src) {
1794 NOT_LP64(assert(VM_Version::supports_sse(), ""));
1795 int encode = simd_prefix_and_encode(dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, /* no_mask_reg */ true);
1796 emit_int8(0x2C);
1797 emit_int8((unsigned char)(0xC0 | encode));
1798 }
1799
1800 void Assembler::decl(Address dst) {
1801 // Don't use it directly. Use MacroAssembler::decrement() instead.
1802 InstructionMark im(this);
1803 prefix(dst);
1804 emit_int8((unsigned char)0xFF);
1805 emit_operand(rcx, dst);
1806 }
1807
1808 void Assembler::divsd(XMMRegister dst, Address src) {
1809 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1810 if (VM_Version::supports_evex()) {
1811 _tuple_type = EVEX_T1S;
1812 _input_size_in_bits = EVEX_64bit;
1813 emit_simd_arith_q(0x5E, dst, src, VEX_SIMD_F2);
1814 } else {
1815 emit_simd_arith(0x5E, dst, src, VEX_SIMD_F2);
1816 }
1817 }
1818
1819 void Assembler::divsd(XMMRegister dst, XMMRegister src) {
1820 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1821 if (VM_Version::supports_evex()) {
1822 emit_simd_arith_q(0x5E, dst, src, VEX_SIMD_F2);
1823 } else {
1824 emit_simd_arith(0x5E, dst, src, VEX_SIMD_F2);
1825 }
1826 }
1827
1828 void Assembler::divss(XMMRegister dst, Address src) {
1829 if (VM_Version::supports_evex()) {
1830 _tuple_type = EVEX_T1S;
1831 _input_size_in_bits = EVEX_32bit;
1832 }
1833 NOT_LP64(assert(VM_Version::supports_sse(), ""));
1834 emit_simd_arith(0x5E, dst, src, VEX_SIMD_F3);
1835 }
1836
1837 void Assembler::divss(XMMRegister dst, XMMRegister src) {
1838 NOT_LP64(assert(VM_Version::supports_sse(), ""));
1839 emit_simd_arith(0x5E, dst, src, VEX_SIMD_F3);
1840 }
1841
1842 void Assembler::emms() {
1843 NOT_LP64(assert(VM_Version::supports_mmx(), ""));
1844 emit_int8(0x0F);
1845 emit_int8(0x77);
1846 }
1847
1848 void Assembler::hlt() {
1849 emit_int8((unsigned char)0xF4);
1850 }
1851
1852 void Assembler::idivl(Register src) {
1853 int encode = prefix_and_encode(src->encoding());
1854 emit_int8((unsigned char)0xF7);
1855 emit_int8((unsigned char)(0xF8 | encode));
1856 }
1857
1858 void Assembler::divl(Register src) { // Unsigned
1859 int encode = prefix_and_encode(src->encoding());
1860 emit_int8((unsigned char)0xF7);
1861 emit_int8((unsigned char)(0xF0 | encode));
1862 }
1863
1864 void Assembler::imull(Register dst, Register src) {
1865 int encode = prefix_and_encode(dst->encoding(), src->encoding());
1866 emit_int8(0x0F);
1867 emit_int8((unsigned char)0xAF);
1868 emit_int8((unsigned char)(0xC0 | encode));
1869 }
1870
1871
1872 void Assembler::imull(Register dst, Register src, int value) {
1873 int encode = prefix_and_encode(dst->encoding(), src->encoding());
1874 if (is8bit(value)) {
1875 emit_int8(0x6B);
1876 emit_int8((unsigned char)(0xC0 | encode));
1877 emit_int8(value & 0xFF);
1878 } else {
1879 emit_int8(0x69);
1880 emit_int8((unsigned char)(0xC0 | encode));
1881 emit_int32(value);
1882 }
1883 }
1884
1885 void Assembler::imull(Register dst, Address src) {
1886 InstructionMark im(this);
1887 prefix(src, dst);
1888 emit_int8(0x0F);
1889 emit_int8((unsigned char) 0xAF);
1890 emit_operand(dst, src);
1891 }
1892
1893
1894 void Assembler::incl(Address dst) {
1895 // Don't use it directly. Use MacroAssembler::increment() instead.
1896 InstructionMark im(this);
1897 prefix(dst);
1898 emit_int8((unsigned char)0xFF);
1899 emit_operand(rax, dst);
1900 }
1901
1902 void Assembler::jcc(Condition cc, Label& L, bool maybe_short) {
1903 InstructionMark im(this);
1904 assert((0 <= cc) && (cc < 16), "illegal cc");
1905 if (L.is_bound()) {
1906 address dst = target(L);
1907 assert(dst != NULL, "jcc most probably wrong");
1908
1909 const int short_size = 2;
1910 const int long_size = 6;
1911 intptr_t offs = (intptr_t)dst - (intptr_t)pc();
1912 if (maybe_short && is8bit(offs - short_size)) {
1913 // 0111 tttn #8-bit disp
1914 emit_int8(0x70 | cc);
1915 emit_int8((offs - short_size) & 0xFF);
1916 } else {
1917 // 0000 1111 1000 tttn #32-bit disp
1918 assert(is_simm32(offs - long_size),
1919 "must be 32bit offset (call4)");
1920 emit_int8(0x0F);
1921 emit_int8((unsigned char)(0x80 | cc));
1922 emit_int32(offs - long_size);
1923 }
1924 } else {
1925 // Note: could eliminate cond. jumps to this jump if condition
1926 // is the same however, seems to be rather unlikely case.
1927 // Note: use jccb() if label to be bound is very close to get
1928 // an 8-bit displacement
1929 L.add_patch_at(code(), locator());
1930 emit_int8(0x0F);
1931 emit_int8((unsigned char)(0x80 | cc));
1932 emit_int32(0);
1933 }
1934 }
1935
1936 void Assembler::jccb(Condition cc, Label& L) {
1937 if (L.is_bound()) {
1938 const int short_size = 2;
1939 address entry = target(L);
1940 #ifdef ASSERT
1941 intptr_t dist = (intptr_t)entry - ((intptr_t)pc() + short_size);
1942 intptr_t delta = short_branch_delta();
1943 if (delta != 0) {
1944 dist += (dist < 0 ? (-delta) :delta);
1945 }
1946 assert(is8bit(dist), "Dispacement too large for a short jmp");
1947 #endif
1948 intptr_t offs = (intptr_t)entry - (intptr_t)pc();
1949 // 0111 tttn #8-bit disp
1950 emit_int8(0x70 | cc);
1951 emit_int8((offs - short_size) & 0xFF);
1952 } else {
1953 InstructionMark im(this);
1954 L.add_patch_at(code(), locator());
1955 emit_int8(0x70 | cc);
1956 emit_int8(0);
1957 }
1958 }
1959
1960 void Assembler::jmp(Address adr) {
1961 InstructionMark im(this);
1962 prefix(adr);
1963 emit_int8((unsigned char)0xFF);
1964 emit_operand(rsp, adr);
1965 }
1966
1967 void Assembler::jmp(Label& L, bool maybe_short) {
1968 if (L.is_bound()) {
1969 address entry = target(L);
1970 assert(entry != NULL, "jmp most probably wrong");
1971 InstructionMark im(this);
1972 const int short_size = 2;
1973 const int long_size = 5;
1974 intptr_t offs = entry - pc();
1975 if (maybe_short && is8bit(offs - short_size)) {
1976 emit_int8((unsigned char)0xEB);
1977 emit_int8((offs - short_size) & 0xFF);
1978 } else {
1979 emit_int8((unsigned char)0xE9);
1980 emit_int32(offs - long_size);
1981 }
1982 } else {
1983 // By default, forward jumps are always 32-bit displacements, since
1984 // we can't yet know where the label will be bound. If you're sure that
1985 // the forward jump will not run beyond 256 bytes, use jmpb to
1986 // force an 8-bit displacement.
1987 InstructionMark im(this);
1988 L.add_patch_at(code(), locator());
1989 emit_int8((unsigned char)0xE9);
1990 emit_int32(0);
1991 }
1992 }
1993
1994 void Assembler::jmp(Register entry) {
1995 int encode = prefix_and_encode(entry->encoding());
1996 emit_int8((unsigned char)0xFF);
1997 emit_int8((unsigned char)(0xE0 | encode));
1998 }
1999
2000 void Assembler::jmp_literal(address dest, RelocationHolder const& rspec) {
2001 InstructionMark im(this);
2002 emit_int8((unsigned char)0xE9);
2003 assert(dest != NULL, "must have a target");
2004 intptr_t disp = dest - (pc() + sizeof(int32_t));
2005 assert(is_simm32(disp), "must be 32bit offset (jmp)");
2006 emit_data(disp, rspec.reloc(), call32_operand);
2007 }
2008
2009 void Assembler::jmpb(Label& L) {
2010 if (L.is_bound()) {
2011 const int short_size = 2;
2012 address entry = target(L);
2013 assert(entry != NULL, "jmp most probably wrong");
2014 #ifdef ASSERT
2015 intptr_t dist = (intptr_t)entry - ((intptr_t)pc() + short_size);
2016 intptr_t delta = short_branch_delta();
2017 if (delta != 0) {
2018 dist += (dist < 0 ? (-delta) :delta);
2019 }
2020 assert(is8bit(dist), "Dispacement too large for a short jmp");
2021 #endif
2022 intptr_t offs = entry - pc();
2023 emit_int8((unsigned char)0xEB);
2024 emit_int8((offs - short_size) & 0xFF);
2025 } else {
2026 InstructionMark im(this);
2027 L.add_patch_at(code(), locator());
2028 emit_int8((unsigned char)0xEB);
2029 emit_int8(0);
2030 }
2031 }
2032
2033 void Assembler::ldmxcsr( Address src) {
2034 NOT_LP64(assert(VM_Version::supports_sse(), ""));
2035 InstructionMark im(this);
2036 prefix(src);
2037 emit_int8(0x0F);
2038 emit_int8((unsigned char)0xAE);
2039 emit_operand(as_Register(2), src);
2040 }
2041
2042 void Assembler::leal(Register dst, Address src) {
2043 InstructionMark im(this);
2044 #ifdef _LP64
2045 emit_int8(0x67); // addr32
2046 prefix(src, dst);
2047 #endif // LP64
2048 emit_int8((unsigned char)0x8D);
2049 emit_operand(dst, src);
2050 }
2051
2052 void Assembler::lfence() {
2053 emit_int8(0x0F);
2054 emit_int8((unsigned char)0xAE);
2055 emit_int8((unsigned char)0xE8);
2056 }
2057
2058 void Assembler::lock() {
2059 emit_int8((unsigned char)0xF0);
2060 }
2061
2062 void Assembler::lzcntl(Register dst, Register src) {
2063 assert(VM_Version::supports_lzcnt(), "encoding is treated as BSR");
2064 emit_int8((unsigned char)0xF3);
2065 int encode = prefix_and_encode(dst->encoding(), src->encoding());
2066 emit_int8(0x0F);
2067 emit_int8((unsigned char)0xBD);
2068 emit_int8((unsigned char)(0xC0 | encode));
2069 }
2070
2071 // Emit mfence instruction
2072 void Assembler::mfence() {
2073 NOT_LP64(assert(VM_Version::supports_sse2(), "unsupported");)
2074 emit_int8(0x0F);
2075 emit_int8((unsigned char)0xAE);
2076 emit_int8((unsigned char)0xF0);
2077 }
2078
2079 void Assembler::mov(Register dst, Register src) {
2080 LP64_ONLY(movq(dst, src)) NOT_LP64(movl(dst, src));
2081 }
2082
2083 void Assembler::movapd(XMMRegister dst, XMMRegister src) {
2084 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2085 if (VM_Version::supports_avx512novl()) {
2086 int vector_len = AVX_512bit;
2087 int dst_enc = dst->encoding();
2088 int src_enc = src->encoding();
2089 int encode = vex_prefix_and_encode(dst_enc, 0, src_enc, VEX_SIMD_66, VEX_OPCODE_0F,
2090 /* vex_w */ true, vector_len, /* legacy_mode */ false, /* no_mask_reg */ false);
2091 emit_int8(0x28);
2092 emit_int8((unsigned char)(0xC0 | encode));
2093 } else if (VM_Version::supports_evex()) {
2094 emit_simd_arith_nonds_q(0x28, dst, src, VEX_SIMD_66);
2095 } else {
2096 emit_simd_arith_nonds(0x28, dst, src, VEX_SIMD_66);
2097 }
2098 }
2099
2100 void Assembler::movaps(XMMRegister dst, XMMRegister src) {
2101 NOT_LP64(assert(VM_Version::supports_sse(), ""));
2102 if (VM_Version::supports_avx512novl()) {
2103 int vector_len = AVX_512bit;
2104 int encode = vex_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, vector_len);
2105 emit_int8(0x28);
2106 emit_int8((unsigned char)(0xC0 | encode));
2107 } else {
2108 emit_simd_arith_nonds(0x28, dst, src, VEX_SIMD_NONE);
2109 }
2110 }
2111
2112 void Assembler::movlhps(XMMRegister dst, XMMRegister src) {
2113 NOT_LP64(assert(VM_Version::supports_sse(), ""));
2114 int encode = simd_prefix_and_encode(dst, src, src, VEX_SIMD_NONE, /* no_mask_reg */ true);
2115 emit_int8(0x16);
2116 emit_int8((unsigned char)(0xC0 | encode));
2117 }
2118
2119 void Assembler::movb(Register dst, Address src) {
2120 NOT_LP64(assert(dst->has_byte_register(), "must have byte register"));
2121 InstructionMark im(this);
2122 prefix(src, dst, true);
2123 emit_int8((unsigned char)0x8A);
2124 emit_operand(dst, src);
2125 }
2126
2127 void Assembler::kmovql(KRegister dst, KRegister src) {
2128 NOT_LP64(assert(VM_Version::supports_evex(), ""));
2129 int encode = kreg_prefix_and_encode(dst, knoreg, src, VEX_SIMD_NONE,
2130 /* no_mask_reg */ true, VEX_OPCODE_0F, /* rex_w */ true);
2131 emit_int8((unsigned char)0x90);
2132 emit_int8((unsigned char)(0xC0 | encode));
2133 }
2134
2135 void Assembler::kmovql(KRegister dst, Address src) {
2136 NOT_LP64(assert(VM_Version::supports_evex(), ""));
2137 int dst_enc = dst->encoding();
2138 int nds_enc = 0;
2139 vex_prefix(src, nds_enc, dst_enc, VEX_SIMD_NONE,
2140 VEX_OPCODE_0F, /* vex_w */ true, AVX_128bit, /* legacy_mode */ true, /* no_reg_mask */ true);
2141 emit_int8((unsigned char)0x90);
2142 emit_operand((Register)dst, src);
2143 }
2144
2145 void Assembler::kmovql(Address dst, KRegister src) {
2146 NOT_LP64(assert(VM_Version::supports_evex(), ""));
2147 int src_enc = src->encoding();
2148 int nds_enc = 0;
2149 vex_prefix(dst, nds_enc, src_enc, VEX_SIMD_NONE,
2150 VEX_OPCODE_0F, /* vex_w */ true, AVX_128bit, /* legacy_mode */ true, /* no_reg_mask */ true);
2151 emit_int8((unsigned char)0x90);
2152 emit_operand((Register)src, dst);
2153 }
2154
2155 void Assembler::kmovql(KRegister dst, Register src) {
2156 NOT_LP64(assert(VM_Version::supports_evex(), ""));
2157 VexSimdPrefix pre = !_legacy_mode_bw ? VEX_SIMD_F2 : VEX_SIMD_NONE;
2158 int encode = kreg_prefix_and_encode(dst, knoreg, src, pre, /* no_mask_reg */ true,
2159 VEX_OPCODE_0F, /* legacy_mode */ !_legacy_mode_bw);
2160 emit_int8((unsigned char)0x92);
2161 emit_int8((unsigned char)(0xC0 | encode));
2162 }
2163
2164 void Assembler::kmovdl(KRegister dst, Register src) {
2165 NOT_LP64(assert(VM_Version::supports_evex(), ""));
2166 VexSimdPrefix pre = !_legacy_mode_bw ? VEX_SIMD_F2 : VEX_SIMD_NONE;
2167 int encode = kreg_prefix_and_encode(dst, knoreg, src, pre, /* no_mask_reg */ true);
2168 emit_int8((unsigned char)0x92);
2169 emit_int8((unsigned char)(0xC0 | encode));
2170 }
2171
2172 void Assembler::kmovwl(KRegister dst, Register src) {
2173 NOT_LP64(assert(VM_Version::supports_evex(), ""));
2174 int encode = kreg_prefix_and_encode(dst, knoreg, src, VEX_SIMD_NONE, /* no_mask_reg */ true);
2175 emit_int8((unsigned char)0x92);
2176 emit_int8((unsigned char)(0xC0 | encode));
2177 }
2178
2179 void Assembler::movb(Address dst, int imm8) {
2180 InstructionMark im(this);
2181 prefix(dst);
2182 emit_int8((unsigned char)0xC6);
2183 emit_operand(rax, dst, 1);
2184 emit_int8(imm8);
2185 }
2186
2187
2188 void Assembler::movb(Address dst, Register src) {
2189 assert(src->has_byte_register(), "must have byte register");
2190 InstructionMark im(this);
2191 prefix(dst, src, true);
2192 emit_int8((unsigned char)0x88);
2193 emit_operand(src, dst);
2194 }
2195
2196 void Assembler::movdl(XMMRegister dst, Register src) {
2197 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2198 int encode = simd_prefix_and_encode(dst, src, VEX_SIMD_66, /* no_mask_reg */ true);
2199 emit_int8(0x6E);
2200 emit_int8((unsigned char)(0xC0 | encode));
2201 }
2202
2203 void Assembler::movdl(Register dst, XMMRegister src) {
2204 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2205 // swap src/dst to get correct prefix
2206 int encode = simd_prefix_and_encode(src, dst, VEX_SIMD_66, /* no_mask_reg */ true);
2207 emit_int8(0x7E);
2208 emit_int8((unsigned char)(0xC0 | encode));
2209 }
2210
2211 void Assembler::movdl(XMMRegister dst, Address src) {
2212 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2213 if (VM_Version::supports_evex()) {
2214 _tuple_type = EVEX_T1S;
2215 _input_size_in_bits = EVEX_32bit;
2216 }
2217 InstructionMark im(this);
2218 simd_prefix(dst, src, VEX_SIMD_66, /* no_reg_mask */ true);
2219 emit_int8(0x6E);
2220 emit_operand(dst, src);
2221 }
2222
2223 void Assembler::movdl(Address dst, XMMRegister src) {
2224 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2225 if (VM_Version::supports_evex()) {
2226 _tuple_type = EVEX_T1S;
2227 _input_size_in_bits = EVEX_32bit;
2228 }
2229 InstructionMark im(this);
2230 simd_prefix(dst, src, VEX_SIMD_66, /* no_reg_mask */ true);
2231 emit_int8(0x7E);
2232 emit_operand(src, dst);
2233 }
2234
2235 void Assembler::movdqa(XMMRegister dst, XMMRegister src) {
2236 _instruction_uses_vl = true;
2237 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2238 emit_simd_arith_nonds(0x6F, dst, src, VEX_SIMD_66);
2239 }
2240
2241 void Assembler::movdqa(XMMRegister dst, Address src) {
2242 _instruction_uses_vl = true;
2243 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2244 if (VM_Version::supports_evex()) {
2245 _tuple_type = EVEX_FVM;
2246 }
2247 emit_simd_arith_nonds(0x6F, dst, src, VEX_SIMD_66);
2248 }
2249
2250 void Assembler::movdqu(XMMRegister dst, Address src) {
2251 _instruction_uses_vl = true;
2252 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2253 if (VM_Version::supports_evex()) {
2254 _tuple_type = EVEX_FVM;
2255 }
2256 emit_simd_arith_nonds(0x6F, dst, src, VEX_SIMD_F3);
2257 }
2258
2259 void Assembler::movdqu(XMMRegister dst, XMMRegister src) {
2260 _instruction_uses_vl = true;
2261 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2262 emit_simd_arith_nonds(0x6F, dst, src, VEX_SIMD_F3);
2263 }
2264
2265 void Assembler::movdqu(Address dst, XMMRegister src) {
2266 _instruction_uses_vl = true;
2267 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2268 if (VM_Version::supports_evex()) {
2269 _tuple_type = EVEX_FVM;
2270 }
2271 InstructionMark im(this);
2272 simd_prefix(dst, src, VEX_SIMD_F3, /* no_mask_reg */ false);
2273 emit_int8(0x7F);
2274 emit_operand(src, dst);
2275 }
2276
2277 // Move Unaligned 256bit Vector
2278 void Assembler::vmovdqu(XMMRegister dst, XMMRegister src) {
2279 _instruction_uses_vl = true;
2280 assert(UseAVX > 0, "");
2281 int vector_len = AVX_256bit;
2282 int encode = vex_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_F3, vector_len);
2283 emit_int8(0x6F);
2284 emit_int8((unsigned char)(0xC0 | encode));
2285 }
2286
2287 void Assembler::vmovdqu(XMMRegister dst, Address src) {
2288 _instruction_uses_vl = true;
2289 assert(UseAVX > 0, "");
2290 if (VM_Version::supports_evex()) {
2291 _tuple_type = EVEX_FVM;
2292 }
2293 InstructionMark im(this);
2294 int vector_len = AVX_256bit;
2295 vex_prefix(dst, xnoreg, src, VEX_SIMD_F3, vector_len);
2296 emit_int8(0x6F);
2297 emit_operand(dst, src);
2298 }
2299
2300 void Assembler::vmovdqu(Address dst, XMMRegister src) {
2301 _instruction_uses_vl = true;
2302 assert(UseAVX > 0, "");
2303 if (VM_Version::supports_evex()) {
2304 _tuple_type = EVEX_FVM;
2305 }
2306 InstructionMark im(this);
2307 int vector_len = AVX_256bit;
2308 // swap src<->dst for encoding
2309 assert(src != xnoreg, "sanity");
2310 vex_prefix(src, xnoreg, dst, VEX_SIMD_F3, vector_len);
2311 emit_int8(0x7F);
2312 emit_operand(src, dst);
2313 }
2314
2315 // Move Unaligned EVEX enabled Vector (programmable : 8,16,32,64)
2316 void Assembler::evmovdqul(XMMRegister dst, XMMRegister src, int vector_len) {
2317 _instruction_uses_vl = true;
2318 assert(UseAVX > 0, "");
2319 int src_enc = src->encoding();
2320 int dst_enc = dst->encoding();
2321 int encode = vex_prefix_and_encode(dst_enc, 0, src_enc, VEX_SIMD_F3, VEX_OPCODE_0F,
2322 /* vex_w */ false, vector_len, /* legacy_mode */ false, /* no_mask_reg */ false);
2323 emit_int8(0x6F);
2324 emit_int8((unsigned char)(0xC0 | encode));
2325 }
2326
2327 void Assembler::evmovdqul(XMMRegister dst, Address src, int vector_len) {
2328 _instruction_uses_vl = true;
2329 assert(UseAVX > 0, "");
2330 InstructionMark im(this);
2331 if (VM_Version::supports_evex()) {
2332 _tuple_type = EVEX_FVM;
2333 }
2334 vex_prefix(dst, xnoreg, src, VEX_SIMD_F3, vector_len);
2335 emit_int8(0x6F);
2336 emit_operand(dst, src);
2337 }
2338
2339 void Assembler::evmovdqul(Address dst, XMMRegister src, int vector_len) {
2340 _instruction_uses_vl = true;
2341 assert(UseAVX > 0, "");
2342 InstructionMark im(this);
2343 assert(src != xnoreg, "sanity");
2344 if (VM_Version::supports_evex()) {
2345 _tuple_type = EVEX_FVM;
2346 }
2347 // swap src<->dst for encoding
2348 vex_prefix(src, xnoreg, dst, VEX_SIMD_F3, vector_len);
2349 emit_int8(0x7F);
2350 emit_operand(src, dst);
2351 }
2352
2353 void Assembler::evmovdquq(XMMRegister dst, XMMRegister src, int vector_len) {
2354 _instruction_uses_vl = true;
2355 assert(UseAVX > 0, "");
2356 int src_enc = src->encoding();
2357 int dst_enc = dst->encoding();
2358 int encode = vex_prefix_and_encode(dst_enc, 0, src_enc, VEX_SIMD_F3, VEX_OPCODE_0F,
2359 /* vex_w */ true, vector_len, /* legacy_mode */ false, /* no_mask_reg */ false);
2360 emit_int8(0x6F);
2361 emit_int8((unsigned char)(0xC0 | encode));
2362 }
2363
2364 void Assembler::evmovdquq(XMMRegister dst, Address src, int vector_len) {
2365 _instruction_uses_vl = true;
2366 assert(UseAVX > 2, "");
2367 InstructionMark im(this);
2368 _tuple_type = EVEX_FVM;
2369 vex_prefix_q(dst, xnoreg, src, VEX_SIMD_F3, vector_len);
2370 emit_int8(0x6F);
2371 emit_operand(dst, src);
2372 }
2373
2374 void Assembler::evmovdquq(Address dst, XMMRegister src, int vector_len) {
2375 _instruction_uses_vl = true;
2376 assert(UseAVX > 2, "");
2377 InstructionMark im(this);
2378 assert(src != xnoreg, "sanity");
2379 _tuple_type = EVEX_FVM;
2380 // swap src<->dst for encoding
2381 vex_prefix_q(src, xnoreg, dst, VEX_SIMD_F3, vector_len);
2382 emit_int8(0x7F);
2383 emit_operand(src, dst);
2384 }
2385
2386 // Uses zero extension on 64bit
2387
2388 void Assembler::movl(Register dst, int32_t imm32) {
2389 int encode = prefix_and_encode(dst->encoding());
2390 emit_int8((unsigned char)(0xB8 | encode));
2391 emit_int32(imm32);
2392 }
2393
2394 void Assembler::movl(Register dst, Register src) {
2395 int encode = prefix_and_encode(dst->encoding(), src->encoding());
2396 emit_int8((unsigned char)0x8B);
2397 emit_int8((unsigned char)(0xC0 | encode));
2398 }
2399
2400 void Assembler::movl(Register dst, Address src) {
2401 InstructionMark im(this);
2402 prefix(src, dst);
2403 emit_int8((unsigned char)0x8B);
2404 emit_operand(dst, src);
2405 }
2406
2407 void Assembler::movl(Address dst, int32_t imm32) {
2408 InstructionMark im(this);
2409 prefix(dst);
2410 emit_int8((unsigned char)0xC7);
2411 emit_operand(rax, dst, 4);
2412 emit_int32(imm32);
2413 }
2414
2415 void Assembler::movl(Address dst, Register src) {
2416 InstructionMark im(this);
2417 prefix(dst, src);
2418 emit_int8((unsigned char)0x89);
2419 emit_operand(src, dst);
2420 }
2421
2422 // New cpus require to use movsd and movss to avoid partial register stall
2423 // when loading from memory. But for old Opteron use movlpd instead of movsd.
2424 // The selection is done in MacroAssembler::movdbl() and movflt().
2425 void Assembler::movlpd(XMMRegister dst, Address src) {
2426 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2427 if (VM_Version::supports_evex()) {
2428 _tuple_type = EVEX_T1S;
2429 _input_size_in_bits = EVEX_32bit;
2430 emit_simd_arith_q(0x12, dst, src, VEX_SIMD_66, /* no_mask_reg */ true);
2431 } else {
2432 emit_simd_arith(0x12, dst, src, VEX_SIMD_66, /* no_mask_reg */ true);
2433 }
2434 }
2435
2436 void Assembler::movq( MMXRegister dst, Address src ) {
2437 assert( VM_Version::supports_mmx(), "" );
2438 emit_int8(0x0F);
2439 emit_int8(0x6F);
2440 emit_operand(dst, src);
2441 }
2442
2443 void Assembler::movq( Address dst, MMXRegister src ) {
2444 assert( VM_Version::supports_mmx(), "" );
2445 emit_int8(0x0F);
2446 emit_int8(0x7F);
2447 // workaround gcc (3.2.1-7a) bug
2448 // In that version of gcc with only an emit_operand(MMX, Address)
2449 // gcc will tail jump and try and reverse the parameters completely
2450 // obliterating dst in the process. By having a version available
2451 // that doesn't need to swap the args at the tail jump the bug is
2452 // avoided.
2453 emit_operand(dst, src);
2454 }
2455
2456 void Assembler::movq(XMMRegister dst, Address src) {
2457 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2458 InstructionMark im(this);
2459 if (VM_Version::supports_evex()) {
2460 _tuple_type = EVEX_T1S;
2461 _input_size_in_bits = EVEX_64bit;
2462 simd_prefix_q(dst, xnoreg, src, VEX_SIMD_F3, /* no_mask_reg */ true);
2463 } else {
2464 simd_prefix(dst, src, VEX_SIMD_F3, /* no_mask_reg */ true);
2465 }
2466 emit_int8(0x7E);
2467 emit_operand(dst, src);
2468 }
2469
2470 void Assembler::movq(Address dst, XMMRegister src) {
2471 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2472 InstructionMark im(this);
2473 if (VM_Version::supports_evex()) {
2474 _tuple_type = EVEX_T1S;
2475 _input_size_in_bits = EVEX_64bit;
2476 simd_prefix(src, xnoreg, dst, VEX_SIMD_66, /* no_mask_reg */ true,
2477 VEX_OPCODE_0F, /* rex_w */ true);
2478 } else {
2479 simd_prefix(dst, src, VEX_SIMD_66, /* no_mask_reg */ true);
2480 }
2481 emit_int8((unsigned char)0xD6);
2482 emit_operand(src, dst);
2483 }
2484
2485 void Assembler::movsbl(Register dst, Address src) { // movsxb
2486 InstructionMark im(this);
2487 prefix(src, dst);
2488 emit_int8(0x0F);
2489 emit_int8((unsigned char)0xBE);
2490 emit_operand(dst, src);
2491 }
2492
2493 void Assembler::movsbl(Register dst, Register src) { // movsxb
2494 NOT_LP64(assert(src->has_byte_register(), "must have byte register"));
2495 int encode = prefix_and_encode(dst->encoding(), false, src->encoding(), true);
2496 emit_int8(0x0F);
2497 emit_int8((unsigned char)0xBE);
2498 emit_int8((unsigned char)(0xC0 | encode));
2499 }
2500
2501 void Assembler::movsd(XMMRegister dst, XMMRegister src) {
2502 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2503 if (VM_Version::supports_evex()) {
2504 emit_simd_arith_q(0x10, dst, src, VEX_SIMD_F2, /* no_mask_reg */ true);
2505 } else {
2506 emit_simd_arith(0x10, dst, src, VEX_SIMD_F2);
2507 }
2508 }
2509
2510 void Assembler::movsd(XMMRegister dst, Address src) {
2511 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2512 if (VM_Version::supports_evex()) {
2513 _tuple_type = EVEX_T1S;
2514 _input_size_in_bits = EVEX_64bit;
2515 emit_simd_arith_nonds_q(0x10, dst, src, VEX_SIMD_F2, /* no_mask_reg */ true);
2516 } else {
2517 emit_simd_arith_nonds(0x10, dst, src, VEX_SIMD_F2);
2518 }
2519 }
2520
2521 void Assembler::movsd(Address dst, XMMRegister src) {
2522 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2523 InstructionMark im(this);
2524 if (VM_Version::supports_evex()) {
2525 _tuple_type = EVEX_T1S;
2526 _input_size_in_bits = EVEX_64bit;
2527 simd_prefix_q(src, xnoreg, dst, VEX_SIMD_F2);
2528 } else {
2529 simd_prefix(src, xnoreg, dst, VEX_SIMD_F2, /* no_mask_reg */ false);
2530 }
2531 emit_int8(0x11);
2532 emit_operand(src, dst);
2533 }
2534
2535 void Assembler::movss(XMMRegister dst, XMMRegister src) {
2536 NOT_LP64(assert(VM_Version::supports_sse(), ""));
2537 emit_simd_arith(0x10, dst, src, VEX_SIMD_F3, /* no_mask_reg */ true);
2538 }
2539
2540 void Assembler::movss(XMMRegister dst, Address src) {
2541 NOT_LP64(assert(VM_Version::supports_sse(), ""));
2542 if (VM_Version::supports_evex()) {
2543 _tuple_type = EVEX_T1S;
2544 _input_size_in_bits = EVEX_32bit;
2545 }
2546 emit_simd_arith_nonds(0x10, dst, src, VEX_SIMD_F3, /* no_mask_reg */ true);
2547 }
2548
2549 void Assembler::movss(Address dst, XMMRegister src) {
2550 NOT_LP64(assert(VM_Version::supports_sse(), ""));
2551 if (VM_Version::supports_evex()) {
2552 _tuple_type = EVEX_T1S;
2553 _input_size_in_bits = EVEX_32bit;
2554 }
2555 InstructionMark im(this);
2556 simd_prefix(dst, src, VEX_SIMD_F3, /* no_mask_reg */ false);
2557 emit_int8(0x11);
2558 emit_operand(src, dst);
2559 }
2560
2561 void Assembler::movswl(Register dst, Address src) { // movsxw
2562 InstructionMark im(this);
2563 prefix(src, dst);
2564 emit_int8(0x0F);
2565 emit_int8((unsigned char)0xBF);
2566 emit_operand(dst, src);
2567 }
2568
2569 void Assembler::movswl(Register dst, Register src) { // movsxw
2570 int encode = prefix_and_encode(dst->encoding(), src->encoding());
2571 emit_int8(0x0F);
2572 emit_int8((unsigned char)0xBF);
2573 emit_int8((unsigned char)(0xC0 | encode));
2574 }
2575
2576 void Assembler::movw(Address dst, int imm16) {
2577 InstructionMark im(this);
2578
2579 emit_int8(0x66); // switch to 16-bit mode
2580 prefix(dst);
2581 emit_int8((unsigned char)0xC7);
2582 emit_operand(rax, dst, 2);
2583 emit_int16(imm16);
2584 }
2585
2586 void Assembler::movw(Register dst, Address src) {
2587 InstructionMark im(this);
2588 emit_int8(0x66);
2589 prefix(src, dst);
2590 emit_int8((unsigned char)0x8B);
2591 emit_operand(dst, src);
2592 }
2593
2594 void Assembler::movw(Address dst, Register src) {
2595 InstructionMark im(this);
2596 emit_int8(0x66);
2597 prefix(dst, src);
2598 emit_int8((unsigned char)0x89);
2599 emit_operand(src, dst);
2600 }
2601
2602 void Assembler::movzbl(Register dst, Address src) { // movzxb
2603 InstructionMark im(this);
2604 prefix(src, dst);
2605 emit_int8(0x0F);
2606 emit_int8((unsigned char)0xB6);
2607 emit_operand(dst, src);
2608 }
2609
2610 void Assembler::movzbl(Register dst, Register src) { // movzxb
2611 NOT_LP64(assert(src->has_byte_register(), "must have byte register"));
2612 int encode = prefix_and_encode(dst->encoding(), false, src->encoding(), true);
2613 emit_int8(0x0F);
2614 emit_int8((unsigned char)0xB6);
2615 emit_int8(0xC0 | encode);
2616 }
2617
2618 void Assembler::movzwl(Register dst, Address src) { // movzxw
2619 InstructionMark im(this);
2620 prefix(src, dst);
2621 emit_int8(0x0F);
2622 emit_int8((unsigned char)0xB7);
2623 emit_operand(dst, src);
2624 }
2625
2626 void Assembler::movzwl(Register dst, Register src) { // movzxw
2627 int encode = prefix_and_encode(dst->encoding(), src->encoding());
2628 emit_int8(0x0F);
2629 emit_int8((unsigned char)0xB7);
2630 emit_int8(0xC0 | encode);
2631 }
2632
2633 void Assembler::mull(Address src) {
2634 InstructionMark im(this);
2635 prefix(src);
2636 emit_int8((unsigned char)0xF7);
2637 emit_operand(rsp, src);
2638 }
2639
2640 void Assembler::mull(Register src) {
2641 int encode = prefix_and_encode(src->encoding());
2642 emit_int8((unsigned char)0xF7);
2643 emit_int8((unsigned char)(0xE0 | encode));
2644 }
2645
2646 void Assembler::mulsd(XMMRegister dst, Address src) {
2647 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2648 if (VM_Version::supports_evex()) {
2649 _tuple_type = EVEX_T1S;
2650 _input_size_in_bits = EVEX_64bit;
2651 emit_simd_arith_q(0x59, dst, src, VEX_SIMD_F2);
2652 } else {
2653 emit_simd_arith(0x59, dst, src, VEX_SIMD_F2);
2654 }
2655 }
2656
2657 void Assembler::mulsd(XMMRegister dst, XMMRegister src) {
2658 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2659 if (VM_Version::supports_evex()) {
2660 emit_simd_arith_q(0x59, dst, src, VEX_SIMD_F2);
2661 } else {
2662 emit_simd_arith(0x59, dst, src, VEX_SIMD_F2);
2663 }
2664 }
2665
2666 void Assembler::mulss(XMMRegister dst, Address src) {
2667 NOT_LP64(assert(VM_Version::supports_sse(), ""));
2668 if (VM_Version::supports_evex()) {
2669 _tuple_type = EVEX_T1S;
2670 _input_size_in_bits = EVEX_32bit;
2671 }
2672 emit_simd_arith(0x59, dst, src, VEX_SIMD_F3);
2673 }
2674
2675 void Assembler::mulss(XMMRegister dst, XMMRegister src) {
2676 NOT_LP64(assert(VM_Version::supports_sse(), ""));
2677 emit_simd_arith(0x59, dst, src, VEX_SIMD_F3);
2678 }
2679
2680 void Assembler::negl(Register dst) {
2681 int encode = prefix_and_encode(dst->encoding());
2682 emit_int8((unsigned char)0xF7);
2683 emit_int8((unsigned char)(0xD8 | encode));
2684 }
2685
2686 void Assembler::nop(int i) {
2687 #ifdef ASSERT
2688 assert(i > 0, " ");
2689 // The fancy nops aren't currently recognized by debuggers making it a
2690 // pain to disassemble code while debugging. If asserts are on clearly
2691 // speed is not an issue so simply use the single byte traditional nop
2692 // to do alignment.
2693
2694 for (; i > 0 ; i--) emit_int8((unsigned char)0x90);
2695 return;
2696
2697 #endif // ASSERT
2698
2699 if (UseAddressNop && VM_Version::is_intel()) {
2700 //
2701 // Using multi-bytes nops "0x0F 0x1F [address]" for Intel
2702 // 1: 0x90
2703 // 2: 0x66 0x90
2704 // 3: 0x66 0x66 0x90 (don't use "0x0F 0x1F 0x00" - need patching safe padding)
2705 // 4: 0x0F 0x1F 0x40 0x00
2706 // 5: 0x0F 0x1F 0x44 0x00 0x00
2707 // 6: 0x66 0x0F 0x1F 0x44 0x00 0x00
2708 // 7: 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00
2709 // 8: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
2710 // 9: 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
2711 // 10: 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
2712 // 11: 0x66 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
2713
2714 // The rest coding is Intel specific - don't use consecutive address nops
2715
2716 // 12: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
2717 // 13: 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
2718 // 14: 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
2719 // 15: 0x66 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
2720
2721 while(i >= 15) {
2722 // For Intel don't generate consecutive addess nops (mix with regular nops)
2723 i -= 15;
2724 emit_int8(0x66); // size prefix
2725 emit_int8(0x66); // size prefix
2726 emit_int8(0x66); // size prefix
2727 addr_nop_8();
2728 emit_int8(0x66); // size prefix
2729 emit_int8(0x66); // size prefix
2730 emit_int8(0x66); // size prefix
2731 emit_int8((unsigned char)0x90);
2732 // nop
2733 }
2734 switch (i) {
2735 case 14:
2736 emit_int8(0x66); // size prefix
2737 case 13:
2738 emit_int8(0x66); // size prefix
2739 case 12:
2740 addr_nop_8();
2741 emit_int8(0x66); // size prefix
2742 emit_int8(0x66); // size prefix
2743 emit_int8(0x66); // size prefix
2744 emit_int8((unsigned char)0x90);
2745 // nop
2746 break;
2747 case 11:
2748 emit_int8(0x66); // size prefix
2749 case 10:
2750 emit_int8(0x66); // size prefix
2751 case 9:
2752 emit_int8(0x66); // size prefix
2753 case 8:
2754 addr_nop_8();
2755 break;
2756 case 7:
2757 addr_nop_7();
2758 break;
2759 case 6:
2760 emit_int8(0x66); // size prefix
2761 case 5:
2762 addr_nop_5();
2763 break;
2764 case 4:
2765 addr_nop_4();
2766 break;
2767 case 3:
2768 // Don't use "0x0F 0x1F 0x00" - need patching safe padding
2769 emit_int8(0x66); // size prefix
2770 case 2:
2771 emit_int8(0x66); // size prefix
2772 case 1:
2773 emit_int8((unsigned char)0x90);
2774 // nop
2775 break;
2776 default:
2777 assert(i == 0, " ");
2778 }
2779 return;
2780 }
2781 if (UseAddressNop && VM_Version::is_amd()) {
2782 //
2783 // Using multi-bytes nops "0x0F 0x1F [address]" for AMD.
2784 // 1: 0x90
2785 // 2: 0x66 0x90
2786 // 3: 0x66 0x66 0x90 (don't use "0x0F 0x1F 0x00" - need patching safe padding)
2787 // 4: 0x0F 0x1F 0x40 0x00
2788 // 5: 0x0F 0x1F 0x44 0x00 0x00
2789 // 6: 0x66 0x0F 0x1F 0x44 0x00 0x00
2790 // 7: 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00
2791 // 8: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
2792 // 9: 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
2793 // 10: 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
2794 // 11: 0x66 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
2795
2796 // The rest coding is AMD specific - use consecutive address nops
2797
2798 // 12: 0x66 0x0F 0x1F 0x44 0x00 0x00 0x66 0x0F 0x1F 0x44 0x00 0x00
2799 // 13: 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00 0x66 0x0F 0x1F 0x44 0x00 0x00
2800 // 14: 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00
2801 // 15: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00
2802 // 16: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
2803 // Size prefixes (0x66) are added for larger sizes
2804
2805 while(i >= 22) {
2806 i -= 11;
2807 emit_int8(0x66); // size prefix
2808 emit_int8(0x66); // size prefix
2809 emit_int8(0x66); // size prefix
2810 addr_nop_8();
2811 }
2812 // Generate first nop for size between 21-12
2813 switch (i) {
2814 case 21:
2815 i -= 1;
2816 emit_int8(0x66); // size prefix
2817 case 20:
2818 case 19:
2819 i -= 1;
2820 emit_int8(0x66); // size prefix
2821 case 18:
2822 case 17:
2823 i -= 1;
2824 emit_int8(0x66); // size prefix
2825 case 16:
2826 case 15:
2827 i -= 8;
2828 addr_nop_8();
2829 break;
2830 case 14:
2831 case 13:
2832 i -= 7;
2833 addr_nop_7();
2834 break;
2835 case 12:
2836 i -= 6;
2837 emit_int8(0x66); // size prefix
2838 addr_nop_5();
2839 break;
2840 default:
2841 assert(i < 12, " ");
2842 }
2843
2844 // Generate second nop for size between 11-1
2845 switch (i) {
2846 case 11:
2847 emit_int8(0x66); // size prefix
2848 case 10:
2849 emit_int8(0x66); // size prefix
2850 case 9:
2851 emit_int8(0x66); // size prefix
2852 case 8:
2853 addr_nop_8();
2854 break;
2855 case 7:
2856 addr_nop_7();
2857 break;
2858 case 6:
2859 emit_int8(0x66); // size prefix
2860 case 5:
2861 addr_nop_5();
2862 break;
2863 case 4:
2864 addr_nop_4();
2865 break;
2866 case 3:
2867 // Don't use "0x0F 0x1F 0x00" - need patching safe padding
2868 emit_int8(0x66); // size prefix
2869 case 2:
2870 emit_int8(0x66); // size prefix
2871 case 1:
2872 emit_int8((unsigned char)0x90);
2873 // nop
2874 break;
2875 default:
2876 assert(i == 0, " ");
2877 }
2878 return;
2879 }
2880
2881 // Using nops with size prefixes "0x66 0x90".
2882 // From AMD Optimization Guide:
2883 // 1: 0x90
2884 // 2: 0x66 0x90
2885 // 3: 0x66 0x66 0x90
2886 // 4: 0x66 0x66 0x66 0x90
2887 // 5: 0x66 0x66 0x90 0x66 0x90
2888 // 6: 0x66 0x66 0x90 0x66 0x66 0x90
2889 // 7: 0x66 0x66 0x66 0x90 0x66 0x66 0x90
2890 // 8: 0x66 0x66 0x66 0x90 0x66 0x66 0x66 0x90
2891 // 9: 0x66 0x66 0x90 0x66 0x66 0x90 0x66 0x66 0x90
2892 // 10: 0x66 0x66 0x66 0x90 0x66 0x66 0x90 0x66 0x66 0x90
2893 //
2894 while(i > 12) {
2895 i -= 4;
2896 emit_int8(0x66); // size prefix
2897 emit_int8(0x66);
2898 emit_int8(0x66);
2899 emit_int8((unsigned char)0x90);
2900 // nop
2901 }
2902 // 1 - 12 nops
2903 if(i > 8) {
2904 if(i > 9) {
2905 i -= 1;
2906 emit_int8(0x66);
2907 }
2908 i -= 3;
2909 emit_int8(0x66);
2910 emit_int8(0x66);
2911 emit_int8((unsigned char)0x90);
2912 }
2913 // 1 - 8 nops
2914 if(i > 4) {
2915 if(i > 6) {
2916 i -= 1;
2917 emit_int8(0x66);
2918 }
2919 i -= 3;
2920 emit_int8(0x66);
2921 emit_int8(0x66);
2922 emit_int8((unsigned char)0x90);
2923 }
2924 switch (i) {
2925 case 4:
2926 emit_int8(0x66);
2927 case 3:
2928 emit_int8(0x66);
2929 case 2:
2930 emit_int8(0x66);
2931 case 1:
2932 emit_int8((unsigned char)0x90);
2933 break;
2934 default:
2935 assert(i == 0, " ");
2936 }
2937 }
2938
2939 void Assembler::notl(Register dst) {
2940 int encode = prefix_and_encode(dst->encoding());
2941 emit_int8((unsigned char)0xF7);
2942 emit_int8((unsigned char)(0xD0 | encode));
2943 }
2944
2945 void Assembler::orl(Address dst, int32_t imm32) {
2946 InstructionMark im(this);
2947 prefix(dst);
2948 emit_arith_operand(0x81, rcx, dst, imm32);
2949 }
2950
2951 void Assembler::orl(Register dst, int32_t imm32) {
2952 prefix(dst);
2953 emit_arith(0x81, 0xC8, dst, imm32);
2954 }
2955
2956 void Assembler::orl(Register dst, Address src) {
2957 InstructionMark im(this);
2958 prefix(src, dst);
2959 emit_int8(0x0B);
2960 emit_operand(dst, src);
2961 }
2962
2963 void Assembler::orl(Register dst, Register src) {
2964 (void) prefix_and_encode(dst->encoding(), src->encoding());
2965 emit_arith(0x0B, 0xC0, dst, src);
2966 }
2967
2968 void Assembler::orl(Address dst, Register src) {
2969 InstructionMark im(this);
2970 prefix(dst, src);
2971 emit_int8(0x09);
2972 emit_operand(src, dst);
2973 }
2974
2975 void Assembler::packuswb(XMMRegister dst, Address src) {
2976 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2977 assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
2978 if (VM_Version::supports_evex()) {
2979 _tuple_type = EVEX_FV;
2980 _input_size_in_bits = EVEX_32bit;
2981 }
2982 emit_simd_arith(0x67, dst, src, VEX_SIMD_66, /* no_mask_reg */ false, /* legacy_mode */ _legacy_mode_bw);
2983 }
2984
2985 void Assembler::packuswb(XMMRegister dst, XMMRegister src) {
2986 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2987 emit_simd_arith(0x67, dst, src, VEX_SIMD_66, /* no_mask_reg */ false, /* legacy_mode */ _legacy_mode_bw);
2988 }
2989
2990 void Assembler::vpackuswb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
2991 assert(UseAVX > 0, "some form of AVX must be enabled");
2992 emit_vex_arith(0x67, dst, nds, src, VEX_SIMD_66, vector_len, /* no_mask_reg */ false, /* legacy_mode */ _legacy_mode_bw);
2993 }
2994
2995 void Assembler::vpermq(XMMRegister dst, XMMRegister src, int imm8, int vector_len) {
2996 _instruction_uses_vl = true;
2997 assert(VM_Version::supports_avx2(), "");
2998 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, /* no_mask_reg */ false,
2999 VEX_OPCODE_0F_3A, /* rex_w */ true, vector_len);
3000 emit_int8(0x00);
3001 emit_int8(0xC0 | encode);
3002 emit_int8(imm8);
3003 }
3004
3005 void Assembler::pause() {
3006 emit_int8((unsigned char)0xF3);
3007 emit_int8((unsigned char)0x90);
3008 }
3009
3010 void Assembler::pcmpestri(XMMRegister dst, Address src, int imm8) {
3011 assert(VM_Version::supports_sse4_2(), "");
3012 InstructionMark im(this);
3013 simd_prefix(dst, xnoreg, src, VEX_SIMD_66, /* no_mask_reg */ false, VEX_OPCODE_0F_3A,
3014 /* rex_w */ false, AVX_128bit, /* legacy_mode */ true);
3015 emit_int8(0x61);
3016 emit_operand(dst, src);
3017 emit_int8(imm8);
3018 }
3019
3020 void Assembler::pcmpestri(XMMRegister dst, XMMRegister src, int imm8) {
3021 assert(VM_Version::supports_sse4_2(), "");
3022 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, /* no_mask_reg */ false,
3023 VEX_OPCODE_0F_3A, /* rex_w */ false, AVX_128bit, /* legacy_mode */ true);
3024 emit_int8(0x61);
3025 emit_int8((unsigned char)(0xC0 | encode));
3026 emit_int8(imm8);
3027 }
3028
3029 void Assembler::pextrd(Register dst, XMMRegister src, int imm8) {
3030 assert(VM_Version::supports_sse4_1(), "");
3031 int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_66, /* no_mask_reg */ true,
3032 VEX_OPCODE_0F_3A, /* rex_w */ false, AVX_128bit, /* legacy_mode */ _legacy_mode_dq);
3033 emit_int8(0x16);
3034 emit_int8((unsigned char)(0xC0 | encode));
3035 emit_int8(imm8);
3036 }
3037
3038 void Assembler::pextrq(Register dst, XMMRegister src, int imm8) {
3039 assert(VM_Version::supports_sse4_1(), "");
3040 int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_66, /* no_mask_reg */ true,
3041 VEX_OPCODE_0F_3A, /* rex_w */ true, AVX_128bit, /* legacy_mode */ _legacy_mode_dq);
3042 emit_int8(0x16);
3043 emit_int8((unsigned char)(0xC0 | encode));
3044 emit_int8(imm8);
3045 }
3046
3047 void Assembler::pinsrd(XMMRegister dst, Register src, int imm8) {
3048 assert(VM_Version::supports_sse4_1(), "");
3049 int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_66, /* no_mask_reg */ true,
3050 VEX_OPCODE_0F_3A, /* rex_w */ false, AVX_128bit, /* legacy_mode */ _legacy_mode_dq);
3051 emit_int8(0x22);
3052 emit_int8((unsigned char)(0xC0 | encode));
3053 emit_int8(imm8);
3054 }
3055
3056 void Assembler::pinsrq(XMMRegister dst, Register src, int imm8) {
3057 assert(VM_Version::supports_sse4_1(), "");
3058 int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_66, /* no_mask_reg */ true,
3059 VEX_OPCODE_0F_3A, /* rex_w */ true, AVX_128bit, /* legacy_mode */ _legacy_mode_dq);
3060 emit_int8(0x22);
3061 emit_int8((unsigned char)(0xC0 | encode));
3062 emit_int8(imm8);
3063 }
3064
3065 void Assembler::pmovzxbw(XMMRegister dst, Address src) {
3066 assert(VM_Version::supports_sse4_1(), "");
3067 if (VM_Version::supports_evex()) {
3068 _tuple_type = EVEX_HVM;
3069 }
3070 InstructionMark im(this);
3071 simd_prefix(dst, src, VEX_SIMD_66, /* no_mask_reg */ false, VEX_OPCODE_0F_38);
3072 emit_int8(0x30);
3073 emit_operand(dst, src);
3074 }
3075
3076 void Assembler::pmovzxbw(XMMRegister dst, XMMRegister src) {
3077 assert(VM_Version::supports_sse4_1(), "");
3078 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, /* no_mask_reg */ false, VEX_OPCODE_0F_38);
3079 emit_int8(0x30);
3080 emit_int8((unsigned char)(0xC0 | encode));
3081 }
3082
3083 // generic
3084 void Assembler::pop(Register dst) {
3085 int encode = prefix_and_encode(dst->encoding());
3086 emit_int8(0x58 | encode);
3087 }
3088
3089 void Assembler::popcntl(Register dst, Address src) {
3090 assert(VM_Version::supports_popcnt(), "must support");
3091 InstructionMark im(this);
3092 emit_int8((unsigned char)0xF3);
3093 prefix(src, dst);
3094 emit_int8(0x0F);
3095 emit_int8((unsigned char)0xB8);
3096 emit_operand(dst, src);
3097 }
3098
3099 void Assembler::popcntl(Register dst, Register src) {
3100 assert(VM_Version::supports_popcnt(), "must support");
3101 emit_int8((unsigned char)0xF3);
3102 int encode = prefix_and_encode(dst->encoding(), src->encoding());
3103 emit_int8(0x0F);
3104 emit_int8((unsigned char)0xB8);
3105 emit_int8((unsigned char)(0xC0 | encode));
3106 }
3107
3108 void Assembler::popf() {
3109 emit_int8((unsigned char)0x9D);
3110 }
3111
3112 #ifndef _LP64 // no 32bit push/pop on amd64
3113 void Assembler::popl(Address dst) {
3114 // NOTE: this will adjust stack by 8byte on 64bits
3115 InstructionMark im(this);
3116 prefix(dst);
3117 emit_int8((unsigned char)0x8F);
3118 emit_operand(rax, dst);
3119 }
3120 #endif
3121
3122 void Assembler::prefetch_prefix(Address src) {
3123 prefix(src);
3124 emit_int8(0x0F);
3125 }
3126
3127 void Assembler::prefetchnta(Address src) {
3128 NOT_LP64(assert(VM_Version::supports_sse(), "must support"));
3129 InstructionMark im(this);
3130 prefetch_prefix(src);
3131 emit_int8(0x18);
3132 emit_operand(rax, src); // 0, src
3133 }
3134
3135 void Assembler::prefetchr(Address src) {
3136 assert(VM_Version::supports_3dnow_prefetch(), "must support");
3137 InstructionMark im(this);
3138 prefetch_prefix(src);
3139 emit_int8(0x0D);
3140 emit_operand(rax, src); // 0, src
3141 }
3142
3143 void Assembler::prefetcht0(Address src) {
3144 NOT_LP64(assert(VM_Version::supports_sse(), "must support"));
3145 InstructionMark im(this);
3146 prefetch_prefix(src);
3147 emit_int8(0x18);
3148 emit_operand(rcx, src); // 1, src
3149 }
3150
3151 void Assembler::prefetcht1(Address src) {
3152 NOT_LP64(assert(VM_Version::supports_sse(), "must support"));
3153 InstructionMark im(this);
3154 prefetch_prefix(src);
3155 emit_int8(0x18);
3156 emit_operand(rdx, src); // 2, src
3157 }
3158
3159 void Assembler::prefetcht2(Address src) {
3160 NOT_LP64(assert(VM_Version::supports_sse(), "must support"));
3161 InstructionMark im(this);
3162 prefetch_prefix(src);
3163 emit_int8(0x18);
3164 emit_operand(rbx, src); // 3, src
3165 }
3166
3167 void Assembler::prefetchw(Address src) {
3168 assert(VM_Version::supports_3dnow_prefetch(), "must support");
3169 InstructionMark im(this);
3170 prefetch_prefix(src);
3171 emit_int8(0x0D);
3172 emit_operand(rcx, src); // 1, src
3173 }
3174
3175 void Assembler::prefix(Prefix p) {
3176 emit_int8(p);
3177 }
3178
3179 void Assembler::pshufb(XMMRegister dst, XMMRegister src) {
3180 assert(VM_Version::supports_ssse3(), "");
3181 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, /* no_mask_reg */ false,
3182 VEX_OPCODE_0F_38, /* rex_w */ false, AVX_128bit, /* legacy_mode */ _legacy_mode_bw);
3183 emit_int8(0x00);
3184 emit_int8((unsigned char)(0xC0 | encode));
3185 }
3186
3187 void Assembler::pshufb(XMMRegister dst, Address src) {
3188 assert(VM_Version::supports_ssse3(), "");
3189 if (VM_Version::supports_evex()) {
3190 _tuple_type = EVEX_FVM;
3191 }
3192 InstructionMark im(this);
3193 simd_prefix(dst, dst, src, VEX_SIMD_66, /* no_mask_reg */ false,
3194 VEX_OPCODE_0F_38, /* rex_w */ false, AVX_128bit, /* legacy_mode */ _legacy_mode_bw);
3195 emit_int8(0x00);
3196 emit_operand(dst, src);
3197 }
3198
3199 void Assembler::pshufd(XMMRegister dst, XMMRegister src, int mode) {
3200 _instruction_uses_vl = true;
3201 assert(isByte(mode), "invalid value");
3202 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3203 emit_simd_arith_nonds(0x70, dst, src, VEX_SIMD_66);
3204 emit_int8(mode & 0xFF);
3205 }
3206
3207 void Assembler::pshufd(XMMRegister dst, Address src, int mode) {
3208 _instruction_uses_vl = true;
3209 assert(isByte(mode), "invalid value");
3210 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3211 assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
3212 if (VM_Version::supports_evex()) {
3213 _tuple_type = EVEX_FV;
3214 _input_size_in_bits = EVEX_32bit;
3215 }
3216 InstructionMark im(this);
3217 simd_prefix(dst, src, VEX_SIMD_66, /* no_mask_reg */ false);
3218 emit_int8(0x70);
3219 emit_operand(dst, src);
3220 emit_int8(mode & 0xFF);
3221 }
3222
3223 void Assembler::pshuflw(XMMRegister dst, XMMRegister src, int mode) {
3224 assert(isByte(mode), "invalid value");
3225 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3226 emit_simd_arith_nonds(0x70, dst, src, VEX_SIMD_F2, /* no_mask_reg */ false, /* legacy_mode */ _legacy_mode_bw);
3227 emit_int8(mode & 0xFF);
3228 }
3229
3230 void Assembler::pshuflw(XMMRegister dst, Address src, int mode) {
3231 assert(isByte(mode), "invalid value");
3232 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3233 assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
3234 if (VM_Version::supports_evex()) {
3235 _tuple_type = EVEX_FVM;
3236 }
3237 InstructionMark im(this);
3238 simd_prefix(dst, xnoreg, src, VEX_SIMD_F2, /* no_mask_reg */ false,
3239 VEX_OPCODE_0F, /* rex_w */ false, AVX_128bit, /* legacy_mode */ _legacy_mode_bw);
3240 emit_int8(0x70);
3241 emit_operand(dst, src);
3242 emit_int8(mode & 0xFF);
3243 }
3244
3245 void Assembler::psrldq(XMMRegister dst, int shift) {
3246 // Shift left 128 bit value in dst XMMRegister by shift number of bytes.
3247 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3248 // XMM3 is for /3 encoding: 66 0F 73 /3 ib
3249 int encode = simd_prefix_and_encode(xmm3, dst, dst, VEX_SIMD_66, /* no_mask_reg */ true,
3250 VEX_OPCODE_0F, /* rex_w */ false, AVX_128bit, /* legacy_mode */ _legacy_mode_bw);
3251 emit_int8(0x73);
3252 emit_int8((unsigned char)(0xC0 | encode));
3253 emit_int8(shift);
3254 }
3255
3256 void Assembler::pslldq(XMMRegister dst, int shift) {
3257 // Shift left 128 bit value in dst XMMRegister by shift number of bytes.
3258 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3259 // XMM7 is for /7 encoding: 66 0F 73 /7 ib
3260 int encode = simd_prefix_and_encode(xmm7, dst, dst, VEX_SIMD_66, /* no_mask_reg */ true,
3261 VEX_OPCODE_0F, /* rex_w */ false, AVX_128bit, /* legacy_mode */ _legacy_mode_bw);
3262 emit_int8(0x73);
3263 emit_int8((unsigned char)(0xC0 | encode));
3264 emit_int8(shift);
3265 }
3266
3267 void Assembler::ptest(XMMRegister dst, Address src) {
3268 assert(VM_Version::supports_sse4_1(), "");
3269 assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
3270 InstructionMark im(this);
3271 simd_prefix(dst, xnoreg, src, VEX_SIMD_66, /* no_mask_reg */ false,
3272 VEX_OPCODE_0F_38, /* rex_w */ false, AVX_128bit, /* legacy_mode */ true);
3273 emit_int8(0x17);
3274 emit_operand(dst, src);
3275 }
3276
3277 void Assembler::ptest(XMMRegister dst, XMMRegister src) {
3278 assert(VM_Version::supports_sse4_1(), "");
3279 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, /* no_mask_reg */ false,
3280 VEX_OPCODE_0F_38, /* rex_w */ false, AVX_128bit, /* legacy_mode */ true);
3281 emit_int8(0x17);
3282 emit_int8((unsigned char)(0xC0 | encode));
3283 }
3284
3285 void Assembler::vptest(XMMRegister dst, Address src) {
3286 assert(VM_Version::supports_avx(), "");
3287 InstructionMark im(this);
3288 int vector_len = AVX_256bit;
3289 assert(dst != xnoreg, "sanity");
3290 int dst_enc = dst->encoding();
3291 // swap src<->dst for encoding
3292 vex_prefix(src, 0, dst_enc, VEX_SIMD_66, VEX_OPCODE_0F_38, /* rex_w */ false,
3293 vector_len, /* legacy_mode */ true, /* no_mask_reg */ false);
3294 emit_int8(0x17);
3295 emit_operand(dst, src);
3296 }
3297
3298 void Assembler::vptest(XMMRegister dst, XMMRegister src) {
3299 assert(VM_Version::supports_avx(), "");
3300 int vector_len = AVX_256bit;
3301 int encode = vex_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, vector_len, VEX_OPCODE_0F_38, /* legacy_mode */ true);
3302 emit_int8(0x17);
3303 emit_int8((unsigned char)(0xC0 | encode));
3304 }
3305
3306 void Assembler::punpcklbw(XMMRegister dst, Address src) {
3307 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3308 assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
3309 if (VM_Version::supports_evex()) {
3310 _tuple_type = EVEX_FVM;
3311 }
3312 emit_simd_arith(0x60, dst, src, VEX_SIMD_66, /* no_mask_reg */ false, /* legacy_mode */ _legacy_mode_vlbw);
3313 }
3314
3315 void Assembler::punpcklbw(XMMRegister dst, XMMRegister src) {
3316 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3317 emit_simd_arith(0x60, dst, src, VEX_SIMD_66, /* no_mask_reg */ false, /* legacy_mode */ _legacy_mode_vlbw);
3318 }
3319
3320 void Assembler::punpckldq(XMMRegister dst, Address src) {
3321 _instruction_uses_vl = true;
3322 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3323 assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
3324 if (VM_Version::supports_evex()) {
3325 _tuple_type = EVEX_FV;
3326 _input_size_in_bits = EVEX_32bit;
3327 }
3328 emit_simd_arith(0x62, dst, src, VEX_SIMD_66);
3329 }
3330
3331 void Assembler::punpckldq(XMMRegister dst, XMMRegister src) {
3332 _instruction_uses_vl = true;
3333 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3334 emit_simd_arith(0x62, dst, src, VEX_SIMD_66);
3335 }
3336
3337 void Assembler::punpcklqdq(XMMRegister dst, XMMRegister src) {
3338 _instruction_uses_vl = true;
3339 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3340 if (VM_Version::supports_evex()) {
3341 emit_simd_arith_q(0x6C, dst, src, VEX_SIMD_66);
3342 } else {
3343 emit_simd_arith(0x6C, dst, src, VEX_SIMD_66);
3344 }
3345 }
3346
3347 void Assembler::push(int32_t imm32) {
3348 // in 64bits we push 64bits onto the stack but only
3349 // take a 32bit immediate
3350 emit_int8(0x68);
3351 emit_int32(imm32);
3352 }
3353
3354 void Assembler::push(Register src) {
3355 int encode = prefix_and_encode(src->encoding());
3356
3357 emit_int8(0x50 | encode);
3358 }
3359
3360 void Assembler::pushf() {
3361 emit_int8((unsigned char)0x9C);
3362 }
3363
3364 #ifndef _LP64 // no 32bit push/pop on amd64
3365 void Assembler::pushl(Address src) {
3366 // Note this will push 64bit on 64bit
3367 InstructionMark im(this);
3368 prefix(src);
3369 emit_int8((unsigned char)0xFF);
3370 emit_operand(rsi, src);
3371 }
3372 #endif
3373
3374 void Assembler::rcll(Register dst, int imm8) {
3375 assert(isShiftCount(imm8), "illegal shift count");
3376 int encode = prefix_and_encode(dst->encoding());
3377 if (imm8 == 1) {
3378 emit_int8((unsigned char)0xD1);
3379 emit_int8((unsigned char)(0xD0 | encode));
3380 } else {
3381 emit_int8((unsigned char)0xC1);
3382 emit_int8((unsigned char)0xD0 | encode);
3383 emit_int8(imm8);
3384 }
3385 }
3386
3387 void Assembler::rdtsc() {
3388 emit_int8((unsigned char)0x0F);
3389 emit_int8((unsigned char)0x31);
3390 }
3391
3392 // copies data from [esi] to [edi] using rcx pointer sized words
3393 // generic
3394 void Assembler::rep_mov() {
3395 emit_int8((unsigned char)0xF3);
3396 // MOVSQ
3397 LP64_ONLY(prefix(REX_W));
3398 emit_int8((unsigned char)0xA5);
3399 }
3400
3401 // sets rcx bytes with rax, value at [edi]
3402 void Assembler::rep_stosb() {
3403 emit_int8((unsigned char)0xF3); // REP
3404 LP64_ONLY(prefix(REX_W));
3405 emit_int8((unsigned char)0xAA); // STOSB
3406 }
3407
3408 // sets rcx pointer sized words with rax, value at [edi]
3409 // generic
3410 void Assembler::rep_stos() {
3411 emit_int8((unsigned char)0xF3); // REP
3412 LP64_ONLY(prefix(REX_W)); // LP64:STOSQ, LP32:STOSD
3413 emit_int8((unsigned char)0xAB);
3414 }
3415
3416 // scans rcx pointer sized words at [edi] for occurance of rax,
3417 // generic
3418 void Assembler::repne_scan() { // repne_scan
3419 emit_int8((unsigned char)0xF2);
3420 // SCASQ
3421 LP64_ONLY(prefix(REX_W));
3422 emit_int8((unsigned char)0xAF);
3423 }
3424
3425 #ifdef _LP64
3426 // scans rcx 4 byte words at [edi] for occurance of rax,
3427 // generic
3428 void Assembler::repne_scanl() { // repne_scan
3429 emit_int8((unsigned char)0xF2);
3430 // SCASL
3431 emit_int8((unsigned char)0xAF);
3432 }
3433 #endif
3434
3435 void Assembler::ret(int imm16) {
3436 if (imm16 == 0) {
3437 emit_int8((unsigned char)0xC3);
3438 } else {
3439 emit_int8((unsigned char)0xC2);
3440 emit_int16(imm16);
3441 }
3442 }
3443
3444 void Assembler::sahf() {
3445 #ifdef _LP64
3446 // Not supported in 64bit mode
3447 ShouldNotReachHere();
3448 #endif
3449 emit_int8((unsigned char)0x9E);
3450 }
3451
3452 void Assembler::sarl(Register dst, int imm8) {
3453 int encode = prefix_and_encode(dst->encoding());
3454 assert(isShiftCount(imm8), "illegal shift count");
3455 if (imm8 == 1) {
3456 emit_int8((unsigned char)0xD1);
3457 emit_int8((unsigned char)(0xF8 | encode));
3458 } else {
3459 emit_int8((unsigned char)0xC1);
3460 emit_int8((unsigned char)(0xF8 | encode));
3461 emit_int8(imm8);
3462 }
3463 }
3464
3465 void Assembler::sarl(Register dst) {
3466 int encode = prefix_and_encode(dst->encoding());
3467 emit_int8((unsigned char)0xD3);
3468 emit_int8((unsigned char)(0xF8 | encode));
3469 }
3470
3471 void Assembler::sbbl(Address dst, int32_t imm32) {
3472 InstructionMark im(this);
3473 prefix(dst);
3474 emit_arith_operand(0x81, rbx, dst, imm32);
3475 }
3476
3477 void Assembler::sbbl(Register dst, int32_t imm32) {
3478 prefix(dst);
3479 emit_arith(0x81, 0xD8, dst, imm32);
3480 }
3481
3482
3483 void Assembler::sbbl(Register dst, Address src) {
3484 InstructionMark im(this);
3485 prefix(src, dst);
3486 emit_int8(0x1B);
3487 emit_operand(dst, src);
3488 }
3489
3490 void Assembler::sbbl(Register dst, Register src) {
3491 (void) prefix_and_encode(dst->encoding(), src->encoding());
3492 emit_arith(0x1B, 0xC0, dst, src);
3493 }
3494
3495 void Assembler::setb(Condition cc, Register dst) {
3496 assert(0 <= cc && cc < 16, "illegal cc");
3497 int encode = prefix_and_encode(dst->encoding(), true);
3498 emit_int8(0x0F);
3499 emit_int8((unsigned char)0x90 | cc);
3500 emit_int8((unsigned char)(0xC0 | encode));
3501 }
3502
3503 void Assembler::shll(Register dst, int imm8) {
3504 assert(isShiftCount(imm8), "illegal shift count");
3505 int encode = prefix_and_encode(dst->encoding());
3506 if (imm8 == 1 ) {
3507 emit_int8((unsigned char)0xD1);
3508 emit_int8((unsigned char)(0xE0 | encode));
3509 } else {
3510 emit_int8((unsigned char)0xC1);
3511 emit_int8((unsigned char)(0xE0 | encode));
3512 emit_int8(imm8);
3513 }
3514 }
3515
3516 void Assembler::shll(Register dst) {
3517 int encode = prefix_and_encode(dst->encoding());
3518 emit_int8((unsigned char)0xD3);
3519 emit_int8((unsigned char)(0xE0 | encode));
3520 }
3521
3522 void Assembler::shrl(Register dst, int imm8) {
3523 assert(isShiftCount(imm8), "illegal shift count");
3524 int encode = prefix_and_encode(dst->encoding());
3525 emit_int8((unsigned char)0xC1);
3526 emit_int8((unsigned char)(0xE8 | encode));
3527 emit_int8(imm8);
3528 }
3529
3530 void Assembler::shrl(Register dst) {
3531 int encode = prefix_and_encode(dst->encoding());
3532 emit_int8((unsigned char)0xD3);
3533 emit_int8((unsigned char)(0xE8 | encode));
3534 }
3535
3536 // copies a single word from [esi] to [edi]
3537 void Assembler::smovl() {
3538 emit_int8((unsigned char)0xA5);
3539 }
3540
3541 void Assembler::sqrtsd(XMMRegister dst, XMMRegister src) {
3542 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3543 if (VM_Version::supports_evex()) {
3544 emit_simd_arith_q(0x51, dst, src, VEX_SIMD_F2);
3545 } else {
3546 emit_simd_arith(0x51, dst, src, VEX_SIMD_F2);
3547 }
3548 }
3549
3550 void Assembler::sqrtsd(XMMRegister dst, Address src) {
3551 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3552 if (VM_Version::supports_evex()) {
3553 _tuple_type = EVEX_T1S;
3554 _input_size_in_bits = EVEX_64bit;
3555 emit_simd_arith_q(0x51, dst, src, VEX_SIMD_F2);
3556 } else {
3557 emit_simd_arith(0x51, dst, src, VEX_SIMD_F2);
3558 }
3559 }
3560
3561 void Assembler::sqrtss(XMMRegister dst, XMMRegister src) {
3562 NOT_LP64(assert(VM_Version::supports_sse(), ""));
3563 emit_simd_arith(0x51, dst, src, VEX_SIMD_F3);
3564 }
3565
3566 void Assembler::std() {
3567 emit_int8((unsigned char)0xFD);
3568 }
3569
3570 void Assembler::sqrtss(XMMRegister dst, Address src) {
3571 NOT_LP64(assert(VM_Version::supports_sse(), ""));
3572 if (VM_Version::supports_evex()) {
3573 _tuple_type = EVEX_T1S;
3574 _input_size_in_bits = EVEX_32bit;
3575 }
3576 emit_simd_arith(0x51, dst, src, VEX_SIMD_F3);
3577 }
3578
3579 void Assembler::stmxcsr( Address dst) {
3580 NOT_LP64(assert(VM_Version::supports_sse(), ""));
3581 InstructionMark im(this);
3582 prefix(dst);
3583 emit_int8(0x0F);
3584 emit_int8((unsigned char)0xAE);
3585 emit_operand(as_Register(3), dst);
3586 }
3587
3588 void Assembler::subl(Address dst, int32_t imm32) {
3589 InstructionMark im(this);
3590 prefix(dst);
3591 emit_arith_operand(0x81, rbp, dst, imm32);
3592 }
3593
3594 void Assembler::subl(Address dst, Register src) {
3595 InstructionMark im(this);
3596 prefix(dst, src);
3597 emit_int8(0x29);
3598 emit_operand(src, dst);
3599 }
3600
3601 void Assembler::subl(Register dst, int32_t imm32) {
3602 prefix(dst);
3603 emit_arith(0x81, 0xE8, dst, imm32);
3604 }
3605
3606 // Force generation of a 4 byte immediate value even if it fits into 8bit
3607 void Assembler::subl_imm32(Register dst, int32_t imm32) {
3608 prefix(dst);
3609 emit_arith_imm32(0x81, 0xE8, dst, imm32);
3610 }
3611
3612 void Assembler::subl(Register dst, Address src) {
3613 InstructionMark im(this);
3614 prefix(src, dst);
3615 emit_int8(0x2B);
3616 emit_operand(dst, src);
3617 }
3618
3619 void Assembler::subl(Register dst, Register src) {
3620 (void) prefix_and_encode(dst->encoding(), src->encoding());
3621 emit_arith(0x2B, 0xC0, dst, src);
3622 }
3623
3624 void Assembler::subsd(XMMRegister dst, XMMRegister src) {
3625 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3626 if (VM_Version::supports_evex()) {
3627 emit_simd_arith_q(0x5C, dst, src, VEX_SIMD_F2);
3628 } else {
3629 emit_simd_arith(0x5C, dst, src, VEX_SIMD_F2);
3630 }
3631 }
3632
3633 void Assembler::subsd(XMMRegister dst, Address src) {
3634 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3635 if (VM_Version::supports_evex()) {
3636 _tuple_type = EVEX_T1S;
3637 _input_size_in_bits = EVEX_64bit;
3638 }
3639 if (VM_Version::supports_evex()) {
3640 emit_simd_arith_q(0x5C, dst, src, VEX_SIMD_F2);
3641 } else {
3642 emit_simd_arith(0x5C, dst, src, VEX_SIMD_F2);
3643 }
3644 }
3645
3646 void Assembler::subss(XMMRegister dst, XMMRegister src) {
3647 NOT_LP64(assert(VM_Version::supports_sse(), ""));
3648 emit_simd_arith(0x5C, dst, src, VEX_SIMD_F3);
3649 }
3650
3651 void Assembler::subss(XMMRegister dst, Address src) {
3652 NOT_LP64(assert(VM_Version::supports_sse(), ""));
3653 if (VM_Version::supports_evex()) {
3654 _tuple_type = EVEX_T1S;
3655 _input_size_in_bits = EVEX_32bit;
3656 }
3657 emit_simd_arith(0x5C, dst, src, VEX_SIMD_F3);
3658 }
3659
3660 void Assembler::testb(Register dst, int imm8) {
3661 NOT_LP64(assert(dst->has_byte_register(), "must have byte register"));
3662 (void) prefix_and_encode(dst->encoding(), true);
3663 emit_arith_b(0xF6, 0xC0, dst, imm8);
3664 }
3665
3666 void Assembler::testl(Register dst, int32_t imm32) {
3667 // not using emit_arith because test
3668 // doesn't support sign-extension of
3669 // 8bit operands
3670 int encode = dst->encoding();
3671 if (encode == 0) {
3672 emit_int8((unsigned char)0xA9);
3673 } else {
3674 encode = prefix_and_encode(encode);
3675 emit_int8((unsigned char)0xF7);
3676 emit_int8((unsigned char)(0xC0 | encode));
3677 }
3678 emit_int32(imm32);
3679 }
3680
3681 void Assembler::testl(Register dst, Register src) {
3682 (void) prefix_and_encode(dst->encoding(), src->encoding());
3683 emit_arith(0x85, 0xC0, dst, src);
3684 }
3685
3686 void Assembler::testl(Register dst, Address src) {
3687 InstructionMark im(this);
3688 prefix(src, dst);
3689 emit_int8((unsigned char)0x85);
3690 emit_operand(dst, src);
3691 }
3692
3693 void Assembler::tzcntl(Register dst, Register src) {
3694 assert(VM_Version::supports_bmi1(), "tzcnt instruction not supported");
3695 emit_int8((unsigned char)0xF3);
3696 int encode = prefix_and_encode(dst->encoding(), src->encoding());
3697 emit_int8(0x0F);
3698 emit_int8((unsigned char)0xBC);
3699 emit_int8((unsigned char)0xC0 | encode);
3700 }
3701
3702 void Assembler::tzcntq(Register dst, Register src) {
3703 assert(VM_Version::supports_bmi1(), "tzcnt instruction not supported");
3704 emit_int8((unsigned char)0xF3);
3705 int encode = prefixq_and_encode(dst->encoding(), src->encoding());
3706 emit_int8(0x0F);
3707 emit_int8((unsigned char)0xBC);
3708 emit_int8((unsigned char)(0xC0 | encode));
3709 }
3710
3711 void Assembler::ucomisd(XMMRegister dst, Address src) {
3712 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3713 if (VM_Version::supports_evex()) {
3714 _tuple_type = EVEX_T1S;
3715 _input_size_in_bits = EVEX_64bit;
3716 emit_simd_arith_nonds_q(0x2E, dst, src, VEX_SIMD_66, /* no_mask_reg */ true);
3717 } else {
3718 emit_simd_arith_nonds(0x2E, dst, src, VEX_SIMD_66);
3719 }
3720 }
3721
3722 void Assembler::ucomisd(XMMRegister dst, XMMRegister src) {
3723 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3724 if (VM_Version::supports_evex()) {
3725 emit_simd_arith_nonds_q(0x2E, dst, src, VEX_SIMD_66, /* no_mask_reg */ true);
3726 } else {
3727 emit_simd_arith_nonds(0x2E, dst, src, VEX_SIMD_66);
3728 }
3729 }
3730
3731 void Assembler::ucomiss(XMMRegister dst, Address src) {
3732 NOT_LP64(assert(VM_Version::supports_sse(), ""));
3733 if (VM_Version::supports_evex()) {
3734 _tuple_type = EVEX_T1S;
3735 _input_size_in_bits = EVEX_32bit;
3736 }
3737 emit_simd_arith_nonds(0x2E, dst, src, VEX_SIMD_NONE, /* no_mask_reg */ true);
3738 }
3739
3740 void Assembler::ucomiss(XMMRegister dst, XMMRegister src) {
3741 NOT_LP64(assert(VM_Version::supports_sse(), ""));
3742 emit_simd_arith_nonds(0x2E, dst, src, VEX_SIMD_NONE, /* no_mask_reg */ true);
3743 }
3744
3745 void Assembler::xabort(int8_t imm8) {
3746 emit_int8((unsigned char)0xC6);
3747 emit_int8((unsigned char)0xF8);
3748 emit_int8((unsigned char)(imm8 & 0xFF));
3749 }
3750
3751 void Assembler::xaddl(Address dst, Register src) {
3752 InstructionMark im(this);
3753 prefix(dst, src);
3754 emit_int8(0x0F);
3755 emit_int8((unsigned char)0xC1);
3756 emit_operand(src, dst);
3757 }
3758
3759 void Assembler::xbegin(Label& abort, relocInfo::relocType rtype) {
3760 InstructionMark im(this);
3761 relocate(rtype);
3762 if (abort.is_bound()) {
3763 address entry = target(abort);
3764 assert(entry != NULL, "abort entry NULL");
3765 intptr_t offset = entry - pc();
3766 emit_int8((unsigned char)0xC7);
3767 emit_int8((unsigned char)0xF8);
3768 emit_int32(offset - 6); // 2 opcode + 4 address
3769 } else {
3770 abort.add_patch_at(code(), locator());
3771 emit_int8((unsigned char)0xC7);
3772 emit_int8((unsigned char)0xF8);
3773 emit_int32(0);
3774 }
3775 }
3776
3777 void Assembler::xchgl(Register dst, Address src) { // xchg
3778 InstructionMark im(this);
3779 prefix(src, dst);
3780 emit_int8((unsigned char)0x87);
3781 emit_operand(dst, src);
3782 }
3783
3784 void Assembler::xchgl(Register dst, Register src) {
3785 int encode = prefix_and_encode(dst->encoding(), src->encoding());
3786 emit_int8((unsigned char)0x87);
3787 emit_int8((unsigned char)(0xC0 | encode));
3788 }
3789
3790 void Assembler::xend() {
3791 emit_int8((unsigned char)0x0F);
3792 emit_int8((unsigned char)0x01);
3793 emit_int8((unsigned char)0xD5);
3794 }
3795
3796 void Assembler::xgetbv() {
3797 emit_int8(0x0F);
3798 emit_int8(0x01);
3799 emit_int8((unsigned char)0xD0);
3800 }
3801
3802 void Assembler::xorl(Register dst, int32_t imm32) {
3803 prefix(dst);
3804 emit_arith(0x81, 0xF0, dst, imm32);
3805 }
3806
3807 void Assembler::xorl(Register dst, Address src) {
3808 InstructionMark im(this);
3809 prefix(src, dst);
3810 emit_int8(0x33);
3811 emit_operand(dst, src);
3812 }
3813
3814 void Assembler::xorl(Register dst, Register src) {
3815 (void) prefix_and_encode(dst->encoding(), src->encoding());
3816 emit_arith(0x33, 0xC0, dst, src);
3817 }
3818
3819
3820 // AVX 3-operands scalar float-point arithmetic instructions
3821
3822 void Assembler::vaddsd(XMMRegister dst, XMMRegister nds, Address src) {
3823 assert(VM_Version::supports_avx(), "");
3824 if (VM_Version::supports_evex()) {
3825 _tuple_type = EVEX_T1S;
3826 _input_size_in_bits = EVEX_64bit;
3827 emit_vex_arith_q(0x58, dst, nds, src, VEX_SIMD_F2, AVX_128bit);
3828 } else {
3829 emit_vex_arith(0x58, dst, nds, src, VEX_SIMD_F2, AVX_128bit);
3830 }
3831 }
3832
3833 void Assembler::vaddsd(XMMRegister dst, XMMRegister nds, XMMRegister src) {
3834 assert(VM_Version::supports_avx(), "");
3835 if (VM_Version::supports_evex()) {
3836 emit_vex_arith_q(0x58, dst, nds, src, VEX_SIMD_F2, AVX_128bit);
3837 } else {
3838 emit_vex_arith(0x58, dst, nds, src, VEX_SIMD_F2, AVX_128bit);
3839 }
3840 }
3841
3842 void Assembler::vaddss(XMMRegister dst, XMMRegister nds, Address src) {
3843 assert(VM_Version::supports_avx(), "");
3844 if (VM_Version::supports_evex()) {
3845 _tuple_type = EVEX_T1S;
3846 _input_size_in_bits = EVEX_32bit;
3847 }
3848 emit_vex_arith(0x58, dst, nds, src, VEX_SIMD_F3, AVX_128bit);
3849 }
3850
3851 void Assembler::vaddss(XMMRegister dst, XMMRegister nds, XMMRegister src) {
3852 assert(VM_Version::supports_avx(), "");
3853 emit_vex_arith(0x58, dst, nds, src, VEX_SIMD_F3, AVX_128bit);
3854 }
3855
3856 void Assembler::vdivsd(XMMRegister dst, XMMRegister nds, Address src) {
3857 assert(VM_Version::supports_avx(), "");
3858 if (VM_Version::supports_evex()) {
3859 _tuple_type = EVEX_T1S;
3860 _input_size_in_bits = EVEX_64bit;
3861 emit_vex_arith_q(0x5E, dst, nds, src, VEX_SIMD_F2, AVX_128bit);
3862 } else {
3863 emit_vex_arith(0x5E, dst, nds, src, VEX_SIMD_F2, AVX_128bit);
3864 }
3865 }
3866
3867 void Assembler::vdivsd(XMMRegister dst, XMMRegister nds, XMMRegister src) {
3868 assert(VM_Version::supports_avx(), "");
3869 if (VM_Version::supports_evex()) {
3870 emit_vex_arith_q(0x5E, dst, nds, src, VEX_SIMD_F2, AVX_128bit);
3871 } else {
3872 emit_vex_arith(0x5E, dst, nds, src, VEX_SIMD_F2, AVX_128bit);
3873 }
3874 }
3875
3876 void Assembler::vdivss(XMMRegister dst, XMMRegister nds, Address src) {
3877 assert(VM_Version::supports_avx(), "");
3878 if (VM_Version::supports_evex()) {
3879 _tuple_type = EVEX_T1S;
3880 _input_size_in_bits = EVEX_32bit;
3881 }
3882 emit_vex_arith(0x5E, dst, nds, src, VEX_SIMD_F3, AVX_128bit);
3883 }
3884
3885 void Assembler::vdivss(XMMRegister dst, XMMRegister nds, XMMRegister src) {
3886 assert(VM_Version::supports_avx(), "");
3887 emit_vex_arith(0x5E, dst, nds, src, VEX_SIMD_F3, AVX_128bit);
3888 }
3889
3890 void Assembler::vmulsd(XMMRegister dst, XMMRegister nds, Address src) {
3891 assert(VM_Version::supports_avx(), "");
3892 if (VM_Version::supports_evex()) {
3893 _tuple_type = EVEX_T1S;
3894 _input_size_in_bits = EVEX_64bit;
3895 emit_vex_arith_q(0x59, dst, nds, src, VEX_SIMD_F2, AVX_128bit);
3896 } else {
3897 emit_vex_arith(0x59, dst, nds, src, VEX_SIMD_F2, AVX_128bit);
3898 }
3899 }
3900
3901 void Assembler::vmulsd(XMMRegister dst, XMMRegister nds, XMMRegister src) {
3902 assert(VM_Version::supports_avx(), "");
3903 if (VM_Version::supports_evex()) {
3904 emit_vex_arith_q(0x59, dst, nds, src, VEX_SIMD_F2, AVX_128bit);
3905 } else {
3906 emit_vex_arith(0x59, dst, nds, src, VEX_SIMD_F2, AVX_128bit);
3907 }
3908 }
3909
3910 void Assembler::vmulss(XMMRegister dst, XMMRegister nds, Address src) {
3911 assert(VM_Version::supports_avx(), "");
3912 if (VM_Version::supports_evex()) {
3913 _tuple_type = EVEX_T1S;
3914 _input_size_in_bits = EVEX_32bit;
3915 }
3916 emit_vex_arith(0x59, dst, nds, src, VEX_SIMD_F3, AVX_128bit);
3917 }
3918
3919 void Assembler::vmulss(XMMRegister dst, XMMRegister nds, XMMRegister src) {
3920 assert(VM_Version::supports_avx(), "");
3921 emit_vex_arith(0x59, dst, nds, src, VEX_SIMD_F3, AVX_128bit);
3922 }
3923
3924 void Assembler::vsubsd(XMMRegister dst, XMMRegister nds, Address src) {
3925 assert(VM_Version::supports_avx(), "");
3926 if (VM_Version::supports_evex()) {
3927 _tuple_type = EVEX_T1S;
3928 _input_size_in_bits = EVEX_64bit;
3929 emit_vex_arith_q(0x5C, dst, nds, src, VEX_SIMD_F2, AVX_128bit);
3930 } else {
3931 emit_vex_arith(0x5C, dst, nds, src, VEX_SIMD_F2, AVX_128bit);
3932 }
3933 }
3934
3935 void Assembler::vsubsd(XMMRegister dst, XMMRegister nds, XMMRegister src) {
3936 assert(VM_Version::supports_avx(), "");
3937 if (VM_Version::supports_evex()) {
3938 emit_vex_arith_q(0x5C, dst, nds, src, VEX_SIMD_F2, AVX_128bit);
3939 } else {
3940 emit_vex_arith(0x5C, dst, nds, src, VEX_SIMD_F2, AVX_128bit);
3941 }
3942 }
3943
3944 void Assembler::vsubss(XMMRegister dst, XMMRegister nds, Address src) {
3945 assert(VM_Version::supports_avx(), "");
3946 if (VM_Version::supports_evex()) {
3947 _tuple_type = EVEX_T1S;
3948 _input_size_in_bits = EVEX_32bit;
3949 }
3950 emit_vex_arith(0x5C, dst, nds, src, VEX_SIMD_F3, AVX_128bit);
3951 }
3952
3953 void Assembler::vsubss(XMMRegister dst, XMMRegister nds, XMMRegister src) {
3954 assert(VM_Version::supports_avx(), "");
3955 emit_vex_arith(0x5C, dst, nds, src, VEX_SIMD_F3, AVX_128bit);
3956 }
3957
3958 //====================VECTOR ARITHMETIC=====================================
3959
3960 // Float-point vector arithmetic
3961
3962 void Assembler::addpd(XMMRegister dst, XMMRegister src) {
3963 _instruction_uses_vl = true;
3964 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3965 if (VM_Version::supports_evex()) {
3966 emit_simd_arith_q(0x58, dst, src, VEX_SIMD_66);
3967 } else {
3968 emit_simd_arith(0x58, dst, src, VEX_SIMD_66);
3969 }
3970 }
3971
3972 void Assembler::addps(XMMRegister dst, XMMRegister src) {
3973 _instruction_uses_vl = true;
3974 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3975 emit_simd_arith(0x58, dst, src, VEX_SIMD_NONE);
3976 }
3977
3978 void Assembler::vaddpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
3979 _instruction_uses_vl = true;
3980 assert(VM_Version::supports_avx(), "");
3981 if (VM_Version::supports_evex()) {
3982 emit_vex_arith_q(0x58, dst, nds, src, VEX_SIMD_66, vector_len);
3983 } else {
3984 emit_vex_arith(0x58, dst, nds, src, VEX_SIMD_66, vector_len);
3985 }
3986 }
3987
3988 void Assembler::vaddps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
3989 _instruction_uses_vl = true;
3990 assert(VM_Version::supports_avx(), "");
3991 emit_vex_arith(0x58, dst, nds, src, VEX_SIMD_NONE, vector_len);
3992 }
3993
3994 void Assembler::vaddpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
3995 _instruction_uses_vl = true;
3996 assert(VM_Version::supports_avx(), "");
3997 if (VM_Version::supports_evex()) {
3998 _tuple_type = EVEX_FV;
3999 _input_size_in_bits = EVEX_64bit;
4000 emit_vex_arith_q(0x58, dst, nds, src, VEX_SIMD_66, vector_len);
4001 } else {
4002 emit_vex_arith(0x58, dst, nds, src, VEX_SIMD_66, vector_len);
4003 }
4004 }
4005
4006 void Assembler::vaddps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4007 _instruction_uses_vl = true;
4008 assert(VM_Version::supports_avx(), "");
4009 if (VM_Version::supports_evex()) {
4010 _tuple_type = EVEX_FV;
4011 _input_size_in_bits = EVEX_32bit;
4012 }
4013 emit_vex_arith(0x58, dst, nds, src, VEX_SIMD_NONE, vector_len);
4014 }
4015
4016 void Assembler::subpd(XMMRegister dst, XMMRegister src) {
4017 _instruction_uses_vl = true;
4018 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4019 if (VM_Version::supports_evex()) {
4020 emit_simd_arith_q(0x5C, dst, src, VEX_SIMD_66);
4021 } else {
4022 emit_simd_arith(0x5C, dst, src, VEX_SIMD_66);
4023 }
4024 }
4025
4026 void Assembler::subps(XMMRegister dst, XMMRegister src) {
4027 _instruction_uses_vl = true;
4028 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4029 emit_simd_arith(0x5C, dst, src, VEX_SIMD_NONE);
4030 }
4031
4032 void Assembler::vsubpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4033 _instruction_uses_vl = true;
4034 assert(VM_Version::supports_avx(), "");
4035 if (VM_Version::supports_evex()) {
4036 emit_vex_arith_q(0x5C, dst, nds, src, VEX_SIMD_66, vector_len);
4037 } else {
4038 emit_vex_arith(0x5C, dst, nds, src, VEX_SIMD_66, vector_len);
4039 }
4040 }
4041
4042 void Assembler::vsubps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4043 _instruction_uses_vl = true;
4044 assert(VM_Version::supports_avx(), "");
4045 emit_vex_arith(0x5C, dst, nds, src, VEX_SIMD_NONE, vector_len);
4046 }
4047
4048 void Assembler::vsubpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4049 _instruction_uses_vl = true;
4050 assert(VM_Version::supports_avx(), "");
4051 if (VM_Version::supports_evex()) {
4052 _tuple_type = EVEX_FV;
4053 _input_size_in_bits = EVEX_64bit;
4054 emit_vex_arith_q(0x5C, dst, nds, src, VEX_SIMD_66, vector_len);
4055 } else {
4056 emit_vex_arith(0x5C, dst, nds, src, VEX_SIMD_66, vector_len);
4057 }
4058 }
4059
4060 void Assembler::vsubps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4061 _instruction_uses_vl = true;
4062 assert(VM_Version::supports_avx(), "");
4063 if (VM_Version::supports_evex()) {
4064 _tuple_type = EVEX_FV;
4065 _input_size_in_bits = EVEX_32bit;
4066 }
4067 emit_vex_arith(0x5C, dst, nds, src, VEX_SIMD_NONE, vector_len);
4068 }
4069
4070 void Assembler::mulpd(XMMRegister dst, XMMRegister src) {
4071 _instruction_uses_vl = true;
4072 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4073 if (VM_Version::supports_evex()) {
4074 emit_simd_arith_q(0x59, dst, src, VEX_SIMD_66);
4075 } else {
4076 emit_simd_arith(0x59, dst, src, VEX_SIMD_66);
4077 }
4078 }
4079
4080 void Assembler::mulps(XMMRegister dst, XMMRegister src) {
4081 _instruction_uses_vl = true;
4082 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4083 emit_simd_arith(0x59, dst, src, VEX_SIMD_NONE);
4084 }
4085
4086 void Assembler::vmulpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4087 _instruction_uses_vl = true;
4088 assert(VM_Version::supports_avx(), "");
4089 if (VM_Version::supports_evex()) {
4090 emit_vex_arith_q(0x59, dst, nds, src, VEX_SIMD_66, vector_len);
4091 } else {
4092 emit_vex_arith(0x59, dst, nds, src, VEX_SIMD_66, vector_len);
4093 }
4094 }
4095
4096 void Assembler::vmulps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4097 _instruction_uses_vl = true;
4098 assert(VM_Version::supports_avx(), "");
4099 emit_vex_arith(0x59, dst, nds, src, VEX_SIMD_NONE, vector_len);
4100 }
4101
4102 void Assembler::vmulpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4103 _instruction_uses_vl = true;
4104 assert(VM_Version::supports_avx(), "");
4105 if (VM_Version::supports_evex()) {
4106 _tuple_type = EVEX_FV;
4107 _input_size_in_bits = EVEX_64bit;
4108 emit_vex_arith_q(0x59, dst, nds, src, VEX_SIMD_66, vector_len);
4109 } else {
4110 emit_vex_arith(0x59, dst, nds, src, VEX_SIMD_66, vector_len);
4111 }
4112 }
4113
4114 void Assembler::vmulps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4115 _instruction_uses_vl = true;
4116 assert(VM_Version::supports_avx(), "");
4117 if (VM_Version::supports_evex()) {
4118 _tuple_type = EVEX_FV;
4119 _input_size_in_bits = EVEX_32bit;
4120 }
4121 emit_vex_arith(0x59, dst, nds, src, VEX_SIMD_NONE, vector_len);
4122 }
4123
4124 void Assembler::divpd(XMMRegister dst, XMMRegister src) {
4125 _instruction_uses_vl = true;
4126 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4127 if (VM_Version::supports_evex()) {
4128 emit_simd_arith_q(0x5E, dst, src, VEX_SIMD_66);
4129 } else {
4130 emit_simd_arith(0x5E, dst, src, VEX_SIMD_66);
4131 }
4132 }
4133
4134 void Assembler::divps(XMMRegister dst, XMMRegister src) {
4135 _instruction_uses_vl = true;
4136 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4137 emit_simd_arith(0x5E, dst, src, VEX_SIMD_NONE);
4138 }
4139
4140 void Assembler::vdivpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4141 _instruction_uses_vl = true;
4142 assert(VM_Version::supports_avx(), "");
4143 if (VM_Version::supports_evex()) {
4144 emit_vex_arith_q(0x5E, dst, nds, src, VEX_SIMD_66, vector_len);
4145 } else {
4146 emit_vex_arith(0x5E, dst, nds, src, VEX_SIMD_66, vector_len);
4147 }
4148 }
4149
4150 void Assembler::vdivps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4151 _instruction_uses_vl = true;
4152 assert(VM_Version::supports_avx(), "");
4153 emit_vex_arith(0x5E, dst, nds, src, VEX_SIMD_NONE, vector_len);
4154 }
4155
4156 void Assembler::vdivpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4157 _instruction_uses_vl = true;
4158 assert(VM_Version::supports_avx(), "");
4159 if (VM_Version::supports_evex()) {
4160 _tuple_type = EVEX_FV;
4161 _input_size_in_bits = EVEX_64bit;
4162 emit_vex_arith_q(0x5E, dst, nds, src, VEX_SIMD_66, vector_len);
4163 } else {
4164 emit_vex_arith(0x5E, dst, nds, src, VEX_SIMD_66, vector_len);
4165 }
4166 }
4167
4168 void Assembler::vdivps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4169 _instruction_uses_vl = true;
4170 assert(VM_Version::supports_avx(), "");
4171 if (VM_Version::supports_evex()) {
4172 _tuple_type = EVEX_FV;
4173 _input_size_in_bits = EVEX_32bit;
4174 }
4175 emit_vex_arith(0x5E, dst, nds, src, VEX_SIMD_NONE, vector_len);
4176 }
4177
4178 void Assembler::vsqrtpd(XMMRegister dst, XMMRegister src, int vector_len) {
4179 _instruction_uses_vl = true;
4180 assert(VM_Version::supports_avx(), "");
4181 if (VM_Version::supports_evex()) {
4182 emit_vex_arith_q(0x51, dst, xnoreg, src, VEX_SIMD_66, vector_len);
4183 } else {
4184 emit_vex_arith(0x51, dst, xnoreg, src, VEX_SIMD_66, vector_len);
4185 }
4186 }
4187
4188 void Assembler::vsqrtpd(XMMRegister dst, Address src, int vector_len) {
4189 _instruction_uses_vl = true;
4190 assert(VM_Version::supports_avx(), "");
4191 if (VM_Version::supports_evex()) {
4192 _tuple_type = EVEX_FV;
4193 _input_size_in_bits = EVEX_64bit;
4194 emit_vex_arith_q(0x51, dst, xnoreg, src, VEX_SIMD_66, vector_len);
4195 } else {
4196 emit_vex_arith(0x51, dst, xnoreg, src, VEX_SIMD_66, vector_len);
4197 }
4198 }
4199
4200 void Assembler::andpd(XMMRegister dst, XMMRegister src) {
4201 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4202 if (VM_Version::supports_avx512dq()) {
4203 emit_simd_arith_q(0x54, dst, src, VEX_SIMD_66);
4204 } else {
4205 emit_simd_arith(0x54, dst, src, VEX_SIMD_66, /* no_mask_reg */ false, /* legacy_mode */ true);
4206 }
4207 }
4208
4209 void Assembler::andps(XMMRegister dst, XMMRegister src) {
4210 NOT_LP64(assert(VM_Version::supports_sse(), ""));
4211 emit_simd_arith(0x54, dst, src, VEX_SIMD_NONE, /* no_mask_reg */ false, /* legacy_mode */ _legacy_mode_dq);
4212 }
4213
4214 void Assembler::andps(XMMRegister dst, Address src) {
4215 NOT_LP64(assert(VM_Version::supports_sse(), ""));
4216 if (VM_Version::supports_evex()) {
4217 _tuple_type = EVEX_FV;
4218 _input_size_in_bits = EVEX_32bit;
4219 }
4220 emit_simd_arith(0x54, dst, src, VEX_SIMD_NONE, /* no_mask_reg */ false, /* legacy_mode */ _legacy_mode_dq);
4221 }
4222
4223 void Assembler::andpd(XMMRegister dst, Address src) {
4224 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4225 if (VM_Version::supports_avx512dq()) {
4226 _tuple_type = EVEX_FV;
4227 _input_size_in_bits = EVEX_64bit;
4228 emit_simd_arith_q(0x54, dst, src, VEX_SIMD_66);
4229 } else {
4230 emit_simd_arith(0x54, dst, src, VEX_SIMD_66, /* no_mask_reg */ false, /* legacy_mode */ true);
4231 }
4232 }
4233
4234 void Assembler::vandpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4235 assert(VM_Version::supports_avx(), "");
4236 if (VM_Version::supports_avx512dq()) {
4237 emit_vex_arith_q(0x54, dst, nds, src, VEX_SIMD_66, vector_len);
4238 } else {
4239 emit_vex_arith(0x54, dst, nds, src, VEX_SIMD_66, vector_len, /* no_mask_reg */ false, /* legacy_mode */ true);
4240 }
4241 }
4242
4243 void Assembler::vandps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4244 assert(VM_Version::supports_avx(), "");
4245 emit_vex_arith(0x54, dst, nds, src, VEX_SIMD_NONE, vector_len, /* no_mask_reg */ false, /* legacy_mode */ _legacy_mode_dq);
4246 }
4247
4248 void Assembler::vandpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4249 assert(VM_Version::supports_avx(), "");
4250 if (VM_Version::supports_avx512dq()) {
4251 _tuple_type = EVEX_FV;
4252 _input_size_in_bits = EVEX_64bit;
4253 emit_vex_arith_q(0x54, dst, nds, src, VEX_SIMD_66, vector_len);
4254 } else {
4255 emit_vex_arith(0x54, dst, nds, src, VEX_SIMD_66, vector_len, /* no_mask_reg */ false, /* legacy_mode */ true);
4256 }
4257 }
4258
4259 void Assembler::vandps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4260 assert(VM_Version::supports_avx(), "");
4261 if (VM_Version::supports_evex()) {
4262 _tuple_type = EVEX_FV;
4263 _input_size_in_bits = EVEX_32bit;
4264 }
4265 emit_vex_arith(0x54, dst, nds, src, VEX_SIMD_NONE, vector_len, /* no_mask_reg */ false, /* legacy_mode */ _legacy_mode_dq);
4266 }
4267
4268 void Assembler::xorpd(XMMRegister dst, XMMRegister src) {
4269 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4270 if (VM_Version::supports_avx512dq()) {
4271 emit_simd_arith_q(0x57, dst, src, VEX_SIMD_66);
4272 } else {
4273 emit_simd_arith(0x57, dst, src, VEX_SIMD_66, /* no_mask_reg */ false, /* legacy_mode */ true);
4274 }
4275 }
4276
4277 void Assembler::xorps(XMMRegister dst, XMMRegister src) {
4278 NOT_LP64(assert(VM_Version::supports_sse(), ""));
4279 emit_simd_arith(0x57, dst, src, VEX_SIMD_NONE, /* no_mask_reg */ false, /* legacy_mode */ _legacy_mode_dq);
4280 }
4281
4282 void Assembler::xorpd(XMMRegister dst, Address src) {
4283 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4284 if (VM_Version::supports_avx512dq()) {
4285 _tuple_type = EVEX_FV;
4286 _input_size_in_bits = EVEX_64bit;
4287 emit_simd_arith_q(0x57, dst, src, VEX_SIMD_66);
4288 } else {
4289 emit_simd_arith(0x57, dst, src, VEX_SIMD_66, /* no_mask_reg */ false, /* legacy_mode */ true);
4290 }
4291 }
4292
4293 void Assembler::xorps(XMMRegister dst, Address src) {
4294 NOT_LP64(assert(VM_Version::supports_sse(), ""));
4295 if (VM_Version::supports_evex()) {
4296 _tuple_type = EVEX_FV;
4297 _input_size_in_bits = EVEX_32bit;
4298 }
4299 emit_simd_arith(0x57, dst, src, VEX_SIMD_NONE, /* no_mask_reg */ false, /* legacy_mode */ _legacy_mode_dq);
4300 }
4301
4302 void Assembler::vxorpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4303 assert(VM_Version::supports_avx(), "");
4304 if (VM_Version::supports_avx512dq()) {
4305 emit_vex_arith_q(0x57, dst, nds, src, VEX_SIMD_66, vector_len);
4306 } else {
4307 emit_vex_arith(0x57, dst, nds, src, VEX_SIMD_66, vector_len, /* no_mask_reg */ false, /* legacy_mode */ true);
4308 }
4309 }
4310
4311 void Assembler::vxorps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4312 assert(VM_Version::supports_avx(), "");
4313 emit_vex_arith(0x57, dst, nds, src, VEX_SIMD_NONE, vector_len, /* no_mask_reg */ false, /* legacy_mode */ _legacy_mode_dq);
4314 }
4315
4316 void Assembler::vxorpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4317 assert(VM_Version::supports_avx(), "");
4318 if (VM_Version::supports_avx512dq()) {
4319 _tuple_type = EVEX_FV;
4320 _input_size_in_bits = EVEX_64bit;
4321 emit_vex_arith_q(0x57, dst, nds, src, VEX_SIMD_66, vector_len);
4322 } else {
4323 emit_vex_arith(0x57, dst, nds, src, VEX_SIMD_66, vector_len, /* no_mask_reg */ false, /* legacy_mode */ true);
4324 }
4325 }
4326
4327 void Assembler::vxorps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4328 assert(VM_Version::supports_avx(), "");
4329 if (VM_Version::supports_evex()) {
4330 _tuple_type = EVEX_FV;
4331 _input_size_in_bits = EVEX_32bit;
4332 }
4333 emit_vex_arith(0x57, dst, nds, src, VEX_SIMD_NONE, vector_len, /* no_mask_reg */ false, /* legacy_mode */ _legacy_mode_dq);
4334 }
4335
4336 // Integer vector arithmetic
4337 void Assembler::vphaddw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4338 assert(VM_Version::supports_avx() && (vector_len == 0) ||
4339 VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
4340 int encode = vex_prefix_and_encode(dst, nds, src, VEX_SIMD_66, vector_len, VEX_OPCODE_0F_38, /* legacy_mode */ true);
4341 emit_int8(0x01);
4342 emit_int8((unsigned char)(0xC0 | encode));
4343 }
4344
4345 void Assembler::vphaddd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4346 assert(VM_Version::supports_avx() && (vector_len == 0) ||
4347 VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
4348 int encode = vex_prefix_and_encode(dst, nds, src, VEX_SIMD_66, vector_len, VEX_OPCODE_0F_38, /* legacy_mode */ true);
4349 emit_int8(0x02);
4350 emit_int8((unsigned char)(0xC0 | encode));
4351 }
4352
4353 void Assembler::paddb(XMMRegister dst, XMMRegister src) {
4354 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4355 emit_simd_arith(0xFC, dst, src, VEX_SIMD_66, /* no_mask_reg */ false, /* legacy_mode */ _legacy_mode_bw);
4356 }
4357
4358 void Assembler::paddw(XMMRegister dst, XMMRegister src) {
4359 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4360 emit_simd_arith(0xFD, dst, src, VEX_SIMD_66, /* no_mask_reg */ false, /* legacy_mode */ _legacy_mode_bw);
4361 }
4362
4363 void Assembler::paddd(XMMRegister dst, XMMRegister src) {
4364 _instruction_uses_vl = true;
4365 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4366 emit_simd_arith(0xFE, dst, src, VEX_SIMD_66);
4367 }
4368
4369 void Assembler::paddq(XMMRegister dst, XMMRegister src) {
4370 _instruction_uses_vl = true;
4371 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4372 if (VM_Version::supports_evex()) {
4373 emit_simd_arith_q(0xD4, dst, src, VEX_SIMD_66);
4374 } else {
4375 emit_simd_arith(0xD4, dst, src, VEX_SIMD_66);
4376 }
4377 }
4378
4379 void Assembler::phaddw(XMMRegister dst, XMMRegister src) {
4380 NOT_LP64(assert(VM_Version::supports_sse3(), ""));
4381 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, /* no_mask_reg */ false,
4382 VEX_OPCODE_0F_38, /* rex_w */ false, AVX_128bit, /* legacy_mode */ true);
4383 emit_int8(0x01);
4384 emit_int8((unsigned char)(0xC0 | encode));
4385 }
4386
4387 void Assembler::phaddd(XMMRegister dst, XMMRegister src) {
4388 NOT_LP64(assert(VM_Version::supports_sse3(), ""));
4389 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, /* no_mask_reg */ false,
4390 VEX_OPCODE_0F_38, /* rex_w */ false, AVX_128bit, /* legacy_mode */ true);
4391 emit_int8(0x02);
4392 emit_int8((unsigned char)(0xC0 | encode));
4393 }
4394
4395 void Assembler::vpaddb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4396 assert(UseAVX > 0, "requires some form of AVX");
4397 emit_vex_arith(0xFC, dst, nds, src, VEX_SIMD_66, vector_len, /* no_mask_reg */ false, /* legacy_mode */ _legacy_mode_bw);
4398 }
4399
4400 void Assembler::vpaddw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4401 assert(UseAVX > 0, "requires some form of AVX");
4402 emit_vex_arith(0xFD, dst, nds, src, VEX_SIMD_66, vector_len, /* no_mask_reg */ false, /* legacy_mode */ _legacy_mode_bw);
4403 }
4404
4405 void Assembler::vpaddd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4406 _instruction_uses_vl = true;
4407 assert(UseAVX > 0, "requires some form of AVX");
4408 emit_vex_arith(0xFE, dst, nds, src, VEX_SIMD_66, vector_len);
4409 }
4410
4411 void Assembler::vpaddq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4412 _instruction_uses_vl = true;
4413 assert(UseAVX > 0, "requires some form of AVX");
4414 if (VM_Version::supports_evex()) {
4415 emit_vex_arith_q(0xD4, dst, nds, src, VEX_SIMD_66, vector_len);
4416 } else {
4417 emit_vex_arith(0xD4, dst, nds, src, VEX_SIMD_66, vector_len);
4418 }
4419 }
4420
4421 void Assembler::vpaddb(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4422 assert(UseAVX > 0, "requires some form of AVX");
4423 if (VM_Version::supports_evex()) {
4424 _tuple_type = EVEX_FVM;
4425 }
4426 emit_vex_arith(0xFC, dst, nds, src, VEX_SIMD_66, vector_len, /* no_mask_reg */ false, /* legacy_mode */ _legacy_mode_bw);
4427 }
4428
4429 void Assembler::vpaddw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4430 assert(UseAVX > 0, "requires some form of AVX");
4431 if (VM_Version::supports_evex()) {
4432 _tuple_type = EVEX_FVM;
4433 }
4434 emit_vex_arith(0xFD, dst, nds, src, VEX_SIMD_66, vector_len, /* no_mask_reg */ false, /* legacy_mode */ _legacy_mode_bw);
4435 }
4436
4437 void Assembler::vpaddd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4438 _instruction_uses_vl = true;
4439 assert(UseAVX > 0, "requires some form of AVX");
4440 if (VM_Version::supports_evex()) {
4441 _tuple_type = EVEX_FV;
4442 _input_size_in_bits = EVEX_32bit;
4443 }
4444 emit_vex_arith(0xFE, dst, nds, src, VEX_SIMD_66, vector_len);
4445 }
4446
4447 void Assembler::vpaddq(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4448 _instruction_uses_vl = true;
4449 assert(UseAVX > 0, "requires some form of AVX");
4450 if (VM_Version::supports_evex()) {
4451 _tuple_type = EVEX_FV;
4452 _input_size_in_bits = EVEX_64bit;
4453 emit_vex_arith_q(0xD4, dst, nds, src, VEX_SIMD_66, vector_len);
4454 } else {
4455 emit_vex_arith(0xD4, dst, nds, src, VEX_SIMD_66, vector_len);
4456 }
4457 }
4458
4459 void Assembler::psubb(XMMRegister dst, XMMRegister src) {
4460 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4461 emit_simd_arith(0xF8, dst, src, VEX_SIMD_66, /* no_mask_reg */ false, /* legacy_mode */ _legacy_mode_bw);
4462 }
4463
4464 void Assembler::psubw(XMMRegister dst, XMMRegister src) {
4465 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4466 emit_simd_arith(0xF9, dst, src, VEX_SIMD_66, /* no_mask_reg */ false, /* legacy_mode */ _legacy_mode_bw);
4467 }
4468
4469 void Assembler::psubd(XMMRegister dst, XMMRegister src) {
4470 _instruction_uses_vl = true;
4471 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4472 emit_simd_arith(0xFA, dst, src, VEX_SIMD_66);
4473 }
4474
4475 void Assembler::psubq(XMMRegister dst, XMMRegister src) {
4476 _instruction_uses_vl = true;
4477 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4478 if (VM_Version::supports_evex()) {
4479 emit_simd_arith_q(0xFB, dst, src, VEX_SIMD_66);
4480 } else {
4481 emit_simd_arith(0xFB, dst, src, VEX_SIMD_66);
4482 }
4483 }
4484
4485 void Assembler::vpsubb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4486 assert(UseAVX > 0, "requires some form of AVX");
4487 emit_vex_arith(0xF8, dst, nds, src, VEX_SIMD_66, vector_len, /* no_mask_reg */ false, /* legacy_mode */ _legacy_mode_bw);
4488 }
4489
4490 void Assembler::vpsubw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4491 assert(UseAVX > 0, "requires some form of AVX");
4492 emit_vex_arith(0xF9, dst, nds, src, VEX_SIMD_66, vector_len, /* no_mask_reg */ false, /* legacy_mode */ _legacy_mode_bw);
4493 }
4494
4495 void Assembler::vpsubd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4496 _instruction_uses_vl = true;
4497 assert(UseAVX > 0, "requires some form of AVX");
4498 emit_vex_arith(0xFA, dst, nds, src, VEX_SIMD_66, vector_len);
4499 }
4500
4501 void Assembler::vpsubq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4502 _instruction_uses_vl = true;
4503 assert(UseAVX > 0, "requires some form of AVX");
4504 if (VM_Version::supports_evex()) {
4505 emit_vex_arith_q(0xFB, dst, nds, src, VEX_SIMD_66, vector_len);
4506 } else {
4507 emit_vex_arith(0xFB, dst, nds, src, VEX_SIMD_66, vector_len);
4508 }
4509 }
4510
4511 void Assembler::vpsubb(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4512 assert(UseAVX > 0, "requires some form of AVX");
4513 if (VM_Version::supports_evex()) {
4514 _tuple_type = EVEX_FVM;
4515 }
4516 emit_vex_arith(0xF8, dst, nds, src, VEX_SIMD_66, vector_len, /* no_mask_reg */ false, /* legacy_mode */ _legacy_mode_bw);
4517 }
4518
4519 void Assembler::vpsubw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4520 assert(UseAVX > 0, "requires some form of AVX");
4521 if (VM_Version::supports_evex()) {
4522 _tuple_type = EVEX_FVM;
4523 }
4524 emit_vex_arith(0xF9, dst, nds, src, VEX_SIMD_66, vector_len, /* no_mask_reg */ false, /* legacy_mode */ _legacy_mode_bw);
4525 }
4526
4527 void Assembler::vpsubd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4528 _instruction_uses_vl = true;
4529 assert(UseAVX > 0, "requires some form of AVX");
4530 if (VM_Version::supports_evex()) {
4531 _tuple_type = EVEX_FV;
4532 _input_size_in_bits = EVEX_32bit;
4533 }
4534 emit_vex_arith(0xFA, dst, nds, src, VEX_SIMD_66, vector_len);
4535 }
4536
4537 void Assembler::vpsubq(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4538 _instruction_uses_vl = true;
4539 assert(UseAVX > 0, "requires some form of AVX");
4540 if (VM_Version::supports_evex()) {
4541 _tuple_type = EVEX_FV;
4542 _input_size_in_bits = EVEX_64bit;
4543 emit_vex_arith_q(0xFB, dst, nds, src, VEX_SIMD_66, vector_len);
4544 } else {
4545 emit_vex_arith(0xFB, dst, nds, src, VEX_SIMD_66, vector_len);
4546 }
4547 }
4548
4549 void Assembler::pmullw(XMMRegister dst, XMMRegister src) {
4550 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4551 emit_simd_arith(0xD5, dst, src, VEX_SIMD_66, /* no_mask_reg */ false, /* legacy_mode */ _legacy_mode_bw);
4552 }
4553
4554 void Assembler::pmulld(XMMRegister dst, XMMRegister src) {
4555 _instruction_uses_vl = true;
4556 assert(VM_Version::supports_sse4_1(), "");
4557 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66,
4558 /* no_mask_reg */ false, VEX_OPCODE_0F_38);
4559 emit_int8(0x40);
4560 emit_int8((unsigned char)(0xC0 | encode));
4561 }
4562
4563 void Assembler::vpmullw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4564 assert(UseAVX > 0, "requires some form of AVX");
4565 emit_vex_arith(0xD5, dst, nds, src, VEX_SIMD_66, vector_len, /* no_mask_reg */ false, /* legacy_mode */ _legacy_mode_bw);
4566 }
4567
4568 void Assembler::vpmulld(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4569 _instruction_uses_vl = true;
4570 assert(UseAVX > 0, "requires some form of AVX");
4571 int encode = vex_prefix_and_encode(dst, nds, src, VEX_SIMD_66, vector_len, VEX_OPCODE_0F_38);
4572 emit_int8(0x40);
4573 emit_int8((unsigned char)(0xC0 | encode));
4574 }
4575
4576 void Assembler::vpmullq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4577 assert(UseAVX > 2, "requires some form of AVX");
4578 int src_enc = src->encoding();
4579 int dst_enc = dst->encoding();
4580 int nds_enc = nds->is_valid() ? nds->encoding() : 0;
4581 int encode = vex_prefix_and_encode(dst_enc, nds_enc, src_enc, VEX_SIMD_66, VEX_OPCODE_0F_38,
4582 /* vex_w */ true, vector_len, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false);
4583 emit_int8(0x40);
4584 emit_int8((unsigned char)(0xC0 | encode));
4585 }
4586
4587 void Assembler::vpmullw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4588 assert(UseAVX > 0, "requires some form of AVX");
4589 if (VM_Version::supports_evex()) {
4590 _tuple_type = EVEX_FVM;
4591 }
4592 emit_vex_arith(0xD5, dst, nds, src, VEX_SIMD_66, vector_len, /* no_mask_reg */ false, /* legacy_mode */ _legacy_mode_bw);
4593 }
4594
4595 void Assembler::vpmulld(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4596 _instruction_uses_vl = true;
4597 assert(UseAVX > 0, "requires some form of AVX");
4598 if (VM_Version::supports_evex()) {
4599 _tuple_type = EVEX_FV;
4600 _input_size_in_bits = EVEX_32bit;
4601 }
4602 InstructionMark im(this);
4603 int dst_enc = dst->encoding();
4604 int nds_enc = nds->is_valid() ? nds->encoding() : 0;
4605 vex_prefix(src, nds_enc, dst_enc, VEX_SIMD_66,
4606 VEX_OPCODE_0F_38, /* vex_w */ false, vector_len);
4607 emit_int8(0x40);
4608 emit_operand(dst, src);
4609 }
4610
4611 void Assembler::vpmullq(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4612 assert(UseAVX > 0, "requires some form of AVX");
4613 if (VM_Version::supports_evex()) {
4614 _tuple_type = EVEX_FV;
4615 _input_size_in_bits = EVEX_64bit;
4616 }
4617 InstructionMark im(this);
4618 int dst_enc = dst->encoding();
4619 int nds_enc = nds->is_valid() ? nds->encoding() : 0;
4620 vex_prefix(src, nds_enc, dst_enc, VEX_SIMD_66,
4621 VEX_OPCODE_0F_38, /* vex_w */ true, vector_len, /* legacy_mode */ _legacy_mode_dq);
4622 emit_int8(0x40);
4623 emit_operand(dst, src);
4624 }
4625
4626 // Shift packed integers left by specified number of bits.
4627 void Assembler::psllw(XMMRegister dst, int shift) {
4628 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4629 // XMM6 is for /6 encoding: 66 0F 71 /6 ib
4630 int encode = simd_prefix_and_encode(xmm6, dst, dst, VEX_SIMD_66, /* no_mask_reg */ false, VEX_OPCODE_0F,
4631 /* rex_w */ false, AVX_128bit, /* legacy_mode */ _legacy_mode_bw);
4632 emit_int8(0x71);
4633 emit_int8((unsigned char)(0xC0 | encode));
4634 emit_int8(shift & 0xFF);
4635 }
4636
4637 void Assembler::pslld(XMMRegister dst, int shift) {
4638 _instruction_uses_vl = true;
4639 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4640 // XMM6 is for /6 encoding: 66 0F 72 /6 ib
4641 int encode = simd_prefix_and_encode(xmm6, dst, dst, VEX_SIMD_66, /* no_mask_reg */ false);
4642 emit_int8(0x72);
4643 emit_int8((unsigned char)(0xC0 | encode));
4644 emit_int8(shift & 0xFF);
4645 }
4646
4647 void Assembler::psllq(XMMRegister dst, int shift) {
4648 _instruction_uses_vl = true;
4649 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4650 // XMM6 is for /6 encoding: 66 0F 73 /6 ib
4651 int encode = simd_prefix_and_encode(xmm6, dst, dst, VEX_SIMD_66, /* no_mask_reg */ false, VEX_OPCODE_0F, /* rex_w */ true);
4652 emit_int8(0x73);
4653 emit_int8((unsigned char)(0xC0 | encode));
4654 emit_int8(shift & 0xFF);
4655 }
4656
4657 void Assembler::psllw(XMMRegister dst, XMMRegister shift) {
4658 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4659 emit_simd_arith(0xF1, dst, shift, VEX_SIMD_66, /* no_mask_reg */ false, /* legacy_mode */ _legacy_mode_bw);
4660 }
4661
4662 void Assembler::pslld(XMMRegister dst, XMMRegister shift) {
4663 _instruction_uses_vl = true;
4664 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4665 emit_simd_arith(0xF2, dst, shift, VEX_SIMD_66);
4666 }
4667
4668 void Assembler::psllq(XMMRegister dst, XMMRegister shift) {
4669 _instruction_uses_vl = true;
4670 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4671 if (VM_Version::supports_evex()) {
4672 emit_simd_arith_q(0xF3, dst, shift, VEX_SIMD_66);
4673 } else {
4674 emit_simd_arith(0xF3, dst, shift, VEX_SIMD_66);
4675 }
4676 }
4677
4678 void Assembler::vpsllw(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
4679 assert(UseAVX > 0, "requires some form of AVX");
4680 // XMM6 is for /6 encoding: 66 0F 71 /6 ib
4681 emit_vex_arith(0x71, xmm6, dst, src, VEX_SIMD_66, vector_len, /* no_mask_reg */ false, /* legacy_mode */ _legacy_mode_bw);
4682 emit_int8(shift & 0xFF);
4683 }
4684
4685 void Assembler::vpslld(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
4686 _instruction_uses_vl = true;
4687 assert(UseAVX > 0, "requires some form of AVX");
4688 // XMM6 is for /6 encoding: 66 0F 72 /6 ib
4689 emit_vex_arith(0x72, xmm6, dst, src, VEX_SIMD_66, vector_len);
4690 emit_int8(shift & 0xFF);
4691 }
4692
4693 void Assembler::vpsllq(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
4694 _instruction_uses_vl = true;
4695 assert(UseAVX > 0, "requires some form of AVX");
4696 // XMM6 is for /6 encoding: 66 0F 73 /6 ib
4697 if (VM_Version::supports_evex()) {
4698 emit_vex_arith_q(0x73, xmm6, dst, src, VEX_SIMD_66, vector_len);
4699 } else {
4700 emit_vex_arith(0x73, xmm6, dst, src, VEX_SIMD_66, vector_len);
4701 }
4702 emit_int8(shift & 0xFF);
4703 }
4704
4705 void Assembler::vpsllw(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
4706 assert(UseAVX > 0, "requires some form of AVX");
4707 emit_vex_arith(0xF1, dst, src, shift, VEX_SIMD_66, vector_len, /* no_mask_reg */ false, /* legacy_mode */ _legacy_mode_bw);
4708 }
4709
4710 void Assembler::vpslld(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
4711 _instruction_uses_vl = true;
4712 assert(UseAVX > 0, "requires some form of AVX");
4713 emit_vex_arith(0xF2, dst, src, shift, VEX_SIMD_66, vector_len);
4714 }
4715
4716 void Assembler::vpsllq(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
4717 _instruction_uses_vl = true;
4718 assert(UseAVX > 0, "requires some form of AVX");
4719 if (VM_Version::supports_evex()) {
4720 emit_vex_arith_q(0xF3, dst, src, shift, VEX_SIMD_66, vector_len);
4721 } else {
4722 emit_vex_arith(0xF3, dst, src, shift, VEX_SIMD_66, vector_len);
4723 }
4724 }
4725
4726 // Shift packed integers logically right by specified number of bits.
4727 void Assembler::psrlw(XMMRegister dst, int shift) {
4728 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4729 // XMM2 is for /2 encoding: 66 0F 71 /2 ib
4730 int encode = simd_prefix_and_encode(xmm2, dst, dst, VEX_SIMD_66, /* no_mask_reg */ false,
4731 VEX_OPCODE_0F, /* rex_w */ false, AVX_128bit, /* legacy_mode */ _legacy_mode_bw);
4732 emit_int8(0x71);
4733 emit_int8((unsigned char)(0xC0 | encode));
4734 emit_int8(shift & 0xFF);
4735 }
4736
4737 void Assembler::psrld(XMMRegister dst, int shift) {
4738 _instruction_uses_vl = true;
4739 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4740 // XMM2 is for /2 encoding: 66 0F 72 /2 ib
4741 int encode = simd_prefix_and_encode(xmm2, dst, dst, VEX_SIMD_66, /* no_mask_reg */ false);
4742 emit_int8(0x72);
4743 emit_int8((unsigned char)(0xC0 | encode));
4744 emit_int8(shift & 0xFF);
4745 }
4746
4747 void Assembler::psrlq(XMMRegister dst, int shift) {
4748 _instruction_uses_vl = true;
4749 // Do not confuse it with psrldq SSE2 instruction which
4750 // shifts 128 bit value in xmm register by number of bytes.
4751 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4752 // XMM2 is for /2 encoding: 66 0F 73 /2 ib
4753 int encode = simd_prefix_and_encode(xmm2, dst, dst, VEX_SIMD_66, /* no_mask_reg */ false,
4754 VEX_OPCODE_0F, /* rex_w */ VM_Version::supports_evex());
4755 emit_int8(0x73);
4756 emit_int8((unsigned char)(0xC0 | encode));
4757 emit_int8(shift & 0xFF);
4758 }
4759
4760 void Assembler::psrlw(XMMRegister dst, XMMRegister shift) {
4761 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4762 emit_simd_arith(0xD1, dst, shift, VEX_SIMD_66, /* no_mask_reg */ false, /* legacy_mode */ _legacy_mode_bw);
4763 }
4764
4765 void Assembler::psrld(XMMRegister dst, XMMRegister shift) {
4766 _instruction_uses_vl = true;
4767 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4768 emit_simd_arith(0xD2, dst, shift, VEX_SIMD_66);
4769 }
4770
4771 void Assembler::psrlq(XMMRegister dst, XMMRegister shift) {
4772 _instruction_uses_vl = true;
4773 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4774 if (VM_Version::supports_evex()) {
4775 emit_simd_arith_q(0xD3, dst, shift, VEX_SIMD_66);
4776 } else {
4777 emit_simd_arith(0xD3, dst, shift, VEX_SIMD_66);
4778 }
4779 }
4780
4781 void Assembler::vpsrlw(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
4782 assert(UseAVX > 0, "requires some form of AVX");
4783 // XMM2 is for /2 encoding: 66 0F 71 /2 ib
4784 emit_vex_arith(0x71, xmm2, dst, src, VEX_SIMD_66, vector_len, /* no_mask_reg */ false, /* legacy_mode */ _legacy_mode_bw);
4785 emit_int8(shift & 0xFF);
4786 }
4787
4788 void Assembler::vpsrld(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
4789 _instruction_uses_vl = true;
4790 assert(UseAVX > 0, "requires some form of AVX");
4791 // XMM2 is for /2 encoding: 66 0F 72 /2 ib
4792 emit_vex_arith(0x72, xmm2, dst, src, VEX_SIMD_66, vector_len);
4793 emit_int8(shift & 0xFF);
4794 }
4795
4796 void Assembler::vpsrlq(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
4797 _instruction_uses_vl = true;
4798 assert(UseAVX > 0, "requires some form of AVX");
4799 // XMM2 is for /2 encoding: 66 0F 73 /2 ib
4800 if (VM_Version::supports_evex()) {
4801 emit_vex_arith_q(0x73, xmm2, dst, src, VEX_SIMD_66, vector_len);
4802 } else {
4803 emit_vex_arith(0x73, xmm2, dst, src, VEX_SIMD_66, vector_len);
4804 }
4805 emit_int8(shift & 0xFF);
4806 }
4807
4808 void Assembler::vpsrlw(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
4809 assert(UseAVX > 0, "requires some form of AVX");
4810 emit_vex_arith(0xD1, dst, src, shift, VEX_SIMD_66, vector_len, /* no_mask_reg */ false, /* legacy_mode */ _legacy_mode_bw);
4811 }
4812
4813 void Assembler::vpsrld(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
4814 _instruction_uses_vl = true;
4815 assert(UseAVX > 0, "requires some form of AVX");
4816 emit_vex_arith(0xD2, dst, src, shift, VEX_SIMD_66, vector_len);
4817 }
4818
4819 void Assembler::vpsrlq(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
4820 _instruction_uses_vl = true;
4821 assert(UseAVX > 0, "requires some form of AVX");
4822 if (VM_Version::supports_evex()) {
4823 emit_vex_arith_q(0xD3, dst, src, shift, VEX_SIMD_66, vector_len);
4824 } else {
4825 emit_vex_arith(0xD3, dst, src, shift, VEX_SIMD_66, vector_len);
4826 }
4827 }
4828
4829 // Shift packed integers arithmetically right by specified number of bits.
4830 void Assembler::psraw(XMMRegister dst, int shift) {
4831 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4832 // XMM4 is for /4 encoding: 66 0F 71 /4 ib
4833 int encode = simd_prefix_and_encode(xmm4, dst, dst, VEX_SIMD_66, /* no_mask_reg */ false,
4834 VEX_OPCODE_0F, /* rex_w */ false, AVX_128bit, /* legacy_mode */ _legacy_mode_bw);
4835 emit_int8(0x71);
4836 emit_int8((unsigned char)(0xC0 | encode));
4837 emit_int8(shift & 0xFF);
4838 }
4839
4840 void Assembler::psrad(XMMRegister dst, int shift) {
4841 _instruction_uses_vl = true;
4842 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4843 // XMM4 is for /4 encoding: 66 0F 72 /4 ib
4844 int encode = simd_prefix_and_encode(xmm4, dst, dst, VEX_SIMD_66, /* no_mask_reg */ false);
4845 emit_int8(0x72);
4846 emit_int8((unsigned char)(0xC0 | encode));
4847 emit_int8(shift & 0xFF);
4848 }
4849
4850 void Assembler::psraw(XMMRegister dst, XMMRegister shift) {
4851 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4852 emit_simd_arith(0xE1, dst, shift, VEX_SIMD_66, /* no_mask_reg */ false, /* legacy_mode */ _legacy_mode_bw);
4853 }
4854
4855 void Assembler::psrad(XMMRegister dst, XMMRegister shift) {
4856 _instruction_uses_vl = true;
4857 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4858 emit_simd_arith(0xE2, dst, shift, VEX_SIMD_66);
4859 }
4860
4861 void Assembler::vpsraw(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
4862 assert(UseAVX > 0, "requires some form of AVX");
4863 // XMM4 is for /4 encoding: 66 0F 71 /4 ib
4864 emit_vex_arith(0x71, xmm4, dst, src, VEX_SIMD_66, vector_len, /* no_mask_reg */ false, /* legacy_mode */ _legacy_mode_bw);
4865 emit_int8(shift & 0xFF);
4866 }
4867
4868 void Assembler::vpsrad(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
4869 _instruction_uses_vl = true;
4870 assert(UseAVX > 0, "requires some form of AVX");
4871 // XMM4 is for /4 encoding: 66 0F 71 /4 ib
4872 emit_vex_arith(0x72, xmm4, dst, src, VEX_SIMD_66, vector_len);
4873 emit_int8(shift & 0xFF);
4874 }
4875
4876 void Assembler::vpsraw(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
4877 assert(UseAVX > 0, "requires some form of AVX");
4878 emit_vex_arith(0xE1, dst, src, shift, VEX_SIMD_66, vector_len, /* no_mask_reg */ false, /* legacy_mode */ _legacy_mode_bw);
4879 }
4880
4881 void Assembler::vpsrad(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
4882 _instruction_uses_vl = true;
4883 assert(UseAVX > 0, "requires some form of AVX");
4884 emit_vex_arith(0xE2, dst, src, shift, VEX_SIMD_66, vector_len);
4885 }
4886
4887
4888 // AND packed integers
4889 void Assembler::pand(XMMRegister dst, XMMRegister src) {
4890 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4891 emit_simd_arith(0xDB, dst, src, VEX_SIMD_66);
4892 }
4893
4894 void Assembler::vpand(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4895 _instruction_uses_vl = true;
4896 assert(UseAVX > 0, "requires some form of AVX");
4897 emit_vex_arith(0xDB, dst, nds, src, VEX_SIMD_66, vector_len);
4898 }
4899
4900 void Assembler::vpand(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4901 _instruction_uses_vl = true;
4902 assert(UseAVX > 0, "requires some form of AVX");
4903 if (VM_Version::supports_evex()) {
4904 _tuple_type = EVEX_FV;
4905 _input_size_in_bits = EVEX_32bit;
4906 }
4907 emit_vex_arith(0xDB, dst, nds, src, VEX_SIMD_66, vector_len);
4908 }
4909
4910 void Assembler::por(XMMRegister dst, XMMRegister src) {
4911 _instruction_uses_vl = true;
4912 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4913 emit_simd_arith(0xEB, dst, src, VEX_SIMD_66);
4914 }
4915
4916 void Assembler::vpor(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4917 _instruction_uses_vl = true;
4918 assert(UseAVX > 0, "requires some form of AVX");
4919 emit_vex_arith(0xEB, dst, nds, src, VEX_SIMD_66, vector_len);
4920 }
4921
4922 void Assembler::vpor(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4923 _instruction_uses_vl = true;
4924 assert(UseAVX > 0, "requires some form of AVX");
4925 if (VM_Version::supports_evex()) {
4926 _tuple_type = EVEX_FV;
4927 _input_size_in_bits = EVEX_32bit;
4928 }
4929 emit_vex_arith(0xEB, dst, nds, src, VEX_SIMD_66, vector_len);
4930 }
4931
4932 void Assembler::pxor(XMMRegister dst, XMMRegister src) {
4933 _instruction_uses_vl = true;
4934 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4935 emit_simd_arith(0xEF, dst, src, VEX_SIMD_66);
4936 }
4937
4938 void Assembler::vpxor(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4939 _instruction_uses_vl = true;
4940 assert(UseAVX > 0, "requires some form of AVX");
4941 emit_vex_arith(0xEF, dst, nds, src, VEX_SIMD_66, vector_len);
4942 }
4943
4944 void Assembler::vpxor(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4945 _instruction_uses_vl = true;
4946 assert(UseAVX > 0, "requires some form of AVX");
4947 if (VM_Version::supports_evex()) {
4948 _tuple_type = EVEX_FV;
4949 _input_size_in_bits = EVEX_32bit;
4950 }
4951 emit_vex_arith(0xEF, dst, nds, src, VEX_SIMD_66, vector_len);
4952 }
4953
4954
4955 void Assembler::vinsertf128h(XMMRegister dst, XMMRegister nds, XMMRegister src) {
4956 assert(VM_Version::supports_avx(), "");
4957 int vector_len = AVX_256bit;
4958 if (VM_Version::supports_evex()) {
4959 vector_len = AVX_512bit;
4960 }
4961 int encode = vex_prefix_and_encode(dst, nds, src, VEX_SIMD_66, vector_len, VEX_OPCODE_0F_3A);
4962 emit_int8(0x18);
4963 emit_int8((unsigned char)(0xC0 | encode));
4964 // 0x00 - insert into lower 128 bits
4965 // 0x01 - insert into upper 128 bits
4966 emit_int8(0x01);
4967 }
4968
4969 void Assembler::vinsertf64x4h(XMMRegister dst, XMMRegister nds, XMMRegister src) {
4970 assert(VM_Version::supports_evex(), "");
4971 int vector_len = AVX_512bit;
4972 int src_enc = src->encoding();
4973 int dst_enc = dst->encoding();
4974 int nds_enc = nds->is_valid() ? nds->encoding() : 0;
4975 int encode = vex_prefix_and_encode(dst_enc, nds_enc, src_enc, VEX_SIMD_66, VEX_OPCODE_0F_3A,
4976 /* vex_w */ true, vector_len, /* legacy_mode */ false, /* no_mask_reg */ false);
4977 emit_int8(0x1A);
4978 emit_int8((unsigned char)(0xC0 | encode));
4979 // 0x00 - insert into lower 256 bits
4980 // 0x01 - insert into upper 256 bits
4981 emit_int8(0x01);
4982 }
4983
4984 void Assembler::vinsertf64x4h(XMMRegister dst, Address src) {
4985 assert(VM_Version::supports_evex(), "");
4986 _tuple_type = EVEX_T4;
4987 _input_size_in_bits = EVEX_64bit;
4988 InstructionMark im(this);
4989 int vector_len = AVX_512bit;
4990 assert(dst != xnoreg, "sanity");
4991 int dst_enc = dst->encoding();
4992 // swap src<->dst for encoding
4993 vex_prefix(src, dst_enc, dst_enc, VEX_SIMD_66, VEX_OPCODE_0F_3A, /* vex_w */ true, vector_len);
4994 emit_int8(0x1A);
4995 emit_operand(dst, src);
4996 // 0x01 - insert into upper 128 bits
4997 emit_int8(0x01);
4998 }
4999
5000 void Assembler::vinsertf32x4h(XMMRegister dst, XMMRegister nds, XMMRegister src, int value) {
5001 assert(VM_Version::supports_evex(), "");
5002 int vector_len = AVX_512bit;
5003 int src_enc = src->encoding();
5004 int dst_enc = dst->encoding();
5005 int nds_enc = nds->is_valid() ? nds->encoding() : 0;
5006 int encode = vex_prefix_and_encode(dst_enc, nds_enc, src_enc, VEX_SIMD_66, VEX_OPCODE_0F_3A,
5007 /* vex_w */ false, vector_len, /* legacy_mode */ false, /* no_mask_reg */ false);
5008 emit_int8(0x18);
5009 emit_int8((unsigned char)(0xC0 | encode));
5010 // 0x00 - insert into q0 128 bits (0..127)
5011 // 0x01 - insert into q1 128 bits (128..255)
5012 // 0x02 - insert into q2 128 bits (256..383)
5013 // 0x03 - insert into q3 128 bits (384..511)
5014 emit_int8(value & 0x3);
5015 }
5016
5017 void Assembler::vinsertf32x4h(XMMRegister dst, Address src, int value) {
5018 assert(VM_Version::supports_evex(), "");
5019 _tuple_type = EVEX_T4;
5020 _input_size_in_bits = EVEX_32bit;
5021 InstructionMark im(this);
5022 int vector_len = AVX_512bit;
5023 assert(dst != xnoreg, "sanity");
5024 int dst_enc = dst->encoding();
5025 // swap src<->dst for encoding
5026 vex_prefix(src, dst_enc, dst_enc, VEX_SIMD_66, VEX_OPCODE_0F_3A, /* vex_w */ false, vector_len);
5027 emit_int8(0x18);
5028 emit_operand(dst, src);
5029 // 0x00 - insert into q0 128 bits (0..127)
5030 // 0x01 - insert into q1 128 bits (128..255)
5031 // 0x02 - insert into q2 128 bits (256..383)
5032 // 0x03 - insert into q3 128 bits (384..511)
5033 emit_int8(value & 0x3);
5034 }
5035
5036 void Assembler::vinsertf128h(XMMRegister dst, Address src) {
5037 assert(VM_Version::supports_avx(), "");
5038 int vector_len = AVX_256bit;
5039 if (VM_Version::supports_evex()) {
5040 _tuple_type = EVEX_T4;
5041 _input_size_in_bits = EVEX_32bit;
5042 vector_len = AVX_512bit;
5043 }
5044 InstructionMark im(this);
5045 assert(dst != xnoreg, "sanity");
5046 int dst_enc = dst->encoding();
5047 // swap src<->dst for encoding
5048 vex_prefix(src, dst_enc, dst_enc, VEX_SIMD_66, VEX_OPCODE_0F_3A, /* vex_w */ false, vector_len);
5049 emit_int8(0x18);
5050 emit_operand(dst, src);
5051 // 0x01 - insert into upper 128 bits
5052 emit_int8(0x01);
5053 }
5054
5055 void Assembler::vextractf128h(XMMRegister dst, XMMRegister src) {
5056 assert(VM_Version::supports_avx(), "");
5057 int vector_len = AVX_256bit;
5058 if (VM_Version::supports_evex()) {
5059 vector_len = AVX_512bit;
5060 }
5061 int encode = vex_prefix_and_encode(src, xnoreg, dst, VEX_SIMD_66, vector_len, VEX_OPCODE_0F_3A);
5062 emit_int8(0x19);
5063 emit_int8((unsigned char)(0xC0 | encode));
5064 // 0x00 - insert into lower 128 bits
5065 // 0x01 - insert into upper 128 bits
5066 emit_int8(0x01);
5067 }
5068
5069 void Assembler::vextractf128h(Address dst, XMMRegister src) {
5070 assert(VM_Version::supports_avx(), "");
5071 int vector_len = AVX_256bit;
5072 if (VM_Version::supports_evex()) {
5073 _tuple_type = EVEX_T4;
5074 _input_size_in_bits = EVEX_32bit;
5075 vector_len = AVX_512bit;
5076 }
5077 InstructionMark im(this);
5078 assert(src != xnoreg, "sanity");
5079 int src_enc = src->encoding();
5080 vex_prefix(dst, 0, src_enc, VEX_SIMD_66, VEX_OPCODE_0F_3A, /* vex_w */ false, vector_len);
5081 emit_int8(0x19);
5082 emit_operand(src, dst);
5083 // 0x01 - extract from upper 128 bits
5084 emit_int8(0x01);
5085 }
5086
5087 void Assembler::vinserti128h(XMMRegister dst, XMMRegister nds, XMMRegister src) {
5088 assert(VM_Version::supports_avx2(), "");
5089 int vector_len = AVX_256bit;
5090 if (VM_Version::supports_evex()) {
5091 vector_len = AVX_512bit;
5092 }
5093 int encode = vex_prefix_and_encode(dst, nds, src, VEX_SIMD_66, vector_len, VEX_OPCODE_0F_3A);
5094 emit_int8(0x38);
5095 emit_int8((unsigned char)(0xC0 | encode));
5096 // 0x00 - insert into lower 128 bits
5097 // 0x01 - insert into upper 128 bits
5098 emit_int8(0x01);
5099 }
5100
5101 void Assembler::vinserti64x4h(XMMRegister dst, XMMRegister nds, XMMRegister src) {
5102 assert(VM_Version::supports_evex(), "");
5103 int vector_len = AVX_512bit;
5104 int src_enc = src->encoding();
5105 int dst_enc = dst->encoding();
5106 int nds_enc = nds->is_valid() ? nds->encoding() : 0;
5107 int encode = vex_prefix_and_encode(dst_enc, nds_enc, src_enc, VEX_SIMD_66, VEX_OPCODE_0F_3A,
5108 /* vex_w */ true, vector_len, /* legacy_mode */ false, /* no_reg_mask */ false);
5109 emit_int8(0x38);
5110 emit_int8((unsigned char)(0xC0 | encode));
5111 // 0x00 - insert into lower 256 bits
5112 // 0x01 - insert into upper 256 bits
5113 emit_int8(0x01);
5114 }
5115
5116 void Assembler::vinserti128h(XMMRegister dst, Address src) {
5117 assert(VM_Version::supports_avx2(), "");
5118 int vector_len = AVX_256bit;
5119 if (VM_Version::supports_evex()) {
5120 _tuple_type = EVEX_T4;
5121 _input_size_in_bits = EVEX_32bit;
5122 vector_len = AVX_512bit;
5123 }
5124 InstructionMark im(this);
5125 assert(dst != xnoreg, "sanity");
5126 int dst_enc = dst->encoding();
5127 // swap src<->dst for encoding
5128 vex_prefix(src, dst_enc, dst_enc, VEX_SIMD_66, VEX_OPCODE_0F_3A, /* vex_w */ false, vector_len);
5129 emit_int8(0x38);
5130 emit_operand(dst, src);
5131 // 0x01 - insert into upper 128 bits
5132 emit_int8(0x01);
5133 }
5134
5135 void Assembler::vextracti128h(XMMRegister dst, XMMRegister src) {
5136 assert(VM_Version::supports_avx(), "");
5137 int vector_len = AVX_256bit;
5138 if (VM_Version::supports_evex()) {
5139 vector_len = AVX_512bit;
5140 }
5141 int encode = vex_prefix_and_encode(src, xnoreg, dst, VEX_SIMD_66, vector_len, VEX_OPCODE_0F_3A);
5142 emit_int8(0x39);
5143 emit_int8((unsigned char)(0xC0 | encode));
5144 // 0x00 - insert into lower 128 bits
5145 // 0x01 - insert into upper 128 bits
5146 emit_int8(0x01);
5147 }
5148
5149 void Assembler::vextracti128h(Address dst, XMMRegister src) {
5150 assert(VM_Version::supports_avx2(), "");
5151 int vector_len = AVX_256bit;
5152 if (VM_Version::supports_evex()) {
5153 _tuple_type = EVEX_T4;
5154 _input_size_in_bits = EVEX_32bit;
5155 vector_len = AVX_512bit;
5156 }
5157 InstructionMark im(this);
5158 assert(src != xnoreg, "sanity");
5159 int src_enc = src->encoding();
5160 vex_prefix(dst, 0, src_enc, VEX_SIMD_66, VEX_OPCODE_0F_3A, /* vex_w */ false, vector_len);
5161 emit_int8(0x39);
5162 emit_operand(src, dst);
5163 // 0x01 - extract from upper 128 bits
5164 emit_int8(0x01);
5165 }
5166
5167 void Assembler::vextracti64x4h(XMMRegister dst, XMMRegister src) {
5168 assert(VM_Version::supports_evex(), "");
5169 int vector_len = AVX_512bit;
5170 int src_enc = src->encoding();
5171 int dst_enc = dst->encoding();
5172 int encode = vex_prefix_and_encode(src_enc, 0, dst_enc, VEX_SIMD_66, VEX_OPCODE_0F_3A,
5173 /* vex_w */ true, vector_len, /* legacy_mode */ false, /* no_mask_reg */ false);
5174 emit_int8(0x3B);
5175 emit_int8((unsigned char)(0xC0 | encode));
5176 // 0x01 - extract from upper 256 bits
5177 emit_int8(0x01);
5178 }
5179
5180 void Assembler::vextracti64x2h(XMMRegister dst, XMMRegister src, int value) {
5181 assert(VM_Version::supports_evex(), "");
5182 int vector_len = AVX_512bit;
5183 int src_enc = src->encoding();
5184 int dst_enc = dst->encoding();
5185 int encode;
5186 if (VM_Version::supports_avx512dq()) {
5187 encode = vex_prefix_and_encode(src_enc, 0, dst_enc, VEX_SIMD_66, VEX_OPCODE_0F_3A,
5188 /* vex_w */ true, vector_len, /* legacy_mode */ false, /* no_mask_reg */ false);
5189 } else {
5190 encode = vex_prefix_and_encode(src_enc, 0, dst_enc, VEX_SIMD_66, VEX_OPCODE_0F_3A,
5191 /* vex_w */ false, vector_len, /* legacy_mode */ true, /* no_mask_reg */ false);
5192 }
5193 emit_int8(0x39);
5194 emit_int8((unsigned char)(0xC0 | encode));
5195 // 0x01 - extract from bits 255:128
5196 // 0x02 - extract from bits 383:256
5197 // 0x03 - extract from bits 511:384
5198 emit_int8(value & 0x3);
5199 }
5200
5201 void Assembler::vextractf64x4h(XMMRegister dst, XMMRegister src) {
5202 assert(VM_Version::supports_evex(), "");
5203 int vector_len = AVX_512bit;
5204 int src_enc = src->encoding();
5205 int dst_enc = dst->encoding();
5206 int encode = vex_prefix_and_encode(src_enc, 0, dst_enc, VEX_SIMD_66, VEX_OPCODE_0F_3A,
5207 /* vex_w */ true, vector_len, /* legacy_mode */ false, /* no_mask_reg */ false);
5208 emit_int8(0x1B);
5209 emit_int8((unsigned char)(0xC0 | encode));
5210 // 0x01 - extract from upper 256 bits
5211 emit_int8(0x01);
5212 }
5213
5214 void Assembler::vextractf64x4h(Address dst, XMMRegister src) {
5215 assert(VM_Version::supports_evex(), "");
5216 _tuple_type = EVEX_T4;
5217 _input_size_in_bits = EVEX_64bit;
5218 InstructionMark im(this);
5219 int vector_len = AVX_512bit;
5220 assert(src != xnoreg, "sanity");
5221 int src_enc = src->encoding();
5222 vex_prefix(dst, 0, src_enc, VEX_SIMD_66, VEX_OPCODE_0F_3A,
5223 /* vex_w */ true, vector_len);
5224 emit_int8(0x1B);
5225 emit_operand(src, dst);
5226 // 0x01 - extract from upper 256 bits
5227 emit_int8(0x01);
5228 }
5229
5230 void Assembler::vextractf32x4h(XMMRegister dst, XMMRegister src, int value) {
5231 assert(VM_Version::supports_evex(), "");
5232 int vector_len = AVX_512bit;
5233 int src_enc = src->encoding();
5234 int dst_enc = dst->encoding();
5235 int encode = vex_prefix_and_encode(src_enc, 0, dst_enc, VEX_SIMD_66, VEX_OPCODE_0F_3A,
5236 /* vex_w */ false, vector_len, /* legacy_mode */ false, /* no_mask_reg */ false);
5237 emit_int8(0x19);
5238 emit_int8((unsigned char)(0xC0 | encode));
5239 // 0x00 - extract from bits 127:0
5240 // 0x01 - extract from bits 255:128
5241 // 0x02 - extract from bits 383:256
5242 // 0x03 - extract from bits 511:384
5243 emit_int8(value & 0x3);
5244 }
5245
5246 void Assembler::vextractf32x4h(Address dst, XMMRegister src, int value) {
5247 assert(VM_Version::supports_evex(), "");
5248 _tuple_type = EVEX_T4;
5249 _input_size_in_bits = EVEX_32bit;
5250 InstructionMark im(this);
5251 int vector_len = AVX_512bit;
5252 assert(src != xnoreg, "sanity");
5253 int src_enc = src->encoding();
5254 vex_prefix(dst, 0, src_enc, VEX_SIMD_66, VEX_OPCODE_0F_3A, /* vex_w */ false, vector_len);
5255 emit_int8(0x19);
5256 emit_operand(src, dst);
5257 // 0x00 - extract from bits 127:0
5258 // 0x01 - extract from bits 255:128
5259 // 0x02 - extract from bits 383:256
5260 // 0x03 - extract from bits 511:384
5261 emit_int8(value & 0x3);
5262 }
5263
5264 void Assembler::vextractf64x2h(XMMRegister dst, XMMRegister src, int value) {
5265 assert(VM_Version::supports_evex(), "");
5266 int vector_len = AVX_512bit;
5267 int src_enc = src->encoding();
5268 int dst_enc = dst->encoding();
5269 int encode = vex_prefix_and_encode(src_enc, 0, dst_enc, VEX_SIMD_66, VEX_OPCODE_0F_3A,
5270 /* vex_w */ !_legacy_mode_dq, vector_len, /* legacy_mode */ false, /* no_mask_reg */ false);
5271 emit_int8(0x19);
5272 emit_int8((unsigned char)(0xC0 | encode));
5273 // 0x01 - extract from bits 255:128
5274 // 0x02 - extract from bits 383:256
5275 // 0x03 - extract from bits 511:384
5276 emit_int8(value & 0x3);
5277 }
5278
5279 // duplicate 4-bytes integer data from src into 8 locations in dest
5280 void Assembler::vpbroadcastd(XMMRegister dst, XMMRegister src) {
5281 _instruction_uses_vl = true;
5282 assert(UseAVX > 1, "");
5283 int vector_len = AVX_256bit;
5284 int encode = vex_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, vector_len, VEX_OPCODE_0F_38);
5285 emit_int8(0x58);
5286 emit_int8((unsigned char)(0xC0 | encode));
5287 }
5288
5289 // duplicate 1-byte integer data from src into 16||32|64 locations in dest : requires AVX512BW and AVX512VL
5290 void Assembler::evpbroadcastb(XMMRegister dst, XMMRegister src, int vector_len) {
5291 _instruction_uses_vl = true;
5292 assert(UseAVX > 1, "");
5293 int encode = vex_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, vector_len, VEX_OPCODE_0F_38);
5294 emit_int8(0x78);
5295 emit_int8((unsigned char)(0xC0 | encode));
5296 }
5297
5298 void Assembler::evpbroadcastb(XMMRegister dst, Address src, int vector_len) {
5299 _instruction_uses_vl = true;
5300 assert(UseAVX > 1, "");
5301 _tuple_type = EVEX_T1S;
5302 _input_size_in_bits = EVEX_8bit;
5303 InstructionMark im(this);
5304 assert(dst != xnoreg, "sanity");
5305 int dst_enc = dst->encoding();
5306 // swap src<->dst for encoding
5307 vex_prefix(src, dst_enc, dst_enc, VEX_SIMD_66, VEX_OPCODE_0F_38, /* vex_w */ false, vector_len);
5308 emit_int8(0x78);
5309 emit_operand(dst, src);
5310 }
5311
5312 // duplicate 2-byte integer data from src into 8|16||32 locations in dest : requires AVX512BW and AVX512VL
5313 void Assembler::evpbroadcastw(XMMRegister dst, XMMRegister src, int vector_len) {
5314 _instruction_uses_vl = true;
5315 assert(UseAVX > 1, "");
5316 int encode = vex_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, vector_len, VEX_OPCODE_0F_38);
5317 emit_int8(0x79);
5318 emit_int8((unsigned char)(0xC0 | encode));
5319 }
5320
5321 void Assembler::evpbroadcastw(XMMRegister dst, Address src, int vector_len) {
5322 _instruction_uses_vl = true;
5323 assert(UseAVX > 1, "");
5324 _tuple_type = EVEX_T1S;
5325 _input_size_in_bits = EVEX_16bit;
5326 InstructionMark im(this);
5327 assert(dst != xnoreg, "sanity");
5328 int dst_enc = dst->encoding();
5329 // swap src<->dst for encoding
5330 vex_prefix(src, dst_enc, dst_enc, VEX_SIMD_66, VEX_OPCODE_0F_38, /* vex_w */ false, vector_len);
5331 emit_int8(0x79);
5332 emit_operand(dst, src);
5333 }
5334
5335 // duplicate 4-byte integer data from src into 4|8|16 locations in dest : requires AVX512VL
5336 void Assembler::evpbroadcastd(XMMRegister dst, XMMRegister src, int vector_len) {
5337 _instruction_uses_vl = true;
5338 assert(UseAVX > 1, "");
5339 int encode = vex_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, vector_len, VEX_OPCODE_0F_38);
5340 emit_int8(0x58);
5341 emit_int8((unsigned char)(0xC0 | encode));
5342 }
5343
5344 void Assembler::evpbroadcastd(XMMRegister dst, Address src, int vector_len) {
5345 _instruction_uses_vl = true;
5346 assert(UseAVX > 1, "");
5347 _tuple_type = EVEX_T1S;
5348 _input_size_in_bits = EVEX_32bit;
5349 InstructionMark im(this);
5350 assert(dst != xnoreg, "sanity");
5351 int dst_enc = dst->encoding();
5352 // swap src<->dst for encoding
5353 vex_prefix(src, dst_enc, dst_enc, VEX_SIMD_66, VEX_OPCODE_0F_38, /* vex_w */ false, vector_len);
5354 emit_int8(0x58);
5355 emit_operand(dst, src);
5356 }
5357
5358 // duplicate 8-byte integer data from src into 4|8|16 locations in dest : requires AVX512VL
5359 void Assembler::evpbroadcastq(XMMRegister dst, XMMRegister src, int vector_len) {
5360 _instruction_uses_vl = true;
5361 assert(UseAVX > 1, "");
5362 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38,
5363 /* vex_w */ true, vector_len, /* legacy_mode */ false, /* no_mask_reg */ false);
5364 emit_int8(0x59);
5365 emit_int8((unsigned char)(0xC0 | encode));
5366 }
5367
5368 void Assembler::evpbroadcastq(XMMRegister dst, Address src, int vector_len) {
5369 _instruction_uses_vl = true;
5370 assert(UseAVX > 1, "");
5371 _tuple_type = EVEX_T1S;
5372 _input_size_in_bits = EVEX_64bit;
5373 InstructionMark im(this);
5374 assert(dst != xnoreg, "sanity");
5375 int dst_enc = dst->encoding();
5376 // swap src<->dst for encoding
5377 vex_prefix(src, dst_enc, dst_enc, VEX_SIMD_66, VEX_OPCODE_0F_38, /* vex_w */ true, vector_len);
5378 emit_int8(0x59);
5379 emit_operand(dst, src);
5380 }
5381
5382 // duplicate single precision fp from src into 4|8|16 locations in dest : requires AVX512VL
5383 void Assembler::evpbroadcastss(XMMRegister dst, XMMRegister src, int vector_len) {
5384 _instruction_uses_vl = true;
5385 assert(UseAVX > 1, "");
5386 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38,
5387 /* vex_w */ false, vector_len, /* legacy_mode */ false, /*no_mask_reg */ false);
5388 emit_int8(0x18);
5389 emit_int8((unsigned char)(0xC0 | encode));
5390 }
5391
5392 void Assembler::evpbroadcastss(XMMRegister dst, Address src, int vector_len) {
5393 assert(UseAVX > 1, "");
5394 _tuple_type = EVEX_T1S;
5395 _input_size_in_bits = EVEX_32bit;
5396 InstructionMark im(this);
5397 assert(dst != xnoreg, "sanity");
5398 int dst_enc = dst->encoding();
5399 // swap src<->dst for encoding
5400 vex_prefix(src, 0, dst_enc, VEX_SIMD_66, VEX_OPCODE_0F_38, /* vex_w */ false, vector_len);
5401 emit_int8(0x18);
5402 emit_operand(dst, src);
5403 }
5404
5405 // duplicate double precision fp from src into 2|4|8 locations in dest : requires AVX512VL
5406 void Assembler::evpbroadcastsd(XMMRegister dst, XMMRegister src, int vector_len) {
5407 _instruction_uses_vl = true;
5408 assert(UseAVX > 1, "");
5409 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38,
5410 /*vex_w */ true, vector_len, /* legacy_mode */ false, /*no_mask_reg */ false);
5411 emit_int8(0x19);
5412 emit_int8((unsigned char)(0xC0 | encode));
5413 }
5414
5415 void Assembler::evpbroadcastsd(XMMRegister dst, Address src, int vector_len) {
5416 _instruction_uses_vl = true;
5417 assert(UseAVX > 1, "");
5418 _tuple_type = EVEX_T1S;
5419 _input_size_in_bits = EVEX_64bit;
5420 InstructionMark im(this);
5421 assert(dst != xnoreg, "sanity");
5422 int dst_enc = dst->encoding();
5423 // swap src<->dst for encoding
5424 vex_prefix(src, 0, dst_enc, VEX_SIMD_66, VEX_OPCODE_0F_38, /* vex_w */ true, vector_len);
5425 emit_int8(0x19);
5426 emit_operand(dst, src);
5427 }
5428
5429 // duplicate 1-byte integer data from src into 16||32|64 locations in dest : requires AVX512BW and AVX512VL
5430 void Assembler::evpbroadcastb(XMMRegister dst, Register src, int vector_len) {
5431 _instruction_uses_vl = true;
5432 assert(VM_Version::supports_evex(), "");
5433 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38,
5434 /*vex_w */ false, vector_len, /* legacy_mode */ false, /*no_mask_reg */ false);
5435 emit_int8(0x7A);
5436 emit_int8((unsigned char)(0xC0 | encode));
5437 }
5438
5439 // duplicate 2-byte integer data from src into 8|16||32 locations in dest : requires AVX512BW and AVX512VL
5440 void Assembler::evpbroadcastw(XMMRegister dst, Register src, int vector_len) {
5441 _instruction_uses_vl = true;
5442 assert(VM_Version::supports_evex(), "");
5443 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38,
5444 /* vex_w */ false, vector_len, /* legacy_mode */ false, /*no_mask_reg */ false);
5445 emit_int8(0x7B);
5446 emit_int8((unsigned char)(0xC0 | encode));
5447 }
5448
5449 // duplicate 4-byte integer data from src into 4|8|16 locations in dest : requires AVX512VL
5450 void Assembler::evpbroadcastd(XMMRegister dst, Register src, int vector_len) {
5451 _instruction_uses_vl = true;
5452 assert(VM_Version::supports_evex(), "");
5453 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38,
5454 /* vex_w */ false, vector_len, /* legacy_mode */ false, /*no_mask_reg */ false);
5455 emit_int8(0x7C);
5456 emit_int8((unsigned char)(0xC0 | encode));
5457 }
5458
5459 // duplicate 8-byte integer data from src into 4|8|16 locations in dest : requires AVX512VL
5460 void Assembler::evpbroadcastq(XMMRegister dst, Register src, int vector_len) {
5461 _instruction_uses_vl = true;
5462 assert(VM_Version::supports_evex(), "");
5463 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38,
5464 /* vex_w */ true, vector_len, /* legacy_mode */ false, /*no_mask_reg */ false);
5465 emit_int8(0x7C);
5466 emit_int8((unsigned char)(0xC0 | encode));
5467 }
5468
5469 // Carry-Less Multiplication Quadword
5470 void Assembler::pclmulqdq(XMMRegister dst, XMMRegister src, int mask) {
5471 assert(VM_Version::supports_clmul(), "");
5472 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, /* no_mask_reg */ false,
5473 VEX_OPCODE_0F_3A, /* rex_w */ false, AVX_128bit, /* legacy_mode */ true);
5474 emit_int8(0x44);
5475 emit_int8((unsigned char)(0xC0 | encode));
5476 emit_int8((unsigned char)mask);
5477 }
5478
5479 // Carry-Less Multiplication Quadword
5480 void Assembler::vpclmulqdq(XMMRegister dst, XMMRegister nds, XMMRegister src, int mask) {
5481 assert(VM_Version::supports_avx() && VM_Version::supports_clmul(), "");
5482 int vector_len = AVX_128bit;
5483 int encode = vex_prefix_and_encode(dst, nds, src, VEX_SIMD_66, vector_len, VEX_OPCODE_0F_3A, /* legacy_mode */ true);
5484 emit_int8(0x44);
5485 emit_int8((unsigned char)(0xC0 | encode));
5486 emit_int8((unsigned char)mask);
5487 }
5488
5489 void Assembler::vzeroupper() {
5490 assert(VM_Version::supports_avx(), "");
5491 if (UseAVX < 3)
5492 {
5493 (void)vex_prefix_and_encode(xmm0, xmm0, xmm0, VEX_SIMD_NONE);
5494 emit_int8(0x77);
5495 }
5496 }
5497
5498
5499 #ifndef _LP64
5500 // 32bit only pieces of the assembler
5501
5502 void Assembler::cmp_literal32(Register src1, int32_t imm32, RelocationHolder const& rspec) {
5503 // NO PREFIX AS NEVER 64BIT
5504 InstructionMark im(this);
5505 emit_int8((unsigned char)0x81);
5506 emit_int8((unsigned char)(0xF8 | src1->encoding()));
5507 emit_data(imm32, rspec, 0);
5508 }
5509
5510 void Assembler::cmp_literal32(Address src1, int32_t imm32, RelocationHolder const& rspec) {
5511 // NO PREFIX AS NEVER 64BIT (not even 32bit versions of 64bit regs
5512 InstructionMark im(this);
5513 emit_int8((unsigned char)0x81);
5514 emit_operand(rdi, src1);
5515 emit_data(imm32, rspec, 0);
5516 }
5517
5518 // The 64-bit (32bit platform) cmpxchg compares the value at adr with the contents of rdx:rax,
5519 // and stores rcx:rbx into adr if so; otherwise, the value at adr is loaded
5520 // into rdx:rax. The ZF is set if the compared values were equal, and cleared otherwise.
5521 void Assembler::cmpxchg8(Address adr) {
5522 InstructionMark im(this);
5523 emit_int8(0x0F);
5524 emit_int8((unsigned char)0xC7);
5525 emit_operand(rcx, adr);
5526 }
5527
5528 void Assembler::decl(Register dst) {
5529 // Don't use it directly. Use MacroAssembler::decrementl() instead.
5530 emit_int8(0x48 | dst->encoding());
5531 }
5532
5533 #endif // _LP64
5534
5535 // 64bit typically doesn't use the x87 but needs to for the trig funcs
5536
5537 void Assembler::fabs() {
5538 emit_int8((unsigned char)0xD9);
5539 emit_int8((unsigned char)0xE1);
5540 }
5541
5542 void Assembler::fadd(int i) {
5543 emit_farith(0xD8, 0xC0, i);
5544 }
5545
5546 void Assembler::fadd_d(Address src) {
5547 InstructionMark im(this);
5548 emit_int8((unsigned char)0xDC);
5549 emit_operand32(rax, src);
5550 }
5551
5552 void Assembler::fadd_s(Address src) {
5553 InstructionMark im(this);
5554 emit_int8((unsigned char)0xD8);
5555 emit_operand32(rax, src);
5556 }
5557
5558 void Assembler::fadda(int i) {
5559 emit_farith(0xDC, 0xC0, i);
5560 }
5561
5562 void Assembler::faddp(int i) {
5563 emit_farith(0xDE, 0xC0, i);
5564 }
5565
5566 void Assembler::fchs() {
5567 emit_int8((unsigned char)0xD9);
5568 emit_int8((unsigned char)0xE0);
5569 }
5570
5571 void Assembler::fcom(int i) {
5572 emit_farith(0xD8, 0xD0, i);
5573 }
5574
5575 void Assembler::fcomp(int i) {
5576 emit_farith(0xD8, 0xD8, i);
5577 }
5578
5579 void Assembler::fcomp_d(Address src) {
5580 InstructionMark im(this);
5581 emit_int8((unsigned char)0xDC);
5582 emit_operand32(rbx, src);
5583 }
5584
5585 void Assembler::fcomp_s(Address src) {
5586 InstructionMark im(this);
5587 emit_int8((unsigned char)0xD8);
5588 emit_operand32(rbx, src);
5589 }
5590
5591 void Assembler::fcompp() {
5592 emit_int8((unsigned char)0xDE);
5593 emit_int8((unsigned char)0xD9);
5594 }
5595
5596 void Assembler::fcos() {
5597 emit_int8((unsigned char)0xD9);
5598 emit_int8((unsigned char)0xFF);
5599 }
5600
5601 void Assembler::fdecstp() {
5602 emit_int8((unsigned char)0xD9);
5603 emit_int8((unsigned char)0xF6);
5604 }
5605
5606 void Assembler::fdiv(int i) {
5607 emit_farith(0xD8, 0xF0, i);
5608 }
5609
5610 void Assembler::fdiv_d(Address src) {
5611 InstructionMark im(this);
5612 emit_int8((unsigned char)0xDC);
5613 emit_operand32(rsi, src);
5614 }
5615
5616 void Assembler::fdiv_s(Address src) {
5617 InstructionMark im(this);
5618 emit_int8((unsigned char)0xD8);
5619 emit_operand32(rsi, src);
5620 }
5621
5622 void Assembler::fdiva(int i) {
5623 emit_farith(0xDC, 0xF8, i);
5624 }
5625
5626 // Note: The Intel manual (Pentium Processor User's Manual, Vol.3, 1994)
5627 // is erroneous for some of the floating-point instructions below.
5628
5629 void Assembler::fdivp(int i) {
5630 emit_farith(0xDE, 0xF8, i); // ST(0) <- ST(0) / ST(1) and pop (Intel manual wrong)
5631 }
5632
5633 void Assembler::fdivr(int i) {
5634 emit_farith(0xD8, 0xF8, i);
5635 }
5636
5637 void Assembler::fdivr_d(Address src) {
5638 InstructionMark im(this);
5639 emit_int8((unsigned char)0xDC);
5640 emit_operand32(rdi, src);
5641 }
5642
5643 void Assembler::fdivr_s(Address src) {
5644 InstructionMark im(this);
5645 emit_int8((unsigned char)0xD8);
5646 emit_operand32(rdi, src);
5647 }
5648
5649 void Assembler::fdivra(int i) {
5650 emit_farith(0xDC, 0xF0, i);
5651 }
5652
5653 void Assembler::fdivrp(int i) {
5654 emit_farith(0xDE, 0xF0, i); // ST(0) <- ST(1) / ST(0) and pop (Intel manual wrong)
5655 }
5656
5657 void Assembler::ffree(int i) {
5658 emit_farith(0xDD, 0xC0, i);
5659 }
5660
5661 void Assembler::fild_d(Address adr) {
5662 InstructionMark im(this);
5663 emit_int8((unsigned char)0xDF);
5664 emit_operand32(rbp, adr);
5665 }
5666
5667 void Assembler::fild_s(Address adr) {
5668 InstructionMark im(this);
5669 emit_int8((unsigned char)0xDB);
5670 emit_operand32(rax, adr);
5671 }
5672
5673 void Assembler::fincstp() {
5674 emit_int8((unsigned char)0xD9);
5675 emit_int8((unsigned char)0xF7);
5676 }
5677
5678 void Assembler::finit() {
5679 emit_int8((unsigned char)0x9B);
5680 emit_int8((unsigned char)0xDB);
5681 emit_int8((unsigned char)0xE3);
5682 }
5683
5684 void Assembler::fist_s(Address adr) {
5685 InstructionMark im(this);
5686 emit_int8((unsigned char)0xDB);
5687 emit_operand32(rdx, adr);
5688 }
5689
5690 void Assembler::fistp_d(Address adr) {
5691 InstructionMark im(this);
5692 emit_int8((unsigned char)0xDF);
5693 emit_operand32(rdi, adr);
5694 }
5695
5696 void Assembler::fistp_s(Address adr) {
5697 InstructionMark im(this);
5698 emit_int8((unsigned char)0xDB);
5699 emit_operand32(rbx, adr);
5700 }
5701
5702 void Assembler::fld1() {
5703 emit_int8((unsigned char)0xD9);
5704 emit_int8((unsigned char)0xE8);
5705 }
5706
5707 void Assembler::fld_d(Address adr) {
5708 InstructionMark im(this);
5709 emit_int8((unsigned char)0xDD);
5710 emit_operand32(rax, adr);
5711 }
5712
5713 void Assembler::fld_s(Address adr) {
5714 InstructionMark im(this);
5715 emit_int8((unsigned char)0xD9);
5716 emit_operand32(rax, adr);
5717 }
5718
5719
5720 void Assembler::fld_s(int index) {
5721 emit_farith(0xD9, 0xC0, index);
5722 }
5723
5724 void Assembler::fld_x(Address adr) {
5725 InstructionMark im(this);
5726 emit_int8((unsigned char)0xDB);
5727 emit_operand32(rbp, adr);
5728 }
5729
5730 void Assembler::fldcw(Address src) {
5731 InstructionMark im(this);
5732 emit_int8((unsigned char)0xD9);
5733 emit_operand32(rbp, src);
5734 }
5735
5736 void Assembler::fldenv(Address src) {
5737 InstructionMark im(this);
5738 emit_int8((unsigned char)0xD9);
5739 emit_operand32(rsp, src);
5740 }
5741
5742 void Assembler::fldlg2() {
5743 emit_int8((unsigned char)0xD9);
5744 emit_int8((unsigned char)0xEC);
5745 }
5746
5747 void Assembler::fldln2() {
5748 emit_int8((unsigned char)0xD9);
5749 emit_int8((unsigned char)0xED);
5750 }
5751
5752 void Assembler::fldz() {
5753 emit_int8((unsigned char)0xD9);
5754 emit_int8((unsigned char)0xEE);
5755 }
5756
5757 void Assembler::flog() {
5758 fldln2();
5759 fxch();
5760 fyl2x();
5761 }
5762
5763 void Assembler::flog10() {
5764 fldlg2();
5765 fxch();
5766 fyl2x();
5767 }
5768
5769 void Assembler::fmul(int i) {
5770 emit_farith(0xD8, 0xC8, i);
5771 }
5772
5773 void Assembler::fmul_d(Address src) {
5774 InstructionMark im(this);
5775 emit_int8((unsigned char)0xDC);
5776 emit_operand32(rcx, src);
5777 }
5778
5779 void Assembler::fmul_s(Address src) {
5780 InstructionMark im(this);
5781 emit_int8((unsigned char)0xD8);
5782 emit_operand32(rcx, src);
5783 }
5784
5785 void Assembler::fmula(int i) {
5786 emit_farith(0xDC, 0xC8, i);
5787 }
5788
5789 void Assembler::fmulp(int i) {
5790 emit_farith(0xDE, 0xC8, i);
5791 }
5792
5793 void Assembler::fnsave(Address dst) {
5794 InstructionMark im(this);
5795 emit_int8((unsigned char)0xDD);
5796 emit_operand32(rsi, dst);
5797 }
5798
5799 void Assembler::fnstcw(Address src) {
5800 InstructionMark im(this);
5801 emit_int8((unsigned char)0x9B);
5802 emit_int8((unsigned char)0xD9);
5803 emit_operand32(rdi, src);
5804 }
5805
5806 void Assembler::fnstsw_ax() {
5807 emit_int8((unsigned char)0xDF);
5808 emit_int8((unsigned char)0xE0);
5809 }
5810
5811 void Assembler::fprem() {
5812 emit_int8((unsigned char)0xD9);
5813 emit_int8((unsigned char)0xF8);
5814 }
5815
5816 void Assembler::fprem1() {
5817 emit_int8((unsigned char)0xD9);
5818 emit_int8((unsigned char)0xF5);
5819 }
5820
5821 void Assembler::frstor(Address src) {
5822 InstructionMark im(this);
5823 emit_int8((unsigned char)0xDD);
5824 emit_operand32(rsp, src);
5825 }
5826
5827 void Assembler::fsin() {
5828 emit_int8((unsigned char)0xD9);
5829 emit_int8((unsigned char)0xFE);
5830 }
5831
5832 void Assembler::fsqrt() {
5833 emit_int8((unsigned char)0xD9);
5834 emit_int8((unsigned char)0xFA);
5835 }
5836
5837 void Assembler::fst_d(Address adr) {
5838 InstructionMark im(this);
5839 emit_int8((unsigned char)0xDD);
5840 emit_operand32(rdx, adr);
5841 }
5842
5843 void Assembler::fst_s(Address adr) {
5844 InstructionMark im(this);
5845 emit_int8((unsigned char)0xD9);
5846 emit_operand32(rdx, adr);
5847 }
5848
5849 void Assembler::fstp_d(Address adr) {
5850 InstructionMark im(this);
5851 emit_int8((unsigned char)0xDD);
5852 emit_operand32(rbx, adr);
5853 }
5854
5855 void Assembler::fstp_d(int index) {
5856 emit_farith(0xDD, 0xD8, index);
5857 }
5858
5859 void Assembler::fstp_s(Address adr) {
5860 InstructionMark im(this);
5861 emit_int8((unsigned char)0xD9);
5862 emit_operand32(rbx, adr);
5863 }
5864
5865 void Assembler::fstp_x(Address adr) {
5866 InstructionMark im(this);
5867 emit_int8((unsigned char)0xDB);
5868 emit_operand32(rdi, adr);
5869 }
5870
5871 void Assembler::fsub(int i) {
5872 emit_farith(0xD8, 0xE0, i);
5873 }
5874
5875 void Assembler::fsub_d(Address src) {
5876 InstructionMark im(this);
5877 emit_int8((unsigned char)0xDC);
5878 emit_operand32(rsp, src);
5879 }
5880
5881 void Assembler::fsub_s(Address src) {
5882 InstructionMark im(this);
5883 emit_int8((unsigned char)0xD8);
5884 emit_operand32(rsp, src);
5885 }
5886
5887 void Assembler::fsuba(int i) {
5888 emit_farith(0xDC, 0xE8, i);
5889 }
5890
5891 void Assembler::fsubp(int i) {
5892 emit_farith(0xDE, 0xE8, i); // ST(0) <- ST(0) - ST(1) and pop (Intel manual wrong)
5893 }
5894
5895 void Assembler::fsubr(int i) {
5896 emit_farith(0xD8, 0xE8, i);
5897 }
5898
5899 void Assembler::fsubr_d(Address src) {
5900 InstructionMark im(this);
5901 emit_int8((unsigned char)0xDC);
5902 emit_operand32(rbp, src);
5903 }
5904
5905 void Assembler::fsubr_s(Address src) {
5906 InstructionMark im(this);
5907 emit_int8((unsigned char)0xD8);
5908 emit_operand32(rbp, src);
5909 }
5910
5911 void Assembler::fsubra(int i) {
5912 emit_farith(0xDC, 0xE0, i);
5913 }
5914
5915 void Assembler::fsubrp(int i) {
5916 emit_farith(0xDE, 0xE0, i); // ST(0) <- ST(1) - ST(0) and pop (Intel manual wrong)
5917 }
5918
5919 void Assembler::ftan() {
5920 emit_int8((unsigned char)0xD9);
5921 emit_int8((unsigned char)0xF2);
5922 emit_int8((unsigned char)0xDD);
5923 emit_int8((unsigned char)0xD8);
5924 }
5925
5926 void Assembler::ftst() {
5927 emit_int8((unsigned char)0xD9);
5928 emit_int8((unsigned char)0xE4);
5929 }
5930
5931 void Assembler::fucomi(int i) {
5932 // make sure the instruction is supported (introduced for P6, together with cmov)
5933 guarantee(VM_Version::supports_cmov(), "illegal instruction");
5934 emit_farith(0xDB, 0xE8, i);
5935 }
5936
5937 void Assembler::fucomip(int i) {
5938 // make sure the instruction is supported (introduced for P6, together with cmov)
5939 guarantee(VM_Version::supports_cmov(), "illegal instruction");
5940 emit_farith(0xDF, 0xE8, i);
5941 }
5942
5943 void Assembler::fwait() {
5944 emit_int8((unsigned char)0x9B);
5945 }
5946
5947 void Assembler::fxch(int i) {
5948 emit_farith(0xD9, 0xC8, i);
5949 }
5950
5951 void Assembler::fyl2x() {
5952 emit_int8((unsigned char)0xD9);
5953 emit_int8((unsigned char)0xF1);
5954 }
5955
5956 void Assembler::frndint() {
5957 emit_int8((unsigned char)0xD9);
5958 emit_int8((unsigned char)0xFC);
5959 }
5960
5961 void Assembler::f2xm1() {
5962 emit_int8((unsigned char)0xD9);
5963 emit_int8((unsigned char)0xF0);
5964 }
5965
5966 void Assembler::fldl2e() {
5967 emit_int8((unsigned char)0xD9);
5968 emit_int8((unsigned char)0xEA);
5969 }
5970
5971 // SSE SIMD prefix byte values corresponding to VexSimdPrefix encoding.
5972 static int simd_pre[4] = { 0, 0x66, 0xF3, 0xF2 };
5973 // SSE opcode second byte values (first is 0x0F) corresponding to VexOpcode encoding.
5974 static int simd_opc[4] = { 0, 0, 0x38, 0x3A };
5975
5976 // Generate SSE legacy REX prefix and SIMD opcode based on VEX encoding.
5977 void Assembler::rex_prefix(Address adr, XMMRegister xreg, VexSimdPrefix pre, VexOpcode opc, bool rex_w) {
5978 if (pre > 0) {
5979 emit_int8(simd_pre[pre]);
5980 }
5981 if (rex_w) {
5982 prefixq(adr, xreg);
5983 } else {
5984 prefix(adr, xreg);
5985 }
5986 if (opc > 0) {
5987 emit_int8(0x0F);
5988 int opc2 = simd_opc[opc];
5989 if (opc2 > 0) {
5990 emit_int8(opc2);
5991 }
5992 }
5993 }
5994
5995 int Assembler::rex_prefix_and_encode(int dst_enc, int src_enc, VexSimdPrefix pre, VexOpcode opc, bool rex_w) {
5996 if (pre > 0) {
5997 emit_int8(simd_pre[pre]);
5998 }
5999 int encode = (rex_w) ? prefixq_and_encode(dst_enc, src_enc) :
6000 prefix_and_encode(dst_enc, src_enc);
6001 if (opc > 0) {
6002 emit_int8(0x0F);
6003 int opc2 = simd_opc[opc];
6004 if (opc2 > 0) {
6005 emit_int8(opc2);
6006 }
6007 }
6008 return encode;
6009 }
6010
6011
6012 void Assembler::vex_prefix(bool vex_r, bool vex_b, bool vex_x, bool vex_w, int nds_enc, VexSimdPrefix pre, VexOpcode opc, int vector_len) {
6013 if (vex_b || vex_x || vex_w || (opc == VEX_OPCODE_0F_38) || (opc == VEX_OPCODE_0F_3A)) {
6014 prefix(VEX_3bytes);
6015
6016 int byte1 = (vex_r ? VEX_R : 0) | (vex_x ? VEX_X : 0) | (vex_b ? VEX_B : 0);
6017 byte1 = (~byte1) & 0xE0;
6018 byte1 |= opc;
6019 emit_int8(byte1);
6020
6021 int byte2 = ((~nds_enc) & 0xf) << 3;
6022 byte2 |= (vex_w ? VEX_W : 0) | ((vector_len > 0) ? 4 : 0) | pre;
6023 emit_int8(byte2);
6024 } else {
6025 prefix(VEX_2bytes);
6026
6027 int byte1 = vex_r ? VEX_R : 0;
6028 byte1 = (~byte1) & 0x80;
6029 byte1 |= ((~nds_enc) & 0xf) << 3;
6030 byte1 |= ((vector_len > 0 ) ? 4 : 0) | pre;
6031 emit_int8(byte1);
6032 }
6033 }
6034
6035 // This is a 4 byte encoding
6036 void Assembler::evex_prefix(bool vex_r, bool vex_b, bool vex_x, bool vex_w, bool evex_r, bool evex_v,
6037 int nds_enc, VexSimdPrefix pre, VexOpcode opc,
6038 bool is_extended_context, bool is_merge_context,
6039 int vector_len, bool no_mask_reg ){
6040 // EVEX 0x62 prefix
6041 prefix(EVEX_4bytes);
6042 _evex_encoding = (vex_w ? VEX_W : 0) | (evex_r ? EVEX_Rb : 0);
6043
6044 // P0: byte 2, initialized to RXBR`00mm
6045 // instead of not'd
6046 int byte2 = (vex_r ? VEX_R : 0) | (vex_x ? VEX_X : 0) | (vex_b ? VEX_B : 0) | (evex_r ? EVEX_Rb : 0);
6047 byte2 = (~byte2) & 0xF0;
6048 // confine opc opcode extensions in mm bits to lower two bits
6049 // of form {0F, 0F_38, 0F_3A}
6050 byte2 |= opc;
6051 emit_int8(byte2);
6052
6053 // P1: byte 3 as Wvvvv1pp
6054 int byte3 = ((~nds_enc) & 0xf) << 3;
6055 // p[10] is always 1
6056 byte3 |= EVEX_F;
6057 byte3 |= (vex_w & 1) << 7;
6058 // confine pre opcode extensions in pp bits to lower two bits
6059 // of form {66, F3, F2}
6060 byte3 |= pre;
6061 emit_int8(byte3);
6062
6063 // P2: byte 4 as zL'Lbv'aaa
6064 int byte4 = (no_mask_reg) ? 0 : 1; // kregs are implemented in the low 3 bits as aaa (hard code k1, it will be initialized for now)
6065 // EVEX.v` for extending EVEX.vvvv or VIDX
6066 byte4 |= (evex_v ? 0: EVEX_V);
6067 // third EXEC.b for broadcast actions
6068 byte4 |= (is_extended_context ? EVEX_Rb : 0);
6069 // fourth EVEX.L'L for vector length : 0 is 128, 1 is 256, 2 is 512, currently we do not support 1024
6070 byte4 |= ((vector_len) & 0x3) << 5;
6071 // last is EVEX.z for zero/merge actions
6072 byte4 |= (is_merge_context ? EVEX_Z : 0);
6073 emit_int8(byte4);
6074 }
6075
6076 void Assembler::vex_prefix(Address adr, int nds_enc, int xreg_enc, VexSimdPrefix pre,
6077 VexOpcode opc, bool vex_w, int vector_len, bool legacy_mode, bool no_mask_reg) {
6078 bool vex_r = ((xreg_enc & 8) == 8) ? 1 : 0;
6079 bool vex_b = adr.base_needs_rex();
6080 bool vex_x = adr.index_needs_rex();
6081 _avx_vector_len = vector_len;
6082
6083 // if vector length is turned off, revert to AVX for vectors smaller than 512-bit
6084 if (_legacy_mode_vl && _instruction_uses_vl) {
6085 switch (vector_len) {
6086 case AVX_128bit:
6087 case AVX_256bit:
6088 legacy_mode = true;
6089 break;
6090 }
6091 }
6092
6093 if ((UseAVX > 2) && (legacy_mode == false))
6094 {
6095 bool evex_r = (xreg_enc >= 16);
6096 bool evex_v = (nds_enc >= 16);
6097 _is_evex_instruction = true;
6098 evex_prefix(vex_r, vex_b, vex_x, vex_w, evex_r, evex_v, nds_enc, pre, opc, false, false, vector_len, no_mask_reg);
6099 } else {
6100 vex_prefix(vex_r, vex_b, vex_x, vex_w, nds_enc, pre, opc, vector_len);
6101 }
6102 _instruction_uses_vl = false;
6103 }
6104
6105 int Assembler::vex_prefix_and_encode(int dst_enc, int nds_enc, int src_enc, VexSimdPrefix pre, VexOpcode opc,
6106 bool vex_w, int vector_len, bool legacy_mode, bool no_mask_reg ) {
6107 bool vex_r = ((dst_enc & 8) == 8) ? 1 : 0;
6108 bool vex_b = ((src_enc & 8) == 8) ? 1 : 0;
6109 bool vex_x = false;
6110 _avx_vector_len = vector_len;
6111
6112 // if vector length is turned off, revert to AVX for vectors smaller than 512-bit
6113 if (_legacy_mode_vl && _instruction_uses_vl) {
6114 switch (vector_len) {
6115 case AVX_128bit:
6116 case AVX_256bit:
6117 legacy_mode = true;
6118 break;
6119 }
6120 }
6121
6122 if ((UseAVX > 2) && (legacy_mode == false))
6123 {
6124 bool evex_r = (dst_enc >= 16);
6125 bool evex_v = (nds_enc >= 16);
6126 // can use vex_x as bank extender on rm encoding
6127 vex_x = (src_enc >= 16);
6128 evex_prefix(vex_r, vex_b, vex_x, vex_w, evex_r, evex_v, nds_enc, pre, opc, false, false, vector_len, no_mask_reg);
6129 } else {
6130 vex_prefix(vex_r, vex_b, vex_x, vex_w, nds_enc, pre, opc, vector_len);
6131 }
6132
6133 _instruction_uses_vl = false;
6134
6135 // return modrm byte components for operands
6136 return (((dst_enc & 7) << 3) | (src_enc & 7));
6137 }
6138
6139
6140 void Assembler::simd_prefix(XMMRegister xreg, XMMRegister nds, Address adr, VexSimdPrefix pre,
6141 bool no_mask_reg, VexOpcode opc, bool rex_w, int vector_len, bool legacy_mode) {
6142 if (UseAVX > 0) {
6143 int xreg_enc = xreg->encoding();
6144 int nds_enc = nds->is_valid() ? nds->encoding() : 0;
6145 vex_prefix(adr, nds_enc, xreg_enc, pre, opc, rex_w, vector_len, legacy_mode, no_mask_reg);
6146 } else {
6147 assert((nds == xreg) || (nds == xnoreg), "wrong sse encoding");
6148 rex_prefix(adr, xreg, pre, opc, rex_w);
6149 }
6150 }
6151
6152 int Assembler::simd_prefix_and_encode(XMMRegister dst, XMMRegister nds, XMMRegister src, VexSimdPrefix pre,
6153 bool no_mask_reg, VexOpcode opc, bool rex_w, int vector_len, bool legacy_mode) {
6154 int dst_enc = dst->encoding();
6155 int src_enc = src->encoding();
6156 if (UseAVX > 0) {
6157 int nds_enc = nds->is_valid() ? nds->encoding() : 0;
6158 return vex_prefix_and_encode(dst_enc, nds_enc, src_enc, pre, opc, rex_w, vector_len, legacy_mode, no_mask_reg);
6159 } else {
6160 assert((nds == dst) || (nds == src) || (nds == xnoreg), "wrong sse encoding");
6161 return rex_prefix_and_encode(dst_enc, src_enc, pre, opc, rex_w);
6162 }
6163 }
6164
6165 int Assembler::kreg_prefix_and_encode(KRegister dst, KRegister nds, KRegister src, VexSimdPrefix pre,
6166 bool no_mask_reg, VexOpcode opc, bool rex_w, int vector_len) {
6167 int dst_enc = dst->encoding();
6168 int src_enc = src->encoding();
6169 int nds_enc = nds->is_valid() ? nds->encoding() : 0;
6170 return vex_prefix_and_encode(dst_enc, nds_enc, src_enc, pre, opc, rex_w, vector_len, true, no_mask_reg);
6171 }
6172
6173 int Assembler::kreg_prefix_and_encode(KRegister dst, KRegister nds, Register src, VexSimdPrefix pre,
6174 bool no_mask_reg, VexOpcode opc, bool rex_w, int vector_len) {
6175 int dst_enc = dst->encoding();
6176 int src_enc = src->encoding();
6177 int nds_enc = nds->is_valid() ? nds->encoding() : 0;
6178 return vex_prefix_and_encode(dst_enc, nds_enc, src_enc, pre, opc, rex_w, vector_len, true, no_mask_reg);
6179 }
6180
6181 void Assembler::emit_simd_arith(int opcode, XMMRegister dst, Address src, VexSimdPrefix pre, bool no_mask_reg, bool legacy_mode) {
6182 InstructionMark im(this);
6183 simd_prefix(dst, dst, src, pre, no_mask_reg, VEX_OPCODE_0F, false, AVX_128bit, legacy_mode);
6184 emit_int8(opcode);
6185 emit_operand(dst, src);
6186 }
6187
6188 void Assembler::emit_simd_arith_q(int opcode, XMMRegister dst, Address src, VexSimdPrefix pre, bool no_mask_reg) {
6189 InstructionMark im(this);
6190 simd_prefix_q(dst, dst, src, pre, no_mask_reg);
6191 emit_int8(opcode);
6192 emit_operand(dst, src);
6193 }
6194
6195 void Assembler::emit_simd_arith(int opcode, XMMRegister dst, XMMRegister src, VexSimdPrefix pre, bool no_mask_reg, bool legacy_mode) {
6196 int encode = simd_prefix_and_encode(dst, dst, src, pre, no_mask_reg, VEX_OPCODE_0F, false, AVX_128bit, legacy_mode);
6197 emit_int8(opcode);
6198 emit_int8((unsigned char)(0xC0 | encode));
6199 }
6200
6201 void Assembler::emit_simd_arith_q(int opcode, XMMRegister dst, XMMRegister src, VexSimdPrefix pre, bool no_mask_reg) {
6202 int encode = simd_prefix_and_encode(dst, dst, src, pre, no_mask_reg, VEX_OPCODE_0F, true, AVX_128bit);
6203 emit_int8(opcode);
6204 emit_int8((unsigned char)(0xC0 | encode));
6205 }
6206
6207 // Versions with no second source register (non-destructive source).
6208 void Assembler::emit_simd_arith_nonds(int opcode, XMMRegister dst, Address src, VexSimdPrefix pre, bool opNoRegMask) {
6209 InstructionMark im(this);
6210 simd_prefix(dst, xnoreg, src, pre, opNoRegMask);
6211 emit_int8(opcode);
6212 emit_operand(dst, src);
6213 }
6214
6215 void Assembler::emit_simd_arith_nonds_q(int opcode, XMMRegister dst, Address src, VexSimdPrefix pre, bool opNoRegMask) {
6216 InstructionMark im(this);
6217 simd_prefix_q(dst, xnoreg, src, pre, opNoRegMask);
6218 emit_int8(opcode);
6219 emit_operand(dst, src);
6220 }
6221
6222 void Assembler::emit_simd_arith_nonds(int opcode, XMMRegister dst, XMMRegister src, VexSimdPrefix pre, bool no_mask_reg, bool legacy_mode) {
6223 int encode = simd_prefix_and_encode(dst, xnoreg, src, pre, no_mask_reg, VEX_OPCODE_0F, false, AVX_128bit, legacy_mode);
6224 emit_int8(opcode);
6225 emit_int8((unsigned char)(0xC0 | encode));
6226 }
6227
6228 void Assembler::emit_simd_arith_nonds_q(int opcode, XMMRegister dst, XMMRegister src, VexSimdPrefix pre, bool no_mask_reg) {
6229 int encode = simd_prefix_and_encode(dst, xnoreg, src, pre, no_mask_reg, VEX_OPCODE_0F, true);
6230 emit_int8(opcode);
6231 emit_int8((unsigned char)(0xC0 | encode));
6232 }
6233
6234 // 3-operands AVX instructions
6235 void Assembler::emit_vex_arith(int opcode, XMMRegister dst, XMMRegister nds, Address src,
6236 VexSimdPrefix pre, int vector_len, bool no_mask_reg, bool legacy_mode) {
6237 InstructionMark im(this);
6238 vex_prefix(dst, nds, src, pre, vector_len, no_mask_reg, legacy_mode);
6239 emit_int8(opcode);
6240 emit_operand(dst, src);
6241 }
6242
6243 void Assembler::emit_vex_arith_q(int opcode, XMMRegister dst, XMMRegister nds,
6244 Address src, VexSimdPrefix pre, int vector_len, bool no_mask_reg) {
6245 InstructionMark im(this);
6246 vex_prefix_q(dst, nds, src, pre, vector_len, no_mask_reg);
6247 emit_int8(opcode);
6248 emit_operand(dst, src);
6249 }
6250
6251 void Assembler::emit_vex_arith(int opcode, XMMRegister dst, XMMRegister nds, XMMRegister src,
6252 VexSimdPrefix pre, int vector_len, bool no_mask_reg, bool legacy_mode) {
6253 int encode = vex_prefix_and_encode(dst, nds, src, pre, vector_len, VEX_OPCODE_0F, legacy_mode, no_mask_reg);
6254 emit_int8(opcode);
6255 emit_int8((unsigned char)(0xC0 | encode));
6256 }
6257
6258 void Assembler::emit_vex_arith_q(int opcode, XMMRegister dst, XMMRegister nds, XMMRegister src,
6259 VexSimdPrefix pre, int vector_len, bool no_mask_reg) {
6260 int src_enc = src->encoding();
6261 int dst_enc = dst->encoding();
6262 int nds_enc = nds->is_valid() ? nds->encoding() : 0;
6263 int encode = vex_prefix_and_encode(dst_enc, nds_enc, src_enc, pre, VEX_OPCODE_0F, true, vector_len, false, no_mask_reg);
6264 emit_int8(opcode);
6265 emit_int8((unsigned char)(0xC0 | encode));
6266 }
6267
6268 #ifndef _LP64
6269
6270 void Assembler::incl(Register dst) {
6271 // Don't use it directly. Use MacroAssembler::incrementl() instead.
6272 emit_int8(0x40 | dst->encoding());
6273 }
6274
6275 void Assembler::lea(Register dst, Address src) {
6276 leal(dst, src);
6277 }
6278
6279 void Assembler::mov_literal32(Address dst, int32_t imm32, RelocationHolder const& rspec) {
6280 InstructionMark im(this);
6281 emit_int8((unsigned char)0xC7);
6282 emit_operand(rax, dst);
6283 emit_data((int)imm32, rspec, 0);
6284 }
6285
6286 void Assembler::mov_literal32(Register dst, int32_t imm32, RelocationHolder const& rspec) {
6287 InstructionMark im(this);
6288 int encode = prefix_and_encode(dst->encoding());
6289 emit_int8((unsigned char)(0xB8 | encode));
6290 emit_data((int)imm32, rspec, 0);
6291 }
6292
6293 void Assembler::popa() { // 32bit
6294 emit_int8(0x61);
6295 }
6296
6297 void Assembler::push_literal32(int32_t imm32, RelocationHolder const& rspec) {
6298 InstructionMark im(this);
6299 emit_int8(0x68);
6300 emit_data(imm32, rspec, 0);
6301 }
6302
6303 void Assembler::pusha() { // 32bit
6304 emit_int8(0x60);
6305 }
6306
6307 void Assembler::set_byte_if_not_zero(Register dst) {
6308 emit_int8(0x0F);
6309 emit_int8((unsigned char)0x95);
6310 emit_int8((unsigned char)(0xE0 | dst->encoding()));
6311 }
6312
6313 void Assembler::shldl(Register dst, Register src) {
6314 emit_int8(0x0F);
6315 emit_int8((unsigned char)0xA5);
6316 emit_int8((unsigned char)(0xC0 | src->encoding() << 3 | dst->encoding()));
6317 }
6318
6319 // 0F A4 / r ib
6320 void Assembler::shldl(Register dst, Register src, int8_t imm8) {
6321 emit_int8(0x0F);
6322 emit_int8((unsigned char)0xA4);
6323 emit_int8((unsigned char)(0xC0 | src->encoding() << 3 | dst->encoding()));
6324 emit_int8(imm8);
6325 }
6326
6327 void Assembler::shrdl(Register dst, Register src) {
6328 emit_int8(0x0F);
6329 emit_int8((unsigned char)0xAD);
6330 emit_int8((unsigned char)(0xC0 | src->encoding() << 3 | dst->encoding()));
6331 }
6332
6333 #else // LP64
6334
6335 void Assembler::set_byte_if_not_zero(Register dst) {
6336 int enc = prefix_and_encode(dst->encoding(), true);
6337 emit_int8(0x0F);
6338 emit_int8((unsigned char)0x95);
6339 emit_int8((unsigned char)(0xE0 | enc));
6340 }
6341
6342 // 64bit only pieces of the assembler
6343 // This should only be used by 64bit instructions that can use rip-relative
6344 // it cannot be used by instructions that want an immediate value.
6345
6346 bool Assembler::reachable(AddressLiteral adr) {
6347 int64_t disp;
6348 // None will force a 64bit literal to the code stream. Likely a placeholder
6349 // for something that will be patched later and we need to certain it will
6350 // always be reachable.
6351 if (adr.reloc() == relocInfo::none) {
6352 return false;
6353 }
6354 if (adr.reloc() == relocInfo::internal_word_type) {
6355 // This should be rip relative and easily reachable.
6356 return true;
6357 }
6358 if (adr.reloc() == relocInfo::virtual_call_type ||
6359 adr.reloc() == relocInfo::opt_virtual_call_type ||
6360 adr.reloc() == relocInfo::static_call_type ||
6361 adr.reloc() == relocInfo::static_stub_type ) {
6362 // This should be rip relative within the code cache and easily
6363 // reachable until we get huge code caches. (At which point
6364 // ic code is going to have issues).
6365 return true;
6366 }
6367 if (adr.reloc() != relocInfo::external_word_type &&
6368 adr.reloc() != relocInfo::poll_return_type && // these are really external_word but need special
6369 adr.reloc() != relocInfo::poll_type && // relocs to identify them
6370 adr.reloc() != relocInfo::runtime_call_type ) {
6371 return false;
6372 }
6373
6374 // Stress the correction code
6375 if (ForceUnreachable) {
6376 // Must be runtimecall reloc, see if it is in the codecache
6377 // Flipping stuff in the codecache to be unreachable causes issues
6378 // with things like inline caches where the additional instructions
6379 // are not handled.
6380 if (CodeCache::find_blob(adr._target) == NULL) {
6381 return false;
6382 }
6383 }
6384 // For external_word_type/runtime_call_type if it is reachable from where we
6385 // are now (possibly a temp buffer) and where we might end up
6386 // anywhere in the codeCache then we are always reachable.
6387 // This would have to change if we ever save/restore shared code
6388 // to be more pessimistic.
6389 disp = (int64_t)adr._target - ((int64_t)CodeCache::low_bound() + sizeof(int));
6390 if (!is_simm32(disp)) return false;
6391 disp = (int64_t)adr._target - ((int64_t)CodeCache::high_bound() + sizeof(int));
6392 if (!is_simm32(disp)) return false;
6393
6394 disp = (int64_t)adr._target - ((int64_t)pc() + sizeof(int));
6395
6396 // Because rip relative is a disp + address_of_next_instruction and we
6397 // don't know the value of address_of_next_instruction we apply a fudge factor
6398 // to make sure we will be ok no matter the size of the instruction we get placed into.
6399 // We don't have to fudge the checks above here because they are already worst case.
6400
6401 // 12 == override/rex byte, opcode byte, rm byte, sib byte, a 4-byte disp , 4-byte literal
6402 // + 4 because better safe than sorry.
6403 const int fudge = 12 + 4;
6404 if (disp < 0) {
6405 disp -= fudge;
6406 } else {
6407 disp += fudge;
6408 }
6409 return is_simm32(disp);
6410 }
6411
6412 // Check if the polling page is not reachable from the code cache using rip-relative
6413 // addressing.
6414 bool Assembler::is_polling_page_far() {
6415 intptr_t addr = (intptr_t)os::get_polling_page();
6416 return ForceUnreachable ||
6417 !is_simm32(addr - (intptr_t)CodeCache::low_bound()) ||
6418 !is_simm32(addr - (intptr_t)CodeCache::high_bound());
6419 }
6420
6421 void Assembler::emit_data64(jlong data,
6422 relocInfo::relocType rtype,
6423 int format) {
6424 if (rtype == relocInfo::none) {
6425 emit_int64(data);
6426 } else {
6427 emit_data64(data, Relocation::spec_simple(rtype), format);
6428 }
6429 }
6430
6431 void Assembler::emit_data64(jlong data,
6432 RelocationHolder const& rspec,
6433 int format) {
6434 assert(imm_operand == 0, "default format must be immediate in this file");
6435 assert(imm_operand == format, "must be immediate");
6436 assert(inst_mark() != NULL, "must be inside InstructionMark");
6437 // Do not use AbstractAssembler::relocate, which is not intended for
6438 // embedded words. Instead, relocate to the enclosing instruction.
6439 code_section()->relocate(inst_mark(), rspec, format);
6440 #ifdef ASSERT
6441 check_relocation(rspec, format);
6442 #endif
6443 emit_int64(data);
6444 }
6445
6446 int Assembler::prefix_and_encode(int reg_enc, bool byteinst) {
6447 if (reg_enc >= 8) {
6448 prefix(REX_B);
6449 reg_enc -= 8;
6450 } else if (byteinst && reg_enc >= 4) {
6451 prefix(REX);
6452 }
6453 return reg_enc;
6454 }
6455
6456 int Assembler::prefixq_and_encode(int reg_enc) {
6457 if (reg_enc < 8) {
6458 prefix(REX_W);
6459 } else {
6460 prefix(REX_WB);
6461 reg_enc -= 8;
6462 }
6463 return reg_enc;
6464 }
6465
6466 int Assembler::prefix_and_encode(int dst_enc, bool dst_is_byte, int src_enc, bool src_is_byte) {
6467 if (dst_enc < 8) {
6468 if (src_enc >= 8) {
6469 prefix(REX_B);
6470 src_enc -= 8;
6471 } else if ((src_is_byte && src_enc >= 4) || (dst_is_byte && dst_enc >= 4)) {
6472 prefix(REX);
6473 }
6474 } else {
6475 if (src_enc < 8) {
6476 prefix(REX_R);
6477 } else {
6478 prefix(REX_RB);
6479 src_enc -= 8;
6480 }
6481 dst_enc -= 8;
6482 }
6483 return dst_enc << 3 | src_enc;
6484 }
6485
6486 int Assembler::prefixq_and_encode(int dst_enc, int src_enc) {
6487 if (dst_enc < 8) {
6488 if (src_enc < 8) {
6489 prefix(REX_W);
6490 } else {
6491 prefix(REX_WB);
6492 src_enc -= 8;
6493 }
6494 } else {
6495 if (src_enc < 8) {
6496 prefix(REX_WR);
6497 } else {
6498 prefix(REX_WRB);
6499 src_enc -= 8;
6500 }
6501 dst_enc -= 8;
6502 }
6503 return dst_enc << 3 | src_enc;
6504 }
6505
6506 void Assembler::prefix(Register reg) {
6507 if (reg->encoding() >= 8) {
6508 prefix(REX_B);
6509 }
6510 }
6511
6512 void Assembler::prefix(Register dst, Register src, Prefix p) {
6513 if (src->encoding() >= 8) {
6514 p = (Prefix)(p | REX_B);
6515 }
6516 if (dst->encoding() >= 8) {
6517 p = (Prefix)( p | REX_R);
6518 }
6519 if (p != Prefix_EMPTY) {
6520 // do not generate an empty prefix
6521 prefix(p);
6522 }
6523 }
6524
6525 void Assembler::prefix(Register dst, Address adr, Prefix p) {
6526 if (adr.base_needs_rex()) {
6527 if (adr.index_needs_rex()) {
6528 assert(false, "prefix(Register dst, Address adr, Prefix p) does not support handling of an X");
6529 } else {
6530 prefix(REX_B);
6531 }
6532 } else {
6533 if (adr.index_needs_rex()) {
6534 assert(false, "prefix(Register dst, Address adr, Prefix p) does not support handling of an X");
6535 }
6536 }
6537 if (dst->encoding() >= 8) {
6538 p = (Prefix)(p | REX_R);
6539 }
6540 if (p != Prefix_EMPTY) {
6541 // do not generate an empty prefix
6542 prefix(p);
6543 }
6544 }
6545
6546 void Assembler::prefix(Address adr) {
6547 if (adr.base_needs_rex()) {
6548 if (adr.index_needs_rex()) {
6549 prefix(REX_XB);
6550 } else {
6551 prefix(REX_B);
6552 }
6553 } else {
6554 if (adr.index_needs_rex()) {
6555 prefix(REX_X);
6556 }
6557 }
6558 }
6559
6560 void Assembler::prefixq(Address adr) {
6561 if (adr.base_needs_rex()) {
6562 if (adr.index_needs_rex()) {
6563 prefix(REX_WXB);
6564 } else {
6565 prefix(REX_WB);
6566 }
6567 } else {
6568 if (adr.index_needs_rex()) {
6569 prefix(REX_WX);
6570 } else {
6571 prefix(REX_W);
6572 }
6573 }
6574 }
6575
6576
6577 void Assembler::prefix(Address adr, Register reg, bool byteinst) {
6578 if (reg->encoding() < 8) {
6579 if (adr.base_needs_rex()) {
6580 if (adr.index_needs_rex()) {
6581 prefix(REX_XB);
6582 } else {
6583 prefix(REX_B);
6584 }
6585 } else {
6586 if (adr.index_needs_rex()) {
6587 prefix(REX_X);
6588 } else if (byteinst && reg->encoding() >= 4 ) {
6589 prefix(REX);
6590 }
6591 }
6592 } else {
6593 if (adr.base_needs_rex()) {
6594 if (adr.index_needs_rex()) {
6595 prefix(REX_RXB);
6596 } else {
6597 prefix(REX_RB);
6598 }
6599 } else {
6600 if (adr.index_needs_rex()) {
6601 prefix(REX_RX);
6602 } else {
6603 prefix(REX_R);
6604 }
6605 }
6606 }
6607 }
6608
6609 void Assembler::prefixq(Address adr, Register src) {
6610 if (src->encoding() < 8) {
6611 if (adr.base_needs_rex()) {
6612 if (adr.index_needs_rex()) {
6613 prefix(REX_WXB);
6614 } else {
6615 prefix(REX_WB);
6616 }
6617 } else {
6618 if (adr.index_needs_rex()) {
6619 prefix(REX_WX);
6620 } else {
6621 prefix(REX_W);
6622 }
6623 }
6624 } else {
6625 if (adr.base_needs_rex()) {
6626 if (adr.index_needs_rex()) {
6627 prefix(REX_WRXB);
6628 } else {
6629 prefix(REX_WRB);
6630 }
6631 } else {
6632 if (adr.index_needs_rex()) {
6633 prefix(REX_WRX);
6634 } else {
6635 prefix(REX_WR);
6636 }
6637 }
6638 }
6639 }
6640
6641 void Assembler::prefix(Address adr, XMMRegister reg) {
6642 if (reg->encoding() < 8) {
6643 if (adr.base_needs_rex()) {
6644 if (adr.index_needs_rex()) {
6645 prefix(REX_XB);
6646 } else {
6647 prefix(REX_B);
6648 }
6649 } else {
6650 if (adr.index_needs_rex()) {
6651 prefix(REX_X);
6652 }
6653 }
6654 } else {
6655 if (adr.base_needs_rex()) {
6656 if (adr.index_needs_rex()) {
6657 prefix(REX_RXB);
6658 } else {
6659 prefix(REX_RB);
6660 }
6661 } else {
6662 if (adr.index_needs_rex()) {
6663 prefix(REX_RX);
6664 } else {
6665 prefix(REX_R);
6666 }
6667 }
6668 }
6669 }
6670
6671 void Assembler::prefixq(Address adr, XMMRegister src) {
6672 if (src->encoding() < 8) {
6673 if (adr.base_needs_rex()) {
6674 if (adr.index_needs_rex()) {
6675 prefix(REX_WXB);
6676 } else {
6677 prefix(REX_WB);
6678 }
6679 } else {
6680 if (adr.index_needs_rex()) {
6681 prefix(REX_WX);
6682 } else {
6683 prefix(REX_W);
6684 }
6685 }
6686 } else {
6687 if (adr.base_needs_rex()) {
6688 if (adr.index_needs_rex()) {
6689 prefix(REX_WRXB);
6690 } else {
6691 prefix(REX_WRB);
6692 }
6693 } else {
6694 if (adr.index_needs_rex()) {
6695 prefix(REX_WRX);
6696 } else {
6697 prefix(REX_WR);
6698 }
6699 }
6700 }
6701 }
6702
6703 void Assembler::adcq(Register dst, int32_t imm32) {
6704 (void) prefixq_and_encode(dst->encoding());
6705 emit_arith(0x81, 0xD0, dst, imm32);
6706 }
6707
6708 void Assembler::adcq(Register dst, Address src) {
6709 InstructionMark im(this);
6710 prefixq(src, dst);
6711 emit_int8(0x13);
6712 emit_operand(dst, src);
6713 }
6714
6715 void Assembler::adcq(Register dst, Register src) {
6716 (void) prefixq_and_encode(dst->encoding(), src->encoding());
6717 emit_arith(0x13, 0xC0, dst, src);
6718 }
6719
6720 void Assembler::addq(Address dst, int32_t imm32) {
6721 InstructionMark im(this);
6722 prefixq(dst);
6723 emit_arith_operand(0x81, rax, dst,imm32);
6724 }
6725
6726 void Assembler::addq(Address dst, Register src) {
6727 InstructionMark im(this);
6728 prefixq(dst, src);
6729 emit_int8(0x01);
6730 emit_operand(src, dst);
6731 }
6732
6733 void Assembler::addq(Register dst, int32_t imm32) {
6734 (void) prefixq_and_encode(dst->encoding());
6735 emit_arith(0x81, 0xC0, dst, imm32);
6736 }
6737
6738 void Assembler::addq(Register dst, Address src) {
6739 InstructionMark im(this);
6740 prefixq(src, dst);
6741 emit_int8(0x03);
6742 emit_operand(dst, src);
6743 }
6744
6745 void Assembler::addq(Register dst, Register src) {
6746 (void) prefixq_and_encode(dst->encoding(), src->encoding());
6747 emit_arith(0x03, 0xC0, dst, src);
6748 }
6749
6750 void Assembler::adcxq(Register dst, Register src) {
6751 //assert(VM_Version::supports_adx(), "adx instructions not supported");
6752 emit_int8((unsigned char)0x66);
6753 int encode = prefixq_and_encode(dst->encoding(), src->encoding());
6754 emit_int8(0x0F);
6755 emit_int8(0x38);
6756 emit_int8((unsigned char)0xF6);
6757 emit_int8((unsigned char)(0xC0 | encode));
6758 }
6759
6760 void Assembler::adoxq(Register dst, Register src) {
6761 //assert(VM_Version::supports_adx(), "adx instructions not supported");
6762 emit_int8((unsigned char)0xF3);
6763 int encode = prefixq_and_encode(dst->encoding(), src->encoding());
6764 emit_int8(0x0F);
6765 emit_int8(0x38);
6766 emit_int8((unsigned char)0xF6);
6767 emit_int8((unsigned char)(0xC0 | encode));
6768 }
6769
6770 void Assembler::andq(Address dst, int32_t imm32) {
6771 InstructionMark im(this);
6772 prefixq(dst);
6773 emit_int8((unsigned char)0x81);
6774 emit_operand(rsp, dst, 4);
6775 emit_int32(imm32);
6776 }
6777
6778 void Assembler::andq(Register dst, int32_t imm32) {
6779 (void) prefixq_and_encode(dst->encoding());
6780 emit_arith(0x81, 0xE0, dst, imm32);
6781 }
6782
6783 void Assembler::andq(Register dst, Address src) {
6784 InstructionMark im(this);
6785 prefixq(src, dst);
6786 emit_int8(0x23);
6787 emit_operand(dst, src);
6788 }
6789
6790 void Assembler::andq(Register dst, Register src) {
6791 (void) prefixq_and_encode(dst->encoding(), src->encoding());
6792 emit_arith(0x23, 0xC0, dst, src);
6793 }
6794
6795 void Assembler::andnq(Register dst, Register src1, Register src2) {
6796 assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
6797 int encode = vex_prefix_0F38_and_encode_q_legacy(dst, src1, src2);
6798 emit_int8((unsigned char)0xF2);
6799 emit_int8((unsigned char)(0xC0 | encode));
6800 }
6801
6802 void Assembler::andnq(Register dst, Register src1, Address src2) {
6803 InstructionMark im(this);
6804 assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
6805 vex_prefix_0F38_q_legacy(dst, src1, src2);
6806 emit_int8((unsigned char)0xF2);
6807 emit_operand(dst, src2);
6808 }
6809
6810 void Assembler::bsfq(Register dst, Register src) {
6811 int encode = prefixq_and_encode(dst->encoding(), src->encoding());
6812 emit_int8(0x0F);
6813 emit_int8((unsigned char)0xBC);
6814 emit_int8((unsigned char)(0xC0 | encode));
6815 }
6816
6817 void Assembler::bsrq(Register dst, Register src) {
6818 int encode = prefixq_and_encode(dst->encoding(), src->encoding());
6819 emit_int8(0x0F);
6820 emit_int8((unsigned char)0xBD);
6821 emit_int8((unsigned char)(0xC0 | encode));
6822 }
6823
6824 void Assembler::bswapq(Register reg) {
6825 int encode = prefixq_and_encode(reg->encoding());
6826 emit_int8(0x0F);
6827 emit_int8((unsigned char)(0xC8 | encode));
6828 }
6829
6830 void Assembler::blsiq(Register dst, Register src) {
6831 assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
6832 int encode = vex_prefix_0F38_and_encode_q_legacy(rbx, dst, src);
6833 emit_int8((unsigned char)0xF3);
6834 emit_int8((unsigned char)(0xC0 | encode));
6835 }
6836
6837 void Assembler::blsiq(Register dst, Address src) {
6838 InstructionMark im(this);
6839 assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
6840 vex_prefix_0F38_q_legacy(rbx, dst, src);
6841 emit_int8((unsigned char)0xF3);
6842 emit_operand(rbx, src);
6843 }
6844
6845 void Assembler::blsmskq(Register dst, Register src) {
6846 assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
6847 int encode = vex_prefix_0F38_and_encode_q_legacy(rdx, dst, src);
6848 emit_int8((unsigned char)0xF3);
6849 emit_int8((unsigned char)(0xC0 | encode));
6850 }
6851
6852 void Assembler::blsmskq(Register dst, Address src) {
6853 InstructionMark im(this);
6854 assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
6855 vex_prefix_0F38_q_legacy(rdx, dst, src);
6856 emit_int8((unsigned char)0xF3);
6857 emit_operand(rdx, src);
6858 }
6859
6860 void Assembler::blsrq(Register dst, Register src) {
6861 assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
6862 int encode = vex_prefix_0F38_and_encode_q_legacy(rcx, dst, src);
6863 emit_int8((unsigned char)0xF3);
6864 emit_int8((unsigned char)(0xC0 | encode));
6865 }
6866
6867 void Assembler::blsrq(Register dst, Address src) {
6868 InstructionMark im(this);
6869 assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
6870 vex_prefix_0F38_q_legacy(rcx, dst, src);
6871 emit_int8((unsigned char)0xF3);
6872 emit_operand(rcx, src);
6873 }
6874
6875 void Assembler::cdqq() {
6876 prefix(REX_W);
6877 emit_int8((unsigned char)0x99);
6878 }
6879
6880 void Assembler::clflush(Address adr) {
6881 prefix(adr);
6882 emit_int8(0x0F);
6883 emit_int8((unsigned char)0xAE);
6884 emit_operand(rdi, adr);
6885 }
6886
6887 void Assembler::cmovq(Condition cc, Register dst, Register src) {
6888 int encode = prefixq_and_encode(dst->encoding(), src->encoding());
6889 emit_int8(0x0F);
6890 emit_int8(0x40 | cc);
6891 emit_int8((unsigned char)(0xC0 | encode));
6892 }
6893
6894 void Assembler::cmovq(Condition cc, Register dst, Address src) {
6895 InstructionMark im(this);
6896 prefixq(src, dst);
6897 emit_int8(0x0F);
6898 emit_int8(0x40 | cc);
6899 emit_operand(dst, src);
6900 }
6901
6902 void Assembler::cmpq(Address dst, int32_t imm32) {
6903 InstructionMark im(this);
6904 prefixq(dst);
6905 emit_int8((unsigned char)0x81);
6906 emit_operand(rdi, dst, 4);
6907 emit_int32(imm32);
6908 }
6909
6910 void Assembler::cmpq(Register dst, int32_t imm32) {
6911 (void) prefixq_and_encode(dst->encoding());
6912 emit_arith(0x81, 0xF8, dst, imm32);
6913 }
6914
6915 void Assembler::cmpq(Address dst, Register src) {
6916 InstructionMark im(this);
6917 prefixq(dst, src);
6918 emit_int8(0x3B);
6919 emit_operand(src, dst);
6920 }
6921
6922 void Assembler::cmpq(Register dst, Register src) {
6923 (void) prefixq_and_encode(dst->encoding(), src->encoding());
6924 emit_arith(0x3B, 0xC0, dst, src);
6925 }
6926
6927 void Assembler::cmpq(Register dst, Address src) {
6928 InstructionMark im(this);
6929 prefixq(src, dst);
6930 emit_int8(0x3B);
6931 emit_operand(dst, src);
6932 }
6933
6934 void Assembler::cmpxchgq(Register reg, Address adr) {
6935 InstructionMark im(this);
6936 prefixq(adr, reg);
6937 emit_int8(0x0F);
6938 emit_int8((unsigned char)0xB1);
6939 emit_operand(reg, adr);
6940 }
6941
6942 void Assembler::cvtsi2sdq(XMMRegister dst, Register src) {
6943 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
6944 int encode = simd_prefix_and_encode_q(dst, dst, src, VEX_SIMD_F2, /* no_mask_reg */ true);
6945 emit_int8(0x2A);
6946 emit_int8((unsigned char)(0xC0 | encode));
6947 }
6948
6949 void Assembler::cvtsi2sdq(XMMRegister dst, Address src) {
6950 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
6951 if (VM_Version::supports_evex()) {
6952 _tuple_type = EVEX_T1S;
6953 _input_size_in_bits = EVEX_32bit;
6954 }
6955 InstructionMark im(this);
6956 simd_prefix_q(dst, dst, src, VEX_SIMD_F2, /* no_mask_reg */ true);
6957 emit_int8(0x2A);
6958 emit_operand(dst, src);
6959 }
6960
6961 void Assembler::cvtsi2ssq(XMMRegister dst, Address src) {
6962 NOT_LP64(assert(VM_Version::supports_sse(), ""));
6963 if (VM_Version::supports_evex()) {
6964 _tuple_type = EVEX_T1S;
6965 _input_size_in_bits = EVEX_32bit;
6966 }
6967 InstructionMark im(this);
6968 simd_prefix_q(dst, dst, src, VEX_SIMD_F3, /* no_mask_reg */ true);
6969 emit_int8(0x2A);
6970 emit_operand(dst, src);
6971 }
6972
6973 void Assembler::cvttsd2siq(Register dst, XMMRegister src) {
6974 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
6975 int encode = simd_prefix_and_encode_q(dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, /* no_mask_reg */ true);
6976 emit_int8(0x2C);
6977 emit_int8((unsigned char)(0xC0 | encode));
6978 }
6979
6980 void Assembler::cvttss2siq(Register dst, XMMRegister src) {
6981 NOT_LP64(assert(VM_Version::supports_sse(), ""));
6982 int encode = simd_prefix_and_encode_q(dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, /* no_mask_reg */ true);
6983 emit_int8(0x2C);
6984 emit_int8((unsigned char)(0xC0 | encode));
6985 }
6986
6987 void Assembler::decl(Register dst) {
6988 // Don't use it directly. Use MacroAssembler::decrementl() instead.
6989 // Use two-byte form (one-byte form is a REX prefix in 64-bit mode)
6990 int encode = prefix_and_encode(dst->encoding());
6991 emit_int8((unsigned char)0xFF);
6992 emit_int8((unsigned char)(0xC8 | encode));
6993 }
6994
6995 void Assembler::decq(Register dst) {
6996 // Don't use it directly. Use MacroAssembler::decrementq() instead.
6997 // Use two-byte form (one-byte from is a REX prefix in 64-bit mode)
6998 int encode = prefixq_and_encode(dst->encoding());
6999 emit_int8((unsigned char)0xFF);
7000 emit_int8(0xC8 | encode);
7001 }
7002
7003 void Assembler::decq(Address dst) {
7004 // Don't use it directly. Use MacroAssembler::decrementq() instead.
7005 InstructionMark im(this);
7006 prefixq(dst);
7007 emit_int8((unsigned char)0xFF);
7008 emit_operand(rcx, dst);
7009 }
7010
7011 void Assembler::fxrstor(Address src) {
7012 prefixq(src);
7013 emit_int8(0x0F);
7014 emit_int8((unsigned char)0xAE);
7015 emit_operand(as_Register(1), src);
7016 }
7017
7018 void Assembler::xrstor(Address src) {
7019 prefixq(src);
7020 emit_int8(0x0F);
7021 emit_int8((unsigned char)0xAE);
7022 emit_operand(as_Register(5), src);
7023 }
7024
7025 void Assembler::fxsave(Address dst) {
7026 prefixq(dst);
7027 emit_int8(0x0F);
7028 emit_int8((unsigned char)0xAE);
7029 emit_operand(as_Register(0), dst);
7030 }
7031
7032 void Assembler::xsave(Address dst) {
7033 prefixq(dst);
7034 emit_int8(0x0F);
7035 emit_int8((unsigned char)0xAE);
7036 emit_operand(as_Register(4), dst);
7037 }
7038
7039 void Assembler::idivq(Register src) {
7040 int encode = prefixq_and_encode(src->encoding());
7041 emit_int8((unsigned char)0xF7);
7042 emit_int8((unsigned char)(0xF8 | encode));
7043 }
7044
7045 void Assembler::imulq(Register dst, Register src) {
7046 int encode = prefixq_and_encode(dst->encoding(), src->encoding());
7047 emit_int8(0x0F);
7048 emit_int8((unsigned char)0xAF);
7049 emit_int8((unsigned char)(0xC0 | encode));
7050 }
7051
7052 void Assembler::imulq(Register dst, Register src, int value) {
7053 int encode = prefixq_and_encode(dst->encoding(), src->encoding());
7054 if (is8bit(value)) {
7055 emit_int8(0x6B);
7056 emit_int8((unsigned char)(0xC0 | encode));
7057 emit_int8(value & 0xFF);
7058 } else {
7059 emit_int8(0x69);
7060 emit_int8((unsigned char)(0xC0 | encode));
7061 emit_int32(value);
7062 }
7063 }
7064
7065 void Assembler::imulq(Register dst, Address src) {
7066 InstructionMark im(this);
7067 prefixq(src, dst);
7068 emit_int8(0x0F);
7069 emit_int8((unsigned char) 0xAF);
7070 emit_operand(dst, src);
7071 }
7072
7073 void Assembler::incl(Register dst) {
7074 // Don't use it directly. Use MacroAssembler::incrementl() instead.
7075 // Use two-byte form (one-byte from is a REX prefix in 64-bit mode)
7076 int encode = prefix_and_encode(dst->encoding());
7077 emit_int8((unsigned char)0xFF);
7078 emit_int8((unsigned char)(0xC0 | encode));
7079 }
7080
7081 void Assembler::incq(Register dst) {
7082 // Don't use it directly. Use MacroAssembler::incrementq() instead.
7083 // Use two-byte form (one-byte from is a REX prefix in 64-bit mode)
7084 int encode = prefixq_and_encode(dst->encoding());
7085 emit_int8((unsigned char)0xFF);
7086 emit_int8((unsigned char)(0xC0 | encode));
7087 }
7088
7089 void Assembler::incq(Address dst) {
7090 // Don't use it directly. Use MacroAssembler::incrementq() instead.
7091 InstructionMark im(this);
7092 prefixq(dst);
7093 emit_int8((unsigned char)0xFF);
7094 emit_operand(rax, dst);
7095 }
7096
7097 void Assembler::lea(Register dst, Address src) {
7098 leaq(dst, src);
7099 }
7100
7101 void Assembler::leaq(Register dst, Address src) {
7102 InstructionMark im(this);
7103 prefixq(src, dst);
7104 emit_int8((unsigned char)0x8D);
7105 emit_operand(dst, src);
7106 }
7107
7108 void Assembler::mov64(Register dst, int64_t imm64) {
7109 InstructionMark im(this);
7110 int encode = prefixq_and_encode(dst->encoding());
7111 emit_int8((unsigned char)(0xB8 | encode));
7112 emit_int64(imm64);
7113 }
7114
7115 void Assembler::mov_literal64(Register dst, intptr_t imm64, RelocationHolder const& rspec) {
7116 InstructionMark im(this);
7117 int encode = prefixq_and_encode(dst->encoding());
7118 emit_int8(0xB8 | encode);
7119 emit_data64(imm64, rspec);
7120 }
7121
7122 void Assembler::mov_narrow_oop(Register dst, int32_t imm32, RelocationHolder const& rspec) {
7123 InstructionMark im(this);
7124 int encode = prefix_and_encode(dst->encoding());
7125 emit_int8((unsigned char)(0xB8 | encode));
7126 emit_data((int)imm32, rspec, narrow_oop_operand);
7127 }
7128
7129 void Assembler::mov_narrow_oop(Address dst, int32_t imm32, RelocationHolder const& rspec) {
7130 InstructionMark im(this);
7131 prefix(dst);
7132 emit_int8((unsigned char)0xC7);
7133 emit_operand(rax, dst, 4);
7134 emit_data((int)imm32, rspec, narrow_oop_operand);
7135 }
7136
7137 void Assembler::cmp_narrow_oop(Register src1, int32_t imm32, RelocationHolder const& rspec) {
7138 InstructionMark im(this);
7139 int encode = prefix_and_encode(src1->encoding());
7140 emit_int8((unsigned char)0x81);
7141 emit_int8((unsigned char)(0xF8 | encode));
7142 emit_data((int)imm32, rspec, narrow_oop_operand);
7143 }
7144
7145 void Assembler::cmp_narrow_oop(Address src1, int32_t imm32, RelocationHolder const& rspec) {
7146 InstructionMark im(this);
7147 prefix(src1);
7148 emit_int8((unsigned char)0x81);
7149 emit_operand(rax, src1, 4);
7150 emit_data((int)imm32, rspec, narrow_oop_operand);
7151 }
7152
7153 void Assembler::lzcntq(Register dst, Register src) {
7154 assert(VM_Version::supports_lzcnt(), "encoding is treated as BSR");
7155 emit_int8((unsigned char)0xF3);
7156 int encode = prefixq_and_encode(dst->encoding(), src->encoding());
7157 emit_int8(0x0F);
7158 emit_int8((unsigned char)0xBD);
7159 emit_int8((unsigned char)(0xC0 | encode));
7160 }
7161
7162 void Assembler::movdq(XMMRegister dst, Register src) {
7163 // table D-1 says MMX/SSE2
7164 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
7165 int encode = simd_prefix_and_encode_q(dst, src, VEX_SIMD_66, /* no_mask_reg */ true);
7166 emit_int8(0x6E);
7167 emit_int8((unsigned char)(0xC0 | encode));
7168 }
7169
7170 void Assembler::movdq(Register dst, XMMRegister src) {
7171 // table D-1 says MMX/SSE2
7172 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
7173 // swap src/dst to get correct prefix
7174 int encode = simd_prefix_and_encode_q(src, dst, VEX_SIMD_66, /* no_mask_reg */ true);
7175 emit_int8(0x7E);
7176 emit_int8((unsigned char)(0xC0 | encode));
7177 }
7178
7179 void Assembler::movq(Register dst, Register src) {
7180 int encode = prefixq_and_encode(dst->encoding(), src->encoding());
7181 emit_int8((unsigned char)0x8B);
7182 emit_int8((unsigned char)(0xC0 | encode));
7183 }
7184
7185 void Assembler::movq(Register dst, Address src) {
7186 InstructionMark im(this);
7187 prefixq(src, dst);
7188 emit_int8((unsigned char)0x8B);
7189 emit_operand(dst, src);
7190 }
7191
7192 void Assembler::movq(Address dst, Register src) {
7193 InstructionMark im(this);
7194 prefixq(dst, src);
7195 emit_int8((unsigned char)0x89);
7196 emit_operand(src, dst);
7197 }
7198
7199 void Assembler::movsbq(Register dst, Address src) {
7200 InstructionMark im(this);
7201 prefixq(src, dst);
7202 emit_int8(0x0F);
7203 emit_int8((unsigned char)0xBE);
7204 emit_operand(dst, src);
7205 }
7206
7207 void Assembler::movsbq(Register dst, Register src) {
7208 int encode = prefixq_and_encode(dst->encoding(), src->encoding());
7209 emit_int8(0x0F);
7210 emit_int8((unsigned char)0xBE);
7211 emit_int8((unsigned char)(0xC0 | encode));
7212 }
7213
7214 void Assembler::movslq(Register dst, int32_t imm32) {
7215 // dbx shows movslq(rcx, 3) as movq $0x0000000049000000,(%rbx)
7216 // and movslq(r8, 3); as movl $0x0000000048000000,(%rbx)
7217 // as a result we shouldn't use until tested at runtime...
7218 ShouldNotReachHere();
7219 InstructionMark im(this);
7220 int encode = prefixq_and_encode(dst->encoding());
7221 emit_int8((unsigned char)(0xC7 | encode));
7222 emit_int32(imm32);
7223 }
7224
7225 void Assembler::movslq(Address dst, int32_t imm32) {
7226 assert(is_simm32(imm32), "lost bits");
7227 InstructionMark im(this);
7228 prefixq(dst);
7229 emit_int8((unsigned char)0xC7);
7230 emit_operand(rax, dst, 4);
7231 emit_int32(imm32);
7232 }
7233
7234 void Assembler::movslq(Register dst, Address src) {
7235 InstructionMark im(this);
7236 prefixq(src, dst);
7237 emit_int8(0x63);
7238 emit_operand(dst, src);
7239 }
7240
7241 void Assembler::movslq(Register dst, Register src) {
7242 int encode = prefixq_and_encode(dst->encoding(), src->encoding());
7243 emit_int8(0x63);
7244 emit_int8((unsigned char)(0xC0 | encode));
7245 }
7246
7247 void Assembler::movswq(Register dst, Address src) {
7248 InstructionMark im(this);
7249 prefixq(src, dst);
7250 emit_int8(0x0F);
7251 emit_int8((unsigned char)0xBF);
7252 emit_operand(dst, src);
7253 }
7254
7255 void Assembler::movswq(Register dst, Register src) {
7256 int encode = prefixq_and_encode(dst->encoding(), src->encoding());
7257 emit_int8((unsigned char)0x0F);
7258 emit_int8((unsigned char)0xBF);
7259 emit_int8((unsigned char)(0xC0 | encode));
7260 }
7261
7262 void Assembler::movzbq(Register dst, Address src) {
7263 InstructionMark im(this);
7264 prefixq(src, dst);
7265 emit_int8((unsigned char)0x0F);
7266 emit_int8((unsigned char)0xB6);
7267 emit_operand(dst, src);
7268 }
7269
7270 void Assembler::movzbq(Register dst, Register src) {
7271 int encode = prefixq_and_encode(dst->encoding(), src->encoding());
7272 emit_int8(0x0F);
7273 emit_int8((unsigned char)0xB6);
7274 emit_int8(0xC0 | encode);
7275 }
7276
7277 void Assembler::movzwq(Register dst, Address src) {
7278 InstructionMark im(this);
7279 prefixq(src, dst);
7280 emit_int8((unsigned char)0x0F);
7281 emit_int8((unsigned char)0xB7);
7282 emit_operand(dst, src);
7283 }
7284
7285 void Assembler::movzwq(Register dst, Register src) {
7286 int encode = prefixq_and_encode(dst->encoding(), src->encoding());
7287 emit_int8((unsigned char)0x0F);
7288 emit_int8((unsigned char)0xB7);
7289 emit_int8((unsigned char)(0xC0 | encode));
7290 }
7291
7292 void Assembler::mulq(Address src) {
7293 InstructionMark im(this);
7294 prefixq(src);
7295 emit_int8((unsigned char)0xF7);
7296 emit_operand(rsp, src);
7297 }
7298
7299 void Assembler::mulq(Register src) {
7300 int encode = prefixq_and_encode(src->encoding());
7301 emit_int8((unsigned char)0xF7);
7302 emit_int8((unsigned char)(0xE0 | encode));
7303 }
7304
7305 void Assembler::mulxq(Register dst1, Register dst2, Register src) {
7306 assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported");
7307 int encode = vex_prefix_and_encode(dst1->encoding(), dst2->encoding(), src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_38,
7308 /* vex_w */ true, AVX_128bit, /* legacy_mode */ true, /* no_mask_reg */ false);
7309 emit_int8((unsigned char)0xF6);
7310 emit_int8((unsigned char)(0xC0 | encode));
7311 }
7312
7313 void Assembler::negq(Register dst) {
7314 int encode = prefixq_and_encode(dst->encoding());
7315 emit_int8((unsigned char)0xF7);
7316 emit_int8((unsigned char)(0xD8 | encode));
7317 }
7318
7319 void Assembler::notq(Register dst) {
7320 int encode = prefixq_and_encode(dst->encoding());
7321 emit_int8((unsigned char)0xF7);
7322 emit_int8((unsigned char)(0xD0 | encode));
7323 }
7324
7325 void Assembler::orq(Address dst, int32_t imm32) {
7326 InstructionMark im(this);
7327 prefixq(dst);
7328 emit_int8((unsigned char)0x81);
7329 emit_operand(rcx, dst, 4);
7330 emit_int32(imm32);
7331 }
7332
7333 void Assembler::orq(Register dst, int32_t imm32) {
7334 (void) prefixq_and_encode(dst->encoding());
7335 emit_arith(0x81, 0xC8, dst, imm32);
7336 }
7337
7338 void Assembler::orq(Register dst, Address src) {
7339 InstructionMark im(this);
7340 prefixq(src, dst);
7341 emit_int8(0x0B);
7342 emit_operand(dst, src);
7343 }
7344
7345 void Assembler::orq(Register dst, Register src) {
7346 (void) prefixq_and_encode(dst->encoding(), src->encoding());
7347 emit_arith(0x0B, 0xC0, dst, src);
7348 }
7349
7350 void Assembler::popa() { // 64bit
7351 movq(r15, Address(rsp, 0));
7352 movq(r14, Address(rsp, wordSize));
7353 movq(r13, Address(rsp, 2 * wordSize));
7354 movq(r12, Address(rsp, 3 * wordSize));
7355 movq(r11, Address(rsp, 4 * wordSize));
7356 movq(r10, Address(rsp, 5 * wordSize));
7357 movq(r9, Address(rsp, 6 * wordSize));
7358 movq(r8, Address(rsp, 7 * wordSize));
7359 movq(rdi, Address(rsp, 8 * wordSize));
7360 movq(rsi, Address(rsp, 9 * wordSize));
7361 movq(rbp, Address(rsp, 10 * wordSize));
7362 // skip rsp
7363 movq(rbx, Address(rsp, 12 * wordSize));
7364 movq(rdx, Address(rsp, 13 * wordSize));
7365 movq(rcx, Address(rsp, 14 * wordSize));
7366 movq(rax, Address(rsp, 15 * wordSize));
7367
7368 addq(rsp, 16 * wordSize);
7369 }
7370
7371 void Assembler::popcntq(Register dst, Address src) {
7372 assert(VM_Version::supports_popcnt(), "must support");
7373 InstructionMark im(this);
7374 emit_int8((unsigned char)0xF3);
7375 prefixq(src, dst);
7376 emit_int8((unsigned char)0x0F);
7377 emit_int8((unsigned char)0xB8);
7378 emit_operand(dst, src);
7379 }
7380
7381 void Assembler::popcntq(Register dst, Register src) {
7382 assert(VM_Version::supports_popcnt(), "must support");
7383 emit_int8((unsigned char)0xF3);
7384 int encode = prefixq_and_encode(dst->encoding(), src->encoding());
7385 emit_int8((unsigned char)0x0F);
7386 emit_int8((unsigned char)0xB8);
7387 emit_int8((unsigned char)(0xC0 | encode));
7388 }
7389
7390 void Assembler::popq(Address dst) {
7391 InstructionMark im(this);
7392 prefixq(dst);
7393 emit_int8((unsigned char)0x8F);
7394 emit_operand(rax, dst);
7395 }
7396
7397 void Assembler::pusha() { // 64bit
7398 // we have to store original rsp. ABI says that 128 bytes
7399 // below rsp are local scratch.
7400 movq(Address(rsp, -5 * wordSize), rsp);
7401
7402 subq(rsp, 16 * wordSize);
7403
7404 movq(Address(rsp, 15 * wordSize), rax);
7405 movq(Address(rsp, 14 * wordSize), rcx);
7406 movq(Address(rsp, 13 * wordSize), rdx);
7407 movq(Address(rsp, 12 * wordSize), rbx);
7408 // skip rsp
7409 movq(Address(rsp, 10 * wordSize), rbp);
7410 movq(Address(rsp, 9 * wordSize), rsi);
7411 movq(Address(rsp, 8 * wordSize), rdi);
7412 movq(Address(rsp, 7 * wordSize), r8);
7413 movq(Address(rsp, 6 * wordSize), r9);
7414 movq(Address(rsp, 5 * wordSize), r10);
7415 movq(Address(rsp, 4 * wordSize), r11);
7416 movq(Address(rsp, 3 * wordSize), r12);
7417 movq(Address(rsp, 2 * wordSize), r13);
7418 movq(Address(rsp, wordSize), r14);
7419 movq(Address(rsp, 0), r15);
7420 }
7421
7422 void Assembler::pushq(Address src) {
7423 InstructionMark im(this);
7424 prefixq(src);
7425 emit_int8((unsigned char)0xFF);
7426 emit_operand(rsi, src);
7427 }
7428
7429 void Assembler::rclq(Register dst, int imm8) {
7430 assert(isShiftCount(imm8 >> 1), "illegal shift count");
7431 int encode = prefixq_and_encode(dst->encoding());
7432 if (imm8 == 1) {
7433 emit_int8((unsigned char)0xD1);
7434 emit_int8((unsigned char)(0xD0 | encode));
7435 } else {
7436 emit_int8((unsigned char)0xC1);
7437 emit_int8((unsigned char)(0xD0 | encode));
7438 emit_int8(imm8);
7439 }
7440 }
7441
7442 void Assembler::rcrq(Register dst, int imm8) {
7443 assert(isShiftCount(imm8 >> 1), "illegal shift count");
7444 int encode = prefixq_and_encode(dst->encoding());
7445 if (imm8 == 1) {
7446 emit_int8((unsigned char)0xD1);
7447 emit_int8((unsigned char)(0xD8 | encode));
7448 } else {
7449 emit_int8((unsigned char)0xC1);
7450 emit_int8((unsigned char)(0xD8 | encode));
7451 emit_int8(imm8);
7452 }
7453 }
7454
7455 void Assembler::rorq(Register dst, int imm8) {
7456 assert(isShiftCount(imm8 >> 1), "illegal shift count");
7457 int encode = prefixq_and_encode(dst->encoding());
7458 if (imm8 == 1) {
7459 emit_int8((unsigned char)0xD1);
7460 emit_int8((unsigned char)(0xC8 | encode));
7461 } else {
7462 emit_int8((unsigned char)0xC1);
7463 emit_int8((unsigned char)(0xc8 | encode));
7464 emit_int8(imm8);
7465 }
7466 }
7467
7468 void Assembler::rorxq(Register dst, Register src, int imm8) {
7469 assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported");
7470 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_3A,
7471 /* vex_w */ true, AVX_128bit, /* legacy_mode */ true, /* no_mask_reg */ false);
7472 emit_int8((unsigned char)0xF0);
7473 emit_int8((unsigned char)(0xC0 | encode));
7474 emit_int8(imm8);
7475 }
7476
7477 void Assembler::sarq(Register dst, int imm8) {
7478 assert(isShiftCount(imm8 >> 1), "illegal shift count");
7479 int encode = prefixq_and_encode(dst->encoding());
7480 if (imm8 == 1) {
7481 emit_int8((unsigned char)0xD1);
7482 emit_int8((unsigned char)(0xF8 | encode));
7483 } else {
7484 emit_int8((unsigned char)0xC1);
7485 emit_int8((unsigned char)(0xF8 | encode));
7486 emit_int8(imm8);
7487 }
7488 }
7489
7490 void Assembler::sarq(Register dst) {
7491 int encode = prefixq_and_encode(dst->encoding());
7492 emit_int8((unsigned char)0xD3);
7493 emit_int8((unsigned char)(0xF8 | encode));
7494 }
7495
7496 void Assembler::sbbq(Address dst, int32_t imm32) {
7497 InstructionMark im(this);
7498 prefixq(dst);
7499 emit_arith_operand(0x81, rbx, dst, imm32);
7500 }
7501
7502 void Assembler::sbbq(Register dst, int32_t imm32) {
7503 (void) prefixq_and_encode(dst->encoding());
7504 emit_arith(0x81, 0xD8, dst, imm32);
7505 }
7506
7507 void Assembler::sbbq(Register dst, Address src) {
7508 InstructionMark im(this);
7509 prefixq(src, dst);
7510 emit_int8(0x1B);
7511 emit_operand(dst, src);
7512 }
7513
7514 void Assembler::sbbq(Register dst, Register src) {
7515 (void) prefixq_and_encode(dst->encoding(), src->encoding());
7516 emit_arith(0x1B, 0xC0, dst, src);
7517 }
7518
7519 void Assembler::shlq(Register dst, int imm8) {
7520 assert(isShiftCount(imm8 >> 1), "illegal shift count");
7521 int encode = prefixq_and_encode(dst->encoding());
7522 if (imm8 == 1) {
7523 emit_int8((unsigned char)0xD1);
7524 emit_int8((unsigned char)(0xE0 | encode));
7525 } else {
7526 emit_int8((unsigned char)0xC1);
7527 emit_int8((unsigned char)(0xE0 | encode));
7528 emit_int8(imm8);
7529 }
7530 }
7531
7532 void Assembler::shlq(Register dst) {
7533 int encode = prefixq_and_encode(dst->encoding());
7534 emit_int8((unsigned char)0xD3);
7535 emit_int8((unsigned char)(0xE0 | encode));
7536 }
7537
7538 void Assembler::shrq(Register dst, int imm8) {
7539 assert(isShiftCount(imm8 >> 1), "illegal shift count");
7540 int encode = prefixq_and_encode(dst->encoding());
7541 emit_int8((unsigned char)0xC1);
7542 emit_int8((unsigned char)(0xE8 | encode));
7543 emit_int8(imm8);
7544 }
7545
7546 void Assembler::shrq(Register dst) {
7547 int encode = prefixq_and_encode(dst->encoding());
7548 emit_int8((unsigned char)0xD3);
7549 emit_int8(0xE8 | encode);
7550 }
7551
7552 void Assembler::subq(Address dst, int32_t imm32) {
7553 InstructionMark im(this);
7554 prefixq(dst);
7555 emit_arith_operand(0x81, rbp, dst, imm32);
7556 }
7557
7558 void Assembler::subq(Address dst, Register src) {
7559 InstructionMark im(this);
7560 prefixq(dst, src);
7561 emit_int8(0x29);
7562 emit_operand(src, dst);
7563 }
7564
7565 void Assembler::subq(Register dst, int32_t imm32) {
7566 (void) prefixq_and_encode(dst->encoding());
7567 emit_arith(0x81, 0xE8, dst, imm32);
7568 }
7569
7570 // Force generation of a 4 byte immediate value even if it fits into 8bit
7571 void Assembler::subq_imm32(Register dst, int32_t imm32) {
7572 (void) prefixq_and_encode(dst->encoding());
7573 emit_arith_imm32(0x81, 0xE8, dst, imm32);
7574 }
7575
7576 void Assembler::subq(Register dst, Address src) {
7577 InstructionMark im(this);
7578 prefixq(src, dst);
7579 emit_int8(0x2B);
7580 emit_operand(dst, src);
7581 }
7582
7583 void Assembler::subq(Register dst, Register src) {
7584 (void) prefixq_and_encode(dst->encoding(), src->encoding());
7585 emit_arith(0x2B, 0xC0, dst, src);
7586 }
7587
7588 void Assembler::testq(Register dst, int32_t imm32) {
7589 // not using emit_arith because test
7590 // doesn't support sign-extension of
7591 // 8bit operands
7592 int encode = dst->encoding();
7593 if (encode == 0) {
7594 prefix(REX_W);
7595 emit_int8((unsigned char)0xA9);
7596 } else {
7597 encode = prefixq_and_encode(encode);
7598 emit_int8((unsigned char)0xF7);
7599 emit_int8((unsigned char)(0xC0 | encode));
7600 }
7601 emit_int32(imm32);
7602 }
7603
7604 void Assembler::testq(Register dst, Register src) {
7605 (void) prefixq_and_encode(dst->encoding(), src->encoding());
7606 emit_arith(0x85, 0xC0, dst, src);
7607 }
7608
7609 void Assembler::xaddq(Address dst, Register src) {
7610 InstructionMark im(this);
7611 prefixq(dst, src);
7612 emit_int8(0x0F);
7613 emit_int8((unsigned char)0xC1);
7614 emit_operand(src, dst);
7615 }
7616
7617 void Assembler::xchgq(Register dst, Address src) {
7618 InstructionMark im(this);
7619 prefixq(src, dst);
7620 emit_int8((unsigned char)0x87);
7621 emit_operand(dst, src);
7622 }
7623
7624 void Assembler::xchgq(Register dst, Register src) {
7625 int encode = prefixq_and_encode(dst->encoding(), src->encoding());
7626 emit_int8((unsigned char)0x87);
7627 emit_int8((unsigned char)(0xc0 | encode));
7628 }
7629
7630 void Assembler::xorq(Register dst, Register src) {
7631 (void) prefixq_and_encode(dst->encoding(), src->encoding());
7632 emit_arith(0x33, 0xC0, dst, src);
7633 }
7634
7635 void Assembler::xorq(Register dst, Address src) {
7636 InstructionMark im(this);
7637 prefixq(src, dst);
7638 emit_int8(0x33);
7639 emit_operand(dst, src);
7640 }
7641
7642 #endif // !LP64