319 bool inline_updateByteBufferAdler32();
320 bool inline_multiplyToLen();
321 bool inline_hasNegatives();
322 bool inline_squareToLen();
323 bool inline_mulAdd();
324 bool inline_montgomeryMultiply();
325 bool inline_montgomerySquare();
326 bool inline_vectorizedMismatch();
327 bool inline_fma(vmIntrinsics::ID id);
328
329 bool inline_profileBoolean();
330 bool inline_isCompileConstant();
331
332 // Vector API support
333 bool inline_vector_nary_operation(int n);
334 bool inline_vector_broadcast_coerced();
335 bool inline_vector_mem_operation(bool is_store);
336 bool inline_vector_reduction();
337 bool inline_vector_test();
338 bool inline_vector_blend();
339 bool inline_vector_compare();
340 bool inline_vector_broadcast_int();
341 bool inline_vector_cast_reinterpret(bool is_cast);
342 bool inline_vector_extract();
343 bool inline_vector_insert();
344 Node* box_vector(Node* in, const TypeInstPtr* vbox_type, BasicType bt, int num_elem);
345 Node* unbox_vector(Node* in, const TypeInstPtr* vbox_type, BasicType bt, int num_elem);
346 Node* shift_count(Node* cnt, int shift_op, BasicType bt, int num_elem);
347 Node* gen_call_to_svml(int vector_api_op_id, BasicType bt, int num_elem, Node* opd1, Node* opd2);
348 void set_vector_result(Node* result, bool set_res = true);
349
350 void clear_upper_avx() {
351 #ifdef X86
352 if (UseAVX >= 2) {
353 C->set_clear_upper_avx(true);
354 }
355 #endif
356 }
357 };
358
891
892 case vmIntrinsics::_VectorUnaryOp:
893 return inline_vector_nary_operation(1);
894 case vmIntrinsics::_VectorBinaryOp:
895 return inline_vector_nary_operation(2);
896 case vmIntrinsics::_VectorTernaryOp:
897 return inline_vector_nary_operation(3);
898
899 case vmIntrinsics::_VectorBroadcastCoerced:
900 return inline_vector_broadcast_coerced();
901 case vmIntrinsics::_VectorLoadOp:
902 return inline_vector_mem_operation(/*is_store=*/false);
903 case vmIntrinsics::_VectorStoreOp:
904 return inline_vector_mem_operation(/*is_store=*/true);
905 case vmIntrinsics::_VectorReductionCoerced:
906 return inline_vector_reduction();
907 case vmIntrinsics::_VectorTest:
908 return inline_vector_test();
909 case vmIntrinsics::_VectorBlend:
910 return inline_vector_blend();
911 case vmIntrinsics::_VectorCompare:
912 return inline_vector_compare();
913 case vmIntrinsics::_VectorBroadcastInt:
914 return inline_vector_broadcast_int();
915 case vmIntrinsics::_VectorReinterpret:
916 return inline_vector_cast_reinterpret(/*is_cast*/ false);
917 case vmIntrinsics::_VectorCast:
918 return inline_vector_cast_reinterpret(/*is_cast*/ true);
919 case vmIntrinsics::_VectorInsert:
920 return inline_vector_insert();
921 case vmIntrinsics::_VectorExtract:
922 return inline_vector_extract();
923
924 default:
925 // If you get here, it may be that someone has added a new intrinsic
926 // to the list in vmSymbols.hpp without implementing it here.
927 #ifndef PRODUCT
928 if ((PrintMiscellaneous && (Verbose || WizardMode)) || PrintOpto) {
929 tty->print_cr("*** Warning: Unimplemented intrinsic %s(%d)",
930 vmIntrinsics::name_at(intrinsic_id()), intrinsic_id());
7263 }
7264
7265 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
7266 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
7267
7268 ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
7269 const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
7270
7271 Node* v1 = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
7272 Node* v2 = unbox_vector(argument(6), vbox_type, elem_bt, num_elem);
7273
7274 if (v1 == NULL || v2 == NULL) {
7275 return false; // operand unboxing failed
7276 }
7277 BoolTest::mask pred = (BoolTest::mask)cond->get_con();
7278 const TypeVect* vt = TypeVect::make(mask_bt, num_elem);
7279 Node* operation = _gvn.transform(new VectorMaskCmpNode(pred, v1, v2, vt));
7280
7281 Node* box = box_vector(operation, mbox_type, mask_bt, num_elem);
7282 set_vector_result(box);
7283
7284 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
7285 return true;
7286 }
7287
7288 Node* LibraryCallKit::shift_count(Node* cnt, int shift_op, BasicType bt, int num_elem) {
7289 assert(bt == T_INT || bt == T_LONG, "Only long and int are supported");
7290 juint mask = (bt == T_INT) ? (BitsPerInt - 1) : (BitsPerLong - 1);
7291 const TypeInt* t = cnt->find_int_type();
7292 if (t != NULL && t->is_con()) {
7293 juint shift = t->get_con();
7294 if (shift > mask) {
7295 return _gvn.transform(ConNode::make(TypeInt::make(shift & mask)));
7296 } else {
7297 return cnt;
7298 }
7299 } else {
7300 Node* maskedcnt = cnt;
7301 if (t == NULL || t->_lo < 0 || t->_hi > (int)mask) {
7302 Node* nmask = _gvn.transform(ConNode::make(TypeInt::make(mask)));
|
319 bool inline_updateByteBufferAdler32();
320 bool inline_multiplyToLen();
321 bool inline_hasNegatives();
322 bool inline_squareToLen();
323 bool inline_mulAdd();
324 bool inline_montgomeryMultiply();
325 bool inline_montgomerySquare();
326 bool inline_vectorizedMismatch();
327 bool inline_fma(vmIntrinsics::ID id);
328
329 bool inline_profileBoolean();
330 bool inline_isCompileConstant();
331
332 // Vector API support
333 bool inline_vector_nary_operation(int n);
334 bool inline_vector_broadcast_coerced();
335 bool inline_vector_mem_operation(bool is_store);
336 bool inline_vector_reduction();
337 bool inline_vector_test();
338 bool inline_vector_blend();
339 bool inline_vector_rearrange();
340 bool inline_vector_compare();
341 bool inline_vector_broadcast_int();
342 bool inline_vector_cast_reinterpret(bool is_cast);
343 bool inline_vector_extract();
344 bool inline_vector_insert();
345 Node* box_vector(Node* in, const TypeInstPtr* vbox_type, BasicType bt, int num_elem);
346 Node* unbox_vector(Node* in, const TypeInstPtr* vbox_type, BasicType bt, int num_elem);
347 Node* shift_count(Node* cnt, int shift_op, BasicType bt, int num_elem);
348 Node* gen_call_to_svml(int vector_api_op_id, BasicType bt, int num_elem, Node* opd1, Node* opd2);
349 void set_vector_result(Node* result, bool set_res = true);
350
351 void clear_upper_avx() {
352 #ifdef X86
353 if (UseAVX >= 2) {
354 C->set_clear_upper_avx(true);
355 }
356 #endif
357 }
358 };
359
892
893 case vmIntrinsics::_VectorUnaryOp:
894 return inline_vector_nary_operation(1);
895 case vmIntrinsics::_VectorBinaryOp:
896 return inline_vector_nary_operation(2);
897 case vmIntrinsics::_VectorTernaryOp:
898 return inline_vector_nary_operation(3);
899
900 case vmIntrinsics::_VectorBroadcastCoerced:
901 return inline_vector_broadcast_coerced();
902 case vmIntrinsics::_VectorLoadOp:
903 return inline_vector_mem_operation(/*is_store=*/false);
904 case vmIntrinsics::_VectorStoreOp:
905 return inline_vector_mem_operation(/*is_store=*/true);
906 case vmIntrinsics::_VectorReductionCoerced:
907 return inline_vector_reduction();
908 case vmIntrinsics::_VectorTest:
909 return inline_vector_test();
910 case vmIntrinsics::_VectorBlend:
911 return inline_vector_blend();
912 case vmIntrinsics::_VectorRearrange:
913 return inline_vector_rearrange();
914 case vmIntrinsics::_VectorCompare:
915 return inline_vector_compare();
916 case vmIntrinsics::_VectorBroadcastInt:
917 return inline_vector_broadcast_int();
918 case vmIntrinsics::_VectorReinterpret:
919 return inline_vector_cast_reinterpret(/*is_cast*/ false);
920 case vmIntrinsics::_VectorCast:
921 return inline_vector_cast_reinterpret(/*is_cast*/ true);
922 case vmIntrinsics::_VectorInsert:
923 return inline_vector_insert();
924 case vmIntrinsics::_VectorExtract:
925 return inline_vector_extract();
926
927 default:
928 // If you get here, it may be that someone has added a new intrinsic
929 // to the list in vmSymbols.hpp without implementing it here.
930 #ifndef PRODUCT
931 if ((PrintMiscellaneous && (Verbose || WizardMode)) || PrintOpto) {
932 tty->print_cr("*** Warning: Unimplemented intrinsic %s(%d)",
933 vmIntrinsics::name_at(intrinsic_id()), intrinsic_id());
7266 }
7267
7268 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
7269 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
7270
7271 ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
7272 const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
7273
7274 Node* v1 = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
7275 Node* v2 = unbox_vector(argument(6), vbox_type, elem_bt, num_elem);
7276
7277 if (v1 == NULL || v2 == NULL) {
7278 return false; // operand unboxing failed
7279 }
7280 BoolTest::mask pred = (BoolTest::mask)cond->get_con();
7281 const TypeVect* vt = TypeVect::make(mask_bt, num_elem);
7282 Node* operation = _gvn.transform(new VectorMaskCmpNode(pred, v1, v2, vt));
7283
7284 Node* box = box_vector(operation, mbox_type, mask_bt, num_elem);
7285 set_vector_result(box);
7286
7287 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
7288 return true;
7289 }
7290
7291 // static
7292 // <V extends Vector, Sh extends Shuffle>
7293 // V rearrangeOp(Class<V> vectorClass, Class<Sh> shuffleClass, Class< ? > elementType, int vlen,
7294 // V v1, Sh sh,
7295 // VectorSwizzleOp<V, Sh, S, E> defaultImpl) { ...
7296
7297 bool LibraryCallKit::inline_vector_rearrange() {
7298 const TypeInstPtr* vector_klass = gvn().type(argument(0))->is_instptr();
7299 const TypeInstPtr* shuffle_klass = gvn().type(argument(1))->is_instptr();
7300 const TypeInstPtr* elem_klass = gvn().type(argument(2))->is_instptr();
7301 const TypeInt* vlen = gvn().type(argument(3))->is_int();
7302
7303 if (shuffle_klass->const_oop() == NULL || vector_klass->const_oop() == NULL ||
7304 elem_klass->const_oop() == NULL || !vlen->is_con()) {
7305 return false; // not enough info for intrinsification
7306 }
7307 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
7308 if (!elem_type->is_primitive_type()) {
7309 return false; // should be primitive type
7310 }
7311 BasicType elem_bt = elem_type->basic_type();
7312 BasicType shuffle_bt = elem_bt;
7313 int num_elem = vlen->get_con();
7314
7315 if (!arch_supports_vector(Op_VectorLoadShuffle, num_elem, elem_bt, VecMaskNotUsed)) {
7316 return false; // not supported
7317 }
7318 if (!arch_supports_vector(Op_VectorRearrange, num_elem, elem_bt, VecMaskNotUsed)) {
7319 return false; // not supported
7320 }
7321 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
7322 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
7323
7324 ciKlass* shbox_klass = shuffle_klass->const_oop()->as_instance()->java_lang_Class_klass();
7325 const TypeInstPtr* shbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, shbox_klass);
7326
7327 Node* v1 = unbox_vector(argument(4), vbox_type, elem_bt, num_elem);
7328 Node* shuffle = unbox_vector(argument(5), shbox_type, shuffle_bt, num_elem);
7329
7330 if (v1 == NULL || shuffle == NULL) {
7331 return false; // operand unboxing failed
7332 }
7333
7334 Node* rearrange = _gvn.transform(new VectorRearrangeNode(v1, shuffle));
7335 Node* box = box_vector(rearrange, vbox_type, elem_bt, num_elem);
7336 set_result(box);
7337
7338 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
7339 return true;
7340 }
7341
7342 Node* LibraryCallKit::shift_count(Node* cnt, int shift_op, BasicType bt, int num_elem) {
7343 assert(bt == T_INT || bt == T_LONG, "Only long and int are supported");
7344 juint mask = (bt == T_INT) ? (BitsPerInt - 1) : (BitsPerLong - 1);
7345 const TypeInt* t = cnt->find_int_type();
7346 if (t != NULL && t->is_con()) {
7347 juint shift = t->get_con();
7348 if (shift > mask) {
7349 return _gvn.transform(ConNode::make(TypeInt::make(shift & mask)));
7350 } else {
7351 return cnt;
7352 }
7353 } else {
7354 Node* maskedcnt = cnt;
7355 if (t == NULL || t->_lo < 0 || t->_hi > (int)mask) {
7356 Node* nmask = _gvn.transform(ConNode::make(TypeInt::make(mask)));
|