1 /*
   2  * Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  *
  23  */
  24 
  25 #include "precompiled.hpp"
  26 #include "opto/castnode.hpp"
  27 #include "opto/graphKit.hpp"
  28 #include "opto/phaseX.hpp"
  29 #include "opto/rootnode.hpp"
  30 #include "opto/vector.hpp"
  31 #include "utilities/macros.hpp"
  32 
  33 void PhaseVector::optimize_vector_boxes() {
  34   Compile::TracePhase tp("vector_elimination", &timers[_t_vector_elimination]);
  35 
  36   assert(C->inlining_incrementally() == false, "sanity");
  37 
  38   C->set_inlining_incrementally(true); // FIXME another way to signal GraphKit it's post-parsing phase?
  39 
  40   C->for_igvn()->clear();
  41   C->initial_gvn()->replace_with(&_igvn);
  42 
  43   expand_vunbox_nodes();
  44 
  45   C->inline_vector_reboxing_calls();
  46 
  47   expand_vbox_nodes();
  48   eliminate_vbox_alloc_nodes();
  49 
  50   C->set_inlining_incrementally(false); // FIXME another way to signal GraphKit it's post-parsing phase?
  51 
  52   do_cleanup();
  53 }
  54 
  55 void PhaseVector::do_cleanup() {
  56   if (C->failing())  return;
  57   {
  58     Compile::TracePhase tp("vector_pru", &timers[_t_vector_pru]);
  59     ResourceMark rm;
  60     PhaseRemoveUseless pru(C->initial_gvn(), C->for_igvn());
  61   }
  62 
  63   if (C->failing())  return;
  64 
  65   {
  66     Compile::TracePhase tp("incrementalInline_igvn", &timers[_t_vector_igvn]);
  67     _igvn = PhaseIterGVN(C->initial_gvn());
  68     _igvn.optimize();
  69   }
  70 }
  71 
  72 void PhaseVector::expand_vbox_nodes() {
  73   if (C->failing())  return;
  74 
  75   int macro_idx = C->macro_count() - 1;
  76   while (macro_idx >= 0) {
  77     Node * n = C->macro_node(macro_idx);
  78     assert(n->is_macro(), "only macro nodes expected here");
  79     if (n->Opcode() == Op_VectorBox) {
  80       VectorBoxNode* vbox = static_cast<VectorBoxNode*>(n);
  81       expand_vbox_node(vbox);
  82       if (C->failing())  return;
  83       C->print_method(PHASE_EXPAND_VBOX, vbox, 3);
  84     }
  85     if (C->failing())  return;
  86     macro_idx = MIN2(macro_idx - 1, C->macro_count() - 1);
  87   }
  88 }
  89 
  90 void PhaseVector::expand_vunbox_nodes() {
  91   if (C->failing())  return;
  92 
  93   int macro_idx = C->macro_count() - 1;
  94   while (macro_idx >= 0) {
  95     Node * n = C->macro_node(macro_idx);
  96     assert(n->is_macro(), "only macro nodes expected here");
  97     if (n->Opcode() == Op_VectorUnbox) {
  98       VectorUnboxNode* vec_unbox = static_cast<VectorUnboxNode*>(n);
  99       expand_vunbox_node(vec_unbox);
 100       if (C->failing())  return;
 101       C->print_method(PHASE_EXPAND_VUNBOX, vec_unbox, 3);
 102     }
 103     if (C->failing())  return;
 104     macro_idx = MIN2(macro_idx - 1, C->macro_count() - 1);
 105   }
 106 }
 107 
 108 void PhaseVector::eliminate_vbox_alloc_nodes() {
 109   if (C->failing())  return;
 110 
 111   int macro_idx = C->macro_count() - 1;
 112   while (macro_idx >= 0) {
 113     Node * n = C->macro_node(macro_idx);
 114     assert(n->is_macro(), "only macro nodes expected here");
 115     if (n->Opcode() == Op_VectorBoxAllocate) {
 116       VectorBoxAllocateNode* vbox_alloc = static_cast<VectorBoxAllocateNode*>(n);
 117       eliminate_vbox_alloc_node(vbox_alloc);
 118       if (C->failing())  return;
 119       C->print_method(PHASE_ELIMINATE_VBOX_ALLOC, vbox_alloc, 3);
 120     }
 121     if (C->failing())  return;
 122     macro_idx = MIN2(macro_idx - 1, C->macro_count() - 1);
 123   }
 124 }
 125 
 126 static JVMState* clone_jvms(Compile* C, SafePointNode* sfpt) {
 127   JVMState* new_jvms = sfpt->jvms()->clone_shallow(C);
 128   uint size = sfpt->req();
 129   SafePointNode* map = new SafePointNode(size, new_jvms);
 130   for (uint i = 0; i < size; i++) {
 131     map->init_req(i, sfpt->in(i));
 132   }
 133   new_jvms->set_map(map);
 134   return new_jvms;
 135 }
 136 
 137 void PhaseVector::expand_vbox_node(VectorBoxNode* vec_box) {
 138   if (vec_box->outcnt() > 0) {
 139     Node* vbox = vec_box->in(VectorBoxNode::Box);
 140     Node* vect = vec_box->in(VectorBoxNode::Value);
 141     Node* result = expand_vbox_node_helper(vbox, vect, vec_box->box_type(), vec_box->vec_type());
 142     C->gvn_replace_by(vec_box, result);
 143   }
 144   C->remove_macro_node(vec_box);
 145 }
 146 
 147 Node* PhaseVector::expand_vbox_node_helper(Node* vbox,
 148                                        Node* vect,
 149                                        const TypeInstPtr* box_type,
 150                                        const TypeVect* vect_type) {
 151   if (vbox->is_Phi() && vect->is_Phi()) {
 152     assert(vbox->as_Phi()->region() == vect->as_Phi()->region(), "");
 153     Node* new_phi = new PhiNode(vbox->as_Phi()->region(), box_type);
 154     for (uint i = 1; i < vbox->req(); i++) {
 155       Node* new_box = expand_vbox_node_helper(vbox->in(i), vect->in(i), box_type, vect_type);
 156       new_phi->set_req(i, new_box);
 157     }
 158     new_phi = C->initial_gvn()->transform(new_phi);
 159     return new_phi;
 160   } else if (vbox->is_Proj() && vbox->in(0)->Opcode() == Op_VectorBoxAllocate) {
 161     VectorBoxAllocateNode* vbox_alloc = static_cast<VectorBoxAllocateNode*>(vbox->in(0));
 162     return expand_vbox_alloc_node(vbox_alloc, vect, box_type, vect_type);
 163   } else {
 164     assert(!vbox->is_Phi(), "");
 165     // TODO: ensure that expanded vbox is initialized with the same value (vect).
 166     return vbox; // already expanded
 167   }
 168 }
 169 
 170 static bool is_vector_mask(ciKlass* klass) {
 171   return klass->is_subclass_of(ciEnv::current()->vector_VectorMask_klass());
 172 }
 173 
 174 static bool is_vector_shuffle(ciKlass* klass) {
 175   return klass->is_subclass_of(ciEnv::current()->vector_VectorShuffle_klass());
 176 }
 177 
 178 Node* PhaseVector::expand_vbox_alloc_node(VectorBoxAllocateNode* vbox_alloc,
 179                                           Node* value,
 180                                           const TypeInstPtr* box_type,
 181                                           const TypeVect* vect_type) {
 182   JVMState* jvms = clone_jvms(C, vbox_alloc);
 183   GraphKit kit(jvms);
 184   PhaseGVN& gvn = kit.gvn();
 185 
 186   ciInstanceKlass* box_klass = box_type->klass()->as_instance_klass();
 187   BasicType bt = vect_type->element_basic_type();
 188   int num_elem = vect_type->length();
 189 
 190   bool is_mask = is_vector_mask(box_klass);
 191   if (is_mask && bt != T_BOOLEAN) {
 192     value = gvn.transform(new VectorStoreMaskNode(value, bt, num_elem));
 193     // Although type of mask depends on its definition, in terms of storage everything is stored in boolean array.
 194     bt = T_BOOLEAN;
 195     assert(value->as_Vector()->bottom_type()->is_vect()->element_basic_type() == bt,
 196            "must be consistent with mask representation");
 197   }
 198 
 199   // Generate array allocation for the field which holds the values.
 200   const TypeKlassPtr* array_klass = TypeKlassPtr::make(ciTypeArrayKlass::make(bt));
 201   Node* arr = kit.new_array(kit.makecon(array_klass), kit.intcon(num_elem), 1);
 202 
 203   // Store the vector value into the array.
 204   // (The store should be captured by InitializeNode and turned into initialized store later.)
 205   Node* arr_adr = kit.array_element_address(arr, kit.intcon(0), bt);
 206   const TypePtr* arr_adr_type = arr_adr->bottom_type()->is_ptr();
 207   Node* arr_mem = kit.memory(arr_adr);
 208   Node* vstore = gvn.transform(StoreVectorNode::make(0,
 209                                                      kit.control(),
 210                                                      arr_mem,
 211                                                      arr_adr,
 212                                                      arr_adr_type,
 213                                                      value,
 214                                                      num_elem));
 215   kit.set_memory(vstore, arr_adr_type);
 216 
 217   C->set_max_vector_size(MAX2(C->max_vector_size(), vect_type->length_in_bytes()));
 218 
 219   // Generate the allocate for the Vector object.
 220   const TypeKlassPtr* klass_type = box_type->as_klass_type();
 221   Node* klass_node = kit.makecon(klass_type);
 222   Node* vec_obj = kit.new_instance(klass_node);
 223 
 224   // Store the allocated array into object.
 225   ciField* field = ciEnv::current()->vector_VectorPayload_klass()->get_field_by_name(ciSymbol::payload_name(),
 226                                                                                      ciSymbol::object_signature(),
 227                                                                                      false);
 228   assert(field != NULL, "");
 229   Node* vec_field = kit.basic_plus_adr(vec_obj, field->offset_in_bytes());
 230   const TypePtr* vec_adr_type = vec_field->bottom_type()->is_ptr();
 231 
 232   // The store should be captured by InitializeNode and turned into initialized store later.
 233   Node* field_store = gvn.transform(kit.access_store_at(vec_obj,
 234                                                             vec_field,
 235                                                             vec_adr_type,
 236                                                             arr,
 237                                                             TypeOopPtr::make_from_klass(field->type()->as_klass()),
 238                                                             T_OBJECT,
 239                                                             IN_HEAP));
 240   kit.set_memory(field_store, vec_adr_type);
 241 
 242   kit.replace_call(vbox_alloc, vec_obj, true);
 243   C->remove_macro_node(vbox_alloc);
 244   return vec_obj;
 245 }
 246 
 247 void PhaseVector::expand_vunbox_node(VectorUnboxNode* vec_unbox) {
 248   if (vec_unbox->outcnt() > 0) {
 249     GraphKit kit;
 250     PhaseGVN& gvn = kit.gvn();
 251 
 252     Node* obj = vec_unbox->obj();
 253     const TypeInstPtr* tinst = gvn.type(obj)->isa_instptr();
 254     ciInstanceKlass* from_kls = tinst->klass()->as_instance_klass();
 255     BasicType bt = vec_unbox->vect_type()->element_basic_type();
 256     BasicType masktype = bt;
 257     BasicType elem_bt;
 258 
 259     if (is_vector_mask(from_kls)) {
 260       bt = T_BOOLEAN;
 261     } else if (is_vector_shuffle(from_kls)) {
 262       if (vec_unbox->is_shuffle_to_vector() == true) {
 263         elem_bt = bt;
 264       }
 265       bt = T_BYTE;
 266     }
 267 
 268     ciField* field = ciEnv::current()->vector_VectorPayload_klass()->get_field_by_name(ciSymbol::payload_name(),
 269                                                                                        ciSymbol::object_signature(),
 270                                                                                        false);
 271     assert(field != NULL, "");
 272     int offset = field->offset_in_bytes();
 273     Node* vec_adr = kit.basic_plus_adr(obj, offset);
 274 
 275     Node* mem = vec_unbox->mem();
 276     Node* ctrl = vec_unbox->in(0);
 277     Node* vec_field_ld = LoadNode::make(gvn,
 278                                         ctrl,
 279                                         mem,
 280                                         vec_adr,
 281                                         vec_adr->bottom_type()->is_ptr(),
 282                                         TypeOopPtr::make_from_klass(field->type()->as_klass()),
 283                                         T_OBJECT,
 284                                         MemNode::unordered);
 285     vec_field_ld = gvn.transform(vec_field_ld);
 286 
 287     // For proper aliasing, attach concrete payload type.
 288     ciKlass* payload_klass = ciTypeArrayKlass::make(bt);
 289     const Type* payload_type = TypeAryPtr::make_from_klass(payload_klass)->cast_to_ptr_type(TypePtr::NotNull);
 290     vec_field_ld = gvn.transform(new CastPPNode(vec_field_ld, payload_type));
 291 
 292     Node* adr = kit.array_element_address(vec_field_ld, gvn.intcon(0), bt);
 293     const TypePtr* adr_type = adr->bottom_type()->is_ptr();
 294     const TypeVect* vt = vec_unbox->bottom_type()->is_vect();
 295     int num_elem = vt->length();
 296     Node* vec_val_load = LoadVectorNode::make(0,
 297                                               ctrl,
 298                                               mem,
 299                                               adr,
 300                                               adr_type,
 301                                               num_elem,
 302                                               bt);
 303     vec_val_load = gvn.transform(vec_val_load);
 304 
 305     C->set_max_vector_size(MAX2(C->max_vector_size(), vt->length_in_bytes()));
 306 
 307     if (is_vector_mask(from_kls) && masktype != T_BOOLEAN) {
 308       assert(vec_unbox->bottom_type()->is_vect()->element_basic_type() == masktype, "expect mask type consistency");
 309       vec_val_load = gvn.transform(new VectorLoadMaskNode(vec_val_load, TypeVect::make(masktype, num_elem)));
 310     } else if (is_vector_shuffle(from_kls)) {
 311       if (vec_unbox->is_shuffle_to_vector() == false) {
 312         assert(vec_unbox->bottom_type()->is_vect()->element_basic_type() == masktype, "expect shuffle type consistency");
 313         vec_val_load = gvn.transform(new VectorLoadShuffleNode(vec_val_load, TypeVect::make(masktype, num_elem)));
 314       } else if (elem_bt != T_BYTE) {
 315         vec_val_load = gvn.transform(VectorCastNode::make(Op_VectorCastB2X, vec_val_load, elem_bt, num_elem));
 316       }
 317     }
 318 
 319     gvn.hash_delete(vec_unbox);
 320     vec_unbox->disconnect_inputs(NULL, C);
 321     C->gvn_replace_by(vec_unbox, vec_val_load);
 322   }
 323   C->remove_macro_node(vec_unbox);
 324 }
 325 
 326 void PhaseVector::eliminate_vbox_alloc_node(VectorBoxAllocateNode* vbox_alloc) {
 327   JVMState* jvms = clone_jvms(C, vbox_alloc);
 328   GraphKit kit(jvms);
 329   // Remove VBA, but leave a safepoint behind.
 330   // Otherwise, it may end up with a loop without any safepoint polls.
 331   kit.replace_call(vbox_alloc, kit.map(), true);
 332   C->remove_macro_node(vbox_alloc);
 333 }