153 return true;
154 }
155 if (n->Opcode() == Op_DecodeN) {
156 return true;
157 }
158 #ifdef ASSERT
159 tty->print("need barrier on?: "); n->dump();
160 #endif
161 return true;
162 }
163
164 bool ShenandoahReadBarrierNode::dominates_memory_rb_impl(PhaseGVN* phase,
165 Node* b1,
166 Node* b2,
167 Node* current,
168 bool linear) {
169 ResourceMark rm;
170 VectorSet visited(Thread::current()->resource_area());
171 Node_Stack phis(0);
172
173 for(;;) {
174 if (current == NULL) {
175 return false;
176 } else if (visited.test_set(current->_idx) || current->is_top() || current == b1) {
177 current = NULL;
178 while (phis.is_nonempty() && current == NULL) {
179 uint idx = phis.index();
180 Node* phi = phis.node();
181 if (idx >= phi->req()) {
182 phis.pop();
183 } else {
184 current = phi->in(idx);
185 phis.set_index(idx+1);
186 }
187 }
188 if (current == NULL) {
189 return true;
190 }
191 } else if (current == phase->C->immutable_memory()) {
192 return false;
193 } else if (current->isa_Phi()) {
210 current = current->in(0);
211 } else if (current->is_Call()) {
212 return false; // TODO: Maybe improve by looking at the call's memory effects?
213 } else if (current->is_MemBar()) {
214 return false; // TODO: Do we need to stop at *any* membar?
215 } else if (current->is_MergeMem()) {
216 // if (true) return false;
217 // tty->print_cr("current == mergemem: "); current->dump();
218 const TypePtr* adr_type = brooks_pointer_type(phase->type(b2));
219 uint alias_idx = phase->C->get_alias_index(adr_type);
220 current = current->as_MergeMem()->memory_at(alias_idx);
221 } else {
222 // tty->print_cr("what else can we see here:");
223 #ifdef ASSERT
224 current->dump();
225 #endif
226 ShouldNotReachHere();
227 return false;
228 }
229 }
230 }
231
232 bool ShenandoahReadBarrierNode::dominates_memory_rb(PhaseGVN* phase, Node* b1, Node* b2, bool linear) {
233 return dominates_memory_rb_impl(phase, b1->in(Memory), b2, b2->in(Memory), linear);
234 }
235
236 bool ShenandoahReadBarrierNode::is_independent(const Type* in_type, const Type* this_type) {
237 assert(in_type->isa_oopptr(), "expect oop ptr");
238 assert(this_type->isa_oopptr(), "expect oop ptr");
239 /*
240 if ((! in_type->isa_oopptr()) || (! this_type->isa_oopptr())) {
241 #ifdef ASSERT
242 tty->print_cr("not oopptr");
243 tty->print("in: "); in_type->dump(); tty->print_cr(" ");
244 tty->print("this: "); this_type->dump(); tty->print_cr(" ");
245 #endif
246 return false;
247 }
248 */
249
250 ciKlass* in_kls = in_type->is_oopptr()->klass();
251 ciKlass* this_kls = this_type->is_oopptr()->klass();
2226
2227 Node* loop = n_loop_head;
2228 if (n_loop_head->is_CountedLoop() && n_loop_head->as_CountedLoop()->is_main_loop()) {
2229 Node* res = try_move_shenandoah_barrier_before_pre_loop(n_loop_head->in(LoopNode::EntryControl), val_ctrl);
2230 if (res != NULL) {
2231 loop = res;
2232 }
2233 }
2234
2235 return try_move_shenandoah_barrier_before_loop_helper(n, loop, val_ctrl, mem);
2236 }
2237 }
2238 }
2239 Node* ctrl = try_move_shenandoah_barrier_before_pre_loop(n->in(0), val_ctrl);
2240 if (ctrl != NULL) {
2241 return try_move_shenandoah_barrier_before_loop_helper(n, ctrl, val_ctrl, n->in(ShenandoahBarrierNode::Memory));
2242 }
2243 }
2244 return NULL;
2245 }
|
153 return true;
154 }
155 if (n->Opcode() == Op_DecodeN) {
156 return true;
157 }
158 #ifdef ASSERT
159 tty->print("need barrier on?: "); n->dump();
160 #endif
161 return true;
162 }
163
164 bool ShenandoahReadBarrierNode::dominates_memory_rb_impl(PhaseGVN* phase,
165 Node* b1,
166 Node* b2,
167 Node* current,
168 bool linear) {
169 ResourceMark rm;
170 VectorSet visited(Thread::current()->resource_area());
171 Node_Stack phis(0);
172
173 for(int i = 0; i < 10; i++) {
174 if (current == NULL) {
175 return false;
176 } else if (visited.test_set(current->_idx) || current->is_top() || current == b1) {
177 current = NULL;
178 while (phis.is_nonempty() && current == NULL) {
179 uint idx = phis.index();
180 Node* phi = phis.node();
181 if (idx >= phi->req()) {
182 phis.pop();
183 } else {
184 current = phi->in(idx);
185 phis.set_index(idx+1);
186 }
187 }
188 if (current == NULL) {
189 return true;
190 }
191 } else if (current == phase->C->immutable_memory()) {
192 return false;
193 } else if (current->isa_Phi()) {
210 current = current->in(0);
211 } else if (current->is_Call()) {
212 return false; // TODO: Maybe improve by looking at the call's memory effects?
213 } else if (current->is_MemBar()) {
214 return false; // TODO: Do we need to stop at *any* membar?
215 } else if (current->is_MergeMem()) {
216 // if (true) return false;
217 // tty->print_cr("current == mergemem: "); current->dump();
218 const TypePtr* adr_type = brooks_pointer_type(phase->type(b2));
219 uint alias_idx = phase->C->get_alias_index(adr_type);
220 current = current->as_MergeMem()->memory_at(alias_idx);
221 } else {
222 // tty->print_cr("what else can we see here:");
223 #ifdef ASSERT
224 current->dump();
225 #endif
226 ShouldNotReachHere();
227 return false;
228 }
229 }
230 return false;
231 }
232
233 bool ShenandoahReadBarrierNode::is_independent(Node* mem) {
234 if (mem->is_Phi() || mem->is_Proj() || mem->is_MergeMem()) {
235 return true;
236 } else if (mem->Opcode() == Op_ShenandoahWriteBarrier) {
237 const Type* mem_type = mem->bottom_type();
238 const Type* this_type = bottom_type();
239 if (is_independent(mem_type, this_type)) {
240 return true;
241 } else {
242 return false;
243 }
244 } else if (mem->is_Call() || mem->is_MemBar()) {
245 return false;
246 }
247 #ifdef ASSERT
248 mem->dump();
249 #endif
250 ShouldNotReachHere();
251 return true;
252 }
253
254
255 bool ShenandoahReadBarrierNode::dominates_memory_rb(PhaseGVN* phase, Node* b1, Node* b2, bool linear) {
256 return dominates_memory_rb_impl(phase, b1->in(Memory), b2, b2->in(Memory), linear);
257 }
258
259 bool ShenandoahReadBarrierNode::is_independent(const Type* in_type, const Type* this_type) {
260 assert(in_type->isa_oopptr(), "expect oop ptr");
261 assert(this_type->isa_oopptr(), "expect oop ptr");
262 /*
263 if ((! in_type->isa_oopptr()) || (! this_type->isa_oopptr())) {
264 #ifdef ASSERT
265 tty->print_cr("not oopptr");
266 tty->print("in: "); in_type->dump(); tty->print_cr(" ");
267 tty->print("this: "); this_type->dump(); tty->print_cr(" ");
268 #endif
269 return false;
270 }
271 */
272
273 ciKlass* in_kls = in_type->is_oopptr()->klass();
274 ciKlass* this_kls = this_type->is_oopptr()->klass();
2249
2250 Node* loop = n_loop_head;
2251 if (n_loop_head->is_CountedLoop() && n_loop_head->as_CountedLoop()->is_main_loop()) {
2252 Node* res = try_move_shenandoah_barrier_before_pre_loop(n_loop_head->in(LoopNode::EntryControl), val_ctrl);
2253 if (res != NULL) {
2254 loop = res;
2255 }
2256 }
2257
2258 return try_move_shenandoah_barrier_before_loop_helper(n, loop, val_ctrl, mem);
2259 }
2260 }
2261 }
2262 Node* ctrl = try_move_shenandoah_barrier_before_pre_loop(n->in(0), val_ctrl);
2263 if (ctrl != NULL) {
2264 return try_move_shenandoah_barrier_before_loop_helper(n, ctrl, val_ctrl, n->in(ShenandoahBarrierNode::Memory));
2265 }
2266 }
2267 return NULL;
2268 }
2269
2270 void PhaseIdealLoop::try_move_shenandoah_read_barrier(Node* n, Node *n_ctrl) {
2271 if (n->Opcode() == Op_ShenandoahReadBarrier) {
2272 ShenandoahReadBarrierNode* rb = (ShenandoahReadBarrierNode*)n;
2273 Node* mem = n->in(MemNode::Memory);
2274 int alias = C->get_alias_index(n->adr_type());
2275 const bool trace = false;
2276
2277 #ifdef ASSERT
2278 if (trace) { tty->print("Trying to move mem of"); n->dump(); }
2279 #endif
2280
2281 Node* new_mem = mem;
2282
2283 ResourceMark rm;
2284 VectorSet seen(Thread::current()->resource_area());
2285 Node_List phis;
2286
2287 for (;;) {
2288 #ifdef ASSERT
2289 if (trace) { tty->print("Looking for dominator from"); mem->dump(); }
2290 #endif
2291 if (mem->is_Proj() && mem->in(0)->is_Start()) {
2292 if (new_mem != n->in(MemNode::Memory)) {
2293 #ifdef ASSERT
2294 if (trace) { tty->print("XXX Setting mem to"); new_mem->dump(); tty->print(" for "); n->dump(); }
2295 #endif
2296 _igvn.replace_input_of(n, MemNode::Memory, new_mem);
2297 }
2298 return;
2299 }
2300
2301 Node* candidate = mem;
2302 do {
2303 if (!rb->is_independent(mem)) {
2304 if (trace) { tty->print_cr("Not independent"); }
2305 if (new_mem != n->in(MemNode::Memory)) {
2306 #ifdef ASSERT
2307 if (trace) { tty->print("XXX Setting mem to"); new_mem->dump(); tty->print(" for "); n->dump(); }
2308 #endif
2309 _igvn.replace_input_of(n, MemNode::Memory, new_mem);
2310 }
2311 return;
2312 }
2313 if (seen.test_set(mem->_idx)) {
2314 if (trace) { tty->print_cr("Already seen"); }
2315 ShouldNotReachHere();
2316 // Strange graph
2317 if (new_mem != n->in(MemNode::Memory)) {
2318 #ifdef ASSERT
2319 if (trace) { tty->print("XXX Setting mem to"); new_mem->dump(); tty->print(" for "); n->dump(); }
2320 #endif
2321 _igvn.replace_input_of(n, MemNode::Memory, new_mem);
2322 }
2323 return;
2324 }
2325 if (mem->is_Phi()) {
2326 phis.push(mem);
2327 }
2328 mem = shenandoah_next_mem(mem, alias);
2329 if (mem->bottom_type() == Type::MEMORY) {
2330 candidate = mem;
2331 }
2332 assert(shenandoah_is_dominator(ctrl_or_self(mem), n_ctrl, mem, n) == is_dominator(ctrl_or_self(mem), n_ctrl), "strange dominator");
2333 #ifdef ASSERT
2334 if (trace) { tty->print("Next mem is"); mem->dump(); }
2335 #endif
2336 } while (mem->bottom_type() != Type::MEMORY || !is_dominator(ctrl_or_self(mem), n_ctrl));
2337
2338 assert(mem->bottom_type() == Type::MEMORY, "bad mem");
2339
2340 bool not_dom = false;
2341 for (uint i = 0; i < phis.size() && !not_dom; i++) {
2342 Node* nn = phis.at(i);
2343
2344 #ifdef ASSERT
2345 if (trace) { tty->print("Looking from phi"); nn->dump(); }
2346 #endif
2347 assert(nn->is_Phi(), "phis only");
2348 for (uint j = 2; j < nn->req() && !not_dom; j++) {
2349 Node* m = nn->in(j);
2350 #ifdef ASSERT
2351 if (trace) { tty->print("Input %d is", j); m->dump(); }
2352 #endif
2353 while (m != mem && !seen.test_set(m->_idx)) {
2354 if (shenandoah_is_dominator(ctrl_or_self(m), ctrl_or_self(mem), m, mem)) {
2355 not_dom = true;
2356 // Scheduling anomaly
2357 #ifdef ASSERT
2358 if (trace) { tty->print("Giving up"); m->dump(); }
2359 #endif
2360 break;
2361 }
2362 if (!rb->is_independent(m)) {
2363 if (trace) { tty->print_cr("Not independent"); }
2364 if (new_mem != n->in(MemNode::Memory)) {
2365 #ifdef ASSERT
2366 if (trace) { tty->print("XXX Setting mem to"); new_mem->dump(); tty->print(" for "); n->dump(); }
2367 #endif
2368 _igvn.replace_input_of(n, MemNode::Memory, new_mem);
2369 }
2370 return;
2371 }
2372 if (m->is_Phi()) {
2373 phis.push(m);
2374 }
2375 m = shenandoah_next_mem(m, alias);
2376 #ifdef ASSERT
2377 if (trace) { tty->print("Next mem is"); m->dump(); }
2378 #endif
2379 }
2380 }
2381 }
2382 if (!not_dom) {
2383 new_mem = mem;
2384 phis.clear();
2385 } else {
2386 seen.Clear();
2387 }
2388 }
2389 }
2390 }
2391
|