--- old/src/share/vm/opto/loopnode.hpp 2016-09-16 09:01:27.920399108 +0200
+++ new/src/share/vm/opto/loopnode.hpp 2016-09-16 09:01:24.300457078 +0200
@@ -1176,6 +1176,7 @@
#endif
bool shenandoah_fix_mem_phis(Node* mem, Node* mem_ctrl, Node* rep_ctrl, int alias);
bool shenandoah_fix_mem_phis_helper(Node* c, Node* mem, Node* mem_ctrl, Node* rep_ctrl, int alias, VectorSet& controls, GrowableArray<Node*>& phis);
+ void try_move_shenandoah_read_barrier(Node* n, Node *n_ctrl);
Node* shenandoah_dom_mem(Node* mem, Node*& mem_ctrl, Node* n, Node* rep_ctrl, int alias);
Node* try_move_shenandoah_barrier_before_pre_loop(Node* c, Node* val_ctrl);
Node* try_move_shenandoah_barrier_before_loop_helper(Node* n, Node* cl, Node* val_ctrl, Node* mem);
--- old/src/share/vm/opto/loopopts.cpp 2016-09-16 09:01:31.720338255 +0200
+++ new/src/share/vm/opto/loopopts.cpp 2016-09-16 09:01:28.196394688 +0200
@@ -956,6 +956,8 @@
return res;
}
+ try_move_shenandoah_read_barrier(n, n_ctrl);
+
// Attempt to remix address expressions for loop invariants
Node *m = remix_address_expressions( n );
if( m ) return m;
@@ -1368,7 +1370,7 @@
// For inner loop uses get the preheader area.
x_ctrl = place_near_use(x_ctrl);
- if (n->is_Load()) {
+ if (n->is_Load() || n->Opcode() == Op_ShenandoahReadBarrier) {
// For loads, add a control edge to a CFG node outside of the loop
// to force them to not combine and return back inside the loop
// during GVN optimization (4641526).
@@ -1376,7 +1378,9 @@
// Because we are setting the actual control input, factor in
// the result from get_late_ctrl() so we respect any
// anti-dependences. (6233005).
- x_ctrl = dom_lca(late_load_ctrl, x_ctrl);
+ if (n->is_Load()) {
+ x_ctrl = dom_lca(late_load_ctrl, x_ctrl);
+ }
// Don't allow the control input to be a CFG splitting node.
// Such nodes should only have ProjNodes as outs, e.g. IfNode
--- old/src/share/vm/opto/shenandoahSupport.cpp 2016-09-16 09:01:36.101268098 +0200
+++ new/src/share/vm/opto/shenandoahSupport.cpp 2016-09-16 09:01:31.983334043 +0200
@@ -170,7 +170,7 @@
VectorSet visited(Thread::current()->resource_area());
Node_Stack phis(0);
- for(;;) {
+ for(int i = 0; i < 10; i++) {
if (current == NULL) {
return false;
} else if (visited.test_set(current->_idx) || current->is_top() || current == b1) {
@@ -227,8 +227,31 @@
return false;
}
}
+ return false;
+}
+
+bool ShenandoahReadBarrierNode::is_independent(Node* mem) {
+ if (mem->is_Phi() || mem->is_Proj() || mem->is_MergeMem()) {
+ return true;
+ } else if (mem->Opcode() == Op_ShenandoahWriteBarrier) {
+ const Type* mem_type = mem->bottom_type();
+ const Type* this_type = bottom_type();
+ if (is_independent(mem_type, this_type)) {
+ return true;
+ } else {
+ return false;
+ }
+ } else if (mem->is_Call() || mem->is_MemBar()) {
+ return false;
+ }
+#ifdef ASSERT
+ mem->dump();
+#endif
+ ShouldNotReachHere();
+ return true;
}
+
bool ShenandoahReadBarrierNode::dominates_memory_rb(PhaseGVN* phase, Node* b1, Node* b2, bool linear) {
return dominates_memory_rb_impl(phase, b1->in(Memory), b2, b2->in(Memory), linear);
}
@@ -2243,3 +2266,126 @@
}
return NULL;
}
+
+void PhaseIdealLoop::try_move_shenandoah_read_barrier(Node* n, Node *n_ctrl) {
+ if (n->Opcode() == Op_ShenandoahReadBarrier) {
+ ShenandoahReadBarrierNode* rb = (ShenandoahReadBarrierNode*)n;
+ Node* mem = n->in(MemNode::Memory);
+ int alias = C->get_alias_index(n->adr_type());
+ const bool trace = false;
+
+#ifdef ASSERT
+ if (trace) { tty->print("Trying to move mem of"); n->dump(); }
+#endif
+
+ Node* new_mem = mem;
+
+ ResourceMark rm;
+ VectorSet seen(Thread::current()->resource_area());
+ Node_List phis;
+
+ for (;;) {
+#ifdef ASSERT
+ if (trace) { tty->print("Looking for dominator from"); mem->dump(); }
+#endif
+ if (mem->is_Proj() && mem->in(0)->is_Start()) {
+ if (new_mem != n->in(MemNode::Memory)) {
+#ifdef ASSERT
+ if (trace) { tty->print("XXX Setting mem to"); new_mem->dump(); tty->print(" for "); n->dump(); }
+#endif
+ _igvn.replace_input_of(n, MemNode::Memory, new_mem);
+ }
+ return;
+ }
+
+ Node* candidate = mem;
+ do {
+ if (!rb->is_independent(mem)) {
+ if (trace) { tty->print_cr("Not independent"); }
+ if (new_mem != n->in(MemNode::Memory)) {
+#ifdef ASSERT
+ if (trace) { tty->print("XXX Setting mem to"); new_mem->dump(); tty->print(" for "); n->dump(); }
+#endif
+ _igvn.replace_input_of(n, MemNode::Memory, new_mem);
+ }
+ return;
+ }
+ if (seen.test_set(mem->_idx)) {
+ if (trace) { tty->print_cr("Already seen"); }
+ ShouldNotReachHere();
+ // Strange graph
+ if (new_mem != n->in(MemNode::Memory)) {
+#ifdef ASSERT
+ if (trace) { tty->print("XXX Setting mem to"); new_mem->dump(); tty->print(" for "); n->dump(); }
+#endif
+ _igvn.replace_input_of(n, MemNode::Memory, new_mem);
+ }
+ return;
+ }
+ if (mem->is_Phi()) {
+ phis.push(mem);
+ }
+ mem = shenandoah_next_mem(mem, alias);
+ if (mem->bottom_type() == Type::MEMORY) {
+ candidate = mem;
+ }
+ assert(shenandoah_is_dominator(ctrl_or_self(mem), n_ctrl, mem, n) == is_dominator(ctrl_or_self(mem), n_ctrl), "strange dominator");
+#ifdef ASSERT
+ if (trace) { tty->print("Next mem is"); mem->dump(); }
+#endif
+ } while (mem->bottom_type() != Type::MEMORY || !is_dominator(ctrl_or_self(mem), n_ctrl));
+
+ assert(mem->bottom_type() == Type::MEMORY, "bad mem");
+
+ bool not_dom = false;
+ for (uint i = 0; i < phis.size() && !not_dom; i++) {
+ Node* nn = phis.at(i);
+
+#ifdef ASSERT
+ if (trace) { tty->print("Looking from phi"); nn->dump(); }
+#endif
+ assert(nn->is_Phi(), "phis only");
+ for (uint j = 2; j < nn->req() && !not_dom; j++) {
+ Node* m = nn->in(j);
+#ifdef ASSERT
+ if (trace) { tty->print("Input %d is", j); m->dump(); }
+#endif
+ while (m != mem && !seen.test_set(m->_idx)) {
+ if (shenandoah_is_dominator(ctrl_or_self(m), ctrl_or_self(mem), m, mem)) {
+ not_dom = true;
+ // Scheduling anomaly
+#ifdef ASSERT
+ if (trace) { tty->print("Giving up"); m->dump(); }
+#endif
+ break;
+ }
+ if (!rb->is_independent(m)) {
+ if (trace) { tty->print_cr("Not independent"); }
+ if (new_mem != n->in(MemNode::Memory)) {
+#ifdef ASSERT
+ if (trace) { tty->print("XXX Setting mem to"); new_mem->dump(); tty->print(" for "); n->dump(); }
+#endif
+ _igvn.replace_input_of(n, MemNode::Memory, new_mem);
+ }
+ return;
+ }
+ if (m->is_Phi()) {
+ phis.push(m);
+ }
+ m = shenandoah_next_mem(m, alias);
+#ifdef ASSERT
+ if (trace) { tty->print("Next mem is"); m->dump(); }
+#endif
+ }
+ }
+ }
+ if (!not_dom) {
+ new_mem = mem;
+ phis.clear();
+ } else {
+ seen.Clear();
+ }
+ }
+ }
+}
+
--- old/src/share/vm/opto/shenandoahSupport.hpp 2016-09-16 09:01:40.598196083 +0200
+++ new/src/share/vm/opto/shenandoahSupport.hpp 2016-09-16 09:01:36.379263646 +0200
@@ -134,6 +134,8 @@
virtual Node* Identity(PhaseGVN* phase);
virtual int Opcode() const;
+ bool is_independent(Node* mem);
+
private:
static bool is_independent(const Type* in_type, const Type* this_type);
static bool dominates_memory_rb(PhaseGVN* phase, Node* b1, Node* b2, bool linear);