Loop split on semi-invariant conditional statement

2019-11-07  Feng Xue <fxue@os.amperecomputing.com>

        PR tree-optimization/89134
        * doc/invoke.texi (min-loop-cond-split-prob): Document new --params.
        * params.def: Add min-loop-cond-split-prob.
        * tree-ssa-loop-split.c (split_loop): Remove niter parameter, move some
        outside checks on loop into the function.
        (split_info): New class.
        (find_vdef_in_loop, get_control_equiv_head_block): New functions.
        (find_control_dep_blocks, vuse_semi_invariant_p): Likewise.
        (ssa_semi_invariant_p, loop_iter_phi_semi_invariant_p): Likewise.
        (control_dep_semi_invariant_p, stmt_semi_invariant_p_1): Likewise.
        (stmt_semi_invariant_p, branch_removable_p): Likewise.
        (get_cond_invariant_branch, compute_added_num_insns): Likewise.
        (get_cond_branch_to_split_loop, do_split_loop_on_cond): Likewise.
        (split_loop_on_cond): Likewise.
        (tree_ssa_split_loops): Add loop split on conditional statement.

2019-11-07  Feng Xue  <fxue@os.amperecomputing.com>

        PR tree-optimization/89134
        * gcc.dg/tree-ssa/loop-cond-split-1.c: New test.
        * g++.dg/tree-ssa/loop-cond-split-1.C: New test.
        * gcc.dg/torture/pr55107.c: Add -fno-split-loops.

From-SVN: r277923

gcc/ChangeLog

+2019-11-07  Feng Xue <fxue@os.amperecomputing.com>
+
+	PR tree-optimization/89134
+	* doc/invoke.texi (min-loop-cond-split-prob): Document new --params.
+	* params.def: Add min-loop-cond-split-prob.
+	* tree-ssa-loop-split.c (split_loop): Remove niter parameter, move some
+	outside checks on loop into the function.
+	(split_info): New class.
+	(find_vdef_in_loop, get_control_equiv_head_block): New functions.
+	(find_control_dep_blocks, vuse_semi_invariant_p): Likewise.
+	(ssa_semi_invariant_p, loop_iter_phi_semi_invariant_p): Likewise.
+	(control_dep_semi_invariant_p, stmt_semi_invariant_p_1): Likewise.
+	(stmt_semi_invariant_p, branch_removable_p): Likewise.
+	(get_cond_invariant_branch, compute_added_num_insns): Likewise.
+	(get_cond_branch_to_split_loop, do_split_loop_on_cond): Likewise.
+	(split_loop_on_cond): Likewise.
+	(tree_ssa_split_loops): Add loop split on conditional statement.
+
 2019-11-07  Andreas Krebbel  <krebbel@linux.ibm.com>
 
 	* config/s390/s390.md ("*cstorecc<mode>_z13"): New insn_and_split

gcc/doc/invoke.texi

@@ -11517,6 +11517,11 @@ The maximum number of branches unswitched in a single loop.
 @item lim-expensive
 The minimum cost of an expensive expression in the loop invariant motion.
 
+@item min-loop-cond-split-prob
+When FDO profile information is available, @option{min-loop-cond-split-prob}
+specifies minimum threshold for probability of semi-invariant condition
+statement to trigger loop split.
+
 @item iv-consider-all-candidates-bound
 Bound on number of candidates for induction variables, below which
 all candidates are considered for each use in induction variable

gcc/params.def

@@ -415,6 +415,12 @@ DEFPARAM(PARAM_MAX_UNSWITCH_LEVEL,
 	"The maximum number of unswitchings in a single loop.",
 	3, 0, 0)
 
+DEFPARAM(PARAM_MIN_LOOP_COND_SPLIT_PROB,
+	"min-loop-cond-split-prob",
+	"The minimum threshold for probability of semi-invariant condition "
+	"statement to trigger loop split.",
+	30, 0, 100)
+
 /* The maximum number of insns in loop header duplicated by the copy loop
    headers pass.  */
 DEFPARAM(PARAM_MAX_LOOP_HEADER_INSNS,

gcc/testsuite/ChangeLog

+2019-11-07  Feng Xue  <fxue@os.amperecomputing.com>
+
+	PR tree-optimization/89134
+	* gcc.dg/tree-ssa/loop-cond-split-1.c: New test.
+	* g++.dg/tree-ssa/loop-cond-split-1.C: New test.
+	* gcc.dg/torture/pr55107.c: Add -fno-split-loops.
+
 2019-11-07  Andreas Krebbel  <krebbel@linux.ibm.com>
 
 	* gcc.target/s390/addsub-signed-overflow-1.c: Expect lochi

gcc/testsuite/g++.dg/tree-ssa/loop-cond-split-1.C

+/* { dg-do compile } */
+/* { dg-options "-O3 -fdump-tree-lsplit-details" } */
+
+#include <string>
+#include <map>
+
+using namespace std;
+
+class  A
+{
+public:
+  bool empty;
+  void set (string s);
+};
+
+class  B
+{
+  map<int, string> m;
+  void f ();
+};
+
+extern A *ga;
+
+void B::f ()
+{
+  for (map<int, string>::iterator iter = m.begin (); iter != m.end (); ++iter)
+    {
+      if (ga->empty)
+        ga->set (iter->second);
+    }
+}
+
+/* { dg-final { scan-tree-dump-times "loop split on semi-invariant condition at false branch" 1 "lsplit" } } */

gcc/testsuite/gcc.dg/torture/pr55107.c

 /* { dg-do compile } */
+/* { dg-additional-options "-fno-split-loops" } */
 
 typedef unsigned short uint16_t;
 

gcc/testsuite/gcc.dg/tree-ssa/loop-cond-split-1.c

+/* { dg-do compile } */
+/* { dg-options "-O3 -fdump-tree-lsplit-details" } */
+
+extern const int step;
+
+int ga, gb;
+
+__attribute__((pure)) __attribute__((noinline)) int inc (int i)
+{
+  return i + step;
+}
+
+extern int do_something (void);
+
+void test1 (int n)
+{
+  int i;
+
+  for (i = 0; i < n; i = inc (i))
+    {
+      if (ga)
+        ga = do_something ();
+    }
+}
+
+void test2 (int n, int p)
+{
+  int i;
+  int v;
+
+  for (i = 0; i < n ; i = inc (i))
+    {
+      if (ga)
+       {
+         v = inc (2);
+         gb += 1;
+       }
+      else
+       {
+         v = p * p;
+         gb *= 3;
+       }
+
+      if (v < 10)
+        ga = do_something ();
+    }
+}
+
+void test3 (int n, int p)
+{
+  int i;
+  int c = p + 1;
+  int v;
+
+  for (i = 0; i < n ; i = inc (i))
+    {
+      if (c)
+       {
+         v = inc (c);
+         gb += 1;
+       }
+      else
+       {
+         v = p * p;
+         gb *= 3;
+       }
+
+      if (v < 10)
+        c = do_something ();
+    }
+}
+
+void test4 (int n, int p)
+{
+  int i;
+  int v;
+
+  for (i = 0; i < n ; i = inc (i))
+    {
+      if (ga)
+       {
+         v = inc (2);
+         if (gb > 16)
+           v = inc (5);  
+       }
+      else
+       {
+         v = p * p;
+         gb += 2;
+       }
+
+      if (v < 10)
+        ga = do_something ();
+    }
+}
+
+/* { dg-final { scan-tree-dump-times "loop split on semi-invariant condition at false branch" 3 "lsplit" } } */

gcc/tree-ssa-loop-split.c

@@ -32,7 +32,10 @@ along with GCC; see the file COPYING3.  If not see
 #include "tree-ssa-loop.h"
 #include "tree-ssa-loop-manip.h"
 #include "tree-into-ssa.h"
+#include "tree-inline.h"
+#include "tree-cfgcleanup.h"
 #include "cfgloop.h"
+#include "params.h"
 #include "tree-scalar-evolution.h"
 #include "gimple-iterator.h"
 #include "gimple-pretty-print.h"
@@ -40,7 +43,9 @@ along with GCC; see the file COPYING3.  If not see
 #include "gimple-fold.h"
 #include "gimplify-me.h"
 
-/* This file implements loop splitting, i.e. transformation of loops like
+/* This file implements two kinds of loop splitting.
+
+   One transformation of loops like:
 
    for (i = 0; i < 100; i++)
      {
@@ -487,8 +492,9 @@ compute_new_first_bound (gimple_seq *stmts, class tree_niter_desc *niter,
    single exit of LOOP.  */
 
 static bool
-split_loop (class loop *loop1, class tree_niter_desc *niter)
+split_loop (class loop *loop1)
 {
+  class tree_niter_desc niter;
   basic_block *bbs;
   unsigned i;
   bool changed = false;
@@ -496,8 +502,28 @@ split_loop (class loop *loop1, class tree_niter_desc *niter)
   tree border = NULL_TREE;
   affine_iv iv;
 
+  if (!single_exit (loop1)
+      /* ??? We could handle non-empty latches when we split the latch edge
+	 (not the exit edge), and put the new exit condition in the new block.
+	 OTOH this executes some code unconditionally that might have been
+	 skipped by the original exit before.  */
+      || !empty_block_p (loop1->latch)
+      || !easy_exit_values (loop1)
+      || !number_of_iterations_exit (loop1, single_exit (loop1), &niter,
+				     false, true)
+      || niter.cmp == ERROR_MARK
+      /* We can't yet handle loops controlled by a != predicate.  */
+      || niter.cmp == NE_EXPR)
+    return false;
+
   bbs = get_loop_body (loop1);
 
+  if (!can_copy_bbs_p (bbs, loop1->num_nodes))
+    {
+      free (bbs);
+      return false;
+    }
+
   /* Find a splitting opportunity.  */
   for (i = 0; i < loop1->num_nodes; i++)
     if ((guard_iv = split_at_bb_p (loop1, bbs[i], &border, &iv)))
@@ -505,8 +531,8 @@ split_loop (class loop *loop1, class tree_niter_desc *niter)
 	/* Handling opposite steps is not implemented yet.  Neither
 	   is handling different step sizes.  */
 	if ((tree_int_cst_sign_bit (iv.step)
-	     != tree_int_cst_sign_bit (niter->control.step))
-	    || !tree_int_cst_equal (iv.step, niter->control.step))
+	     != tree_int_cst_sign_bit (niter.control.step))
+	    || !tree_int_cst_equal (iv.step, niter.control.step))
 	  continue;
 
 	/* Find a loop PHI node that defines guard_iv directly,
@@ -575,7 +601,7 @@ split_loop (class loop *loop1, class tree_niter_desc *niter)
 	   Compute the new bound for the guarding IV and patch the
 	   loop exit to use it instead of original IV and bound.  */
 	gimple_seq stmts = NULL;
-	tree newend = compute_new_first_bound (&stmts, niter, border,
+	tree newend = compute_new_first_bound (&stmts, &niter, border,
 					       guard_code, guard_init);
 	if (stmts)
 	  gsi_insert_seq_on_edge_immediate (loop_preheader_edge (loop1),
@@ -612,6 +638,956 @@ split_loop (class loop *loop1, class tree_niter_desc *niter)
   return changed;
 }
 
+/* Another transformation of loops like:
+
+   for (i = INIT (); CHECK (i); i = NEXT ())
+     {
+       if (expr (a_1, a_2, ..., a_n))  // expr is pure
+         a_j = ...;  // change at least one a_j
+       else
+         S;          // not change any a_j
+     }
+
+   into:
+
+   for (i = INIT (); CHECK (i); i = NEXT ())
+     {
+       if (expr (a_1, a_2, ..., a_n))
+         a_j = ...;
+       else
+         {
+           S;
+           i = NEXT ();
+           break;
+         }
+     }
+
+   for (; CHECK (i); i = NEXT ())
+     {
+       S;
+     }
+
+   */
+
+/* Data structure to hold temporary information during loop split upon
+   semi-invariant conditional statement.  */
+class split_info {
+public:
+  /* Array of all basic blocks in a loop, returned by get_loop_body().  */
+  basic_block *bbs;
+
+  /* All memory store/clobber statements in a loop.  */
+  auto_vec<gimple *> memory_stores;
+
+  /* Whether above memory stores vector has been filled.  */
+  int need_init;
+
+  /* Control dependencies of basic blocks in a loop.  */
+  auto_vec<hash_set<basic_block> *> control_deps;
+
+  split_info () : bbs (NULL),  need_init (true) { }
+
+  ~split_info ()
+    {
+      if (bbs)
+	free (bbs);
+
+      for (unsigned i = 0; i < control_deps.length (); i++)
+	delete control_deps[i];
+    }
+};
+
+/* Find all statements with memory-write effect in LOOP, including memory
+   store and non-pure function call, and keep those in a vector.  This work
+   is only done one time, for the vector should be constant during analysis
+   stage of semi-invariant condition.  */
+
+static void
+find_vdef_in_loop (struct loop *loop)
+{
+  split_info *info = (split_info *) loop->aux;
+  gphi *vphi = get_virtual_phi (loop->header);
+
+  /* Indicate memory store vector has been filled.  */
+  info->need_init = false;
+
+  /* If loop contains memory operation, there must be a virtual PHI node in
+     loop header basic block.  */
+  if (vphi == NULL)
+    return;
+
+  /* All virtual SSA names inside the loop are connected to be a cyclic
+     graph via virtual PHI nodes.  The virtual PHI node in loop header just
+     links the first and the last virtual SSA names, by using the last as
+     PHI operand to define the first.  */
+  const edge latch = loop_latch_edge (loop);
+  const tree first = gimple_phi_result (vphi);
+  const tree last = PHI_ARG_DEF_FROM_EDGE (vphi, latch);
+
+  /* The virtual SSA cyclic graph might consist of only one SSA name, who
+     is defined by itself.
+
+       .MEM_1 = PHI <.MEM_2(loop entry edge), .MEM_1(latch edge)>
+
+     This means the loop contains only memory loads, so we can skip it.  */
+  if (first == last)
+    return;
+
+  auto_vec<gimple *> other_stores;
+  auto_vec<tree> worklist;
+  auto_bitmap visited;
+
+  bitmap_set_bit (visited, SSA_NAME_VERSION (first));
+  bitmap_set_bit (visited, SSA_NAME_VERSION (last));
+  worklist.safe_push (last);
+
+  do
+    {
+      tree vuse = worklist.pop ();
+      gimple *stmt = SSA_NAME_DEF_STMT (vuse);
+
+      /* We mark the first and last SSA names as visited at the beginning,
+	 and reversely start the process from the last SSA name towards the
+	 first, which ensures that this do-while will not touch SSA names
+	 defined outside the loop.  */
+      gcc_assert (gimple_bb (stmt)
+		  && flow_bb_inside_loop_p (loop, gimple_bb (stmt)));
+
+      if (gimple_code (stmt) == GIMPLE_PHI)
+	{
+	  gphi *phi = as_a <gphi *> (stmt);
+
+	  for (unsigned i = 0; i < gimple_phi_num_args (phi); ++i)
+	    {
+	      tree arg = gimple_phi_arg_def (stmt, i);
+
+	      if (bitmap_set_bit (visited, SSA_NAME_VERSION (arg)))
+		worklist.safe_push (arg);
+	    }
+	}
+      else
+	{
+	  tree prev = gimple_vuse (stmt);
+
+	  /* Non-pure call statement is conservatively assumed to impact all
+	     memory locations.  So place call statements ahead of other memory
+	     stores in the vector with an idea of of using them as shortcut
+	     terminators to memory alias analysis.  */
+	  if (gimple_code (stmt) == GIMPLE_CALL)
+	    info->memory_stores.safe_push (stmt);
+	  else
+	    other_stores.safe_push (stmt);
+
+	  if (bitmap_set_bit (visited, SSA_NAME_VERSION (prev)))
+	    worklist.safe_push (prev);
+	}
+    } while (!worklist.is_empty ());
+
+    info->memory_stores.safe_splice (other_stores);
+}
+
+/* Two basic blocks have equivalent control dependency if one dominates to
+   the other, and it is post-dominated by the latter.  Given a basic block
+   BB in LOOP, find farest equivalent dominating basic block.  For BB, there
+   is a constraint that BB does not post-dominate loop header of LOOP, this
+   means BB is control-dependent on at least one basic block in LOOP.  */
+
+static basic_block
+get_control_equiv_head_block (struct loop *loop, basic_block bb)
+{
+  while (!bb->aux)
+    {
+      basic_block dom_bb = get_immediate_dominator (CDI_DOMINATORS, bb);
+
+      gcc_checking_assert (dom_bb && flow_bb_inside_loop_p (loop, dom_bb));
+
+      if (!dominated_by_p (CDI_POST_DOMINATORS, dom_bb, bb))
+	break;
+
+      bb = dom_bb;
+    }
+  return bb;
+}
+
+/* Given a BB in LOOP, find out all basic blocks in LOOP that BB is control-
+   dependent on.  */
+
+static hash_set<basic_block> *
+find_control_dep_blocks (struct loop *loop, basic_block bb)
+{
+  /* BB has same control dependency as loop header, then it is not control-
+     dependent on any basic block in LOOP.  */
+  if (dominated_by_p (CDI_POST_DOMINATORS, loop->header, bb))
+    return NULL;
+
+  basic_block equiv_head = get_control_equiv_head_block (loop, bb);
+
+  if (equiv_head->aux)
+    {
+      /* There is a basic block containing control dependency equivalent
+	 to BB.  No need to recompute that, and also set this information
+	 to other equivalent basic blocks.  */
+      for (; bb != equiv_head;
+	   bb = get_immediate_dominator (CDI_DOMINATORS, bb))
+	bb->aux = equiv_head->aux;
+      return (hash_set<basic_block> *) equiv_head->aux;
+    }
+
+  /* A basic block X is control-dependent on another Y iff there exists
+     a path from X to Y, in which every basic block other than X and Y
+     is post-dominated by Y, but X is not post-dominated by Y.
+
+     According to this rule, traverse basic blocks in the loop backwards
+     starting from BB, if a basic block is post-dominated by BB, extend
+     current post-dominating path to this block, otherwise it is another
+     one that BB is control-dependent on.  */
+
+  auto_vec<basic_block> pdom_worklist;
+  hash_set<basic_block> pdom_visited;
+  hash_set<basic_block> *dep_bbs = new hash_set<basic_block>;
+
+  pdom_worklist.safe_push (equiv_head);
+
+  do
+    {
+      basic_block pdom_bb = pdom_worklist.pop ();
+      edge_iterator ei;
+      edge e;
+
+      if (pdom_visited.add (pdom_bb))
+	continue;
+
+      FOR_EACH_EDGE (e, ei, pdom_bb->preds)
+	{
+	  basic_block pred_bb = e->src;
+
+	  if (!dominated_by_p (CDI_POST_DOMINATORS, pred_bb, bb))
+	    {
+	      dep_bbs->add (pred_bb);
+	      continue;
+	    }
+
+	  pred_bb = get_control_equiv_head_block (loop, pred_bb);
+
+	  if (pdom_visited.contains (pred_bb))
+	    continue;
+
+	  if (!pred_bb->aux)
+	    {
+	      pdom_worklist.safe_push (pred_bb);
+	      continue;
+	    }
+
+	  /* If control dependency of basic block is available, fast extend
+	     post-dominating path using the information instead of advancing
+	     forward step-by-step.  */
+	  hash_set<basic_block> *pred_dep_bbs
+			= (hash_set<basic_block> *) pred_bb->aux;
+
+	  for (hash_set<basic_block>::iterator iter = pred_dep_bbs->begin ();
+	       iter != pred_dep_bbs->end (); ++iter)
+	    {
+	      basic_block pred_dep_bb = *iter;
+
+	      /* Basic blocks can either be in control dependency of BB, or
+		 must be post-dominated by BB, if so, extend the path from
+		 these basic blocks.  */
+	      if (!dominated_by_p (CDI_POST_DOMINATORS, pred_dep_bb, bb))
+		dep_bbs->add (pred_dep_bb);
+	      else if (!pdom_visited.contains (pred_dep_bb))
+		pdom_worklist.safe_push (pred_dep_bb);
+	    }
+	}
+    } while (!pdom_worklist.is_empty ());
+
+  /* Record computed control dependencies in loop so that we can reach them
+     when reclaiming resources.  */
+  ((split_info *) loop->aux)->control_deps.safe_push (dep_bbs);
+
+  /* Associate control dependence with related equivalent basic blocks.  */
+  for (equiv_head->aux = dep_bbs; bb != equiv_head;
+       bb = get_immediate_dominator (CDI_DOMINATORS, bb))
+    bb->aux = dep_bbs;
+
+  return dep_bbs;
+}
+
+/* Forward declaration */
+
+static bool
+stmt_semi_invariant_p_1 (struct loop *loop, gimple *stmt,
+			 const_basic_block skip_head,
+			 hash_map<gimple *, bool> &stmt_stat);
+
+/* Given STMT, memory load or pure call statement, check whether it is impacted
+   by some memory store in LOOP, excluding trace starting from SKIP_HEAD (the
+   trace is composed of SKIP_HEAD and those basic block dominated by it, always
+   corresponds to one branch of a conditional statement).  If SKIP_HEAD is
+   NULL, all basic blocks of LOOP are checked.  */
+
+static bool
+vuse_semi_invariant_p (struct loop *loop, gimple *stmt,
+		       const_basic_block skip_head)
+{
+  split_info *info = (split_info *) loop->aux;
+  tree rhs = NULL_TREE;
+  ao_ref ref;
+  gimple *store;
+  unsigned i;
+
+  /* Collect memory store/clobber statements if haven't done that.  */
+  if (info->need_init)
+    find_vdef_in_loop (loop);
+
+  if (is_gimple_assign (stmt))
+    rhs = gimple_assign_rhs1 (stmt);
+
+  ao_ref_init (&ref, rhs);
+
+  FOR_EACH_VEC_ELT (info->memory_stores, i, store)
+    {
+      /* Skip basic blocks dominated by SKIP_HEAD, if non-NULL.  */
+      if (skip_head
+	  && dominated_by_p (CDI_DOMINATORS, gimple_bb (store), skip_head))
+	continue;
+
+      if (!ref.ref || stmt_may_clobber_ref_p_1 (store, &ref))
+	return false;
+    }
+
+  return true;
+}
+
+/* Suppose one condition branch, led by SKIP_HEAD, is not executed since
+   certain iteration of LOOP, check whether an SSA name (NAME) remains
+   unchanged in next iteration.  We call this characteristic semi-
+   invariantness.  SKIP_HEAD might be NULL, if so, nothing excluded, all basic
+   blocks and control flows in the loop will be considered.  Semi-invariant
+   state of checked statement is cached in hash map STMT_STAT to avoid
+   redundant computation in possible following re-check.  */
+
+static inline bool
+ssa_semi_invariant_p (struct loop *loop, tree name,
+		      const_basic_block skip_head,
+		      hash_map<gimple *, bool> &stmt_stat)
+{
+  gimple *def = SSA_NAME_DEF_STMT (name);
+  const_basic_block def_bb = gimple_bb (def);
+
+  /* An SSA name defined outside loop is definitely semi-invariant.  */
+  if (!def_bb || !flow_bb_inside_loop_p (loop, def_bb))
+    return true;
+
+  return stmt_semi_invariant_p_1 (loop, def, skip_head, stmt_stat);
+}
+
+/* Check whether a loop iteration PHI node (LOOP_PHI) defines a value that is
+   semi-invariant in LOOP.  Basic blocks dominated by SKIP_HEAD (if non-NULL),
+   are excluded from LOOP.  */
+
+static bool
+loop_iter_phi_semi_invariant_p (struct loop *loop, gphi *loop_phi,
+				const_basic_block skip_head)
+{
+  const_edge latch = loop_latch_edge (loop);
+  tree name = gimple_phi_result (loop_phi);
+  tree from = PHI_ARG_DEF_FROM_EDGE (loop_phi, latch);
+
+  gcc_checking_assert (from);
+
+  /* Loop iteration PHI node locates in loop header, and it has two source
+     operands, one is an initial value coming from outside the loop, the other
+     is a value through latch of the loop, which is derived in last iteration,
+     we call the latter latch value.  From the PHI node to definition of latch
+     value, if excluding branch trace starting from SKIP_HEAD, except copy-
+     assignment or likewise, there is no other kind of value redefinition, SSA
+     name defined by the PHI node is semi-invariant.
+
+                         loop entry
+                              |     .--- latch ---.
+                              |     |             |
+                              v     v             |
+                  x_1 = PHI <x_0,  x_3>           |
+                           |                      |
+                           v                      |
+              .------- if (cond) -------.         |
+              |                         |         |
+              |                     [ SKIP ]      |
+              |                         |         |
+              |                     x_2 = ...     |
+              |                         |         |
+              '---- T ---->.<---- F ----'         |
+                           |                      |
+                           v                      |
+                  x_3 = PHI <x_1, x_2>            |
+                           |                      |
+                           '----------------------'
+
+     Suppose in certain iteration, execution flow in above graph goes through
+     true branch, which means that one source value to define x_3 in false
+     branch (x_2) is skipped, x_3 only comes from x_1, and x_1 in next
+     iterations is defined by x_3, we know that x_1 will never changed if COND
+     always chooses true branch from then on.  */
+
+  while (from != name)
+    {
+      /* A new value comes from a CONSTANT.  */
+      if (TREE_CODE (from) != SSA_NAME)
+	return false;
+
+      gimple *stmt = SSA_NAME_DEF_STMT (from);
+      const_basic_block bb = gimple_bb (stmt);
+
+      /* A new value comes from outside the loop.  */
+      if (!bb || !flow_bb_inside_loop_p (loop, bb))
+	return false;
+
+      from = NULL_TREE;
+
+      if (gimple_code (stmt) == GIMPLE_PHI)
+	{
+	  gphi *phi = as_a <gphi *> (stmt);
+
+	  for (unsigned i = 0; i < gimple_phi_num_args (phi); ++i)
+	    {
+	      if (skip_head)
+		{
+		  const_edge e = gimple_phi_arg_edge (phi, i);
+
+		  /* Don't consider redefinitions in excluded basic blocks.  */
+		  if (dominated_by_p (CDI_DOMINATORS, e->src, skip_head))
+		    continue;
+		}
+
+	      tree arg = gimple_phi_arg_def (phi, i);
+
+	      if (!from)
+		from = arg;
+	      else if (!operand_equal_p (from, arg, 0))
+		/* There are more than one source operands that provide
+		   different values to the SSA name, it is variant.  */
+		return false;
+	    }
+	}
+      else if (gimple_code (stmt) == GIMPLE_ASSIGN)
+	{
+	  /* For simple value copy, check its rhs instead.  */
+	  if (gimple_assign_ssa_name_copy_p (stmt))
+	    from = gimple_assign_rhs1 (stmt);
+	}
+
+      /* Any other kind of definition is deemed to introduce a new value
+	 to the SSA name.  */
+      if (!from)
+	return false;
+    }
+  return true;
+}
+
+/* Check whether conditional predicates that BB is control-dependent on, are
+   semi-invariant in LOOP.  Basic blocks dominated by SKIP_HEAD (if non-NULL),
+   are excluded from LOOP.  Semi-invariant state of checked statement is cached
+   in hash map STMT_STAT.  */
+
+static bool
+control_dep_semi_invariant_p (struct loop *loop, basic_block bb,
+			      const_basic_block skip_head,
+			      hash_map<gimple *, bool> &stmt_stat)
+{
+  hash_set<basic_block> *dep_bbs = find_control_dep_blocks (loop, bb);
+
+  if (!dep_bbs)
+    return true;
+
+  for (hash_set<basic_block>::iterator iter = dep_bbs->begin ();
+       iter != dep_bbs->end (); ++iter)
+    {
+      gimple *last = last_stmt (*iter);
+
+      if (!last)
+	return false;
+
+      /* Only check condition predicates.  */
+      if (gimple_code (last) != GIMPLE_COND
+	  && gimple_code (last) != GIMPLE_SWITCH)
+	return false;
+
+      if (!stmt_semi_invariant_p_1 (loop, last, skip_head, stmt_stat))
+	return false;
+    }
+
+  return true;
+}
+
+/* Check whether STMT is semi-invariant in LOOP, iff all its operands are
+   semi-invariant, consequently, all its defined values are semi-invariant.
+   Basic blocks dominated by SKIP_HEAD (if non-NULL), are excluded from LOOP.
+   Semi-invariant state of checked statement is cached in hash map
+   STMT_STAT.  */
+
+static bool
+stmt_semi_invariant_p_1 (struct loop *loop, gimple *stmt,
+			 const_basic_block skip_head,
+			 hash_map<gimple *, bool> &stmt_stat)
+{
+  bool existed;
+  bool &invar = stmt_stat.get_or_insert (stmt, &existed);
+
+  if (existed)
+    return invar;
+
+  /* A statement might depend on itself, which is treated as variant.  So set
+     state of statement under check to be variant to ensure that.  */
+  invar = false;
+
+  if (gimple_code (stmt) == GIMPLE_PHI)
+    {
+      gphi *phi = as_a <gphi *> (stmt);
+
+      if (gimple_bb (stmt) == loop->header)
+	{
+	  invar = loop_iter_phi_semi_invariant_p (loop, phi, skip_head);
+	  return invar;
+	}
+
+      /* For a loop PHI node that does not locate in loop header, it is semi-
+	 invariant only if two conditions are met.  The first is its source
+	 values are derived from CONSTANT (including loop-invariant value), or
+	 from SSA name defined by semi-invariant loop iteration PHI node.  The
+	 second is its source incoming edges are control-dependent on semi-
+	 invariant conditional predicates.  */
+      for (unsigned i = 0; i < gimple_phi_num_args (phi); ++i)
+	{
+	  const_edge e = gimple_phi_arg_edge (phi, i);
+	  tree arg = gimple_phi_arg_def (phi, i);
+
+	  if (TREE_CODE (arg) == SSA_NAME)
+	    {
+	      if (!ssa_semi_invariant_p (loop, arg, skip_head, stmt_stat))
+		return false;
+
+	      /* If source value is defined in location from where the source
+		 edge comes in, no need to check control dependency again
+		 since this has been done in above SSA name check stage.  */
+	      if (e->src == gimple_bb (SSA_NAME_DEF_STMT (arg)))
+		continue;
+	    }
+
+	  if (!control_dep_semi_invariant_p (loop, e->src, skip_head,
+					     stmt_stat))
+	    return false;
+	}
+    }
+  else
+    {
+      ssa_op_iter iter;
+      tree use;
+
+      /* Volatile memory load or return of normal (non-const/non-pure) call
+	 should not be treated as constant in each iteration of loop.  */
+      if (gimple_has_side_effects (stmt))
+	return false;
+
+      /* Check if any memory store may kill memory load at this place.  */
+      if (gimple_vuse (stmt) && !vuse_semi_invariant_p (loop, stmt, skip_head))
+	return false;
+
+      /* Although operand of a statement might be SSA name, CONSTANT or
+	 VARDECL, here we only need to check SSA name operands.  This is
+	 because check on VARDECL operands, which involve memory loads,
+	 must have been done prior to invocation of this function in
+	 vuse_semi_invariant_p.  */
+      FOR_EACH_SSA_TREE_OPERAND (use, stmt, iter, SSA_OP_USE)
+	if (!ssa_semi_invariant_p (loop, use, skip_head, stmt_stat))
+	  return false;
+    }
+
+  if (!control_dep_semi_invariant_p (loop, gimple_bb (stmt), skip_head,
+				     stmt_stat))
+    return false;
+
+  /* Here we SHOULD NOT use invar = true, since hash map might be changed due
+     to new insertion, and thus invar may point to invalid memory.  */
+  stmt_stat.put (stmt, true);
+  return true;
+}
+
+/* A helper function to check whether STMT is semi-invariant in LOOP.  Basic
+   blocks dominated by SKIP_HEAD (if non-NULL), are excluded from LOOP.  */
+
+static bool
+stmt_semi_invariant_p (struct loop *loop, gimple *stmt,
+		       const_basic_block skip_head)
+{
+  hash_map<gimple *, bool> stmt_stat;
+  return stmt_semi_invariant_p_1 (loop, stmt, skip_head, stmt_stat);
+}
+
+/* Determine when conditional statement never transfers execution to one of its
+   branch, whether we can remove the branch's leading basic block (BRANCH_BB)
+   and those basic blocks dominated by BRANCH_BB.  */
+
+static bool
+branch_removable_p (basic_block branch_bb)
+{
+  edge_iterator ei;
+  edge e;
+
+  if (single_pred_p (branch_bb))
+    return true;
+
+  FOR_EACH_EDGE (e, ei, branch_bb->preds)
+    {
+      if (dominated_by_p (CDI_DOMINATORS, e->src, branch_bb))
+	continue;
+
+      if (dominated_by_p (CDI_DOMINATORS, branch_bb, e->src))
+	continue;
+
+       /* The branch can be reached from opposite branch, or from some
+	  statement not dominated by the conditional statement.  */
+      return false;
+    }
+
+  return true;
+}
+
+/* Find out which branch of a conditional statement (COND) is invariant in the
+   execution context of LOOP.  That is: once the branch is selected in certain
+   iteration of the loop, any operand that contributes to computation of the
+   conditional statement remains unchanged in all following iterations.  */
+
+static edge
+get_cond_invariant_branch (struct loop *loop, gcond *cond)
+{
+  basic_block cond_bb = gimple_bb (cond);
+  basic_block targ_bb[2];
+  bool invar[2];
+  unsigned invar_checks = 0;
+
+  for (unsigned i = 0; i < 2; i++)
+    {
+      targ_bb[i] = EDGE_SUCC (cond_bb, i)->dest;
+
+      /* One branch directs to loop exit, no need to perform loop split upon
+	 this conditional statement.  Firstly, it is trivial if the exit branch
+	 is semi-invariant, for the statement is just to break loop.  Secondly,
+	 if the opposite branch is semi-invariant, it means that the statement
+	 is real loop-invariant, which is covered by loop unswitch.  */
+      if (!flow_bb_inside_loop_p (loop, targ_bb[i]))
+	return NULL;
+    }
+
+  for (unsigned i = 0; i < 2; i++)
+    {
+      invar[!i] = false;
+
+      if (!branch_removable_p (targ_bb[i]))
+	continue;
+
+      /* Given a semi-invariant branch, if its opposite branch dominates
+	 loop latch, it and its following trace will only be executed in
+	 final iteration of loop, namely it is not part of repeated body
+	 of the loop.  Similar to the above case that the branch is loop
+	 exit, no need to split loop.  */
+      if (dominated_by_p (CDI_DOMINATORS, loop->latch, targ_bb[i]))
+	continue;
+
+      invar[!i] = stmt_semi_invariant_p (loop, cond, targ_bb[i]);
+      invar_checks++;
+    }
+
+  /* With both branches being invariant (handled by loop unswitch) or
+     variant is not what we want.  */
+  if (invar[0] ^ !invar[1])
+    return NULL;
+
+  /* Found a real loop-invariant condition, do nothing.  */
+  if (invar_checks < 2 && stmt_semi_invariant_p (loop, cond, NULL))
+    return NULL;
+
+  return EDGE_SUCC (cond_bb, invar[0] ? 0 : 1);
+}
+
+/* Calculate increased code size measured by estimated insn number if applying
+   loop split upon certain branch (BRANCH_EDGE) of a conditional statement.  */
+
+static int
+compute_added_num_insns (struct loop *loop, const_edge branch_edge)
+{
+  basic_block cond_bb = branch_edge->src;
+  unsigned branch = EDGE_SUCC (cond_bb, 1) == branch_edge;
+  basic_block opposite_bb = EDGE_SUCC (cond_bb, !branch)->dest;
+  basic_block *bbs = ((split_info *) loop->aux)->bbs;
+  int num = 0;
+
+  for (unsigned i = 0; i < loop->num_nodes; i++)
+    {
+      /* Do no count basic blocks only in opposite branch.  */
+      if (dominated_by_p (CDI_DOMINATORS, bbs[i], opposite_bb))
+	continue;
+
+      num += estimate_num_insns_seq (bb_seq (bbs[i]), &eni_size_weights);
+    }
+
+  /* It is unnecessary to evaluate expression of the conditional statement
+     in new loop that contains only invariant branch.  This expression should
+     be constant value (either true or false).  Exclude code size of insns
+     that contribute to computation of the expression.  */
+
+  auto_vec<gimple *> worklist;
+  hash_set<gimple *> removed;
+  gimple *stmt = last_stmt (cond_bb);
+
+  worklist.safe_push (stmt);
+  removed.add (stmt);
+  num -= estimate_num_insns (stmt, &eni_size_weights);
+
+  do
+    {
+      ssa_op_iter opnd_iter;
+      use_operand_p opnd_p;
+
+      stmt = worklist.pop ();
+      FOR_EACH_PHI_OR_STMT_USE (opnd_p, stmt, opnd_iter, SSA_OP_USE)
+	{
+	  tree opnd = USE_FROM_PTR (opnd_p);
+
+	  if (TREE_CODE (opnd) != SSA_NAME || SSA_NAME_IS_DEFAULT_DEF (opnd))
+	    continue;
+
+	  gimple *opnd_stmt = SSA_NAME_DEF_STMT (opnd);
+	  use_operand_p use_p;
+	  imm_use_iterator use_iter;
+
+	  if (removed.contains (opnd_stmt)
+	      || !flow_bb_inside_loop_p (loop, gimple_bb (opnd_stmt)))
+	    continue;
+
+	  FOR_EACH_IMM_USE_FAST (use_p, use_iter, opnd)
+	    {
+	      gimple *use_stmt = USE_STMT (use_p);
+
+	      if (!is_gimple_debug (use_stmt) && !removed.contains (use_stmt))
+		{
+		  opnd_stmt = NULL;
+		  break;
+		}
+	    }
+
+	  if (opnd_stmt)
+	    {
+	      worklist.safe_push (opnd_stmt);
+	      removed.add (opnd_stmt);
+	      num -= estimate_num_insns (opnd_stmt, &eni_size_weights);
+	    }
+	}
+    } while (!worklist.is_empty ());
+
+  gcc_assert (num >= 0);
+  return num;
+}
+
+/* Find out loop-invariant branch of a conditional statement (COND) if it has,
+   and check whether it is eligible and profitable to perform loop split upon
+   this branch in LOOP.  */
+
+static edge
+get_cond_branch_to_split_loop (struct loop *loop, gcond *cond)
+{
+  edge invar_branch = get_cond_invariant_branch (loop, cond);
+  if (!invar_branch)
+    return NULL;
+
+  /* When accurate profile information is available, and execution
+     frequency of the branch is too low, just let it go.  */
+  profile_probability prob = invar_branch->probability;
+  if (prob.reliable_p ())
+    {
+      int thres = PARAM_VALUE (PARAM_MIN_LOOP_COND_SPLIT_PROB);
+
+      if (prob < profile_probability::always ().apply_scale (thres, 100))
+	return NULL;
+    }
+
+  /* Add a threshold for increased code size to disable loop split.  */
+  if (compute_added_num_insns (loop, invar_branch)
+      > PARAM_VALUE (PARAM_MAX_PEELED_INSNS))
+    return NULL;
+
+  return invar_branch;
+}
+
+/* Given a loop (LOOP1) with a loop-invariant branch (INVAR_BRANCH) of some
+   conditional statement, perform loop split transformation illustrated
+   as the following graph.
+
+               .-------T------ if (true) ------F------.
+               |                    .---------------. |
+               |                    |               | |
+               v                    |               v v
+          pre-header                |            pre-header
+               | .------------.     |                 | .------------.
+               | |            |     |                 | |            |
+               | v            |     |                 | v            |
+             header           |     |               header           |
+               |              |     |                 |              |
+      .--- if (cond) ---.     |     |        .--- if (true) ---.     |
+      |                 |     |     |        |                 |     |
+  invariant             |     |     |    invariant             |     |
+      |                 |     |     |        |                 |     |
+      '---T--->.<---F---'     |     |        '---T--->.<---F---'     |
+               |              |    /                  |              |
+             stmts            |   /                 stmts            |
+               |              F  T                    |              |
+              / \             | /                    / \             |
+     .-------*   *      [ if (cond) ]       .-------*   *            |
+     |           |            |             |           |            |
+     |         latch          |             |         latch          |
+     |           |            |             |           |            |
+     |           '------------'             |           '------------'
+     '------------------------. .-----------'
+             loop1            | |                   loop2
+                              v v
+                             exits
+
+   In the graph, loop1 represents the part derived from original one, and
+   loop2 is duplicated using loop_version (), which corresponds to the part
+   of original one being splitted out.  In original latch edge of loop1, we
+   insert a new conditional statement duplicated from the semi-invariant cond,
+   and one of its branch goes back to loop1 header as a latch edge, and the
+   other branch goes to loop2 pre-header as an entry edge.  And also in loop2,
+   we abandon the variant branch of the conditional statement by setting a
+   constant bool condition, based on which branch is semi-invariant.  */
+
+static bool
+do_split_loop_on_cond (struct loop *loop1, edge invar_branch)
+{
+  basic_block cond_bb = invar_branch->src;
+  bool true_invar = !!(invar_branch->flags & EDGE_TRUE_VALUE);
+  gcond *cond = as_a <gcond *> (last_stmt (cond_bb));
+
+  gcc_assert (cond_bb->loop_father == loop1);
+
+  if (dump_enabled_p ())
+    dump_printf_loc (MSG_OPTIMIZED_LOCATIONS, cond,
+		     "loop split on semi-invariant condition at %s branch\n",
+		     true_invar ? "true" : "false");
+
+  initialize_original_copy_tables ();
+
+  struct loop *loop2 = loop_version (loop1, boolean_true_node, NULL,
+				     profile_probability::always (),
+				     profile_probability::never (),
+				     profile_probability::always (),
+				     profile_probability::always (),
+				     true);
+  if (!loop2)
+    {
+      free_original_copy_tables ();
+      return false;
+    }
+
+  basic_block cond_bb_copy = get_bb_copy (cond_bb);
+  gcond *cond_copy = as_a<gcond *> (last_stmt (cond_bb_copy));
+
+  /* Replace the condition in loop2 with a bool constant to let PassManager
+     remove the variant branch after current pass completes.  */
+  if (true_invar)
+    gimple_cond_make_true (cond_copy);
+  else
+    gimple_cond_make_false (cond_copy);
+
+  update_stmt (cond_copy);
+
+  /* Insert a new conditional statement on latch edge of loop1, its condition
+     is duplicated from the semi-invariant.  This statement acts as a switch
+     to transfer execution from loop1 to loop2, when loop1 enters into
+     invariant state.  */
+  basic_block latch_bb = split_edge (loop_latch_edge (loop1));
+  basic_block break_bb = split_edge (single_pred_edge (latch_bb));
+  gimple *break_cond = gimple_build_cond (gimple_cond_code(cond),
+					  gimple_cond_lhs (cond),
+					  gimple_cond_rhs (cond),
+					  NULL_TREE, NULL_TREE);
+
+  gimple_stmt_iterator gsi = gsi_last_bb (break_bb);
+  gsi_insert_after (&gsi, break_cond, GSI_NEW_STMT);
+
+  edge to_loop1 = single_succ_edge (break_bb);
+  edge to_loop2 = make_edge (break_bb, loop_preheader_edge (loop2)->src, 0);
+
+  to_loop1->flags &= ~EDGE_FALLTHRU;
+  to_loop1->flags |= true_invar ? EDGE_FALSE_VALUE : EDGE_TRUE_VALUE;
+  to_loop2->flags |= true_invar ? EDGE_TRUE_VALUE : EDGE_FALSE_VALUE;
+
+  update_ssa (TODO_update_ssa);
+
+  /* Due to introduction of a control flow edge from loop1 latch to loop2
+     pre-header, we should update PHIs in loop2 to reflect this connection
+     between loop1 and loop2.  */
+  connect_loop_phis (loop1, loop2, to_loop2);
+
+  free_original_copy_tables ();
+
+  rewrite_into_loop_closed_ssa_1 (NULL, 0, SSA_OP_USE, loop1);
+
+  return true;
+}
+
+/* Traverse all conditional statements in LOOP, to find out a good candidate
+   upon which we can do loop split.  */
+
+static bool
+split_loop_on_cond (struct loop *loop)
+{
+  split_info *info = new split_info ();
+  basic_block *bbs = info->bbs = get_loop_body (loop);
+  bool do_split = false;
+
+  /* Allocate an area to keep temporary info, and associate its address
+     with loop aux field.  */
+  loop->aux = info;
+
+  for (unsigned i = 0; i < loop->num_nodes; i++)
+    bbs[i]->aux = NULL;
+
+  for (unsigned i = 0; i < loop->num_nodes; i++)
+    {
+      basic_block bb = bbs[i];
+
+      /* We only consider conditional statement, which be executed at most once
+	 in each iteration of the loop.  So skip statements in inner loops.  */
+      if ((bb->loop_father != loop) || (bb->flags & BB_IRREDUCIBLE_LOOP))
+	continue;
+
+      /* Actually this check is not a must constraint.  With it, we can ensure
+	 conditional statement will always be executed in each iteration.  */
+      if (!dominated_by_p (CDI_DOMINATORS, loop->latch, bb))
+	continue;
+
+      gimple *last = last_stmt (bb);
+
+      if (!last || gimple_code (last) != GIMPLE_COND)
+	continue;
+
+      gcond *cond = as_a <gcond *> (last);
+      edge branch_edge = get_cond_branch_to_split_loop (loop, cond);
+
+      if (branch_edge)
+	{
+	  do_split_loop_on_cond (loop, branch_edge);
+	  do_split = true;
+	  break;
+	}
+    }
+
+  delete info;
+  loop->aux = NULL;
+
+  return do_split;
+}
+
 /* Main entry point.  Perform loop splitting on all suitable loops.  */
 
 static unsigned int
@@ -621,13 +1597,15 @@ tree_ssa_split_loops (void)
   bool changed = false;
 
   gcc_assert (scev_initialized_p ());
+
+  calculate_dominance_info (CDI_POST_DOMINATORS);
+
   FOR_EACH_LOOP (loop, LI_INCLUDE_ROOT)
     loop->aux = NULL;
 
   /* Go through all loops starting from innermost.  */
   FOR_EACH_LOOP (loop, LI_FROM_INNERMOST)
     {
-      class tree_niter_desc niter;
       if (loop->aux)
 	{
 	  /* If any of our inner loops was split, don't split us,
@@ -636,35 +1614,24 @@ tree_ssa_split_loops (void)
 	  continue;
 	}
 
-      if (single_exit (loop)
-	  /* ??? We could handle non-empty latches when we split
-	     the latch edge (not the exit edge), and put the new
-	     exit condition in the new block.  OTOH this executes some
-	     code unconditionally that might have been skipped by the
-	     original exit before.  */
-	  && empty_block_p (loop->latch)
-	  && !optimize_loop_for_size_p (loop)
-	  && easy_exit_values (loop)
-	  && number_of_iterations_exit (loop, single_exit (loop), &niter,
-					false, true)
-	  && niter.cmp != ERROR_MARK
-	  /* We can't yet handle loops controlled by a != predicate.  */
-	  && niter.cmp != NE_EXPR
-	  && can_duplicate_loop_p (loop))
+      if (optimize_loop_for_size_p (loop))
+	continue;
+
+      if (split_loop (loop) || split_loop_on_cond (loop))
 	{
-	  if (split_loop (loop, &niter))
-	    {
-	      /* Mark our containing loop as having had some split inner
-	         loops.  */
-	      loop_outer (loop)->aux = loop;
-	      changed = true;
-	    }
+	  /* Mark our containing loop as having had some split inner loops.  */
+	  loop_outer (loop)->aux = loop;
+	  changed = true;
 	}
     }
 
   FOR_EACH_LOOP (loop, LI_INCLUDE_ROOT)
     loop->aux = NULL;
 
+  clear_aux_for_blocks ();
+
+  free_dominance_info (CDI_POST_DOMINATORS);
+
   if (changed)
     return TODO_cleanup_cfg;
   return 0;