tree-vectorizer.h (vectorizable_condition): Add argument.

        * tree-vectorizer.h (vectorizable_condition): Add argument.
        * tree-vect-loop.c (vectorizable_reduction): Fail for condition
        in SLP.  Update calls to vectorizable_condition.
        * tree-vect-stmts.c (vect_is_simple_cond): Add basic block info
        to the arguments.  Pass it to vect_is_simple_use_1.
        (vectorizable_condition): Add slp_node to the arguments.
        Support vectorization of basic blocks.  Fail for reduction in
        SLP.  Update calls to vect_is_simple_cond and
        vect_is_simple_use.  Support SLP: call vect_get_slp_defs to get
        vector operands.
        (vect_analyze_stmt): Update calls to vectorizable_condition.
        (vect_transform_stmt): Likewise.
        * tree-vect-slp.c (vect_create_new_slp_node): Handle COND_EXPR.
        (vect_get_and_check_slp_defs): Handle COND_EXPR.  Allow pattern
        def stmts.
        (vect_build_slp_tree): Handle COND_EXPR.
        (vect_analyze_slp_instance): Push pattern statements to root
        node.
        (vect_get_constant_vectors): Fix comments.  Handle COND_EXPR.

From-SVN: r181026

gcc/ChangeLog

+2011-11-06  Ira Rosen  <ira.rosen@linaro.org>
+
+	* tree-vectorizer.h (vectorizable_condition): Add argument.
+	* tree-vect-loop.c (vectorizable_reduction): Fail for condition
+	in SLP.  Update calls to vectorizable_condition.
+	* tree-vect-stmts.c (vect_is_simple_cond): Add basic block info to
+	the arguments.  Pass it to vect_is_simple_use_1.
+	(vectorizable_condition): Add slp_node to the arguments.  Support
+	vectorization of basic blocks.  Fail for reduction in SLP.  Update
+	calls to vect_is_simple_cond and vect_is_simple_use.  Support SLP:
+	call vect_get_slp_defs to get vector operands.
+	(vect_analyze_stmt): Update calls to vectorizable_condition.
+	(vect_transform_stmt): Likewise.
+	* tree-vect-slp.c (vect_create_new_slp_node): Handle COND_EXPR.
+	(vect_get_and_check_slp_defs): Handle COND_EXPR.  Allow pattern
+	def stmts.
+	(vect_build_slp_tree): Handle COND_EXPR.
+	(vect_analyze_slp_instance): Push pattern statements to root node.
+	(vect_get_constant_vectors): Fix comments.  Handle COND_EXPR.
+
 2011-11-05  David S. Miller  <davem@davemloft.net>
 
 	* config/sparc/sparc.md (UNSPEC_SHORT_LOAD): New unspec.

gcc/testsuite/ChangeLog

+2011-11-06  Ira Rosen  <ira.rosen@linaro.org>
+
+	* gcc.dg/vect/bb-slp-cond-1.c: New test.
+	* gcc.dg/vect/slp-cond-1.c: New test.
+	* gcc.dg/vect/slp-cond-2.c: New test.
+
 2011-11-05  David S. Miller  <davem@davemloft.net>
 
 	* lib/test-supports.exp

gcc/testsuite/gcc.dg/vect/bb-slp-cond-1.c

+/* { dg-require-effective-target vect_condition } */
+
+#include "tree-vect.h"
+
+#define N 128
+
+__attribute__((noinline, noclone)) void
+foo (int *a, int stride)
+{
+  int i;
+
+  for (i = 0; i < N/stride; i++, a += stride)
+   {
+     a[0] = a[0] ? 1 : 5;
+     a[1] = a[1] ? 2 : 6;
+     a[2] = a[2] ? 3 : 7;
+     a[3] = a[3] ? 4 : 8;
+   }
+}
+
+
+int a[N];
+int main ()
+{
+  int i;
+
+  check_vect ();
+
+  for (i = 0; i < N; i++)
+    a[i] = i;
+
+  foo (a, 4);
+
+  for (i = 1; i < N; i++)
+    if (a[i] != i%4 + 1)
+      abort ();
+
+  if (a[0] != 5)
+    abort ();
+
+  return 0;
+}
+
+/* { dg-final { scan-tree-dump-times "basic block vectorized using SLP" 1 "slp" { target vect_element_align } } } */
+/* { dg-final { cleanup-tree-dump "slp" } } */
+

gcc/testsuite/gcc.dg/vect/slp-cond-1.c

+/* { dg-require-effective-target vect_condition } */
+#include "tree-vect.h"
+
+#define N 32
+int a[N], b[N];
+int d[N], e[N];
+int k[N];
+
+__attribute__((noinline, noclone)) void
+f1 (void)
+{
+  int i;
+  for (i = 0; i < N/4; i++)
+    {
+      k[4*i] = a[4*i] < b[4*i] ? 17 : 0;
+      k[4*i+1] = a[4*i+1] < b[4*i+1] ? 17 : 0;
+      k[4*i+2] = a[4*i+2] < b[4*i+2] ? 17 : 0;
+      k[4*i+3] = a[4*i+3] < b[4*i+3] ? 17 : 0;
+    }
+}
+
+__attribute__((noinline, noclone)) void
+f2 (void)
+{
+  int i;
+  for (i = 0; i < N/2; ++i)
+    {
+      k[2*i] = a[2*i] < b[2*i] ? 0 : 24;
+      k[2*i+1] = a[2*i+1] < b[2*i+1] ? 7 : 4;
+    }
+}
+
+__attribute__((noinline, noclone)) void
+f3 (void)
+{
+  int i;
+  for (i = 0; i < N/2; ++i)
+    {
+      k[2*i] = a[2*i] < b[2*i] ? 51 : 12;
+      k[2*i+1] = a[2*i+1] > b[2*i+1] ? 51 : 12;
+    }
+}
+
+__attribute__((noinline, noclone)) void
+f4 (void)
+{
+  int i;
+  for (i = 0; i < N/2; ++i)
+    {
+      int d0 = d[2*i], e0 = e[2*i];
+      int d1 = d[2*i+1], e1 = e[2*i+1];
+      k[2*i] = a[2*i] >= b[2*i] ? d0 : e0;
+      k[2*i+1] = a[2*i+1] >= b[2*i+1] ? d1 : e1;
+    }
+}
+
+int
+main ()
+{
+  int i;
+
+  check_vect ();
+
+  for (i = 0; i < N; i++)
+    {
+      switch (i % 9)
+	{
+	case 0: asm (""); a[i] = - i - 1; b[i] = i + 1; break;
+	case 1: a[i] = 0; b[i] = 0; break;
+	case 2: a[i] = i + 1; b[i] = - i - 1; break;
+	case 3: a[i] = i; b[i] = i + 7; break;
+	case 4: a[i] = i; b[i] = i; break;
+	case 5: a[i] = i + 16; b[i] = i + 3; break;
+	case 6: a[i] = - i - 5; b[i] = - i; break;
+	case 7: a[i] = - i; b[i] = - i; break;
+	case 8: a[i] = - i; b[i] = - i - 7; break;
+	}
+      d[i] = i;
+      e[i] = 2 * i;
+    }
+  f1 ();
+  for (i = 0; i < N; i++)
+    if (k[i] != ((i % 3) == 0 ? 17 : 0))
+      abort ();
+
+  f2 ();
+  for (i = 0; i < N; i++)
+    {
+      switch (i % 9)
+        {
+        case 0:
+	case 6:
+	  if (k[i] != ((i/9 % 2) == 0 ? 0 : 7))
+	    abort ();
+	  break;
+        case 1:
+        case 5:
+        case 7:
+	  if (k[i] != ((i/9 % 2) == 0 ? 4 : 24))
+            abort ();
+          break;
+        case 2:
+        case 4:
+        case 8:
+	  if (k[i] != ((i/9 % 2) == 0 ? 24 : 4))
+            abort ();
+          break;
+        case 3:
+	  if (k[i] != ((i/9 % 2) == 0 ? 7 : 0))
+            abort ();
+          break;
+        }
+    }
+
+  f3 ();
+
+  f4 ();
+  for (i = 0; i < N; i++)
+    if (k[i] != ((i % 3) == 0 ? e[i] : d[i]))
+      abort ();
+
+  return 0;
+}
+
+/* { dg-final { scan-tree-dump-times "vectorizing stmts using SLP" 3 "vect" } } */
+/* { dg-final { cleanup-tree-dump "vect" } } */

gcc/testsuite/gcc.dg/vect/slp-cond-2.c

+/* { dg-require-effective-target vect_cond_mixed } */
+#include "tree-vect.h"
+
+#define N 32
+int d[N], e[N], f[N];
+unsigned char k[N];
+float a[N], b[N];
+
+__attribute__((noinline, noclone)) void
+f1 (void)
+{
+  int i;
+  for (i = 0; i < N/4; i++)
+    {
+      k[4*i] = a[4*i] < b[4*i] ? 17 : 0;
+      k[4*i+1] = a[4*i+1] < b[4*i+1] ? 17 : 0;
+      k[4*i+2] = a[4*i+2] < b[4*i+2] ? 17 : 0;
+      k[4*i+3] = a[4*i+3] < b[4*i+3] ? 17 : 0;
+    }
+}
+
+__attribute__((noinline, noclone)) void
+f2 (void)
+{
+  int i;
+  for (i = 0; i < N/2; ++i)
+    {
+      k[2*i] = a[2*i] < b[2*i] ? 0 : 24;
+      k[2*i+1] = a[2*i+1] < b[2*i+1] ? 7 : 4;
+    }
+}
+
+__attribute__((noinline, noclone)) void
+f3 (void)
+{
+  int i;
+  for (i = 0; i < N/2; ++i)
+    {
+      k[2*i] = a[2*i] < b[2*i] ? 51 : 12;
+      k[2*i+1] = a[2*i+1] > b[2*i+1] ? 51 : 12;
+    }
+}
+
+__attribute__((noinline, noclone)) void
+f4 (void)
+{
+  int i;
+  for (i = 0; i < N/2; ++i)
+    {
+      int d0 = d[2*i], e0 = e[2*i];
+      int d1 = d[2*i+1], e1 = e[2*i+1];
+      f[2*i] = a[2*i] >= b[2*i] ? d0 : e0;
+      f[2*i+1] = a[2*i+1] >= b[2*i+1] ? d1 : e1;
+    }
+}
+
+int
+main ()
+{
+  int i;
+
+  check_vect ();
+
+  for (i = 0; i < N; i++)
+    {
+      switch (i % 9)
+	{
+	case 0: asm (""); a[i] = - i - 1; b[i] = i + 1; break;
+	case 1: a[i] = 0; b[i] = 0; break;
+	case 2: a[i] = i + 1; b[i] = - i - 1; break;
+	case 3: a[i] = i; b[i] = i + 7; break;
+	case 4: a[i] = i; b[i] = i; break;
+	case 5: a[i] = i + 16; b[i] = i + 3; break;
+	case 6: a[i] = - i - 5; b[i] = - i; break;
+	case 7: a[i] = - i; b[i] = - i; break;
+	case 8: a[i] = - i; b[i] = - i - 7; break;
+	}
+      d[i] = i;
+      e[i] = 2 * i;
+    }
+
+  f1 ();
+  for (i = 0; i < N; i++)
+    if (k[i] != ((i % 3) == 0 ? 17 : 0))
+      abort ();
+
+  f2 ();
+  for (i = 0; i < N; i++)
+    {
+      switch (i % 9)
+        {
+        case 0:
+	case 6:
+	  if (k[i] != ((i/9 % 2) == 0 ? 0 : 7))
+	    abort ();
+	  break;
+        case 1:
+        case 5:
+        case 7:
+	  if (k[i] != ((i/9 % 2) == 0 ? 4 : 24))
+            abort ();
+          break;
+        case 2:
+        case 4:
+        case 8:
+	  if (k[i] != ((i/9 % 2) == 0 ? 24 : 4))
+            abort ();
+          break;
+        case 3:
+	  if (k[i] != ((i/9 % 2) == 0 ? 7 : 0))
+            abort ();
+          break;
+        }
+    }
+
+  f3 ();
+
+  f4 ();
+  for (i = 0; i < N; i++)
+    if (f[i] != ((i % 3) == 0 ? e[i] : d[i]))
+      abort ();
+
+  return 0;
+}
+
+/* { dg-final { scan-tree-dump-times "vectorizing stmts using SLP" 3 "vect" } } */
+/* { dg-final { cleanup-tree-dump "vect" } } */

gcc/tree-vect-loop.c

@@ -4416,6 +4416,9 @@ vectorizable_reduction (gimple stmt, gimple_stmt_iterator *gsi,
       gcc_unreachable ();
     }
 
+  if (code == COND_EXPR && slp_node)
+    return false;
+
   scalar_dest = gimple_assign_lhs (stmt);
   scalar_type = TREE_TYPE (scalar_dest);
   if (!POINTER_TYPE_P (scalar_type) && !INTEGRAL_TYPE_P (scalar_type)
@@ -4502,7 +4505,7 @@ vectorizable_reduction (gimple stmt, gimple_stmt_iterator *gsi,
 
   if (code == COND_EXPR)
     {
-      if (!vectorizable_condition (stmt, gsi, NULL, ops[reduc_index], 0))
+      if (!vectorizable_condition (stmt, gsi, NULL, ops[reduc_index], 0, NULL))
         {
           if (vect_print_dump_info (REPORT_DETAILS))
             fprintf (vect_dump, "unsupported condition in reduction");
@@ -4774,7 +4777,7 @@ vectorizable_reduction (gimple stmt, gimple_stmt_iterator *gsi,
           gcc_assert (!slp_node);
           vectorizable_condition (stmt, gsi, vec_stmt, 
                                   PHI_RESULT (VEC_index (gimple, phis, 0)), 
-                                  reduc_index);
+                                  reduc_index, NULL);
           /* Multiple types are not supported for condition.  */
           break;
         }

gcc/tree-vect-slp.c

@@ -109,7 +109,11 @@ vect_create_new_slp_node (VEC (gimple, heap) *scalar_stmts)
   if (is_gimple_call (stmt))
     nops = gimple_call_num_args (stmt);
   else if (is_gimple_assign (stmt))
-    nops = gimple_num_ops (stmt) - 1;
+    {
+      nops = gimple_num_ops (stmt) - 1;
+      if (gimple_assign_rhs_code (stmt) == COND_EXPR)
+	nops++;
+    }
   else
     return NULL;
 
@@ -191,20 +195,41 @@ vect_get_and_check_slp_defs (loop_vec_info loop_vinfo, bb_vec_info bb_vinfo,
   bool different_types = false;
   bool pattern = false;
   slp_oprnd_info oprnd_info, oprnd0_info, oprnd1_info;
+  int op_idx = 1;
+  tree compare_rhs = NULL_TREE;
 
   if (loop_vinfo)
     loop = LOOP_VINFO_LOOP (loop_vinfo);
 
   if (is_gimple_call (stmt))
     number_of_oprnds = gimple_call_num_args (stmt);
+  else if (is_gimple_assign (stmt))
+    {
+      number_of_oprnds = gimple_num_ops (stmt) - 1;
+      if (gimple_assign_rhs_code (stmt) == COND_EXPR)
+        number_of_oprnds++;
+    }
   else
-    number_of_oprnds = gimple_num_ops (stmt) - 1;
+    return false;
 
   for (i = 0; i < number_of_oprnds; i++)
     {
-      oprnd = gimple_op (stmt, i + 1);
+      if (compare_rhs)
+	{
+	  oprnd = compare_rhs;
+	  compare_rhs = NULL_TREE;
+	}
+      else
+        oprnd = gimple_op (stmt, op_idx++);
+
       oprnd_info = VEC_index (slp_oprnd_info, *oprnds_info, i);
 
+      if (COMPARISON_CLASS_P (oprnd))
+        {
+          compare_rhs = TREE_OPERAND (oprnd, 1);
+          oprnd = TREE_OPERAND (oprnd, 0);
+	}
+
       if (!vect_is_simple_use (oprnd, loop_vinfo, bb_vinfo, &def_stmt, &def,
                                &dt)
 	  || (!def_stmt && dt != vect_constant_def))
@@ -244,8 +269,7 @@ vect_get_and_check_slp_defs (loop_vec_info loop_vinfo, bb_vec_info bb_vinfo,
           def_stmt = STMT_VINFO_RELATED_STMT (vinfo_for_stmt (def_stmt));
           dt = STMT_VINFO_DEF_TYPE (vinfo_for_stmt (def_stmt));
 
-          if (dt == vect_unknown_def_type
-	      || STMT_VINFO_PATTERN_DEF_STMT (vinfo_for_stmt (def_stmt)))
+          if (dt == vect_unknown_def_type)
             {
               if (vect_print_dump_info (REPORT_DETAILS))
                 fprintf (vect_dump, "Unsupported pattern.");
@@ -424,6 +448,7 @@ vect_build_slp_tree (loop_vec_info loop_vinfo, bb_vec_info bb_vinfo,
   VEC (gimple, heap) *stmts = SLP_TREE_SCALAR_STMTS (*node);
   gimple stmt = VEC_index (gimple, stmts, 0);
   enum tree_code first_stmt_code = ERROR_MARK, rhs_code = ERROR_MARK;
+  enum tree_code first_cond_code = ERROR_MARK;
   tree lhs;
   bool stop_recursion = false, need_same_oprnds = false;
   tree vectype, scalar_type, first_op1 = NULL_TREE;
@@ -440,11 +465,18 @@ vect_build_slp_tree (loop_vec_info loop_vinfo, bb_vec_info bb_vinfo,
   VEC (slp_oprnd_info, heap) *oprnds_info;
   unsigned int nops;
   slp_oprnd_info oprnd_info;
+  tree cond;
 
   if (is_gimple_call (stmt))
     nops = gimple_call_num_args (stmt);
+  else if (is_gimple_assign (stmt))
+    {
+      nops = gimple_num_ops (stmt) - 1;
+      if (gimple_assign_rhs_code (stmt) == COND_EXPR)
+	nops++;
+    }
   else
-    nops = gimple_num_ops (stmt) - 1;
+    return false;
 
   oprnds_info = vect_create_oprnd_info (nops, group_size);
 
@@ -485,6 +517,22 @@ vect_build_slp_tree (loop_vec_info loop_vinfo, bb_vec_info bb_vinfo,
 	  return false;
 	}
 
+       if (is_gimple_assign (stmt)
+	   && gimple_assign_rhs_code (stmt) == COND_EXPR
+           && (cond = gimple_assign_rhs1 (stmt))
+           && !COMPARISON_CLASS_P (cond))
+        {
+          if (vect_print_dump_info (REPORT_SLP))
+            {
+              fprintf (vect_dump,
+                       "Build SLP failed: condition is not comparison ");
+              print_gimple_stmt (vect_dump, stmt, 0, TDF_SLIM);
+            }
+
+          vect_free_oprnd_info (&oprnds_info, true);
+          return false;
+        }
+
       scalar_type = vect_get_smallest_scalar_type (stmt, &dummy, &dummy);
       vectype = get_vectype_for_scalar_type (scalar_type);
       if (!vectype)
@@ -737,7 +785,8 @@ vect_build_slp_tree (loop_vec_info loop_vinfo, bb_vec_info bb_vinfo,
 
 	  /* Not memory operation.  */
 	  if (TREE_CODE_CLASS (rhs_code) != tcc_binary
-	      && TREE_CODE_CLASS (rhs_code) != tcc_unary)
+	      && TREE_CODE_CLASS (rhs_code) != tcc_unary
+              && rhs_code != COND_EXPR)
 	    {
 	      if (vect_print_dump_info (REPORT_SLP))
 		{
@@ -750,6 +799,26 @@ vect_build_slp_tree (loop_vec_info loop_vinfo, bb_vec_info bb_vinfo,
 	      return false;
 	    }
 
+          if (rhs_code == COND_EXPR)
+            {
+              tree cond_expr = gimple_assign_rhs1 (stmt);
+
+	      if (i == 0)
+		first_cond_code = TREE_CODE (cond_expr);
+              else if (first_cond_code != TREE_CODE (cond_expr))
+                {
+                  if (vect_print_dump_info (REPORT_SLP))
+                    {
+                      fprintf (vect_dump, "Build SLP failed: different"
+					  " operation");
+                      print_gimple_stmt (vect_dump, stmt, 0, TDF_SLIM);
+                    }
+
+		  vect_free_oprnd_info (&oprnds_info, true);
+                  return false;
+		}
+            }
+
 	  /* Find the def-stmts.  */
 	  if (!vect_get_and_check_slp_defs (loop_vinfo, bb_vinfo, *node, stmt,
 					    ncopies_for_cost, (i == 0),
@@ -1402,7 +1471,12 @@ vect_analyze_slp_instance (loop_vec_info loop_vinfo, bb_vec_info bb_vinfo,
       /* Collect the stores and store them in SLP_TREE_SCALAR_STMTS.  */
       while (next)
         {
-          VEC_safe_push (gimple, heap, scalar_stmts, next);
+	  if (STMT_VINFO_IN_PATTERN_P (vinfo_for_stmt (next))
+	      && STMT_VINFO_RELATED_STMT (vinfo_for_stmt (next)))
+	    VEC_safe_push (gimple, heap, scalar_stmts,
+			STMT_VINFO_RELATED_STMT (vinfo_for_stmt (next)));
+	  else
+            VEC_safe_push (gimple, heap, scalar_stmts, next);
           next = GROUP_NEXT_ELEMENT (vinfo_for_stmt (next));
         }
     }
@@ -1411,7 +1485,7 @@ vect_analyze_slp_instance (loop_vec_info loop_vinfo, bb_vec_info bb_vinfo,
       /* Collect reduction statements.  */
       VEC (gimple, heap) *reductions = LOOP_VINFO_REDUCTIONS (loop_vinfo);
       for (i = 0; VEC_iterate (gimple, reductions, i, next); i++)
-        VEC_safe_push (gimple, heap, scalar_stmts, next);
+	VEC_safe_push (gimple, heap, scalar_stmts, next);
     }
 
   node = vect_create_new_slp_node (scalar_stmts);
@@ -2150,15 +2224,15 @@ vect_get_constant_vectors (tree op, slp_tree slp_node,
 
      For example, we have two scalar operands, s1 and s2 (e.g., group of
      strided accesses of size two), while NUNITS is four (i.e., four scalars
-     of this type can be packed in a vector). The output vector will contain
-     two copies of each scalar operand: {s1, s2, s1, s2}. (NUMBER_OF_COPIES
+     of this type can be packed in a vector).  The output vector will contain
+     two copies of each scalar operand: {s1, s2, s1, s2}.  (NUMBER_OF_COPIES
      will be 2).
 
      If GROUP_SIZE > NUNITS, the scalars will be split into several vectors
      containing the operands.
 
      For example, NUNITS is four as before, and the group size is 8
-     (s1, s2, ..., s8). We will create two vectors {s1, s2, s3, s4} and
+     (s1, s2, ..., s8).  We will create two vectors {s1, s2, s3, s4} and
      {s5, s6, s7, s8}.  */
 
   number_of_copies = least_common_multiple (nunits, group_size) / group_size;
@@ -2170,8 +2244,23 @@ vect_get_constant_vectors (tree op, slp_tree slp_node,
         {
           if (is_store)
             op = gimple_assign_rhs1 (stmt);
-          else
+          else if (gimple_assign_rhs_code (stmt) != COND_EXPR)
             op = gimple_op (stmt, op_num + 1);
+	  else
+	    {
+	      if (op_num == 0 || op_num == 1)
+		{
+		  tree cond = gimple_assign_rhs1 (stmt);
+		  op = TREE_OPERAND (cond, op_num);
+		}
+	      else
+		{
+		  if (op_num == 2)
+		    op = gimple_assign_rhs2 (stmt);
+		  else
+		    op = gimple_assign_rhs3 (stmt);
+		}
+	    }
 
           if (reduc_index != -1)
             {

gcc/tree-vectorizer.h

@@ -837,7 +837,7 @@ extern bool vect_transform_stmt (gimple, gimple_stmt_iterator *,
 extern void vect_remove_stores (gimple);
 extern bool vect_analyze_stmt (gimple, bool *, slp_tree);
 extern bool vectorizable_condition (gimple, gimple_stmt_iterator *, gimple *,
-                                    tree, int);
+                                    tree, int, slp_tree);
 extern void vect_get_load_cost (struct data_reference *, int, bool,
                                 unsigned int *, unsigned int *);
 extern void vect_get_store_cost (struct data_reference *, int, unsigned int *);