tree-parloops.c (loop_parallel_p): Call vect_is_simple_reduction with additional parameter.

	* tree-parloops.c (loop_parallel_p): Call vect_is_simple_reduction
	with additional parameter.
	* tree-vectorizer.h (enum vect_def_type): Add new value 
	vect_nested_cycle.
	(enum vect_relevant): Add comments.
	(vect_is_simple_reduction): Add new argument.
	* tree-vect-loop.c (vect_analyze_scalar_cycles_1): Add comments.
	Detect nested cycles.
	(vect_is_simple_reduction): Update documentation, add an argument to
	distinguish inner-loop reduction from nested cycle, detect nested
	cycles, fix printings and indentation, don't swap operands in case
	of nested cycle.
	(get_initial_def_for_reduction): Handle subtraction.
	(vect_create_epilog_for_reduction): Add new argument to specify
	reduction variable.
	(vect_finalize_reduction): Handle subtraction, fix comments.
	(vectorizable_reduction): Handle nested cycles. In case of nested cycle
	keep track of the reduction variable position. Call 
	vect_is_simple_reduction with additional parameter. Use original 
	statement code in reduction epilogue for nested cycle. Call
	vect_create_epilog_for_reduction with additional parameter.
	* tree-vect-patterns.c (vect_recog_dot_prod_pattern): Assert inner-loop
	vectorization.
	(vect_recog_widen_sum_pattern): Likewise.
	* tree-vect-stmts.c (process_use): Distinguish between nested cycles
	and reductions.
	(vect_mark_stmts_to_be_vectorized): Likewise.
	(vect_get_vec_def_for_operand): Handle nested cycles.

From-SVN: r148518

gcc/ChangeLog

+2009-06-16  Ira Rosen  <irar@il.ibm.com>
+
+	* tree-parloops.c (loop_parallel_p): Call vect_is_simple_reduction
+	with additional parameter.
+	* tree-vectorizer.h (enum vect_def_type): Add new value 
+	vect_nested_cycle.
+	(enum vect_relevant): Add comments.
+	(vect_is_simple_reduction): Add new argument.
+	* tree-vect-loop.c (vect_analyze_scalar_cycles_1): Add comments.
+	Detect nested cycles.
+	(vect_is_simple_reduction): Update documentation, add an argument to
+	distinguish inner-loop reduction from nested cycle, detect nested
+	cycles, fix printings and indentation, don't swap operands in case
+	of nested cycle.
+	(get_initial_def_for_reduction): Handle subtraction.
+	(vect_create_epilog_for_reduction): Add new argument to specify
+	reduction variable.
+	(vect_finalize_reduction): Handle subtraction, fix comments.
+	(vectorizable_reduction): Handle nested cycles. In case of nested cycle
+	keep track of the reduction variable position. Call 
+	vect_is_simple_reduction with additional parameter. Use original 
+	statement code in reduction epilogue for nested cycle. Call
+	vect_create_epilog_for_reduction with additional parameter.
+	* tree-vect-patterns.c (vect_recog_dot_prod_pattern): Assert inner-loop
+	vectorization.
+	(vect_recog_widen_sum_pattern): Likewise.
+	* tree-vect-stmts.c (process_use): Distinguish between nested cycles
+	and reductions.
+	(vect_mark_stmts_to_be_vectorized): Likewise.
+	(vect_get_vec_def_for_operand): Handle nested cycles.
+
 2009-06-16  Ralf Wildenhues  <Ralf.Wildenhues@gmx.de>
 
 	* doc/invoke.texi (Debugging Options): Fix option index entries

gcc/testsuite/ChangeLog

+2009-06-16  Ira Rosen  <irar@il.ibm.com>
+
+	* gcc.dg/vect/vect-outer-4g.c: Don't look for pattern not allowed 
+	printing.
+	* gcc.dg/vect/vect-outer-4k.c, gcc.dg/vect/vect-outer-4l.c,
+	gcc.dg/vect/vect-outer-4f.c: Likewise.
+	* gcc.dg/vect/vect-nest-cycle-1.c: New test.
+	* gcc.dg/vect/vect-nest-cycle-2.c, gcc.dg/vect/vect-nest-cycle-3.c: 
+	Likewise.
+	* gcc.dg/vect/vect-outer-1a.c: Fail because of strided access in outer 
+	loop.
+
 2009-06-16  Tobias Burnus  <burnus@net-b.de>
 
 	PR fortran/40383

gcc/testsuite/gcc.dg/vect/vect-nest-cycle-1.c

+/* { dg-require-effective-target vect_float } */
+
+#include <stdarg.h>
+#include "tree-vect.h"
+
+#define N 16 
+
+float in[N] = {232,132,32,432,532,321,327,323,321,324,322,329,432,832,932,232};
+float out[N];
+float check_res[N] = {112,-4,-120,264,348,121,111,91,73,60,42,33,120,504,588,-128};
+float a[2*N] = {0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31};
+
+/* Outer-loop vectorization.  */
+
+__attribute__ ((noinline)) void 
+foo () 
+{
+  int i, j;
+  float res;
+
+  for (i = 0; i < N; i++)
+    {
+      res = in[i];
+
+      for (j = 0; j < N; j++) 
+        res = res - a[i+j];
+        
+      out[i] = res;
+    }
+
+  for (i = 0; i < N; i++)  
+    if (out[i] != check_res[i])
+      abort ();
+
+}
+
+int main ()
+{
+  check_vect ();
+
+  foo();
+
+  return 0;
+}
+
+/* { dg-final { scan-tree-dump-times "OUTER LOOP VECTORIZED" 1 "vect" } } */
+/* { dg-final { cleanup-tree-dump "vect" } } */
+

gcc/testsuite/gcc.dg/vect/vect-nest-cycle-2.c

+/* { dg-require-effective-target vect_float } */
+
+#include <stdarg.h>
+#include <stdio.h>
+#include "tree-vect.h"
+
+#define N 16 
+
+float out[N];
+float check_res[N] = {880,864,848,832,816,800,784,768,752,736,720,704,688,672,656,640};
+float a[2*N] = {0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31};
+
+/* Outer-loop vectorization.  */
+
+__attribute__ ((noinline)) void 
+foo () 
+{
+  int i, j;
+  float res;
+
+  for (i = 0; i < N; i++)
+    {
+      res = 1000;
+
+      for (j = 0; j < N; j++) 
+        res = res - a[i+j];
+        
+      out[i] = res;
+    }
+
+  for (i = 0; i < N; i++)  
+    if (out[i] != check_res[i])
+      abort ();
+
+}
+
+int main ()
+{
+  check_vect ();
+
+  foo();
+
+  return 0;
+}
+
+/* { dg-final { scan-tree-dump-times "OUTER LOOP VECTORIZED" 1 "vect" } } */
+/* { dg-final { cleanup-tree-dump "vect" } } */
+

gcc/testsuite/gcc.dg/vect/vect-nest-cycle-3.c

+/* { dg-require-effective-target vect_float } */
+
+#include <stdarg.h>
+#include <stdio.h>
+#include "tree-vect.h"
+
+#define N 16
+#define DIFF 82
+
+float c[N][N], b[N][N], a[N];
+
+__attribute__ ((noinline)) int 
+main1 ()
+{
+  int i, j;
+  float diff;
+
+  /* In inner loop vectorization -funsafe-math-optimizations is needed to 
+     vectorize the summation. But in outer loop vectorization the order of
+     calculation doesn't change, therefore, there is no need in that flag.  */
+  for (i = 0; i < N; i++)
+    {
+      diff = 2;
+      for (j = 0; j < N; j++) 
+        diff += (b[j][i] - c[j][i]);
+
+      a[i] = diff;
+    }
+
+  /* Check results:  */
+  for (i = 0; i < N; i++)
+    if (a[i] != DIFF)
+      abort ();
+
+  return 0;
+}
+
+int main (void)
+{ 
+  int i, j;
+
+  for (i = 0; i < N; i++)
+    for (j = 0; j < N; j++)
+      {
+        b[i][j] = i+j+5;
+        c[i][j] = i+j;
+      }
+         
+  check_vect ();
+  
+  main1 ();
+
+  return 0;
+}
+
+/* { dg-final { scan-tree-dump-times "OUTER LOOP VECTORIZED" 1 "vect" } } */
+/* { dg-final { cleanup-tree-dump "vect" } } */

gcc/testsuite/gcc.dg/vect/vect-outer-1a.c

@@ -4,9 +4,7 @@
 signed short image[N][N] __attribute__ ((__aligned__(__BIGGEST_ALIGNMENT__)));
 signed short block[N][N] __attribute__ ((__aligned__(__BIGGEST_ALIGNMENT__)));
 
-/* Can't do outer-loop vectorization because of non-consecutive access.
-   Currently fails to vectorize because the reduction pattern is not
-   recognized.  */
+/* Can't do outer-loop vectorization because of non-consecutive access.  */
 
 int
 foo (){
@@ -22,7 +20,5 @@ foo (){
 }
 
 /* { dg-final { scan-tree-dump-times "OUTER LOOP VECTORIZED" 1 "vect" { xfail *-*-* } } } */
-/* FORNOW */
-/* { dg-final { scan-tree-dump-times "strided access in outer loop" 1 "vect" { xfail *-*-* } } } */
-/* { dg-final { scan-tree-dump-times "unexpected pattern" 1 "vect" } } */
+/* { dg-final { scan-tree-dump-times "strided access in outer loop" 1 "vect" } } */
 /* { dg-final { cleanup-tree-dump "vect" } } */

gcc/testsuite/gcc.dg/vect/vect-outer-4f.c

@@ -66,5 +66,4 @@ int main (void)
 }
 
 /* { dg-final { scan-tree-dump-times "OUTER LOOP VECTORIZED" 1 "vect" { xfail *-*-* } } } */
-/* { dg-final { scan-tree-dump-times "vect_recog_widen_sum_pattern: not allowed" 1 "vect" } } */
 /* { dg-final { cleanup-tree-dump "vect" } } */

gcc/testsuite/gcc.dg/vect/vect-outer-4g.c

@@ -66,5 +66,4 @@ int main (void)
 }
 
 /* { dg-final { scan-tree-dump-times "OUTER LOOP VECTORIZED" 1 "vect" { xfail *-*-* } } } */
-/* { dg-final { scan-tree-dump-times "vect_recog_widen_sum_pattern: not allowed" 1 "vect" } } */
 /* { dg-final { cleanup-tree-dump "vect" } } */

gcc/testsuite/gcc.dg/vect/vect-outer-4k.c

@@ -66,5 +66,4 @@ int main (void)
 }
 
 /* { dg-final { scan-tree-dump-times "OUTER LOOP VECTORIZED" 1 "vect" { xfail *-*-* } } } */
-/* { dg-final { scan-tree-dump-times "vect_recog_widen_sum_pattern: not allowed" 1 "vect" } } */
 /* { dg-final { cleanup-tree-dump "vect" } } */

gcc/testsuite/gcc.dg/vect/vect-outer-4l.c

@@ -66,5 +66,4 @@ int main (void)
 }
 
 /* { dg-final { scan-tree-dump-times "OUTER LOOP VECTORIZED" 1 "vect" { xfail *-*-* } } } */
-/* { dg-final { scan-tree-dump-times "vect_recog_widen_sum_pattern: not allowed" 1 "vect" } } */
 /* { dg-final { cleanup-tree-dump "vect" } } */

gcc/tree-parloops.c

@@ -290,7 +290,7 @@ loop_parallel_p (struct loop *loop, htab_t reduction_list,
       if (!is_gimple_reg (PHI_RESULT (phi)))
 	continue;
       if (simple_loop_info)
-	reduc_stmt = vect_is_simple_reduction (simple_loop_info, phi);
+	reduc_stmt = vect_is_simple_reduction (simple_loop_info, phi, true);
 
       /*  Create a reduction_info struct, initialize it and insert it to 
          the reduction list.  */

gcc/tree-vect-loop.c

@@ -414,7 +414,9 @@ vect_analyze_scalar_cycles_1 (loop_vec_info loop_vinfo, struct loop *loop)
   if (vect_print_dump_info (REPORT_DETAILS))
     fprintf (vect_dump, "=== vect_analyze_scalar_cycles ===");
 
-  /* First - identify all inductions.  */
+  /* First - identify all inductions. Reduction detection assumes that all the
+     inductions have been identified, therefore, this order must not be 
+     changed.  */
   for (gsi = gsi_start_phis  (bb); !gsi_end_p (gsi); gsi_next (&gsi))
     {
       gimple phi = gsi_stmt (gsi);
@@ -456,13 +458,14 @@ vect_analyze_scalar_cycles_1 (loop_vec_info loop_vinfo, struct loop *loop)
     }
 
 
-  /* Second - identify all reductions.  */
+  /* Second - identify all reductions and nested cycles.  */
   while (VEC_length (gimple, worklist) > 0)
     {
       gimple phi = VEC_pop (gimple, worklist);
       tree def = PHI_RESULT (phi);
       stmt_vec_info stmt_vinfo = vinfo_for_stmt (phi);
       gimple reduc_stmt;
+      bool nested_cycle;
 
       if (vect_print_dump_info (REPORT_DETAILS))
         { 
@@ -473,14 +476,28 @@ vect_analyze_scalar_cycles_1 (loop_vec_info loop_vinfo, struct loop *loop)
       gcc_assert (is_gimple_reg (SSA_NAME_VAR (def)));
       gcc_assert (STMT_VINFO_DEF_TYPE (stmt_vinfo) == vect_unknown_def_type);
 
-      reduc_stmt = vect_is_simple_reduction (loop_vinfo, phi);
+      nested_cycle = (loop != LOOP_VINFO_LOOP (loop_vinfo));
+      reduc_stmt = vect_is_simple_reduction (loop_vinfo, phi, !nested_cycle);
       if (reduc_stmt)
         {
-          if (vect_print_dump_info (REPORT_DETAILS))
-            fprintf (vect_dump, "Detected reduction.");
-          STMT_VINFO_DEF_TYPE (stmt_vinfo) = vect_reduction_def;
-          STMT_VINFO_DEF_TYPE (vinfo_for_stmt (reduc_stmt)) =
+          if (nested_cycle)
+            {
+              if (vect_print_dump_info (REPORT_DETAILS))
+                fprintf (vect_dump, "Detected vectorizable nested cycle.");
+
+              STMT_VINFO_DEF_TYPE (stmt_vinfo) = vect_nested_cycle;
+              STMT_VINFO_DEF_TYPE (vinfo_for_stmt (reduc_stmt)) =
+                                                        vect_nested_cycle;
+            }
+          else
+            {
+              if (vect_print_dump_info (REPORT_DETAILS))
+                fprintf (vect_dump, "Detected reduction.");
+
+              STMT_VINFO_DEF_TYPE (stmt_vinfo) = vect_reduction_def;
+              STMT_VINFO_DEF_TYPE (vinfo_for_stmt (reduc_stmt)) =
                                                         vect_reduction_def;
+            }
         }
       else
         if (vect_print_dump_info (REPORT_DETAILS))
@@ -488,7 +505,6 @@ vect_analyze_scalar_cycles_1 (loop_vec_info loop_vinfo, struct loop *loop)
     }
 
   VEC_free (gimple, heap, worklist);
-  return;
 }
 
 
@@ -1501,15 +1517,19 @@ report_vect_op (gimple stmt, const char *msg)
   
    such that:
    1. operation is commutative and associative and it is safe to 
-      change the order of the computation.
+      change the order of the computation (if CHECK_REDUCTION is true)
    2. no uses for a2 in the loop (a2 is used out of the loop)
    3. no uses of a1 in the loop besides the reduction operation.
 
    Condition 1 is tested here.
-   Conditions 2,3 are tested in vect_mark_stmts_to_be_vectorized.  */
+   Conditions 2,3 are tested in vect_mark_stmts_to_be_vectorized.  
+
+   Also detect a cross-iteration def-use cycle in nested loops, i.e., nested
+   cycles, if CHECK_REDUCTION is false.  */
 
 gimple
-vect_is_simple_reduction (loop_vec_info loop_info, gimple phi)
+vect_is_simple_reduction (loop_vec_info loop_info, gimple phi, 
+                          bool check_reduction)
 {
   struct loop *loop = (gimple_bb (phi))->loop_father;
   struct loop *vect_loop = LOOP_VINFO_LOOP (loop_info);
@@ -1524,7 +1544,10 @@ vect_is_simple_reduction (loop_vec_info loop_info, gimple phi)
   imm_use_iterator imm_iter;
   use_operand_p use_p;
 
-  gcc_assert (loop == vect_loop || flow_loop_nested_p (vect_loop, loop));
+  /* If CHECK_REDUCTION is true, we assume inner-most loop vectorization,
+     otherwise, we assume outer loop vectorization.  */
+  gcc_assert ((check_reduction && loop == vect_loop) 
+              || (!check_reduction && flow_loop_nested_p (vect_loop, loop)));
 
   name = PHI_RESULT (phi);
   nloop_uses = 0;
@@ -1587,7 +1610,8 @@ vect_is_simple_reduction (loop_vec_info loop_info, gimple phi)
 
   code = gimple_assign_rhs_code (def_stmt);
 
-  if (!commutative_tree_code (code) || !associative_tree_code (code))
+  if (check_reduction 
+      && (!commutative_tree_code (code) || !associative_tree_code (code)))
     {
       if (vect_print_dump_info (REPORT_DETAILS))
         report_vect_op (def_stmt, "reduction: not commutative/associative: ");
@@ -1610,7 +1634,6 @@ vect_is_simple_reduction (loop_vec_info loop_info, gimple phi)
       return NULL;
     }
 
-  /* Check that it's ok to change the order of the computation.  */
   type = TREE_TYPE (gimple_assign_lhs (def_stmt));
   if (TYPE_MAIN_VARIANT (type) != TYPE_MAIN_VARIANT (TREE_TYPE (op1))
       || TYPE_MAIN_VARIANT (type) != TYPE_MAIN_VARIANT (TREE_TYPE (op2)))
@@ -1627,7 +1650,8 @@ vect_is_simple_reduction (loop_vec_info loop_info, gimple phi)
       return NULL;
     }
 
-  /* Generally, when vectorizing a reduction we change the order of the
+  /* Check that it's ok to change the order of the computation.  
+     Generally, when vectorizing a reduction we change the order of the
      computation.  This may change the behavior of the program in some
      cases, so we need to check that this is ok.  One exception is when 
      vectorizing an outer-loop: the inner-loop is executed sequentially,
@@ -1636,7 +1660,7 @@ vect_is_simple_reduction (loop_vec_info loop_info, gimple phi)
 
   /* CHECKME: check for !flag_finite_math_only too?  */
   if (SCALAR_FLOAT_TYPE_P (type) && !flag_associative_math
-      && !nested_in_vect_loop_p (vect_loop, def_stmt)) 
+      && check_reduction) 
     {
       /* Changing the order of operations changes the semantics.  */
       if (vect_print_dump_info (REPORT_DETAILS))
@@ -1644,14 +1668,14 @@ vect_is_simple_reduction (loop_vec_info loop_info, gimple phi)
       return NULL;
     }
   else if (INTEGRAL_TYPE_P (type) && TYPE_OVERFLOW_TRAPS (type)
-	   && !nested_in_vect_loop_p (vect_loop, def_stmt))
+	   && check_reduction)
     {
       /* Changing the order of operations changes the semantics.  */
       if (vect_print_dump_info (REPORT_DETAILS))
 	report_vect_op (def_stmt, "reduction: unsafe int math optimization: ");
       return NULL;
     }
-  else if (SAT_FIXED_POINT_TYPE_P (type))
+  else if (SAT_FIXED_POINT_TYPE_P (type) && check_reduction)
     {
       /* Changing the order of operations changes the semantics.  */
       if (vect_print_dump_info (REPORT_DETAILS))
@@ -1660,10 +1684,10 @@ vect_is_simple_reduction (loop_vec_info loop_info, gimple phi)
       return NULL;
     }
 
-  /* reduction is safe. we're dealing with one of the following:
+  /* Reduction is safe. We're dealing with one of the following:
      1) integer arithmetic and no trapv
-     2) floating point arithmetic, and special flags permit this optimization.
-   */
+     2) floating point arithmetic, and special flags permit this optimization
+     3) nested cycle (i.e., outer loop vectorization).  */
   def1 = SSA_NAME_DEF_STMT (op1);
   def2 = SSA_NAME_DEF_STMT (op2);
   if (!def1 || !def2 || gimple_nop_p (def1) || gimple_nop_p (def2))
@@ -1683,35 +1707,49 @@ vect_is_simple_reduction (loop_vec_info loop_info, gimple phi)
       && (is_gimple_assign (def1)
 	  || STMT_VINFO_DEF_TYPE (vinfo_for_stmt (def1)) == vect_induction_def
 	  || (gimple_code (def1) == GIMPLE_PHI
-	      && STMT_VINFO_DEF_TYPE (vinfo_for_stmt (def1)) == vect_internal_def
+	      && STMT_VINFO_DEF_TYPE (vinfo_for_stmt (def1)) 
+                  == vect_internal_def
 	      && !is_loop_header_bb_p (gimple_bb (def1)))))
     {
       if (vect_print_dump_info (REPORT_DETAILS))
-	report_vect_op (def_stmt, "detected reduction:");
+	report_vect_op (def_stmt, "detected reduction: ");
       return def_stmt;
     }
   else if (def1 == phi
 	   && flow_bb_inside_loop_p (loop, gimple_bb (def2))
 	   && (is_gimple_assign (def2)
-	       || STMT_VINFO_DEF_TYPE (vinfo_for_stmt (def2)) == vect_induction_def
+	       || STMT_VINFO_DEF_TYPE (vinfo_for_stmt (def2))
+                    == vect_induction_def
 	       || (gimple_code (def2) == GIMPLE_PHI
-		   && STMT_VINFO_DEF_TYPE (vinfo_for_stmt (def2)) == vect_internal_def
+		   && STMT_VINFO_DEF_TYPE (vinfo_for_stmt (def2)) 
+                    == vect_internal_def
 		   && !is_loop_header_bb_p (gimple_bb (def2)))))
     {
-      /* Swap operands (just for simplicity - so that the rest of the code
-	 can assume that the reduction variable is always the last (second)
-	 argument).  */
-      if (vect_print_dump_info (REPORT_DETAILS))
-	report_vect_op (def_stmt ,
-		        "detected reduction: need to swap operands:");
-      swap_tree_operands (def_stmt, gimple_assign_rhs1_ptr (def_stmt),
-			  gimple_assign_rhs2_ptr (def_stmt));
+      if (check_reduction)
+        {
+          /* Swap operands (just for simplicity - so that the rest of the code
+	     can assume that the reduction variable is always the last (second)
+	     argument).  */
+          if (vect_print_dump_info (REPORT_DETAILS))
+	    report_vect_op (def_stmt,
+	  	            "detected reduction: need to swap operands: ");
+
+          swap_tree_operands (def_stmt, gimple_assign_rhs1_ptr (def_stmt),
+ 			      gimple_assign_rhs2_ptr (def_stmt));
+        }
+      else
+        {
+          if (vect_print_dump_info (REPORT_DETAILS))
+            report_vect_op (def_stmt, "detected reduction: ");
+        }
+
       return def_stmt;
     }
   else
     {
       if (vect_print_dump_info (REPORT_DETAILS))
-	report_vect_op (def_stmt, "reduction: unknown pattern.");
+	report_vect_op (def_stmt, "reduction: unknown pattern: ");
+
       return NULL;
     }
 }
@@ -2529,6 +2567,7 @@ get_initial_def_for_reduction (gimple stmt, tree init_val, tree *adjustment_def)
   case WIDEN_SUM_EXPR:
   case DOT_PROD_EXPR:
   case PLUS_EXPR:
+  case MINUS_EXPR:
     if (nested_in_vect_loop)
       *adjustment_def = vect_get_vec_def_for_operand (init_val, stmt, NULL);
     else
@@ -2572,6 +2611,8 @@ get_initial_def_for_reduction (gimple stmt, tree init_val, tree *adjustment_def)
      in vectorizable_operation.
    STMT is the scalar reduction stmt that is being vectorized.
    REDUCTION_PHI is the phi-node that carries the reduction computation.
+   REDUC_INDEX is the index of the operand in the right hand side of the 
+     statement that is defined by REDUCTION_PHI.
 
    This function:
    1. Creates the reduction def-use cycle: sets the arguments for 
@@ -2615,7 +2656,8 @@ static void
 vect_create_epilog_for_reduction (tree vect_def, gimple stmt,
 				  int ncopies,
 				  enum tree_code reduc_code,
-				  gimple reduction_phi)
+				  gimple reduction_phi,
+                                  int reduc_index)
 {
   stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
   stmt_vec_info prev_phi_info;
@@ -2659,14 +2701,16 @@ vect_create_epilog_for_reduction (tree vect_def, gimple stmt,
   switch (get_gimple_rhs_class (gimple_assign_rhs_code (stmt)))
     {
     case GIMPLE_SINGLE_RHS:
-      gcc_assert (TREE_OPERAND_LENGTH (gimple_assign_rhs1 (stmt)) == ternary_op);
-      reduction_op = TREE_OPERAND (gimple_assign_rhs1 (stmt), 2);
+      gcc_assert (TREE_OPERAND_LENGTH (gimple_assign_rhs1 (stmt)) 
+                                       == ternary_op);
+      reduction_op = TREE_OPERAND (gimple_assign_rhs1 (stmt), reduc_index);
       break;
     case GIMPLE_UNARY_RHS:
       reduction_op = gimple_assign_rhs1 (stmt);
       break;
     case GIMPLE_BINARY_RHS:
-      reduction_op = gimple_assign_rhs2 (stmt);
+      reduction_op = reduc_index ? 
+                     gimple_assign_rhs2 (stmt) : gimple_assign_rhs1 (stmt);
       break;
     default:
       gcc_unreachable ();
@@ -2754,6 +2798,7 @@ vect_create_epilog_for_reduction (tree vect_def, gimple stmt,
       SET_PHI_ARG_DEF (phi, single_exit (loop)->dest_idx, def);
       prev_phi_info = vinfo_for_stmt (phi);
     }
+
   exit_gsi = gsi_after_labels (exit_bb);
 
   /* 2.2 Get the relevant tree-code to use in the epilog for schemes 2,3 
@@ -2778,6 +2823,7 @@ vect_create_epilog_for_reduction (tree vect_def, gimple stmt,
       gcc_assert (STMT_VINFO_IN_PATTERN_P (stmt_vinfo));
       gcc_assert (STMT_VINFO_RELATED_STMT (stmt_vinfo) == stmt);
     }
+
   code = gimple_assign_rhs_code (orig_stmt);
   scalar_dest = gimple_assign_lhs (orig_stmt);
   scalar_type = TREE_TYPE (scalar_dest);
@@ -2970,6 +3016,11 @@ vect_finalize_reduction:
     {
       if (nested_in_vect_loop)
 	{
+          /* For MINUS_EXPR we create new_temp = loop_exit_def + adjustment_def
+             since the initial value is [0,0,...,0].  */
+          if (code == MINUS_EXPR)
+            code = PLUS_EXPR;
+
 	  gcc_assert (TREE_CODE (TREE_TYPE (adjustment_def)) == VECTOR_TYPE);
 	  expr = build2 (code, vectype, PHI_RESULT (new_phi), adjustment_def);
 	  new_dest = vect_create_destination_var (scalar_dest, vectype);
@@ -2980,6 +3031,7 @@ vect_finalize_reduction:
 	  expr = build2 (code, scalar_type, new_temp, adjustment_def);
 	  new_dest = vect_create_destination_var (scalar_dest, scalar_type);
 	}
+
       epilog_stmt = gimple_build_assign (new_dest, expr);
       new_temp = make_ssa_name (new_dest, epilog_stmt);
       gimple_assign_set_lhs (epilog_stmt, new_temp);
@@ -3043,7 +3095,7 @@ vect_finalize_reduction:
 
    Check if STMT performs a reduction operation that can be vectorized.
    If VEC_STMT is also passed, vectorize the STMT: create a vectorized
-   stmt to replace it, put it in VEC_STMT, and insert it at BSI.
+   stmt to replace it, put it in VEC_STMT, and insert it at GSI.
    Return FALSE if not a vectorizable STMT, TRUE otherwise.
 
    This function also handles reduction idioms (patterns) that have been 
@@ -3115,9 +3167,16 @@ vectorizable_reduction (gimple stmt, gimple_stmt_iterator *gsi,
   gimple new_stmt = NULL;
   int j;
   tree ops[3];
+  bool nested_cycle = false, found_nested_cycle_def = false;
+  gimple reduc_def_stmt = NULL;
+  /* The default is that the reduction variable is the last in statement.  */
+  int reduc_index = 2;
 
   if (nested_in_vect_loop_p (loop, stmt))
-    loop = loop->inner;
+    {
+      loop = loop->inner;
+      nested_cycle = true;
+    }
 
   gcc_assert (ncopies >= 1);
 
@@ -3138,7 +3197,8 @@ vectorizable_reduction (gimple stmt, gimple_stmt_iterator *gsi,
     return false;
 
   /* Make sure it was already recognized as a reduction computation.  */
-  if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_reduction_def)
+  if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_reduction_def
+      && STMT_VINFO_DEF_TYPE (stmt_info) != vect_nested_cycle)
     return false;
 
   /* 2. Has this been recognized as a reduction pattern? 
@@ -3202,7 +3262,9 @@ vectorizable_reduction (gimple stmt, gimple_stmt_iterator *gsi,
     return false;
 
   /* All uses but the last are expected to be defined in the loop.
-     The last use is the reduction variable.  */
+     The last use is the reduction variable. In case of nested cycle this 
+     assumption is not true: we use reduc_index to record the index of the
+     reduction variable.  */
   for (i = 0; i < op_type-1; i++)
     {
       is_simple_use = vect_is_simple_use (ops[i], loop_vinfo, NULL, &def_stmt,
@@ -3211,21 +3273,39 @@ vectorizable_reduction (gimple stmt, gimple_stmt_iterator *gsi,
       if (dt != vect_internal_def
 	  && dt != vect_external_def
 	  && dt != vect_constant_def
-	  && dt != vect_induction_def)
+	  && dt != vect_induction_def
+          && dt != vect_nested_cycle)
 	return false;
+
+      if (dt == vect_nested_cycle)
+        {
+          found_nested_cycle_def = true;
+          reduc_def_stmt = def_stmt;
+          reduc_index = i;
+        }
     }
 
   is_simple_use = vect_is_simple_use (ops[i], loop_vinfo, NULL, &def_stmt, 
                                       &def, &dt);
   gcc_assert (is_simple_use);
-  gcc_assert (dt == vect_reduction_def);
-  gcc_assert (gimple_code (def_stmt) == GIMPLE_PHI);
+  gcc_assert (dt == vect_reduction_def
+              || dt == vect_nested_cycle
+              || ((dt == vect_internal_def || dt == vect_external_def 
+                   || dt == vect_constant_def || dt == vect_induction_def)
+                   && nested_cycle && found_nested_cycle_def)); 
+  if (!found_nested_cycle_def)
+    reduc_def_stmt = def_stmt;
+
+  gcc_assert (gimple_code (reduc_def_stmt) == GIMPLE_PHI);
   if (orig_stmt) 
-    gcc_assert (orig_stmt == vect_is_simple_reduction (loop_vinfo, def_stmt));
+    gcc_assert (orig_stmt == vect_is_simple_reduction (loop_vinfo, 
+                                                       reduc_def_stmt, 
+                                                       !nested_cycle));
   else
-    gcc_assert (stmt == vect_is_simple_reduction (loop_vinfo, def_stmt));
+    gcc_assert (stmt == vect_is_simple_reduction (loop_vinfo, reduc_def_stmt, 
+                                                  !nested_cycle));
   
-  if (STMT_VINFO_LIVE_P (vinfo_for_stmt (def_stmt)))
+  if (STMT_VINFO_LIVE_P (vinfo_for_stmt (reduc_def_stmt)))
     return false;
 
   /* 4. Supportable by target?  */
@@ -3320,8 +3400,12 @@ vectorizable_reduction (gimple stmt, gimple_stmt_iterator *gsi,
       orig_code = code;
     }
 
-  if (!reduction_code_for_scalar_code (orig_code, &epilog_reduc_code))
-    return false;
+  if (nested_cycle)
+    epilog_reduc_code = orig_code;
+  else
+    if (!reduction_code_for_scalar_code (orig_code, &epilog_reduc_code))
+      return false;
+
   reduc_optab = optab_for_tree_code (epilog_reduc_code, vectype, optab_default);
   if (!reduc_optab)
     {
@@ -3402,13 +3486,20 @@ vectorizable_reduction (gimple stmt, gimple_stmt_iterator *gsi,
       /* Handle uses.  */
       if (j == 0)
         {
-	  loop_vec_def0 = vect_get_vec_def_for_operand (ops[0], stmt, NULL);
+	  loop_vec_def0 = vect_get_vec_def_for_operand (ops[!reduc_index], 
+                                                        stmt, NULL);
           if (op_type == ternary_op)
             {
-	      loop_vec_def1 = vect_get_vec_def_for_operand (ops[1], stmt, NULL);
+              if (reduc_index == 0)
+ 	        loop_vec_def1 = vect_get_vec_def_for_operand (ops[2], stmt, 
+                                                              NULL);
+              else
+                loop_vec_def1 = vect_get_vec_def_for_operand (ops[1], stmt, 
+                                                              NULL);
             }
 
-          /* Get the vector def for the reduction variable from the phi node */
+          /* Get the vector def for the reduction variable from the phi 
+             node.  */
           reduc_def = PHI_RESULT (new_phi);
 	  first_phi = new_phi;
         }
@@ -3427,12 +3518,31 @@ vectorizable_reduction (gimple stmt, gimple_stmt_iterator *gsi,
 	  STMT_VINFO_RELATED_STMT (prev_phi_info) = new_phi;
         }
 
+      
       /* Arguments are ready. create the new vector stmt.  */
       if (op_type == binary_op)
-        expr = build2 (code, vectype, loop_vec_def0, reduc_def);
+        {
+          if (reduc_index == 0)
+            expr = build2 (code, vectype, reduc_def, loop_vec_def0);
+          else
+            expr = build2 (code, vectype, loop_vec_def0, reduc_def);
+        }
       else
-        expr = build3 (code, vectype, loop_vec_def0, loop_vec_def1, 
-		       reduc_def);
+        {
+          if (reduc_index == 0)
+            expr = build3 (code, vectype, reduc_def, loop_vec_def0, 
+                           loop_vec_def1);
+          else 
+            {
+              if (reduc_index == 1)
+                expr = build3 (code, vectype, loop_vec_def0, reduc_def, 
+                               loop_vec_def1);
+              else
+                expr = build3 (code, vectype, loop_vec_def0, loop_vec_def1, 
+	     	               reduc_def);
+            }
+        }
+
       new_stmt = gimple_build_assign (vec_dest, expr);
       new_temp = make_ssa_name (vec_dest, new_stmt);
       gimple_assign_set_lhs (new_stmt, new_temp);
@@ -3451,7 +3561,7 @@ vectorizable_reduction (gimple stmt, gimple_stmt_iterator *gsi,
   if (!single_defuse_cycle)
     new_temp = gimple_assign_lhs (*vec_stmt);
   vect_create_epilog_for_reduction (new_temp, stmt, epilog_copies,
-				    epilog_reduc_code, first_phi);
+				    epilog_reduc_code, first_phi, reduc_index);
   return true;
 }
 

gcc/tree-vect-patterns.c

@@ -319,12 +319,7 @@ vect_recog_dot_prod_pattern (gimple last_stmt, tree *type_in, tree *type_out)
 
   /* We don't allow changing the order of the computation in the inner-loop
      when doing outer-loop vectorization.  */
-  if (nested_in_vect_loop_p (loop, last_stmt))
-    {
-      if (vect_print_dump_info (REPORT_DETAILS))
-        fprintf (vect_dump, "vect_recog_dot_prod_pattern: not allowed.");
-      return NULL;
-    }
+  gcc_assert (!nested_in_vect_loop_p (loop, last_stmt));
 
   return pattern_stmt;
 }
@@ -638,12 +633,7 @@ vect_recog_widen_sum_pattern (gimple last_stmt, tree *type_in, tree *type_out)
 
   /* We don't allow changing the order of the computation in the inner-loop
      when doing outer-loop vectorization.  */
-  if (nested_in_vect_loop_p (loop, last_stmt))
-    {
-      if (vect_print_dump_info (REPORT_DETAILS))
-        fprintf (vect_dump, "vect_recog_widen_sum_pattern: not allowed.");
-      return NULL;
-    }
+  gcc_assert (!nested_in_vect_loop_p (loop, last_stmt));
 
   return pattern_stmt;
 }

gcc/tree-vect-stmts.c

@@ -300,19 +300,24 @@ process_use (gimple stmt, tree use, loop_vec_info loop_vinfo, bool live_p,
     {
       if (vect_print_dump_info (REPORT_DETAILS))
 	fprintf (vect_dump, "outer-loop def-stmt defining inner-loop stmt.");
+
       switch (relevant)
 	{
 	case vect_unused_in_scope:
-	  relevant = (STMT_VINFO_DEF_TYPE (stmt_vinfo) == vect_reduction_def) ?
-			vect_used_by_reduction : vect_unused_in_scope;
+	  relevant = (STMT_VINFO_DEF_TYPE (stmt_vinfo) == vect_nested_cycle) ?
+		      vect_used_in_scope : vect_unused_in_scope;
 	  break;
+
 	case vect_used_in_outer_by_reduction:
+          gcc_assert (STMT_VINFO_DEF_TYPE (stmt_vinfo) != vect_reduction_def);
 	  relevant = vect_used_by_reduction;
 	  break;
+
 	case vect_used_in_outer:
+          gcc_assert (STMT_VINFO_DEF_TYPE (stmt_vinfo) != vect_reduction_def);
 	  relevant = vect_used_in_scope;
 	  break;
-	case vect_used_by_reduction: 
+
 	case vect_used_in_scope:
 	  break;
 
@@ -332,6 +337,7 @@ process_use (gimple stmt, tree use, loop_vec_info loop_vinfo, bool live_p,
     {
       if (vect_print_dump_info (REPORT_DETAILS))
 	fprintf (vect_dump, "inner-loop def-stmt defining outer-loop stmt.");
+
       switch (relevant)
         {
         case vect_unused_in_scope:
@@ -339,10 +345,6 @@ process_use (gimple stmt, tree use, loop_vec_info loop_vinfo, bool live_p,
                       vect_used_in_outer_by_reduction : vect_unused_in_scope;
           break;
 
-        case vect_used_in_outer_by_reduction:
-        case vect_used_in_outer:
-          break;
-
         case vect_used_by_reduction:
           relevant = vect_used_in_outer_by_reduction;
           break;
@@ -461,19 +463,7 @@ vect_mark_stmts_to_be_vectorized (loop_vec_info loop_vinfo)
 	 those that are used by a reduction computation, and those that are 
 	 (also) used by a regular computation. This allows us later on to 
 	 identify stmts that are used solely by a reduction, and therefore the 
-	 order of the results that they produce does not have to be kept.
-
-	 Reduction phis are expected to be used by a reduction stmt, or by
-	 in an outer loop;  Other reduction stmts are expected to be
-	 in the loop, and possibly used by a stmt in an outer loop. 
-	 Here are the expected values of "relevant" for reduction phis/stmts:
-
-	 relevance:				phi	stmt
-	 vect_unused_in_scope				ok
-	 vect_used_in_outer_by_reduction	ok	ok
-	 vect_used_in_outer			ok	ok
-	 vect_used_by_reduction			ok
-	 vect_used_in_scope				              */
+	 order of the results that they produce does not have to be kept.  */
 
       if (STMT_VINFO_DEF_TYPE (stmt_vinfo) == vect_reduction_def)
         {
@@ -485,28 +475,41 @@ vect_mark_stmts_to_be_vectorized (loop_vec_info loop_vinfo)
 	      relevant = vect_used_by_reduction;
 	      break;
 
-	    case vect_used_in_outer_by_reduction:
-	    case vect_used_in_outer:
-	      gcc_assert (gimple_code (stmt) != GIMPLE_ASSIGN
-                          || (gimple_assign_rhs_code (stmt) != WIDEN_SUM_EXPR
-                              && (gimple_assign_rhs_code (stmt)
-                                  != DOT_PROD_EXPR)));
-	      break;
-
 	    case vect_used_by_reduction:
 	      if (gimple_code (stmt) == GIMPLE_PHI)
 		break;
 	      /* fall through */
-	    case vect_used_in_scope:
+
 	    default:
 	      if (vect_print_dump_info (REPORT_DETAILS))
 	        fprintf (vect_dump, "unsupported use of reduction.");
 	      VEC_free (gimple, heap, worklist);
 	      return false;
 	    }
+
 	  live_p = false;	
 	}
+      else if (STMT_VINFO_DEF_TYPE (stmt_vinfo) == vect_nested_cycle)
+        {
+          enum vect_relevant tmp_relevant = relevant;
+          switch (tmp_relevant)
+            {
+              case vect_unused_in_scope:
+              case vect_used_in_outer_by_reduction:
+              case vect_used_in_outer:
+                break;
 
+              default:
+                if (vect_print_dump_info (REPORT_DETAILS))
+                  fprintf (vect_dump, "unsupported use of nested cycle.");
+
+                VEC_free (gimple, heap, worklist);
+                return false;
+            }
+
+          live_p = false; 
+        }
+ 
       FOR_EACH_PHI_OR_STMT_USE (use_p, stmt, iter, SSA_OP_USE)
 	{
 	  tree op = USE_FROM_PTR (use_p);
@@ -971,6 +974,7 @@ vect_get_vec_def_for_operand (tree op, gimple stmt, tree *scalar_def)
 
     /* Case 4: operand is defined by a loop header phi - reduction  */
     case vect_reduction_def:
+    case vect_nested_cycle:
       {
 	struct loop *loop;
 
@@ -3929,6 +3933,7 @@ vect_analyze_stmt (gimple stmt, bool *need_to_vectorize, slp_tree node)
         break;
 
       case vect_reduction_def:
+      case vect_nested_cycle:
          gcc_assert (!bb_vinfo && (relevance == vect_used_in_outer
                      || relevance == vect_used_in_outer_by_reduction
                      || relevance == vect_unused_in_scope));

gcc/tree-vectorizer.h

@@ -61,6 +61,7 @@ enum vect_def_type {
   vect_internal_def,
   vect_induction_def,
   vect_reduction_def,
+  vect_nested_cycle,
   vect_unknown_def_type
 };
 
@@ -339,7 +340,11 @@ enum stmt_vec_info_type {
    block.  */
 enum vect_relevant {
   vect_unused_in_scope = 0,
+  /* The def is in the inner loop, and the use is in the outer loop, and the
+     use is a reduction stmt.  */
   vect_used_in_outer_by_reduction,
+  /* The def is in the inner loop, and the use is in the outer loop (and is
+     not part of reduction).  */
   vect_used_in_outer,
 
   /* defs that feed computations that end up (only) in a reduction. These
@@ -817,7 +822,7 @@ extern tree vect_create_addr_base_for_vector_ref (gimple, gimple_seq *,
 /* In tree-vect-loop.c.  */
 /* FORNOW: Used in tree-parloops.c.  */
 extern void destroy_loop_vec_info (loop_vec_info, bool);
-extern gimple vect_is_simple_reduction (loop_vec_info, gimple);
+extern gimple vect_is_simple_reduction (loop_vec_info, gimple, bool);
 /* Drive for loop analysis stage.  */
 extern loop_vec_info vect_analyze_loop (struct loop *);
 /* Drive for loop transformation stage.  */