tree-vect-loop.c (vect_analyze_loop_operations): Also analyze reduction PHIs.

2016-07-03  Richard Biener  <rguenther@suse.de>

	* tree-vect-loop.c (vect_analyze_loop_operations): Also analyze
	reduction PHIs.
	(vect_force_simple_reduction): Record reduction def -> phi mapping.
	(vectorizable_reduction): Perform reduction PHI creation when
	visiting a reduction PHI and adjust and simplify code generation
	phase of the reduction op.  Cache dts, use fold_binary, not fold_build2.
	(vect_transform_loop): Visit reduction PHIs.
	* tree-vect-slp.c (vect_get_and_check_slp_defs): Record reduction
	defs into the SLP tree.
	(vect_build_slp_tree): Reduction defs terminate the recursion.
	* tree-vect-stmts.c (vect_get_vec_def_for_operand_1): Allow lookup
	of reduction defs.
	(vect_get_vec_defs_for_stmt_copy): Export.
	(vect_get_vec_defs): Likewise.
	* tree-vectorizer.h (struct _stmt_vec_info): Amend reduc_def
	purpose.
	(vect_get_vec_defs_for_stmt_copy): Declare.
	(vect_get_vec_defs): Likewise.

From-SVN: r249897

gcc/ChangeLog

+2016-07-03  Richard Biener  <rguenther@suse.de>
+
+	* tree-vect-loop.c (vect_analyze_loop_operations): Also analyze
+	reduction PHIs.
+	(vect_force_simple_reduction): Record reduction def -> phi mapping.
+	(vectorizable_reduction): Perform reduction PHI creation when
+	visiting a reduction PHI and adjust and simplify code generation
+	phase of the reduction op.  Cache dts, use fold_binary, not fold_build2.
+	(vect_transform_loop): Visit reduction PHIs.
+	* tree-vect-slp.c (vect_get_and_check_slp_defs): Record reduction
+	defs into the SLP tree.
+	(vect_build_slp_tree): Reduction defs terminate the recursion.
+	* tree-vect-stmts.c (vect_get_vec_def_for_operand_1): Allow lookup
+	of reduction defs.
+	(vect_get_vec_defs_for_stmt_copy): Export.
+	(vect_get_vec_defs): Likewise.
+	* tree-vectorizer.h (struct _stmt_vec_info): Amend reduc_def
+	purpose.
+	(vect_get_vec_defs_for_stmt_copy): Declare.
+	(vect_get_vec_defs): Likewise.
+
 2017-07-03  Richard Sandiford  <richard.sandiford@linaro.org>
 
 	* tree-data-ref.c (dr_analyze_innermost): Replace the "nest"

gcc/tree-vect-slp.c

@@ -403,9 +403,9 @@ again:
 	{
 	case vect_constant_def:
 	case vect_external_def:
-	case vect_reduction_def:
 	  break;
 
+	case vect_reduction_def:
 	case vect_induction_def:
 	case vect_internal_def:
 	  oprnd_info->def_stmts.quick_push (def_stmt);
@@ -943,13 +943,15 @@ vect_build_slp_tree (vec_info *vinfo,
   else
     return NULL;
 
-  /* If the SLP node is a PHI (induction), terminate the recursion.  */
+  /* If the SLP node is a PHI (induction or reduction), terminate
+     the recursion.  */
   if (gimple_code (stmt) == GIMPLE_PHI)
     {
-      FOR_EACH_VEC_ELT (stmts, i, stmt)
-	if (stmt != stmts[0])
-	  /* Induction from different IVs is not supported.  */
-	  return NULL;
+      /* Induction from different IVs is not supported.  */
+      if (STMT_VINFO_DEF_TYPE (vinfo_for_stmt (stmt)) == vect_induction_def)
+	FOR_EACH_VEC_ELT (stmts, i, stmt)
+	  if (stmt != stmts[0])
+	    return NULL;
       node = vect_create_new_slp_node (stmts);
       return node;
     }
@@ -1005,6 +1007,7 @@ vect_build_slp_tree (vec_info *vinfo,
       unsigned int j;
 
       if (oprnd_info->first_dt != vect_internal_def
+	  && oprnd_info->first_dt != vect_reduction_def
 	  && oprnd_info->first_dt != vect_induction_def)
         continue;
 

gcc/tree-vect-stmts.c

@@ -1367,14 +1367,10 @@ vect_get_vec_def_for_operand_1 (gimple *def_stmt, enum vect_def_type dt)
         return vec_oprnd;
       }
 
-    /* operand is defined by a loop header phi - reduction  */
+    /* operand is defined by a loop header phi.  */
     case vect_reduction_def:
     case vect_double_reduction_def:
     case vect_nested_cycle:
-      /* Code should use get_initial_def_for_reduction.  */
-      gcc_unreachable ();
-
-    /* operand is defined by loop-header phi - induction.  */
     case vect_induction_def:
       {
 	gcc_assert (gimple_code (def_stmt) == GIMPLE_PHI);
@@ -1535,7 +1531,7 @@ vect_get_vec_def_for_stmt_copy (enum vect_def_type dt, tree vec_oprnd)
 /* Get vectorized definitions for the operands to create a copy of an original
    stmt.  See vect_get_vec_def_for_stmt_copy () for details.  */
 
-static void
+void
 vect_get_vec_defs_for_stmt_copy (enum vect_def_type *dt,
 				 vec<tree> *vec_oprnds0,
 				 vec<tree> *vec_oprnds1)
@@ -1554,11 +1550,9 @@ vect_get_vec_defs_for_stmt_copy (enum vect_def_type *dt,
 }
 
 
-/* Get vectorized definitions for OP0 and OP1.
-   REDUC_INDEX is the index of reduction operand in case of reduction,
-   and -1 otherwise.  */
+/* Get vectorized definitions for OP0 and OP1.  */
 
-static void
+void
 vect_get_vec_defs (tree op0, tree op1, gimple *stmt,
 		   vec<tree> *vec_oprnds0,
 		   vec<tree> *vec_oprnds1,

gcc/tree-vectorizer.h

@@ -647,7 +647,9 @@ typedef struct _stmt_vec_info {
      vect_force_simple_reduction.  */
   enum vect_reduction_type reduc_type;
 
-  /* On a reduction PHI the def returned by vect_force_simple_reduction.  */
+  /* On a reduction PHI the def returned by vect_force_simple_reduction.
+     On the def returned by vect_force_simple_reduction the
+     corresponding PHI.  */
   gimple *reduc_def;
 
   /* The number of scalar stmt references from active SLP instances.  */
@@ -1078,6 +1080,10 @@ extern void vect_finish_stmt_generation (gimple *, gimple *,
 extern bool vect_mark_stmts_to_be_vectorized (loop_vec_info);
 extern tree vect_get_vec_def_for_operand_1 (gimple *, enum vect_def_type);
 extern tree vect_get_vec_def_for_operand (tree, gimple *, tree = NULL);
+extern void vect_get_vec_defs (tree, tree, gimple *, vec<tree> *,
+			       vec<tree> *, slp_tree);
+extern void vect_get_vec_defs_for_stmt_copy (enum vect_def_type *,
+					     vec<tree> *, vec<tree> *);
 extern tree vect_init_vector (gimple *, tree, tree,
                               gimple_stmt_iterator *);
 extern tree vect_get_vec_def_for_stmt_copy (enum vect_def_type, tree);