tree-vect-loop.c (needs_fold_left_reduction_p): Export.

2019-10-17  Richard Biener  <rguenther@suse.de>

	* tree-vect-loop.c (needs_fold_left_reduction_p): Export.
	(vect_is_simple_reduction): Move all validity checks ...
	(vectorizable_reduction): ... here.  Compute whether we
	need a fold-left reduction here.
	* tree-vect-patterns.c (vect_reassociating_reduction_p): Merge
	both overloads, check needs_fold_left_reduction_p directly.
	* tree-vectorizer.h (needs_fold_left_reduction_p): Declare.

From-SVN: r277100

gcc/ChangeLog

 2019-10-17  Richard Biener  <rguenther@suse.de>
 
+	* tree-vect-loop.c (needs_fold_left_reduction_p): Export.
+	(vect_is_simple_reduction): Move all validity checks ...
+	(vectorizable_reduction): ... here.  Compute whether we
+	need a fold-left reduction here.
+	* tree-vect-patterns.c (vect_reassociating_reduction_p): Merge
+	both overloads, check needs_fold_left_reduction_p directly.
+	* tree-vectorizer.h (needs_fold_left_reduction_p): Declare.
+
+2019-10-17  Richard Biener  <rguenther@suse.de>
+
 	* tree-ssa-pre.c (create_component_ref_by_pieces_1): Fix
 	TARGET_MEM_REF creation.
 

gcc/tree-vect-loop.c

@@ -2536,7 +2536,7 @@ report_vect_op (dump_flags_t msg_type, gimple *stmt, const char *msg)
    on type TYPE.  NEED_WRAPPING_INTEGRAL_OVERFLOW is true if integer
    overflow must wrap.  */
 
-static bool
+bool
 needs_fold_left_reduction_p (tree type, tree_code code)
 {
   /* CHECKME: check for !flag_finite_math_only too?  */
@@ -2888,13 +2888,6 @@ vect_is_simple_reduction (loop_vec_info loop_info, stmt_vec_info phi_info,
       op1 = gimple_assign_rhs2 (def_stmt);
       op2 = gimple_assign_rhs3 (def_stmt);
     }
-  else if (!commutative_tree_code (code) || !associative_tree_code (code))
-    {
-      if (dump_enabled_p ())
-	report_vect_op (MSG_MISSED_OPTIMIZATION, def_stmt,
-			"reduction: not commutative/associative: ");
-      return NULL;
-    }
   else if (get_gimple_rhs_class (code) == GIMPLE_BINARY_RHS)
     {
       op1 = gimple_assign_rhs1 (def_stmt);
@@ -2917,18 +2910,6 @@ vect_is_simple_reduction (loop_vec_info loop_info, stmt_vec_info phi_info,
       return NULL;
     }
 
-  /* Check whether it's ok to change the order of the computation.
-     Generally, when vectorizing a reduction we change the order of the
-     computation.  This may change the behavior of the program in some
-     cases, so we need to check that this is ok.  One exception is when
-     vectorizing an outer-loop: the inner-loop is executed sequentially,
-     and therefore vectorizing reductions in the inner-loop during
-     outer-loop vectorization is safe.  */
-  tree type = TREE_TYPE (gimple_assign_lhs (def_stmt));
-  if (STMT_VINFO_REDUC_TYPE (phi_info) == TREE_CODE_REDUCTION
-      && needs_fold_left_reduction_p (type, code))
-    STMT_VINFO_REDUC_TYPE (phi_info) = FOLD_LEFT_REDUCTION;
-
   /* Reduction is safe. We're dealing with one of the following:
      1) integer arithmetic and no trapv
      2) floating point arithmetic, and special flags permit this optimization
@@ -5633,7 +5614,6 @@ vectorizable_reduction (stmt_vec_info stmt_info, slp_tree slp_node,
   loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
   class loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
   enum tree_code code;
-  internal_fn reduc_fn;
   int op_type;
   enum vect_def_type dt, cond_reduc_dt = vect_unknown_def_type;
   stmt_vec_info cond_stmt_vinfo = NULL;
@@ -5873,19 +5853,6 @@ vectorizable_reduction (stmt_vec_info stmt_info, slp_tree slp_node,
      operation in the reduction meta.  */
   STMT_VINFO_REDUC_IDX (reduc_info) = reduc_index;
 
-  /* When vectorizing a reduction chain w/o SLP the reduction PHI is not
-     directy used in stmt.  */
-  if (reduc_index == -1)
-    {
-      if (STMT_VINFO_REDUC_TYPE (phi_info) == FOLD_LEFT_REDUCTION)
-	{
-	  if (dump_enabled_p ())
-	    dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
-			     "in-order reduction chain without SLP.\n");
-	  return false;
-	}
-    }
-
   if (!(reduc_index == -1
 	|| dts[reduc_index] == vect_reduction_def
 	|| dts[reduc_index] == vect_nested_cycle
@@ -6048,17 +6015,6 @@ vectorizable_reduction (stmt_vec_info stmt_info, slp_tree slp_node,
       double_reduc = true;
     }
 
-  vect_reduction_type reduction_type = STMT_VINFO_REDUC_TYPE (reduc_info);
-  if ((double_reduc || reduction_type != TREE_CODE_REDUCTION)
-      && ncopies > 1)
-    {
-      if (dump_enabled_p ())
-	dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
-			 "multiple types in double reduction or condition "
-			 "reduction.\n");
-      return false;
-    }
-
   /* 4.2. Check support for the epilog operation.
 
           If STMT represents a reduction pattern, then the type of the
@@ -6094,38 +6050,75 @@ vectorizable_reduction (stmt_vec_info stmt_info, slp_tree slp_node,
           (and also the same tree-code) when generating the epilog code and
           when generating the code inside the loop.  */
 
-  enum tree_code orig_code;
-  if (orig_stmt_info
-      && (reduction_type == TREE_CODE_REDUCTION
-	  || reduction_type == FOLD_LEFT_REDUCTION))
+  vect_reduction_type reduction_type = STMT_VINFO_REDUC_TYPE (reduc_info);
+  enum tree_code orig_code = ERROR_MARK;
+  if (reduction_type == CONST_COND_REDUCTION
+      || reduction_type == INTEGER_INDUC_COND_REDUCTION)
     {
-      /* This is a reduction pattern: get the vectype from the type of the
-         reduction variable, and get the tree-code from orig_stmt.  */
-      orig_code = gimple_assign_rhs_code (orig_stmt_info->stmt);
-      gcc_assert (vectype_out);
+      /* For simple condition reductions, replace with the actual expression
+	 we want to base our reduction around.  */
+      orig_code = STMT_VINFO_VEC_COND_REDUC_CODE (reduc_info);
+      gcc_assert (orig_code == MAX_EXPR || orig_code == MIN_EXPR);
     }
-  else
+  else if (reduction_type == COND_REDUCTION)
+    orig_code = COND_EXPR;
+  else if (reduction_type == TREE_CODE_REDUCTION
+	   || reduction_type == FOLD_LEFT_REDUCTION)
     {
-      /* Regular reduction: use the same vectype and tree-code as used for
-         the vector code inside the loop can be used for the epilog code. */
-      orig_code = code;
-
-      if (code == MINUS_EXPR)
+      if (orig_stmt_info)
+	orig_code = gimple_assign_rhs_code (orig_stmt_info->stmt);
+      else
+	orig_code = code;
+      gcc_assert (vectype_out);
+      if (orig_code == MINUS_EXPR)
 	orig_code = PLUS_EXPR;
+    }
+  STMT_VINFO_REDUC_CODE (reduc_info) = orig_code;
 
-      /* For simple condition reductions, replace with the actual expression
-	 we want to base our reduction around.  */
-      if (reduction_type == CONST_COND_REDUCTION
-	  || reduction_type == INTEGER_INDUC_COND_REDUCTION)
+  if (reduction_type == TREE_CODE_REDUCTION)
+    {
+      /* Check whether it's ok to change the order of the computation.
+	 Generally, when vectorizing a reduction we change the order of the
+	 computation.  This may change the behavior of the program in some
+	 cases, so we need to check that this is ok.  One exception is when
+	 vectorizing an outer-loop: the inner-loop is executed sequentially,
+	 and therefore vectorizing reductions in the inner-loop during
+	 outer-loop vectorization is safe.  */
+      if (needs_fold_left_reduction_p (scalar_type, orig_code))
+	{
+	  STMT_VINFO_REDUC_TYPE (reduc_info)
+	    = reduction_type = FOLD_LEFT_REDUCTION;
+	  /* When vectorizing a reduction chain w/o SLP the reduction PHI is not
+	     directy used in stmt.  */
+	  if (reduc_index == -1)
+	    {
+	      if (dump_enabled_p ())
+		dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
+				 "in-order reduction chain without SLP.\n");
+	      return false;
+	    }
+	}
+      else if (!commutative_tree_code (orig_code)
+	       || !associative_tree_code (orig_code))
 	{
-	  orig_code = STMT_VINFO_VEC_COND_REDUC_CODE (reduc_info);
-	  gcc_assert (orig_code == MAX_EXPR || orig_code == MIN_EXPR);
+	  if (dump_enabled_p ())
+	    dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
+			    "reduction: not commutative/associative");
+	  return false;
 	}
     }
-  STMT_VINFO_REDUC_CODE (reduc_info) = orig_code;
 
-  reduc_fn = IFN_LAST;
+  if ((double_reduc || reduction_type != TREE_CODE_REDUCTION)
+      && ncopies > 1)
+    {
+      if (dump_enabled_p ())
+	dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
+			 "multiple types in double reduction or condition "
+			 "reduction or fold-left reduction.\n");
+      return false;
+    }
 
+  internal_fn reduc_fn = IFN_LAST;
   if (reduction_type == TREE_CODE_REDUCTION
       || reduction_type == FOLD_LEFT_REDUCTION
       || reduction_type == INTEGER_INDUC_COND_REDUCTION

gcc/tree-vect-patterns.c

@@ -833,19 +833,8 @@ vect_convert_output (stmt_vec_info stmt_info, tree type, gimple *pattern_stmt,
 /* Return true if STMT_VINFO describes a reduction for which reassociation
    is allowed.  If STMT_INFO is part of a group, assume that it's part of
    a reduction chain and optimistically assume that all statements
-   except the last allow reassociation.  */
-
-static bool
-vect_reassociating_reduction_p (stmt_vec_info stmt_vinfo)
-{
-  if (STMT_VINFO_DEF_TYPE (stmt_vinfo) == vect_reduction_def)
-    return (STMT_VINFO_REDUC_TYPE (STMT_VINFO_REDUC_DEF (stmt_vinfo))
-	    != FOLD_LEFT_REDUCTION);
-  else
-    return REDUC_GROUP_FIRST_ELEMENT (stmt_vinfo) != NULL;
-}
-
-/* As above, but also require it to have code CODE and to be a reduction
+   except the last allow reassociation.
+   Also require it to have code CODE and to be a reduction
    in the outermost loop.  When returning true, store the operands in
    *OP0_OUT and *OP1_OUT.  */
 
@@ -867,7 +856,13 @@ vect_reassociating_reduction_p (stmt_vec_info stmt_info, tree_code code,
   if (loop && nested_in_vect_loop_p (loop, stmt_info))
     return false;
 
-  if (!vect_reassociating_reduction_p (stmt_info))
+  if (STMT_VINFO_DEF_TYPE (stmt_info) == vect_reduction_def)
+    {
+      if (needs_fold_left_reduction_p (TREE_TYPE (gimple_assign_lhs (assign)),
+				       code))
+	return false;
+    }
+  else if (REDUC_GROUP_FIRST_ELEMENT (stmt_info) == NULL)
     return false;
 
   *op0_out = gimple_assign_rhs1 (assign);

gcc/tree-vectorizer.h

@@ -1632,6 +1632,7 @@ extern widest_int vect_iv_limit_for_full_masking (loop_vec_info loop_vinfo);
 /* Used in gimple-loop-interchange.c and tree-parloops.c.  */
 extern bool check_reduction_path (dump_user_location_t, loop_p, gphi *, tree,
 				  enum tree_code);
+extern bool needs_fold_left_reduction_p (tree, tree_code);
 /* Drive for loop analysis stage.  */
 extern opt_loop_vec_info vect_analyze_loop (class loop *,
 					    loop_vec_info,