tree-vectorizer.h (_stmt_vec_info::const_cond_reduc_code): Rename to...

2019-09-24  Richard Biener  <rguenther@suse.de>

	* tree-vectorizer.h (_stmt_vec_info::const_cond_reduc_code):
	Rename to...
	(_stmt_vec_info::cond_reduc_code): ... this.
	(_stmt_vec_info::induc_cond_initial_val): Add.
	(STMT_VINFO_VEC_CONST_COND_REDUC_CODE): Rename to...
	(STMT_VINFO_VEC_COND_REDUC_CODE): ... this.
	(STMT_VINFO_VEC_INDUC_COND_INITIAL_VAL): Add.
	* tree-vectorizer.c (vec_info::new_stmt_vec_info): Adjust.
	* tree-vect-loop.c (get_initial_def_for_reduction): Pass in
	the reduction code.
	(vect_create_epilog_for_reduction): Drop special
	induction condition reduction params, pass in reduction code
	and simplify.
	(vectorizable_reduction): Perform condition reduction kind
	selection only at analysis time.  Adjust passing on state.

From-SVN: r276099

gcc/ChangeLog

+2019-09-24  Richard Biener  <rguenther@suse.de>
+
+	* tree-vectorizer.h (_stmt_vec_info::const_cond_reduc_code):
+	Rename to...
+	(_stmt_vec_info::cond_reduc_code): ... this.
+	(_stmt_vec_info::induc_cond_initial_val): Add.
+	(STMT_VINFO_VEC_CONST_COND_REDUC_CODE): Rename to...
+	(STMT_VINFO_VEC_COND_REDUC_CODE): ... this.
+	(STMT_VINFO_VEC_INDUC_COND_INITIAL_VAL): Add.
+	* tree-vectorizer.c (vec_info::new_stmt_vec_info): Adjust.
+	* tree-vect-loop.c (get_initial_def_for_reduction): Pass in
+	the reduction code.
+	(vect_create_epilog_for_reduction): Drop special
+	induction condition reduction params, pass in reduction code
+	and simplify.
+	(vectorizable_reduction): Perform condition reduction kind
+	selection only at analysis time.  Adjust passing on state.
+
 2019-09-24  Kyrylo Tkachov  <kyrylo.tkachov@arm.com>
 
 	* config/aarch64/aarch64.md (mov<mode>): Don't call

gcc/tree-vect-loop.c

@@ -3981,14 +3981,14 @@ vect_model_induction_cost (stmt_vec_info stmt_info, int ncopies,
    A cost model should help decide between these two schemes.  */
 
 static tree
-get_initial_def_for_reduction (stmt_vec_info stmt_vinfo, tree init_val,
+get_initial_def_for_reduction (stmt_vec_info stmt_vinfo,
+			       enum tree_code code, tree init_val,
                                tree *adjustment_def)
 {
   loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_vinfo);
   class loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
   tree scalar_type = TREE_TYPE (init_val);
   tree vectype = get_vectype_for_scalar_type (scalar_type);
-  enum tree_code code = gimple_assign_rhs_code (stmt_vinfo->stmt);
   tree def_for_init;
   tree init_def;
   REAL_VALUE_TYPE real_init_val = dconst0;
@@ -4273,14 +4273,15 @@ static void
 vect_create_epilog_for_reduction (vec<tree> vect_defs,
 				  stmt_vec_info stmt_info,
 				  gimple *reduc_def_stmt,
+				  enum tree_code code,
 				  int ncopies, internal_fn reduc_fn,
 				  vec<stmt_vec_info> reduction_phis,
                                   bool double_reduc, 
 				  slp_tree slp_node,
 				  slp_instance slp_node_instance,
-				  tree induc_val, enum tree_code induc_code,
 				  tree neutral_op)
 {
+  tree induc_val = NULL_TREE;
   stmt_vec_info prev_phi_info;
   tree vectype;
   machine_mode mode;
@@ -4370,17 +4371,22 @@ vect_create_epilog_for_reduction (vec<tree> vect_defs,
       /* Optimize: if initial_def is for REDUC_MAX smaller than the base
 	 and we can't use zero for induc_val, use initial_def.  Similarly
 	 for REDUC_MIN and initial_def larger than the base.  */
+      if (STMT_VINFO_VEC_REDUCTION_TYPE (stmt_info)
+	  == INTEGER_INDUC_COND_REDUCTION)
+	{
+	  induc_val = STMT_VINFO_VEC_INDUC_COND_INITIAL_VAL (stmt_info);
 	  if (TREE_CODE (initial_def) == INTEGER_CST
 	      && (STMT_VINFO_VEC_REDUCTION_TYPE (stmt_info)
 		  == INTEGER_INDUC_COND_REDUCTION)
 	      && !integer_zerop (induc_val)
-	  && ((induc_code == MAX_EXPR
+	      && ((code == MAX_EXPR
 		   && tree_int_cst_lt (initial_def, induc_val))
-	      || (induc_code == MIN_EXPR
+		  || (code == MIN_EXPR
 		      && tree_int_cst_lt (induc_val, initial_def))))
 	    induc_val = initial_def;
-
-      if (double_reduc)
+	  vec_initial_def = build_vector_from_val (vectype, induc_val);
+	}
+      else if (double_reduc)
 	/* In case of double reduction we only create a vector variable
 	   to be put in the reduction phi node.  The actual statement
 	   creation is done later in this function.  */
@@ -4394,7 +4400,7 @@ vect_create_epilog_for_reduction (vec<tree> vect_defs,
 	}
       else
 	vec_initial_def
-	  = get_initial_def_for_reduction (stmt_info, initial_def,
+	  = get_initial_def_for_reduction (stmt_info, code, initial_def,
 					   &adjustment_def);
       vec_initial_defs.create (1);
       vec_initial_defs.quick_push (vec_initial_def);
@@ -4418,22 +4424,6 @@ vect_create_epilog_for_reduction (vec<tree> vect_defs,
 	  /* Set the loop-entry arg of the reduction-phi.  */
 
 	  gphi *phi = as_a <gphi *> (phi_info->stmt);
-	  if (STMT_VINFO_VEC_REDUCTION_TYPE (stmt_info)
-	      == INTEGER_INDUC_COND_REDUCTION)
-	    {
-	      /* Initialise the reduction phi to zero.  This prevents initial
-		 values of non-zero interferring with the reduction op.  */
-	      gcc_assert (ncopies == 1);
-	      gcc_assert (i == 0);
-
-	      tree vec_init_def_type = TREE_TYPE (vec_init_def);
-	      tree induc_val_vec
-		= build_vector_from_val (vec_init_def_type, induc_val);
-
-	      add_phi_arg (phi, induc_val_vec, loop_preheader_edge (loop),
-			   UNKNOWN_LOCATION);
-	    }
-	  else
 	  add_phi_arg (phi, vec_init_def, loop_preheader_edge (loop),
 		       UNKNOWN_LOCATION);
 
@@ -4652,12 +4642,6 @@ vect_create_epilog_for_reduction (vec<tree> vect_defs,
     ;
   else
     {
-  enum tree_code code = gimple_assign_rhs_code (orig_stmt_info->stmt);
-  /* For MINUS_EXPR the initial vector is [init_val,0,...,0], therefore,
-     partial results are added and not subtracted.  */
-  if (code == MINUS_EXPR) 
-    code = PLUS_EXPR;
-
   /* SLP reduction without reduction chain, e.g.,
      # a1 = phi <a2, a0>
      # b1 = phi <b2, b0>
@@ -5049,20 +5033,6 @@ vect_create_epilog_for_reduction (vec<tree> vect_defs,
       bool reduce_with_shift;
       tree vec_temp;
 
-      /* COND reductions all do the final reduction with MAX_EXPR
-	 or MIN_EXPR.  */
-      if (code == COND_EXPR)
-	{
-	  if (STMT_VINFO_VEC_REDUCTION_TYPE (stmt_info)
-	      == INTEGER_INDUC_COND_REDUCTION)
-	    code = induc_code;
-	  else if (STMT_VINFO_VEC_REDUCTION_TYPE (stmt_info)
-		   == CONST_COND_REDUCTION)
-	    code = STMT_VINFO_VEC_CONST_COND_REDUC_CODE (stmt_info);
-	  else
-	    code = MAX_EXPR;
-	}
-
       /* See if the target wants to do the final (shift) reduction
 	 in a vector mode of smaller size and first reduce upper/lower
 	 halves against each other.  */
@@ -5543,7 +5513,7 @@ vect_create_epilog_for_reduction (vec<tree> vect_defs,
                   preheader_arg = PHI_ARG_DEF_FROM_EDGE (use_stmt,
                                              loop_preheader_edge (outer_loop));
                   vect_phi_init = get_initial_def_for_reduction
-		    (stmt_info, preheader_arg, NULL);
+		    (stmt_info, code, preheader_arg, NULL);
 
                   /* Update phi node arguments with vs0 and vs2.  */
                   add_phi_arg (vect_phi, vect_phi_init,
@@ -6021,7 +5991,7 @@ vectorizable_reduction (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
   tree vectype_in = NULL_TREE;
   loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
   class loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
-  enum tree_code code, orig_code;
+  enum tree_code code;
   internal_fn reduc_fn;
   machine_mode vec_mode;
   int op_type;
@@ -6029,7 +5999,6 @@ vectorizable_reduction (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
   tree new_temp = NULL_TREE;
   enum vect_def_type dt, cond_reduc_dt = vect_unknown_def_type;
   stmt_vec_info cond_stmt_vinfo = NULL;
-  enum tree_code cond_reduc_op_code = ERROR_MARK;
   tree scalar_type;
   bool is_simple_use;
   int i;
@@ -6362,9 +6331,11 @@ vectorizable_reduction (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
     = STMT_VINFO_REDUC_TYPE (reduc_def_info);
   stmt_vec_info tmp = STMT_VINFO_REDUC_DEF (reduc_def_info);
 
+  if (!vec_stmt)
     STMT_VINFO_VEC_REDUCTION_TYPE (stmt_info) = v_reduc_type;
   /* If we have a condition reduction, see if we can simplify it further.  */
-  if (v_reduc_type == COND_REDUCTION)
+  if (v_reduc_type == COND_REDUCTION
+      && !vec_stmt)
     {
       /* TODO: We can't yet handle reduction chains, since we need to treat
 	 each COND_EXPR in the chain specially, not just the last one.
@@ -6386,19 +6357,7 @@ vectorizable_reduction (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
 	  return false;
 	}
 
-      /* Loop peeling modifies initial value of reduction PHI, which
-	 makes the reduction stmt to be transformed different to the
-	 original stmt analyzed.  We need to record reduction code for
-	 CONST_COND_REDUCTION type reduction at analyzing stage, thus
-	 it can be used directly at transform stage.  */
-      if (STMT_VINFO_VEC_CONST_COND_REDUC_CODE (stmt_info) == MAX_EXPR
-	  || STMT_VINFO_VEC_CONST_COND_REDUC_CODE (stmt_info) == MIN_EXPR)
-	{
-	  /* Also set the reduction type to CONST_COND_REDUCTION.  */
-	  gcc_assert (cond_reduc_dt == vect_constant_def);
-	  STMT_VINFO_VEC_REDUCTION_TYPE (stmt_info) = CONST_COND_REDUCTION;
-	}
-      else if (direct_internal_fn_supported_p (IFN_FOLD_EXTRACT_LAST,
+      if (direct_internal_fn_supported_p (IFN_FOLD_EXTRACT_LAST,
 					  vectype_in, OPTIMIZE_FOR_SPEED))
 	{
 	  if (dump_enabled_p ())
@@ -6416,6 +6375,7 @@ vectorizable_reduction (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
 	  gcc_assert (TREE_CODE (base) == INTEGER_CST
 		      && TREE_CODE (step) == INTEGER_CST);
 	  cond_reduc_val = NULL_TREE;
+	  enum tree_code cond_reduc_op_code = ERROR_MARK;
 	  tree res = PHI_RESULT (STMT_VINFO_STMT (cond_stmt_vinfo));
 	  if (!types_compatible_p (TREE_TYPE (res), TREE_TYPE (base)))
 	    ;
@@ -6444,10 +6404,10 @@ vectorizable_reduction (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
 	    }
 	  if (cond_reduc_val)
 	    {
-	      if (dump_enabled_p ())
-		dump_printf_loc (MSG_NOTE, vect_location,
-				 "condition expression based on "
-				 "integer induction.\n");
+	      STMT_VINFO_VEC_COND_REDUC_CODE (stmt_info)
+		= cond_reduc_op_code;
+	      STMT_VINFO_VEC_INDUC_COND_INITIAL_VAL (stmt_info)
+		= cond_reduc_val;
 	      STMT_VINFO_VEC_REDUCTION_TYPE (stmt_info)
 		= INTEGER_INDUC_COND_REDUCTION;
 	    }
@@ -6474,7 +6434,7 @@ vectorizable_reduction (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
 				     "condition expression based on "
 				     "compile time constant.\n");
 		  /* Record reduction code at analysis stage.  */
-		  STMT_VINFO_VEC_CONST_COND_REDUC_CODE (stmt_info)
+		  STMT_VINFO_VEC_COND_REDUC_CODE (stmt_info)
 		    = integer_onep (e) ? MAX_EXPR : MIN_EXPR;
 		  STMT_VINFO_VEC_REDUCTION_TYPE (stmt_info)
 		    = CONST_COND_REDUCTION;
@@ -6482,6 +6442,11 @@ vectorizable_reduction (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
 	    }
 	}
     }
+  if (STMT_VINFO_VEC_REDUCTION_TYPE (stmt_info) == INTEGER_INDUC_COND_REDUCTION
+      && dump_enabled_p ())
+    dump_printf_loc (MSG_NOTE, vect_location,
+		     "condition expression based on "
+		     "integer induction.\n");
 
   if (orig_stmt_info)
     gcc_assert (tmp == orig_stmt_info
@@ -6637,6 +6602,7 @@ vectorizable_reduction (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
           (and also the same tree-code) when generating the epilog code and
           when generating the code inside the loop.  */
 
+  enum tree_code orig_code;
   if (orig_stmt_info
       && (reduction_type == TREE_CODE_REDUCTION
 	  || reduction_type == FOLD_LEFT_REDUCTION))
@@ -6658,13 +6624,12 @@ vectorizable_reduction (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
 
       /* For simple condition reductions, replace with the actual expression
 	 we want to base our reduction around.  */
-      if (reduction_type == CONST_COND_REDUCTION)
+      if (reduction_type == CONST_COND_REDUCTION
+	  || reduction_type == INTEGER_INDUC_COND_REDUCTION)
 	{
-	  orig_code = STMT_VINFO_VEC_CONST_COND_REDUC_CODE (stmt_info);
+	  orig_code = STMT_VINFO_VEC_COND_REDUC_CODE (stmt_info);
 	  gcc_assert (orig_code == MAX_EXPR || orig_code == MIN_EXPR);
 	}
-      else if (reduction_type == INTEGER_INDUC_COND_REDUCTION)
-	orig_code = cond_reduc_op_code;
     }
 
   reduc_fn = IFN_LAST;
@@ -7171,9 +7136,8 @@ vectorizable_reduction (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
     vect_defs[0] = gimple_get_lhs ((*vec_stmt)->stmt);
 
   vect_create_epilog_for_reduction (vect_defs, stmt_info, reduc_def_phi,
-				    epilog_copies, reduc_fn, phis,
+				    orig_code, epilog_copies, reduc_fn, phis,
 				    double_reduc, slp_node, slp_node_instance,
-				    cond_reduc_val, cond_reduc_op_code,
 				    neutral_op);
 
   return true;

gcc/tree-vectorizer.c

@@ -638,7 +638,7 @@ vec_info::new_stmt_vec_info (gimple *stmt)
   STMT_VINFO_RELEVANT (res) = vect_unused_in_scope;
   STMT_VINFO_VECTORIZABLE (res) = true;
   STMT_VINFO_VEC_REDUCTION_TYPE (res) = TREE_CODE_REDUCTION;
-  STMT_VINFO_VEC_CONST_COND_REDUC_CODE (res) = ERROR_MARK;
+  STMT_VINFO_VEC_COND_REDUC_CODE (res) = ERROR_MARK;
   STMT_VINFO_REDUC_IDX (res) = -1;
   STMT_VINFO_SLP_VECT_ONLY (res) = false;
 

gcc/tree-vectorizer.h

@@ -934,8 +934,12 @@ public:
   /* For reduction loops, this is the type of reduction.  */
   enum vect_reduction_type v_reduc_type;
 
-  /* For CONST_COND_REDUCTION, record the reduc code.  */
-  enum tree_code const_cond_reduc_code;
+  /* For CONST_COND_REDUCTION and INTEGER_INDUC_COND_REDUCTION, the
+     reduction code.  */
+  enum tree_code cond_reduc_code;
+
+  /* For INTEGER_INDUC_COND_REDUCTION, the initial value to be used.  */
+  tree induc_cond_initial_val;
 
   /* On a reduction PHI the reduction type as detected by
      vect_force_simple_reduction.  */
@@ -1033,7 +1037,8 @@ STMT_VINFO_BB_VINFO (stmt_vec_info stmt_vinfo)
 #define STMT_VINFO_MEMORY_ACCESS_TYPE(S)   (S)->memory_access_type
 #define STMT_VINFO_SIMD_LANE_ACCESS_P(S)   (S)->simd_lane_access_p
 #define STMT_VINFO_VEC_REDUCTION_TYPE(S)   (S)->v_reduc_type
-#define STMT_VINFO_VEC_CONST_COND_REDUC_CODE(S) (S)->const_cond_reduc_code
+#define STMT_VINFO_VEC_COND_REDUC_CODE(S)  (S)->cond_reduc_code
+#define STMT_VINFO_VEC_INDUC_COND_INITIAL_VAL(S) (S)->induc_cond_initial_val
 #define STMT_VINFO_REDUC_IDX(S)		   (S)->reduc_idx
 
 #define STMT_VINFO_DR_WRT_VEC_LOOP(S)      (S)->dr_wrt_vec_loop