tree-vect-loop.c (vectorizable_reduction): Move special cond reduction IV var creation ...

2017-06-28  Richard Biener  <rguenther@suse.de>

	* tree-vect-loop.c (vectorizable_reduction): Move special
	cond reduction IV var creation ...
	(vect_create_epilog_for_reduction): ... here.  Remove induction_index
	parameter.  Use STMT_VINFO_VECTYPE.
	* tree-vect-slp.c (vect_get_constant_vectors): Properly reset
	constant_p.

From-SVN: r249736

gcc/ChangeLog

+2017-06-28  Richard Biener  <rguenther@suse.de>
+
+	* tree-vect-loop.c (vectorizable_reduction): Move special
+	cond reduction IV var creation ...
+	(vect_create_epilog_for_reduction): ... here.  Remove induction_index
+	parameter.  Use STMT_VINFO_VECTYPE.
+	* tree-vect-slp.c (vect_get_constant_vectors): Properly reset
+	constant_p.
+
 2017-06-28  Martin Liska  <mliska@suse.cz>
 
 	PR ipa/81128

gcc/tree-vect-loop.c

@@ -4263,8 +4263,6 @@ get_initial_defs_for_reduction (slp_tree slp_node,
    DOUBLE_REDUC is TRUE if double reduction phi nodes should be handled.
    SLP_NODE is an SLP node containing a group of reduction statements. The 
      first one in this group is STMT.
-   INDUCTION_INDEX is the index of the loop for condition reductions.
-     Otherwise it is undefined.
 
    This function:
    1. Creates the reduction def-use cycles: sets the arguments for 
@@ -4310,7 +4308,7 @@ vect_create_epilog_for_reduction (vec<tree> vect_defs, gimple *stmt,
 				  int ncopies, enum tree_code reduc_code,
 				  vec<gimple *> reduction_phis,
                                   int reduc_index, bool double_reduc, 
-				  slp_tree slp_node, tree induction_index)
+				  slp_tree slp_node)
 {
   stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
   stmt_vec_info prev_phi_info;
@@ -4331,7 +4329,7 @@ vect_create_epilog_for_reduction (vec<tree> vect_defs, gimple *stmt,
   tree bitsize;
   tree adjustment_def = NULL;
   tree vec_initial_def = NULL;
-  tree reduction_op, expr, def, initial_def = NULL;
+  tree expr, def, initial_def = NULL;
   tree orig_name, scalar_result;
   imm_use_iterator imm_iter, phi_imm_iter;
   use_operand_p use_p, phi_use_p;
@@ -4348,6 +4346,7 @@ vect_create_epilog_for_reduction (vec<tree> vect_defs, gimple *stmt,
   bool slp_reduc = false;
   tree new_phi_result;
   gimple *inner_phi = NULL;
+  tree induction_index = NULL_TREE;
 
   if (slp_node)
     group_size = SLP_TREE_SCALAR_STMTS (slp_node).length (); 
@@ -4360,9 +4359,7 @@ vect_create_epilog_for_reduction (vec<tree> vect_defs, gimple *stmt,
       gcc_assert (!slp_node);
     }
 
-  reduction_op = get_reduction_op (stmt, reduc_index);
-
-  vectype = get_vectype_for_scalar_type (TREE_TYPE (reduction_op));
+  vectype = STMT_VINFO_VECTYPE (stmt_info);
   gcc_assert (vectype);
   mode = TYPE_MODE (vectype);
 
@@ -4396,6 +4393,7 @@ vect_create_epilog_for_reduction (vec<tree> vect_defs, gimple *stmt,
     {
       /* Get at the scalar def before the loop, that defines the initial value
 	 of the reduction variable.  */
+      tree reduction_op = get_reduction_op (stmt, reduc_index);
       gimple *def_stmt = SSA_NAME_DEF_STMT (reduction_op);
       initial_def = PHI_ARG_DEF_FROM_EDGE (def_stmt,
 					   loop_preheader_edge (loop));
@@ -4465,6 +4463,95 @@ vect_create_epilog_for_reduction (vec<tree> vect_defs, gimple *stmt,
         }
     }
 
+  /* For cond reductions we want to create a new vector (INDEX_COND_EXPR)
+     which is updated with the current index of the loop for every match of
+     the original loop's cond_expr (VEC_STMT).  This results in a vector
+     containing the last time the condition passed for that vector lane.
+     The first match will be a 1 to allow 0 to be used for non-matching
+     indexes.  If there are no matches at all then the vector will be all
+     zeroes.  */
+  if (STMT_VINFO_VEC_REDUCTION_TYPE (stmt_info) == COND_REDUCTION)
+    {
+      tree indx_before_incr, indx_after_incr;
+      int nunits_out = TYPE_VECTOR_SUBPARTS (vectype);
+      int k;
+
+      gimple *vec_stmt = STMT_VINFO_VEC_STMT (stmt_info);
+      gcc_assert (gimple_assign_rhs_code (vec_stmt) == VEC_COND_EXPR);
+
+      int scalar_precision
+	= GET_MODE_PRECISION (TYPE_MODE (TREE_TYPE (vectype)));
+      tree cr_index_scalar_type = make_unsigned_type (scalar_precision);
+      tree cr_index_vector_type = build_vector_type
+	(cr_index_scalar_type, TYPE_VECTOR_SUBPARTS (vectype));
+
+      /* First we create a simple vector induction variable which starts
+	 with the values {1,2,3,...} (SERIES_VECT) and increments by the
+	 vector size (STEP).  */
+
+      /* Create a {1,2,3,...} vector.  */
+      tree *vtemp = XALLOCAVEC (tree, nunits_out);
+      for (k = 0; k < nunits_out; ++k)
+	vtemp[k] = build_int_cst (cr_index_scalar_type, k + 1);
+      tree series_vect = build_vector (cr_index_vector_type, vtemp);
+
+      /* Create a vector of the step value.  */
+      tree step = build_int_cst (cr_index_scalar_type, nunits_out);
+      tree vec_step = build_vector_from_val (cr_index_vector_type, step);
+
+      /* Create an induction variable.  */
+      gimple_stmt_iterator incr_gsi;
+      bool insert_after;
+      standard_iv_increment_position (loop, &incr_gsi, &insert_after);
+      create_iv (series_vect, vec_step, NULL_TREE, loop, &incr_gsi,
+		 insert_after, &indx_before_incr, &indx_after_incr);
+
+      /* Next create a new phi node vector (NEW_PHI_TREE) which starts
+	 filled with zeros (VEC_ZERO).  */
+
+      /* Create a vector of 0s.  */
+      tree zero = build_zero_cst (cr_index_scalar_type);
+      tree vec_zero = build_vector_from_val (cr_index_vector_type, zero);
+
+      /* Create a vector phi node.  */
+      tree new_phi_tree = make_ssa_name (cr_index_vector_type);
+      new_phi = create_phi_node (new_phi_tree, loop->header);
+      set_vinfo_for_stmt (new_phi,
+			  new_stmt_vec_info (new_phi, loop_vinfo));
+      add_phi_arg (as_a <gphi *> (new_phi), vec_zero,
+		   loop_preheader_edge (loop), UNKNOWN_LOCATION);
+
+      /* Now take the condition from the loops original cond_expr
+	 (VEC_STMT) and produce a new cond_expr (INDEX_COND_EXPR) which for
+	 every match uses values from the induction variable
+	 (INDEX_BEFORE_INCR) otherwise uses values from the phi node
+	 (NEW_PHI_TREE).
+	 Finally, we update the phi (NEW_PHI_TREE) to take the value of
+	 the new cond_expr (INDEX_COND_EXPR).  */
+
+      /* Duplicate the condition from vec_stmt.  */
+      tree ccompare = unshare_expr (gimple_assign_rhs1 (vec_stmt));
+
+      /* Create a conditional, where the condition is taken from vec_stmt
+	 (CCOMPARE), then is the induction index (INDEX_BEFORE_INCR) and
+	 else is the phi (NEW_PHI_TREE).  */
+      tree index_cond_expr = build3 (VEC_COND_EXPR, cr_index_vector_type,
+				     ccompare, indx_before_incr,
+				     new_phi_tree);
+      induction_index = make_ssa_name (cr_index_vector_type);
+      gimple *index_condition = gimple_build_assign (induction_index,
+						     index_cond_expr);
+      gsi_insert_before (&incr_gsi, index_condition, GSI_SAME_STMT);
+      stmt_vec_info index_vec_info = new_stmt_vec_info (index_condition,
+							loop_vinfo);
+      STMT_VINFO_VECTYPE (index_vec_info) = cr_index_vector_type;
+      set_vinfo_for_stmt (index_condition, index_vec_info);
+
+      /* Update the phi with the vec cond.  */
+      add_phi_arg (as_a <gphi *> (new_phi), induction_index,
+		   loop_latch_edge (loop), UNKNOWN_LOCATION);
+    }
+
   /* 2. Create epilog code.
         The reduction epilog code operates across the elements of the vector
         of partial results computed by the vectorized loop.
@@ -6248,100 +6335,14 @@ vectorizable_reduction (gimple *stmt, gimple_stmt_iterator *gsi,
       prev_phi_info = vinfo_for_stmt (new_phi);
     }
 
-  tree indx_before_incr, indx_after_incr, cond_name = NULL;
-
   /* Finalize the reduction-phi (set its arguments) and create the
      epilog reduction code.  */
   if ((!single_defuse_cycle || code == COND_EXPR) && !slp_node)
-    {
-      new_temp = gimple_assign_lhs (*vec_stmt);
-      vect_defs[0] = new_temp;
-
-      /* For cond reductions we want to create a new vector (INDEX_COND_EXPR)
-	 which is updated with the current index of the loop for every match of
-	 the original loop's cond_expr (VEC_STMT).  This results in a vector
-	 containing the last time the condition passed for that vector lane.
-	 The first match will be a 1 to allow 0 to be used for non-matching
-	 indexes.  If there are no matches at all then the vector will be all
-	 zeroes.  */
-      if (STMT_VINFO_VEC_REDUCTION_TYPE (stmt_info) == COND_REDUCTION)
-	{
-	  int nunits_out = TYPE_VECTOR_SUBPARTS (vectype_out);
-	  int k;
-
-	  gcc_assert (gimple_assign_rhs_code (*vec_stmt) == VEC_COND_EXPR);
-
-	  /* First we create a simple vector induction variable which starts
-	     with the values {1,2,3,...} (SERIES_VECT) and increments by the
-	     vector size (STEP).  */
-
-	  /* Create a {1,2,3,...} vector.  */
-	  tree *vtemp = XALLOCAVEC (tree, nunits_out);
-	  for (k = 0; k < nunits_out; ++k)
-	    vtemp[k] = build_int_cst (cr_index_scalar_type, k + 1);
-	  tree series_vect = build_vector (cr_index_vector_type, vtemp);
-
-	  /* Create a vector of the step value.  */
-	  tree step = build_int_cst (cr_index_scalar_type, nunits_out);
-	  tree vec_step = build_vector_from_val (cr_index_vector_type, step);
-
-	  /* Create an induction variable.  */
-	  gimple_stmt_iterator incr_gsi;
-	  bool insert_after;
-	  standard_iv_increment_position (loop, &incr_gsi, &insert_after);
-	  create_iv (series_vect, vec_step, NULL_TREE, loop, &incr_gsi,
-		     insert_after, &indx_before_incr, &indx_after_incr);
-
-	  /* Next create a new phi node vector (NEW_PHI_TREE) which starts
-	     filled with zeros (VEC_ZERO).  */
-
-	  /* Create a vector of 0s.  */
-	  tree zero = build_zero_cst (cr_index_scalar_type);
-	  tree vec_zero = build_vector_from_val (cr_index_vector_type, zero);
-
-	  /* Create a vector phi node.  */
-	  tree new_phi_tree = make_ssa_name (cr_index_vector_type);
-	  new_phi = create_phi_node (new_phi_tree, loop->header);
-	  set_vinfo_for_stmt (new_phi,
-			      new_stmt_vec_info (new_phi, loop_vinfo));
-	  add_phi_arg (new_phi, vec_zero, loop_preheader_edge (loop),
-		       UNKNOWN_LOCATION);
-
-	  /* Now take the condition from the loops original cond_expr
-	     (VEC_STMT) and produce a new cond_expr (INDEX_COND_EXPR) which for
-	     every match uses values from the induction variable
-	     (INDEX_BEFORE_INCR) otherwise uses values from the phi node
-	     (NEW_PHI_TREE).
-	     Finally, we update the phi (NEW_PHI_TREE) to take the value of
-	     the new cond_expr (INDEX_COND_EXPR).  */
-
-	  /* Duplicate the condition from vec_stmt.  */
-	  tree ccompare = unshare_expr (gimple_assign_rhs1 (*vec_stmt));
-
-	  /* Create a conditional, where the condition is taken from vec_stmt
-	     (CCOMPARE), then is the induction index (INDEX_BEFORE_INCR) and
-	     else is the phi (NEW_PHI_TREE).  */
-	  tree index_cond_expr = build3 (VEC_COND_EXPR, cr_index_vector_type,
-					 ccompare, indx_before_incr,
-					 new_phi_tree);
-	  cond_name = make_ssa_name (cr_index_vector_type);
-	  gimple *index_condition = gimple_build_assign (cond_name,
-							 index_cond_expr);
-	  gsi_insert_before (&incr_gsi, index_condition, GSI_SAME_STMT);
-	  stmt_vec_info index_vec_info = new_stmt_vec_info (index_condition,
-							    loop_vinfo);
-	  STMT_VINFO_VECTYPE (index_vec_info) = cr_index_vector_type;
-	  set_vinfo_for_stmt (index_condition, index_vec_info);
-
-	  /* Update the phi with the vec cond.  */
-	  add_phi_arg (new_phi, cond_name, loop_latch_edge (loop),
-		       UNKNOWN_LOCATION);
-	}
-    }
+    vect_defs[0] = gimple_assign_lhs (*vec_stmt);
 
   vect_create_epilog_for_reduction (vect_defs, stmt, epilog_copies,
                                     epilog_reduc_code, phis, reduc_index,
-				    double_reduc, slp_node, cond_name);
+				    double_reduc, slp_node);
 
   return true;
 }

gcc/tree-vect-slp.c

@@ -3016,11 +3016,6 @@ vect_get_constant_vectors (tree op, slp_tree slp_node,
 
   gcc_assert (op);
 
-  if (CONSTANT_CLASS_P (op))
-    constant_p = true;
-  else
-    constant_p = false;
-
   /* NUMBER_OF_COPIES is the number of times we need to use the same values in
      created vectors. It is greater than 1 if unrolling is performed.
 
@@ -3040,6 +3035,7 @@ vect_get_constant_vectors (tree op, slp_tree slp_node,
   number_of_copies = nunits * number_of_vectors / group_size;
 
   number_of_places_left_in_vector = nunits;
+  constant_p = true;
   elts = XALLOCAVEC (tree, nunits);
   bool place_after_defs = false;
   for (j = 0; j < number_of_copies; j++)
@@ -3156,8 +3152,6 @@ vect_get_constant_vectors (tree op, slp_tree slp_node,
 
           if (number_of_places_left_in_vector == 0)
             {
-              number_of_places_left_in_vector = nunits;
-
 	      if (constant_p)
 		vec_cst = build_vector (vector_type, elts);
 	      else
@@ -3188,6 +3182,8 @@ vect_get_constant_vectors (tree op, slp_tree slp_node,
 		}
 	      voprnds.quick_push (init);
 	      place_after_defs = false;
+              number_of_places_left_in_vector = nunits;
+	      constant_p = true;
             }
         }
     }