tree-vect-loop.c (vect_analyze_loop_operations): Also call…

tree-vect-loop.c (vect_analyze_loop_operations): Also call vectorizable_reduction for vect_double_reduction_def.

2019-09-26  Richard Biener  <rguenther@suse.de>

	* tree-vect-loop.c (vect_analyze_loop_operations): Also call
	vectorizable_reduction for vect_double_reduction_def.
	(vect_transform_loop): Likewise.
	(vect_create_epilog_for_reduction): Move double-reduction
	PHI creation and preheader argument setting of PHIs ...
	(vectorizable_reduction): ... here.  Also process
	vect_double_reduction_def PHIs, creating the vectorized
	PHI nodes, remembering the scalar adjustment computed for
	the epilogue in STMT_VINFO_REDUC_EPILOGUE_ADJUSTMENT.
	Remember the original reduction code in STMT_VINFO_REDUC_CODE.
	* tree-vectorizer.c (vec_info::new_stmt_vec_info):
	Initialize STMT_VINFO_REDUC_CODE.
	* tree-vectorizer.h (_stmt_vec_info::reduc_epilogue_adjustment): New.
	(_stmt_vec_info::reduc_code): Likewise.
	(STMT_VINFO_REDUC_EPILOGUE_ADJUSTMENT): Likewise.
	(STMT_VINFO_REDUC_CODE): Likewise.

From-SVN: r276150

gcc/ChangeLog

+2019-09-26  Richard Biener  <rguenther@suse.de>
+
+	* tree-vect-loop.c (vect_analyze_loop_operations): Also call
+	vectorizable_reduction for vect_double_reduction_def.
+	(vect_transform_loop): Likewise.
+	(vect_create_epilog_for_reduction): Move double-reduction
+	PHI creation and preheader argument setting of PHIs ...
+	(vectorizable_reduction): ... here.  Also process
+	vect_double_reduction_def PHIs, creating the vectorized
+	PHI nodes, remembering the scalar adjustment computed for
+	the epilogue in STMT_VINFO_REDUC_EPILOGUE_ADJUSTMENT.
+	Remember the original reduction code in STMT_VINFO_REDUC_CODE.
+	* tree-vectorizer.c (vec_info::new_stmt_vec_info):
+	Initialize STMT_VINFO_REDUC_CODE.
+	* tree-vectorizer.h (_stmt_vec_info::reduc_epilogue_adjustment): New.
+	(_stmt_vec_info::reduc_code): Likewise.
+	(STMT_VINFO_REDUC_EPILOGUE_ADJUSTMENT): Likewise.
+	(STMT_VINFO_REDUC_CODE): Likewise.
+
 2019-09-26  Matt Turner  <mattst88@gmail.com>
 
 	PR driver/69471

gcc/tree-vectorizer.c

@@ -639,6 +639,7 @@ vec_info::new_stmt_vec_info (gimple *stmt)
   STMT_VINFO_VECTORIZABLE (res) = true;
   STMT_VINFO_VEC_REDUCTION_TYPE (res) = TREE_CODE_REDUCTION;
   STMT_VINFO_VEC_COND_REDUC_CODE (res) = ERROR_MARK;
+  STMT_VINFO_REDUC_CODE (res) = ERROR_MARK;
   STMT_VINFO_REDUC_IDX (res) = -1;
   STMT_VINFO_SLP_VECT_ONLY (res) = false;
 

gcc/tree-vectorizer.h

@@ -941,10 +941,16 @@ public:
   /* For INTEGER_INDUC_COND_REDUCTION, the initial value to be used.  */
   tree induc_cond_initial_val;
 
+  /* If not NULL the value to be added to compute final reduction value.  */
+  tree reduc_epilogue_adjustment;
+
   /* On a reduction PHI the reduction type as detected by
      vect_force_simple_reduction.  */
   enum vect_reduction_type reduc_type;
 
+  /* The original reduction code, to be used in the epilogue.  */
+  enum tree_code reduc_code;
+
   /* On a stmt participating in the reduction the index of the operand
      on the reduction SSA cycle.  */
   int reduc_idx;
@@ -1039,6 +1045,7 @@ STMT_VINFO_BB_VINFO (stmt_vec_info stmt_vinfo)
 #define STMT_VINFO_VEC_REDUCTION_TYPE(S)   (S)->v_reduc_type
 #define STMT_VINFO_VEC_COND_REDUC_CODE(S)  (S)->cond_reduc_code
 #define STMT_VINFO_VEC_INDUC_COND_INITIAL_VAL(S) (S)->induc_cond_initial_val
+#define STMT_VINFO_REDUC_EPILOGUE_ADJUSTMENT(S) (S)->reduc_epilogue_adjustment
 #define STMT_VINFO_REDUC_IDX(S)		   (S)->reduc_idx
 
 #define STMT_VINFO_DR_WRT_VEC_LOOP(S)      (S)->dr_wrt_vec_loop
@@ -1070,6 +1077,7 @@ STMT_VINFO_BB_VINFO (stmt_vec_info stmt_vinfo)
 #define STMT_VINFO_MIN_NEG_DIST(S)	(S)->min_neg_dist
 #define STMT_VINFO_NUM_SLP_USES(S)	(S)->num_slp_uses
 #define STMT_VINFO_REDUC_TYPE(S)	(S)->reduc_type
+#define STMT_VINFO_REDUC_CODE(S)	(S)->reduc_code
 #define STMT_VINFO_REDUC_DEF(S)		(S)->reduc_def
 #define STMT_VINFO_SLP_VECT_ONLY(S)     (S)->slp_vect_only_p