[vect]PR 88915: Vectorize epilogues when versioning loops

gcc/ChangeLog:
2019-10-29  Andre Vieira  <andre.simoesdiasvieira@arm.com>

	PR 88915
	* tree-ssa-loop-niter.h (simplify_replace_tree): Change declaration.
	* tree-ssa-loop-niter.c (simplify_replace_tree): Add context parameter
	and make the valueize function pointer also take a void pointer.
	* gcc/tree-ssa-sccvn.c (vn_valueize_wrapper): New function to wrap
	around vn_valueize, to call it without a context.
	(process_bb): Use vn_valueize_wrapper instead of vn_valueize.
	* tree-vect-loop.c (_loop_vec_info): Initialize epilogue_vinfos.
	(~_loop_vec_info): Release epilogue_vinfos.
	(vect_analyze_loop_costing): Use knowledge of main VF to estimate
	number of iterations of epilogue.
	(vect_analyze_loop_2): Adapt to analyse main loop for all supported
	vector sizes when vect-epilogues-nomask=1.  Also keep track of lowest
	versioning threshold needed for main loop.
	(vect_analyze_loop): Likewise.
	(find_in_mapping): New helper function.
	(update_epilogue_loop_vinfo): New function.
	(vect_transform_loop): When vectorizing epilogues re-use analysis done
	on main loop and call update_epilogue_loop_vinfo to update it.
	* tree-vect-loop-manip.c (vect_update_inits_of_drs): No longer insert
	stmts on loop preheader edge.
	(vect_do_peeling): Enable skip-vectors when doing loop versioning if
	we decided to vectorize epilogues.  Update epilogues NITERS and
	construct ADVANCE to update epilogues data references where needed.
	* tree-vectorizer.h (_loop_vec_info): Add epilogue_vinfos.
	(vect_do_peeling, vect_update_inits_of_drs,
	 determine_peel_for_niter, vect_analyze_loop): Add or update
	declarations.
	* tree-vectorizer.c (try_vectorize_loop_1): Make sure to use already
	created loop_vec_info's for epilogues when available.  Otherwise analyse
	epilogue separately.

From-SVN: r277569

gcc/ChangeLog

+2019-10-29  Andre Vieira  <andre.simoesdiasvieira@arm.com>
+
+	PR 88915
+	* tree-ssa-loop-niter.h (simplify_replace_tree): Change declaration.
+	* tree-ssa-loop-niter.c (simplify_replace_tree): Add context parameter
+	and make the valueize function pointer also take a void pointer.
+	* gcc/tree-ssa-sccvn.c (vn_valueize_wrapper): New function to wrap
+	around vn_valueize, to call it without a context.
+	(process_bb): Use vn_valueize_wrapper instead of vn_valueize.
+	* tree-vect-loop.c (_loop_vec_info): Initialize epilogue_vinfos.
+	(~_loop_vec_info): Release epilogue_vinfos.
+	(vect_analyze_loop_costing): Use knowledge of main VF to estimate
+	number of iterations of epilogue.
+	(vect_analyze_loop_2): Adapt to analyse main loop for all supported
+	vector sizes when vect-epilogues-nomask=1.  Also keep track of lowest
+	versioning threshold needed for main loop.
+	(vect_analyze_loop): Likewise.
+	(find_in_mapping): New helper function.
+	(update_epilogue_loop_vinfo): New function.
+	(vect_transform_loop): When vectorizing epilogues re-use analysis done
+	on main loop and call update_epilogue_loop_vinfo to update it.
+	* tree-vect-loop-manip.c (vect_update_inits_of_drs): No longer insert
+	stmts on loop preheader edge.
+	(vect_do_peeling): Enable skip-vectors when doing loop versioning if
+	we decided to vectorize epilogues.  Update epilogues NITERS and
+	construct ADVANCE to update epilogues data references where needed.
+	* tree-vectorizer.h (_loop_vec_info): Add epilogue_vinfos.
+	(vect_do_peeling, vect_update_inits_of_drs,
+	 determine_peel_for_niter, vect_analyze_loop): Add or update
+	declarations.
+	* tree-vectorizer.c (try_vectorize_loop_1): Make sure to use already
+	created loop_vec_info's for epilogues when available.  Otherwise analyse
+	epilogue separately.
+
 2019-10-29  Richard Biener  <rguenther@suse.de>
 
 	* doc/tree-ssa.texi (Immediate Uses): Fix FOR_EACH_IMM_USE_STMT

gcc/tree-ssa-loop-niter.c

@@ -1935,7 +1935,7 @@ number_of_iterations_cond (class loop *loop,
 
 tree
 simplify_replace_tree (tree expr, tree old, tree new_tree,
-		       tree (*valueize) (tree))
+		       tree (*valueize) (tree, void*), void *context)
 {
   unsigned i, n;
   tree ret = NULL_TREE, e, se;
@@ -1951,7 +1951,7 @@ simplify_replace_tree (tree expr, tree old, tree new_tree,
     {
       if (TREE_CODE (expr) == SSA_NAME)
 	{
-	  new_tree = valueize (expr);
+	  new_tree = valueize (expr, context);
 	  if (new_tree != expr)
 	    return new_tree;
 	}
@@ -1967,7 +1967,7 @@ simplify_replace_tree (tree expr, tree old, tree new_tree,
   for (i = 0; i < n; i++)
     {
       e = TREE_OPERAND (expr, i);
-      se = simplify_replace_tree (e, old, new_tree, valueize);
+      se = simplify_replace_tree (e, old, new_tree, valueize, context);
       if (e == se)
 	continue;
 

gcc/tree-ssa-loop-niter.h

@@ -53,7 +53,9 @@ extern bool scev_probably_wraps_p (tree, tree, tree, gimple *,
 				   class loop *, bool);
 extern void free_numbers_of_iterations_estimates (class loop *);
 extern void free_numbers_of_iterations_estimates (function *);
-extern tree simplify_replace_tree (tree, tree, tree, tree (*)(tree) = NULL);
+extern tree simplify_replace_tree (tree, tree,
+				   tree, tree (*)(tree, void *) = NULL,
+				   void * = NULL);
 extern void substitute_in_loop_info (class loop *, tree, tree);
 
 #endif /* GCC_TREE_SSA_LOOP_NITER_H */

gcc/tree-ssa-sccvn.c

@@ -309,6 +309,10 @@ static vn_tables_t valid_info;
 /* Valueization hook.  Valueize NAME if it is an SSA name, otherwise
    just return it.  */
 tree (*vn_valueize) (tree);
+tree vn_valueize_wrapper (tree t, void* context ATTRIBUTE_UNUSED)
+{
+  return vn_valueize (t);
+}
 
 
 /* This represents the top of the VN lattice, which is the universal
@@ -6412,7 +6416,7 @@ process_bb (rpo_elim &avail, basic_block bb,
       if (bb->loop_father->nb_iterations)
 	bb->loop_father->nb_iterations
 	  = simplify_replace_tree (bb->loop_father->nb_iterations,
-				   NULL_TREE, NULL_TREE, vn_valueize);
+				   NULL_TREE, NULL_TREE, &vn_valueize_wrapper);
     }
 
   /* Value-number all defs in the basic-block.  */

gcc/tree-vectorizer.c

@@ -874,6 +874,7 @@ try_vectorize_loop_1 (hash_table<simduid_to_vf> *&simduid_to_vf_htab,
   vec_info_shared shared;
   auto_purge_vect_location sentinel;
   vect_location = find_loop_location (loop);
+
   if (LOCATION_LOCUS (vect_location.get_location_t ()) != UNKNOWN_LOCATION
       && dump_enabled_p ())
     dump_printf (MSG_NOTE | MSG_PRIORITY_INTERNALS,
@@ -881,10 +882,17 @@ try_vectorize_loop_1 (hash_table<simduid_to_vf> *&simduid_to_vf_htab,
 		 LOCATION_FILE (vect_location.get_location_t ()),
 		 LOCATION_LINE (vect_location.get_location_t ()));
 
-  /* Try to analyze the loop, retaining an opt_problem if dump_enabled_p.  */
-  opt_loop_vec_info loop_vinfo
-    = vect_analyze_loop (loop, orig_loop_vinfo, &shared);
-  loop->aux = loop_vinfo;
+  opt_loop_vec_info loop_vinfo = opt_loop_vec_info::success (NULL);
+  /* In the case of epilogue vectorization the loop already has its
+     loop_vec_info set, we do not require to analyze the loop in this case.  */
+  if (loop_vec_info vinfo = loop_vec_info_for_loop (loop))
+    loop_vinfo = opt_loop_vec_info::success (vinfo);
+  else
+    {
+      /* Try to analyze the loop, retaining an opt_problem if dump_enabled_p.  */
+      loop_vinfo = vect_analyze_loop (loop, orig_loop_vinfo, &shared);
+      loop->aux = loop_vinfo;
+    }
 
   if (!loop_vinfo)
     if (dump_enabled_p ())
@@ -1012,8 +1020,13 @@ try_vectorize_loop_1 (hash_table<simduid_to_vf> *&simduid_to_vf_htab,
 
   /* Epilogue of vectorized loop must be vectorized too.  */
   if (new_loop)
-    ret |= try_vectorize_loop_1 (simduid_to_vf_htab, num_vectorized_loops,
-				 new_loop, loop_vinfo, NULL, NULL);
+    {
+      /* Don't include vectorized epilogues in the "vectorized loops" count.
+       */
+      unsigned dont_count = *num_vectorized_loops;
+      ret |= try_vectorize_loop_1 (simduid_to_vf_htab, &dont_count,
+				   new_loop, loop_vinfo, NULL, NULL);
+    }
 
   return ret;
 }

gcc/tree-vectorizer.h

@@ -26,6 +26,7 @@ typedef class _stmt_vec_info *stmt_vec_info;
 #include "tree-data-ref.h"
 #include "tree-hash-traits.h"
 #include "target.h"
+#include <utility>
 
 /* Used for naming of new temporaries.  */
 enum vect_var_kind {
@@ -456,6 +457,8 @@ struct rgroup_masks {
 
 typedef auto_vec<rgroup_masks> vec_loop_masks;
 
+typedef auto_vec<std::pair<data_reference*, tree> > drs_init_vec;
+
 /*-----------------------------------------------------------------*/
 /* Info on vectorized loops.                                       */
 /*-----------------------------------------------------------------*/
@@ -639,6 +642,10 @@ public:
      this points to the original vectorized loop.  Otherwise NULL.  */
   _loop_vec_info *orig_loop_info;
 
+  /* Used to store loop_vec_infos of epilogues of this loop during
+     analysis.  */
+  vec<_loop_vec_info *> epilogue_vinfos;
+
 } *loop_vec_info;
 
 /* Access Functions.  */
@@ -1589,10 +1596,12 @@ class loop *slpeel_tree_duplicate_loop_to_edge_cfg (class loop *,
 						     class loop *, edge);
 class loop *vect_loop_versioning (loop_vec_info);
 extern class loop *vect_do_peeling (loop_vec_info, tree, tree,
-				     tree *, tree *, tree *, int, bool, bool);
+				    tree *, tree *, tree *, int, bool, bool,
+				    tree *, drs_init_vec &);
 extern void vect_prepare_for_masked_peels (loop_vec_info);
 extern dump_user_location_t find_loop_location (class loop *);
 extern bool vect_can_advance_ivs_p (loop_vec_info);
+extern void vect_update_inits_of_drs (loop_vec_info, tree, tree_code);
 
 /* In tree-vect-stmts.c.  */
 extern tree get_vectype_for_scalar_type (vec_info *, tree);
@@ -1700,6 +1709,8 @@ extern tree vect_create_addr_base_for_vector_ref (stmt_vec_info, gimple_seq *,
 
 /* In tree-vect-loop.c.  */
 extern widest_int vect_iv_limit_for_full_masking (loop_vec_info loop_vinfo);
+/* Used in tree-vect-loop-manip.c */
+extern void determine_peel_for_niter (loop_vec_info);
 /* Used in gimple-loop-interchange.c and tree-parloops.c.  */
 extern bool check_reduction_path (dump_user_location_t, loop_p, gphi *, tree,
 				  enum tree_code);