[vect] Keep track of DR_OFFSET advance in dr_vec_info rather than data_reference

gcc/ChangeLog:
2020-01-10  Andre Vieira  <andre.simoesdiasvieira@arm.com>

	* tree-vect-data-refs.c (vect_create_addr_base_for_vector_ref): Use
	get_dr_vinfo_offset
	* tree-vect-loop.c (update_epilogue_loop_vinfo):  Remove orig_drs_init
	parameter and its use to reset DR_OFFSET's.
	(vect_transform_loop): Remove orig_drs_init argument.
	* tree-vect-loop-manip.c (vect_update_init_of_dr): Update the offset
	member of dr_vec_info rather than the offset of the associated
	data_reference's innermost_loop_behavior.
	(vect_update_init_of_dr): Pass dr_vec_info instead of data_reference.
	(vect_do_peeling): Remove orig_drs_init parameter and its construction.
	* tree-vect-stmts.c (check_scan_store): Replace use of DR_OFFSET with
	get_dr_vinfo_offset.
	(vectorizable_store): Likewise.
	(vectorizable_load): Likewise.

From-SVN: r280107

gcc/ChangeLog

+2020-01-10  Andre Vieira  <andre.simoesdiasvieira@arm.com>
+
+	* tree-vect-data-refs.c (vect_create_addr_base_for_vector_ref): Use
+	get_dr_vinfo_offset
+	* tree-vect-loop.c (update_epilogue_loop_vinfo):  Remove orig_drs_init
+	parameter and its use to reset DR_OFFSET's.
+	(vect_transform_loop): Remove orig_drs_init argument.
+	* tree-vect-loop-manip.c (vect_update_init_of_dr): Update the offset
+	member of dr_vec_info rather than the offset of the associated
+	data_reference's innermost_loop_behavior.
+	(vect_update_init_of_dr): Pass dr_vec_info instead of data_reference.
+	(vect_do_peeling): Remove orig_drs_init parameter and its construction.
+	* tree-vect-stmts.c (check_scan_store): Replace use of DR_OFFSET with
+	get_dr_vinfo_offset.
+	(vectorizable_store): Likewise.
+	(vectorizable_load): Likewise.
+
 2020-01-10  Richard Biener  <rguenther@suse.de>
 
 	* gimple-ssa-store-merging

gcc/tree-vect-data-refs.c

@@ -4597,7 +4597,7 @@ vect_create_addr_base_for_vector_ref (stmt_vec_info stmt_info,
   innermost_loop_behavior *drb = vect_dr_behavior (dr_info);
 
   tree data_ref_base = unshare_expr (drb->base_address);
-  tree base_offset = unshare_expr (drb->offset);
+  tree base_offset = unshare_expr (get_dr_vinfo_offset (dr_info, true));
   tree init = unshare_expr (drb->init);
 
   if (loop_vinfo)

gcc/tree-vect-loop-manip.c

@@ -1716,19 +1716,22 @@ vect_gen_prolog_loop_niters (loop_vec_info loop_vinfo,
    iterations before the scalar one (using masking to skip inactive
    elements).  This function updates the information recorded in DR to
    account for the difference.  Specifically, it updates the OFFSET
-   field of DR.  */
+   field of DR_INFO.  */
 
 static void
-vect_update_init_of_dr (struct data_reference *dr, tree niters, tree_code code)
+vect_update_init_of_dr (dr_vec_info *dr_info, tree niters, tree_code code)
 {
-  tree offset = DR_OFFSET (dr);
+  struct data_reference *dr = dr_info->dr;
+  tree offset = dr_info->offset;
+  if (!offset)
+    offset = build_zero_cst (sizetype);
 
   niters = fold_build2 (MULT_EXPR, sizetype,
 			fold_convert (sizetype, niters),
 			fold_convert (sizetype, DR_STEP (dr)));
   offset = fold_build2 (code, sizetype,
 			fold_convert (sizetype, offset), niters);
-  DR_OFFSET (dr) = offset;
+  dr_info->offset = offset;
 }
 
 
@@ -1758,7 +1761,7 @@ vect_update_inits_of_drs (loop_vec_info loop_vinfo, tree niters,
     {
       dr_vec_info *dr_info = loop_vinfo->lookup_dr (dr);
       if (!STMT_VINFO_GATHER_SCATTER_P (dr_info->stmt))
-	vect_update_init_of_dr (dr, niters, code);
+	vect_update_init_of_dr (dr_info, niters, code);
     }
 }
 
@@ -2446,7 +2449,7 @@ vect_do_peeling (loop_vec_info loop_vinfo, tree niters, tree nitersm1,
 		 tree *niters_vector, tree *step_vector,
 		 tree *niters_vector_mult_vf_var, int th,
 		 bool check_profitability, bool niters_no_overflow,
-		 tree *advance, drs_init_vec &orig_drs_init)
+		 tree *advance)
 {
   edge e, guard_e;
   tree type = TREE_TYPE (niters), guard_cond;
@@ -2694,14 +2697,6 @@ vect_do_peeling (loop_vec_info loop_vinfo, tree niters, tree nitersm1,
 	  scale_loop_profile (prolog, prob_prolog, bound_prolog);
 	}
 
-      /* Save original inits for each data_reference before advancing them with
-	 NITERS_PROLOG.  */
-      unsigned int i;
-      struct data_reference *dr;
-      vec<data_reference_p> datarefs = loop_vinfo->shared->datarefs;
-      FOR_EACH_VEC_ELT (datarefs, i, dr)
-	orig_drs_init.safe_push (std::make_pair (dr, DR_OFFSET (dr)));
-
       /* Update init address of DRs.  */
       vect_update_inits_of_drs (loop_vinfo, niters_prolog, PLUS_EXPR);
       /* Update niters for vector loop.  */

gcc/tree-vect-loop.c

@@ -8333,8 +8333,7 @@ find_in_mapping (tree t, void *context)
    prologue of the main loop.  */
 
 static void
-update_epilogue_loop_vinfo (class loop *epilogue, tree advance,
-			    drs_init_vec &orig_drs_init)
+update_epilogue_loop_vinfo (class loop *epilogue, tree advance)
 {
   loop_vec_info epilogue_vinfo = loop_vec_info_for_loop (epilogue);
   auto_vec<gimple *> stmt_worklist;
@@ -8344,16 +8343,10 @@ update_epilogue_loop_vinfo (class loop *epilogue, tree advance,
   gphi_iterator epilogue_phi_gsi;
   stmt_vec_info stmt_vinfo = NULL, related_vinfo;
   basic_block *epilogue_bbs = get_loop_body (epilogue);
+  unsigned i;
 
   LOOP_VINFO_BBS (epilogue_vinfo) = epilogue_bbs;
 
-  /* Restore original data_reference's offset, before the previous loop and its
-     prologue.  */
-  std::pair<data_reference*, tree> *dr_init;
-  unsigned i;
-  for (i = 0; orig_drs_init.iterate (i, &dr_init); i++)
-    DR_OFFSET (dr_init->first) = dr_init->second;
-
   /* Advance data_reference's with the number of iterations of the previous
      loop and its prologue.  */
   vect_update_inits_of_drs (epilogue_vinfo, advance, PLUS_EXPR);
@@ -8569,7 +8562,7 @@ vect_transform_loop (loop_vec_info loop_vinfo)
   epilogue = vect_do_peeling (loop_vinfo, niters, nitersm1, &niters_vector,
 			      &step_vector, &niters_vector_mult_vf, th,
 			      check_profitability, niters_no_overflow,
-			      &advance, orig_drs_init);
+			      &advance);
 
   if (LOOP_VINFO_SCALAR_LOOP (loop_vinfo)
       && LOOP_VINFO_SCALAR_LOOP_SCALING (loop_vinfo).initialized_p ())
@@ -8828,7 +8821,7 @@ vect_transform_loop (loop_vec_info loop_vinfo)
 
   if (epilogue)
     {
-      update_epilogue_loop_vinfo (epilogue, advance, orig_drs_init);
+      update_epilogue_loop_vinfo (epilogue, advance);
 
       epilogue->simduid = loop->simduid;
       epilogue->force_vectorize = loop->force_vectorize;

gcc/tree-vect-stmts.c

@@ -6629,7 +6629,7 @@ check_scan_store (stmt_vec_info stmt_info, tree vectype,
       || loop_vinfo == NULL
       || LOOP_VINFO_FULLY_MASKED_P (loop_vinfo)
       || STMT_VINFO_GROUPED_ACCESS (stmt_info)
-      || !integer_zerop (DR_OFFSET (dr_info->dr))
+      || !integer_zerop (get_dr_vinfo_offset (dr_info))
       || !integer_zerop (DR_INIT (dr_info->dr))
       || !(ref_type = reference_alias_ptr_type (DR_REF (dr_info->dr)))
       || !alias_sets_conflict_p (get_alias_set (vectype),
@@ -7762,6 +7762,7 @@ vectorizable_store (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
       tree running_off;
       tree stride_base, stride_step, alias_off;
       tree vec_oprnd;
+      tree dr_offset;
       unsigned int g;
       /* Checked by get_load_store_type.  */
       unsigned int const_nunits = nunits.to_constant ();
@@ -7769,11 +7770,12 @@ vectorizable_store (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
       gcc_assert (!LOOP_VINFO_FULLY_MASKED_P (loop_vinfo));
       gcc_assert (!nested_in_vect_loop_p (loop, stmt_info));
 
+      dr_offset = get_dr_vinfo_offset (first_dr_info);
       stride_base
 	= fold_build_pointer_plus
 	    (DR_BASE_ADDRESS (first_dr_info->dr),
 	     size_binop (PLUS_EXPR,
-			 convert_to_ptrofftype (DR_OFFSET (first_dr_info->dr)),
+			 convert_to_ptrofftype (dr_offset),
 			 convert_to_ptrofftype (DR_INIT (first_dr_info->dr))));
       stride_step = fold_convert (sizetype, DR_STEP (first_dr_info->dr));
 
@@ -8136,7 +8138,7 @@ vectorizable_store (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
 	      && !loop_masks
 	      && TREE_CODE (DR_BASE_ADDRESS (first_dr_info->dr)) == ADDR_EXPR
 	      && VAR_P (TREE_OPERAND (DR_BASE_ADDRESS (first_dr_info->dr), 0))
-	      && integer_zerop (DR_OFFSET (first_dr_info->dr))
+	      && integer_zerop (get_dr_vinfo_offset (first_dr_info))
 	      && integer_zerop (DR_INIT (first_dr_info->dr))
 	      && alias_sets_conflict_p (get_alias_set (aggr_type),
 					get_alias_set (TREE_TYPE (ref_type))))
@@ -8830,6 +8832,7 @@ vectorizable_load (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
       /* Checked by get_load_store_type.  */
       unsigned int const_nunits = nunits.to_constant ();
       unsigned HOST_WIDE_INT cst_offset = 0;
+      tree dr_offset;
 
       gcc_assert (!LOOP_VINFO_FULLY_MASKED_P (loop_vinfo));
       gcc_assert (!nested_in_vect_loop);
@@ -8860,11 +8863,12 @@ vectorizable_load (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
 	  ref_type = reference_alias_ptr_type (DR_REF (dr_info->dr));
 	}
 
+      dr_offset = get_dr_vinfo_offset (first_dr_info);
       stride_base
 	= fold_build_pointer_plus
 	    (DR_BASE_ADDRESS (first_dr_info->dr),
 	     size_binop (PLUS_EXPR,
-			 convert_to_ptrofftype (DR_OFFSET (first_dr_info->dr)),
+			 convert_to_ptrofftype (dr_offset),
 			 convert_to_ptrofftype (DR_INIT (first_dr_info->dr))));
       stride_step = fold_convert (sizetype, DR_STEP (first_dr_info->dr));
 
@@ -9329,7 +9333,7 @@ vectorizable_load (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
 	  if (simd_lane_access_p
 	      && TREE_CODE (DR_BASE_ADDRESS (first_dr_info->dr)) == ADDR_EXPR
 	      && VAR_P (TREE_OPERAND (DR_BASE_ADDRESS (first_dr_info->dr), 0))
-	      && integer_zerop (DR_OFFSET (first_dr_info->dr))
+	      && integer_zerop (get_dr_vinfo_offset (first_dr_info))
 	      && integer_zerop (DR_INIT (first_dr_info->dr))
 	      && alias_sets_conflict_p (get_alias_set (aggr_type),
 					get_alias_set (TREE_TYPE (ref_type)))

gcc/tree-vectorizer.h

@@ -910,6 +910,10 @@ public:
   /* If true the alignment of base_decl needs to be increased.  */
   bool base_misaligned;
   tree base_decl;
+
+  /* Stores current vectorized loop's offset.  To be added to the DR's
+     offset to calculate current offset of data reference.  */
+  tree offset;
 };
 
 typedef struct data_reference *dr_p;
@@ -1485,6 +1489,26 @@ vect_dr_behavior (dr_vec_info *dr_info)
     return &STMT_VINFO_DR_WRT_VEC_LOOP (stmt_info);
 }
 
+inline tree
+get_dr_vinfo_offset (dr_vec_info *dr_info, bool check_outer = false)
+{
+  innermost_loop_behavior *base;
+  if (check_outer)
+    base = vect_dr_behavior (dr_info);
+  else
+    base = &dr_info->dr->innermost;
+
+  tree offset = base->offset;
+
+  if (!dr_info->offset)
+    return offset;
+
+  offset = fold_convert (sizetype, offset);
+  return fold_build2 (PLUS_EXPR, TREE_TYPE (dr_info->offset), offset,
+		      dr_info->offset);
+}
+
+
 /* Return true if the vect cost model is unlimited.  */
 static inline bool
 unlimited_cost_model (loop_p loop)
@@ -1655,7 +1679,7 @@ class loop *slpeel_tree_duplicate_loop_to_edge_cfg (class loop *,
 class loop *vect_loop_versioning (loop_vec_info);
 extern class loop *vect_do_peeling (loop_vec_info, tree, tree,
 				    tree *, tree *, tree *, int, bool, bool,
-				    tree *, drs_init_vec &);
+				    tree *);
 extern void vect_prepare_for_masked_peels (loop_vec_info);
 extern dump_user_location_t find_loop_location (class loop *);
 extern bool vect_can_advance_ivs_p (loop_vec_info);