tree-vectorizer.h (struct vec_info_shared): New structure with parts split out…

tree-vectorizer.h (struct vec_info_shared): New structure with parts split out from struct vec_info and loop_nest from...

2018-06-25  Richard Biener  <rguenther@suse.de>

	* tree-vectorizer.h (struct vec_info_shared): New structure
	with parts split out from struct vec_info and loop_nest from
	struct _loop_vec_info.
	(struct vec_info): Adjust accordingly.
	(struct _loop_vec_info): Likewise.
	(LOOP_VINFO_LOOP_NEST): Adjust.
	(LOOP_VINFO_DATAREFS): Likewise.
	(LOOP_VINFO_DDRS): Likewise.
	(struct _bb_vec_info): Likewise.
	(BB_VINFO_DATAREFS): Likewise.
	(BB_VINFO_DDRS): Likewise.
	(struct _stmt_vec_info): Add dr_aux member.
	(DR_VECT_AUX): Adjust to refer to member of DR_STMTs vinfo.
	(DR_MISALIGNMENT_UNINITIALIZED): New.
	(set_dr_misalignment): Adjust.
	(dr_misalignment): Assert misalign isn't DR_MISALIGNMENT_UNINITIALIZED.
	(vect_analyze_loop): Adjust prototype.
	(vect_analyze_loop_form): Likewise.
	* tree-vect-data-refs.c (vect_analyze_data_ref_dependences):
	Compute dependences lazily.
	(vect_record_base_alignments): Use shared datarefs/ddrs.
	(vect_verify_datarefs_alignment): Likewise.
	(vect_analyze_data_refs_alignment): Likewise.
	(vect_analyze_data_ref_accesses): Likewise.
	(vect_analyze_data_refs): Likewise.
	* tree-vect-loop.c (_loop_vec_info::_loop_vec_info): Add
	constructor parameter for shared part.
	(vect_analyze_loop_form): Pass in shared part and adjust.
	(vect_analyze_loop_2): Pass in storage for the number of
	stmts.  Move loop nest finding to the caller.  Compute
	datarefs lazily.
	(vect_analyze_loop): Pass in shared part.
	(vect_transform_loop): Verify shared datarefs are unchanged.
	* tree-vect-slp.c (_bb_vec_info::_bb_vec_info): Add
	constructor parameter for shared part.
	(vect_slp_analyze_bb_1): Pass in shared part and adjust.
	(vect_slp_bb): Verify shared datarefs are unchanged before
	transform.
	* tree-vect-stmts.c (ensure_base_align): Adjust for DR_AUX
	change.
	(new_stmt_vec_info): Initialize DR_AUX misalignment to
	DR_MISALIGNMENT_UNINITIALIZED.
	* tree-vectorizer.c (vec_info::vec_info): Add constructor
	parameter for shared part.
	(vec_info::~vec_info): Adjust.
	(vec_info_shared::vec_info_shared): New.
	(vec_info_shared::~vec_info_shared): Likewise.
	(vec_info_shared::save_datarefs): Likewise.
	(vec_info_shared::check_datarefs): Likewise.
	(try_vectorize_loop_1): Construct shared part live for analyses
	of a single loop for multiple vector sizes.
	* tree-parloops.c (gather_scalar_reductions): Adjust.

From-SVN: r262009

gcc/ChangeLog

 2018-06-25  Richard Biener  <rguenther@suse.de>
 
+	* tree-vectorizer.h (struct vec_info_shared): New structure
+	with parts split out from struct vec_info and loop_nest from
+	struct _loop_vec_info.
+	(struct vec_info): Adjust accordingly.
+	(struct _loop_vec_info): Likewise.
+	(LOOP_VINFO_LOOP_NEST): Adjust.
+	(LOOP_VINFO_DATAREFS): Likewise.
+	(LOOP_VINFO_DDRS): Likewise.
+	(struct _bb_vec_info): Likewise.
+	(BB_VINFO_DATAREFS): Likewise.
+	(BB_VINFO_DDRS): Likewise.
+	(struct _stmt_vec_info): Add dr_aux member.
+	(DR_VECT_AUX): Adjust to refer to member of DR_STMTs vinfo.
+	(DR_MISALIGNMENT_UNINITIALIZED): New.
+	(set_dr_misalignment): Adjust.
+	(dr_misalignment): Assert misalign isn't DR_MISALIGNMENT_UNINITIALIZED.
+	(vect_analyze_loop): Adjust prototype.
+	(vect_analyze_loop_form): Likewise.
+	* tree-vect-data-refs.c (vect_analyze_data_ref_dependences):
+	Compute dependences lazily.
+	(vect_record_base_alignments): Use shared datarefs/ddrs.
+	(vect_verify_datarefs_alignment): Likewise.
+	(vect_analyze_data_refs_alignment): Likewise.
+	(vect_analyze_data_ref_accesses): Likewise.
+	(vect_analyze_data_refs): Likewise.
+	* tree-vect-loop.c (_loop_vec_info::_loop_vec_info): Add
+	constructor parameter for shared part.
+	(vect_analyze_loop_form): Pass in shared part and adjust.
+	(vect_analyze_loop_2): Pass in storage for the number of
+	stmts.  Move loop nest finding to the caller.  Compute
+	datarefs lazily.
+	(vect_analyze_loop): Pass in shared part.
+	(vect_transform_loop): Verify shared datarefs are unchanged.
+	* tree-vect-slp.c (_bb_vec_info::_bb_vec_info): Add
+	constructor parameter for shared part.
+	(vect_slp_analyze_bb_1): Pass in shared part and adjust.
+	(vect_slp_bb): Verify shared datarefs are unchanged before
+	transform.
+	* tree-vect-stmts.c (ensure_base_align): Adjust for DR_AUX
+	change.
+	(new_stmt_vec_info): Initialize DR_AUX misalignment to
+	DR_MISALIGNMENT_UNINITIALIZED.
+	* tree-vectorizer.c (vec_info::vec_info): Add constructor
+	parameter for shared part.
+	(vec_info::~vec_info): Adjust.
+	(vec_info_shared::vec_info_shared): New.
+	(vec_info_shared::~vec_info_shared): Likewise.
+	(vec_info_shared::save_datarefs): Likewise.
+	(vec_info_shared::check_datarefs): Likewise.
+	(try_vectorize_loop_1): Construct shared part live for analyses
+	of a single loop for multiple vector sizes.
+	* tree-parloops.c (gather_scalar_reductions): Adjust.
+
+2018-06-25  Richard Biener  <rguenther@suse.de>
+
 	* tree-vect-data-refs.c (vect_find_stmt_data_reference): Modify
 	DR for SIMD lane accesses here and mark DR with (void *)-1 aux.
 	(vect_analyze_data_refs): Remove similar code from here and

gcc/tree-parloops.c

@@ -2597,7 +2597,8 @@ gather_scalar_reductions (loop_p loop, reduction_info_table_type *reduction_list
   stmt_vec_infos.create (50);
   set_stmt_vec_info_vec (&stmt_vec_infos);
 
-  simple_loop_info = vect_analyze_loop_form (loop);
+  vec_info_shared shared;
+  simple_loop_info = vect_analyze_loop_form (loop, &shared);
   if (simple_loop_info == NULL)
     goto gather_done;
 
@@ -2636,7 +2637,8 @@ gather_scalar_reductions (loop_p loop, reduction_info_table_type *reduction_list
 
   if (!double_reduc_phis.is_empty ())
     {
-      simple_loop_info = vect_analyze_loop_form (loop->inner);
+      vec_info_shared shared;
+      simple_loop_info = vect_analyze_loop_form (loop->inner, &shared);
       if (simple_loop_info)
 	{
 	  gphi *phi;

gcc/tree-vect-data-refs.c

@@ -568,15 +568,20 @@ vect_analyze_data_ref_dependences (loop_vec_info loop_vinfo,
 
   DUMP_VECT_SCOPE ("vect_analyze_data_ref_dependences");
 
-  LOOP_VINFO_DDRS (loop_vinfo)
-    .create (LOOP_VINFO_DATAREFS (loop_vinfo).length ()
-	     * LOOP_VINFO_DATAREFS (loop_vinfo).length ());
+  if (!LOOP_VINFO_DDRS (loop_vinfo).exists ())
+    {
+      LOOP_VINFO_DDRS (loop_vinfo)
+	.create (LOOP_VINFO_DATAREFS (loop_vinfo).length ()
+		 * LOOP_VINFO_DATAREFS (loop_vinfo).length ());
+      /* We need read-read dependences to compute
+	 STMT_VINFO_SAME_ALIGN_REFS.  */
+      if (!compute_all_dependences (LOOP_VINFO_DATAREFS (loop_vinfo),
+				    &LOOP_VINFO_DDRS (loop_vinfo),
+				    LOOP_VINFO_LOOP_NEST (loop_vinfo), true))
+	return false;
+    }
+
   LOOP_VINFO_NO_DATA_DEPENDENCIES (loop_vinfo) = true;
-  /* We need read-read dependences to compute STMT_VINFO_SAME_ALIGN_REFS.  */
-  if (!compute_all_dependences (LOOP_VINFO_DATAREFS (loop_vinfo),
-				&LOOP_VINFO_DDRS (loop_vinfo),
-				LOOP_VINFO_LOOP_NEST (loop_vinfo), true))
-    return false;
 
   /* For epilogues we either have no aliases or alias versioning
      was applied to original loop.  Therefore we may just get max_vf
@@ -833,7 +838,7 @@ vect_record_base_alignments (vec_info *vinfo)
   struct loop *loop = loop_vinfo ? LOOP_VINFO_LOOP (loop_vinfo) : NULL;
   data_reference *dr;
   unsigned int i;
-  FOR_EACH_VEC_ELT (vinfo->datarefs, i, dr)
+  FOR_EACH_VEC_ELT (vinfo->shared->datarefs, i, dr)
     {
       gimple *stmt = vect_dr_stmt (dr);
       stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
@@ -1155,7 +1160,7 @@ verify_data_ref_alignment (data_reference_p dr)
 bool
 vect_verify_datarefs_alignment (loop_vec_info vinfo)
 {
-  vec<data_reference_p> datarefs = vinfo->datarefs;
+  vec<data_reference_p> datarefs = vinfo->shared->datarefs;
   struct data_reference *dr;
   unsigned int i;
 
@@ -2353,14 +2358,14 @@ vect_analyze_data_refs_alignment (loop_vec_info vinfo)
 
   /* Mark groups of data references with same alignment using
      data dependence information.  */
-  vec<ddr_p> ddrs = vinfo->ddrs;
+  vec<ddr_p> ddrs = vinfo->shared->ddrs;
   struct data_dependence_relation *ddr;
   unsigned int i;
 
   FOR_EACH_VEC_ELT (ddrs, i, ddr)
     vect_find_same_alignment_drs (ddr);
 
-  vec<data_reference_p> datarefs = vinfo->datarefs;
+  vec<data_reference_p> datarefs = vinfo->shared->datarefs;
   struct data_reference *dr;
 
   vect_record_base_alignments (vinfo);
@@ -2914,7 +2919,7 @@ bool
 vect_analyze_data_ref_accesses (vec_info *vinfo)
 {
   unsigned int i;
-  vec<data_reference_p> datarefs = vinfo->datarefs;
+  vec<data_reference_p> datarefs = vinfo->shared->datarefs;
   struct data_reference *dr;
 
   DUMP_VECT_SCOPE ("vect_analyze_data_ref_accesses");
@@ -4131,7 +4136,7 @@ vect_analyze_data_refs (vec_info *vinfo, poly_uint64 *min_vf)
   /* Go through the data-refs, check that the analysis succeeded.  Update
      pointer from stmt_vec_info struct to DR and vectype.  */
 
-  vec<data_reference_p> datarefs = vinfo->datarefs;
+  vec<data_reference_p> datarefs = vinfo->shared->datarefs;
   FOR_EACH_VEC_ELT (datarefs, i, dr)
     {
       gimple *stmt;

gcc/tree-vect-loop.c

@@ -803,8 +803,8 @@ bb_in_loop_p (const_basic_block bb, const void *data)
 /* Create and initialize a new loop_vec_info struct for LOOP_IN, as well as
    stmt_vec_info structs for all the stmts in LOOP_IN.  */
 
-_loop_vec_info::_loop_vec_info (struct loop *loop_in)
-  : vec_info (vec_info::loop, init_cost (loop_in)),
+_loop_vec_info::_loop_vec_info (struct loop *loop_in, vec_info_shared *shared)
+  : vec_info (vec_info::loop, init_cost (loop_in), shared),
     loop (loop_in),
     bbs (XCNEWVEC (basic_block, loop->num_nodes)),
     num_itersm1 (NULL_TREE),
@@ -1351,7 +1351,7 @@ vect_analyze_loop_form_1 (struct loop *loop, gcond **loop_cond,
 /* Analyze LOOP form and return a loop_vec_info if it is of suitable form.  */
 
 loop_vec_info
-vect_analyze_loop_form (struct loop *loop)
+vect_analyze_loop_form (struct loop *loop, vec_info_shared *shared)
 {
   tree assumptions, number_of_iterations, number_of_iterationsm1;
   gcond *loop_cond, *inner_loop_cond = NULL;
@@ -1361,7 +1361,7 @@ vect_analyze_loop_form (struct loop *loop)
 				  &number_of_iterations, &inner_loop_cond))
     return NULL;
 
-  loop_vec_info loop_vinfo = new _loop_vec_info (loop);
+  loop_vec_info loop_vinfo = new _loop_vec_info (loop, shared);
   LOOP_VINFO_NITERSM1 (loop_vinfo) = number_of_iterationsm1;
   LOOP_VINFO_NITERS (loop_vinfo) = number_of_iterations;
   LOOP_VINFO_NITERS_UNCHANGED (loop_vinfo) = number_of_iterations;
@@ -1822,7 +1822,7 @@ vect_get_datarefs_in_loop (loop_p loop, basic_block *bbs,
    for it.  The different analyses will record information in the
    loop_vec_info struct.  */
 static bool
-vect_analyze_loop_2 (loop_vec_info loop_vinfo, bool &fatal)
+vect_analyze_loop_2 (loop_vec_info loop_vinfo, bool &fatal, unsigned *n_stmts)
 {
   bool ok;
   int res;
@@ -1836,29 +1836,25 @@ vect_analyze_loop_2 (loop_vec_info loop_vinfo, bool &fatal)
      and analyze their evolution in the loop.  */
 
   loop_p loop = LOOP_VINFO_LOOP (loop_vinfo);
-  if (!find_loop_nest (loop, &LOOP_VINFO_LOOP_NEST (loop_vinfo)))
-    {
-      if (dump_enabled_p ())
-	dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
-			 "not vectorized: loop nest containing two "
-			 "or more consecutive inner loops cannot be "
-			 "vectorized\n");
-      return false;
-    }
 
   /* Gather the data references and count stmts in the loop.  */
-  unsigned int n_stmts;
-  if (!vect_get_datarefs_in_loop (loop, LOOP_VINFO_BBS (loop_vinfo),
-				  &LOOP_VINFO_DATAREFS (loop_vinfo),
-				  &n_stmts))
+  if (!LOOP_VINFO_DATAREFS (loop_vinfo).exists ())
     {
-      if (dump_enabled_p ())
-	dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
-			 "not vectorized: loop contains function "
-			 "calls or data references that cannot "
-			 "be analyzed\n");
-      return false;
+      if (!vect_get_datarefs_in_loop (loop, LOOP_VINFO_BBS (loop_vinfo),
+				      &LOOP_VINFO_DATAREFS (loop_vinfo),
+				      n_stmts))
+	{
+	  if (dump_enabled_p ())
+	    dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
+			     "not vectorized: loop contains function "
+			     "calls or data references that cannot "
+			     "be analyzed\n");
+	  return false;
+	}
+      loop_vinfo->shared->save_datarefs ();
     }
+  else
+    loop_vinfo->shared->check_datarefs ();
 
   /* Analyze the data references and also adjust the minimal
      vectorization factor according to the loads and stores.  */
@@ -1947,7 +1943,7 @@ vect_analyze_loop_2 (loop_vec_info loop_vinfo, bool &fatal)
   unsigned th;
 
   /* Check the SLP opportunities in the loop, analyze and build SLP trees.  */
-  ok = vect_analyze_slp (loop_vinfo, n_stmts);
+  ok = vect_analyze_slp (loop_vinfo, *n_stmts);
   if (!ok)
     return false;
 
@@ -2286,7 +2282,8 @@ again:
    loop_vec_info struct.  If ORIG_LOOP_VINFO is not NULL epilogue must
    be vectorized.  */
 loop_vec_info
-vect_analyze_loop (struct loop *loop, loop_vec_info orig_loop_vinfo)
+vect_analyze_loop (struct loop *loop, loop_vec_info orig_loop_vinfo,
+		   vec_info_shared *shared)
 {
   loop_vec_info loop_vinfo;
   auto_vector_sizes vector_sizes;
@@ -2308,11 +2305,22 @@ vect_analyze_loop (struct loop *loop, loop_vec_info orig_loop_vinfo)
       return NULL;
     }
 
+  if (!find_loop_nest (loop, &shared->loop_nest))
+    {
+      if (dump_enabled_p ())
+	dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
+			 "not vectorized: loop nest containing two "
+			 "or more consecutive inner loops cannot be "
+			 "vectorized\n");
+      return NULL;
+    }
+
+  unsigned n_stmts;
   poly_uint64 autodetected_vector_size = 0;
   while (1)
     {
       /* Check the CFG characteristics of the loop (nesting, entry/exit).  */
-      loop_vinfo = vect_analyze_loop_form (loop);
+      loop_vinfo = vect_analyze_loop_form (loop, shared);
       if (!loop_vinfo)
 	{
 	  if (dump_enabled_p ())
@@ -2326,7 +2334,7 @@ vect_analyze_loop (struct loop *loop, loop_vec_info orig_loop_vinfo)
       if (orig_loop_vinfo)
 	LOOP_VINFO_ORIG_LOOP_INFO (loop_vinfo) = orig_loop_vinfo;
 
-      if (vect_analyze_loop_2 (loop_vinfo, fatal))
+      if (vect_analyze_loop_2 (loop_vinfo, fatal, &n_stmts))
 	{
 	  LOOP_VINFO_VECTORIZABLE_P (loop_vinfo) = 1;
 
@@ -8385,6 +8393,8 @@ vect_transform_loop (loop_vec_info loop_vinfo)
 
   DUMP_VECT_SCOPE ("vec_transform_loop");
 
+  loop_vinfo->shared->check_datarefs ();
+
   /* Use the more conservative vectorization threshold.  If the number
      of iterations is constant assume the cost check has been performed
      by our caller.  If the threshold makes all loops profitable that

gcc/tree-vect-slp.c

@@ -2440,8 +2440,9 @@ vect_detect_hybrid_slp (loop_vec_info loop_vinfo)
    REGION_BEGIN_IN (inclusive) and REGION_END_IN (exclusive).  */
 
 _bb_vec_info::_bb_vec_info (gimple_stmt_iterator region_begin_in,
-			    gimple_stmt_iterator region_end_in)
-  : vec_info (vec_info::bb, init_cost (NULL)),
+			    gimple_stmt_iterator region_end_in,
+			    vec_info_shared *shared)
+  : vec_info (vec_info::bb, init_cost (NULL), shared),
     bb (gsi_bb (region_begin_in)),
     region_begin (region_begin_in),
     region_end (region_end_in)
@@ -2803,7 +2804,7 @@ static bb_vec_info
 vect_slp_analyze_bb_1 (gimple_stmt_iterator region_begin,
 		       gimple_stmt_iterator region_end,
 		       vec<data_reference_p> datarefs, int n_stmts,
-		       bool &fatal)
+		       bool &fatal, vec_info_shared *shared)
 {
   bb_vec_info bb_vinfo;
   slp_instance instance;
@@ -2823,11 +2824,12 @@ vect_slp_analyze_bb_1 (gimple_stmt_iterator region_begin,
       return NULL;
     }
 
-  bb_vinfo = new _bb_vec_info (region_begin, region_end);
+  bb_vinfo = new _bb_vec_info (region_begin, region_end, shared);
   if (!bb_vinfo)
     return NULL;
 
   BB_VINFO_DATAREFS (bb_vinfo) = datarefs;
+  bb_vinfo->shared->save_datarefs ();
 
   /* Analyze the data references.  */
 
@@ -3018,14 +3020,16 @@ vect_slp_bb (basic_block bb)
 
       bool vectorized = false;
       bool fatal = false;
+      vec_info_shared shared;
       bb_vinfo = vect_slp_analyze_bb_1 (region_begin, region_end,
-					datarefs, insns, fatal);
+					datarefs, insns, fatal, &shared);
       if (bb_vinfo
 	  && dbg_cnt (vect_slp))
 	{
 	  if (dump_enabled_p ())
 	    dump_printf_loc (MSG_NOTE, vect_location, "SLPing BB part\n");
 
+	  bb_vinfo->shared->check_datarefs ();
 	  vect_schedule_slp (bb_vinfo);
 
 	  unsigned HOST_WIDE_INT bytes;

gcc/tree-vect-stmts.c

@@ -6123,7 +6123,7 @@ vectorizable_operation (gimple *stmt, gimple_stmt_iterator *gsi,
 static void
 ensure_base_align (struct data_reference *dr)
 {
-  if (!dr->aux)
+  if (DR_VECT_AUX (dr)->misalignment == DR_MISALIGNMENT_UNINITIALIZED)
     return;
 
   if (DR_VECT_AUX (dr)->base_misaligned)
@@ -9831,6 +9831,9 @@ new_stmt_vec_info (gimple *stmt, vec_info *vinfo)
   res->gap = 0; /* GROUP_GAP */
   res->same_dr_stmt = NULL; /* GROUP_SAME_DR_STMT */
 
+  /* This is really "uninitialized" until vect_compute_data_ref_alignment.  */
+  res->dr_aux.misalignment = DR_MISALIGNMENT_UNINITIALIZED;
+
   return res;
 }
 

gcc/tree-vectorizer.c

@@ -450,10 +450,10 @@ shrink_simd_arrays
 /* Initialize the vec_info with kind KIND_IN and target cost data
    TARGET_COST_DATA_IN.  */
 
-vec_info::vec_info (vec_info::vec_kind kind_in, void *target_cost_data_in)
+vec_info::vec_info (vec_info::vec_kind kind_in, void *target_cost_data_in,
+		    vec_info_shared *shared_)
   : kind (kind_in),
-    datarefs (vNULL),
-    ddrs (vNULL),
+    shared (shared_),
     target_cost_data (target_cost_data_in)
 {
   stmt_vec_infos.create (50);
@@ -463,25 +463,50 @@ vec_info::vec_info (vec_info::vec_kind kind_in, void *target_cost_data_in)
 vec_info::~vec_info ()
 {
   slp_instance instance;
-  struct data_reference *dr;
   unsigned int i;
 
-  FOR_EACH_VEC_ELT (datarefs, i, dr)
-    if (dr->aux)
-      {
-        free (dr->aux);
-        dr->aux = NULL;
-      }
-
   FOR_EACH_VEC_ELT (slp_instances, i, instance)
     vect_free_slp_instance (instance);
 
-  free_data_refs (datarefs);
-  free_dependence_relations (ddrs);
   destroy_cost_data (target_cost_data);
   free_stmt_vec_infos (&stmt_vec_infos);
 }
 
+vec_info_shared::vec_info_shared ()
+  : datarefs (vNULL),
+    datarefs_copy (vNULL),
+    ddrs (vNULL)
+{
+}
+
+vec_info_shared::~vec_info_shared ()
+{
+  free_data_refs (datarefs);
+  free_dependence_relations (ddrs);
+  datarefs_copy.release ();
+}
+
+void
+vec_info_shared::save_datarefs ()
+{
+  if (!flag_checking)
+    return;
+  datarefs_copy.reserve_exact (datarefs.length ());
+  for (unsigned i = 0; i < datarefs.length (); ++i)
+    datarefs_copy.quick_push (*datarefs[i]);
+}
+
+void
+vec_info_shared::check_datarefs ()
+{
+  if (!flag_checking)
+    return;
+  gcc_assert (datarefs.length () == datarefs_copy.length ());
+  for (unsigned i = 0; i < datarefs.length (); ++i)
+    if (memcmp (&datarefs_copy[i], datarefs[i], sizeof (data_reference)) != 0)
+      gcc_unreachable ();
+}
+
 /* A helper function to free scev and LOOP niter information, as well as
    clear loop constraint LOOP_C_FINITE.  */
 
@@ -669,6 +694,7 @@ try_vectorize_loop_1 (hash_table<simduid_to_vf> *&simduid_to_vf_htab,
 		      gimple *loop_dist_alias_call)
 {
   unsigned ret = 0;
+  vec_info_shared shared;
   vect_location = find_loop_location (loop);
   if (LOCATION_LOCUS (vect_location) != UNKNOWN_LOCATION
       && dump_enabled_p ())
@@ -676,7 +702,7 @@ try_vectorize_loop_1 (hash_table<simduid_to_vf> *&simduid_to_vf_htab,
 		 LOCATION_FILE (vect_location),
 		 LOCATION_LINE (vect_location));
 
-  loop_vec_info loop_vinfo = vect_analyze_loop (loop, orig_loop_vinfo);
+  loop_vec_info loop_vinfo = vect_analyze_loop (loop, orig_loop_vinfo, &shared);
   loop->aux = loop_vinfo;
 
   if (!loop_vinfo || !LOOP_VINFO_VECTORIZABLE_P (loop_vinfo))

gcc/tree-vectorizer.h

@@ -187,33 +187,50 @@ struct vec_lower_bound {
   poly_uint64 min_value;
 };
 
+/* Vectorizer state shared between different analyses like vector sizes
+   of the same CFG region.  */
+struct vec_info_shared {
+  vec_info_shared();
+  ~vec_info_shared();
+
+  void save_datarefs();
+  void check_datarefs();
+
+  /* All data references.  Freed by free_data_refs, so not an auto_vec.  */
+  vec<data_reference_p> datarefs;
+  vec<data_reference> datarefs_copy;
+
+  /* The loop nest in which the data dependences are computed.  */
+  auto_vec<loop_p> loop_nest;
+
+  /* All data dependences.  Freed by free_dependence_relations, so not
+     an auto_vec.  */
+  vec<ddr_p> ddrs;
+};
+
 /* Vectorizer state common between loop and basic-block vectorization.  */
 struct vec_info {
   enum vec_kind { bb, loop };
 
-  vec_info (vec_kind, void *);
+  vec_info (vec_kind, void *, vec_info_shared *);
   ~vec_info ();
 
   /* The type of vectorization.  */
   vec_kind kind;
 
+  /* Shared vectorizer state.  */
+  vec_info_shared *shared;
+
   /* The mapping of GIMPLE UID to stmt_vec_info.  */
   vec<struct _stmt_vec_info *> stmt_vec_infos;
 
   /* All SLP instances.  */
   auto_vec<slp_instance> slp_instances;
 
-  /* All data references.  Freed by free_data_refs, so not an auto_vec.  */
-  vec<data_reference_p> datarefs;
-
   /* Maps base addresses to an innermost_loop_behavior that gives the maximum
      known alignment for that base.  */
   vec_base_alignments base_alignments;
 
-  /* All data dependences.  Freed by free_dependence_relations, so not
-     an auto_vec.  */
-  vec<ddr_p> ddrs;
-
   /* All interleaving chains of stores, represented by the first
      stmt in the chain.  */
   auto_vec<gimple *> grouped_stores;
@@ -342,7 +359,7 @@ typedef auto_vec<rgroup_masks> vec_loop_masks;
 /* Info on vectorized loops.                                       */
 /*-----------------------------------------------------------------*/
 typedef struct _loop_vec_info : public vec_info {
-  _loop_vec_info (struct loop *);
+  _loop_vec_info (struct loop *, vec_info_shared *);
   ~_loop_vec_info ();
 
   /* The loop to which this info struct refers to.  */
@@ -408,9 +425,6 @@ typedef struct _loop_vec_info : public vec_info {
   /* The mask used to check the alignment of pointers or arrays.  */
   int ptr_mask;
 
-  /* The loop nest in which the data dependences are computed.  */
-  auto_vec<loop_p> loop_nest;
-
   /* Data Dependence Relations defining address ranges that are candidates
      for a run-time aliasing check.  */
   auto_vec<ddr_p> may_alias_ddrs;
@@ -531,9 +545,9 @@ typedef struct _loop_vec_info : public vec_info {
 #define LOOP_VINFO_MASK_SKIP_NITERS(L)     (L)->mask_skip_niters
 #define LOOP_VINFO_MASK_COMPARE_TYPE(L)    (L)->mask_compare_type
 #define LOOP_VINFO_PTR_MASK(L)             (L)->ptr_mask
-#define LOOP_VINFO_LOOP_NEST(L)            (L)->loop_nest
-#define LOOP_VINFO_DATAREFS(L)             (L)->datarefs
-#define LOOP_VINFO_DDRS(L)                 (L)->ddrs
+#define LOOP_VINFO_LOOP_NEST(L)            (L)->shared->loop_nest
+#define LOOP_VINFO_DATAREFS(L)             (L)->shared->datarefs
+#define LOOP_VINFO_DDRS(L)                 (L)->shared->ddrs
 #define LOOP_VINFO_INT_NITERS(L)           (TREE_INT_CST_LOW ((L)->num_iters))
 #define LOOP_VINFO_PEELING_FOR_ALIGNMENT(L) (L)->peeling_for_alignment
 #define LOOP_VINFO_UNALIGNED_DR(L)         (L)->unaligned_dr
@@ -596,7 +610,7 @@ nested_in_vect_loop_p (struct loop *loop, gimple *stmt)
 
 typedef struct _bb_vec_info : public vec_info
 {
-  _bb_vec_info (gimple_stmt_iterator, gimple_stmt_iterator);
+  _bb_vec_info (gimple_stmt_iterator, gimple_stmt_iterator, vec_info_shared *);
   ~_bb_vec_info ();
 
   basic_block bb;
@@ -607,8 +621,8 @@ typedef struct _bb_vec_info : public vec_info
 #define BB_VINFO_BB(B)               (B)->bb
 #define BB_VINFO_GROUPED_STORES(B)   (B)->grouped_stores
 #define BB_VINFO_SLP_INSTANCES(B)    (B)->slp_instances
-#define BB_VINFO_DATAREFS(B)         (B)->datarefs
-#define BB_VINFO_DDRS(B)             (B)->ddrs
+#define BB_VINFO_DATAREFS(B)         (B)->shared->datarefs
+#define BB_VINFO_DDRS(B)             (B)->shared->ddrs
 #define BB_VINFO_TARGET_COST_DATA(B) (B)->target_cost_data
 
 static inline bb_vec_info
@@ -734,6 +748,17 @@ enum vect_memory_access_type {
   VMAT_GATHER_SCATTER
 };
 
+struct dataref_aux {
+  /* The misalignment in bytes of the reference, or -1 if not known.  */
+  int misalignment;
+  /* The byte alignment that we'd ideally like the reference to have,
+     and the value that misalignment is measured against.  */
+  int target_alignment;
+  /* If true the alignment of base_decl needs to be increased.  */
+  bool base_misaligned;
+  tree base_decl;
+};
+
 typedef struct data_reference *dr_p;
 
 typedef struct _stmt_vec_info {
@@ -772,6 +797,8 @@ typedef struct _stmt_vec_info {
      relative to the inner-most containing loop.  */
   struct data_reference *data_ref_info;
 
+  dataref_aux dr_aux;
+
   /* Information about the data-ref relative to this loop
      nest (the loop that is being considered for vectorization).  */
   innermost_loop_behavior dr_wrt_vec_loop;
@@ -981,18 +1008,7 @@ STMT_VINFO_BB_VINFO (stmt_vec_info stmt_vinfo)
 #define PURE_SLP_STMT(S)                  ((S)->slp_type == pure_slp)
 #define STMT_SLP_TYPE(S)                   (S)->slp_type
 
-struct dataref_aux {
-  /* The misalignment in bytes of the reference, or -1 if not known.  */
-  int misalignment;
-  /* The byte alignment that we'd ideally like the reference to have,
-     and the value that misalignment is measured against.  */
-  int target_alignment;
-  /* If true the alignment of base_decl needs to be increased.  */
-  bool base_misaligned;
-  tree base_decl;
-};
-
-#define DR_VECT_AUX(dr) ((dataref_aux *)(dr)->aux)
+#define DR_VECT_AUX(dr) (&vinfo_for_stmt (DR_STMT (dr))->dr_aux)
 
 #define VECT_MAX_COST 1000
 
@@ -1230,31 +1246,28 @@ add_stmt_costs (void *data, stmt_vector_for_cost *cost_vec)
 /*-----------------------------------------------------------------*/
 /* Info on data references alignment.                              */
 /*-----------------------------------------------------------------*/
+#define DR_MISALIGNMENT_UNKNOWN (-1)
+#define DR_MISALIGNMENT_UNINITIALIZED (-2)
+
 inline void
 set_dr_misalignment (struct data_reference *dr, int val)
 {
   dataref_aux *data_aux = DR_VECT_AUX (dr);
-
-  if (!data_aux)
-    {
-      data_aux = XCNEW (dataref_aux);
-      dr->aux = data_aux;
-    }
-
   data_aux->misalignment = val;
 }
 
 inline int
 dr_misalignment (struct data_reference *dr)
 {
-  return DR_VECT_AUX (dr)->misalignment;
+  int misalign = DR_VECT_AUX (dr)->misalignment;
+  gcc_assert (misalign != DR_MISALIGNMENT_UNINITIALIZED);
+  return misalign;
 }
 
 /* Reflects actual alignment of first access in the vectorized loop,
    taking into account peeling/versioning if applied.  */
 #define DR_MISALIGNMENT(DR) dr_misalignment (DR)
 #define SET_DR_MISALIGNMENT(DR, VAL) set_dr_misalignment (DR, VAL)
-#define DR_MISALIGNMENT_UNKNOWN (-1)
 
 /* Only defined once DR_MISALIGNMENT is defined.  */
 #define DR_TARGET_ALIGNMENT(DR) DR_VECT_AUX (DR)->target_alignment
@@ -1572,7 +1585,8 @@ extern gimple *vect_force_simple_reduction (loop_vec_info, gimple *,
 extern bool check_reduction_path (location_t, loop_p, gphi *, tree,
 				  enum tree_code);
 /* Drive for loop analysis stage.  */
-extern loop_vec_info vect_analyze_loop (struct loop *, loop_vec_info);
+extern loop_vec_info vect_analyze_loop (struct loop *, loop_vec_info,
+					vec_info_shared *);
 extern tree vect_build_loop_niters (loop_vec_info, bool * = NULL);
 extern void vect_gen_vector_loop_niters (loop_vec_info, tree, tree *,
 					 tree *, bool);
@@ -1585,7 +1599,7 @@ extern tree vect_get_loop_mask (gimple_stmt_iterator *, vec_loop_masks *,
 
 /* Drive for loop transformation stage.  */
 extern struct loop *vect_transform_loop (loop_vec_info);
-extern loop_vec_info vect_analyze_loop_form (struct loop *);
+extern loop_vec_info vect_analyze_loop_form (struct loop *, vec_info_shared *);
 extern bool vectorizable_live_operation (gimple *, gimple_stmt_iterator *,
 					 slp_tree, int, gimple **,
 					 stmt_vector_for_cost *);