Vector peeling cost model 2/6

gcc/ChangeLog:

2017-05-30  Robin Dapp  <rdapp@linux.vnet.ibm.com>

	* tree-vect-data-refs.c (vect_update_misalignment_for_peel): Rename.
	(vect_get_peeling_costs_all_drs): Create function.
	(vect_peeling_hash_get_lowest_cost):
	Use vect_get_peeling_costs_all_drs.
	(vect_peeling_supportable): Create function.
	(vect_enhance_data_refs_alignment): Use vect_peeling_supportable.

From-SVN: r248676

gcc/ChangeLog

 2017-05-30  Robin Dapp  <rdapp@linux.vnet.ibm.com>
 
+	* tree-vect-data-refs.c (vect_update_misalignment_for_peel): Rename.
+	(vect_get_peeling_costs_all_drs): Create function.
+	(vect_peeling_hash_get_lowest_cost):
+	Use vect_get_peeling_costs_all_drs.
+	(vect_peeling_supportable): Create function.
+	(vect_enhance_data_refs_alignment): Use vect_peeling_supportable.
+
+2017-05-30  Robin Dapp  <rdapp@linux.vnet.ibm.com>
+
 	* tree-vect-data-refs.c (vect_compute_data_ref_alignment): Create
 	DR_HAS_NEGATIVE_STEP.
 	(vect_update_misalignment_for_peel): Define DR_MISALIGNMENT.

gcc/tree-vect-data-refs.c

@@ -903,7 +903,11 @@ vect_compute_data_ref_alignment (struct data_reference *dr)
 }
 
 
-/* Function vect_update_misalignment_for_peel
+/* Function vect_update_misalignment_for_peel.
+   Sets DR's misalignment
+   - to 0 if it has the same alignment as DR_PEEL,
+   - to the misalignment computed using NPEEL if DR's salignment is known,
+   - to -1 (unknown) otherwise.
 
    DR - the data reference whose misalignment is to be adjusted.
    DR_PEEL - the data reference whose misalignment is being made
@@ -916,7 +920,7 @@ vect_update_misalignment_for_peel (struct data_reference *dr,
                                    struct data_reference *dr_peel, int npeel)
 {
   unsigned int i;
-  vec<dr_p> same_align_drs;
+  vec<dr_p> same_aligned_drs;
   struct data_reference *current_dr;
   int dr_size = GET_MODE_SIZE (TYPE_MODE (TREE_TYPE (DR_REF (dr))));
   int dr_peel_size = GET_MODE_SIZE (TYPE_MODE (TREE_TYPE (DR_REF (dr_peel))));
@@ -932,9 +936,9 @@ vect_update_misalignment_for_peel (struct data_reference *dr,
 
   /* It can be assumed that the data refs with the same alignment as dr_peel
      are aligned in the vector loop.  */
-  same_align_drs
+  same_aligned_drs
     = STMT_VINFO_SAME_ALIGN_REFS (vinfo_for_stmt (DR_STMT (dr_peel)));
-  FOR_EACH_VEC_ELT (same_align_drs, i, current_dr)
+  FOR_EACH_VEC_ELT (same_aligned_drs, i, current_dr)
     {
       if (current_dr != dr)
         continue;
@@ -1234,27 +1238,23 @@ vect_peeling_hash_get_most_frequent (_vect_peel_info **slot,
   return 1;
 }
 
+/* Get the costs of peeling NPEEL iterations checking data access costs
+   for all data refs. */
 
-/* Traverse peeling hash table and calculate cost for each peeling option.
-   Find the one with the lowest cost.  */
-
-int
-vect_peeling_hash_get_lowest_cost (_vect_peel_info **slot,
-				   _vect_peel_extended_info *min)
+static void
+vect_get_peeling_costs_all_drs (struct data_reference *dr0,
+				unsigned int *inside_cost,
+				unsigned int *outside_cost,
+				stmt_vector_for_cost *body_cost_vec,
+				unsigned int npeel, unsigned int vf)
 {
-  vect_peel_info elem = *slot;
-  int save_misalignment, dummy;
-  unsigned int inside_cost = 0, outside_cost = 0, i;
-  gimple *stmt = DR_STMT (elem->dr);
+  gimple *stmt = DR_STMT (dr0);
   stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
   loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
   vec<data_reference_p> datarefs = LOOP_VINFO_DATAREFS (loop_vinfo);
-  struct data_reference *dr;
-  stmt_vector_for_cost prologue_cost_vec, body_cost_vec, epilogue_cost_vec;
 
-  prologue_cost_vec.create (2);
-  body_cost_vec.create (2);
-  epilogue_cost_vec.create (2);
+  unsigned i;
+  data_reference *dr;
 
   FOR_EACH_VEC_ELT (datarefs, i, dr)
     {
@@ -1272,12 +1272,40 @@ vect_peeling_hash_get_lowest_cost (_vect_peel_info **slot,
 	  && !STMT_VINFO_GROUPED_ACCESS (stmt_info))
 	continue;
 
+      int save_misalignment;
       save_misalignment = DR_MISALIGNMENT (dr);
-      vect_update_misalignment_for_peel (dr, elem->dr, elem->npeel);
-      vect_get_data_access_cost (dr, &inside_cost, &outside_cost,
-				 &body_cost_vec);
+      if (dr == dr0 && npeel == vf / 2)
+	SET_DR_MISALIGNMENT (dr, 0);
+      else
+	vect_update_misalignment_for_peel (dr, dr0, npeel);
+      vect_get_data_access_cost (dr, inside_cost, outside_cost,
+				 body_cost_vec);
       SET_DR_MISALIGNMENT (dr, save_misalignment);
     }
+}
+
+/* Traverse peeling hash table and calculate cost for each peeling option.
+   Find the one with the lowest cost.  */
+
+int
+vect_peeling_hash_get_lowest_cost (_vect_peel_info **slot,
+				   _vect_peel_extended_info *min)
+{
+  vect_peel_info elem = *slot;
+  int dummy;
+  unsigned int inside_cost = 0, outside_cost = 0;
+  gimple *stmt = DR_STMT (elem->dr);
+  stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
+  loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
+  stmt_vector_for_cost prologue_cost_vec, body_cost_vec,
+		       epilogue_cost_vec;
+
+  prologue_cost_vec.create (2);
+  body_cost_vec.create (2);
+  epilogue_cost_vec.create (2);
+
+  vect_get_peeling_costs_all_drs (elem->dr, &inside_cost, &outside_cost,
+				  &body_cost_vec, elem->npeel, 0);
 
   outside_cost += vect_get_known_peeling_cost
     (loop_vinfo, elem->npeel, &dummy,
@@ -1292,7 +1320,8 @@ vect_peeling_hash_get_lowest_cost (_vect_peel_info **slot,
   epilogue_cost_vec.release ();
 
   if (inside_cost < min->inside_cost
-      || (inside_cost == min->inside_cost && outside_cost < min->outside_cost))
+      || (inside_cost == min->inside_cost
+	  && outside_cost < min->outside_cost))
     {
       min->inside_cost = inside_cost;
       min->outside_cost = outside_cost;
@@ -1300,6 +1329,7 @@ vect_peeling_hash_get_lowest_cost (_vect_peel_info **slot,
       min->body_cost_vec = body_cost_vec;
       min->peel_info.dr = elem->dr;
       min->peel_info.npeel = elem->npeel;
+      min->peel_info.count = elem->count;
     }
   else
     body_cost_vec.release ();
@@ -1342,6 +1372,52 @@ vect_peeling_hash_choose_best_peeling (hash_table<peel_info_hasher> *peeling_hta
    return res.peel_info.dr;
 }
 
+/* Return true if the new peeling NPEEL is supported.  */
+
+static bool
+vect_peeling_supportable (loop_vec_info loop_vinfo, struct data_reference *dr0,
+			  unsigned npeel)
+{
+  unsigned i;
+  struct data_reference *dr = NULL;
+  vec<data_reference_p> datarefs = LOOP_VINFO_DATAREFS (loop_vinfo);
+  gimple *stmt;
+  stmt_vec_info stmt_info;
+  enum dr_alignment_support supportable_dr_alignment;
+
+  /* Ensure that all data refs can be vectorized after the peel.  */
+  FOR_EACH_VEC_ELT (datarefs, i, dr)
+    {
+      int save_misalignment;
+
+      if (dr == dr0)
+	continue;
+
+      stmt = DR_STMT (dr);
+      stmt_info = vinfo_for_stmt (stmt);
+      /* For interleaving, only the alignment of the first access
+	 matters.  */
+      if (STMT_VINFO_GROUPED_ACCESS (stmt_info)
+	  && GROUP_FIRST_ELEMENT (stmt_info) != stmt)
+	continue;
+
+      /* Strided accesses perform only component accesses, alignment is
+	 irrelevant for them.  */
+      if (STMT_VINFO_STRIDED_P (stmt_info)
+	  && !STMT_VINFO_GROUPED_ACCESS (stmt_info))
+	continue;
+
+      save_misalignment = DR_MISALIGNMENT (dr);
+      vect_update_misalignment_for_peel (dr, dr0, npeel);
+      supportable_dr_alignment = vect_supportable_dr_alignment (dr, false);
+      SET_DR_MISALIGNMENT (dr, save_misalignment);
+
+      if (!supportable_dr_alignment)
+	return false;
+    }
+
+  return true;
+}
 
 /* Function vect_enhance_data_refs_alignment
 
@@ -1780,40 +1856,11 @@ vect_enhance_data_refs_alignment (loop_vec_info loop_vinfo)
                              "Try peeling by %d\n", npeel);
         }
 
-      /* Ensure that all data refs can be vectorized after the peel.  */
-      FOR_EACH_VEC_ELT (datarefs, i, dr)
-        {
-          int save_misalignment;
-
-	  if (dr == dr0)
-	    continue;
-
-	  stmt = DR_STMT (dr);
-	  stmt_info = vinfo_for_stmt (stmt);
-	  /* For interleaving, only the alignment of the first access
-            matters.  */
-	  if (STMT_VINFO_GROUPED_ACCESS (stmt_info)
-	      && GROUP_FIRST_ELEMENT (stmt_info) != stmt)
-	    continue;
-
-	  /* Strided accesses perform only component accesses, alignment is
-	     irrelevant for them.  */
-	  if (STMT_VINFO_STRIDED_P (stmt_info)
-	      && !STMT_VINFO_GROUPED_ACCESS (stmt_info))
-	    continue;
-
-	  save_misalignment = DR_MISALIGNMENT (dr);
-	  vect_update_misalignment_for_peel (dr, dr0, npeel);
-	  supportable_dr_alignment = vect_supportable_dr_alignment (dr, false);
-	  SET_DR_MISALIGNMENT (dr, save_misalignment);
-
-	  if (!supportable_dr_alignment)
-	    {
-	      do_peeling = false;
-	      break;
-	    }
-	}
+      /* Ensure that all datarefs can be vectorized after the peel.  */
+      if (!vect_peeling_supportable (loop_vinfo, dr0, npeel))
+	do_peeling = false;
 
+      /* Check if all datarefs are supportable and log.  */
       if (do_peeling && known_alignment_for_access_p (dr0) && npeel == 0)
         {
           stat = vect_verify_datarefs_alignment (loop_vinfo);