tree-vect-data-refs.c (vect_analyze_data_ref_dependence): Only add RW dependence…

tree-vect-data-refs.c (vect_analyze_data_ref_dependence): Only add RW dependence for dependence distance zero.

2010-04-13  Richard Guenther  <rguenther@suse.de>

	* tree-vect-data-refs.c (vect_analyze_data_ref_dependence):
	Only add RW dependence for dependence distance zero.
	Adjust maximal vectorization factor according to dependences.
	Move alignment handling ...
	(vect_find_same_alignment_drs): ... here.  New function.
	(vect_analyze_data_ref_dependences): Adjust.
	(vect_analyze_data_refs_alignment): Call vect_find_same_alignment_drs.
	(vect_analyze_data_refs): Adjust minimal vectorization factor
	according to data references.
	* tree-vect-loop.c (vect_analyze_loop): Analyze data-ref
	dependences before determining the vectorization factor.
	Analyze alignment after determining the vectorization factor.
	* tree-vect-slp.c ((vect_slp_analyze_bb): Analyze data-ref
	dependences before alignment.
	* tree-vectorizer.h (vect_analyze_data_ref_dependences):
	Adjust prototype.
	(vect_analyze_data_refs): Likewise.
	(MAX_VECTORIZATION_FACTOR): New define.

	* gcc.dg/vect/no-vfa-vect-depend-1.c: Adjust.

From-SVN: r158259

gcc/ChangeLog

+2010-04-13  Richard Guenther  <rguenther@suse.de>
+
+	* tree-vect-data-refs.c (vect_analyze_data_ref_dependence):
+	Only add RW dependence for dependence distance zero.
+	Adjust maximal vectorization factor according to dependences.
+	Move alignment handling ...
+	(vect_find_same_alignment_drs): ... here.  New function.
+	(vect_analyze_data_ref_dependences): Adjust.
+	(vect_analyze_data_refs_alignment): Call vect_find_same_alignment_drs.
+	(vect_analyze_data_refs): Adjust minimal vectorization factor
+	according to data references.
+	* tree-vect-loop.c (vect_analyze_loop): Analyze data-ref
+	dependences before determining the vectorization factor.
+	Analyze alignment after determining the vectorization factor.
+	* tree-vect-slp.c ((vect_slp_analyze_bb): Analyze data-ref
+	dependences before alignment.
+	* tree-vectorizer.h (vect_analyze_data_ref_dependences):
+	Adjust prototype.
+	(vect_analyze_data_refs): Likewise.
+	(MAX_VECTORIZATION_FACTOR): New define.
+
 2010-04-13  Duncan Sands  <baldrick@free.fr>
 
 	* except.h (lang_eh_type_covers): Remove.

gcc/testsuite/ChangeLog

+2010-04-13  Richard Guenther  <rguenther@suse.de>
+
+	* gcc.dg/vect/no-vfa-vect-depend-1.c: Adjust.
+
 2010-04-13  Eric Botcazou  <ebotcazou@adacore.com>
 
 	* gnat.dg/derived_type1.adb: New test.

gcc/testsuite/gcc.dg/vect/no-vfa-vect-depend-1.c

@@ -51,6 +51,6 @@ int main (void)
 }
 
 /* { dg-final { scan-tree-dump-times "vectorized 1 loops" 1 "vect" {xfail vect_no_align } } } */
-/* { dg-final { scan-tree-dump-times "dependence distance >= VF or negative" 1 "vect"  } } */
+/* { dg-final { scan-tree-dump-times "dependence distance negative" 1 "vect"  } } */
 /* { dg-final { cleanup-tree-dump "vect" } } */
 

gcc/tree-vect-data-refs.c

@@ -487,21 +487,19 @@ vect_mark_for_runtime_alias_test (ddr_p ddr, loop_vec_info loop_vinfo)
 
    Return TRUE if there (might) exist a dependence between a memory-reference
    DRA and a memory-reference DRB.  When versioning for alias may check a
-   dependence at run-time, return FALSE.  */
+   dependence at run-time, return FALSE.  Adjust *MAX_VF according to
+   the data dependence.  */
 
 static bool
 vect_analyze_data_ref_dependence (struct data_dependence_relation *ddr,
-                                  loop_vec_info loop_vinfo)
+                                  loop_vec_info loop_vinfo, int *max_vf)
 {
   unsigned int i;
   struct loop *loop = NULL;
-  int vectorization_factor = 0;
   struct data_reference *dra = DDR_A (ddr);
   struct data_reference *drb = DDR_B (ddr);
   stmt_vec_info stmtinfo_a = vinfo_for_stmt (DR_STMT (dra));
   stmt_vec_info stmtinfo_b = vinfo_for_stmt (DR_STMT (drb));
-  int dra_size = GET_MODE_SIZE (TYPE_MODE (TREE_TYPE (DR_REF (dra))));
-  int drb_size = GET_MODE_SIZE (TYPE_MODE (TREE_TYPE (DR_REF (drb))));
   lambda_vector dist_v;
   unsigned int loop_depth;
 
@@ -513,10 +511,7 @@ vect_analyze_data_ref_dependence (struct data_dependence_relation *ddr,
     }
 
   if (loop_vinfo)
-    {
-      loop = LOOP_VINFO_LOOP (loop_vinfo);
-      vectorization_factor = LOOP_VINFO_VECT_FACTOR (loop_vinfo);
-    }
+    loop = LOOP_VINFO_LOOP (loop_vinfo);
 
   if ((DR_IS_READ (dra) && DR_IS_READ (drb) && loop_vinfo) || dra == drb)
     return false;
@@ -595,17 +590,11 @@ vect_analyze_data_ref_dependence (struct data_dependence_relation *ddr,
       if (vect_print_dump_info (REPORT_DR_DETAILS))
 	fprintf (vect_dump, "dependence distance  = %d.", dist);
 
-      /* Same loop iteration.  */
-      if (dist % vectorization_factor == 0 && dra_size == drb_size)
+      if (dist == 0)
 	{
-	  /* Two references with distance zero have the same alignment.  */
-	  VEC_safe_push (dr_p, heap, STMT_VINFO_SAME_ALIGN_REFS (stmtinfo_a), drb);
-	  VEC_safe_push (dr_p, heap, STMT_VINFO_SAME_ALIGN_REFS (stmtinfo_b), dra);
-	  if (vect_print_dump_info (REPORT_ALIGNMENT))
-	    fprintf (vect_dump, "accesses have the same alignment.");
 	  if (vect_print_dump_info (REPORT_DR_DETAILS))
 	    {
-	      fprintf (vect_dump, "dependence distance modulo vf == 0 between ");
+	      fprintf (vect_dump, "dependence distance == 0 between ");
 	      print_generic_expr (vect_dump, DR_REF (dra), TDF_SLIM);
 	      fprintf (vect_dump, " and ");
 	      print_generic_expr (vect_dump, DR_REF (drb), TDF_SLIM);
@@ -621,18 +610,36 @@ vect_analyze_data_ref_dependence (struct data_dependence_relation *ddr,
                 DR_GROUP_READ_WRITE_DEPENDENCE (stmtinfo_b) = true;
 	    }
 
-          continue;
+	  continue;
+	}
+
+      if (dist > 0 && DDR_REVERSED_P (ddr))
+	{
+	  /* If DDR_REVERSED_P the order of the data-refs in DDR was
+	     reversed (to make distance vector positive), and the actual
+	     distance is negative.  */
+	  if (vect_print_dump_info (REPORT_DR_DETAILS))
+	    fprintf (vect_dump, "dependence distance negative.");
+	  continue;
+	}
+
+      if (abs (dist) >= 2
+	  && abs (dist) < *max_vf)
+	{
+	  /* The dependence distance requires reduction of the maximal
+	     vectorization factor.  */
+	  *max_vf = abs (dist);
+	  if (vect_print_dump_info (REPORT_DR_DETAILS))
+	    fprintf (vect_dump, "adjusting maximal vectorization factor to %i",
+		     *max_vf);
 	}
 
-      if (abs (dist) >= vectorization_factor
-          || (dist > 0 && DDR_REVERSED_P (ddr)))
+      if (abs (dist) >= *max_vf)
 	{
 	  /* Dependence distance does not create dependence, as far as
-	     vectorization is concerned, in this case. If DDR_REVERSED_P the
-	     order of the data-refs in DDR was reversed (to make distance
-	     vector positive), and the actual distance is negative.  */
+	     vectorization is concerned, in this case.  */
 	  if (vect_print_dump_info (REPORT_DR_DETAILS))
-	    fprintf (vect_dump, "dependence distance >= VF or negative.");
+	    fprintf (vect_dump, "dependence distance >= VF.");
 	  continue;
 	}
 
@@ -654,11 +661,12 @@ vect_analyze_data_ref_dependence (struct data_dependence_relation *ddr,
 /* Function vect_analyze_data_ref_dependences.
 
    Examine all the data references in the loop, and make sure there do not
-   exist any data dependences between them.  */
+   exist any data dependences between them.  Set *MAX_VF according to
+   the maximum vectorization factor the data dependences allow.  */
 
 bool
 vect_analyze_data_ref_dependences (loop_vec_info loop_vinfo,
-                                   bb_vec_info bb_vinfo)
+                                   bb_vec_info bb_vinfo, int *max_vf)
 {
   unsigned int i;
   VEC (ddr_p, heap) *ddrs = NULL;
@@ -673,7 +681,7 @@ vect_analyze_data_ref_dependences (loop_vec_info loop_vinfo,
     ddrs = BB_VINFO_DDRS (bb_vinfo);
 
   for (i = 0; VEC_iterate (ddr_p, ddrs, i, ddr); i++)
-    if (vect_analyze_data_ref_dependence (ddr, loop_vinfo))
+    if (vect_analyze_data_ref_dependence (ddr, loop_vinfo, max_vf))
       return false;
 
   return true;
@@ -1410,6 +1418,69 @@ vect_enhance_data_refs_alignment (loop_vec_info loop_vinfo)
 }
 
 
+/* Function vect_find_same_alignment_drs.
+
+   Update group and alignment relations according to the chosen
+   vectorization factor.  */
+
+static void
+vect_find_same_alignment_drs (struct data_dependence_relation *ddr,
+			      loop_vec_info loop_vinfo)
+{
+  unsigned int i;
+  struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
+  int vectorization_factor = LOOP_VINFO_VECT_FACTOR (loop_vinfo);
+  struct data_reference *dra = DDR_A (ddr);
+  struct data_reference *drb = DDR_B (ddr);
+  stmt_vec_info stmtinfo_a = vinfo_for_stmt (DR_STMT (dra));
+  stmt_vec_info stmtinfo_b = vinfo_for_stmt (DR_STMT (drb));
+  int dra_size = GET_MODE_SIZE (TYPE_MODE (TREE_TYPE (DR_REF (dra))));
+  int drb_size = GET_MODE_SIZE (TYPE_MODE (TREE_TYPE (DR_REF (drb))));
+  lambda_vector dist_v;
+  unsigned int loop_depth;
+
+  if (DDR_ARE_DEPENDENT (ddr) == chrec_known)
+    return;
+
+  if ((DR_IS_READ (dra) && DR_IS_READ (drb)) || dra == drb)
+    return;
+
+  if (DDR_ARE_DEPENDENT (ddr) == chrec_dont_know)
+    return;
+
+  /* Loop-based vectorization and known data dependence.  */
+  if (DDR_NUM_DIST_VECTS (ddr) == 0)
+    return;
+
+  loop_depth = index_in_loop_nest (loop->num, DDR_LOOP_NEST (ddr));
+  for (i = 0; VEC_iterate (lambda_vector, DDR_DIST_VECTS (ddr), i, dist_v); i++)
+    {
+      int dist = dist_v[loop_depth];
+
+      if (vect_print_dump_info (REPORT_DR_DETAILS))
+	fprintf (vect_dump, "dependence distance  = %d.", dist);
+
+      /* Same loop iteration.  */
+      if (dist == 0
+	  || (dist % vectorization_factor == 0 && dra_size == drb_size))
+	{
+	  /* Two references with distance zero have the same alignment.  */
+	  VEC_safe_push (dr_p, heap, STMT_VINFO_SAME_ALIGN_REFS (stmtinfo_a), drb);
+	  VEC_safe_push (dr_p, heap, STMT_VINFO_SAME_ALIGN_REFS (stmtinfo_b), dra);
+	  if (vect_print_dump_info (REPORT_ALIGNMENT))
+	    fprintf (vect_dump, "accesses have the same alignment.");
+	  if (vect_print_dump_info (REPORT_DR_DETAILS))
+	    {
+	      fprintf (vect_dump, "dependence distance modulo vf == 0 between ");
+	      print_generic_expr (vect_dump, DR_REF (dra), TDF_SLIM);
+	      fprintf (vect_dump, " and ");
+	      print_generic_expr (vect_dump, DR_REF (drb), TDF_SLIM);
+	    }
+	}
+    }
+}
+
+
 /* Function vect_analyze_data_refs_alignment
 
    Analyze the alignment of the data-references in the loop.
@@ -1422,6 +1493,18 @@ vect_analyze_data_refs_alignment (loop_vec_info loop_vinfo,
   if (vect_print_dump_info (REPORT_DETAILS))
     fprintf (vect_dump, "=== vect_analyze_data_refs_alignment ===");
 
+  /* Mark groups of data references with same alignment using
+     data dependence information.  */
+  if (loop_vinfo)
+    {
+      VEC (ddr_p, heap) *ddrs = LOOP_VINFO_DDRS (loop_vinfo);
+      struct data_dependence_relation *ddr;
+      unsigned int i;
+
+      for (i = 0; VEC_iterate (ddr_p, ddrs, i, ddr); i++)
+	vect_find_same_alignment_drs (ddr, loop_vinfo);
+    }
+
   if (!vect_compute_data_refs_alignment (loop_vinfo, bb_vinfo))
     {
       if (vect_print_dump_info (REPORT_UNVECTORIZED_LOCATIONS))
@@ -1852,7 +1935,9 @@ vect_prune_runtime_alias_test_list (loop_vec_info loop_vinfo)
 */
 
 bool
-vect_analyze_data_refs (loop_vec_info loop_vinfo, bb_vec_info bb_vinfo)
+vect_analyze_data_refs (loop_vec_info loop_vinfo,
+			bb_vec_info bb_vinfo,
+			int *min_vf)
 {
   struct loop *loop = NULL;
   basic_block bb = NULL;
@@ -1907,6 +1992,7 @@ vect_analyze_data_refs (loop_vec_info loop_vinfo, bb_vec_info bb_vinfo)
       gimple stmt;
       stmt_vec_info stmt_info;
       tree base, offset, init;
+      int vf;
 
       if (!dr || !DR_REF (dr))
         {
@@ -2079,6 +2165,12 @@ vect_analyze_data_refs (loop_vec_info loop_vinfo, bb_vec_info bb_vinfo)
             }
           return false;
         }
+
+      /* Adjust the minimal vectorization factor according to the
+	 vector type.  */
+      vf = TYPE_VECTOR_SUBPARTS (STMT_VINFO_VECTYPE (stmt_info));
+      if (vf > *min_vf)
+	*min_vf = vf;
     }
 
   return true;

gcc/tree-vect-loop.c

@@ -1354,6 +1354,8 @@ vect_analyze_loop (struct loop *loop)
 {
   bool ok;
   loop_vec_info loop_vinfo;
+  int max_vf = MAX_VECTORIZATION_FACTOR;
+  int min_vf = 2;
 
   if (vect_print_dump_info (REPORT_DETAILS))
     fprintf (vect_dump, "===== analyze_loop_nest =====");
@@ -1378,12 +1380,13 @@ vect_analyze_loop (struct loop *loop)
     }
 
   /* Find all data references in the loop (which correspond to vdefs/vuses)
-     and analyze their evolution in the loop.
+     and analyze their evolution in the loop.  Also adjust the minimal
+     vectorization factor according to the loads and stores.
 
      FORNOW: Handle only simple, array references, which
      alignment can be forced, and aligned pointer-references.  */
 
-  ok = vect_analyze_data_refs (loop_vinfo, NULL);
+  ok = vect_analyze_data_refs (loop_vinfo, NULL, &min_vf);
   if (!ok)
     {
       if (vect_print_dump_info (REPORT_DETAILS))
@@ -1410,14 +1413,17 @@ vect_analyze_loop (struct loop *loop)
       return NULL;
     }
 
-  /* Analyze the alignment of the data-refs in the loop.
-     Fail if a data reference is found that cannot be vectorized.  */
+  /* Analyze data dependences between the data-refs in the loop
+     and adjust the maximum vectorization factor according to
+     the dependences.
+     FORNOW: fail at the first data dependence that we encounter.  */
 
-  ok = vect_analyze_data_refs_alignment (loop_vinfo, NULL);
-  if (!ok)
+  ok = vect_analyze_data_ref_dependences (loop_vinfo, NULL, &max_vf);
+  if (!ok
+      || max_vf < min_vf)
     {
       if (vect_print_dump_info (REPORT_DETAILS))
-	fprintf (vect_dump, "bad data alignment.");
+	fprintf (vect_dump, "bad data dependence.");
       destroy_loop_vec_info (loop_vinfo, true);
       return NULL;
     }
@@ -1430,15 +1436,22 @@ vect_analyze_loop (struct loop *loop)
       destroy_loop_vec_info (loop_vinfo, true);
       return NULL;
     }
+  if (max_vf < LOOP_VINFO_VECT_FACTOR (loop_vinfo))
+    {
+      if (vect_print_dump_info (REPORT_DETAILS))
+	fprintf (vect_dump, "bad data dependence.");
+      destroy_loop_vec_info (loop_vinfo, true);
+      return NULL;
+    }
 
-  /* Analyze data dependences between the data-refs in the loop.
-     FORNOW: fail at the first data dependence that we encounter.  */
+  /* Analyze the alignment of the data-refs in the loop.
+     Fail if a data reference is found that cannot be vectorized.  */
 
-  ok = vect_analyze_data_ref_dependences (loop_vinfo, NULL);
+  ok = vect_analyze_data_refs_alignment (loop_vinfo, NULL);
   if (!ok)
     {
       if (vect_print_dump_info (REPORT_DETAILS))
-	fprintf (vect_dump, "bad data dependence.");
+	fprintf (vect_dump, "bad data alignment.");
       destroy_loop_vec_info (loop_vinfo, true);
       return NULL;
     }

gcc/tree-vect-slp.c

@@ -1270,6 +1270,8 @@ vect_slp_analyze_bb (basic_block bb)
   slp_instance instance;
   int i, insns = 0;
   gimple_stmt_iterator gsi;
+  int min_vf = 2;
+  int max_vf = MAX_VECTORIZATION_FACTOR;
 
   if (vect_print_dump_info (REPORT_DETAILS))
     fprintf (vect_dump, "===vect_slp_analyze_bb===\n");
@@ -1296,7 +1298,7 @@ vect_slp_analyze_bb (basic_block bb)
   if (!bb_vinfo)
     return NULL;
 
-  if (!vect_analyze_data_refs (NULL, bb_vinfo))
+  if (!vect_analyze_data_refs (NULL, bb_vinfo, &min_vf))
     {
       if (vect_print_dump_info (REPORT_UNVECTORIZED_LOCATIONS))
         fprintf (vect_dump, "not vectorized: unhandled data-ref in basic "
@@ -1317,6 +1319,17 @@ vect_slp_analyze_bb (basic_block bb)
       return NULL;
     }
 
+   if (!vect_analyze_data_ref_dependences (NULL, bb_vinfo, &max_vf)
+       || min_vf > max_vf)
+     {
+       if (vect_print_dump_info (REPORT_UNVECTORIZED_LOCATIONS))
+	 fprintf (vect_dump, "not vectorized: unhandled data dependence "
+		  "in basic block.\n");
+
+       destroy_bb_vec_info (bb_vinfo);
+       return NULL;
+     }
+
   if (!vect_analyze_data_refs_alignment (NULL, bb_vinfo))
     {
       if (vect_print_dump_info (REPORT_UNVECTORIZED_LOCATIONS))
@@ -1327,16 +1340,6 @@ vect_slp_analyze_bb (basic_block bb)
       return NULL;
     }
 
-   if (!vect_analyze_data_ref_dependences (NULL, bb_vinfo))
-    {
-     if (vect_print_dump_info (REPORT_UNVECTORIZED_LOCATIONS))
-       fprintf (vect_dump, "not vectorized: unhandled data dependence in basic"
-                           " block.\n");
-
-      destroy_bb_vec_info (bb_vinfo);
-      return NULL;
-    }
-
   if (!vect_analyze_data_ref_accesses (NULL, bb_vinfo))
     {
      if (vect_print_dump_info (REPORT_UNVECTORIZED_LOCATIONS))

gcc/tree-vectorizer.h

@@ -601,6 +601,9 @@ typedef struct _stmt_vec_info {
    conversion.  */
 #define MAX_INTERM_CVT_STEPS         3
 
+/* The maximum vectorization factor supported by any target (V32QI).  */
+#define MAX_VECTORIZATION_FACTOR 32
+
 /* Avoid GTY(()) on stmt_vec_info.  */
 typedef void *vec_void_p;
 DEF_VEC_P (vec_void_p);
@@ -802,13 +805,14 @@ extern enum dr_alignment_support vect_supportable_dr_alignment
                                            (struct data_reference *);
 extern tree vect_get_smallest_scalar_type (gimple, HOST_WIDE_INT *,
                                            HOST_WIDE_INT *);
-extern bool vect_analyze_data_ref_dependences (loop_vec_info, bb_vec_info);
+extern bool vect_analyze_data_ref_dependences (loop_vec_info, bb_vec_info,
+					       int *);
 extern bool vect_enhance_data_refs_alignment (loop_vec_info);
 extern bool vect_analyze_data_refs_alignment (loop_vec_info, bb_vec_info);
 extern bool vect_verify_datarefs_alignment (loop_vec_info, bb_vec_info);
 extern bool vect_analyze_data_ref_accesses (loop_vec_info, bb_vec_info);
 extern bool vect_prune_runtime_alias_test_list (loop_vec_info);
-extern bool vect_analyze_data_refs (loop_vec_info, bb_vec_info);
+extern bool vect_analyze_data_refs (loop_vec_info, bb_vec_info, int *);
 extern tree vect_create_data_ref_ptr (gimple, struct loop *, tree, tree *,
                                       gimple *, bool, bool *);
 extern tree bump_vector_ptr (tree, gimple, gimple_stmt_iterator *, gimple, tree);