tree-vectorizer.c (try_vectorize_loop_1): Split out of ...

2018-06-19  Richard Biener  <rguenther@suse.de>

	* tree-vectorizer.c (try_vectorize_loop_1): Split out of ...
	(vectorize_loops): ... here.  Fix dbgcnt handling.
	(try_vectorize_loop): Wrap try_vectorize_loop_1.

From-SVN: r261740

gcc/ChangeLog

+2018-06-19  Richard Biener  <rguenther@suse.de>
+
+	* tree-vectorizer.c (try_vectorize_loop_1): Split out of ...
+	(vectorize_loops): ... here.  Fix dbgcnt handling.
+	(try_vectorize_loop): Wrap try_vectorize_loop_1.
+
 2018-06-19  Segher Boessenkool  <segher@kernel.crashing.org>
 
 	PR target/86197

gcc/tree-vectorizer.c

@@ -659,95 +659,16 @@ set_uid_loop_bbs (loop_vec_info loop_vinfo, gimple *loop_vectorized_call)
   free (bbs);
 }
 
-/* Function vectorize_loops.
-
-   Entry point to loop vectorization phase.  */
+/* Try to vectorize LOOP.  */
 
-unsigned
-vectorize_loops (void)
+static unsigned
+try_vectorize_loop_1 (hash_table<simduid_to_vf> *&simduid_to_vf_htab,
+		      unsigned *num_vectorized_loops,
+		      loop_p loop, loop_vec_info orig_loop_vinfo,
+		      gimple *loop_vectorized_call,
+		      gimple *loop_dist_alias_call)
 {
-  unsigned int i;
-  unsigned int num_vectorized_loops = 0;
-  unsigned int vect_loops_num;
-  struct loop *loop;
-  hash_table<simduid_to_vf> *simduid_to_vf_htab = NULL;
-  hash_table<simd_array_to_simduid> *simd_array_to_simduid_htab = NULL;
-  bool any_ifcvt_loops = false;
   unsigned ret = 0;
-  struct loop *new_loop;
-
-  vect_loops_num = number_of_loops (cfun);
-
-  /* Bail out if there are no loops.  */
-  if (vect_loops_num <= 1)
-    return 0;
-
-  if (cfun->has_simduid_loops)
-    note_simd_array_uses (&simd_array_to_simduid_htab);
-
-  set_stmt_vec_info_vec (NULL);
-
-  /*  ----------- Analyze loops. -----------  */
-
-  /* If some loop was duplicated, it gets bigger number
-     than all previously defined loops.  This fact allows us to run
-     only over initial loops skipping newly generated ones.  */
-  FOR_EACH_LOOP (loop, 0)
-    if (loop->dont_vectorize)
-      {
-	any_ifcvt_loops = true;
-	/* If-conversion sometimes versions both the outer loop
-	   (for the case when outer loop vectorization might be
-	   desirable) as well as the inner loop in the scalar version
-	   of the loop.  So we have:
-	    if (LOOP_VECTORIZED (1, 3))
-	      {
-		loop1
-		  loop2
-	      }
-	    else
-	      loop3 (copy of loop1)
-		if (LOOP_VECTORIZED (4, 5))
-		  loop4 (copy of loop2)
-		else
-		  loop5 (copy of loop4)
-	   If FOR_EACH_LOOP gives us loop3 first (which has
-	   dont_vectorize set), make sure to process loop1 before loop4;
-	   so that we can prevent vectorization of loop4 if loop1
-	   is successfully vectorized.  */
-	if (loop->inner)
-	  {
-	    gimple *loop_vectorized_call
-	      = vect_loop_vectorized_call (loop);
-	    if (loop_vectorized_call
-		&& vect_loop_vectorized_call (loop->inner))
-	      {
-		tree arg = gimple_call_arg (loop_vectorized_call, 0);
-		struct loop *vector_loop
-		  = get_loop (cfun, tree_to_shwi (arg));
-		if (vector_loop && vector_loop != loop)
-		  {
-		    loop = vector_loop;
-		    /* Make sure we don't vectorize it twice.  */
-		    loop->dont_vectorize = true;
-		    goto try_vectorize;
-		  }
-	      }
-	  }
-      }
-    else
-      {
-	loop_vec_info loop_vinfo, orig_loop_vinfo;
-	gimple *loop_vectorized_call, *loop_dist_alias_call;
-       try_vectorize:
-	if (!((flag_tree_loop_vectorize
-	       && optimize_loop_nest_for_speed_p (loop))
-	      || loop->force_vectorize))
-	  continue;
-	orig_loop_vinfo = NULL;
-	loop_vectorized_call = vect_loop_vectorized_call (loop);
-	loop_dist_alias_call = vect_loop_dist_alias_call (loop);
-       vectorize_epilogue:
   vect_location = find_loop_location (loop);
   if (LOCATION_LOCUS (vect_location) != UNKNOWN_LOCATION
       && dump_enabled_p ())
@@ -755,7 +676,7 @@ vectorize_loops (void)
 		 LOCATION_FILE (vect_location),
 		 LOCATION_LINE (vect_location));
 
-	loop_vinfo = vect_analyze_loop (loop, orig_loop_vinfo);
+  loop_vec_info loop_vinfo = vect_analyze_loop (loop, orig_loop_vinfo);
   loop->aux = loop_vinfo;
 
   if (!loop_vinfo || !LOOP_VINFO_VECTORIZABLE_P (loop_vinfo))
@@ -808,21 +729,16 @@ vectorize_loops (void)
 	 loop version.  */
       if (loop_vectorized_call && loop->inner)
 	loop->inner->dont_vectorize = true;
-	    continue;
+      return ret;
     }
 
   if (!dbg_cnt (vect_loop))
     {
-	    /* We may miss some if-converted loops due to
-	       debug counter.  Set any_ifcvt_loops to visit
-	       them at finalization.  */
-	    any_ifcvt_loops = true;
       /* Free existing information if loop is analyzed with some
 	 assumptions.  */
       if (loop_constraint_set_p (loop, LOOP_C_FINITE))
 	vect_free_loop_info_assumptions (loop);
-
-	    break;
+      return ret;
     }
 
   if (loop_vectorized_call)
@@ -838,8 +754,8 @@ vectorize_loops (void)
     dump_printf_loc (MSG_OPTIMIZED_LOCATIONS, vect_location,
 		     "loop vectorized using variable length vectors\n");
 
-	new_loop = vect_transform_loop (loop_vinfo);
-	num_vectorized_loops++;
+  loop_p new_loop = vect_transform_loop (loop_vinfo);
+  (*num_vectorized_loops)++;
   /* Now that the loop has been vectorized, allow it to be unrolled
      etc.  */
   loop->force_vectorize = false;
@@ -869,15 +785,111 @@ vectorize_loops (void)
       ret |= TODO_cleanup_cfg;
     }
 
-	if (new_loop)
-	  {
   /* Epilogue of vectorized loop must be vectorized too.  */
+  if (new_loop)
+    ret |= try_vectorize_loop_1 (simduid_to_vf_htab, num_vectorized_loops,
+				 new_loop, loop_vinfo, NULL, NULL);
+
+  return ret;
+}
+
+/* Try to vectorize LOOP.  */
+
+static unsigned
+try_vectorize_loop (hash_table<simduid_to_vf> *&simduid_to_vf_htab,
+		    unsigned *num_vectorized_loops, loop_p loop)
+{
+  if (!((flag_tree_loop_vectorize
+	 && optimize_loop_nest_for_speed_p (loop))
+	|| loop->force_vectorize))
+    return 0;
+
+  return try_vectorize_loop_1 (simduid_to_vf_htab, num_vectorized_loops,
+			       loop, NULL,
+			       vect_loop_vectorized_call (loop),
+			       vect_loop_dist_alias_call (loop));
+}
+
+
+/* Function vectorize_loops.
+
+   Entry point to loop vectorization phase.  */
+
+unsigned
+vectorize_loops (void)
+{
+  unsigned int i;
+  unsigned int num_vectorized_loops = 0;
+  unsigned int vect_loops_num;
+  struct loop *loop;
+  hash_table<simduid_to_vf> *simduid_to_vf_htab = NULL;
+  hash_table<simd_array_to_simduid> *simd_array_to_simduid_htab = NULL;
+  bool any_ifcvt_loops = false;
+  unsigned ret = 0;
+
   vect_loops_num = number_of_loops (cfun);
-	    loop = new_loop;
-	    orig_loop_vinfo = loop_vinfo;  /* To pass vect_analyze_loop.  */
-	    goto vectorize_epilogue;
+
+  /* Bail out if there are no loops.  */
+  if (vect_loops_num <= 1)
+    return 0;
+
+  if (cfun->has_simduid_loops)
+    note_simd_array_uses (&simd_array_to_simduid_htab);
+
+  set_stmt_vec_info_vec (NULL);
+
+  /*  ----------- Analyze loops. -----------  */
+
+  /* If some loop was duplicated, it gets bigger number
+     than all previously defined loops.  This fact allows us to run
+     only over initial loops skipping newly generated ones.  */
+  FOR_EACH_LOOP (loop, 0)
+    if (loop->dont_vectorize)
+      {
+	any_ifcvt_loops = true;
+	/* If-conversion sometimes versions both the outer loop
+	   (for the case when outer loop vectorization might be
+	   desirable) as well as the inner loop in the scalar version
+	   of the loop.  So we have:
+	    if (LOOP_VECTORIZED (1, 3))
+	      {
+		loop1
+		  loop2
+	      }
+	    else
+	      loop3 (copy of loop1)
+		if (LOOP_VECTORIZED (4, 5))
+		  loop4 (copy of loop2)
+		else
+		  loop5 (copy of loop4)
+	   If FOR_EACH_LOOP gives us loop3 first (which has
+	   dont_vectorize set), make sure to process loop1 before loop4;
+	   so that we can prevent vectorization of loop4 if loop1
+	   is successfully vectorized.  */
+	if (loop->inner)
+	  {
+	    gimple *loop_vectorized_call
+	      = vect_loop_vectorized_call (loop);
+	    if (loop_vectorized_call
+		&& vect_loop_vectorized_call (loop->inner))
+	      {
+		tree arg = gimple_call_arg (loop_vectorized_call, 0);
+		struct loop *vector_loop
+		  = get_loop (cfun, tree_to_shwi (arg));
+		if (vector_loop && vector_loop != loop)
+		  {
+		    /* Make sure we don't vectorize it twice.  */
+		    vector_loop->dont_vectorize = true;
+		    ret |= try_vectorize_loop (simduid_to_vf_htab,
+					       &num_vectorized_loops,
+					       vector_loop);
+		  }
+	      }
 	  }
       }
+    else
+      ret |= try_vectorize_loop (simduid_to_vf_htab, &num_vectorized_loops,
+				 loop);
 
   vect_location = UNKNOWN_LOCATION;
 
@@ -914,17 +926,15 @@ vectorize_loops (void)
 	  }
       }
 
-  for (i = 1; i < vect_loops_num; i++)
+  for (i = 1; i < number_of_loops (cfun); i++)
     {
       loop_vec_info loop_vinfo;
       bool has_mask_store;
 
       loop = get_loop (cfun, i);
-      if (!loop)
+      if (!loop || !loop->aux)
 	continue;
       loop_vinfo = (loop_vec_info) loop->aux;
-      has_mask_store = false;
-      if (loop_vinfo)
       has_mask_store = LOOP_VINFO_HAS_MASK_STORE (loop_vinfo);
       delete loop_vinfo;
       if (has_mask_store