tree-vectorizer.c (vect_is_simple_use): Support induction.

2007-02-06  Dorit Nuzman  <dorit@il.ibm.com>
            Victor Kaplansky  <victork@il.ibm.com>

        * tree-vectorizer.c (vect_is_simple_use): Support induction.
        (vect_is_simple_reduction): Support reduction with induction as
        one of the operands.
        (vect_is_simple_iv_evolution): Fix formatting.
        * tree-vect-analyze.c (vect_mark_stmts_to_be_vectorized): Fix
        formatting.  Don't mark induction phis for vectorization.
        (vect_analyze_scalar_cycles): Analyze all inductions, then reductions.
        * tree-vect-transform.c (get_initial_def_for_induction): New function.
        (vect_get_vec_def_for_operand): Support induction.
        (vect_get_vec_def_for_stmt_copy): Fix formatting and add check for
        induction case.
        (vectorizable_reduction): Support reduction with induction as one of
        the operands.
        (vectorizable_type_demotion): Use def-type of stmt argument rather
        than dummy def-type.

        * tree-ssa-loop.c (gate_scev_const_prop): Return the value of
        flag_tree_scev_cprop.
        * common.opt (tree-scev-cprop): New flag.

        * tree-vect-transform.c (vect_create_destination_var): Use 'kind' in
        call to vect_get_new_vect_var.


Co-Authored-By: Victor Kaplansky <victork@il.ibm.com>

From-SVN: r121643

gcc/ChangeLog

+2007-02-06  Dorit Nuzman  <dorit@il.ibm.com>
+	    Victor Kaplansky  <victork@il.ibm.com>
+
+	* tree-vectorizer.c (vect_is_simple_use): Support induction.
+	(vect_is_simple_reduction): Support reduction with induction as
+	one of the operands.
+	(vect_is_simple_iv_evolution): Fix formatting.
+	* tree-vect-analyze.c (vect_mark_stmts_to_be_vectorized): Fix 
+	formatting.  Don't mark induction phis for vectorization.
+	(vect_analyze_scalar_cycles): Analyze all inductions, then reductions.
+	* tree-vect-transform.c (get_initial_def_for_induction): New function.
+	(vect_get_vec_def_for_operand): Support induction.
+	(vect_get_vec_def_for_stmt_copy): Fix formatting and add check for
+	induction case.
+	(vectorizable_reduction): Support reduction with induction as one of 
+	the operands. 
+	(vectorizable_type_demotion): Use def-type of stmt argument rather
+	than dummy def-type.
+
+	* tree-ssa-loop.c (gate_scev_const_prop): Return the value of
+	flag_tree_scev_cprop.
+	* common.opt (tree-scev-cprop): New flag.
+
+	* tree-vect-transform.c (vect_create_destination_var): Use 'kind' in
+	call to vect_get_new_vect_var.
+
 2007-02-06  Ira Rosen  <irar@il.ibm.com>
 
 	* tree-vect-patterns.c (vect_recog_widen_mult_pattern): Check that 

gcc/common.opt

@@ -1080,6 +1080,10 @@ ftree-vectorizer-verbose=
 Common RejectNegative Joined
 -ftree-vectorizer-verbose=<number>	Set the verbosity level of the vectorizer
 
+ftree-scev-cprop
+Common Report Var(flag_tree_scev_cprop) Init(1)
+Enable copy propagation of scalar-evolution information.
+
 ; -fverbose-asm causes extra commentary information to be produced in
 ; the generated assembly code (to make it more readable).  This option
 ; is generally only of use to those who actually need to read the

gcc/testsuite/ChangeLog

+2007-02-06  Dorit Nuzman  <dorit@il.ibm.com>
+
+	* gcc.dg/vect/vect.exp: Add support for -fno-tree-scev-cprop tests.
+	* gcc.dg/vect/vect-iv-1.c: New test.
+	* gcc.dg/vect/vect-iv-2.c: New test.
+	* gcc.dg/vect/vect-iv-3.c: New test.
+	* gcc.dg/vect/vect-iv-4.c: New test.
+	* gcc.dg/vect/vect-iv-5.c: New test.
+	* gcc.dg/vect/vect-iv-6.c: New test.
+	* gcc.dg/vect/vect-iv-7.c: New test.
+	* gcc.dg/vect/vect-iv-8.c: New test.
+	* gcc.dg/vect/vect-iv-9.c: New test.
+	* gcc.dg/vect/vect-iv-10.c: New test.
+	* gcc.dg/vect/vect-iv-11.c: New test.
+	* gcc.dg/vect/no-tree-scev-cprop-vect-iv-1.c: New test.
+	* gcc.dg/vect/no-tree-scev-cprop-vect-iv-2.c: New test.
+	* gcc.dg/vect/vect-14.c: Now vectorizable.
+	* gcc.dg/vect/pr21591.c: Additional loop vectorized (initilization loop).
+	* gcc.dg/vect/vect-27.c: Likewise.
+	* gcc.dg/vect/vect-29.c Likewise.
+	* gcc.dg/vect/vect-dv-2.c: Likewise.
+	* gcc.dg/vect/vect-reduc-dot-u16a.c: Likewise.
+	* gcc.dg/vect/vect-reduc-dot-u16b.c: Likewise.
+	* gcc.dg/vect/vect-widen-mult-u16.c: Likewise.
+
 2007-02-06  Ira Rosen  <irar@il.ibm.com> 
 
 	* gcc.dg/vect/fast-math-vect-pow-2.c: New test.

gcc/testsuite/gcc.dg/vect/no-tree-scev-cprop-vect-iv-1.c

+/* { dg-require-effective-target vect_int } */
+
+#include <stdarg.h>
+#include "tree-vect.h"
+
+#define N 26
+ 
+int main1 (int X)
+{  
+  int s = X;
+  int i;
+
+  /* vectorization of reduction with induction. 
+     Need -fno-tree-scev-cprop or else the loop is eliminated.  */
+  for (i = 0; i < N; i++)
+    s += i;
+
+  return s;
+}
+
+int main (void)
+{ 
+  int s;
+  check_vect ();
+  
+  s = main1 (3);
+  if (s != 328)
+    abort ();
+
+  return 0;
+} 
+
+/* { dg-final { scan-tree-dump-times "vectorized 1 loops" 1 "vect" } } */
+/* { dg-final { cleanup-tree-dump "vect" } } */

gcc/testsuite/gcc.dg/vect/no-tree-scev-cprop-vect-iv-2.c

+/* { dg-require-effective-target vect_int } */
+
+#include <stdarg.h>
+#include "tree-vect.h"
+
+#define N 16
+ 
+int main1 ()
+{  
+  int arr1[N];
+  int k = 0;
+  int m = 3, i = 0;
+  
+  /* Vectorization of induction that is used after the loop.  
+     Currently vectorizable because scev_ccp disconnects the
+     use-after-the-loop from the iv def inside the loop.  */
+
+   do { 
+        k = k + 2;
+        arr1[i] = k;
+	m = m + k;
+	i++;
+   } while (i < N);
+
+  /* check results:  */
+  for (i = 0; i < N; i++)
+    { 
+      if (arr1[i] != 2+2*i)
+        abort ();
+    }
+
+  return m + k;
+}
+
+int main (void)
+{ 
+  int res;
+
+  check_vect ();
+  
+  res = main1 ();
+  if (res != 32 + 275)
+    abort ();
+
+  return 0;
+} 
+
+/* { dg-final { scan-tree-dump-times "vectorized 1 loops" 1 "vect" { xfail *-*-* } } } */
+/* { dg-final { cleanup-tree-dump "vect" } } */

gcc/testsuite/gcc.dg/vect/pr21591.c

@@ -30,6 +30,6 @@ void f(void)
    free(c);
 }
 
-/* { dg-final { scan-tree-dump-times "vectorized 1 loops" 1 "vect" } } */
+/* { dg-final { scan-tree-dump-times "vectorized 2 loops" 1 "vect" } } */
 /* { dg-final { cleanup-tree-dump "vect" } } */
 

gcc/testsuite/gcc.dg/vect/vect-14.c

@@ -10,8 +10,7 @@ int main1 ()
   int i;
   int ia[N];
 
-
-  /* Not vectorizable yet (induction).  */
+  /* Induction.  */
   for ( i = 0; i < N; i++) {
     ia[i] = i;
   }
@@ -33,5 +32,5 @@ int main (void)
   return main1 ();
 }
 
-/* { dg-final { scan-tree-dump-times "vectorized 1 loops" 1 "vect" { xfail *-*-* } } } */
+/* { dg-final { scan-tree-dump-times "vectorized 1 loops" 1 "vect" } } */
 /* { dg-final { cleanup-tree-dump "vect" } } */

gcc/testsuite/gcc.dg/vect/vect-27.c

@@ -40,7 +40,9 @@ int main (void)
   return main1 ();
 }
 
-/* { dg-final { scan-tree-dump-times "vectorized 1 loops" 1 "vect" { xfail vect_no_align } } } */
+/* The initialization induction loop (with aligned access) is also vectorized.  */
+/* { dg-final { scan-tree-dump-times "vectorized 2 loops" 1 "vect" { xfail vect_no_align } } } */
+/* { dg-final { scan-tree-dump-times "vectorized 1 loops" 1 "vect" { target vect_no_align } } } */
 /* { dg-final { scan-tree-dump-times "Vectorizing an unaligned access" 1 "vect" { xfail vect_no_align } } } */
 /* { dg-final { scan-tree-dump-times "Alignment of access forced using peeling" 0 "vect" } } */
 /* { dg-final { cleanup-tree-dump "vect" } } */

gcc/testsuite/gcc.dg/vect/vect-29.c

@@ -46,7 +46,8 @@ int main (void)
 /* For targets that don't support misaligned loads we version for the load.
    (The store is aligned).  */
 
-/* { dg-final { scan-tree-dump-times "vectorized 1 loops" 1 "vect" } } */
+/* The initialization induction loop (with aligned access) is also vectorized.  */
+/* { dg-final { scan-tree-dump-times "vectorized 2 loops" 1 "vect" } } */
 /* { dg-final { scan-tree-dump-times "Vectorizing an unaligned access" 1 "vect" { xfail vect_no_align } } } */
 /* { dg-final { scan-tree-dump-times "Alignment of access forced using peeling" 0 "vect" } } */
 /* { dg-final { scan-tree-dump-times "Alignment of access forced using versioning." 1 "vect" {target vect_no_align } } } */

gcc/testsuite/gcc.dg/vect/vect-dv-2.c

@@ -71,7 +71,7 @@ int main ()
 }
 
 
-
-/* { dg-final { scan-tree-dump-times "vectorized 2 loops" 1 "vect" } } */
+/* The initialization induction loop (with aligned access) is also vectorized.  */
+/* { dg-final { scan-tree-dump-times "vectorized 3 loops" 1 "vect" } } */
 /* { dg-final { scan-tree-dump-times "accesses have the same alignment." 2 "vect" } } */
 /* { dg-final { cleanup-tree-dump "vect" } } */

gcc/testsuite/gcc.dg/vect/vect-iv-1.c

+/* { dg-require-effective-target vect_int } */
+
+#include <stdarg.h>
+#include "tree-vect.h"
+
+#define N 16
+int result[N] = {8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30, 32, 34, 36, 38};
+ 
+int main1 (int X)
+{  
+  int arr[N];
+  int k = X;
+  int m, i=0;
+  
+   /* vectorization of induction.  */
+
+   do { 
+        m = k + 5;
+        arr[i] = m;
+        k = k + 2;
+	i++;
+   } while (i < N);
+
+  /* check results:  */
+  for (i = 0; i < N; i++)
+    {
+      if (arr[i] != result[i])
+	abort ();
+    }
+
+  return 0;
+}
+
+int main (void)
+{ 
+  check_vect ();
+  
+  return main1 (3);
+} 
+
+/* { dg-final { scan-tree-dump-times "vectorized 1 loops" 1 "vect" } } */
+/* { dg-final { cleanup-tree-dump "vect" } } */

gcc/testsuite/gcc.dg/vect/vect-iv-10.c

+/* { dg-require-effective-target vect_int } */
+
+#include <stdarg.h>
+#include "tree-vect.h"
+
+#define N 16
+
+int main1 ()
+{
+  int i,j;
+  int ia[N];
+
+  /* Induction.  */
+  for (j=0,i=N;  j<N,i>0;  i--,j++) {
+    ia[j] = i;
+  }
+
+  /* check results:  */
+  for (j=0,i=N;  j<N,i>0;  i--,j++) {
+      if (ia[j] != i)
+        abort ();
+    }
+
+  return 0;
+}
+
+int main (void)
+{ 
+  check_vect();
+  
+  return main1 ();
+}
+
+/* { dg-final { scan-tree-dump-times "vectorized 1 loops" 1 "vect" } } */
+/* { dg-final { cleanup-tree-dump "vect" } } */

gcc/testsuite/gcc.dg/vect/vect-iv-11.c

+/* { dg-require-effective-target vect_int } */
+
+#include <stdio.h>
+#include <stdarg.h>
+#include "tree-vect.h"
+
+int main1 (int len)
+{  
+  int s = 0;
+  int i = len;
+
+  /* vectorization of induction with reduction.  */
+  for ( ; i > 1; i -=2)
+    s += i;
+
+  return s;
+}
+
+int main (void)
+{ 
+  int s;
+  check_vect ();
+  
+  s = main1 (26);
+  if (s != 182)
+    abort ();
+
+  return 0;
+} 
+
+/* { dg-final { scan-tree-dump-times "vectorized 1 loops" 1 "vect" { xfail *-*-* } } } */
+/* { dg-final { cleanup-tree-dump "vect" } } */

gcc/testsuite/gcc.dg/vect/vect-iv-2.c

+/* { dg-require-effective-target vect_int } */
+
+#include <stdarg.h>
+#include "tree-vect.h"
+
+#define N 16
+ 
+int main1 ()
+{  
+  int arr1[N];
+  int k = 0;
+  int m = 3, i = 0;
+  
+  /* Vectorization of induction that is used after the loop.  
+     Currently vectorizable because scev_ccp disconnects the
+     use-after-the-loop from the iv def inside the loop.  */
+
+   do { 
+        k = k + 2;
+        arr1[i] = k;
+	m = m + k;
+	i++;
+   } while (i < N);
+
+  /* check results:  */
+  for (i = 0; i < N; i++)
+    { 
+      if (arr1[i] != 2+2*i)
+        abort ();
+    }
+
+  return m + k;
+}
+
+int main (void)
+{ 
+  int res;
+
+  check_vect ();
+  
+  res = main1 ();
+  if (res != 32 + 275)
+    abort ();
+
+  return 0;
+} 
+
+/* { dg-final { scan-tree-dump-times "vectorized 1 loops" 1 "vect" } } */
+/* { dg-final { cleanup-tree-dump "vect" } } */

gcc/testsuite/gcc.dg/vect/vect-iv-3.c

+/* { dg-require-effective-target vect_int } */
+
+#include <stdarg.h>
+#include "tree-vect.h"
+
+#define N 16
+ 
+int main1 ()
+{  
+  int arr1[N];
+  int arr2[N];
+  int k = 0;
+  int m = 3, i = 0;
+  
+  /* vectorization of induction. 
+     Peeling to align the store is also applied.  */
+
+   do { 
+        k = k + 2;
+        arr1[i] = k;
+	m = k + 3;
+	arr2[i] = m;
+	i++;
+   } while (i < N);
+
+  /* check results:  */
+  for (i = 0; i < N; i++)
+    { 
+      if (arr1[i] != 2+2*i || arr2[i] != 5 + 2*i)
+        abort ();
+    }
+
+  return 0;
+}
+
+int main (void)
+{ 
+  check_vect ();
+  
+  return main1 ();
+} 
+
+/* { dg-final { scan-tree-dump-times "vectorized 1 loops" 1 "vect" } } */
+/* { dg-final { cleanup-tree-dump "vect" } } */

gcc/testsuite/gcc.dg/vect/vect-iv-4.c

+/* { dg-require-effective-target vect_int } */
+
+#include <stdarg.h>
+#include "tree-vect.h"
+
+#define N 16
+ 
+int main1 ()
+{  
+  unsigned int arr1[N];
+  unsigned short arr2[N];
+  unsigned int k = 0;
+  unsigned short m = 3;
+  int i = 0;
+  
+  /* Vectorization of induction with multiple data types.  */
+
+   do { 
+        k = k + 2;
+        arr1[i] = k;
+	m = k + 3;
+	arr2[i] = m;
+	i++;
+   } while (i < N);
+
+  /* check results:  */
+  for (i = 0; i < N; i++)
+    { 
+      if (arr1[i] != 2+2*i || arr2[i] != 5 + 2*i)
+        abort ();
+    }
+
+  return 0;
+}
+
+int main (void)
+{ 
+  check_vect ();
+  
+  return main1 ();
+} 
+
+/* { dg-final { scan-tree-dump-times "vectorized 1 loops" 1 "vect" } } */
+/* { dg-final { cleanup-tree-dump "vect" } } */

gcc/testsuite/gcc.dg/vect/vect-iv-5.c

+/* { dg-require-effective-target vect_int } */
+
+#include <stdarg.h>
+#include "tree-vect.h"
+
+#define N 16
+ 
+int main1 ()
+{  
+  float arr[N];
+  float f = 1.0;
+  int i;
+  
+  /* Vectorization of fp induction.  */
+
+  for (i=0; i<N; i++)
+    {
+      arr[i] = f;
+      f += 2.0;
+    }
+
+  /* check results:  */
+  for (i = 0; i < N; i++)
+    {
+      if (arr[i] != 1.0 + 2.0*i)
+	abort ();
+    }
+
+  return 0;
+}
+
+int main (void)
+{ 
+  check_vect ();
+  
+  return main1 ();
+} 
+
+/* { dg-final { scan-tree-dump-times "vectorized 1 loops" 1 "vect" { xfail *-*-* } } } */
+/* { dg-final { cleanup-tree-dump "vect" } } */

gcc/testsuite/gcc.dg/vect/vect-iv-6.c

+/* { dg-require-effective-target vect_int } */
+#include <stdio.h>
+#include <stdarg.h>
+#include "tree-vect.h"
+
+#define N 16
+ 
+int main1 (int X)
+{  
+  int arr1[N+1];
+  int arr2[N+1];
+  int k = X;
+  int m, i=0;
+  
+   /* Vectorization of induction with non-constant initial condition X. 
+      Also we have here two uses of the induction-variable k as defined
+      by the loop-header phi (as opposed to the other uses of k that are
+      defined in the loop), in which case we exercise the fact that we
+      reuse the same vector def-use-cycle for both uses. 
+      Peeling to align the store is also applied.  */
+
+   do { 
+	arr2[i+1] = 2*k;
+        k = k + 2;
+        arr1[i+1] = k;
+        k = k + 4;
+	i++;
+   } while (i < N);
+
+  /* check results:  */
+  for (i = 0; i < N; i++)
+    {
+      if (arr1[i+1] != X+6*i+2
+	  || arr2[i+1] != 2*(X+6*i))
+	abort ();
+    }
+
+  return 0;
+}
+
+int main (void)
+{ 
+  check_vect ();
+  
+  return main1 (3);
+} 
+
+/* { dg-final { scan-tree-dump-times "vectorized 1 loops" 1 "vect" { xfail vect_no_align } } } */
+/* { dg-final { cleanup-tree-dump "vect" } } */

gcc/testsuite/gcc.dg/vect/vect-iv-7.c

+/* { dg-require-effective-target vect_int } */
+
+#include <stdarg.h>
+#include "tree-vect.h"
+
+#define N 16
+int result[N] = {8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30, 32, 34, 36, 38};
+ 
+int main1 (int X)
+{  
+  int arr[N];
+  int k = 3;
+  int m, i=0;
+  
+   /* Vectorization of induction with non-constant step X.  */
+
+   do { 
+        m = k + 5;
+        arr[i] = m;
+        k = k + X;
+	i++;
+   } while (i < N);
+
+  /* check results:  */
+  for (i = 0; i < N; i++)
+    {
+      if (arr[i] != result[i])
+	abort ();
+    }
+
+  return 0;
+}
+
+int main (void)
+{ 
+  check_vect ();
+  
+  return main1 (2);
+} 
+
+/* { dg-final { scan-tree-dump-times "vectorized 1 loops" 1 "vect" { xfail *-*-* } } } */
+/* { dg-final { cleanup-tree-dump "vect" } } */

gcc/testsuite/gcc.dg/vect/vect-iv-8.c

+/* { dg-require-effective-target vect_int } */
+
+#include <stdarg.h>
+#include "tree-vect.h"
+
+#define N 26
+ 
+int main1 (short X)
+{  
+  unsigned char a[N];
+  unsigned short b[N];
+  unsigned int c[N];
+  int i;
+
+  /* vectorization of induction with type conversions.  */
+  for (i = 0; i < N; i++)
+  {
+    a[i] = (unsigned char)X;
+    b[i] = X;
+    c[i] = (unsigned int)X;
+  }
+
+  /* check results:  */
+  for (i = 0; i < N; i++)
+    {
+      if (a[i] != (char)X || b[i] != X || c[i] != (int)X)
+	abort ();
+    }
+
+  return 0;
+}
+
+int main (void)
+{ 
+  check_vect ();
+  
+  return main1 (3);
+} 
+
+/* { dg-final { scan-tree-dump-times "vectorized 1 loops" 1 "vect" { target { vect_pack_mod && vect_unpack } } } } */
+/* { dg-final { cleanup-tree-dump "vect" } } */

gcc/testsuite/gcc.dg/vect/vect-iv-9.c

+/* { dg-require-effective-target vect_int } */
+
+#include <stdarg.h>
+#include "tree-vect.h"
+
+#define N 26
+int a[N];
+ 
+int main1 (int X)
+{  
+  int s = X;
+  int i;
+
+  /* vectorization of reduction with induction.  */
+  for (i = 0; i < N; i++)
+    s += (i + a[i]);
+
+  return s;
+}
+
+int main (void)
+{ 
+  int s, i;
+  check_vect ();
+  
+  for (i = 0; i < N; i++)
+    a[i] = 2*i;
+
+  s = main1 (3);
+  if (s != 978)
+    abort ();
+
+  return 0;
+} 
+
+/* { dg-final { scan-tree-dump-times "vectorized 1 loops" 2 "vect" } } */
+/* { dg-final { cleanup-tree-dump "vect" } } */

gcc/testsuite/gcc.dg/vect/vect-reduc-dot-u16a.c

@@ -46,7 +46,7 @@ int main (void)
   return 0;
 }
 
+/* The initialization loop in main also gets vectorized.  */
 /* { dg-final { scan-tree-dump-times "vect_recog_dot_prod_pattern: detected" 1 "vect" { xfail *-*-* } } } */
-/* { dg-final { scan-tree-dump-times "vectorized 1 loops" 1 "vect" { target { vect_short_mult && vect_widen_sum_hi_to_si } } } } */ 
-
+/* { dg-final { scan-tree-dump-times "vectorized 1 loops" 2 "vect" { target { vect_short_mult && vect_widen_sum_hi_to_si } } } } */ 
 /* { dg-final { cleanup-tree-dump "vect" } } */

gcc/testsuite/gcc.dg/vect/vect-reduc-dot-u16b.c

@@ -51,6 +51,8 @@ int main (void)
 /* Once the dot-product pattern is detected, we expect
    that loop to be vectorized on vect_udot_hi targets (targets that support 
    dot-product of unsigned shorts) and targets that support widening multiplication.  */
-/* { dg-final { scan-tree-dump-times "vectorized 1 loops" 1 "vect" { xfail *-*-* } } } */ 
+/* The induction loop in main is vectorized.  */
+/* { dg-final { scan-tree-dump-times "vectorized 1 loops" 2 "vect" { xfail *-*-* } } } */ 
+/* { dg-final { scan-tree-dump-times "vectorized 1 loops" 1 "vect" } } */ 
 
 /* { dg-final { cleanup-tree-dump "vect" } } */

gcc/testsuite/gcc.dg/vect/vect-widen-mult-u16.c

@@ -42,6 +42,8 @@ int main (void)
   return 0;
 }
 
-/* { dg-final { scan-tree-dump-times "vectorized 1 loops" 1 "vect" { xfail *-*-* } } } */
+/*The induction loop is vectorized  */
+/* { dg-final { scan-tree-dump-times "vectorized 1 loops" 2 "vect" { xfail *-*-* } } } */
+/* { dg-final { scan-tree-dump-times "vectorized 1 loops" 1 "vect" } } */
 /* { dg-final { cleanup-tree-dump "vect" } } */
 

gcc/testsuite/gcc.dg/vect/vect.exp

@@ -156,6 +156,12 @@ lappend DEFAULT_VECTCFLAGS "-fno-trapping-math"
 dg-runtest [lsort [glob -nocomplain $srcdir/$subdir/no-trapping-math-*.\[cS\]]]  \
 	"" $DEFAULT_VECTCFLAGS
 
+# -fno-tree-scev-cprop
+set DEFAULT_VECTCFLAGS $SAVED_DEFAULT_VECTCFLAGS
+lappend DEFAULT_VECTCFLAGS "-fno-tree-scev-cprop"
+dg-runtest [lsort [glob -nocomplain $srcdir/$subdir/no-tree-scev-cprop-*.\[cS\]]]  \
+	"" $DEFAULT_VECTCFLAGS
+
 # With -Os
 lappend DEFAULT_VECTCFLAGS "-Os"
 dg-runtest [lsort [glob -nocomplain $srcdir/$subdir/Os-vect-*.\[cS\]]]  \

gcc/tree-ssa-loop.c

@@ -280,7 +280,7 @@ struct tree_opt_pass pass_iv_canon =
 static bool
 gate_scev_const_prop (void)
 {
-  return true;
+  return flag_tree_scev_cprop;
 }
 
 struct tree_opt_pass pass_scev_cprop =

gcc/tree-vect-analyze.c

@@ -509,58 +509,69 @@ vect_analyze_scalar_cycles (loop_vec_info loop_vinfo)
   tree phi;
   struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
   basic_block bb = loop->header;
-  tree dummy;
+  tree dumy;
+  VEC(tree,heap) *worklist = VEC_alloc (tree, heap, 64);
 
   if (vect_print_dump_info (REPORT_DETAILS))
     fprintf (vect_dump, "=== vect_analyze_scalar_cycles ===");
 
+  /* First - identify all inductions.  */
   for (phi = phi_nodes (bb); phi; phi = PHI_CHAIN (phi))
     {
       tree access_fn = NULL;
       tree def = PHI_RESULT (phi);
       stmt_vec_info stmt_vinfo = vinfo_for_stmt (phi);
-      tree reduc_stmt;
 
       if (vect_print_dump_info (REPORT_DETAILS))
 	{
-          fprintf (vect_dump, "Analyze phi: ");
-          print_generic_expr (vect_dump, phi, TDF_SLIM);
+	  fprintf (vect_dump, "Analyze phi: ");
+	  print_generic_expr (vect_dump, phi, TDF_SLIM);
 	}
 
       /* Skip virtual phi's. The data dependences that are associated with
          virtual defs/uses (i.e., memory accesses) are analyzed elsewhere.  */
-
       if (!is_gimple_reg (SSA_NAME_VAR (def)))
-	{
-	  if (vect_print_dump_info (REPORT_DETAILS))
-	    fprintf (vect_dump, "virtual phi. skip.");
-	  continue;
-	}
+	continue;
 
       STMT_VINFO_DEF_TYPE (stmt_vinfo) = vect_unknown_def_type;
 
       /* Analyze the evolution function.  */
-
       access_fn = analyze_scalar_evolution (loop, def);
+      if (access_fn && vect_print_dump_info (REPORT_DETAILS))
+	{
+	  fprintf (vect_dump, "Access function of PHI: ");
+	  print_generic_expr (vect_dump, access_fn, TDF_SLIM);
+	}
 
-      if (!access_fn)
-	continue;
+      if (!access_fn
+	  || !vect_is_simple_iv_evolution (loop->num, access_fn, &dumy, &dumy)) 
+	{
+	  VEC_safe_push (tree, heap, worklist, phi);	  
+	  continue;
+	}
 
       if (vect_print_dump_info (REPORT_DETAILS))
-        {
-           fprintf (vect_dump, "Access function of PHI: ");
-           print_generic_expr (vect_dump, access_fn, TDF_SLIM);
-        }
+	fprintf (vect_dump, "Detected induction.");
+      STMT_VINFO_DEF_TYPE (stmt_vinfo) = vect_induction_def;
+    }
 
-      if (vect_is_simple_iv_evolution (loop->num, access_fn, &dummy, &dummy))
-	{
-	  if (vect_print_dump_info (REPORT_DETAILS))
-	    fprintf (vect_dump, "Detected induction.");
-	  STMT_VINFO_DEF_TYPE (stmt_vinfo) = vect_induction_def;
-          continue;
-	}
 
-      /* TODO: handle invariant phis  */
+  /* Second - identify all reductions.  */
+  while (VEC_length (tree, worklist) > 0)
+    {
+      tree phi = VEC_pop (tree, worklist);
+      tree def = PHI_RESULT (phi);
+      stmt_vec_info stmt_vinfo = vinfo_for_stmt (phi);
+      tree reduc_stmt;
+
+      if (vect_print_dump_info (REPORT_DETAILS))
+        { 
+          fprintf (vect_dump, "Analyze phi: ");
+          print_generic_expr (vect_dump, phi, TDF_SLIM);
+        }
+
+      gcc_assert (is_gimple_reg (SSA_NAME_VAR (def)));
+      gcc_assert (STMT_VINFO_DEF_TYPE (stmt_vinfo) == vect_unknown_def_type);
 
       reduc_stmt = vect_is_simple_reduction (loop, phi);
       if (reduc_stmt)
@@ -574,9 +585,9 @@ vect_analyze_scalar_cycles (loop_vec_info loop_vinfo)
       else
         if (vect_print_dump_info (REPORT_DETAILS))
           fprintf (vect_dump, "Unknown def-use cycle pattern.");
-
     }
 
+  VEC_free (tree, heap, worklist);
   return;
 }
 
@@ -1908,7 +1919,7 @@ vect_analyze_data_refs (loop_vec_info loop_vinfo)
   tree scalar_type;
 
   if (vect_print_dump_info (REPORT_DETAILS))
-    fprintf (vect_dump, "=== vect_analyze_data_refs ===");
+    fprintf (vect_dump, "=== vect_analyze_data_refs ===\n");
 
   compute_data_dependences_for_loop (loop, true,
                                      &LOOP_VINFO_DATAREFS (loop_vinfo),
@@ -1933,7 +1944,7 @@ vect_analyze_data_refs (loop_vec_info loop_vinfo)
       /* Update DR field in stmt_vec_info struct.  */
       stmt = DR_STMT (dr);
       stmt_info = vinfo_for_stmt (stmt);
-  
+
       if (STMT_VINFO_DATA_REF (stmt_info))
         {
           if (vect_print_dump_info (REPORT_UNVECTORIZED_LOOPS))
@@ -2249,14 +2260,21 @@ vect_mark_stmts_to_be_vectorized (loop_vec_info loop_vinfo)
 
       /* case 2.2:  */
       if (STMT_VINFO_DEF_TYPE (stmt_vinfo) == vect_reduction_def)
-        {
-          gcc_assert (relevant == vect_unused_in_loop && live_p);
-          relevant = vect_used_by_reduction;
-          live_p = false;
-        }
+	{
+	  gcc_assert (relevant == vect_unused_in_loop && live_p);
+	  relevant = vect_used_by_reduction;
+	  live_p = false;
+	}
 
+      i = 0;
       FOR_EACH_SSA_TREE_OPERAND (use, stmt, iter, SSA_OP_USE)
 	{
+	  if (vect_print_dump_info (REPORT_DETAILS))
+	    {
+	      fprintf (vect_dump, "worklist: examine use %d: ", i++);
+	      print_generic_expr (vect_dump, use, TDF_SLIM);
+	    }
+
 	  /* case 1: we are only interested in uses that need to be vectorized. 
 	     Uses that are used for address computation are not considered 
 	     relevant.
@@ -2265,25 +2283,19 @@ vect_mark_stmts_to_be_vectorized (loop_vec_info loop_vinfo)
 	    continue;
 
 	  if (!vect_is_simple_use (use, loop_vinfo, &def_stmt, &def, &dt))
-            {
-              if (vect_print_dump_info (REPORT_UNVECTORIZED_LOOPS))
-                fprintf (vect_dump, "not vectorized: unsupported use in stmt.");
+	    {
+	      if (vect_print_dump_info (REPORT_UNVECTORIZED_LOOPS))
+		fprintf (vect_dump, "not vectorized: unsupported use in stmt.");
 	      VEC_free (tree, heap, worklist);
-              return false;
+	      return false;
             }
 
 	  if (!def_stmt || IS_EMPTY_STMT (def_stmt))
 	    continue;
 
-          if (vect_print_dump_info (REPORT_DETAILS))
-            {
-              fprintf (vect_dump, "worklist: examine use %d: ", i);
-              print_generic_expr (vect_dump, use, TDF_SLIM);
-            }
-
 	  bb = bb_for_stmt (def_stmt);
-          if (!flow_bb_inside_loop_p (loop, bb))
-            continue;
+	  if (!flow_bb_inside_loop_p (loop, bb))
+	    continue;
 
 	  /* case 2.1: the reduction-use does not mark the defining-phi
 	     as relevant.  */
@@ -2291,6 +2303,9 @@ vect_mark_stmts_to_be_vectorized (loop_vec_info loop_vinfo)
 	      && TREE_CODE (def_stmt) == PHI_NODE)
 	    continue;
 
+	  if (dt == vect_induction_def && TREE_CODE (def_stmt) == PHI_NODE)
+	    continue;
+
 	  vect_mark_relevant (&worklist, def_stmt, relevant, live_p);
 	}
     }				/* while worklist */

gcc/tree-vect-transform.c

@@ -462,7 +462,7 @@ vect_create_destination_var (tree scalar_dest, tree vectype)
   new_name = get_name (scalar_dest);
   if (!new_name)
     new_name = "var_";
-  vec_dest = vect_get_new_vect_var (type, vect_simple_var, new_name);
+  vec_dest = vect_get_new_vect_var (type, kind, new_name);
   add_referenced_var (vec_dest);
 
   return vec_dest;
@@ -510,6 +510,189 @@ vect_init_vector (tree stmt, tree vector_var, tree vector_type)
 }
 
 
+/* Function get_initial_def_for_induction
+
+   Input:
+   STMT - a stmt that performs an induction operation in the loop.
+   IV_PHI - the initial value of the induction variable
+
+   Output:
+   Return a vector variable, initialized with the first VF values of
+   the induction variable. E.g., for an iv with IV_PHI='X' and
+   evolution S, for a vector of 4 units, we want to return: 
+   [X, X + S, X + 2*S, X + 3*S].  */
+
+static tree
+get_initial_def_for_induction (tree stmt, tree iv_phi)
+{
+  stmt_vec_info stmt_vinfo = vinfo_for_stmt (stmt);
+  loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_vinfo);
+  struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
+  tree scalar_type = TREE_TYPE (iv_phi);
+  tree vectype = get_vectype_for_scalar_type (scalar_type);
+  int nunits = GET_MODE_NUNITS (TYPE_MODE (vectype));
+  edge pe = loop_preheader_edge (loop);
+  basic_block new_bb;
+  block_stmt_iterator bsi;
+  tree vec, vec_init, vec_step, t;
+  tree access_fn;
+  tree new_var;
+  tree new_name;
+  tree init_stmt;
+  tree induction_phi, induc_def, new_stmt, vec_def, vec_dest;
+  tree init_expr, step_expr;
+  int vf = LOOP_VINFO_VECT_FACTOR (loop_vinfo);
+  int i;
+  bool ok;
+  int ncopies = vf / nunits;
+  tree expr;
+  stmt_vec_info phi_info = vinfo_for_stmt (iv_phi);
+
+  gcc_assert (phi_info);
+
+  if (STMT_VINFO_VEC_STMT (phi_info))
+    {
+      induction_phi = STMT_VINFO_VEC_STMT (phi_info);
+      gcc_assert (TREE_CODE (induction_phi) == PHI_NODE);
+
+      if (vect_print_dump_info (REPORT_DETAILS))
+	{
+	  fprintf (vect_dump, "induction already vectorized:");
+	  print_generic_expr (vect_dump, iv_phi, TDF_SLIM);
+	  fprintf (vect_dump, "\n");
+	  print_generic_expr (vect_dump, induction_phi, TDF_SLIM);
+	}
+
+      return PHI_RESULT (induction_phi);
+    }
+
+  gcc_assert (ncopies >= 1);
+ 
+  access_fn = analyze_scalar_evolution (loop, PHI_RESULT (iv_phi));
+  gcc_assert (access_fn);
+  ok = vect_is_simple_iv_evolution (loop->num, access_fn, &init_expr, &step_expr);
+  gcc_assert (ok);
+
+  /* Create the vector that holds the initial_value of the induction.  */
+  new_name = init_expr;
+  t = NULL_TREE;
+  t = tree_cons (NULL_TREE, init_expr, t);
+  for (i = 1; i < nunits; i++)
+    {
+      /* Create: new_name = new_name + step_expr  */
+      new_var = vect_get_new_vect_var (scalar_type, vect_scalar_var, "var_");
+      add_referenced_var (new_var);
+      init_stmt = build2 (GIMPLE_MODIFY_STMT, void_type_node, new_var,
+                          fold_build2 (PLUS_EXPR, scalar_type, new_name, step_expr));
+      new_name = make_ssa_name (new_var, init_stmt);
+      GIMPLE_STMT_OPERAND (init_stmt, 0) = new_name;
+
+      new_bb = bsi_insert_on_edge_immediate (pe, init_stmt);
+      gcc_assert (!new_bb);
+
+      if (vect_print_dump_info (REPORT_DETAILS))
+        {
+          fprintf (vect_dump, "created new init_stmt: ");
+          print_generic_expr (vect_dump, init_stmt, TDF_SLIM);
+        }
+      t = tree_cons (NULL_TREE, new_name, t);
+    }
+  vec = build_constructor_from_list (vectype, nreverse (t));
+  vec_init = vect_init_vector (stmt, vec, vectype);
+
+
+  /* Create the vector that holds the step of the induction.  */
+  expr = build_int_cst (scalar_type, vf);
+  new_name = fold_build2 (MULT_EXPR, scalar_type, expr, step_expr);
+  t = NULL_TREE;
+  for (i = 0; i < nunits; i++)
+    t = tree_cons (NULL_TREE, unshare_expr (new_name), t);
+  vec = build_constructor_from_list (vectype, t);
+  vec_step = vect_init_vector (stmt, vec, vectype);
+
+
+  /* Create the following def-use cycle:
+     loop prolog:
+         vec_init = [X, X+S, X+2*S, X+3*S]
+	 vec_step = [VF*S, VF*S, VF*S, VF*S]
+     loop:
+         vec_iv = PHI <vec_init, vec_loop>
+         ...
+         STMT
+         ...
+         vec_loop = vec_iv + vec_step;  */
+
+  /* Create the induction-phi that defines the induction-operand.  */
+  vec_dest = vect_get_new_vect_var (vectype, vect_simple_var, "vec_iv_");
+  add_referenced_var (vec_dest);
+  induction_phi = create_phi_node (vec_dest, loop->header);
+  set_stmt_info (get_stmt_ann (induction_phi),
+                 new_stmt_vec_info (induction_phi, loop_vinfo));
+  induc_def = PHI_RESULT (induction_phi);
+
+  /* Create the iv update inside the loop  */
+  new_stmt = build2 (GIMPLE_MODIFY_STMT, void_type_node, NULL_TREE,
+                     build2 (PLUS_EXPR, vectype, induc_def, vec_step));
+  vec_def = make_ssa_name (vec_dest, new_stmt);
+  GIMPLE_STMT_OPERAND (new_stmt, 0) = vec_def;
+  bsi = bsi_for_stmt (stmt);
+  vect_finish_stmt_generation (stmt, new_stmt, &bsi);
+
+  /* Set the arguments of the phi node:  */
+  add_phi_arg (induction_phi, vec_init, loop_preheader_edge (loop));
+  add_phi_arg (induction_phi, vec_def, loop_latch_edge (loop));
+
+
+  /* In case the vectorization factor (VF) is bigger than the number
+     of elements that we can fit in a vectype (nunits), we have to generate
+     more than one vector stmt - i.e - we need to "unroll" the
+     vector stmt by a factor VF/nunits.  For more details see documentation
+     in vectorizable_operation.  */
+  
+  if (ncopies > 1)
+    {
+      stmt_vec_info prev_stmt_vinfo;
+
+      /* Create the vector that holds the step of the induction.  */
+      expr = build_int_cst (scalar_type, nunits);
+      new_name = fold_build2 (MULT_EXPR, scalar_type, expr, step_expr);
+      t = NULL_TREE;
+      for (i = 0; i < nunits; i++)
+	t = tree_cons (NULL_TREE, unshare_expr (new_name), t);
+      vec = build_constructor_from_list (vectype, t);
+      vec_step = vect_init_vector (stmt, vec, vectype);
+
+      vec_def = induc_def;
+      prev_stmt_vinfo = vinfo_for_stmt (induction_phi);
+      for (i = 1; i < ncopies; i++)
+	{
+	  /* vec_i = vec_prev + vec_{step*nunits}  */
+			 
+	  new_stmt = build2 (GIMPLE_MODIFY_STMT, void_type_node, NULL_TREE,
+			build2 (PLUS_EXPR, vectype, vec_def, vec_step));
+	  vec_def = make_ssa_name (vec_dest, new_stmt);
+	  GIMPLE_STMT_OPERAND (new_stmt, 0) = vec_def;
+	  bsi = bsi_for_stmt (stmt);
+	  vect_finish_stmt_generation (stmt, new_stmt, &bsi);
+
+	  STMT_VINFO_RELATED_STMT (prev_stmt_vinfo) = new_stmt;
+	  prev_stmt_vinfo = vinfo_for_stmt (new_stmt); 
+	}
+    }
+
+  if (vect_print_dump_info (REPORT_DETAILS))
+    {
+      fprintf (vect_dump, "transform induction: created def-use cycle:");
+      print_generic_expr (vect_dump, induction_phi, TDF_SLIM);
+      fprintf (vect_dump, "\n");
+      print_generic_expr (vect_dump, SSA_NAME_DEF_STMT (vec_def), TDF_SLIM);
+    }
+
+  STMT_VINFO_VEC_STMT (phi_info) = induction_phi;
+  return induc_def;
+}
+
+
 /* Function vect_get_vec_def_for_operand.
 
    OP is an operand in STMT. This function returns a (vector) def that will be
@@ -634,9 +817,10 @@ vect_get_vec_def_for_operand (tree op, tree stmt, tree *scalar_def)
     /* Case 5: operand is defined by loop-header phi - induction.  */
     case vect_induction_def:
       {
-        if (vect_print_dump_info (REPORT_DETAILS))
-          fprintf (vect_dump, "induction - unsupported.");
-        internal_error ("no support for induction"); /* FORNOW */
+	gcc_assert (TREE_CODE (def_stmt) == PHI_NODE);
+
+	/* Get the def before the loop  */
+	return get_initial_def_for_induction (stmt, def_stmt);
       }
 
     default:
@@ -707,14 +891,14 @@ vect_get_vec_def_for_stmt_copy (enum vect_def_type dt, tree vec_oprnd)
   tree vec_stmt_for_operand;
   stmt_vec_info def_stmt_info;
 
-  if (dt == vect_invariant_def || dt == vect_constant_def)
-    {
-      /* Do nothing; can reuse same def.  */ ;
-      return vec_oprnd;
-    }
+  /* Do nothing; can reuse same def.  */
+  if (dt == vect_invariant_def || dt == vect_constant_def )
+    return vec_oprnd;
 
   vec_stmt_for_operand = SSA_NAME_DEF_STMT (vec_oprnd);
   def_stmt_info = vinfo_for_stmt (vec_stmt_for_operand);
+  if (dt == vect_induction_def)
+    gcc_assert (TREE_CODE (vec_stmt_for_operand) == PHI_NODE);  
   gcc_assert (def_stmt_info);
   vec_stmt_for_operand = STMT_VINFO_RELATED_STMT (def_stmt_info);
   gcc_assert (vec_stmt_for_operand);
@@ -1386,8 +1570,11 @@ vectorizable_reduction (tree stmt, block_stmt_iterator *bsi, tree *vec_stmt)
       op = TREE_OPERAND (operation, i);
       is_simple_use = vect_is_simple_use (op, loop_vinfo, &def_stmt, &def, &dt);
       gcc_assert (is_simple_use);
-      gcc_assert (dt == vect_loop_def || dt == vect_invariant_def ||
-                  dt == vect_constant_def);
+      if (dt != vect_loop_def
+	  && dt != vect_invariant_def
+	  && dt != vect_constant_def
+	  && dt != vect_induction_def)
+	return false;
     }
 
   op = TREE_OPERAND (operation, i);
@@ -2260,9 +2447,8 @@ vectorizable_type_demotion (tree stmt, block_stmt_iterator *bsi,
       /* Handle uses.  */
       if (j == 0)
 	{
-	  enum vect_def_type dt = vect_unknown_def_type; /* Dummy */
 	  vec_oprnd0 = vect_get_vec_def_for_operand (op0, stmt, NULL);
-	  vec_oprnd1 = vect_get_vec_def_for_stmt_copy (dt, vec_oprnd0);
+	  vec_oprnd1 = vect_get_vec_def_for_stmt_copy (dt0, vec_oprnd0);
 	}
       else
 	{

gcc/tree-vectorizer.c

@@ -1745,13 +1745,6 @@ vect_is_simple_use (tree operand, loop_vec_info loop_vinfo, tree *def_stmt,
       return false;
     }
 
-  if (*dt == vect_induction_def)
-    {
-      if (vect_print_dump_info (REPORT_DETAILS))
-        fprintf (vect_dump, "induction not supported.");
-      return false;
-    }
-
   return true;
 }
 
@@ -2050,7 +2043,7 @@ vect_is_simple_reduction (struct loop *loop, tree phi)
    */
   def1 = SSA_NAME_DEF_STMT (op1);
   def2 = SSA_NAME_DEF_STMT (op2);
-  if (!def1 || !def2)
+  if (!def1 || !def2 || IS_EMPTY_STMT (def1) || IS_EMPTY_STMT (def2))
     {
       if (vect_print_dump_info (REPORT_DETAILS))
         {
@@ -2060,9 +2053,15 @@ vect_is_simple_reduction (struct loop *loop, tree phi)
       return NULL_TREE;
     }
 
-  if (TREE_CODE (def1) == GIMPLE_MODIFY_STMT
+
+  /* Check that one def is the reduction def, defined by PHI,
+     the other def is either defined in the loop by a GIMPLE_MODIFY_STMT,
+     or it's an induction (defined by some phi node).  */
+
+  if (def2 == phi
       && flow_bb_inside_loop_p (loop, bb_for_stmt (def1))
-      && def2 == phi)
+      && (TREE_CODE (def1) == GIMPLE_MODIFY_STMT 
+	  || STMT_VINFO_DEF_TYPE (vinfo_for_stmt (def1)) == vect_induction_def))
     {
       if (vect_print_dump_info (REPORT_DETAILS))
         {
@@ -2071,9 +2070,10 @@ vect_is_simple_reduction (struct loop *loop, tree phi)
         }
       return def_stmt;
     }
-  else if (TREE_CODE (def2) == GIMPLE_MODIFY_STMT
-      && flow_bb_inside_loop_p (loop, bb_for_stmt (def2))
-      && def1 == phi)
+  else if (def1 == phi
+	   && flow_bb_inside_loop_p (loop, bb_for_stmt (def2))
+	   && (TREE_CODE (def2) == GIMPLE_MODIFY_STMT 
+	       || STMT_VINFO_DEF_TYPE (vinfo_for_stmt (def2)) == vect_induction_def))
     {
       /* Swap operands (just for simplicity - so that the rest of the code
 	 can assume that the reduction variable is always the last (second)
@@ -2110,7 +2110,6 @@ vect_is_simple_iv_evolution (unsigned loop_nb, tree access_fn, tree * init,
 {
   tree init_expr;
   tree step_expr;
-  
   tree evolution_part = evolution_part_in_loop_num (access_fn, loop_nb);
 
   /* When there is no evolution in this loop, the evolution function
@@ -2124,8 +2123,7 @@ vect_is_simple_iv_evolution (unsigned loop_nb, tree access_fn, tree * init,
     return false;
   
   step_expr = evolution_part;
-  init_expr = unshare_expr (initial_condition_in_loop_num (access_fn,
-                                                           loop_nb));
+  init_expr = unshare_expr (initial_condition_in_loop_num (access_fn, loop_nb));
 
   if (vect_print_dump_info (REPORT_DETAILS))
     {
@@ -2139,7 +2137,7 @@ vect_is_simple_iv_evolution (unsigned loop_nb, tree access_fn, tree * init,
   *step = step_expr;
 
   if (TREE_CODE (step_expr) != INTEGER_CST)
-    {
+    { 
       if (vect_print_dump_info (REPORT_DETAILS))
         fprintf (vect_dump, "step unknown.");
       return false;