Support for vectorizing conditional expressions

2015-10-23  Alan Hayward <alan.hayward@arm.com>

gcc/
	PR tree-optimization/65947
	* tree-vect-loop.c
	(vect_is_simple_reduction_1): Find condition reductions.
	(vect_model_reduction_cost): Add condition reduction costs.
	(get_initial_def_for_reduction): Add condition reduction initial var.
	(vect_create_epilog_for_reduction): Add condition reduction epilog.
	(vectorizable_reduction): Condition reduction support.
	* tree-vect-stmts.c (vectorizable_condition): Add vect reduction arg
	* doc/sourcebuild.texi (Vector-specific attributes): Document
	vect_max_reduc

gcc/testsuite
	PR tree-optimization/65947
	* lib/target-supports.exp
	(check_effective_target_vect_max_reduc): Add.
	* gcc.dg/vect/pr65947-1.c: New test.
	* gcc.dg/vect/pr65947-2.c: New test.
	* gcc.dg/vect/pr65947-3.c: New test.
	* gcc.dg/vect/pr65947-4.c: New test.
	* gcc.dg/vect/pr65947-5.c: New test.
	* gcc.dg/vect/pr65947-6.c: New test.
	* gcc.dg/vect/pr65947-7.c: New test.
	* gcc.dg/vect/pr65947-8.c: New test.
	* gcc.dg/vect/pr65947-9.c: New test.
	* gcc.dg/vect/pr65947-10.c: New test.
	* gcc.dg/vect/pr65947-11.c: New test.

From-SVN: r229245

gcc/ChangeLog

+2015-10-23  Alan Hayward <alan.hayward@arm.com>
+
+	PR tree-optimization/65947
+	* tree-vect-loop.c
+	(vect_is_simple_reduction_1): Find condition reductions.
+	(vect_model_reduction_cost): Add condition reduction costs.
+	(get_initial_def_for_reduction): Add condition reduction initial var.
+	(vect_create_epilog_for_reduction): Add condition reduction epilog.
+	(vectorizable_reduction): Condition reduction support.
+	* tree-vect-stmts.c (vectorizable_condition): Add vect reduction arg
+	* doc/sourcebuild.texi (Vector-specific attributes): Document
+	vect_max_reduc
+
 2015-10-23  Richard Biener  <rguenther@suse.de>
 
 	* Makefile.in (build/genmatch.o): Properly depend on is-a.h, tree.def

gcc/doc/sourcebuild.texi

@@ -1466,6 +1466,9 @@ Target supports conversion from @code{float} to @code{signed int}.
 
 @item vect_floatuint_cvt
 Target supports conversion from @code{float} to @code{unsigned int}.
+
+@item vect_max_reduc
+Target supports max reduction for vectors.
 @end table
 
 @subsubsection Thread Local Storage attributes

gcc/testsuite/ChangeLog

+2015-10-23  Alan Hayward <alan.hayward@arm.com>
+
+	PR tree-optimization/65947
+	* lib/target-supports.exp
+	(check_effective_target_vect_max_reduc): Add.
+	* gcc.dg/vect/pr65947-1.c: New test.
+	* gcc.dg/vect/pr65947-2.c: New test.
+	* gcc.dg/vect/pr65947-3.c: New test.
+	* gcc.dg/vect/pr65947-4.c: New test.
+	* gcc.dg/vect/pr65947-5.c: New test.
+	* gcc.dg/vect/pr65947-6.c: New test.
+	* gcc.dg/vect/pr65947-7.c: New test.
+	* gcc.dg/vect/pr65947-8.c: New test.
+	* gcc.dg/vect/pr65947-9.c: New test.
+	* gcc.dg/vect/pr65947-10.c: New test.
+	* gcc.dg/vect/pr65947-11.c: New test.
+
 2015-10-23  Richard Sandiford  <richard.sandiford@arm.com>
 
 	* gcc.c-torture/execute/20030125-1.c (floor, floorf, sin, sinf):

gcc/testsuite/gcc.dg/vect/pr65947-1.c

+/* { dg-require-effective-target vect_condition } */
+
+extern void abort (void) __attribute__ ((noreturn));
+
+#define N 32
+
+/* Simple condition reduction.  */
+
+int
+condition_reduction (int *a, int min_v)
+{
+  int last = -1;
+
+  for (int i = 0; i < N; i++)
+    if (a[i] < min_v)
+      last = i;
+
+  return last;
+}
+
+int
+main (void)
+{
+  int a[N] = {
+  11, -12, 13, 14, 15, 16, 17, 18, 19, 20,
+  1, 2, -3, 4, 5, 6, 7, -8, 9, 10,
+  21, 22, 23, 24, 25, 26, 27, 28, 29, 30,
+  31, 32
+  };
+
+  int ret = condition_reduction (a, 16);
+
+  if (ret != 19)
+    abort ();
+
+  return 0;
+}
+
+/* { dg-final { scan-tree-dump-times "LOOP VECTORIZED" 2 "vect" { xfail { ! vect_max_reduc } } } } */

gcc/testsuite/gcc.dg/vect/pr65947-10.c

+/* { dg-require-effective-target vect_condition } */
+
+extern void abort (void) __attribute__ ((noreturn));
+
+#define N 32
+
+/* Non-integer data types.  */
+
+float
+condition_reduction (float *a, float min_v)
+{
+  float last = 0;
+
+  for (int i = 0; i < N; i++)
+    if (a[i] < min_v)
+      last = a[i];
+
+  return last;
+}
+
+int
+main (void)
+{
+  float a[N] = {
+  11.5, 12.2, 13.22, 14.1, 15.2, 16.3, 17, 18.7, 19, 20,
+  1, 2, 3.3, 4.3333, 5.5, 6.23, 7, 8.63, 9, 10.6,
+  21, 22.12, 23.55, 24.76, 25, 26, 27.34, 28.765, 29, 30,
+  31.111, 32.322
+  };
+
+  float ret = condition_reduction (a, 16.7);
+
+  if (ret != (float)10.6)
+    abort ();
+
+  return 0;
+}
+
+/* { dg-final { scan-tree-dump-times "LOOP VECTORIZED" 2 "vect" { xfail { ! vect_max_reduc } } } } */
+

gcc/testsuite/gcc.dg/vect/pr65947-11.c

+/* { dg-require-effective-target vect_condition } */
+
+extern void abort (void) __attribute__ ((noreturn));
+
+#define N 37
+
+/* Re-use the result of the condition inside the loop.  Will fail to
+   vectorize.  */
+
+unsigned int
+condition_reduction (unsigned int *a, unsigned int min_v, unsigned int *b)
+{
+  unsigned int last = N + 65;
+
+  for (unsigned int i = 0; i < N; i++)
+    {
+      if (b[i] < min_v)
+	last = i;
+      a[i] = last;
+    }
+  return last;
+}
+
+int
+main (void)
+{
+  unsigned int a[N] = {
+  31, 32, 33, 34, 35, 36, 37,
+  1, 2, 3, 4, 5, 6, 7, 8, 9, 10,
+  21, 22, 23, 24, 25, 26, 27, 28, 29, 30,
+  11, 12, 13, 14, 15, 16, 17, 18, 19, 20
+  };
+  unsigned int b[N] = {
+  11, 12, 13, 14, 15, 16, 17, 18, 19, 20,
+  21, 22, 23, 24, 25, 26, 27, 28, 29, 30,
+  1, 2, 3, 4, 5, 6, 7, 8, 9, 10,
+  31, 32, 33, 34, 35, 36, 37
+  };
+
+  unsigned int ret = condition_reduction (a, 16, b);
+
+  if (ret != 29)
+    abort ();
+
+  return 0;
+}
+
+/* { dg-final { scan-tree-dump-not "LOOP VECTORIZED" "vect" } } */

gcc/testsuite/gcc.dg/vect/pr65947-2.c

+/* { dg-require-effective-target vect_condition } */
+
+extern void abort (void) __attribute__ ((noreturn));
+
+#define N 254
+
+/* Non-simple condition reduction.  */
+
+unsigned char
+condition_reduction (unsigned char *a, unsigned char min_v)
+{
+  unsigned char last = 65;
+
+  for (unsigned char i = 0; i < N; i++)
+    if (a[i] < min_v)
+      last = a[i];
+
+  return last;
+}
+
+int
+main (void)
+{
+  unsigned char a[N] = {
+  11, 12, 13, 14, 15, 16, 17, 18, 19, 20,
+  1, 2, 3, 4, 5, 6, 7, 8, 9, 10,
+  21, 22, 23, 24, 25, 26, 27, 28, 29, 30,
+  31, 32
+  };
+  __builtin_memset (a+32, 43, N-32);
+
+  unsigned char ret = condition_reduction (a, 16);
+
+  if (ret != 10)
+    abort ();
+
+  return 0;
+}
+
+/* { dg-final { scan-tree-dump-times "LOOP VECTORIZED" 2 "vect" { xfail { ! vect_max_reduc } } } } */

gcc/testsuite/gcc.dg/vect/pr65947-3.c

+/* { dg-require-effective-target vect_condition } */
+
+extern void abort (void) __attribute__ ((noreturn));
+
+#define N 37
+
+/* Non-simple condition reduction with additional variable and unsigned
+   types.  */
+
+unsigned int
+condition_reduction (unsigned int *a, unsigned int min_v, unsigned int *b)
+{
+  unsigned int last = N + 65;
+  unsigned int aval;
+
+  for (unsigned int i = 0; i < N; i++)
+    {
+      aval = a[i];
+      if (b[i] < min_v)
+	last = aval;
+    }
+  return last;
+}
+
+
+int
+main (void)
+{
+  unsigned int a[N] = {
+  31, 32, 33, 34, 35, 36, 37,
+  1, 2, 3, 4, 5, 6, 7, 8, 9, 10,
+  21, 22, 23, 24, 25, 26, 27, 28, 29, 30,
+  11, 12, 13, 14, 15, 16, 17, 18, 19, 20
+  };
+  unsigned int b[N] = {
+  11, 12, 13, 14, 15, 16, 17, 18, 19, 20,
+  21, 22, 23, 24, 25, 26, 27, 28, 29, 30,
+  1, 2, 3, 4, 5, 6, 7, 8, 9, 10,
+  31, 32, 33, 34, 35, 36, 37
+  };
+
+  unsigned int ret = condition_reduction (a, 16, b);
+
+  if (ret != 13)
+    abort ();
+
+  return 0;
+}
+
+/* { dg-final { scan-tree-dump-times "LOOP VECTORIZED" 2 "vect" { xfail { ! vect_max_reduc } } } } */

gcc/testsuite/gcc.dg/vect/pr65947-4.c

+/* { dg-require-effective-target vect_condition } */
+
+extern void abort (void) __attribute__ ((noreturn));
+
+#define N 27
+
+/* Condition reduction with no valid matches at runtime.  */
+
+int
+condition_reduction (int *a, int min_v)
+{
+  int last = N + 96;
+
+  for (int i = 0; i < N; i++)
+    if (a[i] > min_v)
+      last = i;
+
+  return last;
+}
+
+int
+main (void)
+{
+  int a[N] = {
+  11, 12, 13, 14, 15, 16, 17, 18, 19, 20,
+  1, 2, 3, 4, 5, 6, 7, 8, 9, 10,
+  21, 22, 23, 24, 25, 26, 27
+  };
+
+  int ret = condition_reduction (a, 46);
+
+  /* loop should never have found a value.  */
+  if (ret != N + 96)
+    abort ();
+
+  return 0;
+}
+
+/* { dg-final { scan-tree-dump-times "LOOP VECTORIZED" 2 "vect" { xfail { ! vect_max_reduc } } } } */
+

gcc/testsuite/gcc.dg/vect/pr65947-5.c

+/* { dg-require-effective-target vect_condition } */
+
+extern void abort (void) __attribute__ ((noreturn));
+
+#define N 32
+
+/* Condition reduction where loop size is not known at compile time.  Will fail
+   to vectorize.  Version inlined into main loop will vectorize.  */
+
+unsigned char
+condition_reduction (unsigned char *a, unsigned char min_v, int count)
+{
+  unsigned char last = 65;
+
+  for (int i = 0; i < count; i++)
+    if (a[i] < min_v)
+      last = a[i];
+
+  return last;
+}
+
+int
+main (void)
+{
+  unsigned char a[N] = {
+  11, 12, 13, 14, 15, 16, 17, 18, 19, 20,
+  1, 2, 3, 4, 5, 6, 7, 8, 9, 10,
+  21, 22, 23, 24, 25, 26, 27, 28, 29, 30,
+  31, 32
+  };
+
+  unsigned char ret = condition_reduction (a, 16, N);
+
+  if (ret != 10)
+    abort ();
+
+  return 0;
+}
+
+/* { dg-final { scan-tree-dump-times "LOOP VECTORIZED" 1 "vect" { xfail { ! vect_max_reduc } } } } */
+/* { dg-final { scan-tree-dump "loop size is greater than data size" "vect" { xfail { ! vect_max_reduc } } } } */

gcc/testsuite/gcc.dg/vect/pr65947-6.c

+/* { dg-require-effective-target vect_condition } */
+
+extern void abort (void) __attribute__ ((noreturn));
+
+#define N 30
+
+/* Condition reduction where loop type is different than the data type.  */
+
+int
+condition_reduction (int *a, int min_v)
+{
+  int last = N + 65;
+
+  for (char i = 0; i < N; i++)
+    if (a[i] < min_v)
+      last = a[i];
+
+  return last;
+}
+
+
+int
+main (void)
+{
+  int a[N] = {
+  67, 32, 45, 43, 21, -11, 12, 3, 4, 5,
+  6, 76, -32, 56, -32, -1, 4, 5, 6, 99,
+  43, 22, -3, 22, 16, 34, 55, 31, 87, 324
+  };
+
+  int ret = condition_reduction (a, 16);
+
+  if (ret != -3)
+    abort ();
+
+  return 0;
+}
+
+/* { dg-final { scan-tree-dump-times "LOOP VECTORIZED" 2 "vect" { xfail { ! vect_max_reduc } } } } */

gcc/testsuite/gcc.dg/vect/pr65947-7.c

+/* { dg-require-effective-target vect_condition } */
+
+extern void abort (void) __attribute__ ((noreturn));
+
+#define N 43
+
+/* Condition reduction with comparison is a different type to the data.  Will
+   fail to vectorize.  */
+
+int
+condition_reduction (short *a, int min_v, int *b)
+{
+  int last = N + 65;
+  short aval;
+
+  for (int i = 0; i < N; i++)
+    {
+      aval = a[i];
+      if (b[i] < min_v)
+	last = aval;
+    }
+  return last;
+}
+
+int
+main (void)
+{
+  short a[N] = {
+  31, -32, 133, 324, 335, 36, 37, 45, 11, 65,
+  1, -28, 3, 48, 5, -68, 7, 88, 89, 180,
+  121, -122, 123, 124, -125, 126, 127, 128, 129, 130,
+  11, 12, 13, 14, -15, -16, 17, 18, 19, 20,
+  33, 27, 99
+  };
+  int b[N] = {
+  11, -12, -13, 14, 15, 16, 17, 18, 19, 20,
+  21, -22, 23, 24, -25, 26, 27, 28, 29, 30,
+  1, 62, 3, 14, -15, 6, 37, 48, 99, 10,
+  31, -32, 33, 34, -35, 36, 37, 56, 54, 22,
+  73, 2, 87
+  };
+
+  int ret = condition_reduction (a, 16, b);
+
+  if (ret != 27)
+    abort ();
+
+  return 0;
+}
+
+/* { dg-final { scan-tree-dump-not "LOOP VECTORIZED" "vect" } } */

gcc/testsuite/gcc.dg/vect/pr65947-8.c

+/* { dg-require-effective-target vect_condition } */
+
+extern void abort (void) __attribute__ ((noreturn));
+
+#define N 27
+
+/* Condition reduction with multiple types in the comparison.  Will fail to
+   vectorize.  */
+
+int
+condition_reduction (char *a, int min_v)
+{
+  int last = N + 65;
+
+  for (int i = 0; i < N; i++)
+    if (a[i] < min_v)
+      last = a[i];
+
+  return last;
+}
+
+
+int
+main (void)
+{
+  char a[N] = {
+  1, 28, 3, 48, 5, 68, 7, -88, 89, 180,
+  121, 122, -123, 124, 12, -12, 12, 67, 84, 122,
+  67, 55, 112, 22, 45, 23, 111
+  };
+
+  int ret = condition_reduction (a, 16);
+
+  if (ret != 12)
+    abort ();
+
+  return 0;
+}
+
+/* { dg-final { scan-tree-dump-not "LOOP VECTORIZED" "vect" } } */
+/* { dg-final { scan-tree-dump "multiple types in double reduction or condition reduction" "vect" { xfail { ! vect_max_reduc } } } } */

gcc/testsuite/gcc.dg/vect/pr65947-9.c

+/* { dg-require-effective-target vect_condition } */
+
+extern void abort (void) __attribute__ ((noreturn));
+
+#define N 255
+
+/* Condition reduction with maximum possible loop size.  Will fail to
+   vectorize because the vectorisation requires a slot for default values.  */
+
+char
+condition_reduction (char *a, char min_v)
+{
+  char last = -72;
+
+  for (int i = 0; i < N; i++)
+    if (a[i] < min_v)
+      last = a[i];
+
+  return last;
+}
+
+char
+main (void)
+{
+  char a[N] = {
+  11, 12, 13, 14, 15, 16, 17, 18, 19, 20,
+  1, 2, 3, 4, 5, 6, 7, 8, 9, 10,
+  21, 22, 23, 24, 25, 26, 27, 28, 29, 30,
+  31, 32
+  };
+  __builtin_memset (a+32, 43, N-32);
+
+  char ret = condition_reduction (a, 16);
+
+  if (ret != 10)
+    abort ();
+
+  return 0;
+}
+
+/* { dg-final { scan-tree-dump-not "LOOP VECTORIZED" "vect" } } */
+/* { dg-final { scan-tree-dump "loop size is greater than data size" "vect" { xfail { ! vect_max_reduc } } } } */

gcc/testsuite/lib/target-supports.exp

@@ -6485,3 +6485,12 @@ proc check_effective_target_builtin_eh_return { } {
 	}
     } "" ]
 }
+
+# Return 1 if the target supports max reduction for vectors.
+
+proc check_effective_target_vect_max_reduc { } {
+    if { [istarget aarch64*-*-*] || [istarget arm*-*-*] } {
+	return 1
+    }
+    return 0
+}

gcc/tree-vect-stmts.c

@@ -7202,21 +7202,24 @@ vectorizable_condition (gimple *stmt, gimple_stmt_iterator *gsi,
   if (reduc_index && STMT_SLP_TYPE (stmt_info))
     return false;
 
-  if (!STMT_VINFO_RELEVANT_P (stmt_info) && !bb_vinfo)
-    return false;
+  if (STMT_VINFO_VEC_REDUCTION_TYPE (stmt_info) == TREE_CODE_REDUCTION)
+    {
+      if (!STMT_VINFO_RELEVANT_P (stmt_info) && !bb_vinfo)
+	return false;
 
-  if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_internal_def
-      && !(STMT_VINFO_DEF_TYPE (stmt_info) == vect_nested_cycle
-           && reduc_def))
-    return false;
+      if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_internal_def
+	  && !(STMT_VINFO_DEF_TYPE (stmt_info) == vect_nested_cycle
+	       && reduc_def))
+	return false;
 
-  /* FORNOW: not yet supported.  */
-  if (STMT_VINFO_LIVE_P (stmt_info))
-    {
-      if (dump_enabled_p ())
-        dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
-                         "value used after loop.\n");
-      return false;
+      /* FORNOW: not yet supported.  */
+      if (STMT_VINFO_LIVE_P (stmt_info))
+	{
+	  if (dump_enabled_p ())
+	    dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
+			     "value used after loop.\n");
+	  return false;
+	}
     }
 
   /* Is vectorizable conditional operation?  */
@@ -7865,6 +7868,7 @@ new_stmt_vec_info (gimple *stmt, vec_info *vinfo)
   STMT_VINFO_RELATED_STMT (res) = NULL;
   STMT_VINFO_PATTERN_DEF_SEQ (res) = NULL;
   STMT_VINFO_DATA_REF (res) = NULL;
+  STMT_VINFO_VEC_REDUCTION_TYPE (res) = TREE_CODE_REDUCTION;
 
   STMT_VINFO_DR_BASE_ADDRESS (res) = NULL;
   STMT_VINFO_DR_OFFSET (res) = NULL;
@@ -8118,8 +8122,8 @@ vect_is_simple_use (tree operand, vec_info *vinfo,
   if (TREE_CODE (operand) != SSA_NAME)
     {
       if (dump_enabled_p ())
-        dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
-                         "not ssa-name.\n");
+	dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
+			 "not ssa-name.\n");
       return false;
     }
 

gcc/tree-vectorizer.h

@@ -60,6 +60,12 @@ enum vect_def_type {
   vect_unknown_def_type
 };
 
+/* Define type of reduction.  */
+enum vect_reduction_type {
+  TREE_CODE_REDUCTION,
+  COND_REDUCTION
+};
+
 #define VECTORIZABLE_CYCLE_DEF(D) (((D) == vect_reduction_def)           \
                                    || ((D) == vect_double_reduction_def) \
                                    || ((D) == vect_nested_cycle))
@@ -581,6 +587,10 @@ typedef struct _stmt_vec_info {
 
   /* For both loads and stores.  */
   bool simd_lane_access_p;
+
+  /* For reduction loops, this is the type of reduction.  */
+  enum vect_reduction_type v_reduc_type;
+
 } *stmt_vec_info;
 
 /* Access Functions.  */
@@ -609,6 +619,7 @@ STMT_VINFO_BB_VINFO (stmt_vec_info stmt_vinfo)
 #define STMT_VINFO_GATHER_SCATTER_P(S)	   (S)->gather_scatter_p
 #define STMT_VINFO_STRIDED_P(S)	   	   (S)->strided_p
 #define STMT_VINFO_SIMD_LANE_ACCESS_P(S)   (S)->simd_lane_access_p
+#define STMT_VINFO_VEC_REDUCTION_TYPE(S)   (S)->v_reduc_type
 
 #define STMT_VINFO_DR_BASE_ADDRESS(S)      (S)->dr_base_address
 #define STMT_VINFO_DR_INIT(S)              (S)->dr_init