omp-low.c (struct omp_context): Add scan_inclusive field.

	* omp-low.c (struct omp_context): Add scan_inclusive field.
	(scan_omp_1_stmt) <case GIMPLE_OMP_SCAN>: Set ctx->scan_inclusive
	if inclusive scan.
	(struct omplow_simd_context): Add lastlane member.
	(lower_rec_simd_input_clauses): Add rvar argument, handle inscan
	reductions.  Build 2 or 3 argument .GOMP_SIMD_LANE calls rather than
	1 or 2 argument.
	(lower_rec_input_clauses): Handle inscan reductions in simd contexts.
	(lower_lastprivate_clauses): Set TREE_THIS_NOTRAP on the ARRAY_REF.
	(lower_omp_scan): New function.
	(lower_omp_1) <case GIMPLE_OMP_SCAN>: Use lower_omp_scan.
	* tree-ssa-dce.c (eliminate_unnecessary_stmts): For IFN_GOMP_SIMD_LANE
	check 3rd argument if present rather than 2nd.
	* tree-vectorizer.h (struct _loop_vec_info): Add scan_map member.
	(struct _stmt_vec_info): Change simd_lane_access_p from bool into
	2-bit bitfield.
	* tree-vect-loop.c (_loop_vec_info::_loop_vec_info): Initialize
	scan_map.  For IFN_GOMP_SIMD_LANE check 3rd argument if present rather
	than 2nd.
	(_loop_vec_info::~_loop_vec_info): Delete scan_map.
	* tree-vect-data-refs.c (vect_analyze_data_ref_accesses): Allow two
	different STMT_VINFO_SIMD_LANE_ACCESS_P refs if they have the same
	init.
	(vect_find_stmt_data_reference): Encode in ->aux the 2nd
	IFN_GOMP_SIMD_LANE argument.
	(vect_analyze_data_refs): Set STMT_VINFO_SIMD_LANE_ACCESS_P from the
	encoded ->aux value.
	* tree-vect-stmts.c: Include attribs.h.
	(vectorizable_call): Adjust comment about IFN_GOMP_SIMD_LANE.
	(scan_operand_equal_p, check_scan_store, vectorizable_scan_store): New
	functions.
	(vectorizable_load): For STMT_VINFO_SIMD_LANE_ACCESS_P tests use != 0.
	(vectorizable_store): Handle STMT_VINFO_SIMD_LANE_ACCESS_P > 1.
cp/
	* semantics.c (finish_omp_clauses): For OMP_CLAUSE_REDUCTION_INSCAN
	set need_copy_assignment.
testsuite/
	* gcc.dg/vect/vect-simd-8.c: New test.
	* gcc.dg/vect/vect-simd-9.c: New test.
	* g++.dg/vect/simd-2.cc: New test.
	* g++.dg/gomp/scan-1.C: New test.

From-SVN: r272399

gcc/ChangeLog

+2019-06-17  Jakub Jelinek  <jakub@redhat.com>
+
+	* omp-low.c (struct omp_context): Add scan_inclusive field.
+	(scan_omp_1_stmt) <case GIMPLE_OMP_SCAN>: Set ctx->scan_inclusive
+	if inclusive scan.
+	(struct omplow_simd_context): Add lastlane member.
+	(lower_rec_simd_input_clauses): Add rvar argument, handle inscan
+	reductions.  Build 2 or 3 argument .GOMP_SIMD_LANE calls rather than
+	1 or 2 argument.
+	(lower_rec_input_clauses): Handle inscan reductions in simd contexts.
+	(lower_lastprivate_clauses): Set TREE_THIS_NOTRAP on the ARRAY_REF.
+	(lower_omp_scan): New function.
+	(lower_omp_1) <case GIMPLE_OMP_SCAN>: Use lower_omp_scan.
+	* tree-ssa-dce.c (eliminate_unnecessary_stmts): For IFN_GOMP_SIMD_LANE
+	check 3rd argument if present rather than 2nd.
+	* tree-vectorizer.h (struct _loop_vec_info): Add scan_map member.
+	(struct _stmt_vec_info): Change simd_lane_access_p from bool into
+	2-bit bitfield.
+	* tree-vect-loop.c (_loop_vec_info::_loop_vec_info): Initialize
+	scan_map.  For IFN_GOMP_SIMD_LANE check 3rd argument if present rather
+	than 2nd.
+	(_loop_vec_info::~_loop_vec_info): Delete scan_map.
+	* tree-vect-data-refs.c (vect_analyze_data_ref_accesses): Allow two
+	different STMT_VINFO_SIMD_LANE_ACCESS_P refs if they have the same
+	init.
+	(vect_find_stmt_data_reference): Encode in ->aux the 2nd
+	IFN_GOMP_SIMD_LANE argument.
+	(vect_analyze_data_refs): Set STMT_VINFO_SIMD_LANE_ACCESS_P from the
+	encoded ->aux value.
+	* tree-vect-stmts.c: Include attribs.h.
+	(vectorizable_call): Adjust comment about IFN_GOMP_SIMD_LANE.
+	(scan_operand_equal_p, check_scan_store, vectorizable_scan_store): New
+	functions.
+	(vectorizable_load): For STMT_VINFO_SIMD_LANE_ACCESS_P tests use != 0.
+	(vectorizable_store): Handle STMT_VINFO_SIMD_LANE_ACCESS_P > 1.
+
 2019-06-17  Uroš Bizjak  <ubizjak@gmail.com>
 
 	PR target/62055

gcc/cp/ChangeLog

+2019-06-17  Jakub Jelinek  <jakub@redhat.com>
+
+	* semantics.c (finish_omp_clauses): For OMP_CLAUSE_REDUCTION_INSCAN
+	set need_copy_assignment.
+
 2019-06-17  Marek Polacek  <polacek@redhat.com>
 
 	PR c++/83820 - excessive attribute arguments not detected.

gcc/cp/semantics.c

@@ -7688,6 +7688,8 @@ finish_omp_clauses (tree clauses, enum c_omp_region_type ort)
 	case OMP_CLAUSE_REDUCTION:
 	  if (reduction_seen == -2)
 	    OMP_CLAUSE_REDUCTION_INSCAN (c) = 0;
+	  if (OMP_CLAUSE_REDUCTION_INSCAN (c))
+	    need_copy_assignment = true;
 	  need_implicitly_determined = true;
 	  break;
 	case OMP_CLAUSE_IN_REDUCTION:

gcc/testsuite/ChangeLog

+2019-06-17  Jakub Jelinek  <jakub@redhat.com>
+
+	* gcc.dg/vect/vect-simd-8.c: New test.
+	* gcc.dg/vect/vect-simd-9.c: New test.
+	* g++.dg/vect/simd-2.cc: New test.
+	* g++.dg/gomp/scan-1.C: New test.
+
 2019-06-17  Uroš Bizjak  <ubizjak@gmail.com>
 
 	PR target/62055

gcc/testsuite/g++.dg/gomp/scan-1.C

+// { dg-do compile { target c++11 } }
+
+struct S { S (); ~S (); S &operator = (const S &) = delete; int s; };	// { dg-message "declared here" }
+#pragma omp declare reduction (+ : S : omp_out.s += omp_in.s)
+
+S s;
+
+void
+foo (void)
+{
+  #pragma omp simd reduction (+: s)
+  for (int i = 0; i < 64; ++i)
+    s.s += i;
+}
+
+void
+bar (int *x)
+{
+  #pragma omp simd reduction (inscan, +: s)	// { dg-error "use of deleted function" }
+  for (int i = 0; i < 64; ++i)
+    {
+      s.s += i;
+      #pragma omp scan inclusive (s)		// { dg-error "" }
+      x[i] = s.s;
+    }
+}

gcc/testsuite/g++.dg/vect/simd-2.cc

+// { dg-require-effective-target size32plus }
+// { dg-additional-options "-fopenmp-simd" }
+// { dg-additional-options "-mavx" { target avx_runtime } }
+// { dg-final { scan-tree-dump-times "vectorized \[1-3] loops" 2 "vect" { xfail *-*-* } } }
+
+#include "../../gcc.dg/vect/tree-vect.h"
+
+struct S {
+  inline S ();
+  inline ~S ();
+  inline S (const S &);
+  inline S & operator= (const S &);
+  int s;
+};
+
+S::S () : s (0)
+{
+}
+
+S::~S ()
+{
+}
+
+S::S (const S &x)
+{
+  s = x.s;
+}
+
+S &
+S::operator= (const S &x)
+{
+  s = x.s;
+  return *this;
+}
+
+static inline void
+ini (S &x)
+{
+  x.s = 0;
+}
+
+S r, a[1024], b[1024];
+
+#pragma omp declare reduction (+: S: omp_out.s += omp_in.s)
+#pragma omp declare reduction (plus: S: omp_out.s += omp_in.s) initializer (ini (omp_priv))
+
+__attribute__((noipa)) void
+foo (S *a, S *b)
+{
+  #pragma omp simd reduction (inscan, +:r)
+  for (int i = 0; i < 1024; i++)
+    {
+      r.s += a[i].s;
+      #pragma omp scan inclusive(r)
+      b[i] = r;
+    }
+}
+
+__attribute__((noipa)) S
+bar (void)
+{
+  S s;
+  #pragma omp simd reduction (inscan, plus:s)
+  for (int i = 0; i < 1024; i++)
+    {
+      s.s += 2 * a[i].s;
+      #pragma omp scan inclusive(s)
+      b[i] = s;
+    }
+  return S (s);
+}
+
+__attribute__((noipa)) void
+baz (S *a, S *b)
+{
+  #pragma omp simd reduction (inscan, +:r) simdlen(1)
+  for (int i = 0; i < 1024; i++)
+    {
+      r.s += a[i].s;
+      #pragma omp scan inclusive(r)
+      b[i] = r;
+    }
+}
+
+__attribute__((noipa)) S
+qux (void)
+{
+  S s;
+  #pragma omp simd if (0) reduction (inscan, plus:s)
+  for (int i = 0; i < 1024; i++)
+    {
+      s.s += 2 * a[i].s;
+      #pragma omp scan inclusive(s)
+      b[i] = s;
+    }
+  return S (s);
+}
+
+int
+main ()
+{
+  S s;
+  check_vect ();
+  for (int i = 0; i < 1024; ++i)
+    {
+      a[i].s = i;
+      b[i].s = -1;
+      asm ("" : "+g" (i));
+    }
+  foo (a, b);
+  if (r.s != 1024 * 1023 / 2)
+    abort ();
+  for (int i = 0; i < 1024; ++i)
+    {
+      s.s += i;
+      if (b[i].s != s.s)
+	abort ();
+      else
+	b[i].s = 25;
+    }
+  if (bar ().s != 1024 * 1023)
+    abort ();
+  s.s = 0;
+  for (int i = 0; i < 1024; ++i)
+    {
+      s.s += 2 * i;
+      if (b[i].s != s.s)
+	abort ();
+    }
+  r.s = 0;
+  baz (a, b);
+  if (r.s != 1024 * 1023 / 2)
+    abort ();
+  s.s = 0;
+  for (int i = 0; i < 1024; ++i)
+    {
+      s.s += i;
+      if (b[i].s != s.s)
+	abort ();
+      else
+	b[i].s = 25;
+    }
+  if (qux ().s != 1024 * 1023)
+    abort ();
+  s.s = 0;
+  for (int i = 0; i < 1024; ++i)
+    {
+      s.s += 2 * i;
+      if (b[i].s != s.s)
+	abort ();
+    }
+  return 0;
+}

gcc/testsuite/gcc.dg/vect/vect-simd-8.c

+/* { dg-require-effective-target size32plus } */
+/* { dg-additional-options "-fopenmp-simd" } */
+/* { dg-additional-options "-mavx" { target avx_runtime } } */
+/* { dg-final { scan-tree-dump-times "vectorized \[1-3] loops" 2 "vect" { target i?86-*-* x86_64-*-* } } } */
+
+#include "tree-vect.h"
+
+int r, a[1024], b[1024];
+
+__attribute__((noipa)) void
+foo (int *a, int *b)
+{
+  #pragma omp simd reduction (inscan, +:r)
+  for (int i = 0; i < 1024; i++)
+    {
+      r += a[i];
+      #pragma omp scan inclusive(r)
+      b[i] = r;
+    }
+}
+
+__attribute__((noipa)) int
+bar (void)
+{
+  int s = 0;
+  #pragma omp simd reduction (inscan, +:s)
+  for (int i = 0; i < 1024; i++)
+    {
+      s += 2 * a[i];
+      #pragma omp scan inclusive(s)
+      b[i] = s;
+    }
+  return s;
+}
+
+__attribute__((noipa)) void
+baz (int *a, int *b)
+{
+  #pragma omp simd reduction (inscan, +:r) if (simd: 0)
+  for (int i = 0; i < 1024; i++)
+    {
+      r += a[i];
+      #pragma omp scan inclusive(r)
+      b[i] = r;
+    }
+}
+
+__attribute__((noipa)) int
+qux (void)
+{
+  int s = 0;
+  #pragma omp simd reduction (inscan, +:s) simdlen (1)
+  for (int i = 0; i < 1024; i++)
+    {
+      s += 2 * a[i];
+      #pragma omp scan inclusive(s)
+      b[i] = s;
+    }
+  return s;
+}
+
+int
+main ()
+{
+  int s = 0;
+  check_vect ();
+  for (int i = 0; i < 1024; ++i)
+    {
+      a[i] = i;
+      b[i] = -1;
+      asm ("" : "+g" (i));
+    }
+  foo (a, b);
+  if (r != 1024 * 1023 / 2)
+    abort ();
+  for (int i = 0; i < 1024; ++i)
+    {
+      s += i;
+      if (b[i] != s)
+	abort ();
+      else
+	b[i] = 25;
+    }
+  if (bar () != 1024 * 1023)
+    abort ();
+  s = 0;
+  for (int i = 0; i < 1024; ++i)
+    {
+      s += 2 * i;
+      if (b[i] != s)
+	abort ();
+      else
+	b[i] = -1;
+    }
+  r = 0;
+  baz (a, b);
+  if (r != 1024 * 1023 / 2)
+    abort ();
+  s = 0;
+  for (int i = 0; i < 1024; ++i)
+    {
+      s += i;
+      if (b[i] != s)
+	abort ();
+      else
+	b[i] = -25;
+    }
+  if (qux () != 1024 * 1023)
+    abort ();
+  s = 0;
+  for (int i = 0; i < 1024; ++i)
+    {
+      s += 2 * i;
+      if (b[i] != s)
+	abort ();
+    }
+  return 0;
+}

gcc/testsuite/gcc.dg/vect/vect-simd-9.c

+/* { dg-require-effective-target size32plus } */
+/* { dg-additional-options "-fopenmp-simd" } */
+/* { dg-additional-options "-mavx" { target avx_runtime } } */
+/* { dg-final { scan-tree-dump-times "vectorized \[1-3] loops" 2 "vect" { target i?86-*-* x86_64-*-* } } } */
+
+#include "tree-vect.h"
+
+int r, a[1024], b[1024];
+
+#pragma omp declare reduction (foo: int: omp_out += omp_in) initializer (omp_priv = 0)
+
+__attribute__((noipa)) void
+foo (int *a, int *b)
+{
+  #pragma omp simd reduction (inscan, foo:r)
+  for (int i = 0; i < 1024; i++)
+    {
+      r += a[i];
+      #pragma omp scan inclusive(r)
+      b[i] = r;
+    }
+}
+
+__attribute__((noipa)) int
+bar (void)
+{
+  int s = 0;
+  #pragma omp simd reduction (inscan, foo:s)
+  for (int i = 0; i < 1024; i++)
+    {
+      s += 2 * a[i];
+      #pragma omp scan inclusive(s)
+      b[i] = s;
+    }
+  return s;
+}
+
+__attribute__((noipa)) void
+baz (int *a, int *b)
+{
+  #pragma omp simd reduction (inscan, foo:r) if (simd: 0)
+  for (int i = 0; i < 1024; i++)
+    {
+      r += a[i];
+      #pragma omp scan inclusive(r)
+      b[i] = r;
+    }
+}
+
+__attribute__((noipa)) int
+qux (void)
+{
+  int s = 0;
+  #pragma omp simd reduction (inscan, foo:s) simdlen (1)
+  for (int i = 0; i < 1024; i++)
+    {
+      s += 2 * a[i];
+      #pragma omp scan inclusive(s)
+      b[i] = s;
+    }
+  return s;
+}
+
+int
+main ()
+{
+  int s = 0;
+  check_vect ();
+  for (int i = 0; i < 1024; ++i)
+    {
+      a[i] = i;
+      b[i] = -1;
+      asm ("" : "+g" (i));
+    }
+  foo (a, b);
+  if (r != 1024 * 1023 / 2)
+    abort ();
+  for (int i = 0; i < 1024; ++i)
+    {
+      s += i;
+      if (b[i] != s)
+	abort ();
+      else
+	b[i] = 25;
+    }
+  if (bar () != 1024 * 1023)
+    abort ();
+  s = 0;
+  for (int i = 0; i < 1024; ++i)
+    {
+      s += 2 * i;
+      if (b[i] != s)
+	abort ();
+      else
+	b[i] = -1;
+    }
+  r = 0;
+  baz (a, b);
+  if (r != 1024 * 1023 / 2)
+    abort ();
+  s = 0;
+  for (int i = 0; i < 1024; ++i)
+    {
+      s += i;
+      if (b[i] != s)
+	abort ();
+      else
+	b[i] = -25;
+    }
+  if (qux () != 1024 * 1023)
+    abort ();
+  s = 0;
+  for (int i = 0; i < 1024; ++i)
+    {
+      s += 2 * i;
+      if (b[i] != s)
+	abort ();
+    }
+  return 0;
+}

gcc/tree-ssa-dce.c

@@ -1339,14 +1339,14 @@ eliminate_unnecessary_stmts (void)
 		  update_stmt (stmt);
 		  release_ssa_name (name);
 
-		  /* GOMP_SIMD_LANE (unless two argument) or ASAN_POISON
+		  /* GOMP_SIMD_LANE (unless three argument) or ASAN_POISON
 		     without lhs is not needed.  */
 		  if (gimple_call_internal_p (stmt))
 		    switch (gimple_call_internal_fn (stmt))
 		      {
 		      case IFN_GOMP_SIMD_LANE:
-			if (gimple_call_num_args (stmt) >= 2
-			    && !integer_nonzerop (gimple_call_arg (stmt, 1)))
+			if (gimple_call_num_args (stmt) >= 3
+			    && !integer_nonzerop (gimple_call_arg (stmt, 2)))
 			  break;
 			/* FALLTHRU */
 		      case IFN_ASAN_POISON:

gcc/tree-vect-data-refs.c

@@ -3003,6 +3003,13 @@ vect_analyze_data_ref_accesses (vec_info *vinfo)
 	      || TREE_CODE (DR_INIT (drb)) != INTEGER_CST)
 	    break;
 
+	  /* Different .GOMP_SIMD_LANE calls still give the same lane,
+	     just hold extra information.  */
+	  if (STMT_VINFO_SIMD_LANE_ACCESS_P (stmtinfo_a)
+	      && STMT_VINFO_SIMD_LANE_ACCESS_P (stmtinfo_b)
+	      && data_ref_compare_tree (DR_INIT (dra), DR_INIT (drb)) == 0)
+	    break;
+
 	  /* Sorting has ensured that DR_INIT (dra) <= DR_INIT (drb).  */
 	  HOST_WIDE_INT init_a = TREE_INT_CST_LOW (DR_INIT (dra));
 	  HOST_WIDE_INT init_b = TREE_INT_CST_LOW (DR_INIT (drb));
@@ -4101,7 +4108,8 @@ vect_find_stmt_data_reference (loop_p loop, gimple *stmt,
 			  DR_STEP_ALIGNMENT (newdr)
 			    = highest_pow2_factor (step);
 			  /* Mark as simd-lane access.  */
-			  newdr->aux = (void *)-1;
+			  tree arg2 = gimple_call_arg (def, 1);
+			  newdr->aux = (void *) (-1 - tree_to_uhwi (arg2));
 			  free_data_ref (dr);
 			  datarefs->safe_push (newdr);
 			  return opt_result::success ();
@@ -4210,14 +4218,17 @@ vect_analyze_data_refs (vec_info *vinfo, poly_uint64 *min_vf)
         }
 
       /* See if this was detected as SIMD lane access.  */
-      if (dr->aux == (void *)-1)
+      if (dr->aux == (void *)-1
+	  || dr->aux == (void *)-2
+	  || dr->aux == (void *)-3)
 	{
 	  if (nested_in_vect_loop_p (loop, stmt_info))
 	    return opt_result::failure_at (stmt_info->stmt,
 					   "not vectorized:"
 					   " data ref analysis failed: %G",
 					   stmt_info->stmt);
-	  STMT_VINFO_SIMD_LANE_ACCESS_P (stmt_info) = true;
+	  STMT_VINFO_SIMD_LANE_ACCESS_P (stmt_info)
+	    = -(uintptr_t) dr->aux;
 	}
 
       tree base = get_base_address (DR_REF (dr));

gcc/tree-vect-loop.c

@@ -824,6 +824,7 @@ _loop_vec_info::_loop_vec_info (struct loop *loop_in, vec_info_shared *shared)
     peeling_for_alignment (0),
     ptr_mask (0),
     ivexpr_map (NULL),
+    scan_map (NULL),
     slp_unrolling_factor (1),
     single_scalar_iteration_cost (0),
     vectorizable (false),
@@ -863,8 +864,8 @@ _loop_vec_info::_loop_vec_info (struct loop *loop_in, vec_info_shared *shared)
 	  gimple *stmt = gsi_stmt (si);
 	  gimple_set_uid (stmt, 0);
 	  add_stmt (stmt);
-	  /* If .GOMP_SIMD_LANE call for the current loop has 2 arguments, the
-	     second argument is the #pragma omp simd if (x) condition, when 0,
+	  /* If .GOMP_SIMD_LANE call for the current loop has 3 arguments, the
+	     third argument is the #pragma omp simd if (x) condition, when 0,
 	     loop shouldn't be vectorized, when non-zero constant, it should
 	     be vectorized normally, otherwise versioned with vectorized loop
 	     done if the condition is non-zero at runtime.  */
@@ -872,12 +873,12 @@ _loop_vec_info::_loop_vec_info (struct loop *loop_in, vec_info_shared *shared)
 	      && is_gimple_call (stmt)
 	      && gimple_call_internal_p (stmt)
 	      && gimple_call_internal_fn (stmt) == IFN_GOMP_SIMD_LANE
-	      && gimple_call_num_args (stmt) >= 2
+	      && gimple_call_num_args (stmt) >= 3
 	      && TREE_CODE (gimple_call_arg (stmt, 0)) == SSA_NAME
 	      && (loop_in->simduid
 		  == SSA_NAME_VAR (gimple_call_arg (stmt, 0))))
 	    {
-	      tree arg = gimple_call_arg (stmt, 1);
+	      tree arg = gimple_call_arg (stmt, 2);
 	      if (integer_zerop (arg) || TREE_CODE (arg) == SSA_NAME)
 		simd_if_cond = arg;
 	      else
@@ -959,6 +960,7 @@ _loop_vec_info::~_loop_vec_info ()
 
   release_vec_loop_masks (&masks);
   delete ivexpr_map;
+  delete scan_map;
 
   loop->aux = NULL;
 }

gcc/tree-vectorizer.h

@@ -491,6 +491,10 @@ typedef struct _loop_vec_info : public vec_info {
   /* Map of IV base/step expressions to inserted name in the preheader.  */
   hash_map<tree_operand_hash, tree> *ivexpr_map;
 
+  /* Map of OpenMP "omp simd array" scan variables to corresponding
+     rhs of the store of the initializer.  */
+  hash_map<tree, tree> *scan_map;
+
   /* The unrolling factor needed to SLP the loop. In case of that pure SLP is
      applied to the loop, i.e., no unrolling is needed, this is 1.  */
   poly_uint64 slp_unrolling_factor;
@@ -913,7 +917,7 @@ struct _stmt_vec_info {
   bool strided_p;
 
   /* For both loads and stores.  */
-  bool simd_lane_access_p;
+  unsigned simd_lane_access_p : 2;
 
   /* Classifies how the load or store is going to be implemented
      for loop vectorization.  */