[ARM] PR target/69875 Fix atomic_loaddi expansion

	PR target/69875
	* config/arm/arm.h (TARGET_HAVE_LPAE): Define.
	* config/arm/unspecs.md (VUNSPEC_LDRD_ATOMIC): New value.
	* config/arm/sync.md (arm_atomic_loaddi2_ldrd): New pattern.
	(atomic_loaddi_1): Delete.
	(atomic_loaddi): Rewrite expander using the above changes.

	* gcc.target/arm/atomic_loaddi_acquire.x: New file.
	* gcc.target/arm/atomic_loaddi_relaxed.x: Likewise.
	* gcc.target/arm/atomic_loaddi_seq_cst.x: Likewise.
	* gcc.target/arm/atomic_loaddi_1.c: New test.
	* gcc.target/arm/atomic_loaddi_2.c: Likewise.
	* gcc.target/arm/atomic_loaddi_3.c: Likewise.
	* gcc.target/arm/atomic_loaddi_4.c: Likewise.
	* gcc.target/arm/atomic_loaddi_5.c: Likewise.
	* gcc.target/arm/atomic_loaddi_6.c: Likewise.
	* gcc.target/arm/atomic_loaddi_7.c: Likewise.
	* gcc.target/arm/atomic_loaddi_8.c: Likewise.
	* gcc.target/arm/atomic_loaddi_9.c: Likewise.

From-SVN: r233658

gcc/ChangeLog

+2016-02-24  Kyrylo Tkachov  <kyrylo.tkachov@arm.com>
+
+	PR target/69875
+	* config/arm/arm.h (TARGET_HAVE_LPAE): Define.
+	* config/arm/unspecs.md (VUNSPEC_LDRD_ATOMIC): New value.
+	* config/arm/sync.md (arm_atomic_loaddi2_ldrd): New pattern.
+	(atomic_loaddi_1): Delete.
+	(atomic_loaddi): Rewrite expander using the above changes.
+
 2016-02-24  Jakub Jelinek  <jakub@redhat.com>
 
 	PR c/69918

gcc/config/arm/arm.h

@@ -253,6 +253,10 @@ extern void (*arm_lang_output_object_attributes_hook)(void);
 /* Nonzero if this chip supports ldrex and strex */
 #define TARGET_HAVE_LDREX        ((arm_arch6 && TARGET_ARM) || arm_arch7)
 
+/* Nonzero if this chip supports LPAE.  */
+#define TARGET_HAVE_LPAE						\
+  (arm_arch7 && ARM_FSET_HAS_CPU1 (insn_flags, FL_FOR_ARCH7VE))
+
 /* Nonzero if this chip supports ldrex{bh} and strex{bh}.  */
 #define TARGET_HAVE_LDREXBH ((arm_arch6k && TARGET_ARM) || arm_arch7)
 

gcc/config/arm/sync.md

@@ -96,32 +96,62 @@
   [(set_attr "predicable" "yes")
    (set_attr "predicable_short_it" "no")])
 
-;; Note that ldrd and vldr are *not* guaranteed to be single-copy atomic,
-;; even for a 64-bit aligned address.  Instead we use a ldrexd unparied
-;; with a store.
+;; An LDRD instruction usable by the atomic_loaddi expander on LPAE targets
+
+(define_insn "arm_atomic_loaddi2_ldrd"
+  [(set (match_operand:DI 0 "register_operand" "=r")
+	(unspec_volatile:DI
+	  [(match_operand:DI 1 "arm_sync_memory_operand" "Q")]
+	    VUNSPEC_LDRD_ATOMIC))]
+  "ARM_DOUBLEWORD_ALIGN && TARGET_HAVE_LPAE"
+  "ldrd%?\t%0, %H0, %C1"
+  [(set_attr "predicable" "yes")
+   (set_attr "predicable_short_it" "no")])
+
+;; There are three ways to expand this depending on the architecture
+;; features available.  As for the barriers, a load needs a barrier
+;; after it on all non-relaxed memory models except when the load
+;; has acquire semantics (for ARMv8-A).
+
 (define_expand "atomic_loaddi"
   [(match_operand:DI 0 "s_register_operand")		;; val out
    (match_operand:DI 1 "mem_noofs_operand")		;; memory
    (match_operand:SI 2 "const_int_operand")]		;; model
-  "TARGET_HAVE_LDREXD && ARM_DOUBLEWORD_ALIGN"
+  "(TARGET_HAVE_LDREXD || TARGET_HAVE_LPAE || TARGET_HAVE_LDACQ)
+   && ARM_DOUBLEWORD_ALIGN"
 {
-  enum memmodel model = memmodel_from_int (INTVAL (operands[2]));
-  expand_mem_thread_fence (model);
-  emit_insn (gen_atomic_loaddi_1 (operands[0], operands[1]));
-  if (is_mm_seq_cst (model))
+  memmodel model = memmodel_from_int (INTVAL (operands[2]));
+
+  /* For ARMv8-A we can use an LDAEXD to atomically load two 32-bit registers
+     when acquire or stronger semantics are needed.  When the relaxed model is
+     used this can be relaxed to a normal LDRD.  */
+  if (TARGET_HAVE_LDACQ)
+    {
+      if (is_mm_relaxed (model))
+	emit_insn (gen_arm_atomic_loaddi2_ldrd (operands[0], operands[1]));
+      else
+	emit_insn (gen_arm_load_acquire_exclusivedi (operands[0], operands[1]));
+
+      DONE;
+    }
+
+  /* On LPAE targets LDRD and STRD accesses to 64-bit aligned
+     locations are 64-bit single-copy atomic.  We still need barriers in the
+     appropriate places to implement the ordering constraints.  */
+  if (TARGET_HAVE_LPAE)
+    emit_insn (gen_arm_atomic_loaddi2_ldrd (operands[0], operands[1]));
+  else
+    emit_insn (gen_arm_load_exclusivedi (operands[0], operands[1]));
+
+
+  /* All non-relaxed models need a barrier after the load when load-acquire
+     instructions are not available.  */
+  if (!is_mm_relaxed (model))
     expand_mem_thread_fence (model);
+
   DONE;
 })
 
-(define_insn "atomic_loaddi_1"
-  [(set (match_operand:DI 0 "s_register_operand" "=r")
-	(unspec:DI [(match_operand:DI 1 "mem_noofs_operand" "Ua")]
-		   UNSPEC_LL))]
-  "TARGET_HAVE_LDREXD && ARM_DOUBLEWORD_ALIGN"
-  "ldrexd%?\t%0, %H0, %C1"
-  [(set_attr "predicable" "yes")
-   (set_attr "predicable_short_it" "no")])
-
 (define_expand "atomic_compare_and_swap<mode>"
   [(match_operand:SI 0 "s_register_operand" "")		;; bool out
    (match_operand:QHSD 1 "s_register_operand" "")	;; val out

gcc/config/arm/unspecs.md

@@ -139,6 +139,7 @@
   VUNSPEC_ATOMIC_XCHG	; Represent an atomic exchange.
   VUNSPEC_ATOMIC_OP	; Represent an atomic operation.
   VUNSPEC_LL		; Represent a load-register-exclusive.
+  VUNSPEC_LDRD_ATOMIC	; Represent an LDRD used as an atomic DImode load.
   VUNSPEC_SC		; Represent a store-register-exclusive.
   VUNSPEC_LAX		; Represent a load-register-acquire-exclusive.
   VUNSPEC_SLX		; Represent a store-register-release-exclusive.

gcc/testsuite/ChangeLog

+2016-02-24  Kyrylo Tkachov  <kyrylo.tkachov@arm.com>
+
+	PR target/69875
+	* gcc.target/arm/atomic_loaddi_acquire.x: New file.
+	* gcc.target/arm/atomic_loaddi_relaxed.x: Likewise.
+	* gcc.target/arm/atomic_loaddi_seq_cst.x: Likewise.
+	* gcc.target/arm/atomic_loaddi_1.c: New test.
+	* gcc.target/arm/atomic_loaddi_2.c: Likewise.
+	* gcc.target/arm/atomic_loaddi_3.c: Likewise.
+	* gcc.target/arm/atomic_loaddi_4.c: Likewise.
+	* gcc.target/arm/atomic_loaddi_5.c: Likewise.
+	* gcc.target/arm/atomic_loaddi_6.c: Likewise.
+	* gcc.target/arm/atomic_loaddi_7.c: Likewise.
+	* gcc.target/arm/atomic_loaddi_8.c: Likewise.
+	* gcc.target/arm/atomic_loaddi_9.c: Likewise.
+
 2016-02-24  Jakub Jelinek  <jakub@redhat.com>
 
 	PR middle-end/69909

gcc/testsuite/gcc.target/arm/atomic_loaddi_1.c

+/* { dg-do compile } */
+/* { dg-options "-std=c11 -O" } */
+/* { dg-require-effective-target arm_arch_v7a_ok } */
+/* { dg-add-options arm_arch_v7a } */
+
+#include "atomic_loaddi_acquire.x"
+
+/* { dg-final { scan-assembler-times "ldrexd\tr\[0-9\]+, r\[0-9\]+, \\\[r\[0-9\]+\\\]" 1 } } */
+/* { dg-final { scan-assembler-times "dmb\tish" 1 } } */

gcc/testsuite/gcc.target/arm/atomic_loaddi_2.c

+/* { dg-do compile } */
+/* { dg-options "-std=c11 -O" } */
+/* { dg-require-effective-target arm_arch_v7ve_ok } */
+/* { dg-add-options arm_arch_v7ve } */
+
+#include "atomic_loaddi_acquire.x"
+
+/* { dg-final { scan-assembler-times "ldrd\tr\[0-9\]+, r\[0-9\]+, \\\[r\[0-9\]+\\\]" 1 } } */
+/* { dg-final { scan-assembler-times "dmb\tish" 1 } } */

gcc/testsuite/gcc.target/arm/atomic_loaddi_3.c

+/* { dg-do compile } */
+/* { dg-options "-std=c11 -O" } */
+/* { dg-require-effective-target arm_arch_v8a_ok } */
+/* { dg-add-options arm_arch_v8a } */
+
+#include "atomic_loaddi_acquire.x"
+
+/* { dg-final { scan-assembler-times "ldaexd\tr\[0-9\]+, r\[0-9\]+, \\\[r\[0-9\]+\\\]" 1 } } */
+/* { dg-final { scan-assembler-not "dmb\tish" } } */

gcc/testsuite/gcc.target/arm/atomic_loaddi_4.c

+/* { dg-do compile } */
+/* { dg-options "-std=c11 -O" } */
+/* { dg-require-effective-target arm_arch_v7a_ok } */
+/* { dg-add-options arm_arch_v7a } */
+
+#include "atomic_loaddi_relaxed.x"
+
+/* { dg-final { scan-assembler-times "ldrexd\tr\[0-9\]+, r\[0-9\]+, \\\[r\[0-9\]+\\\]" 1 } } */
+/* { dg-final { scan-assembler-not "dmb\tish" } } */

gcc/testsuite/gcc.target/arm/atomic_loaddi_5.c

+/* { dg-do compile } */
+/* { dg-options "-std=c11 -O" } */
+/* { dg-require-effective-target arm_arch_v7ve_ok } */
+/* { dg-add-options arm_arch_v7ve } */
+
+#include "atomic_loaddi_relaxed.x"
+
+/* { dg-final { scan-assembler-times "ldrd\tr\[0-9\]+, r\[0-9\]+, \\\[r\[0-9\]+\\\]" 1 } } */
+/* { dg-final { scan-assembler-not "dmb\tish" } } */
\ No newline at end of file

gcc/testsuite/gcc.target/arm/atomic_loaddi_6.c

+/* { dg-do compile } */
+/* { dg-options "-std=c11 -O" } */
+/* { dg-require-effective-target arm_arch_v8a_ok } */
+/* { dg-add-options arm_arch_v8a } */
+
+#include "atomic_loaddi_relaxed.x"
+
+/* { dg-final { scan-assembler-times "ldrd\tr\[0-9\]+, r\[0-9\]+, \\\[r\[0-9\]+\\\]" 1 } } */
+/* { dg-final { scan-assembler-not "dmb\tish" } } */
\ No newline at end of file

gcc/testsuite/gcc.target/arm/atomic_loaddi_7.c

+/* { dg-do compile } */
+/* { dg-options "-std=c11 -O" } */
+/* { dg-require-effective-target arm_arch_v7a_ok } */
+/* { dg-add-options arm_arch_v7a } */
+
+#include "atomic_loaddi_seq_cst.x"
+
+/* { dg-final { scan-assembler-times "ldrexd\tr\[0-9\]+, r\[0-9\]+, \\\[r\[0-9\]+\\\]" 1 } } */
+/* { dg-final { scan-assembler-times "dmb\tish" 1 } } */

gcc/testsuite/gcc.target/arm/atomic_loaddi_8.c

+/* { dg-do compile } */
+/* { dg-options "-std=c11 -O" } */
+/* { dg-require-effective-target arm_arch_v7ve_ok } */
+/* { dg-add-options arm_arch_v7ve } */
+
+#include "atomic_loaddi_seq_cst.x"
+
+/* { dg-final { scan-assembler-times "ldrd\tr\[0-9\]+, r\[0-9\]+, \\\[r\[0-9\]+\\\]" 1 } } */
+/* { dg-final { scan-assembler-times "dmb\tish" 1 } } */

gcc/testsuite/gcc.target/arm/atomic_loaddi_9.c

+/* { dg-do compile } */
+/* { dg-options "-std=c11 -O" } */
+/* { dg-require-effective-target arm_arch_v8a_ok } */
+/* { dg-add-options arm_arch_v8a } */
+
+#include "atomic_loaddi_seq_cst.x"
+
+/* { dg-final { scan-assembler-times "ldaexd\tr\[0-9\]+, r\[0-9\]+, \\\[r\[0-9\]+\\\]" 1 } } */
+/* { dg-final { scan-assembler-not "dmb\tish" } } */

gcc/testsuite/gcc.target/arm/atomic_loaddi_acquire.x

+#include <stdatomic.h>
+
+atomic_ullong foo;
+int glob;
+
+int
+main (void)
+{
+  atomic_load_explicit (&foo, memory_order_acquire);
+  return glob;
+}

gcc/testsuite/gcc.target/arm/atomic_loaddi_relaxed.x

+#include <stdatomic.h>
+
+atomic_ullong foo;
+int glob;
+
+int
+main (void)
+{
+  atomic_load_explicit (&foo, memory_order_relaxed);
+  return glob;
+}

gcc/testsuite/gcc.target/arm/atomic_loaddi_seq_cst.x

+#include <stdatomic.h>
+
+atomic_ullong foo;
+int glob;
+
+int
+main (void)
+{
+  atomic_load_explicit (&foo, memory_order_seq_cst);
+  return glob;
+}