re PR target/44452 (gcc.target/i386/abi-2.c and gcc.target/i386/pr22076.c fail…

re PR target/44452 (gcc.target/i386/abi-2.c and gcc.target/i386/pr22076.c fail on 32-bit Solaris 10+/x86)

	gcc/testsuite:
	PR target/44452
	* gcc.target/i386/pr22076.c: Add -mno-vect8-ret-in-mem on
	i?86-*-solaris2.[89], *-*-vxworks*.
	* gcc.target/i386/pr22152.c: Likewise.
	* gcc.target/i386/vect8-ret.c: New test.

	gcc:
	PR target/44452
	* config/i386/i386.opt (mvect8-ret-in-mem): Define.
	* config/i386/i386.c (ix86_target_string): Handle -mvect8-ret-in-mem.
	(ix86_solaris_return_in_memory): Remove.
	* config/i386/i386-protos.h (ix86_solaris_return_in_memory): Remove.
	* config/i386/sol2.h (SUBTARGET_RETURN_IN_MEMORY): Remove.
	(TARGET_SUBTARGET_DEFAULT): Redefine.
	* config/i386/sol2-10.h (TARGET_SUBTARGET_DEFAULT): Update comment.
	* config/i386/vx-common.h (SUBTARGET_RETURN_IN_MEMORY): Remove.
	(TARGET_SUBTARGET_DEFAULT): Redefine.
	* doc/invoke.texi (Option Summary, i386 and x86-64 Options): Add
	-mvect8-ret-in-mem.
	(i386 and x86-64 Options): Document -mvect8-ret-in-mem.

From-SVN: r164687

gcc/ChangeLog

+2010-09-28  Rainer Orth  <ro@CeBiTec.Uni-Bielefeld.DE>
+
+	PR target/44452
+	* config/i386/i386.opt (mvect8-ret-in-mem): Define.
+	* config/i386/i386.c (ix86_target_string): Handle -mvect8-ret-in-mem.
+	(ix86_solaris_return_in_memory): Remove.
+	* config/i386/i386-protos.h (ix86_solaris_return_in_memory): Remove.
+	* config/i386/sol2.h (SUBTARGET_RETURN_IN_MEMORY): Remove.
+	(TARGET_SUBTARGET_DEFAULT): Redefine.
+	* config/i386/sol2-10.h (TARGET_SUBTARGET_DEFAULT): Update comment.
+	* config/i386/vx-common.h (SUBTARGET_RETURN_IN_MEMORY): Remove.
+	(TARGET_SUBTARGET_DEFAULT): Redefine.
+	* doc/invoke.texi (Option Summary, i386 and x86-64 Options): Add
+	-mvect8-ret-in-mem.
+	(i386 and x86-64 Options): Document -mvect8-ret-in-mem.
+
 2010-09-29  Alan Modra  <amodra@gmail.com>
 
 	PR target/45807

gcc/config/i386/i386-protos.h

@@ -140,7 +140,6 @@ extern rtx ix86_libcall_value (enum machine_mode);
 extern bool ix86_function_arg_regno_p (int);
 extern void ix86_asm_output_function_label (FILE *, const char *, tree);
 extern int ix86_function_arg_boundary (enum machine_mode, const_tree);
-extern bool ix86_solaris_return_in_memory (const_tree, const_tree);
 extern rtx ix86_force_to_memory (enum machine_mode, rtx);
 extern void ix86_free_from_memory (enum machine_mode);
 extern void ix86_call_abi_override (const_tree);

gcc/config/i386/i386.c

@@ -2632,6 +2632,7 @@ ix86_target_string (int isa, int flags, const char *arch, const char *tune,
     { "-msseregparm",			MASK_SSEREGPARM },
     { "-mstack-arg-probe",		MASK_STACK_PROBE },
     { "-mtls-direct-seg-refs",		MASK_TLS_DIRECT_SEG_REFS },
+    { "-mvect8-ret-in-mem",		MASK_VECT8_RETURNS },
     { "-m8bit-idiv",			MASK_USE_8BIT_IDIV },
   };
 
@@ -6849,9 +6850,9 @@ return_in_memory_32 (const_tree type, enum machine_mode mode)
 	return false;
 
       /* MMX/3dNow values are returned in MM0,
-	 except when it doesn't exits.  */
+	 except when it doesn't exits or the ABI prescribes otherwise.  */
       if (size == 8)
-	return !TARGET_MMX;
+	return !TARGET_MMX || TARGET_VECT8_RETURNS;
 
       /* SSE values are returned in XMM0, except when it doesn't exist.  */
       if (size == 16)
@@ -6915,43 +6916,6 @@ ix86_return_in_memory (const_tree type, const_tree fntype ATTRIBUTE_UNUSED)
 #endif
 }
 
-/* Return false iff TYPE is returned in memory.  This version is used
-   on Solaris 2.  It is similar to the generic ix86_return_in_memory,
-   but differs notably in that when MMX is available, 8-byte vectors
-   are returned in memory, rather than in MMX registers.  */
-
-bool
-ix86_solaris_return_in_memory (const_tree type, const_tree fntype ATTRIBUTE_UNUSED)
-{
-  int size;
-  enum machine_mode mode = type_natural_mode (type, NULL);
-
-  if (TARGET_64BIT)
-    return return_in_memory_64 (type, mode);
-
-  if (mode == BLKmode)
-    return 1;
-
-  size = int_size_in_bytes (type);
-
-  if (VECTOR_MODE_P (mode))
-    {
-      /* Return in memory only if MMX registers *are* available.  This
-	 seems backwards, but it is consistent with the existing
-	 Solaris x86 ABI.  */
-      if (size == 8)
-	return TARGET_MMX;
-      if (size == 16)
-	return !TARGET_SSE;
-    }
-  else if (mode == TImode)
-    return !TARGET_SSE;
-  else if (mode == XFmode)
-    return 0;
-
-  return size > 12;
-}
-
 /* When returning SSE vector types, we have a choice of either
      (1) being abi incompatible with a -march switch, or
      (2) generating an error.

gcc/config/i386/i386.opt

@@ -244,6 +244,10 @@ mveclibabi=
 Target RejectNegative Joined Var(ix86_veclibabi_string)
 Vector library ABI to use
 
+mvect8-ret-in-mem
+Target Report Mask(VECT8_RETURNS) Save
+Return 8-byte vectors in memory
+
 mrecip
 Target Report Mask(RECIP) Save
 Generate reciprocals instead of divss and sqrtss.

gcc/config/i386/sol2-10.h

@@ -89,9 +89,12 @@ along with GCC; see the file COPYING3.  If not see
     }								\
   while (0)
 
+/* Override i386/sol2.h version: return 8-byte vectors in MMX registers if
+   possible, matching Sun Studio 12 Update 1+ compilers and other x86
+   targets.  */
 #undef TARGET_SUBTARGET_DEFAULT
-#define TARGET_SUBTARGET_DEFAULT (MASK_80387 | MASK_IEEE_FP	\
-				  | MASK_FLOAT_RETURNS)
+#define TARGET_SUBTARGET_DEFAULT \
+	(MASK_80387 | MASK_IEEE_FP | MASK_FLOAT_RETURNS)
 
 #define SUBTARGET_OPTIMIZATION_OPTIONS			\
   do							\

gcc/config/i386/sol2.h

@@ -140,9 +140,15 @@ along with GCC; see the file COPYING3.  If not see
 /* Register the Solaris-specific #pragma directives.  */
 #define REGISTER_SUBTARGET_PRAGMAS() solaris_register_pragmas ()
 
+/* Undo i386/sysv4.h version.  */
 #undef SUBTARGET_RETURN_IN_MEMORY
-#define SUBTARGET_RETURN_IN_MEMORY(TYPE, FNTYPE) \
-	ix86_solaris_return_in_memory (TYPE, FNTYPE)
+
+/* Augment i386/unix.h version to return 8-byte vectors in memory, matching
+   Sun Studio compilers until version 12, the only ones supported on
+   Solaris 8 and 9.  */
+#undef TARGET_SUBTARGET_DEFAULT
+#define TARGET_SUBTARGET_DEFAULT \
+	(MASK_80387 | MASK_IEEE_FP | MASK_FLOAT_RETURNS | MASK_VECT8_RETURNS)
 
 /* Output a simple call for .init/.fini.  */
 #define ASM_OUTPUT_CALL(FILE, FN)				\

gcc/config/i386/vx-common.h

@@ -20,10 +20,11 @@ along with GCC; see the file COPYING3.  If not see
 #define ASM_OUTPUT_ALIGNED_BSS(FILE, DECL, NAME, SIZE, ALIGN) \
   asm_output_aligned_bss (FILE, DECL, NAME, SIZE, ALIGN)
 
-/* VxWorks uses the same ABI as Solaris 2.  */
+/* VxWorks uses the same ABI as Solaris 2, so use i386/sol2.h version.  */
 
-#define SUBTARGET_RETURN_IN_MEMORY(TYPE, FNTYPE) \
-	ix86_solaris_return_in_memory (TYPE, FNTYPE)
+#undef TARGET_SUBTARGET_DEFAULT
+#define TARGET_SUBTARGET_DEFAULT \
+	(MASK_80387 | MASK_IEEE_FP | MASK_FLOAT_RETURNS | MASK_VECT8_RETURNS)
 
 /* Provide our target specific DBX_REGISTER_NUMBER, as advertised by the
    common svr4.h.  VxWorks relies on the SVR4 numbering.  */

gcc/doc/invoke.texi

@@ -601,7 +601,8 @@ Objective-C and Objective-C++ Dialects}.
 -minline-stringops-dynamically -mstringop-strategy=@var{alg} @gol
 -mpush-args  -maccumulate-outgoing-args  -m128bit-long-double @gol
 -m96bit-long-double  -mregparm=@var{num}  -msseregparm @gol
--mveclibabi=@var{type} -mpc32 -mpc64 -mpc80 -mstackrealign @gol
+-mveclibabi=@var{type} -mvect8-ret-in-mem @gol
+-mpc32 -mpc64 -mpc80 -mstackrealign @gol
 -momit-leaf-frame-pointer  -mno-red-zone -mno-tls-direct-seg-refs @gol
 -mcmodel=@var{code-model} -mabi=@var{name} @gol
 -m32  -m64 -mlarge-data-threshold=@var{num} @gol
@@ -12246,6 +12247,16 @@ function by using the function attribute @samp{sseregparm}.
 modules with the same value, including any libraries.  This includes
 the system libraries and startup modules.
 
+@item -mvect8-ret-in-mem
+@opindex mvect8-ret-in-mem
+Return 8-byte vectors in memory instead of MMX registers.  This is the
+default on Solaris~8 and 9 and VxWorks to match the ABI of the Sun
+Studio compilers until version 12.  Later compiler versions (starting
+with Studio 12 Update~1) follow the ABI used by other x86 targets, which
+is the default on Solaris~10 and later.  @emph{Only} use this option if
+you need to remain compatible with existing code produced by those
+previous compiler versions or older versions of GCC.
+
 @item -mpc32
 @itemx -mpc64
 @itemx -mpc80

gcc/testsuite/ChangeLog

+2010-09-28  Rainer Orth  <ro@CeBiTec.Uni-Bielefeld.DE>
+
+	PR target/44452
+	* gcc.target/i386/pr22076.c: Add -mno-vect8-ret-in-mem on
+	i?86-*-solaris2.[89], *-*-vxworks*.
+	* gcc.target/i386/pr22152.c: Likewise.
+	* gcc.target/i386/vect8-ret.c: New test.
+
 2010-09-28  Jie Zhang  <jie@codesourcery.com>
 
 	* gcc.dg/Wcxx-compat-12.c: Add -fno-short-enums.

gcc/testsuite/gcc.target/i386/pr22076.c

 /* { dg-do compile } */
 /* { dg-options "-O2 -fomit-frame-pointer -flax-vector-conversions -mmmx" } */
+/* { dg-options "-O2 -fomit-frame-pointer -flax-vector-conversions -mmmx -mno-vect8-ret-in-mem" { target i?86-*-solaris2.[89] *-*-vxworks* } } */
 
 #include <mmintrin.h>
 

gcc/testsuite/gcc.target/i386/pr22152.c

 /* { dg-do compile } */
 /* { dg-options "-O2 -msse2" } */
+/* { dg-options "-O2 -msse2 -mno-vect8-ret-in-mem" { target i?86-*-solaris2.[89] *-*-vxworks* } } */
 
 #include <mmintrin.h>
 

gcc/testsuite/gcc.target/i386/vect8-ret.c

+/* { dg-do compile { target ilp32 } } */
+/* { dg-options "-mmmx" { target i?86-*-solaris2.[89] *-*-vxworks* } } */
+/* { dg-options "-mmmx -mvect8-ret-in-mem" } */
+
+#include <mmintrin.h>
+
+__m64
+vecret (__m64 vect)
+{
+  return vect;
+}
+
+/* { dg-final { scan-assembler-times "movq" 1 } } */