i386: Fix up *one_cmplv*2* insn with avx512f [PR94343]

This define_insn has two issues.
One is that with -mavx512f -mno-avx512vl it can emit an AVX512VL-only insn
- 128-bit or 256-bit EVEX encoded vpternlog{d,q}.
Another one is that because there is no vpternlog{b,w}, we emit vpternlogd
instead, but then we shouldn't pretend we support masking of that, because
we don't.
The first one can be fixed by forcing the use of %zmm* registers instead of
%xmm* or %ymm* if AVX512F but not AVX512VL, like we do for a couple of other
insns (although that is primarily done in order to support %xmm16+ regs).
But we need to make sure that in that case the input operand isn't memory,
because while we can read and store the higher bits of registers, we don't
want to read from memory more bytes than what we should read.

A variant to these two if_then_else set attrs, condition in the output and
larger condition would be 4 different define_insns (one with something like
VI48_AVX512VL iterator, masking, no g modifiers and "vm" input constraint,
another one with VI48_AVX iterator, !TARGET_AVX512VL in condition,
no masking, g modifiers and "v" input constraint, one with VI12_AVX512VL
iterator, no masking, no g modifiers and "vm" input constraint and last one with
VI12_AVX2 iterator, !TARGET_AVX512VL in condition, no masking, g modifiers
and "v" input constraint, but I think having one pattern is shorter than
that.

2020-03-30  Jakub Jelinek  <jakub@redhat.com>

	PR target/94343
	* config/i386/sse.md (<mask_codefor>one_cmpl<mode>2<mask_name>): If
	!TARGET_AVX512VL, use 512-bit vpternlog and make sure the input
	operand is a register.  Don't enable masked variants for V*[QH]Imode.

	* gcc.target/i386/avx512f-pr94343.c: New test.
	* gcc.target/i386/avx512vl-pr94343.c: New test.

gcc/ChangeLog

 2020-03-30  Jakub Jelinek  <jakub@redhat.com>
 
+	PR target/94343
+	* config/i386/sse.md (<mask_codefor>one_cmpl<mode>2<mask_name>): If
+	!TARGET_AVX512VL, use 512-bit vpternlog and make sure the input
+	operand is a register.  Don't enable masked variants for V*[QH]Imode.
+
 	PR target/93069
 	* config/i386/sse.md (vec_extract_lo_<mode><mask_name>): Use
 	<store_mask_constraint> instead of m in output operand constraint.

gcc/config/i386/sse.md

@@ -12798,14 +12798,29 @@
 })
 
 (define_insn "<mask_codefor>one_cmpl<mode>2<mask_name>"
-  [(set (match_operand:VI 0 "register_operand" "=v")
-	(xor:VI (match_operand:VI 1 "nonimmediate_operand" "vm")
-		(match_operand:VI 2 "vector_all_ones_operand" "BC")))]
-  "TARGET_AVX512F"
-  "vpternlog<ternlogsuffix>\t{$0x55, %1, %0, %0<mask_operand3>|%0<mask_operand3>, %0, %1, 0x55}"
+  [(set (match_operand:VI 0 "register_operand" "=v,v")
+	(xor:VI (match_operand:VI 1 "nonimmediate_operand" "v,m")
+		(match_operand:VI 2 "vector_all_ones_operand" "BC,BC")))]
+  "TARGET_AVX512F
+   && (!<mask_applied>
+       || <ssescalarmode>mode == SImode
+       || <ssescalarmode>mode == DImode)"
+{
+  if (TARGET_AVX512VL)
+    return "vpternlog<ternlogsuffix>\t{$0x55, %1, %0, %0<mask_operand3>|%0<mask_operand3>, %0, %1, 0x55}";
+  else
+    return "vpternlog<ternlogsuffix>\t{$0x55, %g1, %g0, %g0<mask_operand3>|%g0<mask_operand3>, %g0, %g1, 0x55}";
+}
   [(set_attr "type" "sselog")
    (set_attr "prefix" "evex")
-   (set_attr "mode" "<sseinsnmode>")])
+   (set (attr "mode")
+        (if_then_else (match_test "TARGET_AVX512VL")
+		      (const_string "<sseinsnmode>")
+		      (const_string "XI")))
+   (set (attr "enabled")
+	(if_then_else (eq_attr "alternative" "1")
+		      (symbol_ref "<MODE_SIZE> == 64 || TARGET_AVX512VL")
+		      (const_int 1)))])
 
 (define_expand "<sse2_avx2>_andnot<mode>3"
   [(set (match_operand:VI_AVX2 0 "register_operand")

gcc/testsuite/ChangeLog

+2020-03-30  Jakub Jelinek  <jakub@redhat.com>
+
+	PR target/94343
+	* gcc.target/i386/avx512f-pr94343.c: New test.
+	* gcc.target/i386/avx512vl-pr94343.c: New test.
+
 2020-03-30  Martin Liska  <mliska@suse.cz>
 
 	PR rtl-optimization/87716

gcc/testsuite/gcc.target/i386/avx512f-pr94343.c

+/* PR target/94343 */
+/* { dg-do compile } */
+/* { dg-options "-O2 -mavx512f -mno-avx512vl" } */
+/* { dg-final { scan-assembler-not "vpternlogd\[^\n\r]*xmm\[0-9]*" } } */
+
+typedef int __v4si __attribute__((vector_size (16)));
+
+__v4si
+foo (__v4si a)
+{
+  return ~a;
+}

gcc/testsuite/gcc.target/i386/avx512vl-pr94343.c

+/* PR target/94343 */
+/* { dg-do compile } */
+/* { dg-options "-O2 -mavx512vl" } */
+/* { dg-final { scan-assembler "vpternlogd\[^\n\r]*xmm\[0-9]*" } } */
+
+typedef int __v4si __attribute__((vector_size (16)));
+
+__v4si
+foo (__v4si a)
+{
+  return ~a;
+}