re PR target/89112 (Incorrect code generated by rs6000 memcmp expansion)

2019-02-05  Aaron Sawdey  <acsawdey@linux.ibm.com>

	PR target/89112
	* config/rs6000/rs6000-string.c (do_ifelse, expand_cmp_vec_sequence,
	expand_compare_loop, expand_block_compare_gpr,
	expand_strncmp_align_check, expand_strncmp_gpr_sequence): Insert
	REG_BR_PROB notes in inline expansion of memcmp/strncmp. Add
	#include "profile-count.h" and "predict.h" for types and functions
	needed to work with REG_BR_PROB notes.

From-SVN: r268547

gcc/ChangeLog

 2019-02-05  Aaron Sawdey  <acsawdey@linux.ibm.com>
 
 	PR target/89112
+	* config/rs6000/rs6000-string.c (do_ifelse, expand_cmp_vec_sequence,
+	expand_compare_loop, expand_block_compare_gpr,
+	expand_strncmp_align_check, expand_strncmp_gpr_sequence): Insert
+	REG_BR_PROB notes in inline expansion of memcmp/strncmp. Add
+	#include "profile-count.h" and "predict.h" for types and functions
+	needed to work with REG_BR_PROB notes.
+
+2019-02-05  Aaron Sawdey  <acsawdey@linux.ibm.com>
+
+	PR target/89112
 	* config/rs6000/rs6000.md (<bd>tf_<mode>): Generate a local label
 	for the long branch case.
 

gcc/config/rs6000/rs6000-string.c

@@ -35,6 +35,8 @@
 #include "expr.h"
 #include "output.h"
 #include "target.h"
+#include "profile-count.h"
+#include "predict.h"
 
 /* Expand a block clear operation, and return 1 if successful.  Return 0
    if we should let the compiler generate normal code.
@@ -369,6 +371,7 @@ do_load_for_compare_from_addr (machine_mode mode, rtx dest, rtx addr,
    B is the second thing to be compared.
    CR is the condition code reg input, or NULL_RTX.
    TRUE_LABEL is the label to branch to if the condition is true.
+   P is the estimated branch probability for the branch.
 
    The return value is the CR used for the comparison.
    If CR is null_rtx, then a new register of CMPMODE is generated.
@@ -377,7 +380,7 @@ do_load_for_compare_from_addr (machine_mode mode, rtx dest, rtx addr,
 
 static void
 do_ifelse (machine_mode cmpmode, rtx_code comparison,
-	   rtx a, rtx b, rtx cr, rtx true_label)
+	   rtx a, rtx b, rtx cr, rtx true_label, profile_probability br_prob)
 {
   gcc_assert ((a == NULL_RTX && b == NULL_RTX && cr != NULL_RTX)
 	      || (a != NULL_RTX && b != NULL_RTX));
@@ -395,7 +398,8 @@ do_ifelse (machine_mode cmpmode, rtx_code comparison,
   rtx cmp_rtx = gen_rtx_fmt_ee (comparison, VOIDmode, cr, const0_rtx);
 
   rtx ifelse = gen_rtx_IF_THEN_ELSE (VOIDmode, cmp_rtx, label_ref, pc_rtx);
-  rtx j = emit_jump_insn (gen_rtx_SET (pc_rtx, ifelse));
+  rtx_insn *j = emit_jump_insn (gen_rtx_SET (pc_rtx, ifelse));
+  add_reg_br_prob_note (j, br_prob);
   JUMP_LABEL (j) = true_label;
   LABEL_NUSES (true_label) += 1;
 }
@@ -781,7 +785,8 @@ expand_cmp_vec_sequence (unsigned HOST_WIDE_INT bytes_to_compare,
       rtx lab_ref = gen_rtx_LABEL_REF (VOIDmode, dst_label);
       rtx ifelse = gen_rtx_IF_THEN_ELSE (VOIDmode, cmp_rtx,
 					 lab_ref, pc_rtx);
-      rtx j2 = emit_jump_insn (gen_rtx_SET (pc_rtx, ifelse));
+      rtx_insn *j2 = emit_jump_insn (gen_rtx_SET (pc_rtx, ifelse));
+      add_reg_br_prob_note (j2, profile_probability::likely ());
       JUMP_LABEL (j2) = dst_label;
       LABEL_NUSES (dst_label) += 1;
 
@@ -1036,7 +1041,7 @@ expand_compare_loop (rtx operands[])
 
   /* Difference found is stored here before jump to diff_label.  */
   rtx diff = gen_reg_rtx (word_mode);
-  rtx j;
+  rtx_insn *j;
 
   /* Example of generated code for 35 bytes aligned 1 byte.
 
@@ -1120,11 +1125,11 @@ expand_compare_loop (rtx operands[])
       /* Check for > max_bytes bytes.  We want to bail out as quickly as
 	 possible if we have to go over to memcmp.  */
       do_ifelse (CCmode, GT, bytes_rtx, GEN_INT (max_bytes),
-		 NULL_RTX, library_call_label);
+		 NULL_RTX, library_call_label, profile_probability::even ());
 
       /* Check for < loop_bytes bytes.  */
       do_ifelse (CCmode, LT, bytes_rtx, GEN_INT (loop_bytes),
-		 NULL_RTX, cleanup_label);
+		 NULL_RTX, cleanup_label, profile_probability::even ());
 
       /* Loop compare bytes and iterations if bytes>max_bytes.  */
       rtx mb_reg = gen_reg_rtx (word_mode);
@@ -1165,7 +1170,7 @@ expand_compare_loop (rtx operands[])
 	{
 	  rtx lab_after = gen_label_rtx ();
 	  do_ifelse (CCmode, LE, bytes_rtx, GEN_INT (max_bytes),
-		     NULL_RTX, lab_after);
+		     NULL_RTX, lab_after, profile_probability::even ());
 	  emit_move_insn (loop_cmp, mb_reg);
 	  emit_move_insn (iter, mi_reg);
 	  emit_label (lab_after);
@@ -1236,7 +1241,7 @@ expand_compare_loop (rtx operands[])
 	}
 
       do_ifelse (GET_MODE (dcond), NE, NULL_RTX, NULL_RTX,
-		 dcond, diff_label);
+		 dcond, diff_label, profile_probability::unlikely ());
 
       if (TARGET_P9_MISC)
 	{
@@ -1260,6 +1265,7 @@ expand_compare_loop (rtx operands[])
       else
 	j = emit_jump_insn (gen_bdnztf_si (loop_top_label, ctr, ctr,
 					   eqrtx, dcond));
+      add_reg_br_prob_note (j, profile_probability::likely ());
       JUMP_LABEL (j) = loop_top_label;
       LABEL_NUSES (loop_top_label) += 1;
     }
@@ -1272,9 +1278,11 @@ expand_compare_loop (rtx operands[])
      code.  If we exit here with a nonzero diff, it is
      because the second word differed.  */
   if (TARGET_P9_MISC)
-    do_ifelse (CCUNSmode, NE, NULL_RTX, NULL_RTX, dcond, diff_label);
+    do_ifelse (CCUNSmode, NE, NULL_RTX, NULL_RTX, dcond,
+	       diff_label, profile_probability::unlikely ());
   else
-    do_ifelse (CCmode, NE, diff, const0_rtx, NULL_RTX, diff_label);
+    do_ifelse (CCmode, NE, diff, const0_rtx, NULL_RTX,
+	       diff_label, profile_probability::unlikely ());
 
   if (library_call_label != NULL && bytes_is_const && bytes > max_bytes)
     {
@@ -1317,7 +1325,7 @@ expand_compare_loop (rtx operands[])
 	     loop with a branch to cleanup_label.  */
 	  emit_move_insn (target, const0_rtx);
 	  do_ifelse (CCmode, EQ, cmp_rem, const0_rtx,
-		     NULL_RTX, final_label);
+		     NULL_RTX, final_label, profile_probability::unlikely ());
 	}
 
       rtx final_cleanup = gen_label_rtx ();
@@ -1327,9 +1335,12 @@ expand_compare_loop (rtx operands[])
 	{
 	  /* If remainder length < word length, branch to final
 	     cleanup compare.  */
+	  
 	  if (!bytes_is_const)
-	    do_ifelse (CCmode, LT, cmp_rem, GEN_INT (load_mode_size),
-		       NULL_RTX, final_cleanup);
+	    {
+	      do_ifelse (CCmode, LT, cmp_rem, GEN_INT (load_mode_size),
+			 NULL_RTX, final_cleanup, profile_probability::even ());
+	    }
 
 	  /* load and compare 8B */
 	  do_load_for_compare_from_addr (load_mode, d1_1,
@@ -1354,7 +1365,7 @@ expand_compare_loop (rtx operands[])
 	    }
 
 	  do_ifelse (GET_MODE (dcond), NE, NULL_RTX, NULL_RTX,
-		     dcond, diff_label);
+		     dcond, diff_label, profile_probability::even ());
 
 	  do_add3 (src1_addr, src1_addr, GEN_INT (load_mode_size));
 	  do_add3 (src2_addr, src2_addr, GEN_INT (load_mode_size));
@@ -1365,9 +1376,8 @@ expand_compare_loop (rtx operands[])
 	  else
 	    /* See if remaining length is now zero.  We previously set
 	       target to 0 so we can just jump to the end.  */
-	    do_ifelse (CCmode, EQ, cmp_rem, const0_rtx,
-		       NULL_RTX, final_label);
-
+	    do_ifelse (CCmode, EQ, cmp_rem, const0_rtx, NULL_RTX,
+		       final_label, profile_probability::unlikely ());
 	}
 
       /* Cases:
@@ -1450,7 +1460,7 @@ expand_compare_loop (rtx operands[])
 	     than one loop iteration, in which case go do the overlap
 	     load compare path.  */
 	  do_ifelse (CCmode, GT, bytes_rtx, GEN_INT (loop_bytes),
-		     NULL_RTX, nonconst_overlap);
+		     NULL_RTX, nonconst_overlap, profile_probability::even ());
 
 	  rtx rem4k = gen_reg_rtx (word_mode);
 	  rtx dist1 = gen_reg_rtx (word_mode);
@@ -1460,12 +1470,14 @@ expand_compare_loop (rtx operands[])
 	    emit_insn (gen_andsi3 (dist1, src1_addr, GEN_INT (0xfff)));
 	  else
 	    emit_insn (gen_anddi3 (dist1, src1_addr, GEN_INT (0xfff)));
-	  do_ifelse (CCmode, LE, dist1, rem4k, NULL_RTX, handle4k_label);
+	  do_ifelse (CCmode, LE, dist1, rem4k, NULL_RTX,
+		     handle4k_label, profile_probability::very_unlikely ());
 	  if (word_mode == SImode)
 	    emit_insn (gen_andsi3 (dist2, src2_addr, GEN_INT (0xfff)));
 	  else
 	    emit_insn (gen_anddi3 (dist2, src2_addr, GEN_INT (0xfff)));
-	  do_ifelse (CCmode, LE, dist2, rem4k, NULL_RTX, handle4k_label);
+	  do_ifelse (CCmode, LE, dist2, rem4k, NULL_RTX,
+		     handle4k_label, profile_probability::very_unlikely ());
 
 	  /* We don't have a 4k boundary to deal with, so do
 	     a load/shift/compare and jump to diff.  */
@@ -1817,7 +1829,8 @@ expand_block_compare_gpr(unsigned HOST_WIDE_INT bytes, unsigned int base_align,
 	      rtx ne_rtx = gen_rtx_NE (VOIDmode, cr, const0_rtx);
 	      rtx ifelse = gen_rtx_IF_THEN_ELSE (VOIDmode, ne_rtx,
 						 fin_ref, pc_rtx);
-	      rtx j = emit_jump_insn (gen_rtx_SET (pc_rtx, ifelse));
+	      rtx_insn *j = emit_jump_insn (gen_rtx_SET (pc_rtx, ifelse));
+	      add_reg_br_prob_note (j, profile_probability::unlikely ());
 	      JUMP_LABEL (j) = final_label;
 	      LABEL_NUSES (final_label) += 1;
 	    }
@@ -2095,7 +2108,8 @@ expand_strncmp_align_check (rtx strncmp_label, rtx src_addr, HOST_WIDE_INT bytes
 
   rtx ifelse = gen_rtx_IF_THEN_ELSE (VOIDmode, cmp_rtx,
 				     lab_ref, pc_rtx);
-  rtx j = emit_jump_insn (gen_rtx_SET (pc_rtx, ifelse));
+  rtx_insn *j = emit_jump_insn (gen_rtx_SET (pc_rtx, ifelse));
+  add_reg_br_prob_note (j, profile_probability::unlikely ());
   JUMP_LABEL (j) = strncmp_label;
   LABEL_NUSES (strncmp_label) += 1;
 }
@@ -2265,7 +2279,8 @@ expand_strncmp_gpr_sequence (unsigned HOST_WIDE_INT bytes_to_compare,
 
 	      rtx ifelse = gen_rtx_IF_THEN_ELSE (VOIDmode, cmp_rtx,
 						 lab_ref, pc_rtx);
-	      rtx j = emit_jump_insn (gen_rtx_SET (pc_rtx, ifelse));
+	      rtx_insn *j = emit_jump_insn (gen_rtx_SET (pc_rtx, ifelse));
+	      add_reg_br_prob_note (j, profile_probability::unlikely ());
 	      JUMP_LABEL (j) = final_move_label;
 	      LABEL_NUSES (final_move_label) += 1;
 
@@ -2282,7 +2297,8 @@ expand_strncmp_gpr_sequence (unsigned HOST_WIDE_INT bytes_to_compare,
 
 	      rtx ifelse0 = gen_rtx_IF_THEN_ELSE (VOIDmode, cmp0eq_rtx,
 						 lab_ref, pc_rtx);
-	      rtx j0 = emit_jump_insn (gen_rtx_SET (pc_rtx, ifelse0));
+	      rtx_insn *j0 = emit_jump_insn (gen_rtx_SET (pc_rtx, ifelse0));
+	      add_reg_br_prob_note (j0, profile_probability::unlikely ());
 	      JUMP_LABEL (j0) = final_move_label;
 	      LABEL_NUSES (final_move_label) += 1;
 	    }
@@ -2325,7 +2341,8 @@ expand_strncmp_gpr_sequence (unsigned HOST_WIDE_INT bytes_to_compare,
 
 	  rtx ifelse = gen_rtx_IF_THEN_ELSE (VOIDmode, cmp_rtx,
 					     lab_ref, pc_rtx);
-	  rtx j = emit_jump_insn (gen_rtx_SET (pc_rtx, ifelse));
+	  rtx_insn *j = emit_jump_insn (gen_rtx_SET (pc_rtx, ifelse));
+	  add_reg_br_prob_note (j, profile_probability::unlikely ());
 	  JUMP_LABEL (j) = dst_label;
 	  LABEL_NUSES (dst_label) += 1;
 	}