re PR middle-end/38572 (ICE in set_value_range, at tree-vrp.c:398)

2008-12-30  Paolo Bonzini  <bonzini@gnu.org>

	PR tree-optimization/38572
	* tree-vrp.c (vrp_visit_phi_node): Look out for invalid ranges
	and change them to VARYING.

From-SVN: r142962

gcc/ChangeLog

+2008-12-30  Paolo Bonzini  <bonzini@gnu.org>
+
+	PR tree-optimization/38572
+	* tree-vrp.c (vrp_visit_phi_node): Look out for invalid ranges
+	and change them to VARYING.
+
 2008-12-30  Richard Guenther  <rguenther@suse.de>
 
 	PR middle-end/38564

gcc/testsuite/g++.dg/tree-ssa/pr38572.C

+// PR tree-optimization/38572
+// { dg-do compile }
+// { dg-options "-O2" }
+
+// Crash caused by the out-of-bounds enum values (all the remaining cruft
+// is needed only to trigger the appropriate code path in tree-vrp.c).
+enum JSOp
+{
+  JSOP_GETELEM = 5,
+  JSOP_LIMIT
+};
+extern void g ();
+void f (char *pc, char *endpc, int format, char ***fp, enum JSOp op)
+{
+  while (pc <= endpc)
+    {
+      if ((fp && *fp && pc == **fp) || pc == endpc)
+	{
+	  if (format == 1)
+	    op = (JSOp) 256;
+	  else if (format == 2)
+	    op = (JSOp) 257;
+	  else
+	    op = JSOP_GETELEM;
+	}
+      if (op >= JSOP_LIMIT)
+	{
+	  if (format)
+	    g ();
+	}
+    }
+}

gcc/tree-vrp.c

@@ -6361,9 +6361,12 @@ vrp_visit_phi_node (gimple phi)
 	     minimums.  */
 	  if (cmp_min > 0 || cmp_min < 0)
 	    {
-	      /* If we will end up with a (-INF, +INF) range, set it
-		 to VARYING.  */
-	      if (vrp_val_is_max (vr_result.max))
+	      /* If we will end up with a (-INF, +INF) range, set it to
+		 VARYING.  Same if the previous max value was invalid for
+		 the type and we'd end up with vr_result.min > vr_result.max.  */
+	      if (vrp_val_is_max (vr_result.max)
+		  || compare_values (TYPE_MIN_VALUE (TREE_TYPE (vr_result.min)),
+				     vr_result.max) > 0)
 		goto varying;
 
 	      if (!needs_overflow_infinity (TREE_TYPE (vr_result.min))
@@ -6380,9 +6383,12 @@ vrp_visit_phi_node (gimple phi)
 	     the previous one, go all the way to +INF.  */
 	  if (cmp_max < 0 || cmp_max > 0)
 	    {
-	      /* If we will end up with a (-INF, +INF) range, set it
-		 to VARYING.  */
-	      if (vrp_val_is_min (vr_result.min))
+	      /* If we will end up with a (-INF, +INF) range, set it to
+		 VARYING.  Same if the previous min value was invalid for
+		 the type and we'd end up with vr_result.max < vr_result.min.  */
+	      if (vrp_val_is_min (vr_result.min)
+		  || compare_values (TYPE_MAX_VALUE (TREE_TYPE (vr_result.max)),
+				     vr_result.min) < 0)
 		goto varying;
 
 	      if (!needs_overflow_infinity (TREE_TYPE (vr_result.max))