natClass.cc (_Jv_IsAssignableFrom): Ensure that ancestors table index is within allowed bounds.

2001-06-05  Martin Kahlert  <martin.kahlert@infineon.com>
	    Bryce McKinlay  <bryce@waitaki.otago.ac.nz>

	* java/lang/natClass.cc (_Jv_IsAssignableFrom): Ensure that ancestors
	table index is within allowed bounds. Ensure that we dont try to access
	class itable at a negative offset. Avoid an ancestor table lookup if
	source is a primitive type class.
	(isInstance): Remove redundant isPrimitive() check.

Co-Authored-By: Bryce McKinlay <bryce@waitaki.otago.ac.nz>

From-SVN: r42898

libjava/ChangeLog

+2001-06-05  Martin Kahlert  <martin.kahlert@infineon.com>
+	    Bryce McKinlay  <bryce@waitaki.otago.ac.nz>
+
+	* java/lang/natClass.cc (_Jv_IsAssignableFrom): Ensure that ancestors
+	table index is within allowed bounds. Ensure that we dont try to access
+	class itable at a negative offset. Avoid an ancestor table lookup if
+	source is a primitive type class.
+	(isInstance): Remove redundant isPrimitive() check.
+
 2001-06-04  Tom Tromey  <tromey@redhat.com>
 
 	* java/security/PublicKey.java: Extend Key.

libjava/java/lang/natClass.cc

@@ -633,7 +633,7 @@ java::lang::Class::isAssignableFrom (jclass klass)
 jboolean
 java::lang::Class::isInstance (jobject obj)
 {
-  if (__builtin_expect (! obj || isPrimitive (), false))
+  if (! obj)
     return false;
   _Jv_InitClass (this);
   return _Jv_IsAssignableFrom (this, JV_CLASS (obj));
@@ -939,19 +939,29 @@ _Jv_IsAssignableFrom (jclass target, jclass source)
       if (cl_iindex < if_idt->iface.ioffsets[0])
         {
 	  jshort offset = if_idt->iface.ioffsets[cl_iindex];
-	  if (offset < cl_idt->cls.itable_length
+	  if (offset != -1 && offset < cl_idt->cls.itable_length
 	      && cl_idt->cls.itable[offset] == target)
 	    return true;
 	}
       return false;
     }
      
-  if ((target == &ObjectClass && !source->isPrimitive())
-      || (source->ancestors != NULL 
-	  && source->ancestors[source->depth - target->depth] == target))
+  // Primitive TYPE classes are only assignable to themselves.
+  if (__builtin_expect (target->isPrimitive(), false))
+    return false;
+    
+  if (target == &ObjectClass)
+    {
+      if (source->isPrimitive())
+        return false;
+      return true;
+    }
+  else if (source->ancestors != NULL 
+           && source->depth >= target->depth
+	   && source->ancestors[source->depth - target->depth] == target)
     return true;
       
- return false;
+  return false;
 }
 
 // Interface type checking, the slow way. Returns TRUE if IFACE is a