Detect overflow in size calculations in __cxa_vec_new{2,3}

2012-11-03  Florian Weimer  <fweimer@redhat.com>

	* libsupc++/vec.cc (compute_size): New.
	(__cxa_vec_new2, __cxa_vec_new3): Use it.
	* testsuite/18_support/cxa_vec.cc: New.

From-SVN: r193174

libstdc++-v3/ChangeLog

+2012-11-03  Florian Weimer  <fweimer@redhat.com>
+
+	* libsupc++/vec.cc (compute_size): New.
+	(__cxa_vec_new2, __cxa_vec_new3): Use it.
+	* testsuite/18_support/cxa_vec.cc: New.
+
 2012-11-03  Robert Mason  <rbmj@verizon.net>
 
 	* config/os/vxworks/os_defines.h:

libstdc++-v3/libsupc++/vec.cc

 // New abi Support -*- C++ -*-
 
-// Copyright (C) 2000, 2001, 2003, 2004, 2009, 2011
-// Free Software Foundation, Inc.
+// Copyright (C) 2000-2012 Free Software Foundation, Inc.
 //  
 // This file is part of GCC.
 //
@@ -59,6 +58,19 @@ namespace __cxxabiv1
       globals->caughtExceptions = p->nextException;
       globals->uncaughtExceptions += 1;
     }
+
+    // Compute the total size with overflow checking.
+    std::size_t compute_size(std::size_t element_count,
+			     std::size_t element_size,
+			     std::size_t padding_size)
+    {
+      if (element_size && element_count > std::size_t(-1) / element_size)
+	throw std::bad_alloc();
+      std::size_t size = element_count * element_size;
+      if (size + padding_size < size)
+	throw std::bad_alloc();
+      return size + padding_size;
+    }
   }
 
   // Allocate and construct array.
@@ -83,7 +95,8 @@ namespace __cxxabiv1
 		 void *(*alloc) (std::size_t),
 		 void (*dealloc) (void *))
   {
-    std::size_t size = element_count * element_size + padding_size;
+    std::size_t size
+      = compute_size(element_count, element_size, padding_size);
     char *base = static_cast <char *> (alloc (size));
     if (!base)
       return base;
@@ -124,7 +137,8 @@ namespace __cxxabiv1
 		 void *(*alloc) (std::size_t),
 		 void (*dealloc) (void *, std::size_t))
   {
-    std::size_t size = element_count * element_size + padding_size;
+    std::size_t size
+      = compute_size(element_count, element_size, padding_size);
     char *base = static_cast<char *>(alloc (size));
     if (!base)
       return base;

libstdc++-v3/testsuite/18_support/cxa_vec.cc

+// { dg-do run }
+// Avoid use of none-overridable new/delete operators in shared
+// { dg-options "-static" { target *-*-mingw* } }
+// Test __cxa_vec routines
+// Copyright (C) 2000-2012 Free Software Foundation, Inc.
+// Contributed by Nathan Sidwell 7 Apr 2000 <nathan@nathan@codesourcery.com>
+
+#include <cxxabi.h>
+#include <stdio.h>
+#include <new>
+#include <stdlib.h>
+#include <setjmp.h>
+
+// Allocate enough padding to hold an array cookie.
+#ifdef __ARM_EABI__
+static const size_t padding = 8;
+#else
+static const size_t padding = (sizeof (std::size_t));
+#endif
+
+// our pseudo ctors and dtors
+static abi::__cxa_cdtor_return_type ctor (void *x)
+{
+  abort ();
+}
+
+static abi::__cxa_cdtor_return_type dtor (void *x)
+{
+  abort ();
+}
+
+// allocate an array whose size causes an overflow during multiplication
+void test1 ()
+{
+  static const std::size_t large_size =
+    std::size_t(1) << (sizeof(std::size_t) * 8 - 2);
+  try
+    {
+      abi::__cxa_vec_new (large_size, 8, 0, ctor, dtor);
+      abort ();
+    }
+  catch (std::bad_alloc &)
+    {
+    }
+}
+
+// allocate an array whose size causes an overflow during addition
+void test2 ()
+{
+  try
+    {
+      abi::__cxa_vec_new (std::size_t(-1) / 4, 4, padding, ctor, dtor);
+      abort ();
+    }
+  catch (std::bad_alloc &)
+    {
+    }
+}
+
+int main ()
+{
+  test1 ();
+  test2 ();
+}