re PR preprocessor/20078 (Gcc doesn't complain about non-benign macro definitions)

libcpp:
	PR preprocessor/20078
	* include/cpp-id-data.h (struct cpp_macro): Add extra_tokens
	field.
	* include/cpplib.h (SP_DIGRAPH, SP_PREV_WHITE): Define.
	(struct cpp_token): Change flags to unsigned short.
	* lex.c (_cpp_lex_direct): Initialize arg_no for CPP_PASTE tokens.
	(_cpp_equiv_tokens): Check arg_no for CPP_PASTE tokens.
	(cpp_token_val_index): Return CPP_TOKEN_FLD_ARG_NO for CPP_PASTE
	tokens.
	* macro.c (macro_real_token_count): New.
	(enter_macro_context, replace_args): Use macro_real_token_count.
	(create_iso_definition): Record whitespace surrounding and digraph
	spelling of # and ## tokens using SP_PREV_WHITE and SP_DIGRAPH.
	Set extra_tokens and save CPP_PASTE tokens with arg_no set for
	multiple consecutive ## tokens.
	(_cpp_create_definition): Initialize extra_tokens.
	(cpp_macro_definition): Use macro_real_token_count.

gcc/testsuite:
	* gcc.dg/cpp/paste16.c, gcc.dg/cpp/redef4.c: New tests.

From-SVN: r146352

gcc/testsuite/ChangeLog

+2009-04-19  Joseph Myers  <joseph@codesourcery.com>
+
+	PR preprocessor/20078
+	* gcc.dg/cpp/paste16.c, gcc.dg/cpp/redef4.c: New tests.
+
 2009-04-19  Eric Botcazou  <ebotcazou@adacore.com>
 
 	* lib/gnat.exp (local_find_gnatmake): Pass --LINK to gnatlink.

gcc/testsuite/gcc.dg/cpp/paste16.c

+/* Test multiple consecutive ## tokens.  */
+/* { dg-do compile } */
+/* { dg-options "" } */
+#define cat(x,y) x##########y
+int abcd;
+int *p = &cat(ab,cd);

libcpp/ChangeLog

+2009-04-19  Joseph Myers  <joseph@codesourcery.com>
+
+	PR preprocessor/20078
+	* include/cpp-id-data.h (struct cpp_macro): Add extra_tokens
+	field.
+	* include/cpplib.h (SP_DIGRAPH, SP_PREV_WHITE): Define.
+	(struct cpp_token): Change flags to unsigned short.
+	* lex.c (_cpp_lex_direct): Initialize arg_no for CPP_PASTE tokens.
+	(_cpp_equiv_tokens): Check arg_no for CPP_PASTE tokens.
+	(cpp_token_val_index): Return CPP_TOKEN_FLD_ARG_NO for CPP_PASTE
+	tokens.
+	* macro.c (macro_real_token_count): New.
+	(enter_macro_context, replace_args): Use macro_real_token_count.
+	(create_iso_definition): Record whitespace surrounding and digraph
+	spelling of # and ## tokens using SP_PREV_WHITE and SP_DIGRAPH.
+	Set extra_tokens and save CPP_PASTE tokens with arg_no set for
+	multiple consecutive ## tokens.
+	(_cpp_create_definition): Initialize extra_tokens.
+	(cpp_macro_definition): Use macro_real_token_count.
+
 2009-04-18  Joseph Myers  <joseph@codesourcery.com>
 
 	* directives.c (parse_include): Pass true to check_eol.

libcpp/include/cpp-id-data.h

@@ -75,4 +75,9 @@ struct cpp_macro GTY(())
 
   /* Indicate which field of 'exp' is in use.  */
   unsigned int traditional : 1;
+
+  /* Indicate whether the tokens include extra CPP_PASTE tokens at the
+     end to track invalid redefinitions with consecutive CPP_PASTE
+     tokens.  */
+  unsigned int extra_tokens : 1;
 };

libcpp/include/cpplib.h

@@ -178,6 +178,10 @@ struct cpp_string GTY(())
 #define BOL		(1 << 6) /* Token at beginning of line.  */
 #define PURE_ZERO	(1 << 7) /* Single 0 digit, used by the C++ frontend,
 				    set in c-lex.c.  */
+#define SP_DIGRAPH	(1 << 8) /* # or ## token was a digraph.  */
+#define SP_PREV_WHITE	(1 << 9) /* If whitespace before a ##
+				    operator, or before this token
+				    after a # operator.  */
 
 /* Specify which field, if any, of the cpp_token union is used.  */
 
@@ -196,7 +200,7 @@ struct cpp_token GTY(())
 {
   source_location src_loc;	/* Location of first char of token.  */
   ENUM_BITFIELD(cpp_ttype) type : CHAR_BIT;  /* token type */
-  unsigned char flags;		/* flags - see above */
+  unsigned short flags;		/* flags - see above */
 
   union cpp_token_u
   {

libcpp/lex.c

@@ -1244,7 +1244,7 @@ _cpp_lex_direct (cpp_reader *pfile)
 	      result->flags |= DIGRAPH;
 	      result->type = CPP_HASH;
 	      if (*buffer->cur == '%' && buffer->cur[1] == ':')
-		buffer->cur += 2, result->type = CPP_PASTE;
+		buffer->cur += 2, result->type = CPP_PASTE, result->val.arg_no = 0;
 	    }
 	  else if (*buffer->cur == '>')
 	    {
@@ -1325,7 +1325,7 @@ _cpp_lex_direct (cpp_reader *pfile)
     case '=': IF_NEXT_IS ('=', CPP_EQ_EQ, CPP_EQ); break;
     case '!': IF_NEXT_IS ('=', CPP_NOT_EQ, CPP_NOT); break;
     case '^': IF_NEXT_IS ('=', CPP_XOR_EQ, CPP_XOR); break;
-    case '#': IF_NEXT_IS ('#', CPP_PASTE, CPP_HASH); break;
+    case '#': IF_NEXT_IS ('#', CPP_PASTE, CPP_HASH); result->val.arg_no = 0; break;
 
     case '?': result->type = CPP_QUERY; break;
     case '~': result->type = CPP_COMPL; break;
@@ -1572,7 +1572,9 @@ _cpp_equiv_tokens (const cpp_token *a, const cpp_token *b)
       {
       default:			/* Keep compiler happy.  */
       case SPELL_OPERATOR:
-	return 1;
+	/* arg_no is used to track where multiple consecutive ##
+	   tokens were originally located.  */
+	return (a->type != CPP_PASTE || a->val.arg_no == b->val.arg_no);
       case SPELL_NONE:
 	return (a->type != CPP_MACRO_ARG || a->val.arg_no == b->val.arg_no);
       case SPELL_IDENT:
@@ -1886,6 +1888,11 @@ cpp_token_val_index (cpp_token *tok)
       return CPP_TOKEN_FLD_NODE;
     case SPELL_LITERAL:
       return CPP_TOKEN_FLD_STR;
+    case SPELL_OPERATOR:
+      if (tok->type == CPP_PASTE)
+	return CPP_TOKEN_FLD_ARG_NO;
+      else
+	return CPP_TOKEN_FLD_NONE;
     case SPELL_NONE:
       if (tok->type == CPP_MACRO_ARG)
 	return CPP_TOKEN_FLD_ARG_NO;

libcpp/macro.c

@@ -802,6 +802,19 @@ funlike_invocation_p (cpp_reader *pfile, cpp_hashnode *node,
   return NULL;
 }
 
+/* Return the real number of tokens in the expansion of MACRO.  */
+static inline unsigned int
+macro_real_token_count (const cpp_macro *macro)
+{
+  unsigned int i;
+  if (__builtin_expect (!macro->extra_tokens, true))
+    return macro->count;
+  for (i = 0; i < macro->count; i++)
+    if (macro->exp.tokens[i].type == CPP_PASTE)
+      return i;
+  abort ();
+}
+
 /* Push the context of a macro with hash entry NODE onto the context
    stack.  If we can successfully expand the macro, we push a context
    containing its yet-to-be-rescanned replacement list and return one.
@@ -874,7 +887,8 @@ enter_macro_context (cpp_reader *pfile, cpp_hashnode *node,
       macro->used = 1;
 
       if (macro->paramc == 0)
-	_cpp_push_token_context (pfile, node, macro->exp.tokens, macro->count);
+	_cpp_push_token_context (pfile, node, macro->exp.tokens,
+				 macro_real_token_count (macro));
 
       if (pragma_buff)
 	{
@@ -914,13 +928,15 @@ replace_args (cpp_reader *pfile, cpp_hashnode *node, cpp_macro *macro, macro_arg
   const cpp_token **dest, **first;
   macro_arg *arg;
   _cpp_buff *buff;
+  unsigned int count;
 
   /* First, fully macro-expand arguments, calculating the number of
      tokens in the final expansion as we go.  The ordering of the if
      statements below is subtle; we must handle stringification before
      pasting.  */
-  total = macro->count;
-  limit = macro->exp.tokens + macro->count;
+  count = macro_real_token_count (macro);
+  total = count;
+  limit = macro->exp.tokens + count;
 
   for (src = macro->exp.tokens; src < limit; src++)
     if (src->type == CPP_MACRO_ARG)
@@ -1630,6 +1646,7 @@ create_iso_definition (cpp_reader *pfile, cpp_macro *macro)
   bool following_paste_op = false;
   const char *paste_op_error_msg =
     N_("'##' cannot appear at either end of a macro expansion");
+  unsigned int num_extra_tokens = 0;
 
   /* Get the first token of the expansion (or the '(' of a
      function-like macro).  */
@@ -1707,6 +1724,10 @@ create_iso_definition (cpp_reader *pfile, cpp_macro *macro)
 	{
 	  if (token->type == CPP_MACRO_ARG)
 	    {
+	      if (token->flags & PREV_WHITE)
+		token->flags |= SP_PREV_WHITE;
+	      if (token[-1].flags & DIGRAPH)
+		token->flags |= SP_DIGRAPH;
 	      token->flags &= ~PREV_WHITE;
 	      token->flags |= STRINGIFY_ARG;
 	      token->flags |= token[-1].flags & PREV_WHITE;
@@ -1746,8 +1767,21 @@ create_iso_definition (cpp_reader *pfile, cpp_macro *macro)
 	      return false;
 	    }
 
-	  --macro->count;
-	  token[-1].flags |= PASTE_LEFT;
+	  if (token[-1].flags & PASTE_LEFT)
+	    {
+	      macro->extra_tokens = 1;
+	      num_extra_tokens++;
+	      token->val.arg_no = macro->count - 1;
+	    }
+	  else
+	    {
+	      --macro->count;
+	      token[-1].flags |= PASTE_LEFT;
+	      if (token->flags & DIGRAPH)
+		token[-1].flags |= SP_DIGRAPH;
+	      if (token->flags & PREV_WHITE)
+		token[-1].flags |= SP_PREV_WHITE;
+	    }
 	}
 
       following_paste_op = (token->type == CPP_PASTE);
@@ -1770,7 +1804,27 @@ create_iso_definition (cpp_reader *pfile, cpp_macro *macro)
       cpp_token *tokns =
         (cpp_token *) pfile->hash_table->alloc_subobject (sizeof (cpp_token)
                                                           * macro->count);
-      memcpy (tokns, macro->exp.tokens, sizeof (cpp_token) * macro->count);
+      if (num_extra_tokens)
+	{
+	  /* Place second and subsequent ## or %:%: tokens in
+	     sequences of consecutive such tokens at the end of the
+	     list to preserve information about where they appear, how
+	     they are spelt and whether they are preceded by
+	     whitespace without otherwise interfering with macro
+	     expansion.  */
+	  cpp_token *normal_dest = tokns;
+	  cpp_token *extra_dest = tokns + macro->count - num_extra_tokens;
+	  unsigned int i;
+	  for (i = 0; i < macro->count; i++)
+	    {
+	      if (macro->exp.tokens[i].type == CPP_PASTE)
+		*extra_dest++ = macro->exp.tokens[i];
+	      else
+		*normal_dest++ = macro->exp.tokens[i];
+	    }
+	}
+      else
+	memcpy (tokns, macro->exp.tokens, sizeof (cpp_token) * macro->count);
       macro->exp.tokens = tokns;
     }
   else
@@ -1799,6 +1853,7 @@ _cpp_create_definition (cpp_reader *pfile, cpp_hashnode *node)
   macro->used = !CPP_OPTION (pfile, warn_unused_macros);
   macro->count = 0;
   macro->fun_like = 0;
+  macro->extra_tokens = 0;
   /* To suppress some diagnostics.  */
   macro->syshdr = pfile->buffer && pfile->buffer->sysp != 0;
 
@@ -1946,7 +2001,8 @@ cpp_macro_definition (cpp_reader *pfile, const cpp_hashnode *node)
     len += _cpp_replacement_text_len (macro);
   else
     {
-      for (i = 0; i < macro->count; i++)
+      unsigned int count = macro_real_token_count (macro);
+      for (i = 0; i < count; i++)
 	{
 	  cpp_token *token = &macro->exp.tokens[i];
 
@@ -2010,7 +2066,8 @@ cpp_macro_definition (cpp_reader *pfile, const cpp_hashnode *node)
   else if (macro->count)
   /* Expansion tokens.  */
     {
-      for (i = 0; i < macro->count; i++)
+      unsigned int count = macro_real_token_count (macro);
+      for (i = 0; i < count; i++)
 	{
 	  cpp_token *token = &macro->exp.tokens[i];