c-parse.in (_yylex): Use _cpp_backup_tokens.

	* c-parse.in (_yylex): Use _cpp_backup_tokens.
	* cpphash.h (struct tokenrun): Add prev.
	(struct lexer_state): Remove bol.
	(struct cpp_reader): Remove old lookahead stuff, add lookaheads.
	(_cpp_free_lookaheads, _cpp_release_lookahead, _cpp_push_token)
	: Remove.
	* cppinit.c (cpp_create_reader): Don't set bol.
	(cpp_destroy): Don't free lookaheads.
	* cpplex.c (lex_directive): Remove.
	(next_tokenrun): Update.
	(_cpp_lex_token): Clean up logic.
	(lex_token): Update to return a pointer to lexed token, since it
	can move to the start of the buffer.  Simpify newline handling.
	* cpplib.c (SEEN_EOL): Update.
	(skip_rest_of_line): Remove lookahead stuff.
	(end_directive): Line numbers are already incremented.  Revert
	to start of lexed token buffer if we can.
	(_cpp_handle_directive, do_pragma, do_pragma_dependency,
	parse_answer): Use _cpp_backup_tokens.
	(run_directive, cpp_pop_buffer): Don't set bol, set saved_flags
	instead.  Don't check for EOL.
	(do_include_common, do_line, do_pragma_system_header): Use
	skip_rest_of_line.
	* cpplib.h (BOL, _cpp_backup_tokens): New.
	* cppmacro.c (save_lookahead_token, take_lookahead_token,
	alloc_lookahead, free_lookahead, _cpp_free_lookaheads,
	cpp_start_lookahead, cpp_stop_lookahead, _cpp_push_token): Remove.
	(builtin_macro): Don't use cpp_get_line.
	(cpp_get_line): Short term kludge.
	(parse_arg): Handle directives in arguments here.  Back up when
	appropriate.  Store EOF at end of argument list.
	(funlike_invocation_p): Use _cpp_backup_tokens.
	(push_arg_context): Account for EOF at end of list.
	(cpp_get_token): Remove lookahead stuff.  Update.

	* gcc.dg/cpp/directiv.c: Update.
	* gcc.dg/cpp/undef1.c: Update.

From-SVN: r45582

gcc/ChangeLog

+2001-09-13  Neil Booth  <neil@daikokuya.demon.co.uk>
+
+	* c-parse.in (_yylex): Use _cpp_backup_tokens.
+	* cpphash.h (struct tokenrun): Add prev.
+	(struct lexer_state): Remove bol.
+	(struct cpp_reader): Remove old lookahead stuff, add lookaheads.
+	(_cpp_free_lookaheads, _cpp_release_lookahead, _cpp_push_token)
+	: Remove.
+	* cppinit.c (cpp_create_reader): Don't set bol.
+	(cpp_destroy): Don't free lookaheads.
+	* cpplex.c (lex_directive): Remove.
+	(next_tokenrun): Update.
+	(_cpp_lex_token): Clean up logic.
+	(lex_token): Update to return a pointer to lexed token, since it
+	can move to the start of the buffer.  Simpify newline handling.
+	* cpplib.c (SEEN_EOL): Update.
+	(skip_rest_of_line): Remove lookahead stuff.
+	(end_directive): Line numbers are already incremented.  Revert
+	to start of lexed token buffer if we can.
+	(_cpp_handle_directive, do_pragma, do_pragma_dependency,
+	parse_answer): Use _cpp_backup_tokens.
+	(run_directive, cpp_pop_buffer): Don't set bol, set saved_flags
+	instead.  Don't check for EOL.
+	(do_include_common, do_line, do_pragma_system_header): Use
+	skip_rest_of_line.
+	* cpplib.h (BOL, _cpp_backup_tokens): New.
+	* cppmacro.c (save_lookahead_token, take_lookahead_token,
+	alloc_lookahead, free_lookahead, _cpp_free_lookaheads,
+	cpp_start_lookahead, cpp_stop_lookahead, _cpp_push_token): Remove.
+	(builtin_macro): Don't use cpp_get_line.
+	(cpp_get_line): Short term kludge.
+	(parse_arg): Handle directives in arguments here.  Back up when
+	appropriate.  Store EOF at end of argument list.
+	(funlike_invocation_p): Use _cpp_backup_tokens.
+	(push_arg_context): Account for EOF at end of list.
+	(cpp_get_token): Remove lookahead stuff.  Update.
+
 2001-09-13  Kaveh R. Ghazi  <ghazi@caip.rutgers.edu>
 
 	* c-parse.in (yyerror): Const-ification and/or static-ization.

gcc/c-parse.in

@@ -3788,19 +3788,17 @@ ifobjc
 	tree after_at;
 	enum cpp_ttype after_at_type;
 
-	cpp_start_lookahead (parse_in);
 	after_at_type = c_lex (&after_at);
 
 	if (after_at_type == CPP_NAME
 	    && C_IS_RESERVED_WORD (after_at)
 	    && OBJC_IS_AT_KEYWORD (C_RID_CODE (after_at)))
 	  {
-	    cpp_stop_lookahead (parse_in, 1);  /* accept this token */
 	    yylval.ttype = after_at;
 	    last_token = after_at_type;
 	    return rid_to_yy [(int) C_RID_CODE (after_at)];
 	  }
-	cpp_stop_lookahead (parse_in, 0);  /* put back this token */
+	_cpp_backup_tokens (parse_in, 1);
 	return '@';
       }
 end ifobjc

gcc/cpphash.h

@@ -105,7 +105,7 @@ struct toklist
 typedef struct tokenrun tokenrun;
 struct tokenrun
 {
-  tokenrun *next;
+  tokenrun *next, *prev;
   cpp_token *base, *limit;
 };
 
@@ -131,9 +131,6 @@ struct lexer_state
   /* True if we are skipping a failed conditional group.  */
   unsigned char skipping;
 
-  /* Nonzero if next token is the start of a line.  */
-  unsigned char bol;
-
   /* Nonzero if in a directive that takes angle-bracketed headers.  */
   unsigned char angled_headers;
 
@@ -271,16 +268,11 @@ struct cpp_reader
   /* Lexing.  */
   cpp_token *cur_token;
   tokenrun base_run, *cur_run;
+  unsigned int lookaheads;
 
   /* Non-zero prevents the lexer from re-using the token runs.  */
   unsigned int keep_tokens;
 
-  /* Token lookahead.  */
-  struct cpp_lookahead *la_read;	/* Read from this lookahead.  */
-  struct cpp_lookahead *la_write;	/* Write to this lookahead.  */
-  struct cpp_lookahead *la_unused;	/* Free store.  */
-  struct cpp_lookahead *la_saved;	/* Backup when entering directive.  */
-
   /* Error counter for exit code.  */
   unsigned int errors;
 
@@ -382,10 +374,6 @@ extern int _cpp_begin_message PARAMS ((cpp_reader *, enum error_type,
 extern void _cpp_free_definition	PARAMS ((cpp_hashnode *));
 extern int _cpp_create_definition	PARAMS ((cpp_reader *, cpp_hashnode *));
 extern void _cpp_pop_context		PARAMS ((cpp_reader *));
-extern void _cpp_free_lookaheads	PARAMS ((cpp_reader *));
-extern void _cpp_release_lookahead	PARAMS ((cpp_reader *));
-extern void _cpp_push_token		PARAMS ((cpp_reader *, const cpp_token *,
-						 const cpp_lexer_pos *));
 
 /* In cpphash.c */
 extern void _cpp_init_hashtable		PARAMS ((cpp_reader *, hash_table *));

gcc/cppinit.c

@@ -515,7 +515,6 @@ cpp_create_reader (table, lang)
   _cpp_init_tokenrun (&pfile->base_run, 250);
   pfile->cur_run = &pfile->base_run;
   pfile->cur_token = pfile->base_run.base;
-  pfile->state.bol = 1;
 
   /* Initialise the base context.  */
   pfile->context = &pfile->base_context;
@@ -581,7 +580,6 @@ cpp_destroy (pfile)
 
   _cpp_destroy_hashtable (pfile);
   _cpp_cleanup_includes (pfile);
-  _cpp_free_lookaheads (pfile);
 
   _cpp_free_pool (&pfile->ident_pool);
   _cpp_free_pool (&pfile->macro_pool);

gcc/cpplex.c

@@ -102,8 +102,7 @@ static void lex_dot PARAMS ((cpp_reader *, cpp_token *));
 static int name_p PARAMS ((cpp_reader *, const cpp_string *));
 static int maybe_read_ucs PARAMS ((cpp_reader *, const unsigned char **,
 				   const unsigned char *, unsigned int *));
-static int lex_directive PARAMS ((cpp_reader *));
-static void lex_token PARAMS ((cpp_reader *, cpp_token *, int));
+static cpp_token *lex_token PARAMS ((cpp_reader *, cpp_token *));
 static tokenrun *next_tokenrun PARAMS ((tokenrun *));
 
 static cpp_chunk *new_chunk PARAMS ((unsigned int));
@@ -925,114 +924,69 @@ next_tokenrun (run)
   if (run->next == NULL)
     {
       run->next = xnew (tokenrun);
+      run->next->prev = run;
       _cpp_init_tokenrun (run->next, 250);
     }
 
   return run->next;
 }
 
-static int
-lex_directive (pfile)
-     cpp_reader *pfile;
-{
-  /* 6.10.3 paragraph 11: If there are sequences of preprocessing
-     tokens within the list of arguments that would otherwise act as
-     preprocessing directives, the behavior is undefined.
-
-     This implementation will report a hard error, terminate the macro
-     invocation, and proceed to process the directive.  */
-  if (pfile->state.parsing_args)
-    {
-      pfile->lexer_pos.output_line = pfile->line;
-      if (pfile->state.parsing_args == 2)
-	{
-	  cpp_error (pfile,
-		     "directives may not be used inside a macro argument");
-	  pfile->state.bol = 1;
-	  pfile->buffer->cur = pfile->buffer->line_base;
-	  pfile->buffer->read_ahead = EOF;
-	  pfile->cur_token->type = CPP_EOF;
-	}
-
-      return 0;
-    }
-
-  /* This is a directive.  If the return value is false, it is an
-     assembler #.  */
-  {
-    /* FIXME: short-term kludge only - it doesn't handle the case that
-       the # is at the end of a run and we moved to the start of the
-       next one.  Easily fixed once we kill lookaheads.  */
-    cpp_token *token = pfile->cur_token++;
-    if (_cpp_handle_directive (pfile, token->flags & PREV_WHITE))
-      return 1;
-    pfile->cur_token = token;
-    return 0;
-  }
-}
-
 /* Lex a token into RESULT (external interface).  */
 void
-_cpp_lex_token (pfile, result)
+_cpp_lex_token (pfile, dest)
      cpp_reader *pfile;
-     cpp_token *result;
+     cpp_token *dest;
 {
-  if (pfile->cur_token == pfile->cur_run->limit)
-    {
-      pfile->cur_run = next_tokenrun (pfile->cur_run);
-      pfile->cur_token = pfile->cur_run->base;
-    }
+  cpp_token *result;
 
- next_token:
-  if (pfile->state.bol)
+  for (;;)
     {
-    start_new_line:
-      pfile->state.bol = 0;
-
-      /* Return lexer back to base.  */
-      if (!pfile->keep_tokens)
+      if (pfile->cur_token == pfile->cur_run->limit)
 	{
-	  pfile->cur_run = &pfile->base_run;
-	  pfile->cur_token = pfile->base_run.base;
+	  pfile->cur_run = next_tokenrun (pfile->cur_run);
+	  pfile->cur_token = pfile->cur_run->base;
 	}
+      result = pfile->cur_token++;
 
-      lex_token (pfile, pfile->cur_token, 1);
-      pfile->lexer_pos.output_line = pfile->cur_token->line;
-      if (pfile->cur_token->type == CPP_HASH && lex_directive (pfile))
-	goto start_new_line;
-    }
-  else
-    {
-      lex_token (pfile, pfile->cur_token, 0);
-      if (pfile->cur_token->type == CPP_EOF)
+      if (pfile->lookaheads)
+	pfile->lookaheads--;
+      else
+	result = lex_token (pfile, result);
+
+      if (result->flags & BOL)
 	{
-	  if (!pfile->state.in_directive)
-	    goto start_new_line;
-	  /* Decrementing pfile->line allows directives to recognise
-	     that the newline has been seen, and also means that
-	     diagnostics don't point to the next line.  */
-	  pfile->lexer_pos.output_line = pfile->line--;
+	  pfile->lexer_pos.output_line = result->line;
+	  /* Is this a directive.  If _cpp_handle_directive returns
+	     false, it is an assembler #.  */
+	  if (result->type == CPP_HASH
+	      && !pfile->state.parsing_args
+	      && _cpp_handle_directive (pfile, result->flags & PREV_WHITE))
+	    continue;
 	}
-    }
 
-  if (!pfile->state.in_directive)
-    {
-      if (pfile->state.skipping && pfile->cur_token->type != CPP_EOF)
-	goto next_token;
+      /* We don't skip tokens in directives.  */
+      if (pfile->state.in_directive)
+	break;
 
-      /* Outside a directive, invalidate controlling macros.  */
+      /* Outside a directive, invalidate controlling macros.  At file
+	 EOF, lex_token takes care of popping the buffer, so we never
+	 get here and MI optimisation works.  */
       pfile->mi_valid = false;
+
+      if (!pfile->state.skipping || result->type == CPP_EOF)
+	break;
     }
 
-  *result = *pfile->cur_token++;
+  *dest = *result;
 }
 
-/* Lex a token into RESULT (internal interface).  */
-static void
-lex_token (pfile, result, skip_newlines)
+/* Lex a token into RESULT.  When meeting a newline, returns CPP_EOF
+   if parsing a directive, otherwise returns to the start of the token
+   buffer if permissible.  Returns the location of the lexed token.  */
+static cpp_token *
+lex_token (pfile, result)
      cpp_reader *pfile;
      cpp_token *result;
-     int skip_newlines;
 {
   cppchar_t c;
   cpp_buffer *buffer;
@@ -1058,21 +1012,10 @@ lex_token (pfile, result, skip_newlines)
   switch (c)
     {
     case EOF:
+      buffer->saved_flags = BOL;
       if (!pfile->state.parsing_args && !pfile->state.in_directive)
 	{
-	  if (buffer->cur == buffer->line_base)
-	    {
-	      /* Don't pop the last buffer.  */
-	      if (buffer->prev)
-		{
-		  unsigned char stop = buffer->return_at_eof;
-
-		  _cpp_pop_buffer (pfile);
-		  if (!stop)
-		    goto fresh_line;
-		}
-	    }
-	  else
+	  if (buffer->cur != buffer->line_base)
 	    {
 	      /* Non-empty files should end in a newline.  Don't warn
 		 for command line and _Pragma buffers.  */
@@ -1080,6 +1023,16 @@ lex_token (pfile, result, skip_newlines)
 		cpp_pedwarn (pfile, "no newline at end of file");
 	      handle_newline (pfile, '\n');
 	    }
+
+	  /* Don't pop the last buffer.  */
+	  if (buffer->prev)
+	    {
+	      unsigned char stop = buffer->return_at_eof;
+
+	      _cpp_pop_buffer (pfile);
+	      if (!stop)
+		goto fresh_line;
+	    }
 	}
       result->type = CPP_EOF;
       break;
@@ -1090,13 +1043,17 @@ lex_token (pfile, result, skip_newlines)
       goto skipped_white;
 
     case '\n': case '\r':
-      if (pfile->state.in_directive && pfile->state.parsing_args)
-	buffer->read_ahead = c;
-      else
+      handle_newline (pfile, c);
+      buffer->saved_flags = BOL;
+      if (! pfile->state.in_directive)
 	{
-	  handle_newline (pfile, c);
-	  if (skip_newlines)
-	    goto fresh_line;
+	  if (!pfile->keep_tokens)
+	    {
+	      pfile->cur_run = &pfile->base_run;
+	      result = pfile->base_run.base;
+	      pfile->cur_token = result + 1;
+	    }
+	  goto fresh_line;
 	}
       result->type = CPP_EOF;
       break;
@@ -1228,7 +1185,7 @@ lex_token (pfile, result, skip_newlines)
       /* Save the comment as a token in its own right.  */
       save_comment (pfile, result, comment_start);
       /* Don't do MI optimisation.  */
-      return;
+      break;
 
     case '<':
       if (pfile->state.angled_headers)
@@ -1397,6 +1354,8 @@ lex_token (pfile, result, skip_newlines)
       result->val.c = c;
       break;
     }
+
+  return result;
 }
 
 /* An upper bound on the number of bytes needed to spell a token,

gcc/cpplib.c

@@ -176,7 +176,7 @@ DIRECTIVE_TABLE
 #undef D
 #undef DIRECTIVE_TABLE
 
-#define SEEN_EOL() (pfile->lexer_pos.output_line > pfile->line)
+#define SEEN_EOL() (pfile->cur_token[-1].type == CPP_EOF)
 
 /* Skip any remaining tokens in a directive.  */
 static void
@@ -185,10 +185,6 @@ skip_rest_of_line (pfile)
 {
   cpp_token token;
 
-  /* Discard all input lookaheads.  */
-  while (pfile->la_read)
-    _cpp_release_lookahead (pfile);
-
   /* Discard all stacked contexts.  */
   while (pfile->context != &pfile->base_context)
     _cpp_pop_context (pfile);
@@ -227,10 +223,6 @@ start_directive (pfile)
   pfile->directive_pos = pfile->lexer_pos;
   pfile->directive_pos.line = pfile->line;
   pfile->directive_line = pfile->line;
-
-  /* Don't save directive tokens for external clients.  */
-  pfile->la_saved = pfile->la_write;
-  pfile->la_write = 0;
 }
 
 /* Called when leaving a directive, _Pragma or command-line directive.  */
@@ -243,12 +235,14 @@ end_directive (pfile, skip_line)
   if (skip_line)
     {
       skip_rest_of_line (pfile);
-      /*  "Accept" the newline now.  */
-      pfile->line++;
+      if (!pfile->keep_tokens)
+	{
+	  pfile->cur_run = &pfile->base_run;
+	  pfile->cur_token = pfile->base_run.base;
+	}
     }
 
   /* Restore state.  */
-  pfile->la_write = pfile->la_saved;
   pfile->state.save_comments = ! CPP_OPTION (pfile, discard_comments);
   pfile->state.in_directive = 0;
   pfile->state.angled_headers = 0;
@@ -289,7 +283,7 @@ _cpp_handle_directive (pfile, indented)
 	{
 	  dir = &dtable[T_LINE];
 	  pfile->state.line_extension = 1;
-	  _cpp_push_token (pfile, &dname, &pfile->directive_pos);
+	  _cpp_backup_tokens (pfile, 1);
 	  if (CPP_PEDANTIC (pfile) && ! CPP_OPTION (pfile, preprocessed))
 	    cpp_pedwarn (pfile, "# followed by integer");
 	}
@@ -324,7 +318,7 @@ _cpp_handle_directive (pfile, indented)
 	      /* We don't want to process this directive.  Put back the
 		 tokens so caller will see them (and issue an error,
 		 probably).  */
-	      _cpp_push_token (pfile, &dname, &pfile->directive_pos);
+	      _cpp_backup_tokens (pfile, 1);
 	      skip = 0;
 	    }
 	}
@@ -376,8 +370,8 @@ _cpp_handle_directive (pfile, indented)
 	 directives in skipped conditional groups (6.10 p4).  */
       if (CPP_OPTION (pfile, lang) == CLK_ASM)
 	{
-	  /* Output the # and lookahead token for the assembler.  */
-	  _cpp_push_token (pfile, &dname, &pfile->directive_pos);
+	  /* Output the # and this token for the assembler.  */
+	  _cpp_backup_tokens (pfile, 1);
 	  skip = 0;
 	}
       else
@@ -402,12 +396,11 @@ run_directive (pfile, dir_no, buf, count)
   cpp_push_buffer (pfile, (const U_CHAR *) buf, count,
 		   /* from_stage3 */ true, 1);
   start_directive (pfile);
-  pfile->state.bol = 0;
+  pfile->buffer->saved_flags = 0; /* We don't want to recognise directives.  */
   pfile->state.prevent_expansion++;
   pfile->directive = &dtable[dir_no];
   (void) (*pfile->directive->handler) (pfile);
   pfile->state.prevent_expansion--;
-  check_eol (pfile);
   end_directive (pfile, 1);
   _cpp_pop_buffer (pfile);
 }
@@ -618,7 +611,7 @@ do_include_common (pfile, type)
 	{
 	  check_eol (pfile);
 	  /* Get out of macro context, if we are.  */
-	  end_directive (pfile, 1);
+	  skip_rest_of_line (pfile);
 	  if (pfile->cb.include)
 	    (*pfile->cb.include) (pfile, pfile->directive_line,
 				  pfile->directive->name, &header);
@@ -772,7 +765,7 @@ do_line (pfile)
       return;
     }
 
-  end_directive (pfile, 1);
+  skip_rest_of_line (pfile);
   _cpp_do_file_change (pfile, reason, new_file, new_lineno, new_sysp);
 }
 
@@ -961,12 +954,13 @@ do_pragma (pfile)
   pragma_cb handler = NULL;
   const struct pragma_entry *p;
   cpp_token tok;
+  unsigned int count = 0;
 
   p = pfile->pragmas;
   pfile->state.prevent_expansion++;
-  cpp_start_lookahead (pfile);
 
  new_space:
+  count++;
   cpp_get_token (pfile, &tok);
   if (tok.type == CPP_NAME)
     {
@@ -993,13 +987,14 @@ do_pragma (pfile)
 	}
     }
 
-  cpp_stop_lookahead (pfile, handler != NULL);
   pfile->state.prevent_expansion--;
-
   if (handler)
     (*handler) (pfile);
   else if (pfile->cb.def_pragma)
-    (*pfile->cb.def_pragma) (pfile, pfile->directive_line);
+    {
+      _cpp_backup_tokens (pfile, count);
+      (*pfile->cb.def_pragma) (pfile, pfile->directive_line);
+    }
 }
 
 static void
@@ -1066,7 +1061,7 @@ do_pragma_system_header (pfile)
   else
     {
       check_eol (pfile);
-      end_directive (pfile, 1);
+      skip_rest_of_line (pfile);
       cpp_make_system_header (pfile, 1, 0);
     }
 }
@@ -1092,11 +1087,12 @@ do_pragma_dependency (pfile)
     {
       cpp_warning (pfile, "current file is older than %s",
 		   cpp_token_as_text (pfile, &header));
-      cpp_start_lookahead (pfile);
       cpp_get_token (pfile, &msg);
-      cpp_stop_lookahead (pfile, msg.type == CPP_EOF);
       if (msg.type != CPP_EOF)
-	do_diagnostic (pfile, WARNING, 0);
+	{
+	  _cpp_backup_tokens (pfile, 1);
+	  do_diagnostic (pfile, WARNING, 0);
+	}
     }
 }
 
@@ -1387,11 +1383,7 @@ parse_answer (pfile, answerp, type)
 
   /* In a conditional, it is legal to not have an open paren.  We
      should save the following token in this case.  */
-  if (type == T_IF)
-    cpp_start_lookahead (pfile);
   cpp_get_token (pfile, &paren);
-  if (type == T_IF)
-    cpp_stop_lookahead (pfile, paren.type == CPP_OPEN_PAREN);
 
   /* If not a paren, see if we're OK.  */
   if (paren.type != CPP_OPEN_PAREN)
@@ -1399,7 +1391,10 @@ parse_answer (pfile, answerp, type)
       /* In a conditional no answer is a test for any answer.  It
          could be followed by any token.  */
       if (type == T_IF)
-	return 0;
+	{
+	  _cpp_backup_tokens (pfile, 1);
+	  return 0;
+	}
 
       /* #unassert with no answer is valid - it removes all answers.  */
       if (type == T_UNASSERT && paren.type == CPP_EOF)
@@ -1755,6 +1750,7 @@ cpp_push_buffer (pfile, buffer, len, from_stage3, return_at_eof)
   new->from_stage3 = from_stage3;
   new->prev = pfile->buffer;
   new->return_at_eof = return_at_eof;
+  new->saved_flags = BOL;
 
   pfile->buffer = new;
 
@@ -1783,7 +1779,6 @@ _cpp_pop_buffer (pfile)
      case of a missing #endif.  */
   pfile->lexer_pos.output_line = pfile->line;
   pfile->state.skipping = 0;
-  pfile->state.bol = 1;
 
   /* Update the reader's buffer before _cpp_do_file_change.  */
   pfile->buffer = buffer->prev;

gcc/cpplib.h

@@ -167,6 +167,7 @@ struct cpp_string
 #define NAMED_OP	(1 << 4) /* C++ named operators.  */
 #define NO_EXPAND	(1 << 5) /* Do not macro-expand this token.  */
 #define AVOID_LPASTE	(1 << 6) /* Check left for accidental pastes.  */
+#define BOL		(1 << 7) /* Token at beginning of line.  */
 
 /* A preprocessing token.  This has been carefully packed and should
    occupy 12 bytes on 32-bit hosts and 16 bytes on 64-bit hosts.  */
@@ -524,6 +525,7 @@ extern void cpp_get_token PARAMS ((cpp_reader *, cpp_token *));
 extern const cpp_lexer_pos *cpp_get_line PARAMS ((cpp_reader *));
 extern const unsigned char *cpp_macro_definition PARAMS ((cpp_reader *,
 						  const cpp_hashnode *));
+extern void _cpp_backup_tokens PARAMS ((cpp_reader *, unsigned int));
 
 /* Evaluate a CPP_CHAR or CPP_WCHAR token.  */
 extern HOST_WIDE_INT

gcc/cppmacro.c

@@ -77,13 +77,6 @@ static int funlike_invocation_p PARAMS ((cpp_reader *, const cpp_hashnode *,
 static void replace_args PARAMS ((cpp_reader *, cpp_macro *, macro_arg *,
 				  struct toklist *));
 
-/* Lookaheads.  */
-
-static void save_lookahead_token PARAMS ((cpp_reader *, const cpp_token *));
-static void take_lookahead_token PARAMS ((cpp_reader *, cpp_token *));
-static cpp_lookahead *alloc_lookahead PARAMS ((cpp_reader *));
-static void free_lookahead PARAMS ((cpp_lookahead *));
-
 /* #define directive parsing and handling.  */
 
 static cpp_token *lex_expansion_token PARAMS ((cpp_reader *, cpp_macro *));
@@ -175,7 +168,7 @@ builtin_macro (pfile, token)
 	 line of the macro's invocation, not its definition.
 	 Otherwise things like assert() will not work properly.  */
       make_number_token (pfile, token,
-			 SOURCE_LINE (pfile->map, cpp_get_line (pfile)->line));
+			 SOURCE_LINE (pfile->map, pfile->cur_token[-1].line));
       break;
 
     case BT_STDC:
@@ -224,6 +217,12 @@ const cpp_lexer_pos *
 cpp_get_line (pfile)
      cpp_reader *pfile;
 {
+  if (pfile->context->prev == NULL)
+    {
+      pfile->lexer_pos.line = pfile->cur_token[-1].line;
+      pfile->lexer_pos.col = pfile->cur_token[-1].col;
+    }
+
   return &pfile->lexer_pos;
 }
 
@@ -486,10 +485,12 @@ parse_arg (pfile, arg, variadic)
       /* Newlines in arguments are white space (6.10.3.10).  */
       line = pfile->line;
       cpp_get_token (pfile, token);
+
       if (line != pfile->line)
 	token->flags |= PREV_WHITE;
 
       result = token->type;
+
       if (result == CPP_OPEN_PAREN)
 	paren++;
       else if (result == CPP_CLOSE_PAREN && paren-- == 0)
@@ -498,11 +499,37 @@ parse_arg (pfile, arg, variadic)
       else if (result == CPP_COMMA && paren == 0 && !variadic)
 	break;
       else if (result == CPP_EOF)
-	break;		/* Error reported by caller.  */
+	{
+	  /* We still need the EOF (added below) to end pre-expansion
+	     and directives.  */
+	  if (pfile->context->prev || pfile->state.in_directive)
+	    _cpp_backup_tokens (pfile, 1);
+	  /* Error reported by caller.  */
+	  break;
+	}
+      else if (result == CPP_HASH && token->flags & BOL)
+	{
+	  /* 6.10.3 paragraph 11: If there are sequences of
+	     preprocessing tokens within the list of arguments that
+	     would otherwise act as preprocessing directives, the
+	     behavior is undefined.
+
+	     This implementation will report a hard error, terminate
+	     the macro invocation, and proceed to process the
+	     directive.  */
+	  cpp_error (pfile,
+		     "directives may not be used inside a macro argument");
+	  _cpp_backup_tokens (pfile, 1);
+	  result = CPP_EOF;
+	  break;
+	}
     }
 
-  /* Commit the memory used to store the arguments.  */
-  POOL_COMMIT (&pfile->argument_pool, arg->count * sizeof (cpp_token));
+  /* Commit the memory used to store the arguments.  We make the last
+     argument a CPP_EOF, so that it terminates macro pre-expansion,
+     but it is not included in arg->count.  */
+  arg->first[arg->count].type = CPP_EOF;  
+  POOL_COMMIT (&pfile->argument_pool, (arg->count + 1) * sizeof (cpp_token));
 
   return result;
 }
@@ -600,17 +627,19 @@ funlike_invocation_p (pfile, node, list)
   pfile->state.prevent_expansion++;
 
   pfile->keep_tokens++;
-  cpp_start_lookahead (pfile);
   cpp_get_token (pfile, &maybe_paren);
-  cpp_stop_lookahead (pfile, maybe_paren.type == CPP_OPEN_PAREN);
   pfile->state.parsing_args = 2;
 
   if (maybe_paren.type == CPP_OPEN_PAREN)
     args = parse_args (pfile, node);
-  else if (CPP_WTRADITIONAL (pfile) && ! node->value.macro->syshdr)
-    cpp_warning (pfile,
-	 "function-like macro \"%s\" must be used with arguments in traditional C",
-		 NODE_NAME (node));
+  else
+    {
+      _cpp_backup_tokens (pfile, 1);
+      if (CPP_WTRADITIONAL (pfile) && ! node->value.macro->syshdr)
+	cpp_warning (pfile,
+ "function-like macro \"%s\" must be used with arguments in traditional C",
+		     NODE_NAME (node));
+    }
 
   pfile->state.prevent_expansion--;
   pfile->state.parsing_args = 0;
@@ -623,13 +652,7 @@ funlike_invocation_p (pfile, node, list)
   if (args)
     {
       if (node->value.macro->paramc > 0)
-	{
-	  /* Don't save tokens during pre-expansion.  */
-	  struct cpp_lookahead *la_saved = pfile->la_write;
-	  pfile->la_write = 0;
-	  replace_args (pfile, node->value.macro, args, list);
-	  pfile->la_write = la_saved;
-	}
+	replace_args (pfile, node->value.macro, args, list);
       free (args);
     }
 
@@ -838,7 +861,7 @@ push_arg_context (pfile, arg)
   cpp_context *context = next_context (pfile);
   context->macro = 0;
   context->list.first = arg->first;
-  context->list.limit = arg->first + arg->count;
+  context->list.limit = arg->first + arg->count + 1;
 
   return context;
 }
@@ -908,10 +931,8 @@ cpp_get_token (pfile, token)
     {
       cpp_context *context = pfile->context;
 
-      if (pfile->la_read)
-	take_lookahead_token (pfile, token);
       /* Context->prev == 0 <=> base context.  */
-      else if (!context->prev)
+      if (!context->prev)
 	_cpp_lex_token (pfile, token);
       else if (context->list.first != context->list.limit)
 	{
@@ -928,17 +949,13 @@ cpp_get_token (pfile, token)
 	}
       else
 	{
-	  if (context->macro)
-	    {
-	      /* Avoid accidental paste at the end of a macro.  */
-	      pfile->buffer->saved_flags |= AVOID_LPASTE;
-	      _cpp_pop_context (pfile);
-	      continue;
-	    }
-	  /* End of argument pre-expansion.  */
-	  token->type = CPP_EOF;
-	  token->flags = 0;
-	  return;
+	  if (!context->macro)
+	    cpp_ice (pfile, "context->macro == 0");
+
+	  /* Avoid accidental paste at the end of a macro.  */
+	  pfile->buffer->saved_flags |= AVOID_LPASTE;
+	  _cpp_pop_context (pfile);
+	  continue;
 	}
 
       if (token->type != CPP_NAME)
@@ -983,9 +1000,6 @@ cpp_get_token (pfile, token)
          since this token came from either the lexer or a macro.  */
       _cpp_do__Pragma (pfile);
     }
-
-  if (pfile->la_write)
-    save_lookahead_token (pfile, token);
 }
 
 /* Returns true if we're expanding an object-like macro that was
@@ -1013,154 +1027,36 @@ cpp_scan_nooutput (pfile)
   while (token.type != CPP_EOF);
 }
 
-/* Lookahead handling.  */
-
-static void
-save_lookahead_token (pfile, token)
-     cpp_reader *pfile;
-     const cpp_token *token;
-{
-  cpp_lookahead *la = pfile->la_write;
-  cpp_token_with_pos *twp;
-
-  if (la->count == la->cap)
-    {
-      la->cap += la->cap + 8;
-      la->tokens = (cpp_token_with_pos *)
-	xrealloc (la->tokens, la->cap * sizeof (cpp_token_with_pos));
-    }
-
-  twp = &la->tokens[la->count++];
-  twp->token = *token;
-  twp->pos = *cpp_get_line (pfile);
-}
-
-static void
-take_lookahead_token (pfile, token)
-     cpp_reader *pfile;
-     cpp_token *token;
-{
-  cpp_lookahead *la = pfile->la_read;
-  cpp_token_with_pos *twp = &la->tokens[la->cur];
-
-  *token = twp->token;
-  pfile->lexer_pos = twp->pos;
-
-  if (++la->cur == la->count)
-    _cpp_release_lookahead (pfile);
-}
-
-/* Moves the lookahead at the front of the read list to the free store.  */
+/* Step back one (or more) tokens.  Can only step mack more than 1 if
+   they are from the lexer, and not from macro expansion.  */
 void
-_cpp_release_lookahead (pfile)
+_cpp_backup_tokens (pfile, count)
      cpp_reader *pfile;
+     unsigned int count;
 {
-  cpp_lookahead *la = pfile->la_read;
-
-  pfile->la_read = la->next;
-  la->next = pfile->la_unused;
-  pfile->la_unused = la;
-  unlock_pools (pfile);
-}
-
-/* Take a new lookahead from the free store, or allocate one if none.  */
-static cpp_lookahead *
-alloc_lookahead (pfile)
-     cpp_reader *pfile;
-{
-  cpp_lookahead *la = pfile->la_unused;
-
-  if (la)
-    pfile->la_unused = la->next;
-  else
+  if (pfile->context->prev == NULL)
     {
-      la = xnew (cpp_lookahead);
-      la->tokens = 0;
-      la->cap = 0;
+      pfile->lookaheads += count;
+      while (count--)
+	{
+	  pfile->cur_token--;
+	  if (pfile->cur_token == pfile->cur_run->base)
+	    {
+	      if (pfile->cur_run == NULL)
+		abort ();
+	      pfile->cur_run = pfile->cur_run->prev;
+	      pfile->cur_token = pfile->cur_run->limit;
+	    }
+	}
     }
-
-  la->cur = la->count = 0;
-  return la;
-}
-
-/* Free memory associated with a lookahead list.  */
-static void
-free_lookahead (la)
-     cpp_lookahead *la;
-{
-  if (la->tokens)
-    free ((PTR) la->tokens);
-  free ((PTR) la);
-}
-
-/* Free all the lookaheads of a cpp_reader.  */
-void
-_cpp_free_lookaheads (pfile)
-     cpp_reader *pfile;
-{
-  cpp_lookahead *la, *lan;
-
-  if (pfile->la_read)
-    free_lookahead (pfile->la_read);
-  if (pfile->la_write)
-    free_lookahead (pfile->la_write);
-
-  for (la = pfile->la_unused; la; la = lan)
+  else
     {
-      lan = la->next;
-      free_lookahead (la);
+      if (count != 1)
+	abort ();
+      pfile->context->list.first--;
     }
 }
 
-/* Allocate a lookahead and move it to the front of the write list.  */
-void
-cpp_start_lookahead (pfile)
-     cpp_reader *pfile;
-{
-  cpp_lookahead *la = alloc_lookahead (pfile);
-
-  la->next = pfile->la_write;
-  pfile->la_write = la;
-
-  la->pos = *cpp_get_line (pfile);
-
-  /* Don't allow memory pools to be re-used whilst we're reading ahead.  */
-  lock_pools (pfile);
-}
-
-/* Stop reading ahead - either step back, or drop the read ahead.  */
-void
-cpp_stop_lookahead (pfile, drop)
-     cpp_reader *pfile;
-     int drop;
-{
-  cpp_lookahead *la = pfile->la_write;
-
-  pfile->la_write = la->next;
-  la->next = pfile->la_read;
-  pfile->la_read = la;
-
-  if (drop || la->count == 0)
-    _cpp_release_lookahead (pfile);
-  else
-    pfile->lexer_pos = la->pos;
-}
-
-/* Push a single token back to the front of the queue.  Only to be
-   used by cpplib, and only then when necessary.  POS is the position
-   to report for the preceding token.  */
-void
-_cpp_push_token (pfile, token, pos)
-     cpp_reader *pfile;
-     const cpp_token *token;
-     const cpp_lexer_pos *pos;
-{
-  cpp_start_lookahead (pfile);
-  save_lookahead_token (pfile, token);
-  cpp_stop_lookahead (pfile, 0);
-  pfile->lexer_pos = *pos;
-}
-
 /* #define directive parsing and handling.  */
 
 /* Returns non-zero if a macro redefinition warning is required.  */

gcc/testsuite/ChangeLog

+2001-09-13  Neil Booth  <neil@daikokuya.demon.co.uk>
+
+	* gcc.dg/cpp/directiv.c: Update.
+	* gcc.dg/cpp/undef1.c: Update.
+
 2001-09-12  Jakub Jelinek  <jakub@redhat.com>
 
 	* gcc.dg/20010912-1.c: New test.

gcc/testsuite/gcc.dg/cpp/directiv.c

@@ -28,7 +28,7 @@ EMPTY #define bar
 /* Check that directives always start a line, even if in middle of
    macro expansion.  */
 #define func(x) x
-func (2		/* { dg-error "unterminated" "" { target *-*-* } 32 } */
+func (2		/* { dg-error "unterminated" "" } */
 #define foobar	/* { dg-error "directives may not" } */
 
 /* Check newlines end directives, even in function-like macro

gcc/testsuite/gcc.dg/cpp/undef1.c

@@ -9,6 +9,6 @@
 
 #define foo(bar) bar
 
-foo( blah	/* { dg-error "unterminated" "" { target *-*-* } 13 } */
+foo( blah	/* { dg-error "unterminated" "" } */
 #undef foo	/* { dg-error "may not be used inside" "foo(#undef foo)" } */
      blah )