cpplex.c (_cpp_lex_line): Merge vertical space.

	* cpplex.c (_cpp_lex_line): Merge vertical space.  Flag
	first token of a line BOL.  Update EOF code for this.
	Remove illegal directive check - it appears in the (not
	yet committed) caller.
	* cpplib.h (BOL): New flag.

From-SVN: r34216

gcc/ChangeLog

 2000-05-28  Neil Booth  <NeilB@earthling.net>
 
+	* cpplex.c (_cpp_lex_line): Merge vertical space.  Flag
+	first token of a line BOL.  Update EOF code for this.
+	Remove illegal directive check - it appears in the (not
+	yet committed) caller.
+	* cpplib.h (BOL): New flag.
+
+2000-05-28  Neil Booth  <NeilB@earthling.net>
+
         * cpplex.c (_cpp_init_toklist): No comment space to initialise.
 	(_cpp_free_toklist): No comment space to free.
 	(expand_comment_space): Remove.

gcc/cpplex.c

@@ -62,7 +62,6 @@ static void pedantic_whitespace	PARAMS ((cpp_reader *, U_CHAR *,
 
 #ifdef NEW_LEXER
 
-static void expand_comment_space PARAMS ((cpp_toklist *));
 void init_trigraph_map PARAMS ((void));
 static unsigned char* trigraph_replace PARAMS ((cpp_reader *, unsigned char *,
 						unsigned char *));
@@ -76,7 +75,8 @@ static void parse_number PARAMS ((cpp_reader *, cpp_toklist *, cpp_name *));
 static void parse_string2 PARAMS ((cpp_reader *, cpp_toklist *, cpp_name *,
 				  unsigned int, int));
 static int trigraph_ok PARAMS ((cpp_reader *, const unsigned char *));
-static void save_comment PARAMS ((cpp_toklist *, cpp_token *, unsigned char *,
+static void save_comment PARAMS ((cpp_toklist *, cpp_token *,
+				  const unsigned char *,
 				  unsigned int, unsigned int));
 void _cpp_lex_line PARAMS ((cpp_reader *, cpp_toklist *));
 
@@ -120,8 +120,8 @@ typedef unsigned int (* speller) PARAMS ((unsigned char *, cpp_toklist *,
 
 /* An upper bound on the number of bytes needed to spell a token,
    including preceding whitespace.  */
-#define TOKEN_LEN(token) (5 + (token_spellings[token->type].type > \
-		               SPELL_NONE ? token->val.name.len: 0))
+#define TOKEN_LEN(token) (5 + (token_spellings[(token)->type].type > \
+		               SPELL_NONE ? (token)->val.name.len: 0))
 
 #endif
 
@@ -2866,9 +2866,10 @@ save_comment (list, token, from, len, type)
 
 /*
  *  The tokenizer's main loop.  Returns a token list, representing a
- *  logical line in the input file, terminated with a CPP_VSPACE
- *  token.  On EOF, a token list containing the single CPP_EOF token
- *  is returned.
+ *  logical line in the input file.  On EOF after some tokens have
+ *  been processed, we return immediately.  Then in next call, or if
+ *  EOF occurred at the beginning of a logical line, a single CPP_EOF
+ *  token is placed in the list.
  *
  *  Implementation relies almost entirely on lookback, rather than
  *  looking forwards.  This means that tokenization requires just
@@ -2891,6 +2892,7 @@ _cpp_lex_line (pfile, list)
   unsigned char flags = 0;
   unsigned int first_token = list->tokens_used;
 
+  list->line = CPP_BUF_LINE (buffer);
   pfile->col_adjust = 0;
  expanded:
   token_limit = list->tokens + list->tokens_cap;
@@ -3122,8 +3124,12 @@ _cpp_lex_line (pfile, list)
 	      buffer->cur = cur;
 	      cpp_warning (pfile, "backslash and newline separated by space");
 	    }
-	  PUSH_TOKEN (CPP_VSPACE);
-	  goto out;
+	  /* Skip vertical space until we have at least one token to
+             return.  */
+	  if (cur_token != &list->tokens[first_token])
+	    goto out;
+	  list->line = CPP_BUF_LINE (buffer);
+	  break;
 
 	case '-':
 	  if (IMMED_TOKEN () && PREV_TOKEN_TYPE == CPP_MINUS)
@@ -3327,35 +3333,19 @@ _cpp_lex_line (pfile, list)
       goto expanded;
     }
 
-  cur_token->type = CPP_EOF;
   cur_token->flags = flags;
-
-  if (cur_token != &list->tokens[first_token])
+  if (cur_token == &list->tokens[first_token])
     {
-      /* Next call back will get just a CPP_EOF.  */
-      buffer->cur = cur;
-      cpp_warning (pfile, "no newline at end of file");
-      PUSH_TOKEN (CPP_VSPACE);
+      /* FIXME: move this warning to callers who care.  */
+      if (cur > buffer->buf && !IS_NEWLINE (cur[-1]))
+	cpp_warning (pfile, "no newline at end of file");
+      cur_token++->type = CPP_EOF;
     }
 
  out:
+  list->tokens[first_token].flags |= BOL;
   buffer->cur = cur;
-
   list->tokens_used = cur_token - list->tokens;
-
-  /* FIXME:  take this check out and put it in the caller.
-     list->directive == 0 indicates an unknown directive (but null
-     directive is OK).  This is the first time we can be sure the
-     directive is invalid, and thus warn about it, because it might
-     have been split by escaped newlines.  Also, don't complain about
-     invalid directives in assembly source, we don't know where the
-     comments are, and # may introduce assembler pseudo-ops.  */
-
-  if (IS_DIRECTIVE (list) && list->dirno == -1
-      && list->tokens[1].type != CPP_VSPACE
-      && !CPP_OPTION (pfile, lang_asm))
-    cpp_error_with_line (pfile, list->line, list->tokens[1].col,
-			 "invalid preprocessing directive");
 }
 
 /* Write the spelling of a token TOKEN to BUFFER.  The buffer must
@@ -3460,15 +3450,13 @@ _cpp_output_list (pfile, list)
      cpp_reader *pfile;
      cpp_toklist *list;
 {
-  cpp_token *token;
+  unsigned int i;
 
-  token = &list->tokens[0];
-  do
+  for (i = 0; i < list->tokens_used; i++)
     {
-      CPP_RESERVE (pfile, TOKEN_LEN (token));
-      pfile->limit = spell_token (pfile, token, pfile->limit, 1);
+      CPP_RESERVE (pfile, TOKEN_LEN (&list->tokens[i]));
+      pfile->limit = spell_token (pfile, &list->tokens[i], pfile->limit, 1);
     }
-  while (token++->type != CPP_VSPACE);
 }
 
 #endif

gcc/cpplib.h

@@ -121,11 +121,11 @@ typedef struct cpp_name cpp_name;
   I(CPP_COMMENT,	0)	/* Only if output comments.  */ \
   N(CPP_MACRO_ARG,      0)	/* Macro argument.  */          \
   N(CPP_SUBLIST,        0)	/* Sublist.  */                 \
-  T(CPP_VSPACE,		"\n")	/* End of line.  */		\
   N(CPP_EOF,		0)	/* End of file.  */		\
   N(CPP_HEADER_NAME,	0)	/* <stdio.h> in #include */	\
 \
   /* Obsolete - will be removed when no code uses them still.  */	\
+  T(CPP_VSPACE,		"\n")	/* End of line.  */		\
   N(CPP_HSPACE,		0)	/* Horizontal white space.  */	\
   N(CPP_DIRECTIVE,	0)	/* #define and the like */	\
   N(CPP_MACRO,		0)	/* Like a NAME, but expanded.  */
@@ -168,8 +168,9 @@ struct cpp_name
 
 /* Flags for the cpp_token structure.  */
 #define PREV_WHITESPACE     1	/* If whitespace before this token.  */
-#define DIGRAPH             2	/* If it was a digraph.  */
-#define UNSIGNED_INT        4   /* If int preprocessing token unsigned.  */
+#define BOL		    2   /* Beginning of line.  */
+#define DIGRAPH             4	/* If it was a digraph.  */
+#define UNSIGNED_INT        8   /* If int preprocessing token unsigned.  */
 
 /* A preprocessing token.  This has been carefully packed and should
    occupy 16 bytes on both 32- and 64-bit hosts.  */