Handle C2x attributes in Objective-C.

When adding the initial support for C2x attributes, I deferred the
unbounded lookahead support required to support such attributes in
Objective-C (except for the changes to string literal handling, which
were the riskier piece of preparation for such lookahead support).
This patch adds that remaining ObjC support.

For C, the parser continues to work exactly as it did before.  For
ObjC, however, when checking for whether '[[' starts attributes, it
lexes however many tokens are needed to check for a matching ']]', but
in a raw mode that omits all the context-sensitive processing that
c_lex_with_flags normally does, so that that processing can be done
later when the right context-sensitive flags are set.  Those tokens
are saved in a separate raw_tokens vector in the parser, and normal
c_lex_one_token calls will get tokens from there and perform the
remaining processing on them, if any tokens are found there, so all
parsing not using the new interfaces gets the same tokens as it did
before.  (For C, this raw lexing never occurs and the vector of raw
tokens is always NULL.)

Bootstrapped with no regressions for x86_64-pc-linux-gnu.

gcc/c:
	* c-parser.c (struct c_parser): Add members raw_tokens and
	raw_tokens_used.
	(c_lex_one_token): Add argument raw.  Handle lexing raw tokens and
	using previously-lexed raw tokens.
	(c_parser_peek_nth_token_raw)
	(c_parser_check_balanced_raw_token_sequence): New functions.
	(c_parser_nth_token_starts_std_attributes): Use
	c_parser_check_balanced_raw_token_sequence for Objective-C.

gcc/testsuite:
	* objc.dg/attributes/gnu2x-attr-syntax-1.m: New test.

From-SVN: r278827

gcc/c/ChangeLog

+2019-11-29  Joseph Myers  <joseph@codesourcery.com>
+
+	* c-parser.c (struct c_parser): Add members raw_tokens and
+	raw_tokens_used.
+	(c_lex_one_token): Add argument raw.  Handle lexing raw tokens and
+	using previously-lexed raw tokens.
+	(c_parser_peek_nth_token_raw)
+	(c_parser_check_balanced_raw_token_sequence): New functions.
+	(c_parser_nth_token_starts_std_attributes): Use
+	c_parser_check_balanced_raw_token_sequence for Objective-C.
+
 2019-11-25  Joseph Myers  <joseph@codesourcery.com>
 
 	PR c/91985

gcc/c/c-parser.c

@@ -176,6 +176,12 @@ struct GTY(()) c_parser {
   /* How many look-ahead tokens are available (0 - 4, or
      more if parsing from pre-lexed tokens).  */
   unsigned int tokens_avail;
+  /* Raw look-ahead tokens, used only for checking in Objective-C
+     whether '[[' starts attributes.  */
+  vec<c_token, va_gc> *raw_tokens;
+  /* The number of raw look-ahead tokens that have since been fully
+     lexed.  */
+  unsigned int raw_tokens_used;
   /* True if a syntax error is being recovered from; false otherwise.
      c_parser_error sets this flag.  It should clear this flag when
      enough tokens have been consumed to recover from the error.  */
@@ -251,20 +257,39 @@ c_parser_set_error (c_parser *parser, bool err)
 
 static GTY (()) c_parser *the_parser;
 
-/* Read in and lex a single token, storing it in *TOKEN.  */
+/* Read in and lex a single token, storing it in *TOKEN.  If RAW,
+   context-sensitive postprocessing of the token is not done.  */
 
 static void
-c_lex_one_token (c_parser *parser, c_token *token)
+c_lex_one_token (c_parser *parser, c_token *token, bool raw = false)
 {
   timevar_push (TV_LEX);
 
-  token->type = c_lex_with_flags (&token->value, &token->location,
-				  &token->flags,
-				  (parser->lex_joined_string
-				   ? 0 : C_LEX_STRING_NO_JOIN));
-  token->id_kind = C_ID_NONE;
-  token->keyword = RID_MAX;
-  token->pragma_kind = PRAGMA_NONE;
+  if (raw || vec_safe_length (parser->raw_tokens) == 0)
+    {
+      token->type = c_lex_with_flags (&token->value, &token->location,
+				      &token->flags,
+				      (parser->lex_joined_string
+				       ? 0 : C_LEX_STRING_NO_JOIN));
+      token->id_kind = C_ID_NONE;
+      token->keyword = RID_MAX;
+      token->pragma_kind = PRAGMA_NONE;
+    }
+  else
+    {
+      /* Use a token previously lexed as a raw look-ahead token, and
+	 complete the processing on it.  */
+      *token = (*parser->raw_tokens)[parser->raw_tokens_used];
+      ++parser->raw_tokens_used;
+      if (parser->raw_tokens_used == vec_safe_length (parser->raw_tokens))
+	{
+	  vec_free (parser->raw_tokens);
+	  parser->raw_tokens_used = 0;
+	}
+    }
+
+  if (raw)
+    goto out;
 
   switch (token->type)
     {
@@ -434,6 +459,7 @@ c_lex_one_token (c_parser *parser, c_token *token)
     default:
       break;
     }
+ out:
   timevar_pop (TV_LEX);
 }
 
@@ -484,6 +510,32 @@ c_parser_peek_nth_token (c_parser *parser, unsigned int n)
   return &parser->tokens[n - 1];
 }
 
+/* Return a pointer to the Nth token from PARSER, reading it in as a
+   raw look-ahead token if necessary.  The N-1th token is already read
+   in.  Raw look-ahead tokens remain available for when the non-raw
+   functions above are called.  */
+
+c_token *
+c_parser_peek_nth_token_raw (c_parser *parser, unsigned int n)
+{
+  /* N is 1-based, not zero-based.  */
+  gcc_assert (n > 0);
+
+  if (parser->tokens_avail >= n)
+    return &parser->tokens[n - 1];
+  unsigned int raw_len = vec_safe_length (parser->raw_tokens);
+  unsigned int raw_avail
+    = parser->tokens_avail + raw_len - parser->raw_tokens_used;
+  gcc_assert (raw_avail >= n - 1);
+  if (raw_avail >= n)
+    return &(*parser->raw_tokens)[parser->raw_tokens_used
+				  + n - 1 - parser->tokens_avail];
+  vec_safe_reserve (parser->raw_tokens, 1);
+  parser->raw_tokens->quick_grow (raw_len + 1);
+  c_lex_one_token (parser, &(*parser->raw_tokens)[raw_len], true);
+  return &(*parser->raw_tokens)[raw_len];
+}
+
 bool
 c_keyword_starts_typename (enum rid keyword)
 {
@@ -4968,6 +5020,80 @@ c_parser_std_attribute_specifier (c_parser *parser, bool for_tm)
   return nreverse (attributes);
 }
 
+/* Look past an optional balanced token sequence of raw look-ahead
+   tokens starting with the *Nth token.  *N is updated to point to the
+   following token.  Return true if such a sequence was found, false
+   if the tokens parsed were not balanced.  */
+
+static bool
+c_parser_check_balanced_raw_token_sequence (c_parser *parser, unsigned int *n)
+{
+  while (true)
+    {
+      c_token *token = c_parser_peek_nth_token_raw (parser, *n);
+      switch (token->type)
+	{
+	case CPP_OPEN_BRACE:
+	  {
+	    ++*n;
+	    if (c_parser_check_balanced_raw_token_sequence (parser, n))
+	      {
+		token = c_parser_peek_nth_token_raw (parser, *n);
+		if (token->type == CPP_CLOSE_BRACE)
+		  ++*n;
+		else
+		  return false;
+	      }
+	    else
+	      return false;
+	    break;
+	  }
+
+	case CPP_OPEN_PAREN:
+	  {
+	    ++*n;
+	    if (c_parser_check_balanced_raw_token_sequence (parser, n))
+	      {
+		token = c_parser_peek_nth_token_raw (parser, *n);
+		if (token->type == CPP_CLOSE_PAREN)
+		  ++*n;
+		else
+		  return false;
+	      }
+	    else
+	      return false;
+	    break;
+	  }
+
+	case CPP_OPEN_SQUARE:
+	  {
+	    ++*n;
+	    if (c_parser_check_balanced_raw_token_sequence (parser, n))
+	      {
+		token = c_parser_peek_nth_token_raw (parser, *n);
+		if (token->type == CPP_CLOSE_SQUARE)
+		  ++*n;
+		else
+		  return false;
+	      }
+	    else
+	      return false;
+	    break;
+	  }
+
+	case CPP_CLOSE_BRACE:
+	case CPP_CLOSE_PAREN:
+	case CPP_CLOSE_SQUARE:
+	case CPP_EOF:
+	  return true;
+
+	default:
+	  ++*n;
+	  break;
+	}
+    }
+}
+
 /* Return whether standard attributes start with the Nth token.  */
 
 static bool
@@ -4976,10 +5102,18 @@ c_parser_nth_token_starts_std_attributes (c_parser *parser, unsigned int n)
   if (!(c_parser_peek_nth_token (parser, n)->type == CPP_OPEN_SQUARE
 	&& c_parser_peek_nth_token (parser, n + 1)->type == CPP_OPEN_SQUARE))
     return false;
-  /* In C, '[[' must start attributes.  In Objective-C, identifying
-     whether those tokens start attributes requires unbounded
-     lookahead, which is not yet implemented.  */
-  return !c_dialect_objc ();
+  /* In C, '[[' must start attributes.  In Objective-C, we need to
+     check whether '[[' is matched by ']]'.  */
+  if (!c_dialect_objc ())
+    return true;
+  n += 2;
+  if (!c_parser_check_balanced_raw_token_sequence (parser, &n))
+    return false;
+  c_token *token = c_parser_peek_nth_token_raw (parser, n);
+  if (token->type != CPP_CLOSE_SQUARE)
+    return false;
+  token = c_parser_peek_nth_token_raw (parser, n + 1);
+  return token->type == CPP_CLOSE_SQUARE;
 }
 
 static tree

gcc/testsuite/ChangeLog

+2019-11-29  Joseph Myers  <joseph@codesourcery.com>
+
+	* objc.dg/attributes/gnu2x-attr-syntax-1.m: New test.
+
 2019-11-28  Martin Liska  <mliska@suse.cz>
 
 	PR lto/92609

gcc/testsuite/objc.dg/attributes/gnu2x-attr-syntax-1.m

+/* Test C2x attribute syntax.  Test empty attributes accepted.  */
+/* { dg-do compile } */
+/* { dg-options "-std=gnu2x" } */
+
+#include "../../gcc.dg/c2x-attr-syntax-1.c"