re GNATS gcj/33 (gcj mangles composed characters)

	Fix for PR gcj/33:
	* jv-scan.c (help): Document --encoding.
	(options): Added `encoding' entry.
	(OPT_ENCODING): New define.
	(main): Handle --encoding.
	Include <langinfo.h> if nl_langinfo exists.
	* lang-options.h: Document --classpath, --CLASSPATH, --main, and
	--encoding.
	* jcf-parse.c Include <langinfo.h> if we have nl_langinfo.
	(parse_source_file): Correctly call java_init_lex.  Added `finput'
	argument.  Use nl_langinfo to determine default encoding.
	* java-tree.h (current_encoding): Declare.
	* parse.y (java_parser_context_restore_global): Don't restore
	`finput'.
	(java_parser_context_save_global): Don't set `finput' field.
	(java_pop_parser_context): Don't restore `finput'.  Free old lexer
	if required.
	* lang.c (current_encoding): New global.
	(lang_decode_option): Recognize `-fencoding='.
	(finish_parse): Don't close finput.
	* parse.h (struct parser_ctxt): Removed `finput' and
	`unget_utf8_value' fields.  Added `lexer' field.
	(java_init_lex): Fixed declaration.
	* lex.c (java_new_lexer): New function.
	(java_destroy_lexer): Likewise.
	(java_read_char): Added `lex' argument.  Handle iconv case.
	(java_read_unicode): Added `lex' argument.  Count backslashes in
	lexer structure.
	(java_init_lex): Added `finput' and `encoding' arguments.  Set
	`lexer' field in ctxp.
	(BAD_UTF8_VALUE): Removed.
	(java_lex): Handle seeing UEOF in the middle of a string literal.
	* lex.h: Include <iconv.h> if HAVE_ICONV defined.
	(java_lexer): New structure.
	(UNGETC): Removed.
	(GETC): Removed.
	(DEFAULT_ENCODING): New define.
	(java_destroy_lexer): Declare.

From-SVN: r36377

gcc/java/ChangeLog

 2000-09-12  Tom Tromey  <tromey@cygnus.com>
 
+	Fix for PR gcj/33:
+	* jv-scan.c (help): Document --encoding.
+	(options): Added `encoding' entry.
+	(OPT_ENCODING): New define.
+	(main): Handle --encoding.
+	Include <langinfo.h> if nl_langinfo exists.
+	* lang-options.h: Document --classpath, --CLASSPATH, --main, and
+	--encoding.
+	* jcf-parse.c Include <langinfo.h> if we have nl_langinfo.
+	(parse_source_file): Correctly call java_init_lex.  Added `finput'
+	argument.  Use nl_langinfo to determine default encoding.
+	* java-tree.h (current_encoding): Declare.
+	* parse.y (java_parser_context_restore_global): Don't restore
+	`finput'.
+	(java_parser_context_save_global): Don't set `finput' field.
+	(java_pop_parser_context): Don't restore `finput'.  Free old lexer
+	if required.
+	* lang.c (current_encoding): New global.
+	(lang_decode_option): Recognize `-fencoding='.
+	(finish_parse): Don't close finput.
+	* parse.h (struct parser_ctxt): Removed `finput' and
+	`unget_utf8_value' fields.  Added `lexer' field.
+	(java_init_lex): Fixed declaration.
+	* lex.c (java_new_lexer): New function.
+	(java_destroy_lexer): Likewise.
+	(java_read_char): Added `lex' argument.  Handle iconv case.
+	(java_read_unicode): Added `lex' argument.  Count backslashes in
+	lexer structure.
+	(java_init_lex): Added `finput' and `encoding' arguments.  Set
+	`lexer' field in ctxp.
+	(BAD_UTF8_VALUE): Removed.
+	(java_lex): Handle seeing UEOF in the middle of a string literal.
+	* lex.h: Include <iconv.h> if HAVE_ICONV defined.
+	(java_lexer): New structure.
+	(UNGETC): Removed.
+	(GETC): Removed.
+	(DEFAULT_ENCODING): New define.
+	(java_destroy_lexer): Declare.
+
+2000-09-12  Tom Tromey  <tromey@cygnus.com>
+
 	Fix for PR gcj/343:
 	* lex.c (java_init_lex): Initialize java_io_serializable.
 	* parse.y (java_io_serializable): New global.

gcc/java/java-tree.h

@@ -169,6 +169,9 @@ extern int flag_use_boehm_gc;
    object to its synchronization structure.  */
 extern int flag_hash_synchronization;
 
+/* Encoding used for source files.  */
+extern char *current_encoding;
+
 /* The Java .class file that provides main_class;  the main input file. */
 extern struct JCF *current_jcf;
 

gcc/java/jcf-parse.c

@@ -35,6 +35,10 @@ The Free Software Foundation is independent of Sun Microsystems, Inc.  */
 #include "toplev.h"
 #include "parse.h"
 
+#ifdef HAVE_NL_LANGINFO
+#include <langinfo.h>
+#endif
+
 /* A CONSTANT_Utf8 element is converted to an IDENTIFIER_NODE at parse time. */
 #define JPOOL_UTF(JCF, INDEX) CPOOL_UTF(&(JCF)->cpool, INDEX)
 #define JPOOL_UTF_LENGTH(JCF, INDEX) IDENTIFIER_LENGTH (JPOOL_UTF (JCF, INDEX))
@@ -83,7 +87,7 @@ static struct JCF main_jcf[1];
 static tree give_name_to_class PARAMS ((JCF *jcf, int index));
 static void parse_zip_file_entries PARAMS ((void));
 static void process_zip_dir PARAMS ((void));
-static void parse_source_file PARAMS ((tree));
+static void parse_source_file PARAMS ((tree, FILE *));
 static void jcf_parse_source PARAMS ((void));
 static int jcf_figure_file_type PARAMS ((JCF *));
 static int find_in_current_zip PARAMS ((const char *, struct JCF **));
@@ -564,6 +568,7 @@ static void
 jcf_parse_source ()
 {
   tree file;
+  FILE *finput;
 
   java_parser_context_save_global ();
   java_push_parser_context ();
@@ -576,7 +581,7 @@ jcf_parse_source ()
       if (!(finput = fopen (input_filename, "r")))
 	fatal ("input file `%s' just disappeared - jcf_parse_source",
 	       input_filename);
-      parse_source_file (file);
+      parse_source_file (file, finput);
       if (fclose (finput))
 	fatal ("can't close input file `%s' stream - jcf_parse_source",
 	       input_filename);
@@ -754,8 +759,9 @@ parse_class_file ()
 /* Parse a source file, as pointed by the current value of INPUT_FILENAME. */
 
 static void
-parse_source_file (file)
+parse_source_file (file, finput)
      tree file;
+     FILE *finput;
 {
   int save_error_count = java_error_count;
   /* Mark the file as parsed */
@@ -765,7 +771,21 @@ parse_source_file (file)
 
   lang_init_source (1);		    /* Error msgs have no method prototypes */
 
-  java_init_lex ();		    /* Initialize the parser */
+  /* There's no point in trying to find the current encoding unless we
+     are going to do something intelligent with it -- hence the test
+     for iconv.  */
+#ifdef HAVE_ICONV
+#ifdef HAVE_NL_LANGINFO
+  setlocale (LC_CTYPE, "");
+  if (current_encoding == NULL)
+    current_encoding = nl_langinfo (CODESET);
+#endif /* HAVE_NL_LANGINFO */
+#endif /* HAVE_ICONV */
+  if (current_encoding == NULL || *current_encoding == '\0')
+    current_encoding = DEFAULT_ENCODING;
+
+  /* Initialize the parser */
+  java_init_lex (finput, current_encoding);
   java_parse_abort_on_error ();
 
   java_parse ();		    /* Parse and build partial tree nodes. */
@@ -796,6 +816,7 @@ yyparse ()
   int several_files = 0;
   char *list = xstrdup (input_filename), *next;
   tree node, current_file_list = NULL_TREE;
+  FILE *finput;
 
   do 
     {
@@ -901,7 +922,7 @@ yyparse ()
 	case JCF_SOURCE:
 	  java_push_parser_context ();
 	  java_parser_context_save_global ();
-	  parse_source_file (name);
+	  parse_source_file (name, finput);
 	  java_parser_context_restore_global ();
 	  java_pop_parser_context (1);
 	  break;

gcc/java/jv-scan.c

@@ -26,6 +26,10 @@ Boston, MA 02111-1307, USA.  */
 
 #include "version.h"
 
+#ifdef HAVE_NL_LANGINFO
+#include <langinfo.h>
+#endif
+
 #include <getopt.h>
 
 void fatal PARAMS ((const char *s, ...)) ATTRIBUTE_PRINTF_1 ATTRIBUTE_NORETURN;
@@ -61,6 +65,7 @@ int flag_list_filename = 0;
 
 #define OPT_HELP      LONG_OPT (0)
 #define OPT_VERSION   LONG_OPT (1)
+#define OPT_ENCODING  LONG_OPT (2)
 
 static struct option options[] =
 {
@@ -69,6 +74,7 @@ static struct option options[] =
   { "print-main", no_argument,      &flag_find_main, 1 },
   { "list-filename", no_argument,   &flag_list_filename, 1 },
   { "list-class", no_argument,      &flag_dump_class, 1 },
+  { "encoding",  required_argument, NULL, OPT_ENCODING },
   { NULL,        no_argument,       NULL, 0 }
 };
 
@@ -84,6 +90,7 @@ help ()
 {
   printf ("Usage: jv-scan [OPTION]... FILE...\n\n");
   printf ("Print useful information read from Java source files.\n\n");
+  printf ("  --encoding NAME         Specify encoding of input file\n");
   printf ("  --print-main            Print name of class containing `main'\n");
   printf ("  --list-class            List all classes defined in file\n");
   printf ("  --list-filename         Print input filename when listing class names\n");
@@ -114,6 +121,7 @@ DEFUN (main, (argc, argv),
 {
   int i = 1;
   const char *output_file = NULL;
+  const char *encoding = NULL;
   long ft;
   int opt;
 
@@ -144,6 +152,10 @@ DEFUN (main, (argc, argv),
 	  version ();
 	  break;
 
+	case OPT_ENCODING:
+	  encoding = optarg;
+	  break;
+
 	default:
 	  usage ();
 	  break;
@@ -172,7 +184,20 @@ DEFUN (main, (argc, argv),
 	input_filename = argv [i];
 	if ( (finput = fopen (argv [i], "r")) )
 	  {
-	    java_init_lex ();
+	    /* There's no point in trying to find the current encoding
+	       unless we are going to do something intelligent with it
+	       -- hence the test for iconv.  */
+#ifdef HAVE_ICONV
+#ifdef HAVE_NL_LANGINFO
+	    setlocale (LC_CTYPE, "");
+	    if (encoding == NULL)
+	      encoding = nl_langinfo (CODESET);
+#endif /* HAVE_NL_LANGINFO */
+#endif /* HAVE_ICONV */
+	    if (encoding == NULL || *encoding == '\0')
+	      encoding = DEFAULT_ENCODING;
+
+	    java_init_lex (finput, encoding);
 	    yyparse ();
 	    if (ftell (out) != ft)
 	      fputc ('\n', out);

gcc/java/lang-options.h

@@ -42,8 +42,10 @@ DEFINE_LANG_NAME ("Java")
   { "-M", "Print dependencies to stdout" },
   { "-MM", "Print dependencies to stdout" },
 #endif /* ! USE_CPPLIB */
-  { "-fclasspath", "Set class path and suppress system path" },
-  { "-fCLASSPATH", "Set class path" },
+  { "--classpath", "Set class path and suppress system path" },
+  { "--CLASSPATH", "Set class path" },
+  { "--main", "Choose class whose main method should be used" },
+  { "--encoding", "Choose input encoding (default is UTF-8)" },
   { "-I", "Add directory to class path" },
   { "-foutput-class-dir", "Directory where class files should be written" },
   { "-fuse-divide-subroutine", "" },

gcc/java/lang.c

@@ -121,6 +121,9 @@ int flag_hash_synchronization;
    JNI, not CNI.  */
 int flag_jni = 0;
 
+/* The encoding of the source file.  */
+char *current_encoding = NULL;
+
 /* When non zero, report the now deprecated empty statements.  */
 int flag_extraneous_semicolon;
 
@@ -222,6 +225,13 @@ lang_decode_option (argc, argv)
       return 1;
     }
 #undef ARG
+#define ARG "-fencoding="
+  if (strncmp (p, ARG, sizeof (ARG) - 1) == 0)
+    {
+      current_encoding = p + sizeof (ARG) - 1;
+      return 1;
+    }
+#undef ARG
 
   if (p[0] == '-' && p[1] == 'f')
     {
@@ -309,7 +319,9 @@ lang_decode_option (argc, argv)
   return 0;
 }
 
+/* Global open file.  */
 FILE *finput;
+
 const char *
 init_parse (filename)
      const char *filename;
@@ -362,6 +374,7 @@ init_parse (filename)
 	    }
 	}
     }
+
   init_lex ();
 
   return filename;
@@ -370,7 +383,6 @@ init_parse (filename)
 void
 finish_parse ()
 {
-  fclose (finput);
   jcf_dependency_write ();
 }
 

gcc/java/lex.h

@@ -35,6 +35,13 @@ extern int   lineno;
 /* A Unicode character, as read from the input file  */
 typedef unsigned short unicode_t;
 
+#ifdef HAVE_ICONV
+#include <iconv.h>
+#endif /* HAVE_ICONV */
+
+/* Default encoding to use if no encoding is specified.  */
+#define DEFAULT_ENCODING "UTF-8"
+
 /* Debug macro to print-out what we match  */
 #ifdef JAVA_LEX_DEBUG
 #ifdef JAVA_LEX_DEBUG_CHAR
@@ -96,12 +103,38 @@ typedef struct _java_lc {
   int col;
 } java_lc;
 
+typedef struct java_lexer
+{
+  /* The file from which we're reading.  */
+  FILE *finput;
 
-#define JAVA_LINE_MAX 80
+  /* Number of consecutive backslashes we've read.  */
+  int bs_count;
+
+  /* If nonzero, a value that was pushed back.  */
+  unicode_t unget_value;
+
+#ifdef HAVE_ICONV
+  /* The handle for the iconv converter we're using.  */
+  iconv_t handle;
 
-/* Macro to read and unread bytes */
-#define UNGETC(c) ungetc(c, finput)
-#define GETC()    getc(finput)
+  /* Bytes we've read from the file but have not sent to iconv.  */
+  char buffer[1024];
+
+  /* Index of first valid character in buffer, -1 if no valid
+     characters.  */
+  int first;
+
+  /* Index of last valid character in buffer, plus one.  -1 if no
+     valid characters in buffer.  */
+  int last;
+#endif /* HAVE_ICONV */
+} java_lexer;
+
+/* Destroy a lexer object.  */
+extern void java_destroy_lexer PARAMS ((java_lexer *));
+
+#define JAVA_LINE_MAX 80
 
 /* Build a location compound integer */
 #define BUILD_LOCATION() ((ctxp->elc.line << 12) | (ctxp->elc.col & 0xfff))

gcc/java/parse.h

@@ -728,13 +728,12 @@ typedef struct _jdeplist {
 struct parser_ctxt {
 
   const char *filename;		    /* Current filename */
-  FILE *finput;			    /* Current file input stream */
   struct parser_ctxt *next;
 
+  java_lexer *lexer;		     /* Current lexer state */
   char marker_begining;		     /* Marker. Should be a sub-struct */
   struct java_line *p_line, *c_line; /* Previous and current line */
   java_lc elc;			     /* Error's line column info */
-  unicode_t unget_utf8_value;        /* An unget utf8 value */
   int ccb_indent;		     /* Keep track of {} indent, lexer */
   int first_ccb_indent1;	     /* First { at ident level 1 */
   int last_ccb_indent1;		     /* Last } at ident level 1 */
@@ -928,7 +927,7 @@ extern void reset_report PARAMS ((void));
 /* Always in use, no matter what you compile */
 void java_push_parser_context PARAMS ((void));
 void java_pop_parser_context PARAMS ((int));
-void java_init_lex PARAMS ((void));
+void java_init_lex PARAMS ((FILE *, const char *));
 extern void java_parser_context_save_global PARAMS ((void));
 extern void java_parser_context_restore_global PARAMS ((void));
 int yyparse PARAMS ((void));

gcc/java/parse.y

@@ -2618,10 +2618,13 @@ java_pop_parser_context (generate)
       next->incomplete_class = ctxp->incomplete_class;
       next->gclass_list = ctxp->gclass_list;
       lineno = ctxp->lineno;
-      finput = ctxp->finput;
       current_class = ctxp->current_class;
     }
 
+  /* If the old and new lexers differ, then free the old one.  */
+  if (ctxp->lexer && next && ctxp->lexer != next->lexer)
+    java_destroy_lexer (ctxp->lexer);
+
   /* Set the single import class file flag to 0 for the current list
      of imported things */
   for (current = ctxp->import_list; current; current = TREE_CHAIN (current))
@@ -2661,7 +2664,6 @@ java_parser_context_save_global ()
   else if (ctxp->saved_data)
     create_new_parser_context (1);
 
-  ctxp->finput = finput;
   ctxp->lineno = lineno;
   ctxp->current_class = current_class;
   ctxp->filename = input_filename;
@@ -2675,7 +2677,6 @@ java_parser_context_save_global ()
 void
 java_parser_context_restore_global ()
 {
-  finput = ctxp->finput;
   lineno = ctxp->lineno;
   current_class = ctxp->current_class;
   input_filename = ctxp->filename;