compiler: export indexed type data, read unexported types lazily

    
    Introduce a new "types" command to the export data to record the
    number of types and the size of their export data. It is immediately
    followed by new "type" commands that can be indexed. Parse all the
    exported types immediately so that we register them, but parse other
    type data only as needed.
    
    Reviewed-on: https://go-review.googlesource.com/c/143022

From-SVN: r265409

gcc/go/gofrontend/MERGE

-e1dc92a6037a3f81ea1b8ea8fb6207af33505f0c
+6db7e35d3bcd75ab3cb15296a5ddc5178038c9c1
 
 The first line of this file holds the git revision number of the last
 merge done from the gofrontend repository.

gcc/go/gofrontend/export.h

@@ -11,6 +11,7 @@
 
 class Go_sha1_helper;
 class Gogo;
+class Named_object;
 class Import_init;
 class Named_object;
 class Bindings;
@@ -154,6 +155,10 @@ class Export : public String_dump
 		 const Import_init_set& imported_init_fns,
 		 const Bindings* bindings);
 
+  // Set the index of a type.
+  bool
+  set_type_index(Type*);
+
   // Write a string to the export stream.
   void
   write_string(const std::string& s)
@@ -196,7 +201,7 @@ class Export : public String_dump
   Export& operator=(const Export&);
 
   // Prepare types for exporting.
-  void
+  int
   prepare_types(const std::vector<Named_object*>* exports,
 		Unordered_set(const Package*)* imports);
 
@@ -224,24 +229,27 @@ class Export : public String_dump
   write_imported_init_fns(const std::string& package_name,
 			  const std::string&, const Import_init_set&);
 
+  // Write out all types.
+  void
+  write_types(int unexported_type_index);
+
+  // Write out one type definition.
+  void
+  write_type_definition(const Type* type, int index);
+
   // Register one builtin type.
   void
   register_builtin_type(Gogo*, const char* name, Builtin_code);
 
-  // Mapping from Type objects to a constant index.
-  typedef Unordered_map(const Type*, int) Type_refs;
-
   // The stream to which we are writing data.
   Stream* stream_;
-  // Type mappings.
-  Type_refs type_refs_;
   // Index number of next type.
   int type_index_;
   // Packages we have written out.
   Unordered_set(const Package*) packages_;
 };
 
-// An export streamer which puts the export stream in a named section.
+// An export streamer that puts the export stream in a named section.
 
 class Stream_to_section : public Export::Stream
 {
@@ -256,4 +264,26 @@ class Stream_to_section : public Export::Stream
   Backend* backend_;
 };
 
+// An export streamer that puts the export stream in a string.
+
+class Stream_to_string : public Export::Stream
+{
+ public:
+  Stream_to_string()
+    : string_()
+  {}
+
+  const std::string&
+  string() const
+  { return this->string_; }
+
+ protected:
+  void
+  do_write(const char* s, size_t len)
+  { this->string_.append(s, len); }
+
+ private:
+  std::string string_;
+};
+
 #endif // !defined(GO_EXPORT_H)

gcc/go/gofrontend/gogo.cc

@@ -7511,8 +7511,8 @@ Named_object::export_named_object(Export* exp) const
       break;
 
     case NAMED_OBJECT_TYPE:
-      this->type_value()->export_named_type(exp, this->name_);
-      break;
+      // Types are handled by export::write_types.
+      go_unreachable();
 
     case NAMED_OBJECT_TYPE_DECLARATION:
       go_error_at(this->type_declaration_value()->location(),

gcc/go/gofrontend/gogo.h

@@ -12,9 +12,7 @@
 class Traverse;
 class Statement_inserter;
 class Type;
-class Type_hash_identical;
 class Type_equal;
-class Type_identical;
 class Typed_identifier;
 class Typed_identifier_list;
 class Function_type;

gcc/go/gofrontend/import.cc

@@ -236,7 +236,7 @@ Import::find_export_data(const std::string& filename, int fd, Location location)
     }
 
   char buf[len];
-  ssize_t c = read(fd, buf, len);
+  ssize_t c = ::read(fd, buf, len);
   if (c < len)
     return NULL;
 
@@ -288,7 +288,7 @@ Import::find_object_export_data(const std::string& filename,
 
 Import::Import(Stream* stream, Location location)
   : gogo_(NULL), stream_(stream), location_(location), package_(NULL),
-    add_to_globals_(false),
+    add_to_globals_(false), type_data_(), type_pos_(0), type_offsets_(),
     builtin_types_((- SMALLEST_BUILTIN_CODE) + 1),
     types_(), version_(EXPORT_FORMAT_UNKNOWN)
 {
@@ -403,6 +403,12 @@ Import::import(Gogo* gogo, const std::string& local_name,
       if (stream->match_c_string("init"))
 	this->read_import_init_fns(gogo);
 
+      if (stream->match_c_string("types "))
+	{
+	  if (!this->read_types())
+	    return NULL;
+	}
+
       // Loop over all the input data for this package.
       while (!stream->saw_error())
 	{
@@ -585,6 +591,86 @@ Import::read_import_init_fns(Gogo* gogo)
     }
 }
 
+// Import the types.  Starting in export format version 3 all the
+// types are listed first.
+
+bool
+Import::read_types()
+{
+  this->require_c_string("types ");
+  std::string str = this->read_identifier();
+  int maxp1;
+  if (!this->string_to_int(str, false, &maxp1))
+    return false;
+
+  this->require_c_string(" ");
+  str = this->read_identifier();
+  int exportedp1;
+  if (!this->string_to_int(str, false, &exportedp1))
+    return false;
+
+  this->type_offsets_.resize(maxp1, std::make_pair<size_t, size_t>(0, 0));
+  size_t total_type_size = 0;
+  // Start at 1 because type index 0 not used.
+  for (int i = 1; i < maxp1; i++)
+    {
+      this->require_c_string(" ");
+      str = this->read_identifier();
+      int v;
+      if (!this->string_to_int(str, false, &v))
+	return false;
+      size_t vs = static_cast<size_t>(v);
+      this->type_offsets_[i] = std::make_pair(total_type_size, vs);
+      total_type_size += vs;
+    }
+
+  this->require_c_string("\n");
+
+  // Types can refer to each other in an unpredictable order.  Read
+  // all the type data into type_data_.  The type_offsets_ vector we
+  // just initialized provides indexes into type_data_.
+
+  this->type_pos_ = this->stream_->pos();
+  const char* type_data;
+  if (!this->stream_->peek(total_type_size, &type_data))
+    return false;
+  this->type_data_ = std::string(type_data, total_type_size);
+  this->advance(total_type_size);
+
+  this->types_.resize(maxp1, NULL);
+
+  // Parse all the exported types now, so that the names are properly
+  // bound and visible to the parser.  Parse unexported types lazily.
+
+  // Start at 1 because there is no type 0.
+  for (int i = 1; i < exportedp1; i++)
+    {
+      // We may have already parsed this type when we parsed an
+      // earlier type.
+      Type* type = this->types_[i];
+      if (type == NULL)
+	{
+	  if (!this->parse_type(i))
+	    return false;
+	  type = this->types_[i];
+	  go_assert(type != NULL);
+	}
+      Named_type* nt = type->named_type();
+      if (nt == NULL)
+	{
+	  go_error_at(this->location_,
+		      "error in import data: exported unnamed type %d",
+		      i);
+	  return false;
+	}
+      nt->set_is_visible();
+      if (this->add_to_globals_)
+	this->gogo_->add_named_type(nt);
+    }
+
+  return true;
+}
+
 // Import a constant.
 
 void
@@ -605,6 +691,18 @@ Import::import_const()
 void
 Import::import_type()
 {
+  if (this->version_ >= EXPORT_FORMAT_V3)
+    {
+      if (!this->stream_->saw_error())
+	{
+	  go_error_at(this->location_,
+		    "error in import data at %d: old type syntax",
+		    this->stream_->pos());
+	  this->stream_->set_saw_error();
+	}
+      return;
+    }
+
   Named_type* type;
   Named_type::import_named_type(this, &type);
 
@@ -694,9 +792,73 @@ Import::import_func(Package* package)
   return no;
 }
 
+// Read a type definition and initialize the entry in this->types_.
+// This parses the type definition saved by read_types earlier.  This
+// returns true on success, false on failure.
+
+bool
+Import::parse_type(int i)
+{
+  go_assert(i >= 0 && static_cast<size_t>(i) < this->types_.size());
+  go_assert(this->types_[i] == NULL);
+  size_t offset = this->type_offsets_[i].first;
+  size_t len = this->type_offsets_[i].second;
+
+  Stream* orig_stream = this->stream_;
+
+  Stream_from_string_ref stream(this->type_data_, offset, len);
+  stream.set_pos(this->type_pos_ + offset);
+  this->stream_ = &stream;
+
+  this->require_c_string("type ");
+  std::string str = this->read_identifier();
+  int id;
+  if (!this->string_to_int(str, false, &id))
+    {
+      this->stream_ = orig_stream;
+      return false;
+    }
+  if (i != id)
+    {
+      go_error_at(this->location_,
+		  ("error in import data at %d: "
+		   "type ID mismatch: got %d, want %d"),
+		  stream.pos(), id, i);
+      this->stream_ = orig_stream;
+      return false;
+    }
+
+  this->require_c_string(" ");
+  if (stream.peek_char() == '"')
+    {
+      stream.advance(1);
+      Type* type = this->read_named_type(i);
+      if (type->is_error_type())
+	{
+	  this->stream_ = orig_stream;
+	  return false;
+	}
+    }
+  else
+    {
+      Type* type = Type::import_type(this);
+      if (type->is_error_type())
+	{
+	  this->stream_ = orig_stream;
+	  return false;
+	}
+      this->types_[i] = type;
+
+      this->require_c_string("\n");
+    }
+
+  this->stream_ = orig_stream;
+  return true;
+}
+
 // Read a type in the import stream.  This records the type by the
-// type index.  If the type is named, it registers the name, but marks
-// it as invisible.
+// type index.  If the type is named (which can only happen with older
+// export formats), it registers the name, but marks it as invisible.
 
 Type*
 Import::read_type()
@@ -720,7 +882,28 @@ Import::read_type()
 
   if (c == '>')
     {
-      // This type was already defined.
+      // A reference to a type defined earlier.
+
+      if (index >= 0 && !this->type_data_.empty())
+	{
+	  if (static_cast<size_t>(index) >= this->type_offsets_.size())
+	    {
+	      go_error_at(this->location_,
+			  ("error in import data at %d: "
+			   "bad type index %d >= %d"),
+			  stream->pos(), index,
+			  static_cast<int>(this->type_offsets_.size()));
+	      stream->set_saw_error();
+	      return Type::make_error_type();
+	    }
+
+	  if (this->types_[index] == NULL)
+	    {
+	      if (!this->parse_type(index))
+		return Type::make_error_type();
+	    }
+	}
+
       if (index < 0
 	  ? (static_cast<size_t>(- index) >= this->builtin_types_.size()
 	     || this->builtin_types_[- index] == NULL)
@@ -737,11 +920,21 @@ Import::read_type()
       return index < 0 ? this->builtin_types_[- index] : this->types_[index];
     }
 
+  if (this->version_ >= EXPORT_FORMAT_V3)
+    {
+      if (!stream->saw_error())
+	go_error_at(this->location_,
+		    "error in import data at %d: expected %<>%>",
+		    stream->pos());
+      stream->set_saw_error();
+      return Type::make_error_type();
+    }
+
   if (c != ' ')
     {
       if (!stream->saw_error())
 	go_error_at(this->location_,
-		    "error in import data at %d: expect %< %> or %<>%>'",
+		    "error in import data at %d: expected %< %> or %<>%>'",
 		    stream->pos());
       stream->set_saw_error();
       stream->advance(1);
@@ -774,10 +967,25 @@ Import::read_type()
       return type;
     }
 
-  // This type has a name.
-
   stream->advance(1);
+
+  Type* type = this->read_named_type(index);
+
+  this->require_c_string(">");
+
+  return type;
+}
+
+// Read a named type from the import stream and store it in
+// this->types_[index].  The stream should be positioned immediately
+// after the '"' that starts the name.
+
+Type*
+Import::read_named_type(int index)
+{
+  Stream* stream = this->stream_;
   std::string type_name;
+  int c;
   while ((c = stream->get_char()) != '"')
     type_name += c;
 
@@ -863,7 +1071,7 @@ Import::read_type()
   // If there is no type definition, then this is just a forward
   // declaration of a type defined in some other file.
   Type* type;
-  if (this->match_c_string(">"))
+  if (this->match_c_string(">") || this->match_c_string("\n"))
     type = this->types_[index];
   else
     {
@@ -912,8 +1120,6 @@ Import::read_type()
 	}
     }
 
-  this->require_c_string(">");
-
   return type;
 }
 
@@ -1125,10 +1331,9 @@ Stream_from_file::do_peek(size_t length, const char** bytes)
       *bytes = this->data_.data();
       return true;
     }
-  // Don't bother to handle the general case, since we don't need it.
-  go_assert(length < 64);
-  char buf[64];
-  ssize_t got = read(this->fd_, buf, length);
+
+  this->data_.resize(length);
+  ssize_t got = ::read(this->fd_, &this->data_[0], length);
 
   if (got < 0)
     {
@@ -1149,8 +1354,6 @@ Stream_from_file::do_peek(size_t length, const char** bytes)
   if (static_cast<size_t>(got) < length)
     return false;
 
-  this->data_.assign(buf, got);
-
   *bytes = this->data_.data();
   return true;
 }

gcc/go/gofrontend/import.h

@@ -30,6 +30,11 @@ class Import
     Stream();
     virtual ~Stream();
 
+    // Set the position, for error messages.
+    void
+    set_pos(int pos)
+    { this->pos_ = pos; }
+
     // Return whether we have seen an error.
     bool
     saw_error() const
@@ -249,6 +254,10 @@ class Import
   void
   read_import_init_fns(Gogo*);
 
+  // Read the types.
+  bool
+  read_types();
+
   // Import a constant.
   void
   import_const();
@@ -265,6 +274,14 @@ class Import
   Named_object*
   import_func(Package*);
 
+  // Parse a type definition.
+  bool
+  parse_type(int index);
+
+  // Read a named type and store it at this->type_[index].
+  Type*
+  read_named_type(int index);
+
   // Register a single builtin type.
   void
   register_builtin_type(Gogo*, const char* name, Builtin_code);
@@ -299,6 +316,12 @@ class Import
   // Whether to add new objects to the global scope, rather than to a
   // package scope.
   bool add_to_globals_;
+  // All type data.
+  std::string type_data_;
+  // Position of type data in the stream.
+  int type_pos_;
+  // Mapping from type code to offset/length in type_data_.
+  std::vector<std::pair<size_t, size_t> > type_offsets_;
   // Mapping from negated builtin type codes to Type structures.
   std::vector<Named_type*> builtin_types_;
   // Mapping from exported type codes to Type structures.
@@ -399,4 +422,41 @@ class Stream_from_file : public Import::Stream
   std::string data_;
 };
 
+// Read import data from an offset into a std::string.  This uses a
+// reference to the string, to avoid copying, so the string must be
+// kept alive through some other mechanism.
+
+class Stream_from_string_ref : public Import::Stream
+{
+ public:
+  Stream_from_string_ref(const std::string& str, size_t offset, size_t length)
+    : str_(str), pos_(offset), end_(offset + length)
+  { }
+
+  ~Stream_from_string_ref()
+  {}
+
+ protected:
+  bool
+  do_peek(size_t length, const char** bytes)
+  {
+    if (this->pos_ + length > this->end_)
+      return false;
+    *bytes = &this->str_[this->pos_];
+    return true;
+  }
+
+  void
+  do_advance(size_t length)
+  { this->pos_ += length; }
+
+ private:
+  // A reference to the string we are reading from.
+  const std::string& str_;
+  // The current offset into the string.
+  size_t pos_;
+  // The index after the last byte we can read.
+  size_t end_;
+};
+
 #endif // !defined(GO_IMPORT_H)

gcc/go/gofrontend/types.cc

@@ -10865,19 +10865,8 @@ Named_type::append_reflection_type_name(Gogo* gogo, bool use_alias,
   ret->append(Gogo::unpack_hidden_name(this->named_object_->name()));
 }
 
-// Export the type.  This is called to export a global type.
-
-void
-Named_type::export_named_type(Export* exp, const std::string&) const
-{
-  // We don't need to write the name of the type here, because it will
-  // be written by Export::write_type anyhow.
-  exp->write_c_string("type ");
-  exp->write_type(this);
-  exp->write_c_string("\n");
-}
-
-// Import a named type.
+// Import a named type.  This is only used for export format versions
+// before version 3.
 
 void
 Named_type::import_named_type(Import* imp, Named_type** ptype)
@@ -10891,12 +10880,15 @@ Named_type::import_named_type(Import* imp, Named_type** ptype)
 }
 
 // Export the type when it is referenced by another type.  In this
-// case Export::export_type will already have issued the name.
+// case Export::export_type will already have issued the name.  The
+// output always ends with a newline, since that is convenient if
+// there are methods.
 
 void
 Named_type::do_export(Export* exp) const
 {
   exp->write_type(this->type_);
+  exp->write_c_string("\n");
 
   // To save space, we only export the methods directly attached to
   // this type.
@@ -10904,7 +10896,6 @@ Named_type::do_export(Export* exp) const
   if (methods == NULL)
     return;
 
-  exp->write_c_string("\n");
   for (Bindings::const_definitions_iterator p = methods->begin_definitions();
        p != methods->end_definitions();
        ++p)

gcc/go/gofrontend/types.h

@@ -3445,10 +3445,6 @@ class Named_type : public Type
   void
   append_mangled_type_name(Gogo*, bool use_alias, std::string*) const;
 
-  // Export the type.
-  void
-  export_named_type(Export*, const std::string& name) const;
-
   // Import a named type.
   static void
   import_named_type(Import*, Named_type**);

libgo/go/go/internal/gccgoimporter/parser.go

@@ -18,7 +18,7 @@ import (
 )
 
 type parser struct {
-	scanner  scanner.Scanner
+	scanner  *scanner.Scanner
 	version  string                    // format version
 	tok      rune                      // current token
 	lit      string                    // literal string; only valid for Ident, Int, String tokens
@@ -27,18 +27,24 @@ type parser struct {
 	pkg      *types.Package            // reference to imported package
 	imports  map[string]*types.Package // package path -> package object
 	typeMap  map[int]types.Type        // type number -> type
+	typeData []string                  // unparsed type data
 	initdata InitData                  // package init priority data
 }
 
 func (p *parser) init(filename string, src io.Reader, imports map[string]*types.Package) {
+	p.scanner = new(scanner.Scanner)
+	p.initScanner(filename, src)
+	p.imports = imports
+	p.typeMap = make(map[int]types.Type)
+}
+
+func (p *parser) initScanner(filename string, src io.Reader) {
 	p.scanner.Init(src)
 	p.scanner.Error = func(_ *scanner.Scanner, msg string) { p.error(msg) }
 	p.scanner.Mode = scanner.ScanIdents | scanner.ScanInts | scanner.ScanFloats | scanner.ScanStrings | scanner.ScanComments | scanner.SkipComments
 	p.scanner.Whitespace = 1<<'\t' | 1<<' '
 	p.scanner.Filename = filename // for good error messages
 	p.next()
-	p.imports = imports
-	p.typeMap = make(map[int]types.Type)
 }
 
 type importError struct {
@@ -720,6 +726,9 @@ func (p *parser) parseType(pkg *types.Package) (t types.Type) {
 		n := p.parseInt()
 
 		if p.tok == '>' {
+			if len(p.typeData) > 0 && p.typeMap[int(n)] == nil {
+				p.parseSavedType(pkg, int(n))
+			}
 			t = p.typeMap[int(n)]
 		} else {
 			t = p.parseTypeDefinition(pkg, int(n))
@@ -739,6 +748,67 @@ func (p *parser) parseType(pkg *types.Package) (t types.Type) {
 	return
 }
 
+// Types = "types" maxp1 exportedp1 (offset length)* .
+func (p *parser) parseTypes(pkg *types.Package) {
+	maxp1 := p.parseInt()
+	exportedp1 := p.parseInt()
+
+	type typeOffset struct {
+		offset int
+		length int
+	}
+	var typeOffsets []typeOffset
+
+	total := 0
+	for i := 1; i < int(maxp1); i++ {
+		len := int(p.parseInt())
+		typeOffsets = append(typeOffsets, typeOffset{total, len})
+		total += len
+	}
+
+	// We should now have p.tok pointing to the final newline.
+	// The next runes from the scanner should be the type data.
+
+	var sb strings.Builder
+	for sb.Len() < total {
+		r := p.scanner.Next()
+		if r == scanner.EOF {
+			p.error("unexpected EOF")
+		}
+		sb.WriteRune(r)
+	}
+	allTypeData := sb.String()
+
+	p.typeData = []string{""} // type 0, unused
+	for _, to := range typeOffsets {
+		p.typeData = append(p.typeData, allTypeData[to.offset:to.offset+to.length])
+	}
+
+	for i := 1; i < int(exportedp1); i++ {
+		p.parseSavedType(pkg, i)
+	}
+}
+
+// parseSavedType parses one saved type definition.
+func (p *parser) parseSavedType(pkg *types.Package, i int) {
+	defer func(s *scanner.Scanner, tok rune, lit string) {
+		p.scanner = s
+		p.tok = tok
+		p.lit = lit
+	}(p.scanner, p.tok, p.lit)
+
+	p.scanner = new(scanner.Scanner)
+	p.initScanner(p.scanner.Filename, strings.NewReader(p.typeData[i]))
+	p.expectKeyword("type")
+	id := int(p.parseInt())
+	if id != i {
+		p.errorf("type ID mismatch: got %d, want %d", id, i)
+	}
+	if p.typeMap[i] == nil {
+		p.typeMap[i] = p.parseTypeDefinition(pkg, i)
+	}
+}
+
 // PackageInit = unquotedString unquotedString int .
 func (p *parser) parsePackageInit() PackageInit {
 	name := p.parseUnquotedString()
@@ -883,6 +953,11 @@ func (p *parser) parseDirective() {
 		p.getPkg(pkgpath, pkgname)
 		p.expectEOL()
 
+	case "types":
+		p.next()
+		p.parseTypes(p.pkg)
+		p.expectEOL()
+
 	case "func":
 		p.next()
 		fun := p.parseFunc(p.pkg)