Commit d79a22ed by Ian Lance Taylor

libgo: update to final 1.14.2 release

Reviewed-on: https://go-review.googlesource.com/c/gofrontend/+/227551
parent d09f80ae
b31fbf7d8f23508cfbd578c5c44b13eefd8f359e 89fbf55a409d37ae898e5c4ea4250035f86bed1b
The first line of this file holds the git revision number of the last The first line of this file holds the git revision number of the last
merge done from the gofrontend repository. merge done from the gofrontend repository.
edea4a79e8d7dea2456b688f492c8af33d381dc2 96745b980cfde139e8611772e2bc0c59a8e6cdf7
The first line of this file holds the git revision number of the The first line of this file holds the git revision number of the
last merge done from the master library sources. last merge done from the master library sources.
...@@ -2506,13 +2506,21 @@ ...@@ -2506,13 +2506,21 @@
// The "go get" command remains permitted to update go.mod even with -mod=readonly, // The "go get" command remains permitted to update go.mod even with -mod=readonly,
// and the "go mod" commands do not take the -mod flag (or any other build flags). // and the "go mod" commands do not take the -mod flag (or any other build flags).
// //
// If invoked with -mod=vendor, the go command assumes that the vendor // If invoked with -mod=vendor, the go command loads packages from the main
// directory holds the correct copies of dependencies and ignores // module's vendor directory instead of downloading modules to and loading packages
// the dependency descriptions in go.mod. // from the module cache. The go command assumes the vendor directory holds
// correct copies of dependencies, and it does not compute the set of required
// module versions from go.mod files. However, the go command does check that
// vendor/modules.txt (generated by 'go mod vendor') contains metadata consistent
// with go.mod.
// //
// If invoked with -mod=mod, the go command loads modules from the module cache // If invoked with -mod=mod, the go command loads modules from the module cache
// even if there is a vendor directory present. // even if there is a vendor directory present.
// //
// If the go command is not invoked with a -mod flag and the vendor directory
// is present and the "go" version in go.mod is 1.14 or higher, the go command
// will act as if it were invoked with -mod=vendor.
//
// Pseudo-versions // Pseudo-versions
// //
// The go.mod file and the go command more generally use semantic versions as // The go.mod file and the go command more generally use semantic versions as
...@@ -2710,22 +2718,28 @@ ...@@ -2710,22 +2718,28 @@
// //
// Modules and vendoring // Modules and vendoring
// //
// When using modules, the go command completely ignores vendor directories. // When using modules, the go command typically satisfies dependencies by
// // downloading modules from their sources and using those downloaded copies
// By default, the go command satisfies dependencies by downloading modules // (after verification, as described in the previous section). Vendoring may
// from their sources and using those downloaded copies (after verification, // be used to allow interoperation with older versions of Go, or to ensure
// as described in the previous section). To allow interoperation with older // that all files used for a build are stored together in a single file tree.
// versions of Go, or to ensure that all files used for a build are stored //
// together in a single file tree, 'go mod vendor' creates a directory named // The command 'go mod vendor' constructs a directory named vendor in the main
// vendor in the root directory of the main module and stores there all the // module's root directory that contains copies of all packages needed to support
// packages from dependency modules that are needed to support builds and // builds and tests of packages in the main module. 'go mod vendor' also
// tests of packages in the main module. // creates the file vendor/modules.txt that contains metadata about vendored
// // packages and module versions. This file should be kept consistent with go.mod:
// To build using the main module's top-level vendor directory to satisfy // when vendoring is used, 'go mod vendor' should be run after go.mod is updated.
// dependencies (disabling use of the usual network sources and local //
// caches), use 'go build -mod=vendor'. Note that only the main module's // If the vendor directory is present in the main module's root directory, it will
// top-level vendor directory is used; vendor directories in other locations // be used automatically if the "go" version in the main module's go.mod file is
// are still ignored. // 1.14 or higher. Build commands like 'go build' and 'go test' will load packages
// from the vendor directory instead of accessing the network or the local module
// cache. To explicitly enable vendoring, invoke the go command with the flag
// -mod=vendor. To disable vendoring, use the flag -mod=mod.
//
// Unlike vendoring in GOPATH, the go command ignores vendor directories in
// locations other than the main module's root directory.
// //
// //
// Module authentication using go.sum // Module authentication using go.sum
......
...@@ -6,6 +6,7 @@ package modcmd ...@@ -6,6 +6,7 @@ package modcmd
import ( import (
"bytes" "bytes"
"errors"
"fmt" "fmt"
"io/ioutil" "io/ioutil"
"os" "os"
...@@ -67,12 +68,10 @@ func verifyMod(mod module.Version) bool { ...@@ -67,12 +68,10 @@ func verifyMod(mod module.Version) bool {
_, zipErr = os.Stat(zip) _, zipErr = os.Stat(zip)
} }
dir, dirErr := modfetch.DownloadDir(mod) dir, dirErr := modfetch.DownloadDir(mod)
if dirErr == nil {
_, dirErr = os.Stat(dir)
}
data, err := ioutil.ReadFile(zip + "hash") data, err := ioutil.ReadFile(zip + "hash")
if err != nil { if err != nil {
if zipErr != nil && os.IsNotExist(zipErr) && dirErr != nil && os.IsNotExist(dirErr) { if zipErr != nil && errors.Is(zipErr, os.ErrNotExist) &&
dirErr != nil && errors.Is(dirErr, os.ErrNotExist) {
// Nothing downloaded yet. Nothing to verify. // Nothing downloaded yet. Nothing to verify.
return true return true
} }
...@@ -81,7 +80,7 @@ func verifyMod(mod module.Version) bool { ...@@ -81,7 +80,7 @@ func verifyMod(mod module.Version) bool {
} }
h := string(bytes.TrimSpace(data)) h := string(bytes.TrimSpace(data))
if zipErr != nil && os.IsNotExist(zipErr) { if zipErr != nil && errors.Is(zipErr, os.ErrNotExist) {
// ok // ok
} else { } else {
hZ, err := dirhash.HashZip(zip, dirhash.DefaultHash) hZ, err := dirhash.HashZip(zip, dirhash.DefaultHash)
...@@ -93,7 +92,7 @@ func verifyMod(mod module.Version) bool { ...@@ -93,7 +92,7 @@ func verifyMod(mod module.Version) bool {
ok = false ok = false
} }
} }
if dirErr != nil && os.IsNotExist(dirErr) { if dirErr != nil && errors.Is(dirErr, os.ErrNotExist) {
// ok // ok
} else { } else {
hD, err := dirhash.HashDir(dir, mod.Path+"@"+mod.Version, dirhash.DefaultHash) hD, err := dirhash.HashDir(dir, mod.Path+"@"+mod.Version, dirhash.DefaultHash)
......
...@@ -7,6 +7,7 @@ package modfetch ...@@ -7,6 +7,7 @@ package modfetch
import ( import (
"bytes" "bytes"
"encoding/json" "encoding/json"
"errors"
"fmt" "fmt"
"io" "io"
"io/ioutil" "io/ioutil"
...@@ -56,8 +57,11 @@ func CachePath(m module.Version, suffix string) (string, error) { ...@@ -56,8 +57,11 @@ func CachePath(m module.Version, suffix string) (string, error) {
return filepath.Join(dir, encVer+"."+suffix), nil return filepath.Join(dir, encVer+"."+suffix), nil
} }
// DownloadDir returns the directory to which m should be downloaded. // DownloadDir returns the directory to which m should have been downloaded.
// Note that the directory may not yet exist. // An error will be returned if the module path or version cannot be escaped.
// An error satisfying errors.Is(err, os.ErrNotExist) will be returned
// along with the directory if the directory does not exist or if the directory
// is not completely populated.
func DownloadDir(m module.Version) (string, error) { func DownloadDir(m module.Version) (string, error) {
if PkgMod == "" { if PkgMod == "" {
return "", fmt.Errorf("internal error: modfetch.PkgMod not set") return "", fmt.Errorf("internal error: modfetch.PkgMod not set")
...@@ -76,9 +80,39 @@ func DownloadDir(m module.Version) (string, error) { ...@@ -76,9 +80,39 @@ func DownloadDir(m module.Version) (string, error) {
if err != nil { if err != nil {
return "", err return "", err
} }
return filepath.Join(PkgMod, enc+"@"+encVer), nil
dir := filepath.Join(PkgMod, enc+"@"+encVer)
if fi, err := os.Stat(dir); os.IsNotExist(err) {
return dir, err
} else if err != nil {
return dir, &DownloadDirPartialError{dir, err}
} else if !fi.IsDir() {
return dir, &DownloadDirPartialError{dir, errors.New("not a directory")}
}
partialPath, err := CachePath(m, "partial")
if err != nil {
return dir, err
}
if _, err := os.Stat(partialPath); err == nil {
return dir, &DownloadDirPartialError{dir, errors.New("not completely extracted")}
} else if !os.IsNotExist(err) {
return dir, err
}
return dir, nil
}
// DownloadDirPartialError is returned by DownloadDir if a module directory
// exists but was not completely populated.
//
// DownloadDirPartialError is equivalent to os.ErrNotExist.
type DownloadDirPartialError struct {
Dir string
Err error
} }
func (e *DownloadDirPartialError) Error() string { return fmt.Sprintf("%s: %v", e.Dir, e.Err) }
func (e *DownloadDirPartialError) Is(err error) bool { return err == os.ErrNotExist }
// lockVersion locks a file within the module cache that guards the downloading // lockVersion locks a file within the module cache that guards the downloading
// and extraction of the zipfile for the given module version. // and extraction of the zipfile for the given module version.
func lockVersion(mod module.Version) (unlock func(), err error) { func lockVersion(mod module.Version) (unlock func(), err error) {
......
...@@ -22,6 +22,7 @@ import ( ...@@ -22,6 +22,7 @@ import (
"cmd/go/internal/lockedfile" "cmd/go/internal/lockedfile"
"cmd/go/internal/par" "cmd/go/internal/par"
"cmd/go/internal/renameio" "cmd/go/internal/renameio"
"cmd/go/internal/robustio"
"golang.org/x/mod/module" "golang.org/x/mod/module"
"golang.org/x/mod/sumdb/dirhash" "golang.org/x/mod/sumdb/dirhash"
...@@ -45,24 +46,27 @@ func Download(mod module.Version) (dir string, err error) { ...@@ -45,24 +46,27 @@ func Download(mod module.Version) (dir string, err error) {
err error err error
} }
c := downloadCache.Do(mod, func() interface{} { c := downloadCache.Do(mod, func() interface{} {
dir, err := DownloadDir(mod) dir, err := download(mod)
if err != nil { if err != nil {
return cached{"", err} return cached{"", err}
} }
if err := download(mod, dir); err != nil {
return cached{"", err}
}
checkMod(mod) checkMod(mod)
return cached{dir, nil} return cached{dir, nil}
}).(cached) }).(cached)
return c.dir, c.err return c.dir, c.err
} }
func download(mod module.Version, dir string) (err error) { func download(mod module.Version) (dir string, err error) {
// If the directory exists, the module has already been extracted. // If the directory exists, and no .partial file exists,
fi, err := os.Stat(dir) // the module has already been completely extracted.
if err == nil && fi.IsDir() { // .partial files may be created when future versions of cmd/go
return nil // extract module zip directories in place instead of extracting
// to a random temporary directory and renaming.
dir, err = DownloadDir(mod)
if err == nil {
return dir, nil
} else if dir == "" || !errors.Is(err, os.ErrNotExist) {
return "", err
} }
// To avoid cluttering the cache with extraneous files, // To avoid cluttering the cache with extraneous files,
...@@ -70,22 +74,24 @@ func download(mod module.Version, dir string) (err error) { ...@@ -70,22 +74,24 @@ func download(mod module.Version, dir string) (err error) {
// Invoke DownloadZip before locking the file. // Invoke DownloadZip before locking the file.
zipfile, err := DownloadZip(mod) zipfile, err := DownloadZip(mod)
if err != nil { if err != nil {
return err return "", err
} }
unlock, err := lockVersion(mod) unlock, err := lockVersion(mod)
if err != nil { if err != nil {
return err return "", err
} }
defer unlock() defer unlock()
// Check whether the directory was populated while we were waiting on the lock. // Check whether the directory was populated while we were waiting on the lock.
fi, err = os.Stat(dir) _, dirErr := DownloadDir(mod)
if err == nil && fi.IsDir() { if dirErr == nil {
return nil return dir, nil
} }
_, dirExists := dirErr.(*DownloadDirPartialError)
// Clean up any remaining temporary directories from previous runs. // Clean up any remaining temporary directories from previous runs, as well
// as partially extracted diectories created by future versions of cmd/go.
// This is only safe to do because the lock file ensures that their writers // This is only safe to do because the lock file ensures that their writers
// are no longer active. // are no longer active.
parentDir := filepath.Dir(dir) parentDir := filepath.Dir(dir)
...@@ -95,6 +101,19 @@ func download(mod module.Version, dir string) (err error) { ...@@ -95,6 +101,19 @@ func download(mod module.Version, dir string) (err error) {
RemoveAll(path) // best effort RemoveAll(path) // best effort
} }
} }
if dirExists {
if err := RemoveAll(dir); err != nil {
return "", err
}
}
partialPath, err := CachePath(mod, "partial")
if err != nil {
return "", err
}
if err := os.Remove(partialPath); err != nil && !os.IsNotExist(err) {
return "", err
}
// Extract the zip file to a temporary directory, then rename it to the // Extract the zip file to a temporary directory, then rename it to the
// final path. That way, we can use the existence of the source directory to // final path. That way, we can use the existence of the source directory to
...@@ -102,11 +121,11 @@ func download(mod module.Version, dir string) (err error) { ...@@ -102,11 +121,11 @@ func download(mod module.Version, dir string) (err error) {
// the entire directory (e.g. as an attempt to prune out file corruption) // the entire directory (e.g. as an attempt to prune out file corruption)
// the module cache will still be left in a recoverable state. // the module cache will still be left in a recoverable state.
if err := os.MkdirAll(parentDir, 0777); err != nil { if err := os.MkdirAll(parentDir, 0777); err != nil {
return err return "", err
} }
tmpDir, err := ioutil.TempDir(parentDir, tmpPrefix) tmpDir, err := ioutil.TempDir(parentDir, tmpPrefix)
if err != nil { if err != nil {
return err return "", err
} }
defer func() { defer func() {
if err != nil { if err != nil {
...@@ -116,11 +135,11 @@ func download(mod module.Version, dir string) (err error) { ...@@ -116,11 +135,11 @@ func download(mod module.Version, dir string) (err error) {
if err := modzip.Unzip(tmpDir, mod, zipfile); err != nil { if err := modzip.Unzip(tmpDir, mod, zipfile); err != nil {
fmt.Fprintf(os.Stderr, "-> %s\n", err) fmt.Fprintf(os.Stderr, "-> %s\n", err)
return err return "", err
} }
if err := os.Rename(tmpDir, dir); err != nil { if err := robustio.Rename(tmpDir, dir); err != nil {
return err return "", err
} }
if !cfg.ModCacheRW { if !cfg.ModCacheRW {
...@@ -128,7 +147,7 @@ func download(mod module.Version, dir string) (err error) { ...@@ -128,7 +147,7 @@ func download(mod module.Version, dir string) (err error) {
// os.Rename was observed to fail for read-only directories on macOS. // os.Rename was observed to fail for read-only directories on macOS.
makeDirsReadOnly(dir) makeDirsReadOnly(dir)
} }
return nil return dir, nil
} }
var downloadZipCache par.Cache var downloadZipCache par.Cache
......
...@@ -148,9 +148,7 @@ func moduleInfo(m module.Version, fromBuildList bool) *modinfo.ModulePublic { ...@@ -148,9 +148,7 @@ func moduleInfo(m module.Version, fromBuildList bool) *modinfo.ModulePublic {
} }
dir, err := modfetch.DownloadDir(mod) dir, err := modfetch.DownloadDir(mod)
if err == nil { if err == nil {
if info, err := os.Stat(dir); err == nil && info.IsDir() { m.Dir = dir
m.Dir = dir
}
} }
} }
} }
......
...@@ -176,13 +176,21 @@ not need updates, such as in a continuous integration and testing system. ...@@ -176,13 +176,21 @@ not need updates, such as in a continuous integration and testing system.
The "go get" command remains permitted to update go.mod even with -mod=readonly, The "go get" command remains permitted to update go.mod even with -mod=readonly,
and the "go mod" commands do not take the -mod flag (or any other build flags). and the "go mod" commands do not take the -mod flag (or any other build flags).
If invoked with -mod=vendor, the go command assumes that the vendor If invoked with -mod=vendor, the go command loads packages from the main
directory holds the correct copies of dependencies and ignores module's vendor directory instead of downloading modules to and loading packages
the dependency descriptions in go.mod. from the module cache. The go command assumes the vendor directory holds
correct copies of dependencies, and it does not compute the set of required
module versions from go.mod files. However, the go command does check that
vendor/modules.txt (generated by 'go mod vendor') contains metadata consistent
with go.mod.
If invoked with -mod=mod, the go command loads modules from the module cache If invoked with -mod=mod, the go command loads modules from the module cache
even if there is a vendor directory present. even if there is a vendor directory present.
If the go command is not invoked with a -mod flag and the vendor directory
is present and the "go" version in go.mod is 1.14 or higher, the go command
will act as if it were invoked with -mod=vendor.
Pseudo-versions Pseudo-versions
The go.mod file and the go command more generally use semantic versions as The go.mod file and the go command more generally use semantic versions as
...@@ -380,22 +388,28 @@ the format of the cached downloaded packages. ...@@ -380,22 +388,28 @@ the format of the cached downloaded packages.
Modules and vendoring Modules and vendoring
When using modules, the go command completely ignores vendor directories. When using modules, the go command typically satisfies dependencies by
downloading modules from their sources and using those downloaded copies
By default, the go command satisfies dependencies by downloading modules (after verification, as described in the previous section). Vendoring may
from their sources and using those downloaded copies (after verification, be used to allow interoperation with older versions of Go, or to ensure
as described in the previous section). To allow interoperation with older that all files used for a build are stored together in a single file tree.
versions of Go, or to ensure that all files used for a build are stored
together in a single file tree, 'go mod vendor' creates a directory named The command 'go mod vendor' constructs a directory named vendor in the main
vendor in the root directory of the main module and stores there all the module's root directory that contains copies of all packages needed to support
packages from dependency modules that are needed to support builds and builds and tests of packages in the main module. 'go mod vendor' also
tests of packages in the main module. creates the file vendor/modules.txt that contains metadata about vendored
packages and module versions. This file should be kept consistent with go.mod:
To build using the main module's top-level vendor directory to satisfy when vendoring is used, 'go mod vendor' should be run after go.mod is updated.
dependencies (disabling use of the usual network sources and local
caches), use 'go build -mod=vendor'. Note that only the main module's If the vendor directory is present in the main module's root directory, it will
top-level vendor directory is used; vendor directories in other locations be used automatically if the "go" version in the main module's go.mod file is
are still ignored. 1.14 or higher. Build commands like 'go build' and 'go test' will load packages
from the vendor directory instead of accessing the network or the local module
cache. To explicitly enable vendoring, invoke the go command with the flag
-mod=vendor. To disable vendoring, use the flag -mod=mod.
Unlike vendoring in GOPATH, the go command ignores vendor directories in
locations other than the main module's root directory.
`, `,
} }
......
...@@ -15,7 +15,7 @@ import ( ...@@ -15,7 +15,7 @@ import (
"time" "time"
) )
const arbitraryTimeout = 500 * time.Millisecond const arbitraryTimeout = 2000 * time.Millisecond
// retry retries ephemeral errors from f up to an arbitrary timeout // retry retries ephemeral errors from f up to an arbitrary timeout
// to work around filesystem flakiness on Windows and Darwin. // to work around filesystem flakiness on Windows and Darwin.
......
...@@ -1941,7 +1941,11 @@ Run: ...@@ -1941,7 +1941,11 @@ Run:
// The bitmask starts at s.startAddr. // The bitmask starts at s.startAddr.
// The result must be deallocated with dematerializeGCProg. // The result must be deallocated with dematerializeGCProg.
func materializeGCProg(ptrdata uintptr, prog *byte) *mspan { func materializeGCProg(ptrdata uintptr, prog *byte) *mspan {
s := mheap_.allocManual((ptrdata/(8*sys.PtrSize)+pageSize-1)/pageSize, &memstats.gc_sys) // Each word of ptrdata needs one bit in the bitmap.
bitmapBytes := divRoundUp(ptrdata, 8*sys.PtrSize)
// Compute the number of pages needed for bitmapBytes.
pages := divRoundUp(bitmapBytes, pageSize)
s := mheap_.allocManual(pages, &memstats.gc_sys)
runGCProg(addb(prog, 4), nil, (*byte)(unsafe.Pointer(s.startAddr)), 1) runGCProg(addb(prog, 4), nil, (*byte)(unsafe.Pointer(s.startAddr)), 1)
return s return s
} }
......
...@@ -250,6 +250,13 @@ func alignDown(n, a uintptr) uintptr { ...@@ -250,6 +250,13 @@ func alignDown(n, a uintptr) uintptr {
return n &^ (a - 1) return n &^ (a - 1)
} }
// divRoundUp returns ceil(n / a).
func divRoundUp(n, a uintptr) uintptr {
// a is generally a power of two. This will get inlined and
// the compiler will optimize the division.
return (n + a - 1) / a
}
// checkASM returns whether assembly runtime checks have passed. // checkASM returns whether assembly runtime checks have passed.
func checkASM() bool { func checkASM() bool {
return true return true
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment