From f2113d94c15dfa82d5afb46bc750f009284e2f64 Mon Sep 17 00:00:00 2001 From: Dmitri Shuralyov Date: Wed, 12 Sep 2018 13:58:18 -0400 Subject: [PATCH 01/13] [release-branch.go1.11] doc/go1.11, cmd/go: elaborate on new GOFLAGS environment variable In Go 1.11, cmd/go gained support for the GOFLAGS environment variable. It was added and described in detail in CL 126656. Mention it in the Go 1.11 release notes, link to the cmd/go documentation, and add more details there. Fixes #27387. Change-Id: Ifc35bfe3e0886a145478d36dde8e80aedd8ec68e Reviewed-on: https://go-review.googlesource.com/135035 Reviewed-by: Bryan C. Mills Reviewed-by: Rob Pike (cherry picked from commit 58c6afe075d74261dd67750e0aab5a1b8460839f) Reviewed-on: https://go-review.googlesource.com/135496 Reviewed-by: Dmitri Shuralyov --- doc/go1.11.html | 14 ++++++++++++++ src/cmd/go/alldocs.go | 6 ++++++ src/cmd/go/internal/help/helpdoc.go | 6 ++++++ 3 files changed, 26 insertions(+) diff --git a/doc/go1.11.html b/doc/go1.11.html index afe19397663af..16b4c904cb332 100644 --- a/doc/go1.11.html +++ b/doc/go1.11.html @@ -348,6 +348,20 @@

Cgo

details.

+

Go command

+ +

+ The environment variable GOFLAGS may now be used + to set default flags for the go command. + This is useful in certain situations. + Linking can be noticeably slower on underpowered systems due to DWARF, + and users may want to set -ldflags=-w by default. + For modules, some users and CI systems will want vendoring always, + so they should set -mod=vendor by default. + For more information, see the go + command documentation. +

+

Godoc

diff --git a/src/cmd/go/alldocs.go b/src/cmd/go/alldocs.go index ebbd154f3e6ff..1060e779c1b6e 100644 --- a/src/cmd/go/alldocs.go +++ b/src/cmd/go/alldocs.go @@ -1449,6 +1449,12 @@ // The directory where the go command will write // temporary source files, packages, and binaries. // +// Each entry in the GOFLAGS list must be a standalone flag. +// Because the entries are space-separated, flag values must +// not contain spaces. In some cases, you can provide multiple flag +// values instead: for example, to set '-ldflags=-s -w' +// you can use 'GOFLAGS=-ldflags=-s -ldflags=-w'. +// // Environment variables for use with cgo: // // CC diff --git a/src/cmd/go/internal/help/helpdoc.go b/src/cmd/go/internal/help/helpdoc.go index aff4ce12f6c9a..e2c4e61615bcc 100644 --- a/src/cmd/go/internal/help/helpdoc.go +++ b/src/cmd/go/internal/help/helpdoc.go @@ -507,6 +507,12 @@ General-purpose environment variables: The directory where the go command will write temporary source files, packages, and binaries. +Each entry in the GOFLAGS list must be a standalone flag. +Because the entries are space-separated, flag values must +not contain spaces. In some cases, you can provide multiple flag +values instead: for example, to set '-ldflags=-s -w' +you can use 'GOFLAGS=-ldflags=-s -ldflags=-w'. + Environment variables for use with cgo: CC From b5ed6ec14092b04156adcbaba101958dc9d9d74b Mon Sep 17 00:00:00 2001 From: Johan Brandhorst Date: Fri, 24 Aug 2018 12:10:01 +0100 Subject: [PATCH 02/13] [release-branch.go1.11] net/http: ensure null body in Fetch response is not read The Fetch API returns a null body if there is no response body, on browsers that support streaming the response body. This change ensures we check for both undefined and null bodies before attempting to read the body. Fixes #27424 Change-Id: I0da86b61284fe394418b4b431495e715a037f335 Reviewed-on: https://go-review.googlesource.com/131236 Reviewed-by: Richard Musiol Run-TryBot: Richard Musiol TryBot-Result: Gobot Gobot (cherry picked from commit ce536837d8e53f1bf0c7ef450d4580d19f7d6f52) Reviewed-on: https://go-review.googlesource.com/136915 --- src/net/http/roundtrip_js.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/net/http/roundtrip_js.go b/src/net/http/roundtrip_js.go index 16b7b891c86b1..38e4f5573e60e 100644 --- a/src/net/http/roundtrip_js.go +++ b/src/net/http/roundtrip_js.go @@ -116,7 +116,9 @@ func (t *Transport) RoundTrip(req *Request) (*Response, error) { b := result.Get("body") var body io.ReadCloser - if b != js.Undefined() { + // The body is undefined when the browser does not support streaming response bodies (Firefox), + // and null in certain error cases, i.e. when the request is blocked because of CORS settings. + if b != js.Undefined() && b != js.Null() { body = &streamReader{stream: b.Call("getReader")} } else { // Fall back to using ArrayBuffer From e535c71009224422ae1853b88f071bb250eac1af Mon Sep 17 00:00:00 2001 From: Keith Randall Date: Mon, 17 Sep 2018 12:25:36 -0700 Subject: [PATCH 03/13] [release-branch.go1.11] runtime: ignore races between close and len/cap They aren't really races, or at least they don't have any observable effect. The spec is silent on whether these are actually races or not. Fix this problem by not using the address of len (or of cap) as the location where channel operations are recorded to occur. Use a random other field of hchan for that. I'm not 100% sure we should in fact fix this. Opinions welcome. Fixes #27778 Change-Id: Ib4efd4b62e0d1ef32fa51e373035ef207a655084 Reviewed-on: https://go-review.googlesource.com/135698 Reviewed-by: Dmitry Vyukov (cherry picked from commit 83dfc3b001245f0b725afdc94c0b540fe1952d21) Reviewed-on: https://go-review.googlesource.com/138179 Run-TryBot: Keith Randall TryBot-Result: Gobot Gobot Reviewed-by: Ian Lance Taylor --- src/runtime/chan.go | 23 ++++++++++++++------ src/runtime/race/testdata/chan_test.go | 30 +++++++++++++++++++------- src/runtime/select.go | 4 ++-- 3 files changed, 40 insertions(+), 17 deletions(-) diff --git a/src/runtime/chan.go b/src/runtime/chan.go index ce71cee4c5b40..bf41dfddaf6d9 100644 --- a/src/runtime/chan.go +++ b/src/runtime/chan.go @@ -92,7 +92,7 @@ func makechan(t *chantype, size int) *hchan { // Queue or element size is zero. c = (*hchan)(mallocgc(hchanSize, nil, true)) // Race detector uses this location for synchronization. - c.buf = unsafe.Pointer(c) + c.buf = c.raceaddr() case elem.kind&kindNoPointers != 0: // Elements do not contain pointers. // Allocate hchan and buf in one call. @@ -151,7 +151,7 @@ func chansend(c *hchan, ep unsafe.Pointer, block bool, callerpc uintptr) bool { } if raceenabled { - racereadpc(unsafe.Pointer(c), callerpc, funcPC(chansend)) + racereadpc(c.raceaddr(), callerpc, funcPC(chansend)) } // Fast path: check for failed non-blocking operation without acquiring the lock. @@ -337,8 +337,8 @@ func closechan(c *hchan) { if raceenabled { callerpc := getcallerpc() - racewritepc(unsafe.Pointer(c), callerpc, funcPC(closechan)) - racerelease(unsafe.Pointer(c)) + racewritepc(c.raceaddr(), callerpc, funcPC(closechan)) + racerelease(c.raceaddr()) } c.closed = 1 @@ -361,7 +361,7 @@ func closechan(c *hchan) { gp := sg.g gp.param = nil if raceenabled { - raceacquireg(gp, unsafe.Pointer(c)) + raceacquireg(gp, c.raceaddr()) } gp.schedlink.set(glist) glist = gp @@ -380,7 +380,7 @@ func closechan(c *hchan) { gp := sg.g gp.param = nil if raceenabled { - raceacquireg(gp, unsafe.Pointer(c)) + raceacquireg(gp, c.raceaddr()) } gp.schedlink.set(glist) glist = gp @@ -457,7 +457,7 @@ func chanrecv(c *hchan, ep unsafe.Pointer, block bool) (selected, received bool) if c.closed != 0 && c.qcount == 0 { if raceenabled { - raceacquire(unsafe.Pointer(c)) + raceacquire(c.raceaddr()) } unlock(&c.lock) if ep != nil { @@ -735,6 +735,15 @@ func (q *waitq) dequeue() *sudog { } } +func (c *hchan) raceaddr() unsafe.Pointer { + // Treat read-like and write-like operations on the channel to + // happen at this address. Avoid using the address of qcount + // or dataqsiz, because the len() and cap() builtins read + // those addresses, and we don't want them racing with + // operations like close(). + return unsafe.Pointer(&c.buf) +} + func racesync(c *hchan, sg *sudog) { racerelease(chanbuf(c, 0)) raceacquireg(sg.g, chanbuf(c, 0)) diff --git a/src/runtime/race/testdata/chan_test.go b/src/runtime/race/testdata/chan_test.go index 7f349c42ed783..60e55ed66a4b0 100644 --- a/src/runtime/race/testdata/chan_test.go +++ b/src/runtime/race/testdata/chan_test.go @@ -577,18 +577,32 @@ func TestRaceChanItselfCap(t *testing.T) { <-compl } -func TestRaceChanCloseLen(t *testing.T) { - v := 0 - _ = v +func TestNoRaceChanCloseLen(t *testing.T) { c := make(chan int, 10) - c <- 0 + r := make(chan int, 10) + go func() { + r <- len(c) + }() go func() { - v = 1 close(c) + r <- 0 }() - time.Sleep(1e7) - _ = len(c) - v = 2 + <-r + <-r +} + +func TestNoRaceChanCloseCap(t *testing.T) { + c := make(chan int, 10) + r := make(chan int, 10) + go func() { + r <- cap(c) + }() + go func() { + close(c) + r <- 0 + }() + <-r + <-r } func TestRaceChanCloseSend(t *testing.T) { diff --git a/src/runtime/select.go b/src/runtime/select.go index 3a3ac6b7ac0bd..2729c2ecf98a2 100644 --- a/src/runtime/select.go +++ b/src/runtime/select.go @@ -245,7 +245,7 @@ loop: case caseSend: if raceenabled { - racereadpc(unsafe.Pointer(c), cas.pc, chansendpc) + racereadpc(c.raceaddr(), cas.pc, chansendpc) } if c.closed != 0 { goto sclose @@ -462,7 +462,7 @@ rclose: typedmemclr(c.elemtype, cas.elem) } if raceenabled { - raceacquire(unsafe.Pointer(c)) + raceacquire(c.raceaddr()) } goto retc From 05a0c7b4c60e240e521b0840c786cacd69221e68 Mon Sep 17 00:00:00 2001 From: Ian Gudger Date: Wed, 5 Sep 2018 23:53:36 -0700 Subject: [PATCH 04/13] [release-branch.go1.11] net: fail fast for DNS rcode success with no answers of requested type DNS responses which do not contain answers of the requested type return errNoSuchHost, the same error as rcode name error. Prior to golang.org/cl/37879, both cases resulted in no additional name servers being consulted for the question. That CL changed the behavior for both cases. Issue #25336 was filed about the rcode name error case and golang.org/cl/113815 fixed it. This CL fixes the no answers of requested type case as well. Updates #27525 Fixes #27537 Change-Id: I52fadedcd195f16adf62646b76bea2ab3b15d117 Reviewed-on: https://go-review.googlesource.com/133675 Run-TryBot: Ian Gudger TryBot-Result: Gobot Gobot Reviewed-by: Brad Fitzpatrick (cherry picked from commit 94f48ddb96c4dfc919ae024f64df19d764f5fb5b) Reviewed-on: https://go-review.googlesource.com/138175 Run-TryBot: Brad Fitzpatrick Reviewed-by: Ian Lance Taylor --- src/net/dnsclient_unix.go | 115 +++++++++++++++----------- src/net/dnsclient_unix_test.go | 143 ++++++++++++++++++++------------- 2 files changed, 158 insertions(+), 100 deletions(-) diff --git a/src/net/dnsclient_unix.go b/src/net/dnsclient_unix.go index 2fee3346e92dc..9a0b1d69a81b1 100644 --- a/src/net/dnsclient_unix.go +++ b/src/net/dnsclient_unix.go @@ -27,6 +27,20 @@ import ( "golang_org/x/net/dns/dnsmessage" ) +var ( + errLameReferral = errors.New("lame referral") + errCannotUnmarshalDNSMessage = errors.New("cannot unmarshal DNS message") + errCannotMarshalDNSMessage = errors.New("cannot marshal DNS message") + errServerMisbehaving = errors.New("server misbehaving") + errInvalidDNSResponse = errors.New("invalid DNS response") + errNoAnswerFromDNSServer = errors.New("no answer from DNS server") + + // errServerTemporarlyMisbehaving is like errServerMisbehaving, except + // that when it gets translated to a DNSError, the IsTemporary field + // gets set to true. + errServerTemporarlyMisbehaving = errors.New("server misbehaving") +) + func newRequest(q dnsmessage.Question) (id uint16, udpReq, tcpReq []byte, err error) { id = uint16(rand.Int()) ^ uint16(time.Now().UnixNano()) b := dnsmessage.NewBuilder(make([]byte, 2, 514), dnsmessage.Header{ID: id, RecursionDesired: true}) @@ -105,14 +119,14 @@ func dnsStreamRoundTrip(c Conn, id uint16, query dnsmessage.Question, b []byte) var p dnsmessage.Parser h, err := p.Start(b[:n]) if err != nil { - return dnsmessage.Parser{}, dnsmessage.Header{}, errors.New("cannot unmarshal DNS message") + return dnsmessage.Parser{}, dnsmessage.Header{}, errCannotUnmarshalDNSMessage } q, err := p.Question() if err != nil { - return dnsmessage.Parser{}, dnsmessage.Header{}, errors.New("cannot unmarshal DNS message") + return dnsmessage.Parser{}, dnsmessage.Header{}, errCannotUnmarshalDNSMessage } if !checkResponse(id, query, h, q) { - return dnsmessage.Parser{}, dnsmessage.Header{}, errors.New("invalid DNS response") + return dnsmessage.Parser{}, dnsmessage.Header{}, errInvalidDNSResponse } return p, h, nil } @@ -122,7 +136,7 @@ func (r *Resolver) exchange(ctx context.Context, server string, q dnsmessage.Que q.Class = dnsmessage.ClassINET id, udpReq, tcpReq, err := newRequest(q) if err != nil { - return dnsmessage.Parser{}, dnsmessage.Header{}, errors.New("cannot marshal DNS message") + return dnsmessage.Parser{}, dnsmessage.Header{}, errCannotMarshalDNSMessage } for _, network := range []string{"udp", "tcp"} { ctx, cancel := context.WithDeadline(ctx, time.Now().Add(timeout)) @@ -147,31 +161,31 @@ func (r *Resolver) exchange(ctx context.Context, server string, q dnsmessage.Que return dnsmessage.Parser{}, dnsmessage.Header{}, mapErr(err) } if err := p.SkipQuestion(); err != dnsmessage.ErrSectionDone { - return dnsmessage.Parser{}, dnsmessage.Header{}, errors.New("invalid DNS response") + return dnsmessage.Parser{}, dnsmessage.Header{}, errInvalidDNSResponse } if h.Truncated { // see RFC 5966 continue } return p, h, nil } - return dnsmessage.Parser{}, dnsmessage.Header{}, errors.New("no answer from DNS server") + return dnsmessage.Parser{}, dnsmessage.Header{}, errNoAnswerFromDNSServer } // checkHeader performs basic sanity checks on the header. func checkHeader(p *dnsmessage.Parser, h dnsmessage.Header, name, server string) error { + if h.RCode == dnsmessage.RCodeNameError { + return errNoSuchHost + } + _, err := p.AnswerHeader() if err != nil && err != dnsmessage.ErrSectionDone { - return &DNSError{ - Err: "cannot unmarshal DNS message", - Name: name, - Server: server, - } + return errCannotUnmarshalDNSMessage } // libresolv continues to the next server when it receives // an invalid referral response. See golang.org/issue/15434. if h.RCode == dnsmessage.RCodeSuccess && !h.Authoritative && !h.RecursionAvailable && err == dnsmessage.ErrSectionDone { - return &DNSError{Err: "lame referral", Name: name, Server: server} + return errLameReferral } if h.RCode != dnsmessage.RCodeSuccess && h.RCode != dnsmessage.RCodeNameError { @@ -180,11 +194,10 @@ func checkHeader(p *dnsmessage.Parser, h dnsmessage.Header, name, server string) // a name error and we didn't get success, // the server is behaving incorrectly or // having temporary trouble. - err := &DNSError{Err: "server misbehaving", Name: name, Server: server} if h.RCode == dnsmessage.RCodeServerFailure { - err.IsTemporary = true + return errServerTemporarlyMisbehaving } - return err + return errServerMisbehaving } return nil @@ -194,28 +207,16 @@ func skipToAnswer(p *dnsmessage.Parser, qtype dnsmessage.Type, name, server stri for { h, err := p.AnswerHeader() if err == dnsmessage.ErrSectionDone { - return &DNSError{ - Err: errNoSuchHost.Error(), - Name: name, - Server: server, - } + return errNoSuchHost } if err != nil { - return &DNSError{ - Err: "cannot unmarshal DNS message", - Name: name, - Server: server, - } + return errCannotUnmarshalDNSMessage } if h.Type == qtype { return nil } if err := p.SkipAnswer(); err != nil { - return &DNSError{ - Err: "cannot unmarshal DNS message", - Name: name, - Server: server, - } + return errCannotUnmarshalDNSMessage } } } @@ -229,7 +230,7 @@ func (r *Resolver) tryOneName(ctx context.Context, cfg *dnsConfig, name string, n, err := dnsmessage.NewName(name) if err != nil { - return dnsmessage.Parser{}, "", errors.New("cannot marshal DNS message") + return dnsmessage.Parser{}, "", errCannotMarshalDNSMessage } q := dnsmessage.Question{ Name: n, @@ -243,38 +244,62 @@ func (r *Resolver) tryOneName(ctx context.Context, cfg *dnsConfig, name string, p, h, err := r.exchange(ctx, server, q, cfg.timeout) if err != nil { - lastErr = &DNSError{ + dnsErr := &DNSError{ Err: err.Error(), Name: name, Server: server, } if nerr, ok := err.(Error); ok && nerr.Timeout() { - lastErr.(*DNSError).IsTimeout = true + dnsErr.IsTimeout = true } // Set IsTemporary for socket-level errors. Note that this flag // may also be used to indicate a SERVFAIL response. if _, ok := err.(*OpError); ok { - lastErr.(*DNSError).IsTemporary = true + dnsErr.IsTemporary = true } + lastErr = dnsErr continue } - // The name does not exist, so trying another server won't help. - // - // TODO: indicate this in a more obvious way, such as a field on DNSError? - if h.RCode == dnsmessage.RCodeNameError { - return dnsmessage.Parser{}, "", &DNSError{Err: errNoSuchHost.Error(), Name: name, Server: server} - } - - lastErr = checkHeader(&p, h, name, server) - if lastErr != nil { + if err := checkHeader(&p, h, name, server); err != nil { + dnsErr := &DNSError{ + Err: err.Error(), + Name: name, + Server: server, + } + if err == errServerTemporarlyMisbehaving { + dnsErr.IsTemporary = true + } + if err == errNoSuchHost { + // The name does not exist, so trying + // another server won't help. + // + // TODO: indicate this in a more + // obvious way, such as a field on + // DNSError? + return p, server, dnsErr + } + lastErr = dnsErr continue } - lastErr = skipToAnswer(&p, qtype, name, server) - if lastErr == nil { + err = skipToAnswer(&p, qtype, name, server) + if err == nil { return p, server, nil } + lastErr = &DNSError{ + Err: err.Error(), + Name: name, + Server: server, + } + if err == errNoSuchHost { + // The name does not exist, so trying another + // server won't help. + // + // TODO: indicate this in a more obvious way, + // such as a field on DNSError? + return p, server, lastErr + } } } return dnsmessage.Parser{}, "", lastErr diff --git a/src/net/dnsclient_unix_test.go b/src/net/dnsclient_unix_test.go index bb014b903ab93..9e4ebcc7bbef8 100644 --- a/src/net/dnsclient_unix_test.go +++ b/src/net/dnsclient_unix_test.go @@ -1427,28 +1427,35 @@ func TestDNSGoroutineRace(t *testing.T) { } } +func lookupWithFake(fake fakeDNSServer, name string, typ dnsmessage.Type) error { + r := Resolver{PreferGo: true, Dial: fake.DialContext} + + resolvConf.mu.RLock() + conf := resolvConf.dnsConfig + resolvConf.mu.RUnlock() + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + _, _, err := r.tryOneName(ctx, conf, name, typ) + return err +} + // Issue 8434: verify that Temporary returns true on an error when rcode // is SERVFAIL func TestIssue8434(t *testing.T) { - msg := dnsmessage.Message{ - Header: dnsmessage.Header{ - RCode: dnsmessage.RCodeServerFailure, + err := lookupWithFake(fakeDNSServer{ + rh: func(n, _ string, q dnsmessage.Message, _ time.Time) (dnsmessage.Message, error) { + return dnsmessage.Message{ + Header: dnsmessage.Header{ + ID: q.ID, + Response: true, + RCode: dnsmessage.RCodeServerFailure, + }, + Questions: q.Questions, + }, nil }, - } - b, err := msg.Pack() - if err != nil { - t.Fatal("Pack failed:", err) - } - var p dnsmessage.Parser - h, err := p.Start(b) - if err != nil { - t.Fatal("Start failed:", err) - } - if err := p.SkipAllQuestions(); err != nil { - t.Fatal("SkipAllQuestions failed:", err) - } - - err = checkHeader(&p, h, "golang.org", "foo:53") + }, "golang.org.", dnsmessage.TypeALL) if err == nil { t.Fatal("expected an error") } @@ -1464,50 +1471,76 @@ func TestIssue8434(t *testing.T) { } } -// Issue 12778: verify that NXDOMAIN without RA bit errors as -// "no such host" and not "server misbehaving" +// TestNoSuchHost verifies that tryOneName works correctly when the domain does +// not exist. +// +// Issue 12778: verify that NXDOMAIN without RA bit errors as "no such host" +// and not "server misbehaving" // // Issue 25336: verify that NXDOMAIN errors fail fast. -func TestIssue12778(t *testing.T) { - lookups := 0 - fake := fakeDNSServer{ - rh: func(n, _ string, q dnsmessage.Message, _ time.Time) (dnsmessage.Message, error) { - lookups++ - return dnsmessage.Message{ - Header: dnsmessage.Header{ - ID: q.ID, - Response: true, - RCode: dnsmessage.RCodeNameError, - RecursionAvailable: false, - }, - Questions: q.Questions, - }, nil +// +// Issue 27525: verify that empty answers fail fast. +func TestNoSuchHost(t *testing.T) { + tests := []struct { + name string + f func(string, string, dnsmessage.Message, time.Time) (dnsmessage.Message, error) + }{ + { + "NXDOMAIN", + func(n, _ string, q dnsmessage.Message, _ time.Time) (dnsmessage.Message, error) { + return dnsmessage.Message{ + Header: dnsmessage.Header{ + ID: q.ID, + Response: true, + RCode: dnsmessage.RCodeNameError, + RecursionAvailable: false, + }, + Questions: q.Questions, + }, nil + }, + }, + { + "no answers", + func(n, _ string, q dnsmessage.Message, _ time.Time) (dnsmessage.Message, error) { + return dnsmessage.Message{ + Header: dnsmessage.Header{ + ID: q.ID, + Response: true, + RCode: dnsmessage.RCodeSuccess, + RecursionAvailable: false, + Authoritative: true, + }, + Questions: q.Questions, + }, nil + }, }, } - r := Resolver{PreferGo: true, Dial: fake.DialContext} - - resolvConf.mu.RLock() - conf := resolvConf.dnsConfig - resolvConf.mu.RUnlock() - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - _, _, err := r.tryOneName(ctx, conf, ".", dnsmessage.TypeALL) + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + lookups := 0 + err := lookupWithFake(fakeDNSServer{ + rh: func(n, s string, q dnsmessage.Message, d time.Time) (dnsmessage.Message, error) { + lookups++ + return test.f(n, s, q, d) + }, + }, ".", dnsmessage.TypeALL) - if lookups != 1 { - t.Errorf("got %d lookups, wanted 1", lookups) - } + if lookups != 1 { + t.Errorf("got %d lookups, wanted 1", lookups) + } - if err == nil { - t.Fatal("expected an error") - } - de, ok := err.(*DNSError) - if !ok { - t.Fatalf("err = %#v; wanted a *net.DNSError", err) - } - if de.Err != errNoSuchHost.Error() { - t.Fatalf("Err = %#v; wanted %q", de.Err, errNoSuchHost.Error()) + if err == nil { + t.Fatal("expected an error") + } + de, ok := err.(*DNSError) + if !ok { + t.Fatalf("err = %#v; wanted a *net.DNSError", err) + } + if de.Err != errNoSuchHost.Error() { + t.Fatalf("Err = %#v; wanted %q", de.Err, errNoSuchHost.Error()) + } + }) } } From 7544ac632ff5a3272c09ae81042c18e5df60f434 Mon Sep 17 00:00:00 2001 From: Ian Lance Taylor Date: Sun, 26 Aug 2018 20:38:43 -0700 Subject: [PATCH 05/13] [release-branch.go1.11] cmd/go: add GOMIPS value to build id for mipsle Strip a trailing "le" from the GOARCH value when calculating the GOxxx environment variable that affects it. Updates #27260 Fixes #27420 Change-Id: I081f30d5dc19281901551823f4f56be028b5f71a Reviewed-on: https://go-review.googlesource.com/131379 Reviewed-by: Brad Fitzpatrick (cherry picked from commit 61318d7ffe8a49e9dedc5aa8195a164a3821465c) Reviewed-on: https://go-review.googlesource.com/138176 Run-TryBot: Brad Fitzpatrick TryBot-Result: Gobot Gobot Reviewed-by: Ian Lance Taylor --- src/cmd/go/internal/work/exec.go | 4 +- .../go/testdata/script/build_cache_gomips.txt | 37 +++++++++++++++++++ 2 files changed, 40 insertions(+), 1 deletion(-) create mode 100644 src/cmd/go/testdata/script/build_cache_gomips.txt diff --git a/src/cmd/go/internal/work/exec.go b/src/cmd/go/internal/work/exec.go index 42fa0e64ac007..12e15276f4cc3 100644 --- a/src/cmd/go/internal/work/exec.go +++ b/src/cmd/go/internal/work/exec.go @@ -224,7 +224,9 @@ func (b *Builder) buildActionID(a *Action) cache.ActionID { if len(p.SFiles) > 0 { fmt.Fprintf(h, "asm %q %q %q\n", b.toolID("asm"), forcedAsmflags, p.Internal.Asmflags) } - fmt.Fprintf(h, "GO$GOARCH=%s\n", os.Getenv("GO"+strings.ToUpper(cfg.BuildContext.GOARCH))) // GO386, GOARM, etc + // GO386, GOARM, GOMIPS, etc. + baseArch := strings.TrimSuffix(cfg.BuildContext.GOARCH, "le") + fmt.Fprintf(h, "GO$GOARCH=%s\n", os.Getenv("GO"+strings.ToUpper(baseArch))) // TODO(rsc): Convince compiler team not to add more magic environment variables, // or perhaps restrict the environment variables passed to subprocesses. diff --git a/src/cmd/go/testdata/script/build_cache_gomips.txt b/src/cmd/go/testdata/script/build_cache_gomips.txt new file mode 100644 index 0000000000000..c77acc3f2f32d --- /dev/null +++ b/src/cmd/go/testdata/script/build_cache_gomips.txt @@ -0,0 +1,37 @@ +# Set up fresh GOCACHE. +env GOCACHE=$WORK/gocache +mkdir $GOCACHE + +# Building for mipsle without setting GOMIPS will use floating point registers. +env GOARCH=mipsle +env GOOS=linux +go build -gcflags=-S f.go +stderr ADDD.F[0-9]+,.F[0-9]+,.F[0-9]+ + +# Clean cache +go clean -cache + +# Building with GOMIPS=softfloat will not use floating point registers +env GOMIPS=softfloat +go build -gcflags=-S f.go +! stderr ADDD.F[0-9]+,.F[0-9]+,.F[0-9]+ + +# Clean cache +go clean -cache + +# Build without setting GOMIPS +env GOMIPS= +go build -gcflags=-S f.go +stderr ADDD.F[0-9]+,.F[0-9]+,.F[0-9]+ + +# Building with GOMIPS should still not use floating point registers. +env GOMIPS=softfloat +go build -gcflags=-S f.go +! stderr ADDD.F[0-9]+,.F[0-9]+,.F[0-9]+ + +-- f.go -- +package f + +func F(x float64) float64 { + return x + x +} From 34e5a852e0c7cf721fda53cd007802e8d1b7bb8c Mon Sep 17 00:00:00 2001 From: Chris Broadfoot Date: Fri, 28 Sep 2018 02:02:22 -0700 Subject: [PATCH 06/13] [release-branch.go1.11] doc: add go1.11 to contrib.html Missing from https://golang.org/project Change-Id: I6cb769ae861a81f0264bae624b5fe8d70aa92497 Reviewed-on: https://go-review.googlesource.com/138356 Reviewed-by: Chris Broadfoot --- doc/contrib.html | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/contrib.html b/doc/contrib.html index df53d480d3856..c7c0e0377efc5 100644 --- a/doc/contrib.html +++ b/doc/contrib.html @@ -34,6 +34,7 @@

Release History

A summary of the changes between Go releases. Notes for the major releases:

    +
  • Go 1.11 (August 2018)
  • Go 1.10 (February 2018)
  • Go 1.9 (August 2017)
  • Go 1.8 (February 2017)
  • From a2f1c8e2ad723589eed8f01056e29f6156869e53 Mon Sep 17 00:00:00 2001 From: Keith Randall Date: Tue, 25 Sep 2018 14:32:44 -0700 Subject: [PATCH 07/13] [release-branch.go1.11] reflect: use correct write barrier operations for method funcs Fix the code to use write barriers on heap memory, and no write barriers on stack memory. These errors were discovered as part of fixing #27695. They may have something to do with that issue, but hard to be sure. The core cause is different, so this fix is a separate CL. Update #27867 Change-Id: Ib005f6b3308de340be83c3d07d049d5e316b1e3c Reviewed-on: https://go-review.googlesource.com/137438 Reviewed-by: Austin Clements (cherry picked from commit e35a41261b19589f40d32bd66274c23ab4b9b32e) Reviewed-on: https://go-review.googlesource.com/138581 Run-TryBot: Ian Lance Taylor TryBot-Result: Gobot Gobot Reviewed-by: Ian Lance Taylor --- src/reflect/type.go | 2 ++ src/reflect/value.go | 39 ++++++++++++++++++++++++++------------- src/runtime/mbarrier.go | 13 +++++++++++++ 3 files changed, 41 insertions(+), 13 deletions(-) diff --git a/src/reflect/type.go b/src/reflect/type.go index 58cfc0e884035..6b0ce431a6771 100644 --- a/src/reflect/type.go +++ b/src/reflect/type.go @@ -3066,6 +3066,8 @@ func funcLayout(t *rtype, rcvr *rtype) (frametype *rtype, argSize, retOffset uin // space no matter how big they actually are. if ifaceIndir(rcvr) || rcvr.pointers() { ptrmap.append(1) + } else { + ptrmap.append(0) } offset += ptrSize } diff --git a/src/reflect/value.go b/src/reflect/value.go index 4e7b1d74db3dc..6a49adef593c5 100644 --- a/src/reflect/value.go +++ b/src/reflect/value.go @@ -453,15 +453,14 @@ func (v Value) call(op string, in []Value) []Value { var ret []Value if nout == 0 { - // This is untyped because the frame is really a - // stack, even though it's a heap object. - memclrNoHeapPointers(args, frametype.size) + typedmemclr(frametype, args) framePool.Put(args) } else { // Zero the now unused input area of args, // because the Values returned by this function contain pointers to the args object, // and will thus keep the args object alive indefinitely. - memclrNoHeapPointers(args, retOffset) + typedmemclrpartial(frametype, args, 0, retOffset) + // Wrap Values around return values in args. ret = make([]Value, nout) off = retOffset @@ -472,6 +471,10 @@ func (v Value) call(op string, in []Value) []Value { if tv.Size() != 0 { fl := flagIndir | flag(tv.Kind()) ret[i] = Value{tv.common(), add(args, off, "tv.Size() != 0"), fl} + // Note: this does introduce false sharing between results - + // if any result is live, they are all live. + // (And the space for the args is live as well, but as we've + // cleared that space it isn't as big a deal.) } else { // For zero-sized return value, args+off may point to the next object. // In this case, return the zero value instead. @@ -660,6 +663,8 @@ func callMethod(ctxt *methodValue, frame unsafe.Pointer) { } // Call. + // Call copies the arguments from args to the stack, calls fn, + // and then copies the results back into args. call(frametype, fn, args, uint32(frametype.size), uint32(retOffset)) // Copy return values. On amd64p32, the beginning of return values @@ -673,16 +678,14 @@ func callMethod(ctxt *methodValue, frame unsafe.Pointer) { if runtime.GOARCH == "amd64p32" { callerRetOffset = align(argSize-ptrSize, 8) } - typedmemmovepartial(frametype, - add(frame, callerRetOffset, "frametype.size > retOffset"), + // This copies to the stack. Write barriers are not needed. + memmove(add(frame, callerRetOffset, "frametype.size > retOffset"), add(args, retOffset, "frametype.size > retOffset"), - retOffset, frametype.size-retOffset) } - // This is untyped because the frame is really a stack, even - // though it's a heap object. - memclrNoHeapPointers(args, frametype.size) + // Put the args scratch space back in the pool. + typedmemclr(frametype, args) framePool.Put(args) // See the comment in callReflect. @@ -2569,6 +2572,10 @@ func call(argtype *rtype, fn, arg unsafe.Pointer, n uint32, retoffset uint32) func ifaceE2I(t *rtype, src interface{}, dst unsafe.Pointer) +// memmove copies size bytes to dst from src. No write barriers are used. +//go:noescape +func memmove(dst, src unsafe.Pointer, size uintptr) + // typedmemmove copies a value of type t to dst from src. //go:noescape func typedmemmove(t *rtype, dst, src unsafe.Pointer) @@ -2578,14 +2585,20 @@ func typedmemmove(t *rtype, dst, src unsafe.Pointer) //go:noescape func typedmemmovepartial(t *rtype, dst, src unsafe.Pointer, off, size uintptr) +// typedmemclr zeros the value at ptr of type t. +//go:noescape +func typedmemclr(t *rtype, ptr unsafe.Pointer) + +// typedmemclrpartial is like typedmemclr but assumes that +// dst points off bytes into the value and only clears size bytes. +//go:noescape +func typedmemclrpartial(t *rtype, ptr unsafe.Pointer, off, size uintptr) + // typedslicecopy copies a slice of elemType values from src to dst, // returning the number of elements copied. //go:noescape func typedslicecopy(elemType *rtype, dst, src sliceHeader) int -//go:noescape -func memclrNoHeapPointers(ptr unsafe.Pointer, n uintptr) - // Dummy annotation marking that the value x escapes, // for use in cases where the reflect code is so clever that // the compiler cannot follow. diff --git a/src/runtime/mbarrier.go b/src/runtime/mbarrier.go index b6c5ee0658061..ed47c6f0ef0b5 100644 --- a/src/runtime/mbarrier.go +++ b/src/runtime/mbarrier.go @@ -320,6 +320,19 @@ func typedmemclr(typ *_type, ptr unsafe.Pointer) { memclrNoHeapPointers(ptr, typ.size) } +//go:linkname reflect_typedmemclr reflect.typedmemclr +func reflect_typedmemclr(typ *_type, ptr unsafe.Pointer) { + typedmemclr(typ, ptr) +} + +//go:linkname reflect_typedmemclrpartial reflect.typedmemclrpartial +func reflect_typedmemclrpartial(typ *_type, ptr unsafe.Pointer, off, size uintptr) { + if typ.kind&kindNoPointers == 0 { + bulkBarrierPreWrite(uintptr(ptr), 0, size) + } + memclrNoHeapPointers(ptr, size) +} + // memclrHasPointers clears n bytes of typed memory starting at ptr. // The caller must ensure that the type of the object at ptr has // pointers, usually by checking typ.kind&kindNoPointers. However, ptr From 3afa9dfa9b00c17c8ad0e42b220ec37329631f2c Mon Sep 17 00:00:00 2001 From: Keith Randall Date: Tue, 25 Sep 2018 15:54:11 -0700 Subject: [PATCH 08/13] [release-branch.go1.11] reflect: ensure correct scanning of return values During a call to a reflect-generated function or method (via makeFuncStub or methodValueCall), when should we scan the return values? When we're starting a reflect call, the space on the stack for the return values is not initialized yet, as it contains whatever junk was on the stack of the caller at the time. The return space must not be scanned during a GC. When we're finishing a reflect call, the return values are initialized, and must be scanned during a GC to make sure that any pointers in the return values are found and their referents retained. When the GC stack walk comes across a reflect call in progress on the stack, it needs to know whether to scan the results or not. It doesn't know the progress of the reflect call, so it can't decide by itself. The reflect package needs to tell it. This CL adds another slot in the frame of makeFuncStub and methodValueCall so we can put a boolean in there which tells the runtime whether to scan the results or not. This CL also adds the args length to reflectMethodValue so the runtime can restrict its scanning to only the args section (not the results) if the reflect package says the results aren't ready yet. Do a delicate dance in the reflect package to set the "results are valid" bit. We need to make sure we set the bit only after we've copied the results back to the stack. But we must set the bit before we drop reflect's copy of the results. Otherwise, we might have a state where (temporarily) no one has a live copy of the results. That's the state we were observing in issue #27695 before this CL. The bitmap used by the runtime currently contains only the args. (Actually, it contains all the bits, but the size is set so we use only the args portion.) This is safe for early in a reflect call, but unsafe late in a reflect call. The test issue27695.go demonstrates this unsafety. We change the bitmap to always include both args and results, and decide at runtime which portion to use. issue27695.go only has a test for method calls. Function calls were ok because there wasn't a safepoint between when reflect dropped its copy of the return values and when the caller is resumed. This may change when we introduce safepoints everywhere. This truncate-to-only-the-args was part of CL 9888 (in 2015). That part of the CL fixed the problem demonstrated in issue27695b.go but introduced the problem demonstrated in issue27695.go. TODO, in another CL: simplify FuncLayout and its test. stack return value is now identical to frametype.ptrdata + frametype.gcdata. Update #27867 Change-Id: I2d49b34e34a82c6328b34f02610587a291b25c5f Reviewed-on: https://go-review.googlesource.com/137440 Run-TryBot: Keith Randall TryBot-Result: Gobot Gobot Reviewed-by: Austin Clements Reviewed-on: https://go-review.googlesource.com/138582 Run-TryBot: Ian Lance Taylor Reviewed-by: Ian Lance Taylor --- src/cmd/compile/internal/types/type.go | 2 +- src/reflect/all_test.go | 4 +- src/reflect/asm_386.s | 10 +++- src/reflect/asm_amd64.s | 10 +++- src/reflect/asm_amd64p32.s | 10 +++- src/reflect/asm_arm.s | 12 ++++- src/reflect/asm_arm64.s | 10 +++- src/reflect/asm_mips64x.s | 10 +++- src/reflect/asm_mipsx.s | 10 +++- src/reflect/asm_ppc64x.s | 10 +++- src/reflect/asm_s390x.s | 10 +++- src/reflect/asm_wasm.s | 10 +++- src/reflect/export_test.go | 4 +- src/reflect/makefunc.go | 27 ++++++----- src/reflect/type.go | 14 ++---- src/reflect/value.go | 61 +++++++++++++++++------- src/runtime/traceback.go | 13 +++++- test/fixedbugs/issue27695.go | 62 +++++++++++++++++++++++++ test/fixedbugs/issue27695b.go | 64 ++++++++++++++++++++++++++ 19 files changed, 288 insertions(+), 65 deletions(-) create mode 100644 test/fixedbugs/issue27695.go create mode 100644 test/fixedbugs/issue27695b.go diff --git a/src/cmd/compile/internal/types/type.go b/src/cmd/compile/internal/types/type.go index d367cd19440e6..25f8f826e607c 100644 --- a/src/cmd/compile/internal/types/type.go +++ b/src/cmd/compile/internal/types/type.go @@ -817,7 +817,7 @@ func (t *Type) ChanArgs() *Type { return t.Extra.(ChanArgs).T } -// FuncArgs returns the channel type for TFUNCARGS type t. +// FuncArgs returns the func type for TFUNCARGS type t. func (t *Type) FuncArgs() *Type { t.wantEtype(TFUNCARGS) return t.Extra.(FuncArgs).T diff --git a/src/reflect/all_test.go b/src/reflect/all_test.go index cf7fe3cf7ae72..bce9053f84149 100644 --- a/src/reflect/all_test.go +++ b/src/reflect/all_test.go @@ -5844,7 +5844,7 @@ func clobber() { type funcLayoutTest struct { rcvr, t Type size, argsize, retOffset uintptr - stack []byte // pointer bitmap: 1 is pointer, 0 is scalar (or uninitialized) + stack []byte // pointer bitmap: 1 is pointer, 0 is scalar gc []byte } @@ -5866,7 +5866,7 @@ func init() { 6 * PtrSize, 4 * PtrSize, 4 * PtrSize, - []byte{1, 0, 1}, + []byte{1, 0, 1, 0, 1}, []byte{1, 0, 1, 0, 1}, }) diff --git a/src/reflect/asm_386.s b/src/reflect/asm_386.s index d827360006c97..e79beb6dc99f0 100644 --- a/src/reflect/asm_386.s +++ b/src/reflect/asm_386.s @@ -9,11 +9,14 @@ // See the comment on the declaration of makeFuncStub in makefunc.go // for more details. // No argsize here, gc generates argsize info at call site. -TEXT ·makeFuncStub(SB),(NOSPLIT|WRAPPER),$8 +TEXT ·makeFuncStub(SB),(NOSPLIT|WRAPPER),$16 NO_LOCAL_POINTERS MOVL DX, 0(SP) LEAL argframe+0(FP), CX MOVL CX, 4(SP) + MOVB $0, 12(SP) + LEAL 12(SP), AX + MOVL AX, 8(SP) CALL ·callReflect(SB) RET @@ -21,10 +24,13 @@ TEXT ·makeFuncStub(SB),(NOSPLIT|WRAPPER),$8 // See the comment on the declaration of methodValueCall in makefunc.go // for more details. // No argsize here, gc generates argsize info at call site. -TEXT ·methodValueCall(SB),(NOSPLIT|WRAPPER),$8 +TEXT ·methodValueCall(SB),(NOSPLIT|WRAPPER),$16 NO_LOCAL_POINTERS MOVL DX, 0(SP) LEAL argframe+0(FP), CX MOVL CX, 4(SP) + MOVB $0, 12(SP) + LEAL 12(SP), AX + MOVL AX, 8(SP) CALL ·callMethod(SB) RET diff --git a/src/reflect/asm_amd64.s b/src/reflect/asm_amd64.s index 1272c489de829..fb28ab87f113e 100644 --- a/src/reflect/asm_amd64.s +++ b/src/reflect/asm_amd64.s @@ -9,11 +9,14 @@ // See the comment on the declaration of makeFuncStub in makefunc.go // for more details. // No arg size here; runtime pulls arg map out of the func value. -TEXT ·makeFuncStub(SB),(NOSPLIT|WRAPPER),$16 +TEXT ·makeFuncStub(SB),(NOSPLIT|WRAPPER),$32 NO_LOCAL_POINTERS MOVQ DX, 0(SP) LEAQ argframe+0(FP), CX MOVQ CX, 8(SP) + MOVB $0, 24(SP) + LEAQ 24(SP), AX + MOVQ AX, 16(SP) CALL ·callReflect(SB) RET @@ -21,10 +24,13 @@ TEXT ·makeFuncStub(SB),(NOSPLIT|WRAPPER),$16 // See the comment on the declaration of methodValueCall in makefunc.go // for more details. // No arg size here; runtime pulls arg map out of the func value. -TEXT ·methodValueCall(SB),(NOSPLIT|WRAPPER),$16 +TEXT ·methodValueCall(SB),(NOSPLIT|WRAPPER),$32 NO_LOCAL_POINTERS MOVQ DX, 0(SP) LEAQ argframe+0(FP), CX MOVQ CX, 8(SP) + MOVB $0, 24(SP) + LEAQ 24(SP), AX + MOVQ AX, 16(SP) CALL ·callMethod(SB) RET diff --git a/src/reflect/asm_amd64p32.s b/src/reflect/asm_amd64p32.s index d827360006c97..e79beb6dc99f0 100644 --- a/src/reflect/asm_amd64p32.s +++ b/src/reflect/asm_amd64p32.s @@ -9,11 +9,14 @@ // See the comment on the declaration of makeFuncStub in makefunc.go // for more details. // No argsize here, gc generates argsize info at call site. -TEXT ·makeFuncStub(SB),(NOSPLIT|WRAPPER),$8 +TEXT ·makeFuncStub(SB),(NOSPLIT|WRAPPER),$16 NO_LOCAL_POINTERS MOVL DX, 0(SP) LEAL argframe+0(FP), CX MOVL CX, 4(SP) + MOVB $0, 12(SP) + LEAL 12(SP), AX + MOVL AX, 8(SP) CALL ·callReflect(SB) RET @@ -21,10 +24,13 @@ TEXT ·makeFuncStub(SB),(NOSPLIT|WRAPPER),$8 // See the comment on the declaration of methodValueCall in makefunc.go // for more details. // No argsize here, gc generates argsize info at call site. -TEXT ·methodValueCall(SB),(NOSPLIT|WRAPPER),$8 +TEXT ·methodValueCall(SB),(NOSPLIT|WRAPPER),$16 NO_LOCAL_POINTERS MOVL DX, 0(SP) LEAL argframe+0(FP), CX MOVL CX, 4(SP) + MOVB $0, 12(SP) + LEAL 12(SP), AX + MOVL AX, 8(SP) CALL ·callMethod(SB) RET diff --git a/src/reflect/asm_arm.s b/src/reflect/asm_arm.s index b721ed28c66f9..cd50d33918fc6 100644 --- a/src/reflect/asm_arm.s +++ b/src/reflect/asm_arm.s @@ -9,11 +9,15 @@ // See the comment on the declaration of makeFuncStub in makefunc.go // for more details. // No argsize here, gc generates argsize info at call site. -TEXT ·makeFuncStub(SB),(NOSPLIT|WRAPPER),$8 +TEXT ·makeFuncStub(SB),(NOSPLIT|WRAPPER),$16 NO_LOCAL_POINTERS MOVW R7, 4(R13) MOVW $argframe+0(FP), R1 MOVW R1, 8(R13) + MOVW $0, R1 + MOVB R1, 16(R13) + ADD $16, R13, R1 + MOVW R1, 12(R13) BL ·callReflect(SB) RET @@ -21,10 +25,14 @@ TEXT ·makeFuncStub(SB),(NOSPLIT|WRAPPER),$8 // See the comment on the declaration of methodValueCall in makefunc.go // for more details. // No argsize here, gc generates argsize info at call site. -TEXT ·methodValueCall(SB),(NOSPLIT|WRAPPER),$8 +TEXT ·methodValueCall(SB),(NOSPLIT|WRAPPER),$16 NO_LOCAL_POINTERS MOVW R7, 4(R13) MOVW $argframe+0(FP), R1 MOVW R1, 8(R13) + MOVW $0, R1 + MOVB R1, 16(R13) + ADD $16, R13, R1 + MOVW R1, 12(R13) BL ·callMethod(SB) RET diff --git a/src/reflect/asm_arm64.s b/src/reflect/asm_arm64.s index d1563709f2f5a..28bb86c2a47eb 100644 --- a/src/reflect/asm_arm64.s +++ b/src/reflect/asm_arm64.s @@ -9,11 +9,14 @@ // See the comment on the declaration of makeFuncStub in makefunc.go // for more details. // No arg size here, runtime pulls arg map out of the func value. -TEXT ·makeFuncStub(SB),(NOSPLIT|WRAPPER),$24 +TEXT ·makeFuncStub(SB),(NOSPLIT|WRAPPER),$40 NO_LOCAL_POINTERS MOVD R26, 8(RSP) MOVD $argframe+0(FP), R3 MOVD R3, 16(RSP) + MOVB $0, 32(RSP) + ADD $32, RSP, R3 + MOVD R3, 24(RSP) BL ·callReflect(SB) RET @@ -21,10 +24,13 @@ TEXT ·makeFuncStub(SB),(NOSPLIT|WRAPPER),$24 // See the comment on the declaration of methodValueCall in makefunc.go // for more details. // No arg size here; runtime pulls arg map out of the func value. -TEXT ·methodValueCall(SB),(NOSPLIT|WRAPPER),$24 +TEXT ·methodValueCall(SB),(NOSPLIT|WRAPPER),$40 NO_LOCAL_POINTERS MOVD R26, 8(RSP) MOVD $argframe+0(FP), R3 MOVD R3, 16(RSP) + MOVB $0, 32(RSP) + ADD $32, RSP, R3 + MOVD R3, 24(RSP) BL ·callMethod(SB) RET diff --git a/src/reflect/asm_mips64x.s b/src/reflect/asm_mips64x.s index 98afb52f6a11b..6f76685567a7a 100644 --- a/src/reflect/asm_mips64x.s +++ b/src/reflect/asm_mips64x.s @@ -13,11 +13,14 @@ // See the comment on the declaration of makeFuncStub in makefunc.go // for more details. // No arg size here, runtime pulls arg map out of the func value. -TEXT ·makeFuncStub(SB),(NOSPLIT|WRAPPER),$16 +TEXT ·makeFuncStub(SB),(NOSPLIT|WRAPPER),$32 NO_LOCAL_POINTERS MOVV REGCTXT, 8(R29) MOVV $argframe+0(FP), R1 MOVV R1, 16(R29) + MOVB R0, 32(R29) + ADDV $32, R29, R1 + MOVV R1, 24(R29) JAL ·callReflect(SB) RET @@ -25,10 +28,13 @@ TEXT ·makeFuncStub(SB),(NOSPLIT|WRAPPER),$16 // See the comment on the declaration of methodValueCall in makefunc.go // for more details. // No arg size here; runtime pulls arg map out of the func value. -TEXT ·methodValueCall(SB),(NOSPLIT|WRAPPER),$16 +TEXT ·methodValueCall(SB),(NOSPLIT|WRAPPER),$32 NO_LOCAL_POINTERS MOVV REGCTXT, 8(R29) MOVV $argframe+0(FP), R1 MOVV R1, 16(R29) + MOVB R0, 32(R29) + ADDV $32, R29, R1 + MOVV R1, 24(R29) JAL ·callMethod(SB) RET diff --git a/src/reflect/asm_mipsx.s b/src/reflect/asm_mipsx.s index b6df4e636e81f..5a5c53ef9f9f1 100644 --- a/src/reflect/asm_mipsx.s +++ b/src/reflect/asm_mipsx.s @@ -13,11 +13,14 @@ // See the comment on the declaration of makeFuncStub in makefunc.go // for more details. // No arg size here, runtime pulls arg map out of the func value. -TEXT ·makeFuncStub(SB),(NOSPLIT|WRAPPER),$8 +TEXT ·makeFuncStub(SB),(NOSPLIT|WRAPPER),$16 NO_LOCAL_POINTERS MOVW REGCTXT, 4(R29) MOVW $argframe+0(FP), R1 MOVW R1, 8(R29) + MOVB R0, 16(R29) + ADD $16, R29, R1 + MOVW R1, 12(R29) JAL ·callReflect(SB) RET @@ -25,10 +28,13 @@ TEXT ·makeFuncStub(SB),(NOSPLIT|WRAPPER),$8 // See the comment on the declaration of methodValueCall in makefunc.go // for more details. // No arg size here; runtime pulls arg map out of the func value. -TEXT ·methodValueCall(SB),(NOSPLIT|WRAPPER),$8 +TEXT ·methodValueCall(SB),(NOSPLIT|WRAPPER),$16 NO_LOCAL_POINTERS MOVW REGCTXT, 4(R29) MOVW $argframe+0(FP), R1 MOVW R1, 8(R29) + MOVB R0, 16(R29) + ADD $16, R29, R1 + MOVW R1, 12(R29) JAL ·callMethod(SB) RET diff --git a/src/reflect/asm_ppc64x.s b/src/reflect/asm_ppc64x.s index 42f57743e6f21..4609f6bb75223 100644 --- a/src/reflect/asm_ppc64x.s +++ b/src/reflect/asm_ppc64x.s @@ -12,11 +12,14 @@ // See the comment on the declaration of makeFuncStub in makefunc.go // for more details. // No arg size here, runtime pulls arg map out of the func value. -TEXT ·makeFuncStub(SB),(NOSPLIT|WRAPPER),$16 +TEXT ·makeFuncStub(SB),(NOSPLIT|WRAPPER),$32 NO_LOCAL_POINTERS MOVD R11, FIXED_FRAME+0(R1) MOVD $argframe+0(FP), R3 MOVD R3, FIXED_FRAME+8(R1) + MOVB R0, FIXED_FRAME+24(R1) + ADD $FIXED_FRAME+24, R1, R3 + MOVD R3, FIXED_FRAME+16(R1) BL ·callReflect(SB) RET @@ -24,10 +27,13 @@ TEXT ·makeFuncStub(SB),(NOSPLIT|WRAPPER),$16 // See the comment on the declaration of methodValueCall in makefunc.go // for more details. // No arg size here; runtime pulls arg map out of the func value. -TEXT ·methodValueCall(SB),(NOSPLIT|WRAPPER),$16 +TEXT ·methodValueCall(SB),(NOSPLIT|WRAPPER),$32 NO_LOCAL_POINTERS MOVD R11, FIXED_FRAME+0(R1) MOVD $argframe+0(FP), R3 MOVD R3, FIXED_FRAME+8(R1) + MOVB R0, FIXED_FRAME+24(R1) + ADD $FIXED_FRAME+24, R1, R3 + MOVD R3, FIXED_FRAME+16(R1) BL ·callMethod(SB) RET diff --git a/src/reflect/asm_s390x.s b/src/reflect/asm_s390x.s index e6b86cfaa9d11..2ab5481c9bf3b 100644 --- a/src/reflect/asm_s390x.s +++ b/src/reflect/asm_s390x.s @@ -9,11 +9,14 @@ // See the comment on the declaration of makeFuncStub in makefunc.go // for more details. // No arg size here, runtime pulls arg map out of the func value. -TEXT ·makeFuncStub(SB),(NOSPLIT|WRAPPER),$16 +TEXT ·makeFuncStub(SB),(NOSPLIT|WRAPPER),$32 NO_LOCAL_POINTERS MOVD R12, 8(R15) MOVD $argframe+0(FP), R3 MOVD R3, 16(R15) + MOVB R0, 32(R15) + ADD $32, R15, R3 + MOVD R3, 24(R15) BL ·callReflect(SB) RET @@ -21,10 +24,13 @@ TEXT ·makeFuncStub(SB),(NOSPLIT|WRAPPER),$16 // See the comment on the declaration of methodValueCall in makefunc.go // for more details. // No arg size here; runtime pulls arg map out of the func value. -TEXT ·methodValueCall(SB),(NOSPLIT|WRAPPER),$16 +TEXT ·methodValueCall(SB),(NOSPLIT|WRAPPER),$32 NO_LOCAL_POINTERS MOVD R12, 8(R15) MOVD $argframe+0(FP), R3 MOVD R3, 16(R15) + MOVB R0, 32(R15) + ADD $32, R15, R3 + MOVD R3, 24(R15) BL ·callMethod(SB) RET diff --git a/src/reflect/asm_wasm.s b/src/reflect/asm_wasm.s index 0f9b5aa130fe0..627e295769cb3 100644 --- a/src/reflect/asm_wasm.s +++ b/src/reflect/asm_wasm.s @@ -9,7 +9,7 @@ // See the comment on the declaration of makeFuncStub in makefunc.go // for more details. // No arg size here; runtime pulls arg map out of the func value. -TEXT ·makeFuncStub(SB),(NOSPLIT|WRAPPER),$16 +TEXT ·makeFuncStub(SB),(NOSPLIT|WRAPPER),$32 NO_LOCAL_POINTERS MOVD CTXT, 0(SP) @@ -21,6 +21,9 @@ TEXT ·makeFuncStub(SB),(NOSPLIT|WRAPPER),$16 I64Add I64Store $8 + MOVB $0, 24(SP) + MOVD $24(SP), 16(SP) + CALL ·callReflect(SB) RET @@ -28,7 +31,7 @@ TEXT ·makeFuncStub(SB),(NOSPLIT|WRAPPER),$16 // See the comment on the declaration of methodValueCall in makefunc.go // for more details. // No arg size here; runtime pulls arg map out of the func value. -TEXT ·methodValueCall(SB),(NOSPLIT|WRAPPER),$16 +TEXT ·methodValueCall(SB),(NOSPLIT|WRAPPER),$32 NO_LOCAL_POINTERS MOVD CTXT, 0(SP) @@ -40,5 +43,8 @@ TEXT ·methodValueCall(SB),(NOSPLIT|WRAPPER),$16 I64Add I64Store $8 + MOVB $0, 24(SP) + MOVD $24(SP), 16(SP) + CALL ·callMethod(SB) RET diff --git a/src/reflect/export_test.go b/src/reflect/export_test.go index 14a6981fdee1b..3c47d6712f172 100644 --- a/src/reflect/export_test.go +++ b/src/reflect/export_test.go @@ -25,9 +25,9 @@ func FuncLayout(t Type, rcvr Type) (frametype Type, argSize, retOffset uintptr, var ft *rtype var s *bitVector if rcvr != nil { - ft, argSize, retOffset, s, _ = funcLayout(t.(*rtype), rcvr.(*rtype)) + ft, argSize, retOffset, s, _ = funcLayout((*funcType)(unsafe.Pointer(t.(*rtype))), rcvr.(*rtype)) } else { - ft, argSize, retOffset, s, _ = funcLayout(t.(*rtype), nil) + ft, argSize, retOffset, s, _ = funcLayout((*funcType)(unsafe.Pointer(t.(*rtype))), nil) } frametype = ft for i := uint32(0); i < s.n; i++ { diff --git a/src/reflect/makefunc.go b/src/reflect/makefunc.go index 885966db6fe92..67dc4859b9736 100644 --- a/src/reflect/makefunc.go +++ b/src/reflect/makefunc.go @@ -12,14 +12,15 @@ import ( // makeFuncImpl is the closure value implementing the function // returned by MakeFunc. -// The first two words of this type must be kept in sync with +// The first three words of this type must be kept in sync with // methodValue and runtime.reflectMethodValue. // Any changes should be reflected in all three. type makeFuncImpl struct { - code uintptr - stack *bitVector - typ *funcType - fn func([]Value) []Value + code uintptr + stack *bitVector // ptrmap for both args and results + argLen uintptr // just args + ftyp *funcType + fn func([]Value) []Value } // MakeFunc returns a new function of the given Type @@ -59,9 +60,9 @@ func MakeFunc(typ Type, fn func(args []Value) (results []Value)) Value { code := **(**uintptr)(unsafe.Pointer(&dummy)) // makeFuncImpl contains a stack map for use by the runtime - _, _, _, stack, _ := funcLayout(t, nil) + _, argLen, _, stack, _ := funcLayout(ftyp, nil) - impl := &makeFuncImpl{code: code, stack: stack, typ: ftyp, fn: fn} + impl := &makeFuncImpl{code: code, stack: stack, argLen: argLen, ftyp: ftyp, fn: fn} return Value{t, unsafe.Pointer(impl), flag(Func)} } @@ -73,12 +74,13 @@ func MakeFunc(typ Type, fn func(args []Value) (results []Value)) Value { // word in the passed-in argument frame. func makeFuncStub() -// The first two words of this type must be kept in sync with +// The first 3 words of this type must be kept in sync with // makeFuncImpl and runtime.reflectMethodValue. // Any changes should be reflected in all three. type methodValue struct { fn uintptr - stack *bitVector + stack *bitVector // ptrmap for both args and results + argLen uintptr // just args method int rcvr Value } @@ -101,7 +103,7 @@ func makeMethodValue(op string, v Value) Value { rcvr := Value{v.typ, v.ptr, fl} // v.Type returns the actual type of the method value. - funcType := v.Type().(*rtype) + ftyp := (*funcType)(unsafe.Pointer(v.Type().(*rtype))) // Indirect Go func value (dummy) to obtain // actual code address. (A Go func value is a pointer @@ -110,11 +112,12 @@ func makeMethodValue(op string, v Value) Value { code := **(**uintptr)(unsafe.Pointer(&dummy)) // methodValue contains a stack map for use by the runtime - _, _, _, stack, _ := funcLayout(funcType, nil) + _, argLen, _, stack, _ := funcLayout(ftyp, nil) fv := &methodValue{ fn: code, stack: stack, + argLen: argLen, method: int(v.flag) >> flagMethodShift, rcvr: rcvr, } @@ -124,7 +127,7 @@ func makeMethodValue(op string, v Value) Value { // but we want Interface() and other operations to fail early. methodReceiver(op, fv.rcvr, fv.method) - return Value{funcType, unsafe.Pointer(fv), v.flag&flagRO | flag(Func)} + return Value{&ftyp.rtype, unsafe.Pointer(fv), v.flag&flagRO | flag(Func)} } // methodValueCall is an assembly function that is the code half of diff --git a/src/reflect/type.go b/src/reflect/type.go index 6b0ce431a6771..d8971d620ef59 100644 --- a/src/reflect/type.go +++ b/src/reflect/type.go @@ -3022,8 +3022,8 @@ func toType(t *rtype) Type { } type layoutKey struct { - t *rtype // function signature - rcvr *rtype // receiver type, or nil if none + ftyp *funcType // function signature + rcvr *rtype // receiver type, or nil if none } type layoutType struct { @@ -3042,7 +3042,7 @@ var layoutCache sync.Map // map[layoutKey]layoutType // The returned type exists only for GC, so we only fill out GC relevant info. // Currently, that's just size and the GC program. We also fill in // the name for possible debugging use. -func funcLayout(t *rtype, rcvr *rtype) (frametype *rtype, argSize, retOffset uintptr, stk *bitVector, framePool *sync.Pool) { +func funcLayout(t *funcType, rcvr *rtype) (frametype *rtype, argSize, retOffset uintptr, stk *bitVector, framePool *sync.Pool) { if t.Kind() != Func { panic("reflect: funcLayout of non-func type") } @@ -3055,8 +3055,6 @@ func funcLayout(t *rtype, rcvr *rtype) (frametype *rtype, argSize, retOffset uin return lt.t, lt.argSize, lt.retOffset, lt.stack, lt.framePool } - tt := (*funcType)(unsafe.Pointer(t)) - // compute gc program & stack bitmap for arguments ptrmap := new(bitVector) var offset uintptr @@ -3071,19 +3069,18 @@ func funcLayout(t *rtype, rcvr *rtype) (frametype *rtype, argSize, retOffset uin } offset += ptrSize } - for _, arg := range tt.in() { + for _, arg := range t.in() { offset += -offset & uintptr(arg.align-1) addTypeBits(ptrmap, offset, arg) offset += arg.size } - argN := ptrmap.n argSize = offset if runtime.GOARCH == "amd64p32" { offset += -offset & (8 - 1) } offset += -offset & (ptrSize - 1) retOffset = offset - for _, res := range tt.out() { + for _, res := range t.out() { offset += -offset & uintptr(res.align-1) addTypeBits(ptrmap, offset, res) offset += res.size @@ -3104,7 +3101,6 @@ func funcLayout(t *rtype, rcvr *rtype) (frametype *rtype, argSize, retOffset uin } else { x.kind |= kindNoPointers } - ptrmap.n = argN var s string if rcvr != nil { diff --git a/src/reflect/value.go b/src/reflect/value.go index 6a49adef593c5..602a37c55dbdf 100644 --- a/src/reflect/value.go +++ b/src/reflect/value.go @@ -325,7 +325,7 @@ var callGC bool // for testing; see TestCallMethodJump func (v Value) call(op string, in []Value) []Value { // Get function pointer, type. - t := v.typ + t := (*funcType)(unsafe.Pointer(v.typ)) var ( fn unsafe.Pointer rcvr Value @@ -499,8 +499,13 @@ func (v Value) call(op string, in []Value) []Value { // NOTE: This function must be marked as a "wrapper" in the generated code, // so that the linker can make it work correctly for panic and recover. // The gc compilers know to do that for the name "reflect.callReflect". -func callReflect(ctxt *makeFuncImpl, frame unsafe.Pointer) { - ftyp := ctxt.typ +// +// ctxt is the "closure" generated by MakeFunc. +// frame is a pointer to the arguments to that closure on the stack. +// retValid points to a boolean which should be set when the results +// section of frame is set. +func callReflect(ctxt *makeFuncImpl, frame unsafe.Pointer, retValid *bool) { + ftyp := ctxt.ftyp f := ctxt.fn // Copy argument frame into Values. @@ -565,6 +570,16 @@ func callReflect(ctxt *makeFuncImpl, frame unsafe.Pointer) { } } + // Announce that the return values are valid. + // After this point the runtime can depend on the return values being valid. + *retValid = true + + // We have to make sure that the out slice lives at least until + // the runtime knows the return values are valid. Otherwise, the + // return values might not be scanned by anyone during a GC. + // (out would be dead, and the return slots not yet alive.) + runtime.KeepAlive(out) + // runtime.getArgInfo expects to be able to find ctxt on the // stack when it finds our caller, makeFuncStub. Make sure it // doesn't get garbage collected. @@ -578,7 +593,7 @@ func callReflect(ctxt *makeFuncImpl, frame unsafe.Pointer) { // The return value rcvrtype gives the method's actual receiver type. // The return value t gives the method type signature (without the receiver). // The return value fn is a pointer to the method code. -func methodReceiver(op string, v Value, methodIndex int) (rcvrtype, t *rtype, fn unsafe.Pointer) { +func methodReceiver(op string, v Value, methodIndex int) (rcvrtype *rtype, t *funcType, fn unsafe.Pointer) { i := methodIndex if v.typ.Kind() == Interface { tt := (*interfaceType)(unsafe.Pointer(v.typ)) @@ -595,7 +610,7 @@ func methodReceiver(op string, v Value, methodIndex int) (rcvrtype, t *rtype, fn } rcvrtype = iface.itab.typ fn = unsafe.Pointer(&iface.itab.fun[i]) - t = tt.typeOff(m.typ) + t = (*funcType)(unsafe.Pointer(tt.typeOff(m.typ))) } else { rcvrtype = v.typ ms := v.typ.exportedMethods() @@ -608,7 +623,7 @@ func methodReceiver(op string, v Value, methodIndex int) (rcvrtype, t *rtype, fn } ifn := v.typ.textOff(m.ifn) fn = unsafe.Pointer(&ifn) - t = v.typ.typeOff(m.mtyp) + t = (*funcType)(unsafe.Pointer(v.typ.typeOff(m.mtyp))) } return } @@ -647,25 +662,31 @@ func align(x, n uintptr) uintptr { // NOTE: This function must be marked as a "wrapper" in the generated code, // so that the linker can make it work correctly for panic and recover. // The gc compilers know to do that for the name "reflect.callMethod". -func callMethod(ctxt *methodValue, frame unsafe.Pointer) { +// +// ctxt is the "closure" generated by makeVethodValue. +// frame is a pointer to the arguments to that closure on the stack. +// retValid points to a boolean which should be set when the results +// section of frame is set. +func callMethod(ctxt *methodValue, frame unsafe.Pointer, retValid *bool) { rcvr := ctxt.rcvr rcvrtype, t, fn := methodReceiver("call", rcvr, ctxt.method) frametype, argSize, retOffset, _, framePool := funcLayout(t, rcvrtype) // Make a new frame that is one word bigger so we can store the receiver. - args := framePool.Get().(unsafe.Pointer) + // This space is used for both arguments and return values. + scratch := framePool.Get().(unsafe.Pointer) // Copy in receiver and rest of args. // Avoid constructing out-of-bounds pointers if there are no args. - storeRcvr(rcvr, args) + storeRcvr(rcvr, scratch) if argSize-ptrSize > 0 { - typedmemmovepartial(frametype, add(args, ptrSize, "argSize > ptrSize"), frame, ptrSize, argSize-ptrSize) + typedmemmovepartial(frametype, add(scratch, ptrSize, "argSize > ptrSize"), frame, ptrSize, argSize-ptrSize) } // Call. - // Call copies the arguments from args to the stack, calls fn, - // and then copies the results back into args. - call(frametype, fn, args, uint32(frametype.size), uint32(retOffset)) + // Call copies the arguments from scratch to the stack, calls fn, + // and then copies the results back into scratch. + call(frametype, fn, scratch, uint32(frametype.size), uint32(retOffset)) // Copy return values. On amd64p32, the beginning of return values // is 64-bit aligned, so the caller's frame layout (which doesn't have @@ -680,13 +701,19 @@ func callMethod(ctxt *methodValue, frame unsafe.Pointer) { } // This copies to the stack. Write barriers are not needed. memmove(add(frame, callerRetOffset, "frametype.size > retOffset"), - add(args, retOffset, "frametype.size > retOffset"), + add(scratch, retOffset, "frametype.size > retOffset"), frametype.size-retOffset) } - // Put the args scratch space back in the pool. - typedmemclr(frametype, args) - framePool.Put(args) + // Tell the runtime it can now depend on the return values + // being properly initialized. + *retValid = true + + // Clear the scratch space and put it back in the pool. + // This must happen after the statement above, so that the return + // values will always be scanned by someone. + typedmemclr(frametype, scratch) + framePool.Put(scratch) // See the comment in callReflect. runtime.KeepAlive(ctxt) diff --git a/src/runtime/traceback.go b/src/runtime/traceback.go index d8c225d975fa2..cdd2a62d203e9 100644 --- a/src/runtime/traceback.go +++ b/src/runtime/traceback.go @@ -557,8 +557,9 @@ func gentraceback(pc0, sp0, lr0 uintptr, gp *g, skip int, pcbuf *uintptr, max in // reflectMethodValue is a partial duplicate of reflect.makeFuncImpl // and reflect.methodValue. type reflectMethodValue struct { - fn uintptr - stack *bitvector // args bitmap + fn uintptr + stack *bitvector // ptrmap for both args and results + argLen uintptr // just args } // getArgInfoFast returns the argument frame information for a call to f. @@ -587,6 +588,7 @@ func getArgInfo(frame *stkframe, f funcInfo, needArgMap bool, ctxt *funcval) (ar // These take a *reflect.methodValue as their // context register. var mv *reflectMethodValue + var retValid bool if ctxt != nil { // This is not an actual call, but a // deferred call. The function value @@ -600,6 +602,10 @@ func getArgInfo(frame *stkframe, f funcInfo, needArgMap bool, ctxt *funcval) (ar // 0(SP). arg0 := frame.sp + sys.MinFrameSize mv = *(**reflectMethodValue)(unsafe.Pointer(arg0)) + // Figure out whether the return values are valid. + // Reflect will update this value after it copies + // in the return values. + retValid = *(*bool)(unsafe.Pointer(arg0 + 3*sys.PtrSize)) } if mv.fn != f.entry { print("runtime: confused by ", funcname(f), "\n") @@ -607,6 +613,9 @@ func getArgInfo(frame *stkframe, f funcInfo, needArgMap bool, ctxt *funcval) (ar } bv := mv.stack arglen = uintptr(bv.n * sys.PtrSize) + if !retValid { + arglen = uintptr(mv.argLen) &^ (sys.PtrSize - 1) + } argmap = bv } } diff --git a/test/fixedbugs/issue27695.go b/test/fixedbugs/issue27695.go new file mode 100644 index 0000000000000..8bd4939e7ecb4 --- /dev/null +++ b/test/fixedbugs/issue27695.go @@ -0,0 +1,62 @@ +// run + +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Make sure return values are always scanned, when +// calling methods (+functions, TODO) with reflect. + +package main + +import ( + "reflect" + "runtime/debug" + "sync" +) + +func main() { + debug.SetGCPercent(1) // run GC frequently + var wg sync.WaitGroup + for i := 0; i < 20; i++ { + wg.Add(1) + go func() { + defer wg.Done() + for i := 0; i < 2000; i++ { + _test() + } + }() + } + wg.Wait() +} + +type Stt struct { + Data interface{} +} + +type My struct { + b byte +} + +func (this *My) Run(rawData []byte) (Stt, error) { + var data string = "hello" + stt := Stt{ + Data: data, + } + return stt, nil +} + +func _test() (interface{}, error) { + f := reflect.ValueOf(&My{}).MethodByName("Run") + if method, ok := f.Interface().(func([]byte) (Stt, error)); ok { + s, e := method(nil) + // The bug in issue27695 happens here, during the return + // from the above call (at the end of reflect.callMethod + // when preparing to return). The result value that + // is assigned to s was not being scanned if GC happens + // to occur there. + i := interface{}(s) + return i, e + } + return nil, nil +} diff --git a/test/fixedbugs/issue27695b.go b/test/fixedbugs/issue27695b.go new file mode 100644 index 0000000000000..d80acfb8b40ee --- /dev/null +++ b/test/fixedbugs/issue27695b.go @@ -0,0 +1,64 @@ +// run + +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Make sure return values aren't scanned until they +// are initialized, when calling functions and methods +// via reflect. + +package main + +import ( + "reflect" + "runtime" + "unsafe" +) + +var badPtr uintptr + +var sink []byte + +func init() { + // Allocate large enough to use largeAlloc. + b := make([]byte, 1<<16-1) + sink = b // force heap allocation + // Any space between the object and the end of page is invalid to point to. + badPtr = uintptr(unsafe.Pointer(&b[len(b)-1])) + 1 +} + +func f(d func() *byte) *byte { + // Initialize callee args section with a bad pointer. + g(badPtr) + + // Then call a function which returns a pointer. + // That return slot starts out holding a bad pointer. + return d() +} + +//go:noinline +func g(x uintptr) { +} + +type T struct { +} + +func (t *T) Foo() *byte { + runtime.GC() + return nil +} + +func main() { + // Functions + d := reflect.MakeFunc(reflect.TypeOf(func() *byte { return nil }), + func(args []reflect.Value) []reflect.Value { + runtime.GC() + return []reflect.Value{reflect.ValueOf((*byte)(nil))} + }).Interface().(func() *byte) + f(d) + + // Methods + e := reflect.ValueOf(&T{}).Method(0).Interface().(func() *byte) + f(e) +} From 58c9bd9bfbedda966d2ae687f3e23f67ec889bc4 Mon Sep 17 00:00:00 2001 From: Keith Randall Date: Sun, 30 Sep 2018 09:04:17 -0700 Subject: [PATCH 09/13] [release-branch.go1.11] reflect: fix s390x reflect method calls R0 isn't the zero register any more. Oops. Update #27867 Change-Id: I46a975ed37d5e570afe2e228d3edf74949e08ad7 Reviewed-on: https://go-review.googlesource.com/138580 Reviewed-by: Michael Munday Reviewed-on: https://go-review.googlesource.com/138583 Run-TryBot: Keith Randall Run-TryBot: Ian Lance Taylor Run-TryBot: Michael Munday TryBot-Result: Gobot Gobot Reviewed-by: Ian Lance Taylor --- src/reflect/asm_s390x.s | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/reflect/asm_s390x.s b/src/reflect/asm_s390x.s index 2ab5481c9bf3b..cb7954c900cb9 100644 --- a/src/reflect/asm_s390x.s +++ b/src/reflect/asm_s390x.s @@ -14,7 +14,7 @@ TEXT ·makeFuncStub(SB),(NOSPLIT|WRAPPER),$32 MOVD R12, 8(R15) MOVD $argframe+0(FP), R3 MOVD R3, 16(R15) - MOVB R0, 32(R15) + MOVB $0, 32(R15) ADD $32, R15, R3 MOVD R3, 24(R15) BL ·callReflect(SB) @@ -29,7 +29,7 @@ TEXT ·methodValueCall(SB),(NOSPLIT|WRAPPER),$32 MOVD R12, 8(R15) MOVD $argframe+0(FP), R3 MOVD R3, 16(R15) - MOVB R0, 32(R15) + MOVB $0, 32(R15) ADD $32, R15, R3 MOVD R3, 24(R15) BL ·callMethod(SB) From 307f8b5a6d21ff6b00f1a09589e1e366450c7509 Mon Sep 17 00:00:00 2001 From: Matthew Waters Date: Mon, 24 Sep 2018 06:08:54 -0400 Subject: [PATCH 10/13] [release-branch.go1.11] net: concatenate multiple TXT strings in single TXT record When go resolver was changed to use dnsmessage.Parser, LookupTXT returned two strings in one record as two different records. This change reverts back to concatenating multiple strings in a single TXT record. Updates #27763 Fixes #27886 Change-Id: Ice226fcb2be4be58853de34ed35b4627acb429ea Reviewed-on: https://go-review.googlesource.com/136955 Reviewed-by: Ian Gudger Reviewed-by: Brad Fitzpatrick Run-TryBot: Ian Gudger TryBot-Result: Gobot Gobot (cherry picked from commit 7b3b160323b56b357832549fbab7a60d27688ec1) Reviewed-on: https://go-review.googlesource.com/138177 Run-TryBot: Brad Fitzpatrick Reviewed-by: Katie Hockman --- src/net/dnsclient_unix_test.go | 53 ++++++++++++++++++++++++++++++++++ src/net/lookup_unix.go | 16 ++++++++-- 2 files changed, 66 insertions(+), 3 deletions(-) diff --git a/src/net/dnsclient_unix_test.go b/src/net/dnsclient_unix_test.go index 9e4ebcc7bbef8..9482fc466f443 100644 --- a/src/net/dnsclient_unix_test.go +++ b/src/net/dnsclient_unix_test.go @@ -1568,3 +1568,56 @@ func TestDNSDialTCP(t *testing.T) { t.Fatal("exhange failed:", err) } } + +// Issue 27763: verify that two strings in one TXT record are concatenated. +func TestTXTRecordTwoStrings(t *testing.T) { + fake := fakeDNSServer{ + rh: func(n, _ string, q dnsmessage.Message, _ time.Time) (dnsmessage.Message, error) { + r := dnsmessage.Message{ + Header: dnsmessage.Header{ + ID: q.Header.ID, + Response: true, + RCode: dnsmessage.RCodeSuccess, + }, + Questions: q.Questions, + Answers: []dnsmessage.Resource{ + { + Header: dnsmessage.ResourceHeader{ + Name: q.Questions[0].Name, + Type: dnsmessage.TypeA, + Class: dnsmessage.ClassINET, + }, + Body: &dnsmessage.TXTResource{ + TXT: []string{"string1 ", "string2"}, + }, + }, + { + Header: dnsmessage.ResourceHeader{ + Name: q.Questions[0].Name, + Type: dnsmessage.TypeA, + Class: dnsmessage.ClassINET, + }, + Body: &dnsmessage.TXTResource{ + TXT: []string{"onestring"}, + }, + }, + }, + } + return r, nil + }, + } + r := Resolver{PreferGo: true, Dial: fake.DialContext} + txt, err := r.lookupTXT(context.Background(), "golang.org") + if err != nil { + t.Fatal("LookupTXT failed:", err) + } + if want := 2; len(txt) != want { + t.Fatalf("len(txt), got %d, want %d", len(txt), want) + } + if want := "string1 string2"; txt[0] != want { + t.Errorf("txt[0], got %q, want %q", txt[0], want) + } + if want := "onestring"; txt[1] != want { + t.Errorf("txt[1], got %q, want %q", txt[1], want) + } +} diff --git a/src/net/lookup_unix.go b/src/net/lookup_unix.go index 2c3191aca8a66..f55ef5969eb16 100644 --- a/src/net/lookup_unix.go +++ b/src/net/lookup_unix.go @@ -299,11 +299,21 @@ func (r *Resolver) lookupTXT(ctx context.Context, name string) ([]string, error) Server: server, } } + // Multiple strings in one TXT record need to be + // concatenated without separator to be consistent + // with previous Go resolver. + n := 0 + for _, s := range txt.TXT { + n += len(s) + } + txtJoin := make([]byte, 0, n) + for _, s := range txt.TXT { + txtJoin = append(txtJoin, s...) + } if len(txts) == 0 { - txts = txt.TXT - } else { - txts = append(txts, txt.TXT...) + txts = make([]string, 0, 1) } + txts = append(txts, string(txtJoin)) } return txts, nil } From 92ae524bc5b62ef8e505a95d65f74ca9a7cb335a Mon Sep 17 00:00:00 2001 From: Taesu Pyo Date: Tue, 28 Aug 2018 15:56:10 +0000 Subject: [PATCH 11/13] [release-branch.go1.11] encoding/json: fix UnmarshalTypeError without field and struct values MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Updates #26444 Updates #27275 Fixes #27318 Change-Id: I9e8cbff79f7643ca8964c572c1a98172b6831730 GitHub-Last-Rev: 7eea2158b67ccab34b45a21e8f4289c36de02d93 GitHub-Pull-Request: golang/go#26719 Reviewed-on: https://go-review.googlesource.com/126897 Reviewed-by: Daniel Martí Run-TryBot: Daniel Martí TryBot-Result: Gobot Gobot Reviewed-on: https://go-review.googlesource.com/138178 Run-TryBot: Brad Fitzpatrick --- src/encoding/json/decode.go | 6 +++--- src/encoding/json/decode_test.go | 17 +++++++++++++++++ 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/src/encoding/json/decode.go b/src/encoding/json/decode.go index 0b29249218a32..7d235087e60f4 100644 --- a/src/encoding/json/decode.go +++ b/src/encoding/json/decode.go @@ -672,6 +672,7 @@ func (d *decodeState) object(v reflect.Value) error { } var mapElem reflect.Value + originalErrorContext := d.errorContext for { // Read opening " of string key or closing }. @@ -832,8 +833,7 @@ func (d *decodeState) object(v reflect.Value) error { return errPhase } - d.errorContext.Struct = "" - d.errorContext.Field = "" + d.errorContext = originalErrorContext } return nil } @@ -991,7 +991,7 @@ func (d *decodeState) literalStore(item []byte, v reflect.Value, fromQuoted bool if fromQuoted { return fmt.Errorf("json: invalid use of ,string struct tag, trying to unmarshal %q into %v", item, v.Type()) } - return &UnmarshalTypeError{Value: "number", Type: v.Type(), Offset: int64(d.readIndex())} + d.saveError(&UnmarshalTypeError{Value: "number", Type: v.Type(), Offset: int64(d.readIndex())}) case reflect.Interface: n, err := d.convertNumber(s) if err != nil { diff --git a/src/encoding/json/decode_test.go b/src/encoding/json/decode_test.go index ab83b81bb3958..5746ddf9861e7 100644 --- a/src/encoding/json/decode_test.go +++ b/src/encoding/json/decode_test.go @@ -371,6 +371,10 @@ func (b *intWithPtrMarshalText) UnmarshalText(data []byte) error { return (*intWithMarshalText)(b).UnmarshalText(data) } +type mapStringToStringData struct { + Data map[string]string `json:"data"` +} + type unmarshalTest struct { in string ptr interface{} @@ -401,6 +405,7 @@ var unmarshalTests = []unmarshalTest{ {in: `"invalid: \uD834x\uDD1E"`, ptr: new(string), out: "invalid: \uFFFDx\uFFFD"}, {in: "null", ptr: new(interface{}), out: nil}, {in: `{"X": [1,2,3], "Y": 4}`, ptr: new(T), out: T{Y: 4}, err: &UnmarshalTypeError{"array", reflect.TypeOf(""), 7, "T", "X"}}, + {in: `{"X": 23}`, ptr: new(T), out: T{}, err: &UnmarshalTypeError{"number", reflect.TypeOf(""), 8, "T", "X"}}, {in: `{"x": 1}`, ptr: new(tx), out: tx{}}, {in: `{"x": 1}`, ptr: new(tx), out: tx{}}, {in: `{"x": 1}`, ptr: new(tx), err: fmt.Errorf("json: unknown field \"x\""), disallowUnknownFields: true}, {in: `{"F1":1,"F2":2,"F3":3}`, ptr: new(V), out: V{F1: float64(1), F2: int32(2), F3: Number("3")}}, @@ -866,6 +871,18 @@ var unmarshalTests = []unmarshalTest{ err: fmt.Errorf("json: unknown field \"extra\""), disallowUnknownFields: true, }, + // issue 26444 + // UnmarshalTypeError without field & struct values + { + in: `{"data":{"test1": "bob", "test2": 123}}`, + ptr: new(mapStringToStringData), + err: &UnmarshalTypeError{Value: "number", Type: reflect.TypeOf(""), Offset: 37, Struct: "mapStringToStringData", Field: "data"}, + }, + { + in: `{"data":{"test1": 123, "test2": "bob"}}`, + ptr: new(mapStringToStringData), + err: &UnmarshalTypeError{Value: "number", Type: reflect.TypeOf(""), Offset: 21, Struct: "mapStringToStringData", Field: "data"}, + }, } func TestMarshal(t *testing.T) { From 52c4bdb65d49be0b8f04a27f228a8611fd57ca3c Mon Sep 17 00:00:00 2001 From: Katie Hockman Date: Mon, 1 Oct 2018 16:08:49 -0400 Subject: [PATCH 12/13] [release-branch.go1.11] doc: document Go 1.11.1 Updates #27953 Change-Id: I2f1a55e15dc5737a5a06bd894c46b2c4705f338c Reviewed-on: https://go-review.googlesource.com/138858 Reviewed-by: Filippo Valsorda (cherry picked from commit f99fc3a119dbb98fa9dddcb2e31a6c51925fde77) Reviewed-on: https://go-review.googlesource.com/138859 Reviewed-by: Dmitri Shuralyov --- doc/devel/release.html | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/doc/devel/release.html b/doc/devel/release.html index a6dd5f9d28223..0448c0d43e71f 100644 --- a/doc/devel/release.html +++ b/doc/devel/release.html @@ -30,6 +30,17 @@

    go1.11 (released 2018/08/24)

    Read the Go 1.11 Release Notes for more information.

    +

    Minor revisions

    + +

    +go1.11.1 (released 2018/10/01) includes fixes to the compiler, documentation, go +command, runtime, and the crypto/x509, encoding/json, +go/types, net, net/http, and +reflect packages. +See the Go +1.11.1 milestone on our issue tracker for details. +

    +

    go1.10 (released 2018/02/16)

    From 26957168c4c0cdcc7ca4f0b19d0eb19474d224ac Mon Sep 17 00:00:00 2001 From: Katie Hockman Date: Mon, 1 Oct 2018 16:40:42 -0400 Subject: [PATCH 13/13] [release-branch.go1.11] go1.11.1 Change-Id: I3cf3e57b11ad02b497276bae1864fc5ade8144b9 Reviewed-on: https://go-review.googlesource.com/138860 Run-TryBot: Katie Hockman TryBot-Result: Gobot Gobot Reviewed-by: Dmitri Shuralyov --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index f3220fb759237..11c4f5950a4da 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -go1.11 +go1.11.1 \ No newline at end of file