lightning: every HTTP retry should use its own request (#47959) (#47968)

close #47930

br/pkg/pdutil/pd.go

@@ -163,14 +163,17 @@ func pdRequestWithCode(
 		return 0, nil, errors.Trace(err)
 	}
 	reqURL := fmt.Sprintf("%s/%s", u, prefix)
-	req, err := http.NewRequestWithContext(ctx, method, reqURL, body)
-	if err != nil {
-		return 0, nil, errors.Trace(err)
-	}
-	var resp *http.Response
+	var (
+		req  *http.Request
+		resp *http.Response
+	)
 	count := 0
 	// the total retry duration: 120*1 = 2min
 	for {
+		req, err = http.NewRequestWithContext(ctx, method, reqURL, body)
+		if err != nil {
+			return 0, nil, errors.Trace(err)
+		}
 		resp, err = cli.Do(req) //nolint:bodyclose
 		count++
 		failpoint.Inject("InjectClosed", func(v failpoint.Value) {

br/pkg/pdutil/pd_serial_test.go

@@ -3,6 +3,7 @@
 package pdutil
 
 import (
+	"bytes"
 	"context"
 	"encoding/hex"
 	"encoding/json"
@@ -186,8 +187,16 @@ func TestPDRequestRetry(t *testing.T) {
 		w.WriteHeader(http.StatusOK)
 	}))
 	cli := http.DefaultClient
+	cli.Transport = http.DefaultTransport.(*http.Transport).Clone()
+	// although the real code doesn't disable keep alive, we need to disable it
+	// in test to avoid the connection being reused and #47930 can't appear. The
+	// real code will only meet #47930 when go's internal http client just dropped
+	// all idle connections.
+	cli.Transport.(*http.Transport).DisableKeepAlives = true
+
 	taddr := ts.URL
-	_, reqErr := pdRequest(ctx, taddr, "", cli, http.MethodGet, nil)
+	body := bytes.NewBuffer([]byte("test"))
+	_, reqErr := pdRequest(ctx, taddr, "", cli, http.MethodPost, body)
 	require.NoError(t, reqErr)
 	ts.Close()
 	count = 0