etcd-tester: request with grpc.FailFast true

[gyuho]

Fix #6221.

Since we close the grpc connection, FailFast false
will block.

When we cancel stresser, we close grpc connection (https://github.com/coreos/etcd/blob/master/tools/functional-tester/etcd-tester/stresser.go#L281). This will cause errConnClosing in grpc side (https://github.com/grpc/grpc-go/blob/master/call.go#L173), hanging with grpc FailFast false.

We should use grpc FailFast true as in clientv3.

[gyuho]

/cc @heyitsanthony Reproducible

package main

import (
    "fmt"
    "log"
    "math/rand"
    "sync"
    "time"

    pb "github.com/coreos/etcd/etcdserver/etcdserverpb"
    "golang.org/x/net/context"
    "google.golang.org/grpc"
)

func randBytes(size int) []byte {
    data := make([]byte, size)
    for i := 0; i < size; i++ {
        data[i] = byte(int('a') + rand.Intn(26))
    }
    return data
}

type stressFunc func(ctx context.Context) error

func newStressRange(kvc pb.KVClient, keySuffixRange int) stressFunc {
    return func(ctx context.Context) error {
        _, err := kvc.Range(ctx, &pb.RangeRequest{
            Key: []byte(fmt.Sprintf("foo%016x", rand.Intn(keySuffixRange))),
        }, grpc.FailFast(false))
        return err
    }
}

func main() {
    conn, err := grpc.Dial("localhost:2379", grpc.WithInsecure())
    if err != nil {
        log.Fatal(err)
    }
    kvc := pb.NewKVClient(conn)
    fc := newStressRange(kvc, 800)

    ctx, cancel := context.WithCancel(context.Background())

    var wg sync.WaitGroup
    wg.Add(1000)

    for i := 0; i < 1000; i++ {
        go func() {
            defer wg.Done()
            now := time.Now()
            sctx, scancel := context.WithTimeout(ctx, time.Second)
            err := fc(sctx)
            scancel()
            fmt.Println("error:", err, "/ took", time.Since(now))
        }()
    }

    go func() {
        time.Sleep(15 * time.Millisecond)
        fmt.Println("canceling!!!!!!!!!!!!!")
        conn.Close()
        cancel()
    }()
    wg.Wait()
}

[heyitsanthony]

@gyuho this is a grpc bug... I think clientv3 is also affected?

[heyitsanthony]

PR inflight grpc/grpc-go#841