sanitize provisioner output strings

The grpc protocol requires strings to be valid utf8, but because
provisioners often don't have control over the command output, invalid
utf8 sequences can make it into the response causing grpc transport
errors.

Replace all invalid utf sequences with the standard utf replacement
character in the provisioner output. The code is a direct copy from the
go1.13 std library, and can be replaced with strings.ToValidUTF8 once
it's available.

helper/plugin/grpc_provisioner.go

@@ -1,7 +1,10 @@
 package plugin
 
 import (
+	"fmt"
 	"log"
+	"strings"
+	"unicode/utf8"
 
 	"github.com/hashicorp/terraform/helper/schema"
 	proto "github.com/hashicorp/terraform/internal/tfplugin5"
@@ -90,7 +93,7 @@ type uiOutput struct {
 
 func (o uiOutput) Output(s string) {
 	err := o.srv.Send(&proto.ProvisionResource_Response{
-		Output: s,
+		Output: toValidUTF8(s, string(utf8.RuneError)),
 	})
 	if err != nil {
 		log.Printf("[ERROR] %s", err)
@@ -113,6 +116,7 @@ func (s *GRPCProvisionerServer) ProvisionResource(req *proto.ProvisionResource_R
 
 	connVal, err := msgpack.Unmarshal(req.Connection.Msgpack, cty.Map(cty.String))
 	if err != nil {
+		fmt.Println(err)
 		srvResp.Diagnostics = convert.AppendProtoDiag(srvResp.Diagnostics, err)
 		srv.Send(srvResp)
 		return nil
@@ -145,3 +149,55 @@ func (s *GRPCProvisionerServer) Stop(_ context.Context, req *proto.Stop_Request)
 
 	return resp, nil
 }
+
+// FIXME: backported from go1.13 strings package, remove once terraform is
+//        using go >= 1.13
+// ToValidUTF8 returns a copy of the string s with each run of invalid UTF-8 byte sequences
+// replaced by the replacement string, which may be empty.
+func toValidUTF8(s, replacement string) string {
+	var b strings.Builder
+
+	for i, c := range s {
+		if c != utf8.RuneError {
+			continue
+		}
+
+		_, wid := utf8.DecodeRuneInString(s[i:])
+		if wid == 1 {
+			b.Grow(len(s) + len(replacement))
+			b.WriteString(s[:i])
+			s = s[i:]
+			break
+		}
+	}
+
+	// Fast path for unchanged input
+	if b.Cap() == 0 { // didn't call b.Grow above
+		return s
+	}
+
+	invalid := false // previous byte was from an invalid UTF-8 sequence
+	for i := 0; i < len(s); {
+		c := s[i]
+		if c < utf8.RuneSelf {
+			i++
+			invalid = false
+			b.WriteByte(c)
+			continue
+		}
+		_, wid := utf8.DecodeRuneInString(s[i:])
+		if wid == 1 {
+			i++
+			if !invalid {
+				invalid = true
+				b.WriteString(replacement)
+			}
+			continue
+		}
+		invalid = false
+		b.WriteString(s[i : i+wid])
+		i += wid
+	}
+
+	return b.String()
+}

helper/plugin/grpc_provisioner_test.go

@@ -1,5 +1,83 @@
 package plugin
 
-import proto "github.com/hashicorp/terraform/internal/tfplugin5"
+import (
+	"testing"
+	"unicode/utf8"
+
+	"github.com/golang/mock/gomock"
+	"github.com/hashicorp/terraform/helper/schema"
+	proto "github.com/hashicorp/terraform/internal/tfplugin5"
+	mockproto "github.com/hashicorp/terraform/plugin/mock_proto"
+	"github.com/hashicorp/terraform/terraform"
+	context "golang.org/x/net/context"
+)
 
 var _ proto.ProvisionerServer = (*GRPCProvisionerServer)(nil)
+
+type validUTF8Matcher string
+
+func (m validUTF8Matcher) Matches(x interface{}) bool {
+	resp := x.(*proto.ProvisionResource_Response)
+	return utf8.Valid([]byte(resp.Output))
+
+}
+
+func (m validUTF8Matcher) String() string {
+	return string(m)
+}
+
+func mockProvisionerServer(t *testing.T) *mockproto.MockProvisioner_ProvisionResourceServer {
+	ctrl := gomock.NewController(t)
+
+	server := mockproto.NewMockProvisioner_ProvisionResourceServer(ctrl)
+
+	// we always need a GetSchema method
+	server.EXPECT().Send(
+		validUTF8Matcher("check for valid utf8"),
+	).Return(nil)
+
+	return server
+}
+
+// ensure that a provsioner cannot return invalid utf8 which isn't allowed in
+// the grpc protocol.
+func TestProvisionerInvalidUTF8(t *testing.T) {
+	p := &schema.Provisioner{
+		ConnSchema: map[string]*schema.Schema{
+			"foo": {
+				Type:     schema.TypeString,
+				Optional: true,
+			},
+		},
+
+		Schema: map[string]*schema.Schema{
+			"foo": {
+				Type:     schema.TypeInt,
+				Optional: true,
+			},
+		},
+
+		ApplyFunc: func(ctx context.Context) error {
+			out := ctx.Value(schema.ProvOutputKey).(terraform.UIOutput)
+			out.Output("invalid \xc3\x28\n")
+			return nil
+		},
+	}
+
+	srv := mockProvisionerServer(t)
+	cfg := &proto.DynamicValue{
+		Msgpack: []byte("\x81\xa3foo\x01"),
+	}
+	conn := &proto.DynamicValue{
+		Msgpack: []byte("\x81\xa3foo\xa4host"),
+	}
+	provisionerServer := NewGRPCProvisionerServerShim(p)
+	req := &proto.ProvisionResource_Request{
+		Config:     cfg,
+		Connection: conn,
+	}
+
+	if err := provisionerServer.ProvisionResource(req, srv); err != nil {
+		t.Fatal(err)
+	}
+}