From fff9ec77052eb1ede4c457fb9d18ab66c17fc0f2 Mon Sep 17 00:00:00 2001 From: harshil Date: Fri, 28 Aug 2020 15:34:31 +0530 Subject: [PATCH] fix(bulk): Batch list in bulk loader to avoid panic --- dgraph/cmd/bulk/reduce.go | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/dgraph/cmd/bulk/reduce.go b/dgraph/cmd/bulk/reduce.go index 02e4f8c51fa..055cd8dba1f 100644 --- a/dgraph/cmd/bulk/reduce.go +++ b/dgraph/cmd/bulk/reduce.go @@ -401,7 +401,16 @@ func (r *reducer) startWriting(ci *countIndexer, writerCh chan *encodeRequest, c // Wait for it to be encoded. start := time.Now() - x.Check(ci.writer.Write(req.list)) + for len(req.list.GetKv()) > 0 { + batchSize := 100 + if len(req.list.Kv) < batchSize { + batchSize = len(req.list.Kv) + } + batch := &bpb.KVList{Kv: req.list.Kv[:batchSize]} + req.list.Kv = req.list.Kv[batchSize:] + x.Check(ci.writer.Write(batch)) + } + if req.splitList != nil && len(req.splitList.Kv) > 0 { splitCh <- req.splitList }