From 6cb558981311a8b3a5801b42737de9f432be8b1d Mon Sep 17 00:00:00 2001 From: harshil Date: Fri, 28 Aug 2020 15:34:31 +0530 Subject: [PATCH] fix(bulk): Batch list in bulk loader to avoid panic --- dgraph/cmd/bulk/reduce.go | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/dgraph/cmd/bulk/reduce.go b/dgraph/cmd/bulk/reduce.go index 6204b220fca..84a85b1d7d1 100644 --- a/dgraph/cmd/bulk/reduce.go +++ b/dgraph/cmd/bulk/reduce.go @@ -402,7 +402,16 @@ func (r *reducer) startWriting(ci *countIndexer, writerCh chan *encodeRequest, c // Wait for it to be encoded. start := time.Now() - x.Check(ci.writer.Write(req.list)) + for len(req.list.GetKv()) > 0 { + batchSize := 100 + if len(req.list.Kv) < batchSize { + batchSize = len(req.list.Kv) + } + batch := &bpb.KVList{Kv: req.list.Kv[:batchSize]} + req.list.Kv = req.list.Kv[batchSize:] + x.Check(ci.writer.Write(batch)) + } + if req.splitList != nil && len(req.splitList.Kv) > 0 { splitCh <- req.splitList }