[tools] Use streaming reads in read_data_files (#3474)

m3db · May 6, 2021 · c4bd395 · c4bd395
1 parent 824d4a9
commit c4bd395
Showing 1 changed file with 9 additions and 8 deletions.
diff --git a/src/cmd/tools/read_data_files/main/main.go b/src/cmd/tools/read_data_files/main/main.go
@@ -136,7 +136,8 @@ func main() {
 			BlockStart:  time.Unix(0, *optBlockstart),
 			VolumeIndex: int(*volume),
 		},
-		FileSetType: fileSetType,
+		FileSetType:      fileSetType,
+		StreamingEnabled: true,
 	}
 
 	err = reader.Open(openOpts)
@@ -145,22 +146,25 @@ func main() {
 	}
 
 	for {
-		id, _, data, _, err := reader.Read()
+		entry, err := reader.StreamingRead()
 		if err == io.EOF {
 			break
 		}
 		if err != nil {
 			log.Fatalf("err reading metadata: %v", err)
 		}
 
+		var (
+			id   = entry.ID
+			data = entry.Data
+		)
+
 		if *idFilter != "" && !strings.Contains(id.String(), *idFilter) {
 			continue
 		}
 
 		if benchMode != benchmarkSeries {
-			data.IncRef()
-
-			iter := m3tsz.NewReaderIterator(xio.NewBytesReader64(data.Bytes()), true, encodingOpts)
+			iter := m3tsz.NewReaderIterator(xio.NewBytesReader64(data), true, encodingOpts)
 			for iter.Next() {
 				dp, _, annotation := iter.Current()
 				if benchMode == benchmarkNone {
@@ -179,11 +183,8 @@ func main() {
 				log.Fatalf("unable to iterate original data: %v", err)
 			}
 			iter.Close()
-
-			data.DecRef()
 		}
 
-		data.Finalize()
 		seriesCount++
 	}