[large-tiles] Full flow deployable

Makefile

@@ -89,6 +89,7 @@ TOOLS :=               \
 	read_data_files      \
 	read_index_files     \
 	read_index_segments  \
+	query_data_files     \
 	clone_fileset        \
 	dtest                \
 	verify_data_files    \

go.mod

@@ -12,6 +12,7 @@ require (
 	github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5 // indirect
 	github.com/RoaringBitmap/roaring v0.4.21
 	github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d // indirect
+	github.com/apache/arrow/go/arrow v0.0.0-20200623135148-0d1b2f8e7158
 	github.com/apache/thrift v0.13.0
 	github.com/apex/log v1.3.0 // indirect
 	github.com/bmatcuk/doublestar v1.3.1 // indirect
@@ -96,7 +97,7 @@ require (
 	github.com/shirou/gopsutil v2.20.5+incompatible // indirect
 	github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72
 	github.com/spf13/cast v1.3.1-0.20190531151931-f31dc0aaab5a // indirect
-	github.com/spf13/cobra v0.0.3
+	github.com/spf13/cobra v0.0.5
 	github.com/spf13/jwalterweatherman v1.1.0 // indirect
 	github.com/spf13/pflag v1.0.5
 	github.com/spf13/viper v1.6.1

go.sum

@@ -61,6 +61,8 @@ github.com/alexbrainman/sspi v0.0.0-20180613141037-e580b900e9f5/go.mod h1:976q2E
 github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239 h1:kFOfPq6dUM1hTo4JG6LR5AXSUEsOjtdm0kw0FtQtMJA=
 github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c=
 github.com/antihax/optional v0.0.0-20180407024304-ca021399b1a6/go.mod h1:V8iCPQYkqmusNa815XgQio277wI47sdRh1dUOLdyC6Q=
+github.com/apache/arrow/go/arrow v0.0.0-20200623135148-0d1b2f8e7158 h1:NF0LS5ch0H8TotJS1pNwq9ikl+KZV9q8olvY9vltnhg=
+github.com/apache/arrow/go/arrow v0.0.0-20200623135148-0d1b2f8e7158/go.mod h1:QNYViu/X0HXDHw7m3KXzWSVXIbfUvJqBFe6Gj8/pYA0=
 github.com/apex/log v1.3.0 h1:1fyfbPvUwD10nMoh3hY6MXzvZShJQn9/ck7ATgAt5pA=
 github.com/apex/log v1.3.0/go.mod h1:jd8Vpsr46WAe3EZSQ/IUMs2qQD/GOycT5rPWCO1yGcs=
 github.com/apex/logs v0.0.4/go.mod h1:XzxuLZ5myVHDy9SAmYpamKKRNApGj54PfYLcFrXqDwo=
@@ -123,6 +125,7 @@ github.com/containerd/continuity v0.0.0-20200413184840-d3ef23f19fbb h1:nXPkFq8X1
 github.com/containerd/continuity v0.0.0-20200413184840-d3ef23f19fbb/go.mod h1:Dq467ZllaHgAtVp4p1xUQWBrFXR9s/wyoTpG8zOJGkY=
 github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=
 github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
+github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk=
 github.com/coreos/go-semver v0.2.0 h1:3Jm3tLmsgAYcjC+4Up7hJrFBPr+n7rAqYeSw/SZazuY=
 github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
 github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
@@ -131,10 +134,13 @@ github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7
 github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
 github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f h1:lBNOc5arjvs8E5mO2tbpBpLoyyu8B6e44T7hJy6potg=
 github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
+github.com/cpuguy83/go-md2man v1.0.10 h1:BSKMNlYxDvnunlTymqtgONjNnaRV1sTpcovwwjF22jk=
+github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE=
 github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d h1:U+s90UTSYgptZMwQh2aRr3LuazLJIa+Pg3Kc1ylSYVY=
 github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
 github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY=
 github.com/davecgh/go-spew v0.0.0-20151105211317-5215b55f46b2/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM=
@@ -274,6 +280,8 @@ github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEW
 github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/btree v1.0.0 h1:0udJVsspx3VBr5FwtLhQQtuAsVc79tTq0ocGIPAU6qo=
 github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
+github.com/google/flatbuffers v1.11.0 h1:O7CEyB8Cb3/DmtxODGtLHcEvpr81Jm5qLg/hsHnxA2A=
+github.com/google/flatbuffers v1.11.0/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8=
 github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
 github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
@@ -467,6 +475,7 @@ github.com/m3dbx/pilosa v0.0.0-20200602205121-7f389745e9ab h1:CpoDBrDayfBBuueJF+
 github.com/m3dbx/pilosa v0.0.0-20200602205121-7f389745e9ab/go.mod h1:muOSChOjsONKXcQTw+pr+bqIvb4bD7tSQHAXJPlerwc=
 github.com/m3dbx/vellum v0.0.0-20200602203954-e10aaedbd934 h1:INQTjUQJZlfKiHvFjACwTSTZd6TxtgoWP3am4+CmPVU=
 github.com/m3dbx/vellum v0.0.0-20200602203954-e10aaedbd934/go.mod h1:DOTAUfV4bzK6Nrb0dboT/oCG0DnQuX+/n0jfZPh6xxI=
+github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
 github.com/magiconair/properties v1.8.1 h1:ZC2Vc7/ZFkGmsVC9KvOjumD+G5lXy2RtTKyzRKO2BQ4=
 github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
 github.com/mailru/easyjson v0.0.0-20160728113105-d5b7844b561a/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
@@ -646,6 +655,7 @@ github.com/rogpeppe/fastuuid v1.1.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6L
 github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
 github.com/rs/cors v1.6.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU=
+github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
 github.com/russross/blackfriday v2.0.0+incompatible h1:cBXrhZNUf9C+La9/YpS+UHpUT8YD6Td9ZMSU9APFcsk=
 github.com/russross/blackfriday v2.0.0+incompatible/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
 github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0RK8m9o+Q=
@@ -695,6 +705,8 @@ github.com/spf13/cast v1.3.1-0.20190531151931-f31dc0aaab5a/go.mod h1:Qx5cxh0v+4U
 github.com/spf13/cobra v0.0.2-0.20171109065643-2da4a54c5cee/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
 github.com/spf13/cobra v0.0.3 h1:ZlrZ4XsMRm04Fr5pSFxBgfND2EBVa1nLpiy1stUsX/8=
 github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
+github.com/spf13/cobra v0.0.5 h1:f0B+LkLX6DtmRH1isoNA9VTtNUK9K8xYd28JNNfOv/s=
+github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU=
 github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
 github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk=
 github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo=
@@ -704,6 +716,7 @@ github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnIn
 github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
 github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
 github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
+github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s=
 github.com/spf13/viper v1.6.1 h1:VPZzIkznI1YhVMRi6vNFLHSwhnhReBfgTxIPccpfdZk=
 github.com/spf13/viper v1.6.1/go.mod h1:t3iDnF5Jlj76alVNuyFBk5oUMCvsrkbvZK0WQdfDi5k=
 github.com/src-d/gcfg v1.4.0 h1:xXbNR5AlLSA315x2UO+fTSSAXCDf+Ar38/6oyGbDKQ4=
@@ -745,6 +758,7 @@ github.com/uber/jaeger-lib v2.2.0+incompatible/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6
 github.com/uber/tchannel-go v1.12.0 h1:xqfXzViTPQxoA/KXBfnkXlwBn7JVpa7MVRUzWjVHMbI=
 github.com/uber/tchannel-go v1.12.0/go.mod h1:Rrgz1eL8kMjW/nEzZos0t+Heq0O4LhnUJVA32OvWKHo=
 github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc=
+github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0=
 github.com/ulikunitz/xz v0.5.5 h1:pFrO0lVpTBXLpYw+pnLj6TbvHuyjXMfjGeCwSqCVwok=
 github.com/ulikunitz/xz v0.5.5/go.mod h1:2bypXElzHzzJZwzH67Y6wb67pO62Rzfn7BSiF4ABRW8=
 github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA=
@@ -799,6 +813,7 @@ go.uber.org/zap v1.13.0 h1:nR6NoDBgAf67s68NhaXbsojM+2gxp3S1hWkHDl27pVU=
 go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM=
 golang.org/x/crypto v0.0.0-20171113213409-9f005a07e0d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
+golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190219172222-a4c6cb3142f2/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
@@ -879,6 +894,7 @@ golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5h
 golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20181122145206-62eef0e2fa9b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190209173611-3b5209105503/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190221075227-b4e8571b14e0/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=

scripts/development/m3_stack/docker-compose.yml

@@ -12,6 +12,7 @@ services:
       - "./schema.proto:/etc/m3dbnode/schema.proto"
     environment:
       - M3DB_HOST_ID=m3db_seed
+      - RAW_NAMESPACE=default
     expose:
       - "9002"
       - "9003"

src/cmd/tools/query_data_files/README.md

@@ -0,0 +1,34 @@
+# query_data_files
+
+`query_data_files` is a utility to calculate tiled sums on data for all timeseries' present in a TSDB file set.
+
+# Usage
+```
+$ git clone [email protected]:m3db/m3.git
+$ make query_data_files
+$ ./bin/query_data_files
+Usage: query_data_files [-b value] [-t value] [-p value] [-s value] [parameters ...]
+ -a                 Use arrow
+ -b, --block-start=value
+                    Block Start Time [in nsec]
+ -f, --fileset-type=value
+                    flush|snapshot
+ -i, --iterations=value
+                    Concurrent iteration count
+ -n, --namespace=value
+                    Namespace [e.g. metrics]
+ -p, --path-prefix=value
+                    Path prefix [e.g. /var/lib/m3db]
+ -s, --shard=value  Shard [expected format uint32]
+ -t, --tile-size=value
+                    Block Start Time [in min]
+ -v, --volume=value
+                    Volume number
+
+# example usage
+# query_data_files -b1480960800000000000 -n metrics -p /var/lib/m3db -s 451 -t 5 -c 3 > /tmp/sample-data.out
+```
+
+# TBH
+- The tool outputs tile aggregations with non-zero values.
+- The code currently assumes the data layout under the hood is `<path-prefix>/data/<namespace>/<shard>/...<block-start>-[index|...].db`. If this is not the file structure under the hood, replicate it to use this tool. Remember to copy checkpoint files along with each index file.

src/cmd/tools/query_data_files/main/main.go

@@ -0,0 +1,207 @@
+// Copyright (c) 2020 Uber Technologies, Inc.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in
+// all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+// THE SOFTWARE.
+
+package main
+
+import (
+	"fmt"
+	"io"
+	"log"
+	"os"
+	"time"
+
+	"github.com/m3db/m3/src/cmd/tools"
+	"github.com/m3db/m3/src/dbnode/encoding"
+	"github.com/m3db/m3/src/dbnode/encoding/arrow/tile"
+	"github.com/m3db/m3/src/dbnode/encoding/m3tsz"
+	"github.com/m3db/m3/src/dbnode/namespace"
+	"github.com/m3db/m3/src/dbnode/persist"
+	"github.com/m3db/m3/src/dbnode/persist/fs"
+	"github.com/m3db/m3/src/x/ident"
+	"github.com/m3db/m3/src/x/instrument"
+	"github.com/m3db/m3/src/x/pool"
+	xtime "github.com/m3db/m3/src/x/time"
+
+	"net/http"
+	_ "net/http/pprof"
+
+	"github.com/pborman/getopt"
+	"go.uber.org/zap"
+)
+
+const snapshotType = "snapshot"
+const flushType = "flush"
+const initValLength = 10
+
+func main() {
+	var (
+		optPathPrefix  = getopt.StringLong("path-prefix", 'p', "", "Path prefix [e.g. /var/lib/m3db]")
+		optNamespace   = getopt.StringLong("namespace", 'n', "", "Namespace [e.g. metrics]")
+		optShard       = getopt.Int32Long("shard", 's', -1, "Shard [expected format uint32]")
+		optBlockstart  = getopt.Int64Long("block-start", 'b', 0, "Block Start Time [in nsec]")
+		optTilesize    = getopt.Int64Long("tile-size", 't', 5, "Block Start Time [in min]")
+		volume         = getopt.Int64Long("volume", 'v', 0, "Volume number")
+		fileSetTypeArg = getopt.StringLong("fileset-type", 'f', flushType, fmt.Sprintf("%s|%s", flushType, snapshotType))
+
+		iterationCount = getopt.IntLong("iterations", 'i', 50, "Concurrent iteration count")
+		optUseArrow    = getopt.Bool('a', "Use arrow")
+	)
+	getopt.Parse()
+
+	go func() {
+		log.Println(http.ListenAndServe("localhost:6060", nil))
+	}()
+
+	rawLogger, err := zap.NewDevelopment()
+	if err != nil {
+		log.Fatalf("unable to create logger: %+v", err)
+	}
+	log := rawLogger.Sugar()
+
+	if *optPathPrefix == "" ||
+		*optNamespace == "" ||
+		*optBlockstart <= 0 ||
+		*optTilesize <= 0 ||
+		*volume < 0 ||
+		*iterationCount < 1 ||
+		(*fileSetTypeArg != snapshotType && *fileSetTypeArg != flushType) {
+		getopt.Usage()
+		os.Exit(1)
+	}
+
+	var fileSetType persist.FileSetType
+	switch *fileSetTypeArg {
+	case flushType:
+		fileSetType = persist.FileSetFlushType
+	case snapshotType:
+		fileSetType = persist.FileSetSnapshotType
+	default:
+		log.Fatalf("unknown fileset type: %s", *fileSetTypeArg)
+	}
+
+	bytesPool := tools.NewCheckedBytesPool()
+	bytesPool.Init()
+
+	var (
+		fsOpts = fs.NewOptions().SetFilePathPrefix(*optPathPrefix)
+		iOpts  = instrument.NewOptions()
+
+		iterations = *iterationCount
+		useArrow   = *optUseArrow
+		readStart  = time.Now()
+	)
+
+	var shards []uint32
+	if *optShard >= 0 {
+		shards = append(shards, uint32(*optShard))
+	} else {
+		for shard := 0; shard < 256; shard++ {
+			path := fmt.Sprintf("%s/data/%s/%d", *optPathPrefix, *optNamespace, shard)
+			if _, err := os.Stat(path); os.IsNotExist(err) {
+				continue
+			}
+			shards = append(shards, uint32(shard))
+		}
+	}
+
+	iterPoolPoolOpts := pool.NewObjectPoolOptions().SetSize(1024)
+	iterPool := encoding.NewReaderIteratorPool(iterPoolPoolOpts)
+	encodingOpts := encoding.NewOptions().SetReaderIteratorPool(iterPool)
+	iterPool.Init(func(r io.Reader, descr namespace.SchemaDescr) encoding.ReaderIterator {
+		return m3tsz.NewReaderIterator(r, m3tsz.DefaultIntOptimizationEnabled, encodingOpts)
+	})
+
+	for iteration := 0; iteration < iterations; iteration++ {
+		for _, shard := range shards {
+			reader, err := fs.NewReader(bytesPool, fsOpts)
+			if err != nil {
+				log.Fatalf("could not create new reader: %v", err)
+			}
+
+			openOpts := fs.DataReaderOpenOptions{
+				Identifier: fs.FileSetFileIdentifier{
+					Namespace:   ident.StringID(*optNamespace),
+					Shard:       shard,
+					BlockStart:  time.Unix(0, *optBlockstart),
+					VolumeIndex: int(*volume),
+				},
+				FileSetType:    fileSetType,
+				OrderedByIndex: true,
+			}
+
+			err = reader.Open(openOpts)
+			if err != nil {
+				log.Fatalf("unable to open reader: %v", err)
+			}
+
+			var (
+				frameSize = xtime.UnixNano(*optTilesize) * xtime.UnixNano(time.Minute)
+				start     = xtime.UnixNano(*optBlockstart)
+			)
+
+			opts := tile.Options{
+				FrameSize:          frameSize,
+				Start:              start,
+				UseArrow:           useArrow,
+				ReaderIteratorPool: iterPool,
+			}
+
+			crossBlockReader, err := fs.NewCrossBlockReader([]fs.DataFileSetReader{reader}, iOpts)
+			if err != nil {
+				fmt.Println("error creating CrossBlockReader", err)
+				return
+			}
+
+			it, err := tile.NewSeriesBlockIterator(crossBlockReader, opts)
+			if err != nil {
+				fmt.Println("error creating block iterator", err)
+				return
+			}
+
+			for it.Next() {
+				frameIter := it.Current()
+				for frameIter.Next() {
+					// No-op for now for better benchmarking underlying iters.
+				}
+			}
+
+			if err := it.Close(); err != nil {
+				fmt.Println("iterator close error:", err)
+			}
+
+			if err := it.Err(); err != nil {
+				fmt.Println("series error:", err)
+			}
+
+			if err := reader.Close(); err != nil {
+				fmt.Println("reader close error:", err)
+			}
+		}
+	}
+
+	frameSize := time.Duration(*optTilesize) * time.Minute
+	if useArrow {
+		fmt.Printf("Using arrow buffers\nIterations: %d\nFrameSize: %v\nTook: %v\n",
+			iterations, frameSize, time.Since(readStart))
+	} else {
+		fmt.Printf("Using flat buffers\nIterations: %d\nFrameSize: %v\nTook: %v\n",
+			iterations, frameSize, time.Since(readStart))
+	}
+}