[large-tiles] Large tiles aggregation flow (#2617)

m3db · Sep 14, 2020 · 876cee8 · 876cee8
1 parent d38ff54
commit 876cee8
Show file tree

Hide file tree

Showing 29 changed files with 2,769 additions and 195 deletions.
diff --git a/src/dbnode/generated/mocks/generate.go b/src/dbnode/generated/mocks/generate.go
@@ -20,7 +20,7 @@
 
 // mockgen rules for generating mocks for exported interfaces (reflection mode)
 
-//go:generate sh -c "mockgen -package=fs $PACKAGE/src/dbnode/persist/fs DataFileSetWriter,DataFileSetReader,DataFileSetSeeker,IndexFileSetWriter,IndexFileSetReader,IndexSegmentFileSetWriter,IndexSegmentFileSet,IndexSegmentFile,SnapshotMetadataFileWriter,DataFileSetSeekerManager,ConcurrentDataFileSetSeeker,MergeWith | genclean -pkg $PACKAGE/src/dbnode/persist/fs -out $GOPATH/src/$PACKAGE/src/dbnode/persist/fs/fs_mock.go"
+//go:generate sh -c "mockgen -package=fs $PACKAGE/src/dbnode/persist/fs DataFileSetWriter,DataFileSetReader,DataFileSetSeeker,IndexFileSetWriter,IndexFileSetReader,IndexSegmentFileSetWriter,IndexSegmentFileSet,IndexSegmentFile,SnapshotMetadataFileWriter,DataFileSetSeekerManager,ConcurrentDataFileSetSeeker,MergeWith,CrossBlockReader | genclean -pkg $PACKAGE/src/dbnode/persist/fs -out $GOPATH/src/$PACKAGE/src/dbnode/persist/fs/fs_mock.go"
 //go:generate sh -c "mockgen -package=xio $PACKAGE/src/dbnode/x/xio SegmentReader,SegmentReaderPool | genclean -pkg $PACKAGE/src/dbnode/x/xio -out $GOPATH/src/$PACKAGE/src/dbnode/x/xio/io_mock.go"
 //go:generate sh -c "mockgen -package=digest -destination=$GOPATH/src/$PACKAGE/src/dbnode/digest/digest_mock.go $PACKAGE/src/dbnode/digest ReaderWithDigest"
 //go:generate sh -c "mockgen -package=series $PACKAGE/src/dbnode/storage/series DatabaseSeries,QueryableBlockRetriever | genclean -pkg $PACKAGE/src/dbnode/storage/series -out $GOPATH/src/$PACKAGE/src/dbnode/storage/series/series_mock.go"

diff --git a/src/dbnode/generated/thrift/rpc.thrift b/src/dbnode/generated/thrift/rpc.thrift
@@ -63,6 +63,8 @@ service Node {
 	void repair() throws (1: Error err)
 	TruncateResult truncate(1: TruncateRequest req) throws (1: Error err)
 
+	AggregateTilesResult aggregateTiles(1: AggregateTilesRequest req) throws (1: Error err)
+
 	// Management endpoints
 	NodeHealthResult health() throws (1: Error err)
 	// NB: bootstrapped is for use with cluster management tools like k8s.
@@ -216,8 +218,8 @@ struct Block {
 }
 
 struct Tag {
-  1: required string name
-  2: required string value
+	1: required string name
+	2: required string value
 }
 
 struct FetchBlocksMetadataRawV2Request {
@@ -455,41 +457,55 @@ struct QueryResultElement {
 }
 
 struct TermQuery {
-  1: required string field
-  2: required string term
+	1: required string field
+	2: required string term
 }
 
 struct RegexpQuery {
-  1: required string field
-  2: required string regexp
+	1: required string field
+	2: required string regexp
 }
 
 struct NegationQuery {
-  1: required Query query
+	1: required Query query
 }
 
 struct ConjunctionQuery {
-  1: required list<Query> queries
+	1: required list<Query> queries
 }
 
 struct DisjunctionQuery {
-  1: required list<Query> queries
+	1: required list<Query> queries
 }
 
 struct AllQuery {}
 
 struct FieldQuery {
-  1: required string field
+	1: required string field
 }
 
 struct Query {
-  1: optional TermQuery        term
-  2: optional RegexpQuery      regexp
-  3: optional NegationQuery    negation
-  4: optional ConjunctionQuery conjunction
-  5: optional DisjunctionQuery disjunction
-  6: optional AllQuery         all
-  7: optional FieldQuery       field
+	1: optional TermQuery        term
+	2: optional RegexpQuery      regexp
+	3: optional NegationQuery    negation
+	4: optional ConjunctionQuery conjunction
+	5: optional DisjunctionQuery disjunction
+	6: optional AllQuery         all
+	7: optional FieldQuery       field
+}
+
+struct AggregateTilesRequest {
+	1: required string sourceNamespace
+	2: required string targetNamespace
+	3: required i64 rangeStart
+	4: required i64 rangeEnd
+	5: required string step
+	6: bool removeResets // FIXME: temporary, remove after metrics type metadata is available.
+	7: optional TimeType rangeType = TimeType.UNIX_SECONDS
+}
+
+struct AggregateTilesResult {
+	1: required i64 processedBlockCount
 }
 
 struct DebugProfileStartRequest {