Fix/deterministic ordering (#308)

* added several new tests for the sqlanalyzer

* fixed various bugs in the database engine, and improved unit tests

* added tool to analyze select cores.  this is needed to support determinism for CTEs

* added the ability to predict attribute types, and generate a table

* rebased main, merged proto

* minor additions

* fixed comment typo

* fixed failing unit test

* fixed gavins feedback

* fixed jons feedback

go.mod

@@ -13,6 +13,7 @@ require (
 	github.com/decred/dcrd/certgen v1.1.2
 	github.com/dgraph-io/badger/v3 v3.2103.5
 	github.com/ethereum/go-ethereum v1.12.0
+	github.com/google/go-cmp v0.5.9
 	github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.0.0-rc.5
 	github.com/grpc-ecosystem/grpc-gateway/v2 v2.12.0
 	github.com/joho/godotenv v1.5.1
@@ -117,7 +118,6 @@ require (
 	github.com/google/btree v1.1.2 // indirect
 	github.com/google/flatbuffers v1.12.1 // indirect
 	github.com/google/gnostic v0.5.7-v3refs // indirect
-	github.com/google/go-cmp v0.5.9 // indirect
 	github.com/google/gofuzz v1.2.0 // indirect
 	github.com/google/orderedcode v0.0.1 // indirect
 	github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect

pkg/engine/dataset/data_test.go

@@ -38,11 +38,11 @@ var (
 						},
 						{
 							Type:  types.MIN_LENGTH,
-							Value: 5,
+							Value: "5",
 						},
 						{
 							Type:  types.MAX_LENGTH,
-							Value: 32,
+							Value: "32",
 						},
 					},
 				},
@@ -55,11 +55,11 @@ var (
 						},
 						{
 							Type:  types.MIN,
-							Value: 13,
+							Value: "13",
 						},
 						{
 							Type:  types.MAX,
-							Value: 200,
+							Value: "200",
 						},
 					},
 				},

pkg/engine/db/db.go

@@ -13,14 +13,18 @@ import (
 	"io"
 	"sync"
 
+	"github.com/kwilteam/kwil-db/pkg/log"
+	"go.uber.org/zap"
+
 	"github.com/kwilteam/kwil-db/pkg/engine/sqlanalyzer"
-	"github.com/kwilteam/kwil-db/pkg/engine/sqlparser"
 	"github.com/kwilteam/kwil-db/pkg/sql"
 )
 
 type DB struct {
 	Sqldb SqlDB
 
+	log log.Logger
+
 	// caches metadata from QueryUnsafe.
 	// this is a really bad practice, but works.
 	// essentially, we cache the metadata the first time it is retrieved, during schema
@@ -30,6 +34,23 @@ type DB struct {
 	mu sync.RWMutex
 }
 
+// NewDB wraps the provided database with a DB abstraction layer.
+// It will initialize the metadata table if it does not exist.
+func NewDB(ctx context.Context, sqldb SqlDB, opts ...DBOpt) (*DB, error) {
+	db := &DB{
+		Sqldb:         sqldb,
+		metadataCache: make(map[metadataType][]*metadata),
+		log:           log.NewNoOp(),
+	}
+
+	err := db.initMetadataTable(ctx)
+	if err != nil {
+		return nil, err
+	}
+
+	return db, nil
+}
+
 func (d *DB) Close() error {
 	return d.Sqldb.Close()
 }
@@ -39,41 +60,28 @@ func (d *DB) Delete() error {
 }
 
 func (d *DB) Prepare(ctx context.Context, query string) (*PreparedStatement, error) {
-	ast, err := sqlparser.Parse(query)
-	if err != nil {
-		return nil, err
-	}
-
 	tables, err := d.ListTables(ctx)
 	if err != nil {
 		return nil, err
 	}
 
-	err = sqlanalyzer.ApplyRules(ast, sqlanalyzer.AllRules, &sqlanalyzer.RuleMetadata{
+	analyzed, err := sqlanalyzer.ApplyRules(query, sqlanalyzer.AllRules, &sqlanalyzer.RuleMetadata{
 		Tables: tables,
 	})
 	if err != nil {
+		d.log.Debug("failed to analyze query", zap.String("query", query), zap.Error(err))
 		return nil, err
 	}
 
-	mutativity, err := sqlanalyzer.IsMutative(ast)
-	if err != nil {
-		return nil, err
-	}
-
-	generatedSql, err := ast.ToSQL()
-	if err != nil {
-		return nil, err
-	}
-
-	prepStmt, err := d.Sqldb.Prepare(generatedSql)
+	prepStmt, err := d.Sqldb.Prepare(analyzed.Statement())
 	if err != nil {
+		d.log.Error("failed to prepare analyzed statement", zap.String("query", query), zap.Error(err))
 		return nil, err
 	}
 
 	return &PreparedStatement{
 		Statement: prepStmt,
-		mutative:  mutativity,
+		mutative:  analyzed.Mutative(),
 	}, nil
 }
 
@@ -85,24 +93,18 @@ func (d *DB) Savepoint() (sql.Savepoint, error) {
 	return d.Sqldb.Savepoint()
 }
 
-func NewDB(ctx context.Context, sqldb SqlDB) (*DB, error) {
-	db := &DB{
-		Sqldb:         sqldb,
-		metadataCache: make(map[metadataType][]*metadata),
-	}
-
-	err := db.initMetadataTable(ctx)
-	if err != nil {
-		return nil, err
-	}
-
-	return db, nil
-}
-
 func (d *DB) CreateSession() (sql.Session, error) {
 	return d.Sqldb.CreateSession()
 }
 
 func (d *DB) ApplyChangeset(changeset io.Reader) error {
 	return d.Sqldb.ApplyChangeset(changeset)
 }
+
+type DBOpt func(*DB)
+
+func WithLogger(logger log.Logger) DBOpt {
+	return func(db *DB) {
+		db.log = logger
+	}
+}