add basic pk deduplication (matrixorigin#1)

Signed-off-by: asuka <[email protected]>

go.mod

@@ -4,6 +4,7 @@ go 1.17
 
 require (
 	github.com/BurntSushi/toml v1.0.0
+	github.com/FastFilter/xorfilter v0.1.1
 	github.com/RoaringBitmap/roaring v0.9.4
 	github.com/axiomhq/hyperloglog v0.0.0-20220105174342-98591331716a
 	github.com/bxcodec/faker/v3 v3.8.0
@@ -20,6 +21,7 @@ require (
 	github.com/matrixorigin/simdcsv v0.0.0-20210926114300-591bf748a770
 	github.com/panjf2000/ants/v2 v2.4.6
 	github.com/pierrec/lz4 v2.6.1+incompatible
+	github.com/plar/go-adaptive-radix-tree v1.0.4
 	github.com/prashantv/gostub v1.1.0
 	github.com/sirupsen/logrus v1.8.1
 	github.com/smartystreets/assertions v1.2.0

go.sum

@@ -57,6 +57,8 @@ github.com/CloudyKit/jet v2.1.3-0.20180809161101-62edd43e4f88+incompatible/go.mo
 github.com/DataDog/zstd v1.4.5/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo=
 github.com/DataDog/zstd v1.5.0 h1:+K/VEwIAaPcHiMtQvpLD4lqW7f0Gk3xdYZmI1hD+CXo=
 github.com/DataDog/zstd v1.5.0/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw=
+github.com/FastFilter/xorfilter v0.1.1 h1:b0x4ms1+is6YROG+v+0II/2ogCcKqKsDODbJFKzf2q4=
+github.com/FastFilter/xorfilter v0.1.1/go.mod h1:RB6+tbWbRN163V4y7z10tNfZec6n1oTsOElP0Tu5hzU=
 github.com/Joker/hpp v1.0.0/go.mod h1:8x5n+M1Hp5hC0g8okX3sR3vFQwynaX/UgSOM9MeBKzY=
 github.com/Joker/jade v1.0.1-0.20190614124447-d475f43051e7/go.mod h1:6E6s8o2AE4KhCrqr6GRJjdC/gNfTdxkIXvuGZZda2VM=
 github.com/K-Phoen/grabana v0.4.1/go.mod h1:dNuhqBkLkjJNDtWMq/UEaX8iRo7otgIQF9PLH0OIE40=
@@ -561,6 +563,8 @@ github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/profile v1.2.1/go.mod h1:hJw3o1OdXxsrSjjVksARp5W95eeEaEfptyVZyv6JUPA=
+github.com/plar/go-adaptive-radix-tree v1.0.4 h1:Ucd8R6RH2E7RW8ZtDKrsWyOD3paG2qqJO0I20WQ8oWQ=
+github.com/plar/go-adaptive-radix-tree v1.0.4/go.mod h1:Ot8d28EII3i7Lv4PSvBlF8ejiD/CtRYDuPsySJbSaK8=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI=

pkg/vm/engine/tae/common/compare.go

@@ -17,50 +17,118 @@ package common
 import (
 	"bytes"
 	"github.com/matrixorigin/matrixone/pkg/container/types"
-	"github.com/matrixorigin/matrixone/pkg/logutil"
+	"github.com/matrixorigin/matrixone/pkg/vm/engine/tae/index/common/errors"
 )
 
-func CompareInterface(a, b interface{}) int64 {
-	if av, ok := a.(int8); ok {
-		return int64(av - b.(int8))
+func CompareGeneric(a, b interface{}, t types.Type) int {
+	switch t.Oid {
+	case types.T_int8:
+		if a.(int8) > b.(int8) {
+			return 1
+		} else if a.(int8) < b.(int8) {
+			return -1
+		} else {
+			return 0
+		}
+	case types.T_int16:
+		if a.(int16) > b.(int16) {
+			return 1
+		} else if a.(int16) < b.(int16) {
+			return -1
+		} else {
+			return 0
+		}
+	case types.T_int32:
+		if a.(int32) > b.(int32) {
+			return 1
+		} else if a.(int32) < b.(int32) {
+			return -1
+		} else {
+			return 0
+		}
+	case types.T_int64:
+		if a.(int64) > b.(int64) {
+			return 1
+		} else if a.(int64) < b.(int64) {
+			return -1
+		} else {
+			return 0
+		}
+	case types.T_uint8:
+		if a.(uint8) > b.(uint8) {
+			return 1
+		} else if a.(uint8) < b.(uint8) {
+			return -1
+		} else {
+			return 0
+		}
+	case types.T_uint16:
+		if a.(uint16) > b.(uint16) {
+			return 1
+		} else if a.(uint16) < b.(uint16) {
+			return -1
+		} else {
+			return 0
+		}
+	case types.T_uint32:
+		if a.(uint32) > b.(uint32) {
+			return 1
+		} else if a.(uint32) < b.(uint32) {
+			return -1
+		} else {
+			return 0
+		}
+	case types.T_uint64:
+		if a.(uint64) > b.(uint64) {
+			return 1
+		} else if a.(uint64) < b.(uint64) {
+			return -1
+		} else {
+			return 0
+		}
+	case types.T_float32:
+		if a.(float32) > b.(float32) {
+			return 1
+		} else if a.(float32) < b.(float32) {
+			return -1
+		} else {
+			return 0
+		}
+	case types.T_float64:
+		if a.(float64) > b.(float64) {
+			return 1
+		} else if a.(float64) < b.(float64) {
+			return -1
+		} else {
+			return 0
+		}
+	case types.T_date:
+		if a.(types.Date) > b.(types.Date) {
+			return 1
+		} else if a.(types.Date) < b.(types.Date) {
+			return -1
+		} else {
+			return 0
+		}
+	case types.T_datetime:
+		if a.(types.Datetime) > b.(types.Datetime) {
+			return 1
+		} else if a.(types.Datetime) < b.(types.Datetime) {
+			return -1
+		} else {
+			return 0
+		}
+	case types.T_char, types.T_varchar:
+		res := bytes.Compare(a.([]byte), b.([]byte))
+		if res > 0 {
+			return 1
+		} else if res < 0 {
+			return -1
+		} else {
+			return 0
+		}
+	default:
+		panic(errors.ErrTypeNotSupported)
 	}
-	if av, ok := a.(int16); ok {
-		return int64(av - b.(int16))
-	}
-	if av, ok := a.(int32); ok {
-		return int64(av - b.(int32))
-	}
-	if av, ok := a.(int64); ok {
-		return av - b.(int64)
-	}
-	if av, ok := a.(uint8); ok {
-		return int64(av - b.(uint8))
-	}
-	if av, ok := a.(uint16); ok {
-		return int64(av - b.(uint16))
-	}
-	if av, ok := a.(uint32); ok {
-		return int64(av - b.(uint32))
-	}
-	if av, ok := a.(uint64); ok {
-		return int64(av - b.(uint64))
-	}
-	if av, ok := a.(float32); ok {
-		return int64(av - b.(float32))
-	}
-	if av, ok := a.(float64); ok {
-		return int64(av - b.(float64))
-	}
-	if av, ok := a.(types.Date); ok {
-		return int64(av - b.(types.Date))
-	}
-	if av, ok := a.(types.Datetime); ok {
-		return int64(av - b.(types.Datetime))
-	}
-	if av, ok := a.([]byte); ok {
-		return int64(bytes.Compare(av, b.([]byte)))
-	}
-	logutil.Infof("%+v\n%+v\n", a, b)
-	panic("invalid type")
 }