diff --git a/cmd/kwild/server/build.go b/cmd/kwild/server/build.go index fb6fedef6..c850891cc 100644 --- a/cmd/kwild/server/build.go +++ b/cmd/kwild/server/build.go @@ -12,6 +12,7 @@ import ( "time" "github.com/kwilteam/kwil-db/cmd/kwild/config" + extensions "github.com/kwilteam/kwil-db/extensions/actions" "github.com/kwilteam/kwil-db/internal/abci" "github.com/kwilteam/kwil-db/internal/abci/cometbft" "github.com/kwilteam/kwil-db/internal/abci/snapshots" @@ -195,9 +196,10 @@ func buildDatasetsModule(d *coreDependencies, eng datasets.Engine, accs datasets } func buildEngine(d *coreDependencies, a *sessions.AtomicCommitter) *engine.Engine { - extensions, err := connectExtensions(d.ctx, d.cfg.AppCfg.ExtensionEndpoints) - if err != nil { - failBuild(err, "failed to connect to extensions") + extensions := extensions.GetRegisteredExtensions() + + for _, ext := range extensions { + d.log.Debug("registered extension", zap.String("name", ext.Name())) } sqlCommitRegister := &sqlCommittableRegister{ diff --git a/cmd/kwild/server/utils.go b/cmd/kwild/server/utils.go index cfb2bbd18..4ff2cc7b2 100644 --- a/cmd/kwild/server/utils.go +++ b/cmd/kwild/server/utils.go @@ -4,7 +4,6 @@ import ( "bytes" "context" "errors" - "fmt" "strings" "github.com/kwilteam/kwil-db/core/log" @@ -26,28 +25,6 @@ import ( cmttypes "github.com/cometbft/cometbft/types" ) -// connectExtensions connects to the provided extension urls. -func connectExtensions(ctx context.Context, urls []string) (map[string]*extensions.Extension, error) { - exts := make(map[string]*extensions.Extension, len(urls)) - - for _, url := range urls { - ext := extensions.New(url) - err := ext.Connect(ctx) - if err != nil { - return nil, fmt.Errorf("failed to connect extension '%s': %w", ext.Name(), err) - } - - _, ok := exts[ext.Name()] - if ok { - return nil, fmt.Errorf("duplicate extension name: %s", ext.Name()) - } - - exts[ext.Name()] = ext - } - - return exts, nil -} - func adaptExtensions(exts map[string]*extensions.Extension) map[string]engine.ExtensionInitializer { adapted := make(map[string]engine.ExtensionInitializer, len(exts)) diff --git a/extensions/actions/builder.go b/extensions/actions/builder.go new file mode 100644 index 000000000..cc577a6ef --- /dev/null +++ b/extensions/actions/builder.go @@ -0,0 +1,51 @@ +package extensions + +import "context" + +type extensionBuilder struct { + extension *Extension +} + +// ExtensionBuilder is the interface for creating an extension server +type ExtensionBuilder interface { + // WithMethods specifies the methods that should be provided + // by the extension + WithMethods(map[string]MethodFunc) ExtensionBuilder + // WithInitializer is a function that initializes a new extension instance. + WithInitializer(InitializeFunc) ExtensionBuilder + // Named specifies the name of the extensions. + Named(string) ExtensionBuilder + + // Build creates the extensions + Build() (*Extension, error) +} + +func Builder() ExtensionBuilder { + return &extensionBuilder{ + extension: &Extension{ + methods: make(map[string]MethodFunc), + initializeFunc: func(ctx context.Context, metadata map[string]string) (map[string]string, error) { + return metadata, nil + }, + }, + } +} + +func (b *extensionBuilder) Named(name string) ExtensionBuilder { + b.extension.name = name + return b +} + +func (b *extensionBuilder) WithMethods(methods map[string]MethodFunc) ExtensionBuilder { + b.extension.methods = methods + return b +} + +func (b *extensionBuilder) WithInitializer(fn InitializeFunc) ExtensionBuilder { + b.extension.initializeFunc = fn + return b +} + +func (b *extensionBuilder) Build() (*Extension, error) { + return b.extension, nil +} diff --git a/extensions/actions/extension.go b/extensions/actions/extension.go index 48251abed..b5587842e 100644 --- a/extensions/actions/extension.go +++ b/extensions/actions/extension.go @@ -3,70 +3,27 @@ package extensions import ( "context" "fmt" - "strings" ) type Extension struct { - name string - url string - methods map[string]struct{} - - client ExtensionClient -} - -func (e *Extension) Name() string { - return e.name + name string + methods map[string]MethodFunc + initializeFunc InitializeFunc } -// New connects to the given extension, and attempts to configure it with the given config. -// If the extension is not available, an error is returned. -func New(url string) *Extension { - return &Extension{ - name: "", - url: url, - methods: make(map[string]struct{}), +func (e *Extension) execute(ctx *ExecutionContext, method string, args ...*ScalarValue) ([]*ScalarValue, error) { + methodFn, ok := e.methods[method] + if !ok { + return nil, fmt.Errorf("method %s not found", method) } -} - -func (e *Extension) Connect(ctx context.Context) error { - extClient, err := ConnectFunc.Connect(ctx, e.url) - if err != nil { - return fmt.Errorf("failed to connect to extension at %s: %w", e.url, err) - } - - name, err := extClient.GetName(ctx) - if err != nil { - return fmt.Errorf("failed to get extension name: %w", err) - } - - e.name = name - e.client = extClient - err = e.loadMethods(ctx) - if err != nil { - return fmt.Errorf("failed to load methods for extension %s: %w", e.name, err) - } - - return nil + return methodFn(ctx, args...) } -func (e *Extension) loadMethods(ctx context.Context) error { - methodList, err := e.client.ListMethods(ctx) - if err != nil { - return fmt.Errorf("failed to list methods for extension '%s' at target '%s': %w", e.name, e.url, err) - } - - e.methods = make(map[string]struct{}) - for _, method := range methodList { - lowerName := strings.ToLower(method) - - _, ok := e.methods[lowerName] - if ok { - return fmt.Errorf("extension %s has duplicate method %s. this is an issue with the extension", e.name, lowerName) - } - - e.methods[lowerName] = struct{}{} - } +func (e *Extension) initialize(ctx context.Context, metadata map[string]string) (map[string]string, error) { + return e.initializeFunc(ctx, metadata) +} - return nil +func (e *Extension) Name() string { + return e.name } diff --git a/extensions/actions/extension_registry.go b/extensions/actions/extension_registry.go new file mode 100644 index 000000000..a88e66afb --- /dev/null +++ b/extensions/actions/extension_registry.go @@ -0,0 +1,19 @@ +package extensions + +import "strings" + +var registeredExtensions = make(map[string]*Extension) + +func RegisterExtension(name string, ext *Extension) error { + name = strings.ToLower(name) + if _, ok := registeredExtensions[name]; ok { + panic("extension of same name already registered: " + name) + } + + registeredExtensions[name] = ext + return nil +} + +func GetRegisteredExtensions() map[string]*Extension { + return registeredExtensions +} diff --git a/extensions/actions/extension_test.go b/extensions/actions/extension_test.go index 7da1c4ca2..8acec5dec 100644 --- a/extensions/actions/extension_test.go +++ b/extensions/actions/extension_test.go @@ -1,3 +1,5 @@ +//go:build ext_test + package extensions_test import ( @@ -5,33 +7,55 @@ import ( "testing" extensions "github.com/kwilteam/kwil-db/extensions/actions" + "github.com/stretchr/testify/assert" ) -// TODO: these tests are pretty bad. -// since this is a prototype, and the package is simple, this is good for now. func Test_Extensions(t *testing.T) { ctx := context.Background() - ext := extensions.New("local:8080") + math_ext, err := extensions.NewMathExtension() + assert.NoError(t, err) - err := ext.Connect(ctx) - if err != nil { - t.Fatal(err) + // Create an instance with incorrect metadata, instance should be created with default metadata: up + incorrectMetadata := map[string]string{ + "roundoff": "up", } + instance1, err := math_ext.CreateInstance(ctx, incorrectMetadata) + assert.NoError(t, err) + assert.NotNil(t, instance1) - instance, err := ext.CreateInstance(ctx, map[string]string{ - "token_address": "0x12345", - "wallet_address": "0xabcd", - }) - if err != nil { - t.Fatal(err) - } + // Verify that the metadata was updated with the default value "round: up" + updatedMetadata := instance1.Metadata() + assert.Equal(t, "up", updatedMetadata["round"]) - results, err := instance.Execute(ctx, "method1", "0x12345") - if err != nil { - t.Fatal(err) + // Create an instance with correct metadata + correctMetadata := map[string]string{ + "round": "down", } - if len(results) != 2 { - t.Fatalf("expected 2 results, got %d", len(results)) - } + instance2, err := math_ext.CreateInstance(ctx, correctMetadata) + assert.NoError(t, err) + + // test that the instance has the correct name + name := instance2.Name() + assert.Equal(t, "math", name) + + // Execute an available method + // Instance1: round: up + result, err := instance1.Execute(ctx, "divide", 1, 2) + assert.NoError(t, err) + assert.Equal(t, int64(1), result[0]) + + // Instance2: round: down + result, err = instance2.Execute(ctx, "divide", 1, 2) + assert.NoError(t, err) + assert.Equal(t, int64(0), result[0]) + + // Check that the methods are case insensitive + result, err = instance2.Execute(ctx, "ADD", 1, 2) + assert.NoError(t, err) + assert.Equal(t, int64(3), result[0]) + + // Execute an unavailable method + _, err = instance2.Execute(ctx, "modulus", 1, 2) + assert.Error(t, err) } diff --git a/extensions/actions/instance.go b/extensions/actions/instance.go index c8bc926f6..d900d360a 100644 --- a/extensions/actions/instance.go +++ b/extensions/actions/instance.go @@ -4,8 +4,6 @@ import ( "context" "fmt" "strings" - - "github.com/kwilteam/kwil-extensions/types" ) // An instance is a single instance of an extension. @@ -16,18 +14,18 @@ import ( type Instance struct { metadata map[string]string - extenstion *Extension + extension *Extension } func (e *Extension) CreateInstance(ctx context.Context, metadata map[string]string) (*Instance, error) { - newMetadata, err := e.client.Initialize(ctx, metadata) + newMetadata, err := e.initialize(ctx, metadata) if err != nil { return nil, err } return &Instance{ - metadata: newMetadata, - extenstion: e, + metadata: newMetadata, + extension: e, }, nil } @@ -36,18 +34,34 @@ func (i *Instance) Metadata() map[string]string { } func (i *Instance) Name() string { - return i.extenstion.name + return i.extension.name } func (i *Instance) Execute(ctx context.Context, method string, args ...any) ([]any, error) { - lowerMethod := strings.ToLower(method) - _, ok := i.extenstion.methods[lowerMethod] - if !ok { - return nil, fmt.Errorf("method '%s' is not available for extension '%s' at target '%s'", lowerMethod, i.extenstion.name, i.extenstion.url) + var encodedArgs []*ScalarValue + for _, arg := range args { + scalarVal, err := NewScalarValue(arg) + if err != nil { + return nil, fmt.Errorf("error encoding argument: %s", err.Error()) + } + + encodedArgs = append(encodedArgs, scalarVal) } - return i.extenstion.client.CallMethod(&types.ExecutionContext{ + execCtx := &ExecutionContext{ Ctx: ctx, Metadata: i.metadata, - }, lowerMethod, args...) + } + + lowerMethod := strings.ToLower(method) + results, err := i.extension.execute(execCtx, lowerMethod, encodedArgs...) + if err != nil { + return nil, err + } + + var outputs []any + for _, result := range results { + outputs = append(outputs, result.Value) + } + return outputs, nil } diff --git a/extensions/actions/interface.go b/extensions/actions/interface.go deleted file mode 100644 index 2adf262c5..000000000 --- a/extensions/actions/interface.go +++ /dev/null @@ -1,31 +0,0 @@ -package extensions - -import ( - "context" - - "github.com/kwilteam/kwil-extensions/client" - "github.com/kwilteam/kwil-extensions/types" -) - -type ExtensionClient interface { - CallMethod(execCtx *types.ExecutionContext, method string, args ...any) ([]any, error) - Close() error - Initialize(ctx context.Context, metadata map[string]string) (map[string]string, error) - GetName(ctx context.Context) (string, error) - ListMethods(ctx context.Context) ([]string, error) -} - -type Connecter interface { - Connect(ctx context.Context, target string, opts ...client.ClientOpt) (ExtensionClient, error) -} - -type extensionConnectFunc func(ctx context.Context, target string, opts ...client.ClientOpt) (*client.ExtensionClient, error) - -func (e extensionConnectFunc) Connect(ctx context.Context, target string, opts ...client.ClientOpt) (ExtensionClient, error) { - return e(ctx, target, opts...) -} - -var ( - // this can be overridden for testing - ConnectFunc Connecter = extensionConnectFunc(client.NewExtensionClient) -) diff --git a/extensions/actions/math.go b/extensions/actions/math.go new file mode 100644 index 000000000..d2be5e6ee --- /dev/null +++ b/extensions/actions/math.go @@ -0,0 +1,188 @@ +//go:build ext_math || ext_test + +package extensions + +import ( + "context" + "fmt" + "math/big" +) + +func init() { + ext, err := NewMathExtension() + if err != nil { + panic(err) + } + + err = RegisterExtension("math", ext) + if err != nil { + panic(err) + } +} + +type MathExtension struct{} + +func NewMathExtension() (*Extension, error) { + mathExt := &MathExtension{} + methods := map[string]MethodFunc{ + "add": mathExt.add, + "subtract": mathExt.subtract, + "multiply": mathExt.multiply, + "divide": mathExt.divide, + } + + ext, err := Builder().Named("math").WithMethods(methods).WithInitializer(initialize).Build() + if err != nil { + return nil, err + } + return ext, nil +} + +func (e *MathExtension) Name() string { + return "math" +} + +// this initialize function checks if round is set. If not, it sets it to "up" +func initialize(ctx context.Context, metadata map[string]string) (map[string]string, error) { + _, ok := metadata["round"] + if !ok { + metadata["round"] = "up" + } + + roundVal := metadata["round"] + if roundVal != "up" && roundVal != "down" { + return nil, fmt.Errorf("round must be either 'up' or 'down'. default is 'up'") + } + + return metadata, nil +} + +func (e *MathExtension) add(ctx *ExecutionContext, values ...*ScalarValue) ([]*ScalarValue, error) { + if len(values) != 2 { + return nil, fmt.Errorf("expected 2 values for method Add, got %d", len(values)) + } + + val0Int, err := values[0].Int() + if err != nil { + return nil, fmt.Errorf("failed to convert value to int: %w. \nreceived value: %v", err, val0Int) + } + + val1Int, err := values[1].Int() + if err != nil { + return nil, fmt.Errorf("failed to convert value to int: %w. \nreceived value: %v", err, val1Int) + } + + return encodeScalarValues(val0Int + val1Int) +} + +func (e *MathExtension) subtract(ctx *ExecutionContext, values ...*ScalarValue) ([]*ScalarValue, error) { + if len(values) != 2 { + return nil, fmt.Errorf("expected 2 values for method Subtract, got %d", len(values)) + } + + val0Int, err := values[0].Int() + if err != nil { + return nil, fmt.Errorf("failed to convert value to int: %w. \nreceived value: %v", err, val0Int) + } + + val1Int, err := values[1].Int() + if err != nil { + return nil, fmt.Errorf("failed to convert value to int: %w. \nreceived value: %v", err, val1Int) + } + + return encodeScalarValues(val0Int - val1Int) +} + +func (e *MathExtension) multiply(ctx *ExecutionContext, values ...*ScalarValue) ([]*ScalarValue, error) { + if len(values) != 2 { + return nil, fmt.Errorf("expected 2 values for method Multiply, got %d", len(values)) + } + + val0Int, err := values[0].Int() + if err != nil { + return nil, fmt.Errorf("failed to convert value to int: %w. \nreceived value: %v", err, val0Int) + } + + val1Int, err := values[1].Int() + if err != nil { + return nil, fmt.Errorf("failed to convert value to int: %w. \nreceived value: %v", err, val1Int) + } + + return encodeScalarValues(val0Int * val1Int) +} + +func (e *MathExtension) divide(ctx *ExecutionContext, values ...*ScalarValue) ([]*ScalarValue, error) { + if len(values) != 2 { + return nil, fmt.Errorf("expected 2 values for method Divide, got %d", len(values)) + } + + val0Int, err := values[0].Int() + if err != nil { + return nil, fmt.Errorf("failed to convert value to int: %w. \nreceived value: %v", err, val0Int) + } + + val1Int, err := values[1].Int() + if err != nil { + return nil, fmt.Errorf("failed to convert value to int: %w. \nreceived value: %v", err, val1Int) + } + + bigVal1 := newBigFloat(float64(val0Int)) + + bigVal2 := newBigFloat(float64(val1Int)) + + result := new(big.Float).Quo(bigVal1, bigVal2) + + var IntResult *big.Int + if ctx.Metadata["round"] == "up" { + IntResult = roundUp(result) + } else { + IntResult = roundDown(result) + } + + return encodeScalarValues(IntResult.Int64()) +} + +// roundUp takes a big.Float and returns a new big.Float rounded up. +func roundUp(f *big.Float) *big.Int { + c := new(big.Float).SetPrec(precision).Copy(f) + r := new(big.Int) + f.Int(r) + + if c.Sub(c, new(big.Float).SetPrec(precision).SetInt(r)).Sign() > 0 { + r.Add(r, big.NewInt(1)) + } + + return r +} + +// roundDown takes a big.Float and returns a new big.Float rounded down. +func roundDown(f *big.Float) *big.Int { + r := new(big.Int) + f.Int(r) + + return r +} + +func encodeScalarValues(values ...any) ([]*ScalarValue, error) { + scalarValues := make([]*ScalarValue, len(values)) + for i, v := range values { + scalarValue, err := NewScalarValue(v) + if err != nil { + return nil, err + } + + scalarValues[i] = scalarValue + } + + return scalarValues, nil +} + +const ( + precision = 128 +) + +func newBigFloat(num float64) *big.Float { + bg := new(big.Float).SetPrec(precision) + + return bg.SetFloat64(num) +} diff --git a/extensions/actions/methods.go b/extensions/actions/methods.go new file mode 100644 index 000000000..0dd70f104 --- /dev/null +++ b/extensions/actions/methods.go @@ -0,0 +1,41 @@ +package extensions + +import ( + "context" + "fmt" +) + +// MethodFunc is a function that executes a method +type MethodFunc func(ctx *ExecutionContext, inputs ...*ScalarValue) ([]*ScalarValue, error) + +// InitializeFunc is a function that creates a new instance of an extension. +// In most cases, this should just validate the metadata being sent. +type InitializeFunc func(ctx context.Context, metadata map[string]string) (map[string]string, error) + +// WithInputsCheck checks the number of inputs. +// If the number of inputs is not equal to numInputs, it returns an error. +func WithInputsCheck(fn MethodFunc, numInputs int) MethodFunc { + return func(ctx *ExecutionContext, inputs ...*ScalarValue) ([]*ScalarValue, error) { + if len(inputs) != numInputs { + return nil, fmt.Errorf("expected %d args, got %d", numInputs, len(inputs)) + } + return fn(ctx, inputs...) + } +} + +// WithOutputsCheck checks the number of outputs. +// If the number of outputs is not equal to numOutputs, it returns an error. +func WithOutputsCheck(fn MethodFunc, numOutputs int) MethodFunc { + return func(ctx *ExecutionContext, inputs ...*ScalarValue) ([]*ScalarValue, error) { + res, err := fn(ctx, inputs...) + if err != nil { + return nil, err + } + + if len(res) != numOutputs { + return nil, fmt.Errorf("expected %d returns, got %d", numOutputs, len(res)) + } + + return res, nil + } +} diff --git a/extensions/actions/mocks_test.go b/extensions/actions/mocks_test.go deleted file mode 100644 index e092d28cc..000000000 --- a/extensions/actions/mocks_test.go +++ /dev/null @@ -1,47 +0,0 @@ -package extensions_test - -import ( - "context" - - extensions "github.com/kwilteam/kwil-db/extensions/actions" - "github.com/kwilteam/kwil-extensions/client" - "github.com/kwilteam/kwil-extensions/types" -) - -func init() { - extensions.ConnectFunc = connecterFunc(mockConnect) -} - -// this is used to inject a mock connection function for testing -func mockConnect(ctx context.Context, target string, opts ...client.ClientOpt) (extensions.ExtensionClient, error) { - return &mockClient{}, nil -} - -type connecterFunc func(ctx context.Context, target string, opts ...client.ClientOpt) (extensions.ExtensionClient, error) - -func (m connecterFunc) Connect(ctx context.Context, target string, opts ...client.ClientOpt) (extensions.ExtensionClient, error) { - return &mockClient{}, nil -} - -// mockClient implements the ExtensionClient interface -type mockClient struct{} - -func (m *mockClient) GetName(ctx context.Context) (string, error) { - return "mock", nil -} - -func (m *mockClient) CallMethod(ctx *types.ExecutionContext, method string, args ...any) ([]any, error) { - return []any{"val1", 2}, nil -} - -func (m *mockClient) Close() error { - return nil -} - -func (m *mockClient) ListMethods(ctx context.Context) ([]string, error) { - return []string{"method1", "method2"}, nil -} - -func (m *mockClient) Initialize(ctx context.Context, metadata map[string]string) (map[string]string, error) { - return metadata, nil -} diff --git a/extensions/actions/types.go b/extensions/actions/types.go new file mode 100644 index 000000000..a39ccc655 --- /dev/null +++ b/extensions/actions/types.go @@ -0,0 +1,44 @@ +package extensions + +import ( + "context" + "fmt" + "reflect" + + "github.com/cstockton/go-conv" +) + +type ScalarValue struct { + Value any +} + +func NewScalarValue(v any) (*ScalarValue, error) { + valueType := reflect.TypeOf(v) + switch valueType.Kind() { + case reflect.String, reflect.Float32, reflect.Float64: + return &ScalarValue{ + Value: v, + }, nil + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64: + return &ScalarValue{ + Value: v, + }, nil + default: + return nil, fmt.Errorf("invalid scalar type: %s", valueType.Kind()) + } +} + +// String returns the string representation of the value. +func (s *ScalarValue) String() (string, error) { + return conv.String(s.Value) +} + +// Int returns the int representation of the value. +func (s *ScalarValue) Int() (int64, error) { + return conv.Int64(s.Value) +} + +type ExecutionContext struct { + Ctx context.Context + Metadata map[string]string +} diff --git a/go.mod b/go.mod index dcf7273b7..def85ec8c 100644 --- a/go.mod +++ b/go.mod @@ -22,7 +22,6 @@ require ( github.com/kwilteam/kuneiform v0.5.0-alpha.0.20231011193347-ab7495c55426 github.com/kwilteam/kwil-db/core v0.0.0 github.com/kwilteam/kwil-db/parse v0.0.0 - github.com/kwilteam/kwil-extensions v0.0.0-20230710163303-bfa03f64ff82 github.com/manifoldco/promptui v0.9.0 github.com/olekukonko/tablewriter v0.0.5 github.com/spf13/cobra v1.7.0 diff --git a/go.sum b/go.sum index b9dfd37fe..0ff8bc69e 100644 --- a/go.sum +++ b/go.sum @@ -323,8 +323,6 @@ github.com/kwilteam/go-sqlite v0.0.0-20230606000142-c7eaa7111421 h1:TewJpDtkIU8Z github.com/kwilteam/go-sqlite v0.0.0-20230606000142-c7eaa7111421/go.mod h1:urRZ5yExms/OcYQHq0IAPLkNoudEbfUuQdlNvhcfrKI= github.com/kwilteam/kuneiform v0.5.0-alpha.0.20231011193347-ab7495c55426 h1:IO3Myedpq5Jr7Yo/ieqQBJPqsObA84/eEwkPexweduw= github.com/kwilteam/kuneiform v0.5.0-alpha.0.20231011193347-ab7495c55426/go.mod h1:MT8wV7wVVMz0UREaaOkkInUyvZMKO7FcHZ7E4cmsgLQ= -github.com/kwilteam/kwil-extensions v0.0.0-20230710163303-bfa03f64ff82 h1:pA0ya2WrncGSxxXB0g3dVq1jZZSm1HO6Qp0/yYn4qks= -github.com/kwilteam/kwil-extensions v0.0.0-20230710163303-bfa03f64ff82/go.mod h1:+BrFrV+3qcdYIfptqjwatE5gT19azuRHJzw77wMPY8c= github.com/kwilteam/sql-grammar-go v0.0.3-0.20230925230724-00685e1bac32 h1:NDMw+6BKSqLxFyfpbbCJNx8EOLB3+ugCUEnMpomXBeQ= github.com/kwilteam/sql-grammar-go v0.0.3-0.20230925230724-00685e1bac32/go.mod h1:OqmGyCwHfBZvYv/sYPrQ5Ih290dhlD5AcKOHDlUSS0Y= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= diff --git a/internal/csv/corn.csv b/internal/csv/corn.csv deleted file mode 100644 index b542190d5..000000000 --- a/internal/csv/corn.csv +++ /dev/null @@ -1,24 +0,0 @@ -"Date","Price","Open","High","Low","Vol.","Change %" -"04/19/2023","678.12","675.00","678.40","672.38","","0.09%" -"04/18/2023","677.50","676.00","682.50","669.75","130.89K","0.15%" -"04/17/2023","676.50","666.00","677.00","662.25","146.90K","1.54%" -"04/14/2023","666.25","653.75","668.00","651.00","154.00K","2.15%" -"04/13/2023","652.25","656.00","661.75","651.50","147.15K","-0.57%" -"04/12/2023","656.00","651.00","658.50","648.75","150.11K","0.77%" -"04/11/2023","651.00","653.50","657.75","646.75","200.26K","-0.46%" -"04/10/2023","654.00","644.50","655.00","640.25","150.10K","1.63%" -"04/07/2023","643.50","643.50","643.50","643.50","","0.00%" -"04/06/2023","643.50","652.25","652.50","641.75","132.98K","-1.42%" -"04/05/2023","652.75","653.75","660.00","646.50","143.56K","-0.15%" -"04/04/2023","653.75","657.00","657.50","648.50","140.10K","-0.61%" -"04/03/2023","657.75","659.50","668.50","656.00","153.87K","-0.42%" -"03/31/2023","660.50","648.75","664.50","647.25","201.89K","1.69%" -"03/30/2023","649.50","649.75","654.75","648.00","119.49K","-0.15%" -"03/29/2023","650.50","646.25","655.50","645.75","156.69K","0.50%" -"03/28/2023","647.25","646.75","651.25","644.50","135.24K","-0.15%" -"03/27/2023","648.25","643.00","648.75","637.00","145.57K","0.82%" -"03/24/2023","643.00","631.00","645.00","627.50","191.26K","1.78%" -"03/23/2023","631.75","633.00","644.00","627.25","186.44K","-0.28%" -"03/22/2023","633.50","631.25","636.00","623.25","154.75K","0.56%" -"03/21/2023","630.00","632.75","638.25","628.75","133.31K","-0.47%" -"03/20/2023","633.00","633.00","634.00","625.00","115.79K","-0.20%" \ No newline at end of file diff --git a/internal/csv/csv.go b/internal/csv/csv.go deleted file mode 100644 index 5961e5ef1..000000000 --- a/internal/csv/csv.go +++ /dev/null @@ -1,151 +0,0 @@ -package csv - -import ( - "encoding/csv" - "io" - "os" - "strings" -) - -type CSVReaderFlag uint8 - -const ( - ContainsHeader CSVReaderFlag = 1 << iota -) - -type CSV struct { - Header []string - Records [][]string -} - -func Read(csvFile *os.File, flags CSVReaderFlag) (*CSV, error) { - reader := csv.NewReader(csvFile) - reader.LazyQuotes = true - csvStruct := &CSV{ - Records: [][]string{}, - } - - if flags&ContainsHeader == ContainsHeader { - err := csvStruct.readAndTrimHeader(reader) - if err != nil { - return nil, err - } - } - - for { - record, err := reader.Read() - if err == io.EOF { - break - } - if err != nil { - return nil, err - } - - csvStruct.Records = append(csvStruct.Records, record) - } - - return csvStruct, nil -} - -func (c *CSV) readAndTrimHeader(reader *csv.Reader) error { - header, err := reader.Read() - if err != nil { - return err - } - - for i, singleHeader := range header { - header[i] = cleanHeader(singleHeader) - } - - c.Header = header - - return nil -} - -// some headers are formatted as "\ufeff\"Date\"" instead of "Date" -// this function will remove the leading \ufeff -func cleanHeader(header string) string { - str := header - if strings.HasPrefix(header, "\ufeff") { - str = strings.Replace(header, "\ufeff", "", 1) - } - - // remove leading and trailing quotes - str = strings.Trim(str, "\"") - - return str -} - -// BuildInputs is the same as BuildInputs, but it takes a schema to ensure that the input -// is valid. If the input is invalid, it will return an error. -// The function takes an action, as well as a map mapping the CSV column name to the -// action input name. -func (c *CSV) BuildInputs(inputNames map[string]string) ([]map[string]any, error) { - resultMap := make([]map[string]any, 0) - err := c.ForEachRecord(func(record []CSVCell) error { - input := make(map[string]any) - - for _, cell := range record { - inputName, ok := inputNames[*cell.Column] - if !ok { - continue - } - - input[inputName] = cell.Value - } - - resultMap = append(resultMap, input) - return nil - }) - if err != nil { - return nil, err - } - - return resultMap, nil -} - -// GetColumnIndex returns the index of the column in the CSV. If the column doesn't exist, -// it will return -1. -func (c *CSV) GetColumnIndex(column string) int { - for i, col := range c.Header { - if col == column { - return i - } - } - return -1 -} - -// ForEachRecord will loop through each record in the CSV and call the function with the -// record as a map of the CSV column name to value. -func (c *CSV) ForEachRecord(fn func([]CSVCell) error) error { - var err error - for _, record := range c.Records { - err = fn(c.buildCSVCells(record)) - if err != nil { - return err - } - } - - return nil -} - -type CSVCell struct { - Column *string - Value string -} - -// buildCSVCells will build a map of the CSV column name to value. -// The values are serialized strings. -// If for some reason it fails to serialize to string, it will panic. -func (c *CSV) buildCSVCells(record []string) []CSVCell { - csvVals := make([]CSVCell, len(record)) - for i, column := range record { - - csvVals[i] = CSVCell{ - Column: &c.Header[i], - Value: column, - } - } - - return csvVals -} diff --git a/internal/csv/csv_test.go b/internal/csv/csv_test.go deleted file mode 100644 index b034ce1a8..000000000 --- a/internal/csv/csv_test.go +++ /dev/null @@ -1,116 +0,0 @@ -package csv_test - -import ( - "os" - "testing" - -<<<<<<<< HEAD:cmd/kwil-cli/csv/csv_test.go - "github.com/kwilteam/kwil-db/cmd/kwil-cli/csv" -======== - "github.com/kwilteam/kwil-db/internal/csv" ->>>>>>>> cdc2e23e (establish app and sdk package isolation with core submodule (these can all move)):internal/csv/csv_test.go -) - -func Test_CSV(t *testing.T) { - file, err := loadTestCSV(t) - if err != nil { - t.Fatal(err) - } - - data, err := csv.Read(file, csv.ContainsHeader) - if err != nil { - t.Fatal(err) - } - - if len(data.Header) != 3 { - t.Fatal(`expected 3 columns, got: `, len(data.Header)) - } -} - -func Test_PrepareInputs(t *testing.T) { - file, err := loadTestCSV(t) - if err != nil { - t.Fatal(err) - } - - data, err := csv.Read(file, csv.ContainsHeader) - if err != nil { - t.Fatal(err) - } - - inputNames := map[string]string{ - "id": "$id", - "full_name": "$name", - "age": "$age", - } - - inputs, err := data.BuildInputs(inputNames) - if err != nil { - t.Fatal(err) - } - - if len(inputs) != 100 { - t.Fatal(`expected 100 records, got: `, len(inputs)) - } - - record := inputs[0] - if len(record) != 3 { - t.Fatal(`expected 3 columns, got: `, len(record)) - } - - row0col0 := record["$id"] - row0col1 := record["$name"] - row0col2 := record["$age"] - - if row0col0 != "1" { - t.Fatal("expected row 0, column 0 to be 1, got: ", record[inputNames["$id"]]) - } - - if row0col1 != "Theodore Berry" { - t.Fatal("expected row 0, column 1 to be Theodore Berry, got: ", record[inputNames["$name"]]) - } - - if row0col2 != "51" { - t.Fatal("expected row 0, column 2 to be 51, got: ", record[inputNames["$age"]]) - } -} - -func loadTestCSV(t *testing.T) (*os.File, error) { - path := "./test.csv" - return os.Open(path) -} - -func loadCornCSV(t *testing.T) (*os.File, error) { - path := "./corn.csv" - return os.Open(path) -} - -func Test_ReadDoubleQuotes(t *testing.T) { - file, err := loadCornCSV(t) - if err != nil { - t.Fatal(err) - } - - data, err := csv.Read(file, csv.ContainsHeader) - if err != nil { - t.Fatal(err) - } - - inputNames := map[string]string{ - "Date": "$dt", - "Price": "$value", - } - - res, err := data.BuildInputs(inputNames) - if err != nil { - t.Fatal(err) - } - - if len(res) != 23 { - t.Fatal(`expected 23 records, got: `, len(res)) - } - - if len(res[0]) != 2 { - t.Fatal(`expected 2 columns, got: `, len(res[0])) - } -} diff --git a/internal/csv/test.csv b/internal/csv/test.csv deleted file mode 100644 index 94e9036aa..000000000 --- a/internal/csv/test.csv +++ /dev/null @@ -1,101 +0,0 @@ -id,full_name,age -1,Theodore Berry,51 -2,Ada Rodriquez,22 -3,Warren Ray,33 -4,Glenn Santiago,27 -5,Jerry Stokes,60 -6,Fred Neal,62 -7,Jeremiah Simpson,44 -8,Eliza Price,31 -9,Estella Steele,44 -10,Herbert Ellis,43 -11,Susie Thomas,29 -12,Alejandro Clayton,55 -13,Jerome Diaz,24 -14,Harry Pittman,62 -15,Martha Guerrero,33 -16,Mittie Moore,23 -17,Eula Ross,56 -18,Jean Maxwell,47 -19,Eugene Mann,44 -20,Warren Park,51 -21,Olga Bush,45 -22,Lillian Marshall,25 -23,Joseph Edwards,29 -24,Daniel Matthews,65 -25,Katie Price,19 -26,Mathilda Carter,26 -27,Chris Hale,29 -28,Frank Patrick,35 -29,Linnie Foster,63 -30,Joe Holt,34 -31,Bobby Phillips,39 -32,Katharine Farmer,42 -33,Sue Caldwell,37 -34,Terry Palmer,28 -35,Jerry Johnson,61 -36,Leah Crawford,57 -37,Craig Mullins,23 -38,Lee Erickson,35 -39,Cecelia Harrison,26 -40,Jonathan Jensen,62 -41,Isaac Harris,28 -42,Sue Cortez,41 -43,Lizzie Fisher,50 -44,Adrian Fowler,51 -45,Mark Walton,40 -46,Alma Bradley,39 -47,Nicholas Murphy,23 -48,Ada Dawson,48 -49,Fannie Fisher,65 -50,Genevieve Freeman,50 -51,Randall Bryant,23 -52,Elmer Alvarez,59 -53,Amanda Leonard,33 -54,Lizzie Estrada,47 -55,Madge Griffith,42 -56,David Tate,28 -57,Jimmy Valdez,51 -58,Flora Perkins,23 -59,Joshua Simpson,41 -60,Ida Chavez,46 -61,Dean Moody,22 -62,Logan Ryan,20 -63,Oscar Boone,25 -64,Sarah Garner,25 -65,Lucille Hall,50 -66,Leila Evans,56 -67,Donald Ryan,62 -68,Rena Sullivan,36 -69,Miguel Hale,47 -70,Elmer White,42 -71,Myra Bell,30 -72,Adele Webster,32 -73,Jonathan Higgins,52 -74,Edgar Cunningham,32 -75,Lois Delgado,25 -76,Ronnie Hammond,35 -77,Bessie Hall,18 -78,Julian Guerrero,59 -79,Robert Carr,48 -80,David Rose,20 -81,Lily Hubbard,41 -82,Florence Hardy,50 -83,Raymond McKinney,49 -84,Sophie Berry,32 -85,Ellen Edwards,37 -86,Randall Webster,58 -87,Mamie Parsons,18 -88,Sophia Mason,55 -89,Caleb Kelly,54 -90,Angel Black,39 -91,Marie Cruz,36 -92,Sam Blair,51 -93,Joel Moore,26 -94,Frederick Jones,35 -95,Lottie Gomez,23 -96,Hilda Knight,48 -97,Ella Norton,30 -98,Mark Bradley,24 -99,Sallie Gibson,21 -100,Bobby Wong,44 diff --git a/nodecfg/generate.go b/nodecfg/generate.go deleted file mode 100644 index 43de2867b..000000000 --- a/nodecfg/generate.go +++ /dev/null @@ -1,254 +0,0 @@ -// Package nodecfg provides functions to assist in the generation of new kwild -// node configurations. This is primarily intended for the kwil-admin commands -// and tests that required dynamic node configuration. -package nodecfg - -import ( - "encoding/hex" - "fmt" - "net" - "os" - "path/filepath" - "strings" - - // NOTE: do not use the types from these internal packages on nodecfg's - // exported API. - "github.com/kwilteam/kwil-db/cmd/kwild/config" - "github.com/kwilteam/kwil-db/internal/abci/cometbft" - - cmtEd "github.com/cometbft/cometbft/crypto/ed25519" -) - -const ( - nodeDirPerm = 0755 - chainIDPrefix = "kwil-chain-" - - abciDir = config.ABCIDirName - abciConfigDir = cometbft.ConfigDir - abciDataDir = cometbft.DataDir -) - -type NodeGenerateConfig struct { - // InitialHeight int64 // ? - OutputDir string - JoinExpiry int64 - WithoutGasCosts bool - WithoutNonces bool -} - -type TestnetGenerateConfig struct { - // InitialHeight int64 - NValidators int - NNonValidators int - ConfigFile string - OutputDir string - NodeDirPrefix string - PopulatePersistentPeers bool - HostnamePrefix string - HostnameSuffix string - StartingIPAddress string - Hostnames []string - P2pPort int - JoinExpiry int64 - WithoutGasCosts bool - WithoutNonces bool -} - -// GenerateNodeConfig is used to generate configuration required for running a -// kwil node. This includes the files: private_key, config.toml, genesis.json. -// -// - The private key is generated if it does not exist. -// - The genesis file is generated if it does not exist. A new genesis file -// will include the node as a validator; existing genesis is not updated. -// - The config.toml file is generated if it does not exist. -func GenerateNodeConfig(genCfg *NodeGenerateConfig) error { - rootDir, err := config.ExpandPath(genCfg.OutputDir) - if err != nil { - return fmt.Errorf("failed to get absolute path for output directory: %w", err) - } - - cfg := config.DefaultConfig() - cfg.RootDir = rootDir - chainRoot := filepath.Join(rootDir, abciDir) - // NOTE: not the fully re-rooted path since this may run in a container. The - // caller can update PrivateKeyPath if desired. - - fullABCIConfigDir := filepath.Join(chainRoot, abciConfigDir) - err = os.MkdirAll(fullABCIConfigDir, nodeDirPerm) - if err != nil { - return err - } - - err = os.MkdirAll(filepath.Join(chainRoot, abciDataDir), nodeDirPerm) - if err != nil { - return err - } - - cfg.AppCfg.PrivateKeyPath = config.PrivateKeyFileName - err = writeConfigFile(filepath.Join(rootDir, config.ConfigFileName), cfg) - if err != nil { - return err - } - - // Load or generate private key. - fullKeyPath := filepath.Join(rootDir, config.PrivateKeyFileName) - _, pubKey, newKey, err := config.ReadOrCreatePrivateKeyFile(fullKeyPath, true) - if err != nil { - return fmt.Errorf("cannot read or create private key: %w", err) - } - if newKey { - fmt.Printf("Generated new private key: %v\n", fullKeyPath) - } - - // Create or update genesis config. - genFile := filepath.Join(fullABCIConfigDir, cometbft.GenesisJSONName) - - _, err = os.Stat(genFile) - if os.IsNotExist(err) { - genesisCfg := config.NewGenesisWithValidator(pubKey) - genCfg.applyGenesisParams(genesisCfg) - return genesisCfg.SaveAs(genFile) - } - - return err -} - -func (genCfg *NodeGenerateConfig) applyGenesisParams(genesisCfg *config.GenesisConfig) { - genesisCfg.ConsensusParams.Validator.JoinExpiry = genCfg.JoinExpiry - genesisCfg.ConsensusParams.WithoutGasCosts = genCfg.WithoutGasCosts - genesisCfg.ConsensusParams.WithoutNonces = genCfg.WithoutNonces -} - -// GenerateTestnetConfig is like GenerateNodeConfig but it generates multiple -// configs for a network of nodes on a LAN. See also TestnetGenerateConfig. -func GenerateTestnetConfig(genCfg *TestnetGenerateConfig) error { - var err error - genCfg.OutputDir, err = config.ExpandPath(genCfg.OutputDir) - if err != nil { - fmt.Println("Error while getting absolute path for output directory: ", err) - return err - } - - nNodes := genCfg.NValidators + genCfg.NNonValidators - if nHosts := len(genCfg.Hostnames); nHosts > 0 && nHosts != nNodes { - return fmt.Errorf( - "testnet needs precisely %d hostnames (number of validators plus nonValidators) if --hostname parameter is used", - nNodes, - ) - } - - // overwrite default config if set and valid - cfg := config.DefaultConfig() - if genCfg.ConfigFile != "" { - if err = cfg.ParseConfig(genCfg.ConfigFile); err != nil { - return fmt.Errorf("failed to parse config file %s: %w", genCfg.ConfigFile, err) - } - } - - privateKeys := make([]cmtEd.PrivKey, nNodes) - for i := range privateKeys { - privateKeys[i] = cmtEd.GenPrivKey() - - nodeDirName := fmt.Sprintf("%s%d", genCfg.NodeDirPrefix, i) - nodeDir := filepath.Join(genCfg.OutputDir, nodeDirName) - chainRoot := filepath.Join(nodeDir, abciDir) - - err := os.MkdirAll(filepath.Join(chainRoot, abciConfigDir), nodeDirPerm) - if err != nil { - _ = os.RemoveAll(genCfg.OutputDir) - return err - } - - err = os.MkdirAll(filepath.Join(chainRoot, abciDataDir), nodeDirPerm) - if err != nil { - _ = os.RemoveAll(genCfg.OutputDir) - return err - } - - privKeyHex := hex.EncodeToString(privateKeys[i][:]) - privKeyFile := filepath.Join(nodeDir, config.PrivateKeyFileName) - err = os.WriteFile(privKeyFile, []byte(privKeyHex), 0644) // permissive for testnet only - if err != nil { - return fmt.Errorf("creating private key file: %w", err) - } - } - - genConfig := config.DefaultGenesisConfig() - for i, pk := range privateKeys[:genCfg.NValidators] { - genConfig.Validators = append(genConfig.Validators, &config.GenesisValidator{ - PubKey: pk.PubKey().Bytes(), - Power: 1, - Name: fmt.Sprintf("validator-%d", i), - }) - } - genCfg.applyGenesisParams(genConfig) - - // write genesis file - for i := 0; i < genCfg.NValidators+genCfg.NNonValidators; i++ { - nodeDir := filepath.Join(genCfg.OutputDir, fmt.Sprintf("%s%d", genCfg.NodeDirPrefix, i)) - chainRoot := filepath.Join(nodeDir, abciDir) - genFile := cometbft.GenesisPath(chainRoot) // filepath.Join(nodeDir, abciDir, abciConfigDir, "genesis.json") - err = genConfig.SaveAs(genFile) - if err != nil { - return fmt.Errorf("failed to write genesis file %v: %w", genFile, err) - } - } - - // Gather persistent peers addresses - var persistentPeers string - if genCfg.PopulatePersistentPeers { - persistentPeers = persistentPeersString(genCfg, privateKeys) - } - - // Overwrite default config - cfg.ChainCfg.P2P.AddrBookStrict = false - cfg.ChainCfg.P2P.AllowDuplicateIP = true - for i := 0; i < genCfg.NValidators+genCfg.NNonValidators; i++ { - nodeDir := filepath.Join(genCfg.OutputDir, fmt.Sprintf("%s%d", genCfg.NodeDirPrefix, i)) - cfg.RootDir = nodeDir - - if genCfg.PopulatePersistentPeers { - cfg.ChainCfg.P2P.PersistentPeers = persistentPeers - } - cfg.AppCfg.PrivateKeyPath = config.PrivateKeyFileName // not abs/rooted because this might be run in a container - writeConfigFile(filepath.Join(nodeDir, config.ConfigFileName), cfg) - } - - fmt.Printf("Successfully initialized %d node directories: %s\n", - genCfg.NValidators+genCfg.NNonValidators, genCfg.OutputDir) - - return nil -} - -func (genCfg *TestnetGenerateConfig) applyGenesisParams(genesisCfg *config.GenesisConfig) { - genesisCfg.ConsensusParams.Validator.JoinExpiry = genCfg.JoinExpiry - genesisCfg.ConsensusParams.WithoutGasCosts = genCfg.WithoutGasCosts - genesisCfg.ConsensusParams.WithoutNonces = genCfg.WithoutNonces -} - -func hostnameOrIP(genCfg *TestnetGenerateConfig, i int) string { - if len(genCfg.Hostnames) > 0 && i < len(genCfg.Hostnames) { - return genCfg.Hostnames[i] - } - if genCfg.StartingIPAddress == "" { - return fmt.Sprintf("%s%d%s", genCfg.HostnamePrefix, i, genCfg.HostnameSuffix) - } - ip := net.ParseIP(genCfg.StartingIPAddress) - ip = ip.To4() - if ip == nil { - panic(fmt.Sprintf("%v: non ipv4 address\n", genCfg.StartingIPAddress)) - } - - ip[3] += byte(i) - return ip.String() -} - -func persistentPeersString(genCfg *TestnetGenerateConfig, privKeys []cmtEd.PrivKey) string { - persistentPeers := make([]string, genCfg.NValidators+genCfg.NNonValidators) - for i := range persistentPeers { - pubKey := privKeys[i].PubKey().(cmtEd.PubKey) - hostPort := fmt.Sprintf("%s:%d", hostnameOrIP(genCfg, i), genCfg.P2pPort) - persistentPeers[i] = cometbft.NodeIDAddressString(pubKey, hostPort) - } - return strings.Join(persistentPeers, ",") -} diff --git a/nodecfg/toml.go b/nodecfg/toml.go deleted file mode 100644 index 573298275..000000000 --- a/nodecfg/toml.go +++ /dev/null @@ -1,213 +0,0 @@ -package nodecfg - -import ( - "bytes" - "fmt" - "os" - "strings" - "text/template" - - "github.com/kwilteam/kwil-db/cmd/kwild/config" -) - -var configTemplate *template.Template - -func init() { - var err error - tmpl := template.New("configFileTemplate").Funcs(template.FuncMap{ - "arrayFormatter": arrayFormatter, - }) - if configTemplate, err = tmpl.Parse(defaultConfigTemplate); err != nil { - panic(err) - } -} - -// arrayFormatter is a template function that formats a array of strings in to `["str1", "str2", ...]` in toml file. -func arrayFormatter(items []string) string { - var formattedStrings []string - for _, word := range items { - formattedStrings = append(formattedStrings, fmt.Sprintf(`"%s"`, word)) - } - return "[" + strings.Join(formattedStrings, ", ") + "]" -} - -// writeConfigFile writes the config to a file. -func writeConfigFile(configFilePath string, cfg *config.KwildConfig) error { - var buffer bytes.Buffer - - if err := configTemplate.Execute(&buffer, cfg); err != nil { - return err - } - - return os.WriteFile(configFilePath, buffer.Bytes(), nodeDirPerm) -} - -const defaultConfigTemplate = ` -# This is a TOML config file. -# For more information, see https://github.com/toml-lang/toml - -# NOTE: Any path below can be absolute (e.g. "/var/myawesomeapp/data") or -# relative to the home directory (e.g. "data") - -# Root Directory Structure: -# RootDir/ -# |- config.toml (app and chain configuration for running the kwild node) -# |- private_key (node's private key) -# |- abci/ -# | |- config/ -# | | |- genesis.json (genesis file for the network) -# | | |- addrbook.json (peer routable addresses for the kwild node) -# | |- data/ -# | | |- blockchain db files/dir (blockstore.db, state.db, etc) -# | |- info/ -# |- application/wal -# |- data -# | |- kwild.db/ -# |- signing/ - -# Only the config.toml and genesis file are required to run the kwild node -# The rest of the files & directories are created by the kwild node on startup - -####################################################################### -### Logging Config Options ### -####################################################################### -[log] -# Output level for logging, default is "info". Other options are "debug", "error", "warn", "trace" -level = "{{ .Logging.Level }}" - -# Output paths for the logger, can be stdout or a file path -output_paths = {{arrayFormatter .Logging.OutputPaths }} - -# Output format: 'plain' or 'json' -format = "{{ .Logging.Format }}" - -# Time format: "epochfloat" (default), "epochmilli", or "rfc3339milli" -time_format = "{{ .Logging.TimeEncoding }}" - -####################################################################### -### App Config Options ### -####################################################################### - -[app] -# Node's Private key -private_key_path = "{{ .AppCfg.PrivateKeyPath }}" - -# TCP or UNIX socket address for the KWILD App's GRPC server to listen on -grpc_listen_addr = "{{ .AppCfg.GrpcListenAddress }}" - -# TCP or UNIX socket address for the KWILD App's HTTP server to listen on -http_listen_addr = "{{ .AppCfg.HTTPListenAddress }}" - -# List of Extension endpoints to be enabled ex: ["localhost:50052", "169.198.102.34:50053"] -extension_endpoints = {{arrayFormatter .AppCfg.ExtensionEndpoints}} - -# KWILD Sqlite database file path -sqlite_file_path = "{{ .AppCfg.SqliteFilePath }}" - -# The path to a file containing certificate that is used to create the HTTPS server. -# Might be either absolute path or path related to the kwild root directory. -# If the certificate is signed by a certificate authority, -# the certFile should be the concatenation of the server's certificate, any intermediates, -# and the CA's certificate. -# NOTE: both tls_cert_file and tls_key_file must be present for CometBFT to create HTTPS server. -# Otherwise, HTTP server is run. -tls_cert_file = "{{ .AppCfg.TLSCertFile }}" - -# The path to a file containing matching private key that is used to create the HTTPS server. -# Might be either absolute path or path related to the kwild root directory. -# NOTE: both tls_cert_file and tls_key_file must be present for CometBFT to create HTTPS server. -# Otherwise, HTTP server is run. -tls_key_file = "{{ .AppCfg.TLSKeyFile }}" - -# Kwild Server hostname -hostname = "" - -####################################################################### -### Chain Main Base Config Options ### -####################################################################### -[chain] - -# A custom human readable name for this node -moniker = "{{ .ChainCfg.Moniker }}" - -####################################################################### -### Advanced Configuration Options ### -####################################################################### - -####################################################### -### RPC Server Configuration Options ### -####################################################### -[chain.rpc] - -# TCP or UNIX socket address for the RPC server to listen on -listen_addr = "{{ .ChainCfg.RPC.ListenAddress }}" - -####################################################### -### Consensus Configuration Options ### -####################################################### -[chain.consensus] - -# How long we wait for a proposal block before prevoting nil -timeout_propose = "{{ .ChainCfg.Consensus.TimeoutPropose }}" - -# How long we wait after receiving +2/3 prevotes for “anything” (ie. not a single block or nil) -timeout_prevote = "{{ .ChainCfg.Consensus.TimeoutPrevote }}" - -# How long we wait after receiving +2/3 precommits for “anything” (ie. not a single block or nil) -timeout_precommit = "{{ .ChainCfg.Consensus.TimeoutPrecommit }}" - -# How long we wait after committing a block, before starting on the new -# height (this gives us a chance to receive some more precommits, even -# though we already have +2/3). -timeout_commit = "{{ .ChainCfg.Consensus.TimeoutCommit }}" - -####################################################### -### P2P Configuration Options ### -####################################################### -[chain.p2p] - -# Address to listen for incoming connections -listen_addr = "{{ .ChainCfg.P2P.ListenAddress }}" - -# Address to advertise to peers for them to dial -# If empty, will use the same port as the listening address, -# and will introspect on the listener or use UPnP -# to figure out the address. ip and port are required -# example: 159.89.10.97:26656 -external_address = "{{ .ChainCfg.P2P.ExternalAddress }}" - -# Comma separated list of nodes to keep persistent connections to (used for bootstrapping) -# Example: "d128266b8b9f64c313de466cf29e0a6182dba54d@172.10.100.2:26656,9440f4a8059cf7ff31454973c4f9c68de65fe526@172.10.100.3:26656" -persistent_peers = "{{ .ChainCfg.P2P.PersistentPeers }}" - -# Set true for strict address routability rules -# Set false for private or local networks -addr_book_strict = {{ .ChainCfg.P2P.AddrBookStrict }} - -# Maximum number of inbound peers -max_num_inbound_peers = {{ .ChainCfg.P2P.MaxNumInboundPeers }} - -# Maximum number of outbound peers to connect to, excluding persistent peers -max_num_outbound_peers = {{ .ChainCfg.P2P.MaxNumOutboundPeers }} - -# List of node IDs, to which a connection will be (re)established ignoring any existing limits -unconditional_peer_ids = "{{ .ChainCfg.P2P.UnconditionalPeerIDs }}" - -# Toggle to disable guard against peers connecting from the same ip. -allow_duplicate_ip = {{ .ChainCfg.P2P.AllowDuplicateIP }} - -####################################################### -### Mempool Configuration Options ### -####################################################### -[chain.mempool] -# Maximum number of transactions in the mempool -size = {{ .ChainCfg.Mempool.Size }} - -# Limit the total size of all txs in the mempool. -# This only accounts for raw transactions (e.g. given 1MB transactions and -# max_txs_bytes=5MB, mempool will only accept 5 transactions). -#max_txs_bytes = xx .ChainCfg.Mempool.MaxTxsBytes xx - -# Size of the cache (used to filter transactions we saw earlier) in transactions -cache_size = {{ .ChainCfg.Mempool.CacheSize }} -` diff --git a/nodecfg/toml_test.go b/nodecfg/toml_test.go deleted file mode 100644 index 8661e11b7..000000000 --- a/nodecfg/toml_test.go +++ /dev/null @@ -1,65 +0,0 @@ -package nodecfg - -import ( - "os" - "testing" - - "github.com/kwilteam/kwil-db/cmd/kwild/config" - - "github.com/stretchr/testify/assert" -) - -func Test_Generate_TOML(t *testing.T) { - cfg := config.DefaultConfig() - - cfg.AppCfg.SqliteFilePath = "sqlite.db/randomPath" - cfg.AppCfg.GrpcListenAddress = "localhost:9000" - cfg.AppCfg.ExtensionEndpoints = []string{"localhost:9001", "localhost:9002"} - cfg.Logging.OutputPaths = []string{"stdout", "file"} - writeConfigFile("test.toml", cfg) - defer os.Remove("test.toml") - - updatedcfg := config.DefaultConfig() - err := updatedcfg.ParseConfig("test.toml") - assert.NoError(t, err) - assert.Equal(t, cfg.AppCfg.SqliteFilePath, updatedcfg.AppCfg.SqliteFilePath) - assert.Equal(t, cfg.AppCfg.GrpcListenAddress, updatedcfg.AppCfg.GrpcListenAddress) - assert.Equal(t, cfg.AppCfg.ExtensionEndpoints, updatedcfg.AppCfg.ExtensionEndpoints) - assert.Equal(t, cfg.Logging.OutputPaths, updatedcfg.Logging.OutputPaths) -} - -func Test_GenerateNodeCfg(t *testing.T) { - genCfg := NodeGenerateConfig{ - // InitialHeight: 0, - OutputDir: "test/trybuild/", - JoinExpiry: 100, - WithoutGasCosts: true, - WithoutNonces: false, - } - - err := GenerateNodeConfig(&genCfg) - if err != nil { - t.Fatal(err) - } - - os.RemoveAll(genCfg.OutputDir) -} - -func Test_GenerateTestnetConfig(t *testing.T) { - genCfg := TestnetGenerateConfig{ - // InitialHeight: 0, - NValidators: 2, - NNonValidators: 1, - OutputDir: "test/testnet/", - StartingIPAddress: "192.168.12.12", - PopulatePersistentPeers: true, - P2pPort: 26656, - } - - err := GenerateTestnetConfig(&genCfg) - if err != nil { - t.Fatal(err) - } - - os.RemoveAll(genCfg.OutputDir) -} diff --git a/test/acceptance/docker-compose.yml b/test/acceptance/docker-compose.yml index c22687d12..1163da69e 100644 --- a/test/acceptance/docker-compose.yml +++ b/test/acceptance/docker-compose.yml @@ -17,29 +17,17 @@ services: - type: bind source: ${KWIL_HOME:-./.testnode} target: /app/kwil - depends_on: - - ext networks: - kwil-act-testnet command: | --root_dir=/app/kwil - --log.level=${LOG_LEVEL:-info} - --app.extension_endpoints=ext:50051 + --log.level=${LOG_LEVEL:-debug} --app.admin_listen_addr=:50151 --app.grpc_listen_addr=:50051 --app.http_listen_addr=:8080 --chain.p2p.external_address=tcp://0.0.0.0:26656 --chain.rpc.listen_addr=tcp://0.0.0.0:26657 - ext: - container_name: math_ext - image: kwilbrennan/extensions-math:multi-arch - ports: - - "50061:50051" - networks: - - kwil-act-testnet - - networks: kwil-act-testnet: name: kwil-act-testnet diff --git a/test/acceptance/helper.go b/test/acceptance/helper.go index b6cd01ac4..f20be7923 100644 --- a/test/acceptance/helper.go +++ b/test/acceptance/helper.go @@ -217,9 +217,6 @@ func (r *ActHelper) runDockerCompose(ctx context.Context) { // before the timeout err = dc. WithEnv(envs). - WaitForService( - "ext", - wait.NewLogStrategy("listening on").WithStartupTimeout(r.cfg.WaitTimeout)). WaitForService( "kwild", wait.NewLogStrategy("Starting Node service").WithStartupTimeout(r.cfg.WaitTimeout)). diff --git a/test/acceptance/test-data/test_db.kf b/test/acceptance/test-data/test_db.kf index 97553a4e5..826fbef09 100644 --- a/test/acceptance/test-data/test_db.kf +++ b/test/acceptance/test-data/test_db.kf @@ -114,8 +114,8 @@ action multi_select() public { } action divide($numerator1, $numerator2, $denominator) public view { - $up = math_up.div(abs($numerator1 + $numerator2), $denominator); - $down = math_down.div(abs($numerator1 + $numerator2), $denominator); + $up = math_up.divide(abs($numerator1 + $numerator2), $denominator); + $down = math_down.divide(abs($numerator1 + $numerator2), $denominator); select $up AS upper_value, $down AS lower_value; } diff --git a/test/integration/docker-compose.yml b/test/integration/docker-compose.yml index f01b5bc6b..1da027b68 100644 --- a/test/integration/docker-compose.yml +++ b/test/integration/docker-compose.yml @@ -19,12 +19,9 @@ services: networks: kwil-int-testnet: ipv4_address: 172.10.100.2 - depends_on: - - ext1 command: | --root_dir=/app/kwil --log.level=${LOG_LEVEL:-debug} - --app.extension_endpoints=ext1:50051 --app.grpc_listen_addr=:50051 --app.http_listen_addr=:8080 --chain.p2p.external_address=tcp://0.0.0.0:26656 @@ -47,12 +44,9 @@ services: networks: kwil-int-testnet: ipv4_address: 172.10.100.3 - depends_on: - - ext1 command: | --root_dir=/app/kwil --log.level=${LOG_LEVEL:-debug} - --app.extension_endpoints=ext1:50051 --app.grpc_listen_addr=:50051 --app.http_listen_addr=:8080 --chain.p2p.external_address=tcp://0.0.0.0:26656 @@ -75,12 +69,9 @@ services: networks: kwil-int-testnet: ipv4_address: 172.10.100.4 - depends_on: - - ext1 command: | --root_dir=/app/kwil --log.level=${LOG_LEVEL:-debug} - --app.extension_endpoints=ext1:50051 --app.grpc_listen_addr=:50051 --app.http_listen_addr=:8080 --chain.p2p.external_address=tcp://0.0.0.0:26656 @@ -115,22 +106,6 @@ services: --chain.p2p.external_address=tcp://0.0.0.0:26656 --chain.rpc.listen_addr=tcp://0.0.0.0:26657 - # this ext is shared by all nodes - # we can make a separate ext for each node if we want - ext1: - container_name: math_ext1 - image: kwilbrennan/extensions-math:multi-arch - ports: - - "50061:50051" - networks: - kwil-int-testnet: - ipv4_address: 172.10.100.101 - healthcheck: - test: [ "CMD", "nc", "-v", "50051" ] - interval: 5s - timeout: 3s - retries: 3 - networks: kwil-int-testnet: diff --git a/test/integration/helper.go b/test/integration/helper.go index 69681421b..2ad7a704e 100644 --- a/test/integration/helper.go +++ b/test/integration/helper.go @@ -51,8 +51,6 @@ var defaultWaitStrategies = map[string]string{ "node3": "Starting Node service", } -const ExtContainer = "ext1" - // IntTestConfig is the config for integration test // This is totally separate from acceptance test type IntTestConfig struct { @@ -242,10 +240,6 @@ func (r *IntHelper) RunDockerComposeWithServices(ctx context.Context, services [ require.NoError(r.t, err, "failed to start kwild cluster") for _, name := range services { - // skip ext1 - if name == ExtContainer { - continue - } container, err := dc.ServiceContainer(ctx, name) require.NoError(r.t, err, "failed to get container for service %s", name) r.containers[name] = container diff --git a/test/integration/kwild_test.go b/test/integration/kwild_test.go index c304a7829..09d0c5755 100644 --- a/test/integration/kwild_test.go +++ b/test/integration/kwild_test.go @@ -13,7 +13,7 @@ import ( var dev = flag.Bool("dev", false, "run for development purpose (no tests)") var drivers = flag.String("drivers", "client,cli", "comma separated list of drivers to run") -var allServices = []string{integration.ExtContainer, "node0", "node1", "node2", "node3"} +var allServices = []string{"node0", "node1", "node2", "node3"} var numServices = len(allServices) func TestKwildDatabaseIntegration(t *testing.T) { diff --git a/test/integration/test-data/test_db.kf b/test/integration/test-data/test_db.kf index d6682ff76..a1baa879c 100644 --- a/test/integration/test-data/test_db.kf +++ b/test/integration/test-data/test_db.kf @@ -113,8 +113,8 @@ action multi_select() public { } action divide($numerator, $denominator) public view { - $up = math_up.div($numerator, $denominator); - $down = math_down.div($numerator, $denominator); + $up = math_up.divide($numerator, $denominator); + $down = math_down.divide($numerator, $denominator); select $up AS upper_value, $down AS lower_value; }