diff --git a/go.mod b/go.mod index 63f0788a91be6..43749d20e3d66 100644 --- a/go.mod +++ b/go.mod @@ -21,6 +21,7 @@ require ( github.com/aws/aws-sdk-go-v2/service/sts v1.33.1 github.com/aws/smithy-go v1.22.1 github.com/axiomhq/hyperloglog v0.0.0-20230201085229-3ddf4bad03dc + github.com/buger/jsonparser v1.1.1 github.com/cakturk/go-netstat v0.0.0-20200220111822-e5b49efee7a5 github.com/cespare/xxhash/v2 v2.3.0 github.com/cockroachdb/errors v1.9.1 @@ -31,7 +32,7 @@ require ( github.com/dolthub/maphash v0.1.0 github.com/dslipak/pdf v0.0.2 github.com/elastic/gosigar v0.14.2 - github.com/extism/go-sdk v1.3.0 + github.com/extism/go-sdk v1.6.0 github.com/fagongzi/goetty/v2 v2.0.3-0.20230628075727-26c9a2fd5fb8 github.com/fagongzi/util v0.0.0-20210923134909-bccc37b5040d github.com/felixge/fgprof v0.9.6-0.20240831122612-49987e680f04 @@ -101,6 +102,7 @@ require ( github.com/clbanning/mxj v1.8.4 // indirect github.com/coreos/go-systemd/v22 v22.3.2 // indirect github.com/dustin/go-humanize v1.0.1 // indirect + github.com/dylibso/observe-sdk/go v0.0.0-20240819160327-2d926c5d788a // indirect github.com/go-ini/ini v1.67.0 // indirect github.com/gobwas/glob v0.2.3 // indirect github.com/goccy/go-json v0.10.3 // indirect @@ -110,6 +112,7 @@ require ( github.com/google/go-querystring v1.1.0 // indirect github.com/gosimple/slug v1.13.1 // indirect github.com/gosimple/unidecode v1.0.1 // indirect + github.com/ianlancetaylor/demangle v0.0.0-20240805132620-81f5be970eca // indirect github.com/itchyny/timefmt-go v0.1.6 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/josharian/native v1.1.0 // indirect @@ -129,7 +132,9 @@ require ( github.com/rs/xid v1.6.0 // indirect github.com/segmentio/asm v1.1.3 // indirect github.com/shoenig/go-m1cpu v0.1.6 // indirect - github.com/tetratelabs/wazero v1.7.3 // indirect + github.com/tetratelabs/wabin v0.0.0-20230304001439-f6f874872834 // indirect + github.com/tetratelabs/wazero v1.8.1-0.20240916092830-1353ca24fef0 // indirect + go.opentelemetry.io/proto/otlp v1.3.1 // indirect golang.org/x/crypto v0.28.0 // indirect golang.org/x/net v0.30.0 // indirect golang.org/x/text v0.19.0 // indirect @@ -204,7 +209,7 @@ require ( go.uber.org/multierr v1.11.0 // indirect golang.org/x/mod v0.21.0 // indirect golang.org/x/tools v0.26.0 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20240604185151-ef581f913117 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240814211410-ddb44dafa142 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/go.sum b/go.sum index 6936615fa8361..5c3e9a3c484de 100644 --- a/go.sum +++ b/go.sum @@ -120,10 +120,12 @@ github.com/bufbuild/protocompile v0.6.0 h1:Uu7WiSQ6Yj9DbkdnOe7U4mNKp58y9WDMKDn28 github.com/bufbuild/protocompile v0.6.0/go.mod h1:YNP35qEYoYGme7QMtz5SBCoN4kL4g12jTtjuzRNdjpE= github.com/buger/goterm v1.0.4 h1:Z9YvGmOih81P0FbVtEYTFF6YsSgxSUKEhf/f9bTMXbY= github.com/buger/goterm v1.0.4/go.mod h1:HiFWV3xnkolgrBV3mY8m0X0Pumt4zg4QhbdOzQtB8tE= +github.com/buger/jsonparser v1.1.1 h1:2PnMjfWD7wBILjqQbt530v576A/cAbQvEW9gGIpYMUs= +github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0= github.com/cakturk/go-netstat v0.0.0-20200220111822-e5b49efee7a5 h1:BjkPE3785EwPhhyuFkbINB+2a1xATwk8SNDWnJiD41g= github.com/cakturk/go-netstat v0.0.0-20200220111822-e5b49efee7a5/go.mod h1:jtAfVaU/2cu1+wdSRPWE2c1N2qeAA3K4RH9pYgqwets= -github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM= -github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= +github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= +github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= @@ -223,6 +225,8 @@ github.com/dslipak/pdf v0.0.2/go.mod h1:2L3SnkI9cQwnAS9gfPz2iUoLC0rUZwbucpbKi5R1 github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/dylibso/observe-sdk/go v0.0.0-20240819160327-2d926c5d788a h1:UwSIFv5g5lIvbGgtf3tVwC7Ky9rmMFBp0RMs+6f6YqE= +github.com/dylibso/observe-sdk/go v0.0.0-20240819160327-2d926c5d788a/go.mod h1:C8DzXehI4zAbrdlbtOByKX6pfivJTBiV9Jjqv56Yd9Q= github.com/eknkc/amber v0.0.0-20171010120322-cdade1c07385/go.mod h1:0vRUJqYpeSZifjYj7uP3BG/gKcuzL9xWVV/Y+cK33KM= github.com/emicklei/go-restful/v3 v3.10.1 h1:rc42Y5YTp7Am7CS630D7JmhRjq4UlEUuEKfrDac4bSQ= github.com/emicklei/go-restful/v3 v3.10.1/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= @@ -232,8 +236,8 @@ github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1m github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/etcd-io/bbolt v1.3.3/go.mod h1:ZF2nL25h33cCyBtcyWeZ2/I3HQOfTP+0PIEvHjkjCrw= -github.com/extism/go-sdk v1.3.0 h1:DBd4FzDBUAL3P01MNqUD2+x8G7qyYdJ7pV96NIrfWXA= -github.com/extism/go-sdk v1.3.0/go.mod h1:tPMWfCSOThie3LSTSZKbrQjRm2oAXxUUjSE4HJWjYQM= +github.com/extism/go-sdk v1.6.0 h1:crFRMhjcPAn6R9M4eIvkjHQs7CLBs3yzPqwnj+uwzdg= +github.com/extism/go-sdk v1.6.0/go.mod h1:yRolc4PvIUQ9J/BBB3QZ5EY1MtXAN2jqBGDGR3Sk54M= github.com/fagongzi/util v0.0.0-20210923134909-bccc37b5040d h1:1pILVCatHj3eVo9i52dZyY4BwjTmSIeN+/hoJh8rD0Y= github.com/fagongzi/util v0.0.0-20210923134909-bccc37b5040d/go.mod h1:5cqSns2zMRcJeVGvAqeTrbXFqh5AqBFr5uVKP9T2kiE= github.com/fasthttp-contrib/websocket v0.0.0-20160511215533-1f3b11f56072/go.mod h1:duJ4Jxv5lDcvg4QuQr0oowTf7dz4/CR8NtyCooz9HL8= @@ -265,8 +269,8 @@ github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A= github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= -github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ= -github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= +github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-martini/martini v0.0.0-20170121215854-22fa46961aab/go.mod h1:/P9AEU963A2AYjv4d1V5eVL1CQbEJq6aCNHDDjibzu8= @@ -376,8 +380,8 @@ github.com/gosimple/unidecode v1.0.1 h1:hZzFTMMqSswvf0LBJZCZgThIZrpDHFXux9KeGmn6 github.com/gosimple/unidecode v1.0.1/go.mod h1:CP0Cr1Y1kogOtx0bJblKzsVWrqYaqfNOnHzpgWw4Awc= github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 h1:+9834+KizmvFV7pXQGSXQTsaWhq2GjuNUt0aUU0YBYw= github.com/grpc-ecosystem/go-grpc-middleware v1.3.0/go.mod h1:z0ButlSOZa5vEBq9m2m2hlwIgKw+rp3sdCBRoJY+30Y= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0 h1:YBftPWNWd4WwGqtY2yeZL2ef8rHAxPBD8KFhJpmcqms= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0/go.mod h1:YN5jB8ie0yfIUg6VvR9Kz84aCaG7AsGZnLjhHbUqwPg= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.21.0 h1:CWyXh/jylQWp2dtiV33mY4iSSp6yf4lmn+c7/tN+ObI= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.21.0/go.mod h1:nCLIt0w3Ept2NwF8ThLmrppXsfT07oC8k0XNDxd8sVU= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= @@ -409,6 +413,8 @@ github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpO github.com/hydrogen18/memlistener v0.0.0-20141126152155-54553eb933fb/go.mod h1:qEIFzExnS6016fRpRfxrExeVn2gbClQA99gQhnIcdhE= github.com/hydrogen18/memlistener v0.0.0-20200120041712-dcc25e7acd91/go.mod h1:qEIFzExnS6016fRpRfxrExeVn2gbClQA99gQhnIcdhE= github.com/ianlancetaylor/demangle v0.0.0-20240312041847-bd984b5ce465/go.mod h1:gx7rwoVhcfuVKG5uya9Hs3Sxj7EIvldVofAWIUtGouw= +github.com/ianlancetaylor/demangle v0.0.0-20240805132620-81f5be970eca h1:T54Ema1DU8ngI+aef9ZhAhNGQhcRTrWxVeG07F+c/Rw= +github.com/ianlancetaylor/demangle v0.0.0-20240805132620-81f5be970eca/go.mod h1:gx7rwoVhcfuVKG5uya9Hs3Sxj7EIvldVofAWIUtGouw= github.com/imdario/mergo v0.3.16 h1:wwQJbIsHYGMUyLSPrEq1CT16AhnhNJQ51+4fdHUnCl4= github.com/imdario/mergo v0.3.16/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY= github.com/imkira/go-interpol v1.1.0/go.mod h1:z0h2/2T3XF8kyEPpRgJ3kmNv+C43p+I/CoI+jC3w2iA= @@ -759,8 +765,10 @@ github.com/testcontainers/testcontainers-go v0.29.1 h1:z8kxdFlovA2y97RWx98v/TQ+t github.com/testcontainers/testcontainers-go v0.29.1/go.mod h1:SnKnKQav8UcgtKqjp/AD8bE1MqZm+3TDb/B8crE3XnI= github.com/testcontainers/testcontainers-go/modules/compose v0.29.1 h1:47ipPM+s+ltCDOP3Sa1j95AkNb+z+WGiHLDbLU8ixuc= github.com/testcontainers/testcontainers-go/modules/compose v0.29.1/go.mod h1:Sqh+Ef2ESdbJQjTJl57UOkEHkOc7gXvQLg1b5xh6f1Y= -github.com/tetratelabs/wazero v1.7.3 h1:PBH5KVahrt3S2AHgEjKu4u+LlDbbk+nsGE3KLucy6Rw= -github.com/tetratelabs/wazero v1.7.3/go.mod h1:ytl6Zuh20R/eROuyDaGPkp82O9C/DJfXAwJfQ3X6/7Y= +github.com/tetratelabs/wabin v0.0.0-20230304001439-f6f874872834 h1:ZF+QBjOI+tILZjBaFj3HgFonKXUcwgJ4djLb6i42S3Q= +github.com/tetratelabs/wabin v0.0.0-20230304001439-f6f874872834/go.mod h1:m9ymHTgNSEjuxvw8E7WWe4Pl4hZQHXONY8wE6dMLaRk= +github.com/tetratelabs/wazero v1.8.1-0.20240916092830-1353ca24fef0 h1:NCRnJ+X6eZt3awiReoHCcDuC6Wf+CgWk6p4IDkIuxTo= +github.com/tetratelabs/wazero v1.8.1-0.20240916092830-1353ca24fef0/go.mod h1:yAI0XTsMBhREkM/YDAK/zNou3GoiAce1P6+rp/wQhjs= github.com/theupdateframework/notary v0.7.0 h1:QyagRZ7wlSpjT5N2qQAh/pN+DVqgekv4DzbAiAiEL3c= github.com/theupdateframework/notary v0.7.0/go.mod h1:c9DRxcmhHmVLDay4/2fUYdISnHqbFDGRSlXPO0AhYWw= github.com/ti-mo/conntrack v0.5.1 h1:opEwkFICnDbQc0BUXl73PHBK0h23jEIFVjXsqvF4GY0= @@ -831,32 +839,32 @@ go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0. go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.45.0/go.mod h1:4m3RnBBb+7dB9d21y510oO1pdB1V4J6smNf14WXcBFQ= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.45.0 h1:x8Z78aZx8cOF0+Kkazoc7lwUNMGy0LrzEMxTm4BbTxg= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.45.0/go.mod h1:62CPTSry9QZtOaSsE3tOzhx6LzDhHnXJ6xHeMNNiM6Q= -go.opentelemetry.io/otel v1.19.0 h1:MuS/TNf4/j4IXsZuJegVzI1cwut7Qc00344rgH7p8bs= -go.opentelemetry.io/otel v1.19.0/go.mod h1:i0QyjOq3UPoTzff0PJB2N66fb4S0+rSbSB15/oyH9fY= +go.opentelemetry.io/otel v1.28.0 h1:/SqNcYk+idO0CxKEUOtKQClMK/MimZihKYMruSMViUo= +go.opentelemetry.io/otel v1.28.0/go.mod h1:q68ijF8Fc8CnMHKyzqL6akLO46ePnjkgfIMIjUIX9z4= go.opentelemetry.io/otel/exporters/otlp/otlpmetric v0.42.0 h1:ZtfnDL+tUrs1F0Pzfwbg2d59Gru9NCH3bgSHBM6LDwU= go.opentelemetry.io/otel/exporters/otlp/otlpmetric v0.42.0/go.mod h1:hG4Fj/y8TR/tlEDREo8tWstl9fO9gcFkn4xrx0Io8xU= go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v0.42.0 h1:NmnYCiR0qNufkldjVvyQfZTHSdzeHoZ41zggMsdMcLM= go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v0.42.0/go.mod h1:UVAO61+umUsHLtYb8KXXRoHtxUkdOPkYidzW3gipRLQ= go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v0.42.0 h1:wNMDy/LVGLj2h3p6zg4d0gypKfWKSWI14E1C4smOgl8= go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v0.42.0/go.mod h1:YfbDdXAAkemWJK3H/DshvlrxqFB2rtW4rY6ky/3x/H0= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.19.0 h1:Mne5On7VWdx7omSrSSZvM4Kw7cS7NQkOOmLcgscI51U= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.19.0/go.mod h1:IPtUMKL4O3tH5y+iXVyAXqpAwMuzC1IrxVS81rummfE= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.19.0 h1:3d+S281UTjM+AbF31XSOYn1qXn3BgIdWl8HNEpx08Jk= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.19.0/go.mod h1:0+KuTDyKL4gjKCF75pHOX4wuzYDUZYfAQdSu43o+Z2I= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.19.0 h1:IeMeyr1aBvBiPVYihXIaeIZba6b8E1bYp7lbdxK8CQg= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.19.0/go.mod h1:oVdCUtjq9MK9BlS7TtucsQwUcXcymNiEDjgDD2jMtZU= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.28.0 h1:3Q/xZUyC1BBkualc9ROb4G8qkH90LXEIICcs5zv1OYY= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.28.0/go.mod h1:s75jGIWA9OfCMzF0xr+ZgfrB5FEbbV7UuYo32ahUiFI= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.28.0 h1:R3X6ZXmNPRR8ul6i3WgFURCHzaXjHdm0karRG/+dj3s= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.28.0/go.mod h1:QWFXnDavXWwMx2EEcZsf3yxgEKAqsxQ+Syjp+seyInw= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.28.0 h1:j9+03ymgYhPKmeXGk5Zu+cIZOlVzd9Zv7QIiyItjFBU= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.28.0/go.mod h1:Y5+XiUG4Emn1hTfciPzGPJaSI+RpDts6BnCIir0SLqk= go.opentelemetry.io/otel/exporters/prometheus v0.42.0 h1:jwV9iQdvp38fxXi8ZC+lNpxjK16MRcZlpDYvbuO1FiA= go.opentelemetry.io/otel/exporters/prometheus v0.42.0/go.mod h1:f3bYiqNqhoPxkvI2LrXqQVC546K7BuRDL/kKuxkujhA= -go.opentelemetry.io/otel/metric v1.19.0 h1:aTzpGtV0ar9wlV4Sna9sdJyII5jTVJEvKETPiOKwvpE= -go.opentelemetry.io/otel/metric v1.19.0/go.mod h1:L5rUsV9kM1IxCj1MmSdS+JQAcVm319EUrDVLrt7jqt8= -go.opentelemetry.io/otel/sdk v1.19.0 h1:6USY6zH+L8uMH8L3t1enZPR3WFEmSTADlqldyHtJi3o= -go.opentelemetry.io/otel/sdk v1.19.0/go.mod h1:NedEbbS4w3C6zElbLdPJKOpJQOrGUJ+GfzpjUvI0v1A= +go.opentelemetry.io/otel/metric v1.28.0 h1:f0HGvSl1KRAU1DLgLGFjrwVyismPlnuU6JD6bOeuA5Q= +go.opentelemetry.io/otel/metric v1.28.0/go.mod h1:Fb1eVBFZmLVTMb6PPohq3TO9IIhUisDsbJoL/+uQW4s= +go.opentelemetry.io/otel/sdk v1.28.0 h1:b9d7hIry8yZsgtbmM0DKyPWMMUMlK9NEKuIG4aBqWyE= +go.opentelemetry.io/otel/sdk v1.28.0/go.mod h1:oYj7ClPUA7Iw3m+r7GeEjz0qckQRJK2B8zjcZEfu7Pg= go.opentelemetry.io/otel/sdk/metric v1.19.0 h1:EJoTO5qysMsYCa+w4UghwFV/ptQgqSL/8Ni+hx+8i1k= go.opentelemetry.io/otel/sdk/metric v1.19.0/go.mod h1:XjG0jQyFJrv2PbMvwND7LwCEhsJzCzV5210euduKcKY= -go.opentelemetry.io/otel/trace v1.19.0 h1:DFVQmlVbfVeOuBRrwdtaehRrWiL1JoVs9CPIQ1Dzxpg= -go.opentelemetry.io/otel/trace v1.19.0/go.mod h1:mfaSyvGyEJEI0nyV2I4qhNQnbBOUUmYZpYojqMnX2vo= -go.opentelemetry.io/proto/otlp v1.0.0 h1:T0TX0tmXU8a3CbNXzEKGeU5mIVOdf0oykP+u2lIVU/I= -go.opentelemetry.io/proto/otlp v1.0.0/go.mod h1:Sy6pihPLfYHkr3NkUbEhGHFhINUSI/v80hjKIs5JXpM= +go.opentelemetry.io/otel/trace v1.28.0 h1:GhQ9cUuQGmNDd5BTCP2dAvv75RdMxEfTmYejp+lkx9g= +go.opentelemetry.io/otel/trace v1.28.0/go.mod h1:jPyXzNPg6da9+38HEwElrQiHlVMTnVfM3/yv2OlIHaI= +go.opentelemetry.io/proto/otlp v1.3.1 h1:TrMUixzpM0yuc/znrFTP9MMRh8trP93mkCiDVeXrui0= +go.opentelemetry.io/proto/otlp v1.3.1/go.mod h1:0X1WI4de4ZsLrrJNLAQbFeLCm3T7yBkR0XqQ7niQU+8= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= @@ -1080,10 +1088,10 @@ google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEY google.golang.org/genproto v0.0.0-20210624195500-8bfb893ecb84/go.mod h1:SzzZ/N+nwJDaO1kznhnlzqS8ocJICar6hYhVyhi++24= google.golang.org/genproto v0.0.0-20240325203815-454cdb8f5daa h1:ePqxpG3LVx+feAUOx8YmR5T7rc0rdzK8DyxM8cQ9zq0= google.golang.org/genproto v0.0.0-20240325203815-454cdb8f5daa/go.mod h1:CnZenrTdRJb7jc+jOm0Rkywq+9wh0QC4U8tyiRbEPPM= -google.golang.org/genproto/googleapis/api v0.0.0-20240528184218-531527333157 h1:7whR9kGa5LUwFtpLm2ArCEejtnxlGeLbAyjFY8sGNFw= -google.golang.org/genproto/googleapis/api v0.0.0-20240528184218-531527333157/go.mod h1:99sLkeliLXfdj2J75X3Ho+rrVCaJze0uwN7zDDkjPVU= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240604185151-ef581f913117 h1:1GBuWVLM/KMVUv1t1En5Gs+gFZCNd360GGb4sSxtrhU= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240604185151-ef581f913117/go.mod h1:EfXuqaE1J41VCDicxHzUDm+8rk+7ZdXzHV0IhO/I6s0= +google.golang.org/genproto/googleapis/api v0.0.0-20240814211410-ddb44dafa142 h1:wKguEg1hsxI2/L3hUYrpo1RVi48K+uTyzKqprwLXsb8= +google.golang.org/genproto/googleapis/api v0.0.0-20240814211410-ddb44dafa142/go.mod h1:d6be+8HhtEtucleCbxpPW9PA9XwISACu8nvpPqF0BVo= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240814211410-ddb44dafa142 h1:e7S5W7MGGLaSu8j3YjdezkZ+m1/Nm0uRVRMEMGk26Xs= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240814211410-ddb44dafa142/go.mod h1:UqMtugtsSgubUsoxbuAoiCXvqvErP7Gf0so0mK9tHxU= google.golang.org/grpc v1.12.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= diff --git a/pkg/frontend/variables.go b/pkg/frontend/variables.go index a0322af0754d5..5c85dad7c7aa1 100644 --- a/pkg/frontend/variables.go +++ b/pkg/frontend/variables.go @@ -2422,6 +2422,14 @@ var gSysVarsDefs = map[string]SystemVariable{ Type: InitSystemVariableIntType("min_examined_row_limit", 0, 4294967295, false), Default: int64(0), }, + "moplugin_allowed_hosts": { + Name: "moplugin_allowed_hosts", + Scope: ScopeGlobal, + Dynamic: true, + SetVarHintApplies: false, + Type: InitSystemVariableStringType("moplugin_allowed_hosts"), + Default: "localhost", + }, "myisam_data_pointer_size": { Name: "myisam_data_pointer_size", Scope: ScopeGlobal, diff --git a/pkg/sql/colexec/table_function/plugin.go b/pkg/sql/colexec/table_function/plugin.go new file mode 100644 index 0000000000000..1efede4c809b8 --- /dev/null +++ b/pkg/sql/colexec/table_function/plugin.go @@ -0,0 +1,334 @@ +// Copyright 2022 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package table_function + +import ( + "encoding/json" + "errors" + "fmt" + "net/url" + "strings" + + "github.com/buger/jsonparser" + extism "github.com/extism/go-sdk" + + "github.com/matrixorigin/matrixone/pkg/common/moerr" + "github.com/matrixorigin/matrixone/pkg/container/batch" + "github.com/matrixorigin/matrixone/pkg/container/bytejson" + "github.com/matrixorigin/matrixone/pkg/container/types" + "github.com/matrixorigin/matrixone/pkg/container/vector" + "github.com/matrixorigin/matrixone/pkg/datalink" + "github.com/matrixorigin/matrixone/pkg/sql/colexec" + "github.com/matrixorigin/matrixone/pkg/vm" + "github.com/matrixorigin/matrixone/pkg/vm/process" +) + +// Plugin is a framework to run an wasm image with input data (datalink or string) and +// output must be a JSON array +// Plugin needs four input columns +// 1. wasm image URL. http://, https://, file:// and stage:// supported +// 2. wasm function name +// 3. wasm configuration in JSON object format, e.g. {"key1":"value1", "key2":"value2", ...} +// 4. input data with datalink or string. With offset and size parameter in datalink, only portion +// of data is read and send to stdin. +// Return: +// output buffer in JSON Array format +// +// One of the example is wikipedia dumps. Wiki dump is a multisteam file which combine multiple bzip2 +// data chunks and its index file has the (offset, ID, title). The advantage is we don't need to unpack +// the whole file before getting the required data. +// Also, for LLM application, you can run the table_function wasm_run_table() to convert the data from datalink +// into mediawiki formatted text. +// All you have to do it to write a plugin wasm in any language that +// 1. read the data from input, unzip data with bzip2, parse xml to get all pages +// 2. return the all pages in mediawiki formatted text in JSON. +// +// To do this, simple run the SQL like this: +// SELECT json_unquote(json_extract(result, '$.id')), json_unquote(json_extract(result, '$.revision.text')) from +// wasm_run_table('https://github.com/pathto/wikidump.wasm', 'get_pages', null, cast('stage://mys3/wiki.bz2?offset=0&size=123456' as datalink)) as f; +// + +type pluginState struct { + inited bool + called bool + plugin *extism.Plugin + allowed_hosts []string + + // holding one call batch, pluginState owns it. + batch *batch.Batch +} + +func (u *pluginState) reset(tf *TableFunction, proc *process.Process) { + if u.batch != nil { + u.batch.CleanOnlyData() + } + u.called = false +} + +func (u *pluginState) call(tf *TableFunction, proc *process.Process) (vm.CallResult, error) { + var res vm.CallResult + if u.called { + return res, nil + } + res.Batch = u.batch + u.called = true + return res, nil +} + +func (u *pluginState) free(tf *TableFunction, proc *process.Process, pipelineFailed bool, err error) { + if u.batch != nil { + u.batch.Clean(proc.Mp()) + } +} + +func pluginPrepare(proc *process.Process, arg *TableFunction) (tvfState, error) { + var err error + st := &pluginState{} + arg.ctr.executorsForArgs, err = colexec.NewExpressionExecutorsFromPlanExpressions(proc, arg.Args) + if err != nil { + return nil, err + } + arg.ctr.argVecs = make([]*vector.Vector, len(arg.Args)) + + for i := range arg.Attrs { + arg.Attrs[i] = strings.ToUpper(arg.Attrs[i]) + } + + val, err := proc.GetResolveVariableFunc()("moplugin_allowed_hosts", true, false) + if err != nil { + return nil, err + } + st.allowed_hosts = strings.Split(val.(string), ",") + + return st, err + +} + +// start calling tvf on nthRow and put the result in u.batch. Note that current tokenize impl will +// always return one batch per nthRow. +func (u *pluginState) start(tf *TableFunction, proc *process.Process, nthRow int, analyzer process.Analyzer) error { + + if !u.inited { + u.batch = tf.createResultBatch() + u.inited = true + } + + u.called = false + // cleanup the batch + u.batch.CleanOnlyData() + + // plugin exec: number of args is always 4. (wasm_url string, func_name string, config map in JSON, datalink) + + // wasm + wasmVec := tf.ctr.argVecs[0] + switch wasmVec.GetType().Oid { + case types.T_varchar, types.T_datalink, types.T_char, types.T_text: + default: + return moerr.NewInternalError(proc.Ctx, "wasm URL only support varchar, char, text and datalink type") + } + + if !wasmVec.IsConst() { + return moerr.NewInternalError(proc.Ctx, "wasm URL must be a constant") + } + + if wasmVec.IsNull(uint64(nthRow)) { + u.batch.SetRowCount(0) + return nil + } + wasmurl := wasmVec.GetStringAt(nthRow) + + // func name + funcVec := tf.ctr.argVecs[1] + if funcVec.IsNull(uint64(nthRow)) { + u.batch.SetRowCount(0) + return nil + } + funcname := funcVec.GetStringAt(nthRow) + + // config + var cfgbytes []byte + cfgVec := tf.ctr.argVecs[2] + + if !cfgVec.IsConst() { + return moerr.NewInternalError(proc.Ctx, "config must be a constant string") + } + + if !cfgVec.IsNull(uint64(nthRow)) { + switch cfgVec.GetType().Oid { + case types.T_json: + + cfg := cfgVec.GetBytesAt(nthRow) + cfgjs := bytejson.ByteJson{} + cfgjs.Unmarshal(cfg) + + if cfgjs.Type != bytejson.TpCodeObject { + return moerr.NewInternalError(proc.Ctx, "config must be a JSON object") + } + + cfgbytes, _ = cfgjs.MarshalJSON() + case types.T_varchar, types.T_text, types.T_char: + cfgbytes = cfgVec.GetBytesAt(nthRow) + default: + return moerr.NewInternalError(proc.Ctx, "config must be a JSON Object or string") + } + } + + cfgmap := make(map[string]string) + + if cfgbytes != nil { + //logutil.Infof("COMMAND %s", string(cmdbytes)) + jsonparser.ObjectEach(cfgbytes, func(key []byte, value []byte, dataType jsonparser.ValueType, offset int) error { + + if dataType != jsonparser.String { + return moerr.NewInternalError(proc.Ctx, "config value is not string") + } + + cfgmap[string(key)] = string(value) + return nil + }) + } + //logutil.Infof("ARGS %v", args) + + // datalink + dlVec := tf.ctr.argVecs[3] + if dlVec.IsNull(uint64(nthRow)) { + u.batch.SetRowCount(0) + return nil + } + src := dlVec.GetStringAt(nthRow) + + var bytes []byte + switch dlVec.GetType().Oid { + case types.T_datalink: + dl, err := datalink.NewDatalink(src, proc) + if err != nil { + return err + } + bytes, err = dl.GetPlainText(proc) + if err != nil { + return err + } + case types.T_varchar, types.T_text, types.T_char: + bytes = []byte(src) + default: + return moerr.NewInternalError(proc.Ctx, "plugin_exec input type not supported") + } + + wurl, err := url.Parse(wasmurl) + if err != nil { + return err + } + + if u.plugin == nil { + var manifest extism.Manifest + if wurl.Scheme == "https" || wurl.Scheme == "http" { + + manifest = extism.Manifest{ + Wasm: []extism.Wasm{ + extism.WasmUrl{ + Url: wasmurl, + }, + }, + Config: cfgmap, + AllowedHosts: u.allowed_hosts, + } + } else { + // treat as datalink + wasmdl, err := datalink.NewDatalink(wasmurl, proc) + if err != nil { + return err + } + image, err := wasmdl.GetBytes(proc) + if err != nil { + return err + } + + manifest = extism.Manifest{ + Wasm: []extism.Wasm{ + extism.WasmData{ + Data: image, + }, + }, + Config: cfgmap, + AllowedHosts: u.allowed_hosts, + } + } + + config := extism.PluginConfig{ + EnableWasi: true, + EnableHttpResponseHeaders: true, + } + u.plugin, err = extism.NewPlugin(proc.Ctx, manifest, config, []extism.HostFunction{}) + if err != nil { + return err + } + } + + exit, out, err := u.plugin.Call(funcname, bytes) + if err != nil { + return moerr.NewInternalError(proc.Ctx, fmt.Sprintf("plugin exit with error %d. err %v", exit, err)) + } + + if len(out) == 0 { + u.batch.SetRowCount(0) + return nil + } + + nitem := 0 + var jserr error + jsonparser.ArrayEach(out, func(value []byte, dataType jsonparser.ValueType, offset int, err error) { + + if err != nil { + jserr = errors.Join(jserr, err) + return + } + + bj := bytejson.ByteJson{} + isnull := false + switch dataType { + case jsonparser.String: + jsdata, err := json.Marshal(string(value)) + if err != nil { + jserr = errors.Join(jserr, err) + return + } + err = bj.UnmarshalJSON(jsdata) + if err != nil { + jserr = errors.Join(jserr, err) + return + } + case jsonparser.Number, jsonparser.Boolean, jsonparser.Object, jsonparser.Array: + err = bj.UnmarshalJSON(value) + if err != nil { + jserr = errors.Join(jserr, err) + return + } + case jsonparser.Null: + isnull = true + default: + jserr = errors.Join(jserr, moerr.NewInternalErrorNoCtx("unknown json type")) + return + } + vector.AppendByteJson(u.batch.Vecs[0], bj, isnull, proc.Mp()) + nitem += 1 + }) + + if jserr != nil { + return jserr + } + + u.batch.SetRowCount(nitem) + return nil +} diff --git a/pkg/sql/colexec/table_function/table_function.go b/pkg/sql/colexec/table_function/table_function.go index 6d2ed6c75b049..64f668515ba1e 100644 --- a/pkg/sql/colexec/table_function/table_function.go +++ b/pkg/sql/colexec/table_function/table_function.go @@ -143,6 +143,8 @@ func (tableFunction *TableFunction) Prepare(proc *process.Process) error { tblArg.ctr.state, err = fulltextIndexTokenizePrepare(proc, tblArg) case "stage_list": tblArg.ctr.state, err = stageListPrepare(proc, tblArg) + case "moplugin_table": + tblArg.ctr.state, err = pluginPrepare(proc, tblArg) default: tblArg.ctr.state = nil err = moerr.NewNotSupported(proc.Ctx, fmt.Sprintf("table function %s is not supported", tblArg.FuncName)) diff --git a/pkg/sql/plan/function/func_builtin_w.go b/pkg/sql/plan/function/func_builtin_w.go index 74b6fd01dc228..3737ed940cf91 100644 --- a/pkg/sql/plan/function/func_builtin_w.go +++ b/pkg/sql/plan/function/func_builtin_w.go @@ -15,12 +15,13 @@ package function import ( - "context" + "net/url" extism "github.com/extism/go-sdk" "github.com/matrixorigin/matrixone/pkg/common/moerr" "github.com/matrixorigin/matrixone/pkg/container/types" "github.com/matrixorigin/matrixone/pkg/container/vector" + "github.com/matrixorigin/matrixone/pkg/datalink" "github.com/matrixorigin/matrixone/pkg/vm/process" ) @@ -45,23 +46,49 @@ func newOpBuiltInWasm() *opBuiltInWasm { return &opBuiltInWasm{} } -func (op *opBuiltInWasm) buildWasm(ctx context.Context, url string) error { +func (op *opBuiltInWasm) buildWasm(proc *process.Process, wasmurl string) error { var err error - // manifest is created from wasm url. - // We will need to handle stage in the future. - manifest := extism.Manifest{ - Wasm: []extism.Wasm{ - extism.WasmUrl{ - Url: url, + u, err := url.Parse(wasmurl) + if err != nil { + return err + } + + var manifest extism.Manifest + if u.Scheme == "http" || u.Scheme == "https" { + // manifest is created from wasm url. + manifest = extism.Manifest{ + Wasm: []extism.Wasm{ + extism.WasmUrl{ + Url: wasmurl, + }, + }, + } + } else { + + // treat as datalink + wasmdl, err := datalink.NewDatalink(wasmurl, proc) + if err != nil { + return err + } + image, err := wasmdl.GetBytes(proc) + if err != nil { + return err + } + manifest = extism.Manifest{ + Wasm: []extism.Wasm{ + extism.WasmData{ + Data: image, + }, }, - }, + } } + // enable wasi: tinygo build wasm need wasi. config := extism.PluginConfig{ EnableWasi: true, } - op.plugin, err = extism.NewPlugin(ctx, manifest, config, []extism.HostFunction{}) + op.plugin, err = extism.NewPlugin(proc.Ctx, manifest, config, []extism.HostFunction{}) return err } @@ -90,7 +117,7 @@ func (op *opBuiltInWasm) tryWasmImpl(params []*vector.Vector, result vector.Func if isnull { return moerr.NewInvalidInput(proc.Ctx, "wasm url cannot be null.") } - if err := op.buildWasm(proc.Ctx, string(url)); err != nil { + if err := op.buildWasm(proc, string(url)); err != nil { return err } diff --git a/pkg/sql/plan/function/function_id.go b/pkg/sql/plan/function/function_id.go index 9caa1f950fd44..d7da9a7b4ebf9 100644 --- a/pkg/sql/plan/function/function_id.go +++ b/pkg/sql/plan/function/function_id.go @@ -614,8 +614,8 @@ var functionIdRegister = map[string]int32{ "json_replace": JSON_REPLACE, "jq": JQ, "try_jq": TRY_JQ, - "wasm": WASM, - "try_wasm": TRY_WASM, + "moplugin": WASM, + "try_moplugin": TRY_WASM, "enable_fault_injection": ENABLE_FAULT_INJECTION, "disable_fault_injection": DISABLE_FAULT_INJECTION, "dense_rank": DENSE_RANK, diff --git a/pkg/sql/plan/plugin.go b/pkg/sql/plan/plugin.go new file mode 100644 index 0000000000000..3e40c0fb2c6bc --- /dev/null +++ b/pkg/sql/plan/plugin.go @@ -0,0 +1,65 @@ +// Copyright 2022 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package plan + +import ( + "github.com/matrixorigin/matrixone/pkg/common/moerr" + "github.com/matrixorigin/matrixone/pkg/container/types" + "github.com/matrixorigin/matrixone/pkg/pb/plan" + "github.com/matrixorigin/matrixone/pkg/sql/parsers/tree" +) + +// coldef shall copy index type +var ( + plugin_exec_func_name = "moplugin_table" + + pluginColdefs = []*plan.ColDef{ + { + Name: "result", + Typ: plan.Type{ + Id: int32(types.T_json), + NotNullable: false, + }, + }, + } +) + +// arg list [command, datalink] +func (builder *QueryBuilder) buildPluginExec(tbl *tree.TableFunction, ctx *BindContext, exprs []*plan.Expr, childId int32) (int32, error) { + + if len(exprs) != 4 { + return 0, moerr.NewInvalidInput(builder.GetContext(), "Invalid number of arguments (NARGS != 4).") + } + + colDefs := _getColDefs(pluginColdefs) + + node := &plan.Node{ + NodeType: plan.Node_FUNCTION_SCAN, + Stats: &plan.Stats{}, + TableDef: &plan.TableDef{ + TableType: "func_table", //test if ok + //Name: tbl.String(), + TblFunc: &plan.TableFunction{ + Name: plugin_exec_func_name, + Param: []byte(""), + }, + Cols: colDefs, + }, + BindingTags: []int32{builder.genNewTag()}, + TblFuncExprList: exprs, + Children: []int32{childId}, + } + return builder.appendNode(node, ctx), nil +} diff --git a/pkg/sql/plan/query_builder.go b/pkg/sql/plan/query_builder.go index cd568d4a717f7..5cc4fb51fb123 100644 --- a/pkg/sql/plan/query_builder.go +++ b/pkg/sql/plan/query_builder.go @@ -4786,6 +4786,8 @@ func (builder *QueryBuilder) buildTableFunction(tbl *tree.TableFunction, ctx *Bi nodeId, err = builder.buildFullTextIndexTokenize(tbl, ctx, exprs, childId) case "stage_list": nodeId, err = builder.buildStageList(tbl, ctx, exprs, childId) + case "moplugin_table": + nodeId, err = builder.buildPluginExec(tbl, ctx, exprs, childId) default: err = moerr.NewNotSupportedf(builder.GetContext(), "table function '%s' not supported", id) } diff --git a/test/distributed/cases/function/func_wasm.result b/test/distributed/cases/function/func_wasm.result index 3dd4cefb841ef..77ea1dc322d5b 100644 --- a/test/distributed/cases/function/func_wasm.result +++ b/test/distributed/cases/function/func_wasm.result @@ -1,25 +1,33 @@ -select wasm('https://github.com/matrixorigin/mojo/raw/main/plugin/hello/hello.wasm', 'mowasm_hello', 'world'); -wasm(https://github.com/matrixorigin/mojo/raw/main/plugin/hello/hello.wasm, mowasm_hello, world) +create stage mystage URL='file:///$resources/plugin/'; +select moplugin('stage://mystage/hello.wasm', 'mowasm_hello', 'world'); +moplugin(stage://mystage/hello.wasm, mowasm_hello, world) Hello world! -select wasm('https://github.com/matrixorigin/mojo/raw/main/plugin/hello/hello.wasm', 'mowasm_add', '[3, 5]'); -wasm(https://github.com/matrixorigin/mojo/raw/main/plugin/hello/hello.wasm, mowasm_add, [3, 5]) +select moplugin('stage://mystage/hello.wasm', 'mowasm_hello', 'world'); +moplugin(stage://mystage/hello.wasm, mowasm_hello, world) +Hello world! +select moplugin('stage://mystage/hello.wasm', 'mowasm_add', '[3, 5]'); +moplugin(stage://mystage/hello.wasm, mowasm_add, [3, 5]) 8 -select wasm('https://github.com/matrixorigin/mojo/raw/main/plugin/hello/notexist.wasm', 'mowasm_add', '[3, 5]'); +select moplugin('https://github.com/matrixorigin/mojo/raw/main/plugin/hello/notexist.wasm', 'mowasm_add', '[3, 5]'); failed to fetch Wasm data from URL -select try_wasm('https://github.com/matrixorigin/mojo/raw/main/plugin/hello/notexist.wasm', 'mowasm_add', '[3, 5]'); +select try_moplugin('https://github.com/matrixorigin/mojo/raw/main/plugin/hello/notexist.wasm', 'mowasm_add', '[3, 5]'); failed to fetch Wasm data from URL -select wasm('https://github.com/matrixorigin/mojo/raw/main/plugin/hello/hello.wasm', 'mowasm_add2', '[3, 5]'); -Unknown function: mowasm_add2 -select try_wasm('https://github.com/matrixorigin/mojo/raw/main/plugin/hello/hello.wasm', 'mowasm_add2', '[3, 5]'); -try_wasm(https://github.com/matrixorigin/mojo/raw/main/plugin/hello/hello.wasm, mowasm_add2, [3, 5]) +select moplugin('stage://mystage/notexist.wasm', 'mowasm_add', '[3, 5]'); +file notexist.wasm is not found +select try_moplugin('stage://mystage/notexist.wasm', 'mowasm_add', '[3, 5]'); +file notexist.wasm is not found +select moplugin('stage://mystage/hello.wasm', 'mowasm_add2', '[3, 5]'); +unknown function: mowasm_add2 +select try_moplugin('stage://mystage/hello.wasm', 'mowasm_add2', '[3, 5]'); +try_moplugin(stage://mystage/hello.wasm, mowasm_add2, [3, 5]) null -select wasm('https://github.com/matrixorigin/mojo/raw/main/plugin/hello/hello.wasm', 'mowasm_add', '[1, 3, 5]'); +select moplugin('stage://mystage/hello.wasm', 'mowasm_add', '[1, 3, 5]'); add takes two float arguments -select try_wasm('https://github.com/matrixorigin/mojo/raw/main/plugin/hello/hello.wasm', 'mowasm_add', '[1, 3, 5]'); -try_wasm(https://github.com/matrixorigin/mojo/raw/main/plugin/hello/hello.wasm, mowasm_add, [1, 3, 5]) +select try_moplugin('stage://mystage/hello.wasm', 'mowasm_add', '[1, 3, 5]'); +try_moplugin(stage://mystage/hello.wasm, mowasm_add, [1, 3, 5]) null -create table wasmt(id int, fn varchar(255), arg varchar(255)); -insert into wasmt values +create table moplugint(id int, fn varchar(255), arg varchar(255)); +insert into moplugint values (1, 'mowasm_hello', '[1, 2]'), (2, 'mowasm_add', '[1, 2]'), (3, 'mowasm_hello', '[1, 2]'), @@ -31,12 +39,12 @@ insert into wasmt values (9, 'mowasm_hello', '[1, 2]'), (10, 'mowasm_add', '[1, 2]') ; -select count(*) from wasmt; +select count(*) from moplugint; count(*) 10 -select id, wasm('https://github.com/matrixorigin/mojo/raw/main/plugin/hello/hello.wasm', fn, arg) -from wasmt; -id wasm(https://github.com/matrixorigin/mojo/raw/main/plugin/hello/hello.wasm, fn, arg) +select id, moplugin('stage://mystage/hello.wasm', fn, arg) +from moplugint; +id moplugin(stage://mystage/hello.wasm, fn, arg) 1 Hello [1, 2]! 2 3 3 Hello [1, 2]! @@ -47,9 +55,9 @@ id wasm(https://github.com/matrixorigin/mojo/raw/main/plugin/hello/hello.wasm 8 3 9 Hello [1, 2]! 10 3 -select id, try_wasm('https://github.com/matrixorigin/mojo/raw/main/plugin/hello/hello.wasm', fn, arg) -from wasmt; -id try_wasm(https://github.com/matrixorigin/mojo/raw/main/plugin/hello/hello.wasm, fn, arg) +select id, try_moplugin('stage://mystage/hello.wasm', fn, arg) +from moplugint; +id try_moplugin(stage://mystage/hello.wasm, fn, arg) 1 Hello [1, 2]! 2 3 3 Hello [1, 2]! @@ -60,4 +68,5 @@ id try_wasm(https://github.com/matrixorigin/mojo/raw/main/plugin/hello/hello. 8 3 9 Hello [1, 2]! 10 3 -drop table wasmt; +drop table moplugint; +drop stage mystage; diff --git a/test/distributed/cases/function/func_wasm.sql b/test/distributed/cases/function/func_wasm.sql index 130f9bae2b4ea..3d5325de6a51e 100644 --- a/test/distributed/cases/function/func_wasm.sql +++ b/test/distributed/cases/function/func_wasm.sql @@ -1,21 +1,28 @@ -- --- wasm test +-- moplugin test -- -select wasm('https://github.com/matrixorigin/mojo/raw/main/plugin/hello/hello.wasm', 'mowasm_hello', 'world'); -select wasm('https://github.com/matrixorigin/mojo/raw/main/plugin/hello/hello.wasm', 'mowasm_add', '[3, 5]'); +create stage mystage URL='file:///$resources/plugin/'; -select wasm('https://github.com/matrixorigin/mojo/raw/main/plugin/hello/notexist.wasm', 'mowasm_add', '[3, 5]'); -select try_wasm('https://github.com/matrixorigin/mojo/raw/main/plugin/hello/notexist.wasm', 'mowasm_add', '[3, 5]'); +select moplugin('stage://mystage/hello.wasm', 'mowasm_hello', 'world'); -select wasm('https://github.com/matrixorigin/mojo/raw/main/plugin/hello/hello.wasm', 'mowasm_add2', '[3, 5]'); -select try_wasm('https://github.com/matrixorigin/mojo/raw/main/plugin/hello/hello.wasm', 'mowasm_add2', '[3, 5]'); +select moplugin('stage://mystage/hello.wasm', 'mowasm_hello', 'world'); +select moplugin('stage://mystage/hello.wasm', 'mowasm_add', '[3, 5]'); -select wasm('https://github.com/matrixorigin/mojo/raw/main/plugin/hello/hello.wasm', 'mowasm_add', '[1, 3, 5]'); -select try_wasm('https://github.com/matrixorigin/mojo/raw/main/plugin/hello/hello.wasm', 'mowasm_add', '[1, 3, 5]'); +select moplugin('https://github.com/matrixorigin/mojo/raw/main/plugin/hello/notexist.wasm', 'mowasm_add', '[3, 5]'); +select try_moplugin('https://github.com/matrixorigin/mojo/raw/main/plugin/hello/notexist.wasm', 'mowasm_add', '[3, 5]'); -create table wasmt(id int, fn varchar(255), arg varchar(255)); -insert into wasmt values +select moplugin('stage://mystage/notexist.wasm', 'mowasm_add', '[3, 5]'); +select try_moplugin('stage://mystage/notexist.wasm', 'mowasm_add', '[3, 5]'); + +select moplugin('stage://mystage/hello.wasm', 'mowasm_add2', '[3, 5]'); +select try_moplugin('stage://mystage/hello.wasm', 'mowasm_add2', '[3, 5]'); + +select moplugin('stage://mystage/hello.wasm', 'mowasm_add', '[1, 3, 5]'); +select try_moplugin('stage://mystage/hello.wasm', 'mowasm_add', '[1, 3, 5]'); + +create table moplugint(id int, fn varchar(255), arg varchar(255)); +insert into moplugint values (1, 'mowasm_hello', '[1, 2]'), (2, 'mowasm_add', '[1, 2]'), (3, 'mowasm_hello', '[1, 2]'), @@ -28,18 +35,13 @@ insert into wasmt values (10, 'mowasm_add', '[1, 2]') ; -select count(*) from wasmt; -select id, wasm('https://github.com/matrixorigin/mojo/raw/main/plugin/hello/hello.wasm', fn, arg) -from wasmt; -select id, try_wasm('https://github.com/matrixorigin/mojo/raw/main/plugin/hello/hello.wasm', fn, arg) -from wasmt; - -drop table wasmt; - - - - - +select count(*) from moplugint; +select id, moplugin('stage://mystage/hello.wasm', fn, arg) +from moplugint; +select id, try_moplugin('stage://mystage/hello.wasm', fn, arg) +from moplugint; +drop table moplugint; +drop stage mystage; diff --git a/test/distributed/cases/plugin/plugin.result b/test/distributed/cases/plugin/plugin.result new file mode 100644 index 0000000000000..285a5eed41beb --- /dev/null +++ b/test/distributed/cases/plugin/plugin.result @@ -0,0 +1,131 @@ +create stage llmstage URL='file:///$resources/plugin/'; +select * from moplugin_table('stage://llmstage/cat.wasm', 'cat', null, '[1,2,3]') as f; +result +1 +2 +3 +select * from moplugin_table(cast('stage://llmstage/cat.wasm' as json), 'cat', null, '[1,2,3]') as f; +invalid input: json text stage://llmstage/cat.wasm +select * from moplugin_table('stage://llmstage/cat.wasm"') as f; +invalid input: Invalid number of arguments (NARGS != 4). +select * from moplugin_table('unknown', 'cat', null, '[1,2,3]') as f; +unsupported url scheme is not yet implemented +select * from moplugin_table('stage://llmstage/cat.wasm', '1') as f; +invalid input: Invalid number of arguments (NARGS != 4). +select * from moplugin_table('stage://llmstage/cat.wasm', 'cat', null, '[1a,2,3]') as f; +internal error: parse json: json: expected token but found 'a': a +select * from moplugin_table('stage://llmstage/cat.wasm', 'cat', +cast('["header", "footer"]' as JSON), '1,2,3') as f; +internal error: config must be a JSON object +select * from moplugin_table(null, 'cat', null, null) as f; +internal error: wasm URL only support varchar, char, text and datalink type +select * from moplugin_table('stage://llmstage/cat.wasm', null, null, null) as f; +result +select * from moplugin_table('stage://llmstage/cat.wasm', 'cat', null, '') as f; +result +select * from moplugin_table('stage://llmstage/cat.wasm', 'cat', +'{"header":"[", "footer":"]"}', '1,2,3') as f; +result +1 +2 +3 +select * from moplugin_table('stage://llmstage/cat.wasm', 'cat', +cast('{"header":"[", "footer":"]"}' as JSON), '1,2,3') as f; +result +1 +2 +3 +select * from moplugin_table('stage://llmstage/cat.wasm', 'cat', null, '[1,2,3]') as f; +result +1 +2 +3 +select * from moplugin_table('stage://llmstage/cat.wasm', 'cat', null, '["1","2","3"]') as f; +result +"1" +"2" +"3" +select * from moplugin_table(cast('stage://llmstage/cat.wasm' as datalink), 'cat', null, '["a","b","c"]') as f; +result +"a" +"b" +"c" +select * from moplugin_table(cast('stage://llmstage/cat.wasm' as datalink), 'cat', null, '["a","b",null]') as f; +result +"a" +"b" +null +select * from moplugin_table('stage://llmstage/cat.wasm', 'cat', null, '[false,true,null]') as f; +result +false +true +null +select json_extract(result, "$.id") from moplugin_table('stage://llmstage/cat.wasm', 'cat', null, '[{"id":1},{"id":2},{"id":3}]') as f; +json_extract(result, $.id) +1 +2 +3 +select * from moplugin_table('stage://llmstage/cat.wasm', 'cat', null, cast('file:///$resources/plugin/result.json' as datalink)) as f; +result +{"chunk": 0, "e": "[1.1, 2.2, 3.3]", "t": "text0"} +{"chunk": "1", "e": "[2.5, 3.5, 4.5]", "t": "text1"} +{"chunk": "2", "e": "[9.1, 8.2, 7.3]", "t": "text2"} +select json_extract(result, "$.chunk"), json_extract(result, "$.e") from moplugin_table('stage://llmstage/cat.wasm', 'cat', null, +cast('file:///$resources/plugin/multistream.json?offset=0&size=155' as datalink) ) as f; +json_extract(result, $.chunk) json_extract(result, $.e) +0 "[1.1, 2.2, 3.3]" +"1" "[2.5, 3.5, 4.5]" +"2" "[9.1, 8.2, 7.3]" +select json_extract(result, "$.chunk"), json_extract(result, "$.e") from moplugin_table('stage://llmstage/cat.wasm', 'cat', null, +cast('file:///$resources/plugin/multistream.json?offset=155&size=164' as datalink) ) as f; +json_extract(result, $.chunk) json_extract(result, $.e) +3 "[10.1, 20.2, 30.3]" +"4" "[20.5, 30.5, 40.5]" +"5" "[90.1, 80.2, 70.3]" +create table t1 (chunk int, e vecf32(3)); +insert into t1 select json_unquote(json_extract(result, "$.chunk")), json_unquote(json_extract(result, "$.e")) +from moplugin_table('stage://llmstage/cat.wasm', 'cat', null, cast('file:///$resources/plugin/result.json' as datalink)) as f; +select * from t1; +chunk e +0 [1.1, 2.2, 3.3] +1 [2.5, 3.5, 4.5] +2 [9.1, 8.2, 7.3] +truncate t1; +insert into t1 select json_unquote(json_extract(result, "$.chunk")), json_unquote(json_extract(result, "$.e")) +from moplugin_table('stage://llmstage/cat.wasm', 'cat', null, +cast('file:///$resources/plugin/multistream.json?offset=0&size=155' as datalink) ) as f; +select * from t1; +chunk e +0 [1.1, 2.2, 3.3] +1 [2.5, 3.5, 4.5] +2 [9.1, 8.2, 7.3] +insert into t1 select json_unquote(json_extract(result, "$.chunk")), json_unquote(json_extract(result, "$.e")) +from moplugin_table('stage://llmstage/cat.wasm', 'cat', null, cast('file:///$resources/plugin/multistream.json?offset=155&size=164' as datalink) ) as f; +select * from t1; +chunk e +0 [1.1, 2.2, 3.3] +1 [2.5, 3.5, 4.5] +2 [9.1, 8.2, 7.3] +3 [10.1, 20.2, 30.3] +4 [20.5, 30.5, 40.5] +5 [90.1, 80.2, 70.3] +drop table t1; +create table src (pkey int primary key, dlink datalink); +create table embed (pkey int, chunk int, e vecf32(3), t varchar); +insert into src values +(0, 'stage://llmstage/multistream.json?offset=0&size=155'), +(1, 'stage://llmstage/multistream.json?offset=155&size=164'); +insert into embed select src.pkey, json_unquote(json_extract(f.result, "$.chunk")), +json_unquote(json_extract(f.result, "$.e")), json_unquote(json_extract(f.result, "$.t")) +from src CROSS APPLY moplugin_table('stage://llmstage/cat.wasm', 'cat', null, src.dlink) as f; +select * from embed; +pkey chunk e t +0 0 [1.1, 2.2, 3.3] text0 +0 1 [2.5, 3.5, 4.5] text1 +0 2 [9.1, 8.2, 7.3] text2 +1 3 [10.1, 20.2, 30.3] text3 +1 4 [20.5, 30.5, 40.5] text4 +1 5 [90.1, 80.2, 70.3] text5 +drop stage llmstage; +drop table src; +drop table embed; diff --git a/test/distributed/cases/plugin/plugin.sql b/test/distributed/cases/plugin/plugin.sql new file mode 100644 index 0000000000000..0035c5adfb947 --- /dev/null +++ b/test/distributed/cases/plugin/plugin.sql @@ -0,0 +1,85 @@ +create stage llmstage URL='file:///$resources/plugin/'; + +-- error +select * from moplugin_table('stage://llmstage/cat.wasm', 'cat', null, '[1,2,3]') as f; +select * from moplugin_table(cast('stage://llmstage/cat.wasm' as json), 'cat', null, '[1,2,3]') as f; +select * from moplugin_table('stage://llmstage/cat.wasm"') as f; +select * from moplugin_table('unknown', 'cat', null, '[1,2,3]') as f; +select * from moplugin_table('stage://llmstage/cat.wasm', '1') as f; +select * from moplugin_table('stage://llmstage/cat.wasm', 'cat', null, '[1a,2,3]') as f; +select * from moplugin_table('stage://llmstage/cat.wasm', 'cat', + cast('["header", "footer"]' as JSON), '1,2,3') as f; + +-- start test +-- index of multistream.json (offset, size) = [(0, 155), (155, 164)] + +select * from moplugin_table(null, 'cat', null, null) as f; +select * from moplugin_table('stage://llmstage/cat.wasm', null, null, null) as f; + +select * from moplugin_table('stage://llmstage/cat.wasm', 'cat', null, '') as f; + +select * from moplugin_table('stage://llmstage/cat.wasm', 'cat', + '{"header":"[", "footer":"]"}', '1,2,3') as f; + +select * from moplugin_table('stage://llmstage/cat.wasm', 'cat', + cast('{"header":"[", "footer":"]"}' as JSON), '1,2,3') as f; + +select * from moplugin_table('stage://llmstage/cat.wasm', 'cat', null, '[1,2,3]') as f; + +select * from moplugin_table('stage://llmstage/cat.wasm', 'cat', null, '["1","2","3"]') as f; + +select * from moplugin_table(cast('stage://llmstage/cat.wasm' as datalink), 'cat', null, '["a","b","c"]') as f; + +select * from moplugin_table(cast('stage://llmstage/cat.wasm' as datalink), 'cat', null, '["a","b",null]') as f; + +select * from moplugin_table('stage://llmstage/cat.wasm', 'cat', null, '[false,true,null]') as f; + +select json_extract(result, "$.id") from moplugin_table('stage://llmstage/cat.wasm', 'cat', null, '[{"id":1},{"id":2},{"id":3}]') as f; + +select * from moplugin_table('stage://llmstage/cat.wasm', 'cat', null, cast('file:///$resources/plugin/result.json' as datalink)) as f; + +select json_extract(result, "$.chunk"), json_extract(result, "$.e") from moplugin_table('stage://llmstage/cat.wasm', 'cat', null, + cast('file:///$resources/plugin/multistream.json?offset=0&size=155' as datalink) ) as f; + +select json_extract(result, "$.chunk"), json_extract(result, "$.e") from moplugin_table('stage://llmstage/cat.wasm', 'cat', null, + cast('file:///$resources/plugin/multistream.json?offset=155&size=164' as datalink) ) as f; + + +create table t1 (chunk int, e vecf32(3)); +insert into t1 select json_unquote(json_extract(result, "$.chunk")), json_unquote(json_extract(result, "$.e")) +from moplugin_table('stage://llmstage/cat.wasm', 'cat', null, cast('file:///$resources/plugin/result.json' as datalink)) as f; +select * from t1; + +truncate t1; + +insert into t1 select json_unquote(json_extract(result, "$.chunk")), json_unquote(json_extract(result, "$.e")) +from moplugin_table('stage://llmstage/cat.wasm', 'cat', null, + cast('file:///$resources/plugin/multistream.json?offset=0&size=155' as datalink) ) as f; + +select * from t1; + +insert into t1 select json_unquote(json_extract(result, "$.chunk")), json_unquote(json_extract(result, "$.e")) +from moplugin_table('stage://llmstage/cat.wasm', 'cat', null, cast('file:///$resources/plugin/multistream.json?offset=155&size=164' as datalink) ) as f; + +select * from t1; + +drop table t1; + + +create table src (pkey int primary key, dlink datalink); + +create table embed (pkey int, chunk int, e vecf32(3), t varchar); + +insert into src values +(0, 'stage://llmstage/multistream.json?offset=0&size=155'), +(1, 'stage://llmstage/multistream.json?offset=155&size=164'); + +insert into embed select src.pkey, json_unquote(json_extract(f.result, "$.chunk")), +json_unquote(json_extract(f.result, "$.e")), json_unquote(json_extract(f.result, "$.t")) +from src CROSS APPLY moplugin_table('stage://llmstage/cat.wasm', 'cat', null, src.dlink) as f; + +select * from embed; + +drop stage llmstage; +drop table src; +drop table embed; diff --git a/test/distributed/resources/plugin/cat.wasm b/test/distributed/resources/plugin/cat.wasm new file mode 100644 index 0000000000000..49c6b3b482db5 Binary files /dev/null and b/test/distributed/resources/plugin/cat.wasm differ diff --git a/test/distributed/resources/plugin/hello.wasm b/test/distributed/resources/plugin/hello.wasm new file mode 100644 index 0000000000000..db6d00109da95 Binary files /dev/null and b/test/distributed/resources/plugin/hello.wasm differ diff --git a/test/distributed/resources/plugin/multistream.json b/test/distributed/resources/plugin/multistream.json new file mode 100644 index 0000000000000..a63483519589c --- /dev/null +++ b/test/distributed/resources/plugin/multistream.json @@ -0,0 +1,6 @@ +[{"chunk": 0, "e":"[1.1, 2.2, 3.3]", "t":"text0"}, +{"chunk": "1", "e":"[2.5, 3.5, 4.5]", "t":"text1"}, +{"chunk": "2", "e":"[9.1, 8.2, 7.3]", "t":"text2"}] +[{"chunk": 3, "e":"[10.1, 20.2, 30.3]", "t":"text3"}, +{"chunk": "4", "e":"[20.5, 30.5, 40.5]", "t":"text4"}, +{"chunk": "5", "e":"[90.1, 80.2, 70.3]", "t":"text5"}] diff --git a/test/distributed/resources/plugin/result.json b/test/distributed/resources/plugin/result.json new file mode 100644 index 0000000000000..497dcac4a21fe --- /dev/null +++ b/test/distributed/resources/plugin/result.json @@ -0,0 +1,3 @@ +[{"chunk": 0, "e":"[1.1, 2.2, 3.3]", "t":"text0"}, +{"chunk": "1", "e":"[2.5, 3.5, 4.5]", "t":"text1"}, +{"chunk": "2", "e":"[9.1, 8.2, 7.3]", "t":"text2"}]